diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow.py index 4602a678..277ad4b7 100644 --- a/src/keri/app/cli/commands/escrow.py +++ b/src/keri/app/cli/commands/escrow.py @@ -56,8 +56,8 @@ def escrows(tymth, tock=0.0, **opts): oots = list() key = ekey = b'' # both start same. when not same means escrows found while True: - for ekey, edig in hby.db.getOoeItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + for ekey, edig in hby.db.getOoeItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: oots.append(eventing.loadEvent(hby.db, pre, edig)) @@ -74,8 +74,8 @@ def escrows(tymth, tock=0.0, **opts): pwes = list() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in hby.db.getPweItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + for ekey, edig in hby.db.getPweItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: pwes.append(eventing.loadEvent(hby.db, pre, edig)) @@ -92,8 +92,8 @@ def escrows(tymth, tock=0.0, **opts): pses = list() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in hby.db.getPseItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + for ekey, edig in hby.db.getPseItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: pses.append(eventing.loadEvent(hby.db, pre, edig)) @@ -110,8 +110,8 @@ def escrows(tymth, tock=0.0, **opts): ldes = list() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in hby.db.getLdeItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + for ekey, edig in hby.db.getLdeItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: ldes.append(eventing.loadEvent(hby.db, pre, edig)) diff --git a/src/keri/app/cli/commands/ipex/grant.py b/src/keri/app/cli/commands/ipex/grant.py index d6e309d7..cc766664 100644 --- a/src/keri/app/cli/commands/ipex/grant.py +++ b/src/keri/app/cli/commands/ipex/grant.py @@ -111,8 +111,8 @@ def grantDo(self, tymth, tock=0.0): iserder = serdering.SerderKERI(raw=bytes(iss)) seqner = coring.Seqner(sn=iserder.sn) - serder = self.hby.db.findAnchoringSealEvent(creder.sad['i'], - seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) + serder = self.hby.db.fetchAllSealingEventByEventSeal(creder.sad['i'], + seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) anc = self.hby.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said) exn, atc = protocoling.ipexGrantExn(hab=self.hab, recp=recp, message=self.message, acdc=acdc, iss=iss, anc=anc, diff --git a/src/keri/app/cli/commands/migrate.py b/src/keri/app/cli/commands/migrate.py index 36d3b3c0..87858779 100644 --- a/src/keri/app/cli/commands/migrate.py +++ b/src/keri/app/cli/commands/migrate.py @@ -148,7 +148,7 @@ def migrateKeys(db): digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", klas=(coring.Prefixer, coring.Seqner)) - for pre, fn, dig in db.getFelItemAllPreIter(key=b''): + for pre, fn, dig in db.getFelItemAllPreIter(): dgkey = dbing.dgKey(pre, dig) # get message if not (raw := db.getEvt(key=dgkey)): raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) diff --git a/src/keri/app/connecting.py b/src/keri/app/connecting.py index 7ab81dd1..e6a2f545 100644 --- a/src/keri/app/connecting.py +++ b/src/keri/app/connecting.py @@ -201,7 +201,7 @@ def setImg(self, pre, typ, stream): stream (file): file-like stream of image data """ - self.hby.db.delTopVal(db=self.hby.db.imgs, key=pre.encode("utf-8")) + self.hby.db.delTopVal(db=self.hby.db.imgs, top=pre.encode("utf-8")) key = f"{pre}.content-type".encode("utf-8") self.hby.db.setVal(db=self.hby.db.imgs, key=key, val=typ.encode("utf-8")) diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index a1cb0a4c..bddb7dd4 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -138,7 +138,7 @@ def processUnanchoredEscrow(self): dkever = self.hby.kevers[kever.delpre] seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - if dserder := self.hby.db.findAnchoringSealEvent(dkever.prefixer.qb64, seal=seal): + if dserder := self.hby.db.fetchAllSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal): seqner = coring.Seqner(sn=dserder.sn) couple = seqner.qb64b + dserder.saidb dgkey = dbing.dgKey(kever.prefixer.qb64b, kever.serder.saidb) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 41f8249c..096798f7 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -171,7 +171,8 @@ def processDelegateEscrow(self): self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) else: # Not witnesser, we need to look for the anchor and then wait for receipts - if serder := self.hby.db.findAnchoringSealEvent(kever.delpre, seal=anchor): + if serder := self.hby.db.fetchAllSealingEventByEventSeal(kever.delpre, + seal=anchor): aseq = coring.Seqner(sn=serder.sn) couple = aseq.qb64b + serder.saidb dgkey = dbing.dgKey(pre, saider.qb64b) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index d9431aa4..d9869970 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -1538,7 +1538,7 @@ def replay(self, pre=None, fn=0): return msgs - def replayAll(self, key=b''): + def replayAll(self): """ Returns replay of FEL first seen event log for all pre starting at key @@ -1547,7 +1547,7 @@ def replayAll(self, key=b''): """ msgs = bytearray() - for msg in self.db.cloneAllPreIter(key=key): + for msg in self.db.cloneAllPreIter(): msgs.extend(msg) return msgs @@ -2284,6 +2284,9 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode= # may want db method that updates .habs. and .prefixes together habord = basing.HabitatRecord(hid=self.pre, name=self.name, domain=self.ns) + # must add self.pre to self.prefixes before calling processEvent so that + # Kever.locallyOwned or Kever.locallyDelegated or Kever.locallyWitnessed + # evaluates correctly when processing own inception event. if not hidden: self.save(habord) self.prefixes.add(self.pre) diff --git a/src/keri/app/notifying.py b/src/keri/app/notifying.py index 4ad03d22..09ac24db 100644 --- a/src/keri/app/notifying.py +++ b/src/keri/app/notifying.py @@ -184,7 +184,8 @@ def getItemIter(self, keys: Union[str, Iterable] = b""): each entry in db """ - for key, val in self.db.getAllItemIter(db=self.sdb, key=self._tokey(keys), split=False): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys)): yield self._tokeys(key), self.klas(raw=bytes(val)) def cntAll(self): diff --git a/src/keri/app/querying.py b/src/keri/app/querying.py index 04b8bf41..b6a401ad 100644 --- a/src/keri/app/querying.py +++ b/src/keri/app/querying.py @@ -141,7 +141,7 @@ def recur(self, tyme, deeds=None): return False kever = self.hab.kevers[self.pre] - if self.hby.db.findAnchoringSealEvent(self.pre, seal=self.anchor): + if self.hby.db.fetchAllSealingEventByEventSeal(self.pre, seal=self.anchor): self.remove([self.witq]) return True diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 297dc97e..80cf0f2f 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -34,6 +34,17 @@ def __init__(self, name="mbx", headDirPath=None, reopen=True, **kwa): perm: reopen: kwa: + + Mailboxer uses two dbs for mailbox messages these are .tpcs and .msgs. + The message index is in .tpcs (topics). + Each .tpcs index key consists of topic.on where topic is bytes + identifier or prefix/topic for message and on is serialized + ordinal number to orders the appearance of a topic message. + Eash .tpcs val is the digest of the message. + The message itself is stored in .msgs where the key is the msg digest + and the value is the serialized messag itself. + Multiple messages can share the same topic but with a different ordinal. + """ self.tpcs = None self.msgs = None @@ -47,88 +58,102 @@ def reopen(self, **kwa): :return: """ super(Mailboxer, self).reopen(**kwa) - - self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True) + self.tpcs = subing.OnSuber(db=self, subkey='tpcs.') self.msgs = subing.Suber(db=self, subkey='msgs.') # key states return self.env - def delTopic(self, key): - """ - Use snKey() - Deletes value at key. - Returns True If key exists in database Else False + def delTopic(self, key, on=0): + """Removes topic index from .tpcs without deleting message from .msgs + + Returns: + result (boo): True if full key consisting of key and serialized on + exists in database so removed + False otherwise (not removed) """ - return self.delIoSetVals(self.tpcs, key) + return self.tpcs.remOn(keys=key, on=on) def appendToTopic(self, topic, val): - """ - Return first seen order number int, fn, of appended entry. - Computes fn as next fn after last entry. + """Appends val to end of db entries with same topic but with on + incremented by 1 relative to last preexisting entry at topic. - Append val to end of db entries with same topic but with fn incremented by - 1 relative to last preexisting entry at pre. + Returns: + on (int): order number int, on, of appended entry. + Computes on as next on after last entry. Parameters: - topic is bytes identifier prefix/topic for message - val is event digest + topic (bytes): topic identifier for message + val (bytes): msg digest """ - return self.appendIoSetVal(db=self.tpcs, key=topic, val=val) + return self.tpcs.appendOn(key=topic, val=val) + def getTopicMsgs(self, topic, fn=0): """ Returns: - ioset (oset): the insertion ordered set of values at same apparent - effective key. - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. + msgs (Iterable[bytes]): belonging to topic indices with same topic but all + on >= fn i.e. all topic.on beginning with fn Parameters: - topic (Option(bytes|str)): Apparent effective key - fn (int) starting index + topic (Option(bytes|str)): key prefix combined with serialized on + to form full actual key. When key is empty then retrieves + whole database. + fn (int): starting index ordinal number used with onKey(pre,on) + to form key at at which to initiate retrieval """ - if hasattr(topic, "encode"): - topic = topic.encode("utf-8") - - digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn) msgs = [] - for dig in digs: + for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn): if msg := self.msgs.get(keys=dig): - msgs.append(msg.encode("utf-8")) + msgs.append(msg.encode()) # want bytes not str return msgs + def storeMsg(self, topic, msg): """ - Add exn event to mailbox of dest identifier + Add exn event to mailbox topic and on that is 1 greater than last msg + at topic. + + Returns: + result (bool): True if msg successfully stored and indexed at topic + False otherwise Parameters: - msg (bytes): - topic (qb64b): + topic (str | bytes): topic (Option(bytes|str)): key prefix combined + with serialized on to form full actual key. + msg (bytes): serialized message """ - if hasattr(topic, "encode"): - topic = topic.encode("utf-8") - if hasattr(msg, "encode"): msg = msg.encode("utf-8") digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b - self.appendToTopic(topic=topic, val=digb) + on = self.tpcs.appendOn(keys=topic, val=digb) return self.msgs.pin(keys=digb, val=msg) + def cloneTopicIter(self, topic, fn=0): """ - Returns iterator of first seen exn messages with attachments for the - identifier prefix pre starting at first seen order number, fn. + Returns: + triple (Iterator[(on, topic, msg): iterator of messages at topic + beginning with ordinal fn. - """ - if hasattr(topic, 'encode'): - topic = topic.encode("utf-8") + topic (Option(bytes|str)): key prefix combined with serialized on + to form full actual key. When key is empty then retrieves + whole database. + fn (int): starting index ordinal number used with onKey(pre,on) + to form key at at which to initiate retrieval - for (key, dig) in self.getIoSetItemsIter(self.tpcs, key=topic, ion=fn): - topic, ion = dbing.unsuffix(key) + + + ToDo looks like misuse of IoSet this should not be IoSet but simply + Ordinal Numbered db. since should not be using hidden ion has not + hidden. + + """ + for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn): if msg := self.msgs.get(keys=dig): - yield ion, topic, msg.encode("utf-8") + yield (on, topic, msg.encode("utf-8")) + class Respondant(doing.DoDoer): diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 607048c0..a7bd6619 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -43,7 +43,7 @@ from ..db import basing, dbing, subing from ..db.basing import KeyStateRecord, StateEERecord, OobiRecord -from ..db.dbing import dgKey, snKey, fnKey, splitKeySN, splitKey +from ..db.dbing import dgKey, snKey, fnKey, splitSnKey, splitKey logger = help.ogler.getLogger() @@ -1494,7 +1494,7 @@ class Kever: def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, db=None, estOnly=None, delseqner=None, delsaider=None, firner=None, - dater=None, cues=None, local=True, check=False): + dater=None, cues=None, eager=False, local=True, check=False): """ Create incepting kever and state from inception serder Verify incepting serder against sigers raises ValidationError if not @@ -1522,6 +1522,11 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, When dater provided then use dater for first seen datetime cues (Deck | None): reference to Kevery.cues Deck when provided i.e. notices of events or requests to respond to + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -1561,16 +1566,18 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel( + serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + delseqner=delseqner, + delsaider=delsaider, + eager=eager, + local=local) self.delpre = delpre # may be None self.delegated = True if self.delpre else False @@ -1579,7 +1586,8 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen # returns fn == None if already logged fn log is non idempotent - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, + wits=wits, first=True if not check else False, seqner=delseqner, saider=delsaider, firner=firner, dater=dater, local=local) @@ -1661,16 +1669,42 @@ def locallyOwned(self, pre: str | None = None): Returns: (bool): True if pre is local hab but not group hab + When pre="" empty then returns False Parameters: pre (str|None): qb64 identifier prefix if any. Default None None means use self.prefixer.qb64 + """ pre = pre if pre is not None else self.prefixer.qb64 return pre in self.prefixes and pre not in self.groups + def locallyDelegated(self, pre: str): + """Returns True if pre is in .prefixes and not in .groups + False otherwise. Use when pre is a delegator for some event and + want to confirm that pre is also locallyOwned thereby making the + associated event locallyDelegated. + + Indicates that provided identifier prefix is controlled by a local + controller from .prefixes but is not a group with local member. + i.e pre is a locally owned (controlled) AID (identifier prefix) + Because delpre may be None, changes the default to "" instead of + self.prefixer.pre because self.prefixer.pre is delegate not delegator + of self. Unaccepted dip events do not have self.delpre set yet. + + Returns: + (bool): True if pre is local hab but not group hab + When pre="" empty or None then returns False + + Parameters: + pre (str): qb64 identifier prefix if any. + """ + pre = pre if pre is not None else "" + return self.locallyOwned(pre=pre) + + def locallyWitnessed(self, *, wits: list[str]=None, serder: (str)=None): """Returns True if a local controller is a witness of this Kever's KEL of wits in serder of if None then current wits for this Kever. @@ -1859,7 +1893,7 @@ def config(self, serder, estOnly=None, doNotDelegate=None): def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None, local=True, check=False): + firner=None, dater=None, eager=False, local=True, check=False): """ Not an inception event. Verify event serder and indexed signatures in sigers and update state @@ -1883,6 +1917,11 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, dater (Dater | None): Dater instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -1920,23 +1959,27 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # returned sigers and wigers are verified signatures # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=tholder, - wigers=wigers, - toader=toader, - wits=wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel( + serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=tholder, + wigers=wigers, + toader=toader, + wits=wits, + delseqner=delseqner, + delsaider=delsaider, + eager=eager, + local=local) # .valSigWigsDel above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, - first=True if not check else False, seqner=delseqner, saider=delsaider, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, + wits=wits, + first=True if not check else False, + seqner=delseqner, saider=delsaider, firner=firner, dater=dater, local=local) # nxt and signatures verify so update state @@ -1985,18 +2028,20 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=self.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local) + sigers, wigers, delpre, _, _ = self.valSigsWigsDel( + serder=serder, + sigers=sigers, + verfers=self.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + eager=eager, + local=local) # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, first=True if not check else False) # First seen accepted # validates so update state @@ -2114,6 +2159,7 @@ def rotate(self, serder): return tholder, toader, wits, cuts, adds + def deriveBacks(self, serder): """Derives and return tuple of (wits, cuts, adds) for backers given current set and any changes provided by serder. @@ -2166,8 +2212,9 @@ def deriveBacks(self, serder): def valSigsWigsDel(self, serder, sigers, verfers, tholder, - wigers, toader, wits, local=True, - delseqner=None, delsaider=None): + wigers, toader, wits, *, + delseqner=None, delsaider=None, eager=False, + local=True): """ Returns triple (sigers, wigers, delegator) where: sigers is unique validated signature verified members of inputed sigers @@ -2192,14 +2239,19 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, toader (Number): instance of backer witness threshold wits (list): of qb64 non-transferable prefixes of witnesses used to derive werfers for wigers - local (bool): event source for validation logic - True means event source is local (protected). - False means event source is remote (unprotected). - Event validation logic is a function of local or remote delseqner (Seqner | None): instance of delegating event sequence number. If this event is not delegated then seqner is ignored delsaider (Saider | None): instance of of delegating event said. If this event is not delegated then saider is ignored + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ if len(verfers) < tholder.size: @@ -2210,8 +2262,7 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # Filters sigers to remove any signatures from locally membered groups # when not local (remote) event source. So that attacker can't source - # compromised signature remotely to satisfy threshold. - + # remotely compromised but locally membered signatures to satisfy threshold. if not local and self.locallyMembered(): # is this Kever's pre a local group if indices := self.locallyContributedIndices(verfers): for siger in list(sigers): # copy so clean del on original elements @@ -2232,16 +2283,34 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, raise ValidationError("No verified signatures for evt = {}." "".format(serder.ked)) - # Misfit check events that must be locally sourced (protected) get - # escrowed in order to repair the protection when appropriate - if (not local and - (self.locallyOwned() or - self.locallyWitnessed(wits=wits))): + # at least one valid controller signature so we can now escrow event + + # Misfit check events that must be locally sourced (protected). + # Misfit escrow process may repair (upgrade) the protection when appropriate + # Put all misfit checks up front so never goes into any escrow but + # misfit if fails any misfit checks. + # get delegator's delpre if any for misfit check + if serder.ilk == Ilks.dip: # dip so delpre in event "di" field + delpre = serder.delpre # delegator from dip event + if not delpre: # empty or None + raise ValidationError(f"Empty or missing delegator for delegated" + f" inception event = {serder.ked}.") + elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation + delpre = self.delpre # delpre in kever state + else: # not delegable event icp, rot, ixn + delpre = None + + # Misfit escrow checks + if (not local and (self.locallyOwned() or + self.locallyWitnessed(wits=wits) or + self.locallyDelegated(pre=delpre))): self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally owned" - f" or locally witnessed event" - f" = {serder.ked}, {wits}, {self.prefixes}") + raise MisfitEventSourceError(f"Nonlocal source for locally owned or" + f"locally witnessed or locally delegated" + f"event={serder.ked}, local aids=" + f"{self.prefixes}, {wits=}, " + f"delgator={delpre}.") werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers # get unique verified wigers and windices lists from wigers list @@ -2250,9 +2319,8 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # escrow if not fully signed vs signing threshold if not tholder.satisfy(indices): # at least one but not enough - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - if delseqner and delsaider: - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" f" on sigs for {[siger.qb64 for siger in sigers]}" f" for evt = {serder.ked}.") @@ -2263,70 +2331,83 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # prior next threshold in .ntholder and digers in .ndigers ondices = self.exposeds(sigers) if not self.ntholder.satisfy(indices=ondices): - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - if delseqner and delsaider: # save in case not attached later - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider,local=local) raise MissingSignatureError(f"Failure satisfying prior nsith=" f"{self.ntholder.sith} with exposed " f"sigs= {[siger.qb64 for siger in sigers]}" f" for new est evt={serder.ked}.") + # this point the sigers have been verified and the wigers have been verified + # even if locallyOwned or locallyMembered or locallyWitnessed. + # But the toad has not yet been verified. + + if not wits: + if toader.num != 0: # bad toad non-zero and not witnessed bad event + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + else: # wits so may need to validate toad + # The toad is only verified against the wigers as fully witnessed if + # not (locallyOwned or locallyMembered or locallyWitnessed) + if (not (self.locallyOwned() or self.locallyMembered() or + self.locallyWitnessed(wits=wits))): + if wits: # is witnessed + if toader.num < 1 or toader.num > len(wits): # out of bounds toad + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + else: # not witnessed + if toader.num != 0: # invalid toad + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + if len(windices) < toader.num: # not fully witnessed yet + if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, + seqner=delseqner, saider=delsaider, + local=local): + # cue to query for witness receipts + self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) + raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " + f"on witness sigs=" + f"{[siger.qb64 for siger in wigers]} " + f"for event={serder.ked}.") + + + # Delegator approves delegation by attaching valid source + # seal and reprocessing event which shows up here as delseqner, delsaider. + # Won't get to here if not local and locallyDelegated(delpre) misfit + # checks above will send nonlocal sourced delegable event to + # misfit escrow first. Mistfit escrow must first promote to local and + # reprocess event before we get to here. Assumes that attached source + # seal in this case can't be malicious since sourced locally. + # Doesn't get to here until fully signed and witnessed. + + if self.locallyDelegated(delpre): # local delegator + # must be local if locallyDelegated or caught above as misfit + if delseqner is None or delsaider is None: # missing delegation seal + # so escrow delegable. So local delegator can approve OOB. + # and create delegator event with valid event seal of this + # delegated event and then reprocess event with attached source + # seal to delegating event, i.e. delseqner, delsaider. + self.escrowDelegableEvent(serder=serder, sigers=sigers, + wigers=wigers, local=local) + raise MissingDelegableApprovalError(f"Missing approval for " + f" delegation by {delpre} of" + f"event = {serder.ked}.") + + # validateDelegation returns (None, None) when delegation validation + # does not apply. Raises ValidationError if validation applies but + # does not validate. + delseqner, delsaider = self.validateDelegation(serder, + sigers=sigers, + wigers=wigers, + wits=wits, + delpre=delpre, + delseqner=delseqner, + delsaider=delsaider, + eager=eager, + local=local) + + return (sigers, wigers, delpre, delseqner, delsaider) - # get delegator if any - if serder.ilk == Ilks.dip: - delpre = serder.delpre # delegator from dip event - if not delpre: # empty or None - raise ValidationError(f"Empty or missing delegator for delegated" - f" inception event = {serder.ked}.") - elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation - delpre = self.delpre - else: # not delegable event icp, rot, ixn - delpre = None - # delpre maybe None so ensure not None to pass into .locallyOwned which - # defaults to self.prefixer.qb64 when None - if not local and self.locallyOwned(delpre if delpre is not None else ''): - self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally" - f" delegated by {delpre} of" - f"event = {serder.ked}.") - - # short circuit witness validation when either locallyOwned or locallyWitnessed - # otherwise must validate fully witnessed - if not (self.locallyOwned() or self.locallyMembered() or self.locallyWitnessed(wits=wits)): - if wits: # is witnessed - if toader.num < 1 or toader.num > len(wits): # out of bounds toad - raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - else: # not witnessed - if toader.num != 0: # invalid toad - raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - - if len(windices) < toader.num: # not fully witnessed yet - if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, - seqner=delseqner, saider=delsaider, - local=local): - # cue to query for witness receipts - self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) - raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " - f"on witness sigs=" - f"{[siger.qb64 for siger in wigers]} " - f"for event={serder.ked}.") - - if delpre: - if not (delseqner and delsaider): - seal = dict(i=serder.ked["i"], s=serder.snh, d=serder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - - self.validateDelegation(serder, sigers=sigers, - wigers=wigers, wits=wits, - local=local, delpre=delpre, - delseqner=delseqner, delsaider=delsaider) - - return sigers, wigers, delpre, delseqner, delsaider def exposeds(self, sigers): @@ -2376,8 +2457,8 @@ def exposeds(self, sigers): return odxs - def validateDelegation(self, serder, sigers, wigers, wits, local=True, - delpre=None, delseqner=None, delsaider=None): + def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, + delseqner=None, delsaider=None, eager=False, local=True): """ Returns delegator's qb64 identifier prefix if validation successful. Assumes that local vs remote source checks have been applied before @@ -2394,6 +2475,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, Location Seal is from Delegate's establishment event Assumes state setup + self is last accepted event if any or yet to be accepted event, + serder is received event + Parameters: serder (SerderKERI): instance of delegated event serder sigers (list[Siger]): of Siger instances of indexed controller sigs of @@ -2402,18 +2486,27 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, delegated event. Assumes wigers is list of unique verified sigs wits (list[str]): of qb64 non-transferable prefixes of witnesses used to derive werfers for wigers - local (bool): event source for validation logic - True means event source is local (protected). - False means event source is remote (unprotected). - Event validation logic is a function of local or remote - delpre (str | None): qb64 prefix of delegator if any + delpre (str): qb64 prefix of delegator delseqner (Seqner | None): instance of delegating event sequence number. If this event is not delegated then ignored delsaider (Saider | None): instance of of delegating event digest. If this event is not delegated ignored + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + eager (bool): True means try harder to validate event by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote Returns: - (str | None): qb64 delegator prefix or None if not delegated + None Process Logic: A delegative event is processed differently for each of four different @@ -2522,17 +2615,20 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, events signed by compromised keys. The result of superseded recovery is that the KEL is forked at the sn of the superseding event. All events in the superseded branch of the fork still exist but, by virtue of being superseded, - are disputed. The set of superseding events in the superseding fork forms the authoritative - branch of the KEL. All the already seen superseded events in the superseded fork + are disputed. The set of superseding events in the superseding fork forms + the authoritative branch of the KEL. + + All the already seen superseded events in the superseded fork still remain in the KEL and may be viewed in order of their original acceptance because the database stores all accepted events in order of acceptance and denotes this order using the first seen ordinal number, fn. - The fn is not the same as the sn (sequence number). + + Recall that the fn is not the same as the sn (sequence number). Each event accepted into a KEL has a unique fn but multiple events due to recovery forks may share the same sn. - Superseding Rules for Recovery at given SN (sequence number) + Superseding Rules for Recovery at given 'sn' (sequence number) A0. Any rotation event may supersede an interaction event at the same sn. where that interaction event is not before any other rotation event. @@ -2547,17 +2643,27 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, at the same sn under either of the following conditions: B1. The superseding rotation's delegating event is later than - the superseded rotation's delegating event in the delegator's KEL, i.e. the - sn of the superseding event's delegation is higher than the superseded event's - delegation. - - B2. The sn of the superseding rotation's delegating event is the same as - the sn of the superseded rotation's delegating event in the delegator's KEL - and the superseding rotation's delegating event is a rotation and the - superseded rotation's delegating event is an interaction, - i.e. the superseding rotation's delegating event is itself a superseding - rotation of the superseded rotations delegating interaction event in the - delgator's KEL + the superseded rotation's delegating event in the delegator's KEL, + i.e. the sn of the superseding event's delegation is higher than + the superseded event's delegation. + + B2. The sn of the superseding rotation's delegating event is the + same as the sn of the superseded rotation's delegating event in the + delegator's KEL both have the same delegating event and the index + of the delegating seal of the superceding delegation seal is later + (higher) than the index of the delegating seal of the supderseded + rotation. In other words the delegating event is the same event but + the superseding delegation seal appears later in the seal list of the + delegating event than the superseded delegation seal. + + B3. The sn of the superseding rotation's delegating event is the + same as the sn of the superseded rotation's delegating event in the + delegator's KEL and the superseding rotation's delegating event is + a rotation and the superseded rotation's delegating event is an + interaction, i.e. the delegating events are not the same event and + the superseding rotation's delegating event is itself a superseding + rotation of the superseded rotations delegating interaction event + in the delgator's KEL. C. IF Neither A nor B is satisfied, then recursively apply rules A. and B. to the delegating events of those delegating events and so on until @@ -2572,23 +2678,55 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, A. or B. must be satisfied, or else the superseding rotation must be discarded. - Note: The latest seen deleagated rotation constraint means that any earlier - delegated rotations can NOT be superseded. This greatly simplifies the + Note: The latest seen delegated rotation constraint means that any earlier + delegated rotations CAN NOT be superseded. This greatly simplifies the validation logic and avoids a potential infinite regress of forks in the - delegated identifier's KEL. + delegated identifier's KEL while allowing the delegate to + detect that a compromised delegation has occurred and give an + opportunity for the delegator to refuse to approve a subsequent + delegated rotation without additional verification with the delegate + that the subsequent delegated rotation was not compromised. In order to capture control of a delegated identifier the attacker must issue a delegated rotation that rotates to keys under the control of the attacker that must be approved by the delegator. A recovery rotation must - therefore superseded the compromised rotation. If the attacker is able - to issue and get approved by the delegator a second rotation + therefore supersede the compromised rotation. If the attacker is able + to issue and get approved by the delegator a subsequent rotation that follows but does not supersede the compromising rotation then recovery is no longer possible because the delegatee would no longer control the privete keys needed to verifiably sign a recovery rotation. + One way that detectability may be assured is when the delegator imposes + a minimum time between approvals of a delegated rotation that is + sufficient for the delgate to detect a compromised rotation recovery. + Attempts to rotate sooner than the minimum time since the immediately + prior rotation are refused until further verification has occurred. + + A delegated rotation that occurs after the minimum time since the + immediately prior delegated rotation might be automatically approved + to minimize latency. While a subsequent delegated rotation that occurs + within the minimum time would not be approed to maximize safety. + The minimum time window is designed to give the delegate enough time + to detect a comprimised or duplicitious superseding rotation and + prevent the additional verification from proceding. + + ToDo: + + Repair the approval source seal couple in the 'aess' database on recursive + climb the kel tree. Once an event has been accepted into its kel. + Later adding a source seal couple to 'aes' should then be OK from a + security perspective since its only making discovery less expensive. + + When malicious source seal couple is received but event is validly + delegated and the delegation source seal is repaired then need to replace + malicious source seal couple with repaired seal so repaired seal not + malicous seal gets written to 'aes' db. When the event is valid but + non-delegated then need to nullify malicous source seal couple so it + does not get written to 'aes' datable + """ if not delpre: # not delegable so no delegation validation needed - return + return (None, None) # non-delegated so delseqner delsaider must be None # if we are the delegatee, accept the event without requiring the # delegator validation via an anchored delegation seal or by requiring @@ -2597,154 +2735,177 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # seal to be anchored in delegator's KEL. # Witness accepts without waiting for delegation seal to be anchored in # delegator's KEL. Witness cue in Kevery will then generate receipt - if self.locallyOwned() or self.locallyWitnessed(wits=wits): - return - - if self.kevers is None or delpre not in self.kevers: # drop event - # ToDo XXXX cue a trigger to get the KEL of the delegator - # the processDelegableEvent should also cue a trigger to get KEL + if (self.locallyOwned() or self.locallyMembered() or + self.locallyWitnessed(wits=wits)): + return (None, None) # not validated so delseqner delsaider must be None + + if self.kevers is None or delpre not in self.kevers: # missing delegator KEL + # ToDo XXXX cue a trigger to get the KEL of the delegator. This may + # require OOBIing with the delegator. + # The processPDEvent should also cue a trigger to get KEL # of delegator if still missing when processing escrow later. - self.escrowDelegableEvent(serder=serder, sigers=sigers, - wigers=wigers,local=local) - raise MissingDelegableApprovalError(f"Missing Kever for delegator" - f" = {delpre} of event" - f" = {serder.ked}.") + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"Missing KEL of delegator " + f"{delpre} of evt = {serder.ked}.") - dkever = self.kevers[delpre] + + dkever = self.kevers[delpre] # get delegator's KEL if dkever.doNotDelegate: # drop event if delegation not allowed raise ValidationError(f"Delegator = {delpre} for evt = {serder.ked}," f" does not allow delegation.") - - # Delegator accepts here without waiting for delegation seal to be anchored in - # delegator's KEL. But has already waited for fully receipted above. - # Once fully receipted, cue in Kevery will then trigger cue to approve - # delegation - - if delseqner is None or delsaider is None: - if self.locallyOwned(delpre): # local delegator so escrow. - # Won't get to here if not local and locallyOwned(delpre) because - # valSigsWigsDel will send nonlocal sourced delegable event to - # misfit escrow first. Mistfit escrow must first - # promote to local and reprocess event before we get to here - self.escrowDelegableEvent(serder=serder, sigers=sigers, - wigers=wigers, local=local) - raise MissingDelegableApprovalError(f"Missing approval for " - f" delegation by {delpre} of" - f"event = {serder.ked}.") - - else: # not local delegator so escrow - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) + dserder = None # no delegation event yet + if delseqner is None or delsaider is None: # missing delegation seal ref + if eager: # walk kel here to find + seal = dict(i=serder.pre, s=serder.snh, d=serder.said) + dserder = self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal) + if dserder is not None: # found seal in dserder + delseqner = coring.Seqner(sn=dserder.sn) # replace with found + delsaider = coring.Saider(qb64=dserder.said) # replace with found + + if not dserder: # just escrow and try later + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegation seal for delegator " f"{delpre} of evt = {serder.ked}.") - ssn = Number(num=delseqner.sn).validate(inceptive=False).sn - # ToDo XXXX need to replace Seqners with Numbers - - # get the dig of the delegating event. Using getKeLast ensures delegating - # event has not already been superceded - key = snKey(pre=delpre, sn=ssn) # database key - raw = self.db.getKeLast(key) # get dig of delegating event - - if raw is None: # no delegating event at key pre, sn - # ToDo XXXX process this cue of query to fetch delegating event from - # delegator - self.cues.push(dict(kin="query", q=dict(pre=delpre, - sn=delseqner.snh, - dig=delsaider.qb64))) - - # escrow event here - inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False - #sn = validateSN(sn=serder.snh, inceptive=inceptive) - sn = Number(num=serder.sn).validate(inceptive=inceptive).sn - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) - raise MissingDelegationError("No delegating event from {} at {} for " - "evt = {}.".format(delpre, - delsaider.qb64, - serder.ked)) - - # get the delegating event from dig - ddig = bytes(raw) - key = dgKey(pre=delpre, dig=ddig) # database key - raw = self.db.getEvt(key) - if raw is None: # drop event - raise ValidationError("Missing delegation from {} at event dig = {} for evt = {}." - "".format(delpre, ddig, serder.ked)) - - dserder = serdering.SerderKERI(raw=bytes(raw)) # delegating event - - - # compare digests to make sure they match here - if not dserder.compare(said=delsaider.qb64): # drop event - raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." - "".format(delpre, ddig, serder.ked)) - - - found = False # find event seal of delegated event in delegating data - # XXXX ToDo need to change logic here to support native CESR seals not just dicts - # for JSON, CBOR, MGPK - # may want to try harder here by walking KEL - for dseal in dserder.seals: # find delegating seal anchor - if tuple(dseal) == SealEvent._fields: - seal = SealEvent(**dseal) - if (seal.i == serder.pre and - seal.s == serder.sner.numh and - serder.compare(said=seal.d)): - found = True - break - - if not found: # drop event - raise ValidationError(f"Missing delegation seal in designated event" - f"from {delpre} in {dserder.seals} for " - f"evt = {serder.ked}.") - - # Found anchor so can assume delegation successful unless its one of - # the superseding condidtions. - # Assumes database is reverified each bootup chain-of-custody of disc broken. + if delseqner and delsaider and not dserder: # given couple not found + # ToDo XXXX need to replace Seqners with Numbers + # Get delegating event from delseqner and delpre + ssn = Number(num=delseqner.sn).validate(inceptive=False).sn + # get the dig of the delegating event. Using getKeLast ensures delegating + # event has not already been superceded + key = snKey(pre=delpre, sn=ssn) # database key + # get dig of last delegating event purported at sn + raw = self.db.getKeLast(key) # last means not disputed or superseded + if raw is None: # no delegatint event yet. no index at key pre, sn + # Have to wait until delegating event at sn shows up in kel + # ToDo XXXX process this cue of query to fetch delegating event from + # delegator + self.cues.push(dict(kin="query", q=dict(pre=delpre, + sn=delseqner.snh, + dig=delsaider.qb64))) + # escrow event here + inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False + sn = Number(num=serder.sn).validate(inceptive=inceptive).sn + # if we were to allow for malicous local delegator source seal then we + # must check for locallyOwned(delpre) first and escrowDelegable. + # otherwise escrowPDEvent + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") + + # get the latest delegating event candidate from dig given by pre,sn index + ddig = bytes(raw) + key = dgKey(pre=delpre, dig=ddig) # database key + raw = self.db.getEvt(key) # get actual last event + if raw is None: # drop event should never happen unless database is broken + raise ValidationError(f"Missing delegation from {delpre} at event " + f"dig = {ddig} for evt = {serder.ked}.") + + dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event + + found = False # find event seal of delegated event in delegating data + # search purported delegating event from source seal couple + + for dseal in dserder.seals: # find delegating seal anchor + # XXXX ToDo need to change logic here to support native CESR seals not just dicts + # for JSON, CBOR, MGPK + if tuple(dseal) == SealEvent._fields: + dseal = SealEvent(**dseal) + if (dseal.i == serder.pre and + dseal.s == serder.sner.numh and + serder.compare(said=dseal.d)): + found = True + break + + if not found: # nullify and escrow to try harder later + # worst case assume source seal was malicious so nullify it and + # attempt to repair by escrowing and eager search later + delseqner = delsaider = None # nullify + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegation seal for delegator " + f"{delpre} of evt = {serder.ked}.") + + ## Found valid anchoring seal of delegator delpre + ## compare saids to ensure match of delegating event and source seal + ## repair source seal to match last event at sn in source seal. + ## Original delegating event in seal may have been disputed or + ## superseded, but if latest matches then we want to repair + #if not dserder.compare(said=delsaider.qb64): # drop event + #raise ValidationError(f"Invalid delegation from {delpre} at event" + #f" dig={ddig} for evt={serder.ked}.") + + delseqner = Seqner(snh=dserder.snh) # replace with found + delsaider = Saider(qb64=dserder.said) # replace with found + + # Since found valid anchoring seal so can confirm delegation successful + # unless its one of the superseding conditions. + # Valid if not superseding drt of drt. + # Assumes database is reverified each bootup or otherwise when + # chain-of-custody of disc has been broken. # Rule for non-supeding delegated rotation of rotation. - # Returning delegator indicates success and eventually results acceptance + # Returning delegator indicates success and eventually results in acceptance # via Kever.logEvent which also writes the delgating event source couple to # db.aess so we can find it later - if ((serder.ilk == Ilks.dip) or # delegated inception - (serder.sner.num == self.sner.num + 1) or # inorder event - (serder.sner.num == self.sner.num and # superseding event - self.ilk == Ilks.ixn and # superseded is ixn - serder.ilk == Ilks.drt)): # recovery rotation superseding ixn - return # delegator # indicates delegation valid with return of delegator + if ((serder.ilk == Ilks.dip) or # superseding is delegated inception or + (serder.sner.num == self.sner.num + 1) or # superseding is inorder later or + (serder.sner.num == self.sner.num and # superseding event at same sn and + self.ilk == Ilks.ixn and # superseded is interaction and + serder.ilk == Ilks.drt)): # superseding is rotation + return (delseqner, delsaider) # indicates delegation valid # get to here means drt rotation superseding another drt rotation # Kever.logEvent saves authorizer (delegator) seal source couple in - # db.aess data base so can use it here to recusively look up delegating - # events + # db.aess (.getAes, .setAes etc) data base so can use it here to + # recusively look up delegating events # set up recursive search for superseding delegations + # get original potentially superseded delegation + serfo = self.serder # original accepted delegated event i.e. serf original + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, original=True, + eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") + # already have new potential superseding delegation serfn = serder # new potentially superseding delegated event i.e. serf new bossn = dserder # new delegating event of superseding delegated event i.e. boss new - serfo = self.serder # original delegated event i.e. serf original - bosso = self.fetchDelegatingEvent(delpre, serfo) while (True): # superseding delegated rotation of rotation recovery rules - # Only get to here if same sn for drt existing and drt superseding - - if (bossn.sn > bosso.sn or # later supersedes - (bossn.Ilk == Ilks.drt and - bosso.Ilk == Ilks.ixn) ): # drt supersedes ixn - return # delegator # valid superseding delegation + # Only get to here if same sn for delegated event and both + # existing and superseding delegated events are rotations + if (bossn.sn > bosso.sn or # superseding delgation is later or + (bossn.Ilk == Ilks.drt and # superseding delegation is rotation and + bosso.Ilk == Ilks.ixn) ): # superseded delegation is interaction + # valid superseding delegation up chain so tail link valid + return (delseqner, delsaider) # tail event's delegation source if bossn.said == bosso.said: # same delegating event nseals = [SealEvent(**seal) for seal in bossn.seals if tuple(seal) == SealEvent._fields] - nindex = nseals.index(SealEvent(i=serfn.pre, s=serfn.snh, d=serfn.said)) + nindex = nseals.index(SealEvent(i=serfn.pre, + s=serfn.snh, + d=serfn.said)) oseals = [SealEvent(**seal) for seal in bosso.seals if tuple(seal) == SealEvent._fields] - oindex = oseals.index(SealEvent(i=serfo.pre, s=serfo.snh, d=serfo.said)) + oindex = oseals.index(SealEvent(i=serfo.pre, + s=serfo.snh, + d=serfo.said)) - if nindex > oindex: # later seal supersedes + if nindex > oindex: # superseding delegation seal is later # assumes index can't be None - return # delegator # valid superseding delegation + # valid superseding delegation up chain so tail link valid + return (delseqner, delsaider) # tail event's delegation source - else: + else: # not superseded # ToDo: XXXX may want to cue up business logic for delegator # if self.mine(delegator): # failed attempt at recovery raise ValidationError(f"Invalid delegation recovery rotation" @@ -2752,40 +2913,145 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # tie condition same sn and drt so need to climb delegation chain serfn = bossn - bossn = self.fetchDelegatingEvent(delpre, serfn) + if not (bossn := self.fetchDelegatingEvent(delpre, serfn, + original=False, + eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") serfo = bosso - bosso = self.fetchDelegatingEvent(delpre, serfo) + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, + original=True, + eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") # repeat + # should never get to here + - def fetchDelegatingEvent(self, delegator, serder): - """Returns delegating event by delegator of delegated event given by - serder otherwise raises ValidationError. - Assumes serder is already delegated event + def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): + """Returns delegating event of delegator given by its aid delpre of + delegated event given by serder otherwise raises ValidationError. + + Assumes delegation event must have been accepted at some point even if + it has subsequently been superseded. Therefore only looks in .aess for + delegating event source seal references. So do not use for fetching + delegating eventsthat have yet to be accepted. + Checks that found delegation events have both been accepted. + + When delegated event in serder has been accepted then will repair its + .aess entry if needed. + + Returns: + dserder (SerderKERI): delegating key event with delegating seal of + delegated event serder. If can't fetch then + returns None when eager==False or raises + ValidationError if can't fetch but eager==True Parameters: - delegator (str): qb64 of identifier prefix of delegator + delpre (str): qb64 of identifier prefix of delegator serder (SerderKERI): delegated serder - - """ - dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key - if (couple := self.db.getAes(dgkey)): # delegation source couple + original (bool): True means delegated event is the original candidate + to be superseded. This means kel walk search should + include superseded or disputed events. + False means the delegated event is new candidate to + supersede. This means kel walk search should not + include superseded or disputed events. + eager (bool): True means do more expensive KEL walk instead of escrow + False means do not do expensive KEL walk now. + + Assumes db.aes source seal couple of delegating event cannot be written + unless that event has been accepted (first seen) into delegator's KEL + even if it has later been superseded. + Confirms this by checking the .fons database of the delegator. + + A malicious escrow of delegating event could only write source seal + couple to escrow in db.udes not in accepted db.aes + + Delegating event may have been superceded but delegated event validator + does not know it yet because db.aes keeps original delegating event + source seal from before is was superseded. + + Therefore lookup of delegating event needs to check delegating event via + db.fons and not last key event in .kels which would only return the + superseding event. + + When walking delegation tree a given delegating event may have + been superceded by another delegating event in the same delegator + KEL. This method does not distinguish between superseding and + superseded so can't assume that the delegating event is the last + event at its sn, i.e. the superseding event. + So this method must use db.fons to ensure that delegating + event was accepted (first seen) even if it has subsequently been + superseded. + + When the .aess entry is missing and eager is True and a valid delegation + is found and if delegate has been accepted then repairs missing .aess + entry. Otherwise does not repair but simply returns found delegation. + + Found delegation may not be superseding so do not repair .aess unless + delegate was already accepted. + """ + dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate + + if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate seqner, saider = deSourceCouple(couple) - dgkey = dgKey(pre=delegator, dig=saider) # event at its said - # get event by dig not by sn at last event because may have been superceded - if not (raw := self.db.getEvt(dgkey)): - # database broken this should never happen so do not supersede + deldig = saider.qb64 # dig of delegating event + # extra careful double check that .aes is valid by getting + # fner = first seen Number instance index + if not self.db.fons.get(keys=(delpre, deldig)): # Not first seen yet? + if original: # should not happen aes database broken + # repair by deleting aes and returning None so it escrows + # and then next time around find below with repair it + self.db.delAes(dgkey) # delete aes so next time repairs it + # superseding may not have happened yet so let it escrow + return None + ddgkey = dgKey(pre=delpre, dig=deldig) # database key of delegation + if not (raw := self.db.getEvt(ddgkey)): # in fons but no event + # database broken this should never happen raise ValidationError(f"Missing delegation event for {serder.ked}") + # original delegating event i.e. boss original + dserder = serdering.SerderKERI(raw=bytes(raw)) + return dserder - dserder = serdering.SerderKERI(raw=bytes(raw)) # original delegating event i.e. boss original - - else: #try to find seal the hard way + elif eager: #missing aes but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict - if not (dserder := self.db.findAnchoringSealEvent(pre=serder.delpre, seal=seal)): - # database broken this should never happen so do not supersede - raise ValidationError(f"Missing delegation source seal for {serder.ked}") + if original: # search all events in delegator's kel not just last + if not (dserder:=self.db.fetchAllSealingEventByEventSeal(pre=delpre, + seal=seal)): + # database broken this should never happen so do not validate + # since original must have been validated so it must have + # all its delegation chain. + raise ValidationError(f"Missing delegation source seal for {serder.ked}") + else: # only search last events in delegator's kel + if not (dserder:=self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal)): + # superseding delegation may not have happened yet so escrow + # ToDo XXXX need to cue up to get latest events in + # delegator's kel. + #raise ValidationError(f"Missing delegation source seal for {serder.ked}") + return None + + # Only repair .aess when found delegation is for delegated event that + # has been accepted delegated event i.e has .fons entry + if self.db.fons.get(keys=(serder.pre, serder.said)): + # Repair .aess of delegated event by writing found source + # seal couple of delegation. This is safe becaause we confirmed + # delegation event was accepted in delegator's kel. + couple = dserder.sner.huge.encode() + dserder.saidb + self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + + return dserder + + else: # not found so return None to escrow and get fixed later + return None - return dserder def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, @@ -2834,10 +3100,22 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, self.db.putEvt(dgkey, serder.raw) # idempotent (maybe already excrowed) # update event source - if seqner and saider: # delegation for authorized delegated or issued event + # delegation for authorized delegated or issued event + # when seqner and saider are provided they are only assured to be valid + # kever for event if kel is delegated and not locallyOwned + # and not locallyWitnessed as the validateDelegation is short circuited + # for non delegated kels, local controllers, and local witnesses. + # These checks prevent ddos via malicious source seal attachments. + # MUST NOT setAes if not delegated or locallyOwned or locallyWitnessed + if (self.delpre and not serder.ilk == Ilks.ixn and not self.locallyOwned() + and not self.locallyWitnessed(wits=wits) and seqner and saider): couple = seqner.qb64b + saider.qb64b self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + #if seqner and saider: + #couple = seqner.qb64b + saider.qb64b + #self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + if esr := self.db.esrs.get(keys=dgkeys): # preexisting esr if local and not esr.local: # local overwrites prexisting remote esr.local = local @@ -2904,8 +3182,9 @@ def escrowMFEvent(self, serder, sigers, wigers=None, if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent res = self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed @@ -2950,7 +3229,9 @@ def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): logger.debug("Kever state: escrowed delegable event=\n%s\n", json.dumps(serder.ked, indent=1)) - def escrowPSEvent(self, serder, sigers, wigers=None, local=True): + + def escrowPSEvent(self, serder, *, sigers=None, wigers=None, + seqner=None, saider=None, local=True): """ Update associated logs for escrow of partially signed event or fully signed delegated event but not yet verified delegation. @@ -2959,6 +3240,8 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): serder is SerderKERI instance of event sigers is list of Siger instances of indexed controller sigs wigers is optional list of Siger instance of indexed witness sigs + seqner is Seqner instance of sn of seal source event of delegator/issuer + saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -2967,9 +3250,12 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent - self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if sigers: + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.putEvt(dgkey, serder.raw) # update event source @@ -2988,43 +3274,71 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): "event = %s\n", serder.ked) - def escrowPACouple(self, serder, seqner, saider, local=True): + def escrowPWEvent(self, serder, *, sigers=None, wigers=None, + seqner=None, saider=None, local=True): """ - Update associated logs for escrow of partially authenticated issued event. - Assuming signatures are provided elsewhere. Partial authentication results - from either a partially signed event or a fully signed delegated event - but whose delegation is not yet verified. - - Escrow allows escrow processor to retrieve serder from key and source - couple from val in order to to re-verify authentication status. Sigs - are escrowed elsewhere. + Update associated logs for escrow of partially witnessed event Parameters: - serder is SerderKERI instance of delegated or issued event + serder is SerderKERI instance of event + sigers is optional list of Siger instances of indexed controller sigs + wigers is list of Siger instance of indexed witness sigs seqner is Seqner instance of sn of seal source event of delegator/issuer - saider is Saider instance of said of delegator/issuer + saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). Event validation logic is a function of local or remote + """ - local = True if local else False # ignored since not escrowing serder here + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent - logger.debug("Kever state: Escrowed source couple for partially signed " - "or delegated event = %s\n", serder.ked) + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent + + if sigers: + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + + self.db.putEvt(dgkey, serder.raw) + # update event source + if (esr := self.db.esrs.get(keys=dgkey)): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + + logger.debug("Kever state: Escrowed partially witnessed " + "event = %s\n", serder.ked) + return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) - def escrowPWEvent(self, serder, wigers, sigers=None, + def escrowPDEvent(self, serder, *, sigers=None, wigers=None, seqner=None, saider=None, local=True): """ - Update associated logs for escrow of partially witnessed event + Update associated logs for escrow of partially delegated or otherwise + authorized issued event. + Assumes sigs (controller signatures) and wigs (witness signatures) are + provided elsewhere. Partial authentication occurs once an event is + fully signed and witnessed but the authorizing (delegating) source + seal in the authorizer's (delegator's) key has not yet been verified. + + The escrow of the authorizing event ref via source seal is not idempotent + so that malicious or erroneous source seals may be nullified and/or + replaced by found source seals. + + Escrow allows escrow processor to retrieve serder from key and source + couple from val in order to to re-verify authentication status. Parameters: serder is SerderKERI instance of event - wigers is list of Siger instance of indexed witness sigs sigers is optional list of Siger instances of indexed controller sigs + wigers is list of Siger instance of indexed witness sigs seqner is Seqner instance of sn of seal source event of delegator/issuer saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic @@ -3036,16 +3350,25 @@ def escrowPWEvent(self, serder, wigers, sigers=None, local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent - if wigers: - self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) - if sigers: + + if sigers: # idempotent self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) - if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) + if wigers: # idempotent + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: # non-idempotent pin to repair replace + self.db.udes.pin(keys=dgkey, val=(seqner, saider)) # non-idempotent + logger.debug(f"Kever state: Replaced escrow source couple sn=" + f"{seqner.sn}, said={saider.qb64} for partially " + f"delegated/authorized event said={serder.said}.") + else: + self.db.udes.rem(keys=dgkey) # nullify non-idempotent + logger.debug(f"Kever state: Nullified escrow source couple for " + f"partially delegated/authorized event said=" + f"{serder.said}.") - self.db.putEvt(dgkey, serder.raw) - # update event source + self.db.putEvt(dgkey, serder.raw) # idempotent + + # update event source local or remote if (esr := self.db.esrs.get(keys=dgkey)): # preexisting esr if local and not esr.local: # local overwrites prexisting remote esr.local = local @@ -3055,9 +3378,9 @@ def escrowPWEvent(self, serder, wigers, sigers=None, esr = basing.EventSourceRecord(local=local) self.db.esrs.put(keys=dgkey, val=esr) - logger.debug("Kever state: Escrowed partially witnessed " - "event = %s\n", serder.ked) - return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) + logger.debug(f"Kever state: Escrowed partially delegated event=\n" + f"{serder.ked}\n.") + return self.db.pdes.add(keys=snKey(serder.preb, serder.sn), val=serder.saidb) def state(self): @@ -3120,6 +3443,8 @@ def fetchPriorDigers(self, sn: int | None = None) -> list | None: pre = self.prefixer.qb64 if sn is None: sn = self.lastEst.s - 1 + if sn < 0: + return None for digb in self.db.getKelBackIter(pre, sn): dgkey = dgKey(pre, digb) @@ -3370,7 +3695,7 @@ def fetchWitnessState(self, pre, sn): def processEvent(self, serder, sigers, *, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None, local=None): + firner=None, dater=None, eager=False, local=None): """ Process one event serder with attached indexd signatures sigers @@ -3390,6 +3715,11 @@ def processEvent(self, serder, sigers, *, wigers=None, dater (Dater|None): instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool|None): True means local (protected) event source. False means remote (unprotected). None means use default .local . @@ -3428,6 +3758,7 @@ def processEvent(self, serder, sigers, *, wigers=None, firner=firner if self.cloned else None, dater=dater if self.cloned else None, cues=self.cues, + eager=eager, local=local, check=self.check) self.kevers[pre] = kever # not exception so add to kevers @@ -3454,28 +3785,6 @@ def processEvent(self, serder, sigers, *, wigers=None, # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if self.local and kever.locallyOwned(): - # ToDo XXXX process this cue of query to send event to delegator - # to trigger generation of anchor in delegating event - # note for remote validators there is query cue in - # validateDelegation to query for anchoring event seal - pass - - # Moved logic to kever.validateDelegation trigger for delegation escrow - #if (self.local and - #kever.locallyOwned(kever.delpre if kever.delpre is not None else '')): # delpre may be None - ## ToDo XXXX need to cue task here to approve delegation by generating - ## an anchoring SealEvent of serder in delegators KEL - ## may include MFA and or business logic for the delegator i.e. is local - ## event that designates this controller as delegator triggers - ## this cue to approave delegation - #self.cues.push(dict(kin="approveDelegation", - #delegator=kever.delpre, - #serder=serder)) - - - - else: # not inception so can't verify sigs etc, add to out-of-order escrow self.escrowOOEvent(serder=serder, sigers=sigers, @@ -3536,7 +3845,7 @@ def processEvent(self, serder, sigers, *, wigers=None, delseqner=delseqner, delsaider=delsaider, firner=firner if self.cloned else None, dater=dater if self.cloned else None, - local=local, check=self.check) + eager=eager, local=local, check=self.check) # At this point the non-inceptive event (rot, drt, or ixn) # given by serder together with its attachments has been @@ -3561,24 +3870,6 @@ def processEvent(self, serder, sigers, *, wigers=None, # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if self.local and kever.locallyOwned(): - # ToDo XXXX process this cue of query to send event to delegator - # to trigger generation of anchor in delegating event - # note for remote validators there is query cue in - # validateDelegation to query for anchoring event seal - pass - - # Moved logic to kever.validateDelegation trigger for delegation escrow - #if (self.local and - #kever.locallyOwned(kever.delpre if kever.delpre is not None else '')): # delpre may be None - ## ToDo XXXX need to cue task here to approve delegation by generating - ## an anchoring SealEvent of serder in delegators KEL - ## may include MFA and or business logic for the delegator i.e. is local - ## event that designates this controller as delegator triggers - ## this cue to approave delegation - #self.cues.push(dict(kin="approveDelegation", - #delegator=kever.delpre, - #serder=serder)) else: # maybe duplicitous # check if duplicate of existing valid accepted event @@ -3805,7 +4096,7 @@ def processAttachedReceiptCouples(self, serder, cigars, firner=None, local=None) # Only accept receipt if event is latest event at sn. Means its been # first seen and is the most recent first seen with that sn if firner: - ldig = self.db.getFe(key=fnKey(pre=pre, sn=firner.sn)) + ldig = self.db.getFe(key=fnKey(pre=pre, fn=firner.sn)) else: ldig = self.db.getKeLast(key=snKey(pre=pre, sn=sn)) # retrieve dig of last event at sn. @@ -3982,7 +4273,7 @@ def processAttachedReceiptQuadruples(self, serder, trqs, firner=None, local=None sn = serder.sn if firner: # retrieve last event by fn ordinal - ldig = self.db.getFe(key=fnKey(pre=pre, sn=firner.sn)) + ldig = self.db.getFe(key=fnKey(pre=pre, fn=firner.sn)) else: # Only accept receipt if for last seen version of receipted event at sn ldig = self.db.getKeLast(key=snKey(pre=pre, sn=sn)) # retrieve dig of last event at sn. @@ -4604,7 +4895,7 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): kever = self.kevers[pre] if anchor: - if not self.db.findAnchoringSealEvent(pre=pre, seal=anchor): + if not self.db.fetchAllSealingEventByEventSeal(pre=pre, seal=anchor): self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) raise QueryNotFoundError("Query not found error={}.".format(ked)) @@ -4678,15 +4969,17 @@ def fetchEstEvent(self, pre, sn): found = False while not found: - dig = bytes(self.db.getKeLast(key=snKey(pre, sn))) + dig = self.db.getKeLast(key=snKey(pre, sn)) if not dig: return None # retrieve event by dig - raw = bytes(self.db.getEvt(key=dgKey(pre=pre, dig=dig))) + dig = bytes(dig) + raw = self.db.getEvt(key=dgKey(pre=pre, dig=dig)) if not raw: return None + raw = bytes(raw) serder = serdering.SerderKERI(raw=raw) # deserialize event raw if serder.ked["t"] in (Ilks.icp, Ilks.dip, Ilks.rot, Ilks.drt): return serder # establishment event so return @@ -4729,8 +5022,9 @@ def escrowMFEvent(self, serder, sigers, wigers=None, if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed logger.debug("Kevery process: escrowed misfit event=\n%s", @@ -4769,8 +5063,9 @@ def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None, l if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.addOoe(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed logger.debug("Kevery process: escrowed out of order event=\n%s", @@ -5030,6 +5325,7 @@ def processEscrows(self): self.processEscrowUnverWitness() self.processEscrowUnverNonTrans() self.processEscrowUnverTrans() + self.processEscrowPartialDels() self.processEscrowPartialWigs() self.processEscrowPartialSigs() self.processEscrowDuplicitous() @@ -5082,9 +5378,9 @@ def processEscrowOutOfOrders(self): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getOoeItemsNextIter(key=key): + for ekey, edig in self.db.getOoeItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below @@ -5183,6 +5479,7 @@ def processEscrowOutOfOrders(self): break key = ekey # setup next while iteration, with key after ekey + def processEscrowPartialSigs(self): """ Process events escrowed by Kever that were only partially fulfilled, @@ -5218,142 +5515,145 @@ def processEscrowPartialSigs(self): If successful then remove from escrow table """ - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.db.getPseItemsNextIter(key=key): - eserder = None - try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) - if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) - - # check date if expired then remove escrow. - dtb = self.db.getDts(dgkey) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) - - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) + #key = ekey = b'' # both start same. when not same means escrows found + #while True: # break when done + for ekey, edig in self.db.getPseItemIter(key=b''): + eserder = None + try: + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) - # get the escrowed event using edig - eraw = self.db.getEvt(dgkey) - if eraw is None: - # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - # get sigs and attach - sigs = self.db.getSigs(dgkey) - if not sigs: # otherwise its a list of sigs - # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - if not wigs: # empty list - # wigs maybe empty while waiting for first witness signature - # which may not arrive until some time after event is fully signed - # so just log for debugging but do not unescrow by raising - # ValidationError - logger.debug("Kevery unescrow wigs: No event wigs yet at." - "dig = %s", bytes(edig)) - - # seal source (delegator issuer if any) - delseqner = delsaider = None - couple = self.db.getPde(dgkey) - if couple is not None: - delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - couple = delseqner.qb64b + delsaider.qb64b - self.db.putPde(dgkey, couple) + # get the escrowed event using edig + eraw = self.db.getEvt(dgkey) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) - # If process does NOT validate sigs or delegation seal (when delegated), - # but there is still one valid signature then process will - # attempt to re-escrow and then raise MissingSignatureError - # or MissingDelegationSealError (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pse escrow is called by - # Kever.self.escrowPSEvent Which calls - # self.db.addPse(snKey(pre, sn), serder.digb) - # which in turn will not enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pse db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + # get sigs and attach + sigs = self.db.getSigs(dgkey) + if not sigs: # otherwise its a list of sigs + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) - except (MissingSignatureError, MissingDelegationError) as ex: - # still waiting on missing sigs or missing seal to validate - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + if not wigs: # empty list wigs witness sigs not wits + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) + + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=dgkey)): + delseqner, delsaider = couple + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + ## idempotent + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) + + # If process does NOT validate sigs or delegation seal (when delegated), + # but there is still one valid signature then process will + # attempt to re-escrow and then raise MissingSignatureError + # or MissingDelegationSealError (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pse escrow is called by + # Kever.self.escrowPSEvent Which calls + # self.db.addPse(snKey(pre, sn), serder.digb) + # which in turn will not enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pse db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + + except MissingSignatureError as ex: # MissingDelegationError) + # still waiting on missing sigs or missing seal to validate + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs so remove from escrow + self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.udes.rem(keys=dgkey) # leave here since could PartialDelegationEscrow - if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - self.cues.push(dict(kin="psUnescrow", serder=eserder)) + if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + self.cues.push(dict(kin="psUnescrow", serder=eserder)) - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + self.db.udes.rem(keys=dgkey) # remove escrow if any - if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - self.cues.push(dict(kin="psUnescrow", serder=eserder)) + if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + self.cues.push(dict(kin="psUnescrow", serder=eserder)) - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey + #if ekey == key: # still same so no escrows found on last while iteration + #break + #key = ekey # setup next while iteration, with key after ekey def processEscrowPartialWigs(self): """ @@ -5391,130 +5691,309 @@ def processEscrowPartialWigs(self): Process event as if it came in over the wire If successful then remove from escrow table """ + for ekey, edig in self.db.getPweItemIter(key=b''): + try: + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.db.getPweItemsNextIter(key=key): - try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) - if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) - # check date if expired then remove escrow. - dtb = self.db.getDts(dgkey) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) - # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) - if eraw is None: - # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + # get sigs + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) - # get sigs - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs - if not sigs: # empty list - # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + # get witness signatures (wigs not wits) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - # get wigs - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + if not wigs: # empty list + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) - if not wigs: # empty list - # wigs maybe empty while waiting for first witness signature - # which may not arrive until some time after event is fully signed - # so just log for debugging but do not unescrow by raising - # ValidationError - logger.debug("Kevery unescrow wigs: No event wigs yet at." - "dig = %s", bytes(edig)) + # raise ValidationError("Missing escrowed evt wigs at " + # "dig = {}.".format(bytes(edig))) - # raise ValidationError("Missing escrowed evt wigs at " - # "dig = {}.".format(bytes(edig))) + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + ## idempotent + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) + + # If process does NOT validate wigs then process will attempt + # to re-escrow and then raise MissingWitnessSignatureError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pwe escrow is called by + # Kever.self.escrowPWEvent Which calls + # self.db.addPwe(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pwe db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except MissingWitnessSignatureError as ex: # MissingDelegationError + # still waiting on missing witness sigs or delegation + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) - # seal source (delegator issuer if any) - delseqner = delsaider = None - couple = self.db.getPde(dgKey(pre, bytes(edig))) - if couple is not None: - delseqner, delsaider = deSourceCouple(couple) + except Exception as ex: # log diagnostics errors etc + # error other than waiting on wigs so remove from escrow + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.udes.rem(keys=dgkey) # leave here since could PartialDelegationEscrow + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + self.db.udes.rem(keys=dgkey) # remove escrow if any + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") - # If process does NOT validate wigs then process will attempt - # to re-escrow and then raise MissingWitnessSignatureError - # (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pwe escrow is called by - # Kever.self.escrowPWEvent Which calls - # self.db.addPwe(snKey(pre, sn), serder.digb) - # which in turn will NOT enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pwe db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. - # Assumes that controller signature validation and delegation - # validation will be successful as event would not be in - # partially witnessed escrow unless they had already validated - except MissingWitnessSignatureError as ex: - # still waiting on missing witness sigs - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + def processEscrowPartialDels(self): + """ + Process delgated events escrowed by Kever that were only partially fulfilled + due to missing or unverified delegation seals from delegators. + Events only make into this escrow after fully signed and if witnessed + fully witnessed. - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + db.pdes is an instance of subing.IoSetSuber and uses instance methods + for access to the underlying database. + Escrowed items in .pdes are indexed in database table keyed by prefix and + sequence number with multiple entries at same key held in insertion order. + This allows FIFO processing of escrowed events with same prefix and sn. - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + Value in each .pdes entry is dgkey (SAID) of event stored in db.evts where + db.evts holds SerderKERI.raw of event. + + Steps: + Each pass (walk index table) + For each prefix,sn + For each escrow item dup at prefix,sn: + Get Event + Get and Attach Signatures + Get and Attach Witness Signatures + Process event as if it came in over the wire + If successful then remove from escrow table + """ + + for (epre,), esn, edig in self.db.pdes.getOnItemIter(keys=b''): + try: + #pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(epre, edig) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) + + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) + + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) + + # get the escrowed event using edig + eraw = self.db.getEvt(dgkey) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) + + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + + # get sigs + sigs = self.db.getSigs(dgkey) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) + + # get witness signatures (wigs not wits) assumes wont be in this + # escrow if wigs not needed because no wits + wigs = self.db.getWigs(dgkey) # list of wigs if any + # may want to checks wits and wigs here. We are assuming that + # never get to this escrow if wits and not wigs + #if wits and not wigs: # non empty wits but empty wigs + ## wigs maybe empty if not wits or if wits while waiting + ## for first witness signature + ## which may not arrive until some time after event is fully signed + ## so just log for debugging but do not unescrow by raising + ## ValidationError + #logger.info("Kevery unescrow error: Missing event wigs at." + #"dig = %s", bytes(edig)) + + #raise ValidationError("Missing escrowed evt wigs at " + #"dig = {}.".format(bytes(edig))) + + # setup parameters to process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + + # seal source (delegator issuer if any) + # If delegator KEL not available should also cue a trigger to + # get it if still missing when processing escrow. + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(epre, edig))): + delseqner, delsaider = couple # provided + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: # found seal in srdr + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) + + # If process does NOT validate delegation then process will attempt + # to re-escrow and then raise MissingDelegationError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pses escrow is called by + # Kever.self.escrowPDEvent Which calls + # self.db.pdes.add(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrows in db.pdes. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except MissingDelegationError as ex: + # still waiting on missing delegation source seal + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + # removes one event escrow at key val + self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow + self.db.udes.rem(keys=dgkey) # remove source seal escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + # removes one event escrow at key val + self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow + self.db.udes.rem(keys=dgkey) # remove source seal escrow if any + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey def processEscrowUnverWitness(self): """ @@ -5568,9 +6047,9 @@ def processEscrowUnverWitness(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, ecouple in self.db.getUweItemsNextIter(key=key): + for ekey, ecouple in self.db.getUweItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow db key + pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key # get escrowed receipt's rdiger of receipted event and # wiger indexed signature of receipted event @@ -5688,9 +6167,9 @@ def processEscrowUnverNonTrans(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, etriplet in self.db.getUreItemsNextIter(key=key): + for ekey, etriplet in self.db.getUreItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item rsaider, sprefixer, cigar = deReceiptTriple(etriplet) cigar.verfer = Verfer(qb64b=sprefixer.qb64b) @@ -6201,9 +6680,9 @@ def processEscrowUnverTrans(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, equinlet in self.db.getVreItemsNextIter(key=key): + for ekey, equinlet in self.db.getVreItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) # check date if expired then remove escrow. @@ -6371,9 +6850,9 @@ def processEscrowDuplicitous(self): """ key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getLdeItemsNextIter(key=key): + for ekey, edig in self.db.getLdeItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 57d1346a..1418272d 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -470,7 +470,7 @@ def processEscrowReply(self): quadruple (prefixer, seqner, diger, siger) """ - for (route, ion), saider in self.db.rpes.getIoItemIter(): + for (route,), saider in self.db.rpes.getItemIter(): try: tsgs = eventing.fetchTsgs(db=self.db.ssgs, saider=saider) diff --git a/src/keri/core/scheming.py b/src/keri/core/scheming.py index 919d8e03..203ec950 100644 --- a/src/keri/core/scheming.py +++ b/src/keri/core/scheming.py @@ -266,7 +266,8 @@ class Schemer: """ - def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.Blake3_256): + def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), + code=MtrDex.Blake3_256, verify=True): """ Initialize instance of Schemer Deserialize if raw provided @@ -274,14 +275,19 @@ def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.B When serializing if kind provided then use kind instead of field in sed Parameters: - raw (bytes): of serialized schema - sed (dict): dict or None - if None its deserialized from raw - typ (JSONSchema): type of schema - kind (serialization): kind string value or None (see namedtuple coring.Serials) - supported kinds are 'json', 'cbor', 'msgpack', 'binary' - if kind (None): then its extracted from ked or raw - code (MtrDex): default digest code + raw (bytes): of serialized schema + sed (dict): dict or None + if None its deserialized from raw + typ (JSONSchema): type of schema + kind (serialization): kind string value or None (see namedtuple coring.Serials) + supported kinds are 'json', 'cbor', 'msgpack', 'binary' + if kind (None): then its extracted from ked or raw + code (MtrDex): default digest code + verify (bool): True means verify said(s) of given raw or sad. + Raises ValidationError if verification fails + False means don't verify. Useful to avoid unnecessary + reverification when deserializing from database + as opposed to over the wire reception. """ @@ -295,7 +301,7 @@ def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.B else: raise ValueError("Improper initialization need raw or sed.") - if not self._verify_schema(): + if verify and not self._verify_schema(): raise ValidationError("invalid kind {} for schema {}" "".format(self.kind, self.sed)) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index abb74a9b..f28ca035 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -561,8 +561,11 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, Either provided by parser from stream genus version or desired when generating Serder instance to stream verify (bool): True means verify said(s) of given raw or sad. - Raises ValidationError if verification fails - Ignore when raw not provided or when raw and saidify is True + False means don't verify. Useful to avoid unnecessary + reverification when deserializing from database + as opposed to over the wire reception. + Raises ValidationError if verification fails + Ignore when raw empty or when raw and saidify is True makify (bool): True means compute fields for sad including size and saids. proto (str | None): desired protocol type str value of Protocols diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 3a2b511f..14e9bfb5 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -606,11 +606,45 @@ class Baser(dbing.LMDBer): kevers (dict): Kever instances indexed by identifier prefix qb64 prefixes (OrderedSet): local prefixes corresponding to habitats for this db - .evts is named sub DB whose values are serialized events + .evts is named sub DB whose values are serialized key events dgKey DB is keyed by identifier prefix plus digest of serialized event Only one value per DB key is allowed + .kels is named sub DB of key event logs as indices that map sequence numbers + to serialized key event digests. + Actual serialized key events are stored in .evts by SAID digest + Uses sequence number or sn. + snKey + Values are digests used to lookup event in .evts sub DB + DB is keyed by identifier prefix plus sequence number of key event + More than one value per DB key is allowed + + .fels is named sub DB of first seen event logs (FEL) as indices that map + first seen ordinal number to digests. + Actual serialized key events are stored in .evts by SAID digest + This indexes events in first 'seen' accepted order for replay and + cloning of event log. + Uses first seen order number or fn. + fnKey + DB is keyed by identifier prefix plus monotonically increasing first + seen order number fn. + Value is digest of serialized event used to lookup event in .evts sub DB + Only one value per DB key is allowed. + Provides append only ordering of accepted first seen events. + + .fons is named subDB CesrSuber + Uses digest + dgKey + Maps prefix and digest to fn value (first seen ordinal number) of + the associated event. So one used pre and event digest, get its fn here + and then use fn to fetch event from .evnts by fn from .fels. + This ensures that any event looked up this way was first seen at + some point in time even if later superseded by a recovery rotation. + Whereas direct lookup in .evts could be escrowed events that may + never have been accepted as first seen. + CesrSuber(db=self, subkey='fons.', klas=core.Number) + .esrs is named sub DB instance of Komer of EventSourceRecord dgKey DB is keyed by identifier prefix plus digest (said) of serialized event @@ -636,8 +670,9 @@ class Baser(dbing.LMDBer): Escrow processing determines if and how to promote event source to local and then reprocess - .delegables is named sub DB instance of CesrIoSetSuber for delegable escrows - subkey "dees." + .delegables is named sub DB instance of CesrIoSetSuber for delegable event + escrows of key event with local delegator that need approval. + subkey "dees." delegable event escrows snKey DB is keyed by event controller prefix plus sn of serialized event where sn is 32 char hex string with leading zeros @@ -648,18 +683,6 @@ class Baser(dbing.LMDBer): source for a delegable event of a local delegator must first pass through the misfit escrow and get promoted to local source. - # delegable events escrows. events with local delegator that need approval - self.delegables = subing.CesrIoSetSuber(db=self, subkey='dees.', klas=coring.Diger) - - .fels is named sub DB of first seen event log table (FEL) of digests - that indexes events in first 'seen' accepted order for replay and - cloning of event log. Only one value per DB key is allowed. - Provides append only ordering of accepted first seen events. - Uses first seen order number or fn. - fnKey - DB is keyed by identifier prefix plus monotonically increasing first - seen order number fn. - Value is digest of serialized event used to lookup event in .evts sub DB .dtss is named sub DB of datetime stamp strings in ISO 8601 format of the datetime when the event was first escrosed and then later first @@ -740,37 +763,45 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .kels is named sub DB of key event log tables that map sequence numbers - to serialized event digests. + .pses is named sub DB of partially signed key event escrows + that each map pre + sequence number to serialized event digest. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .pses is named sub DB of partially signed escrowed event tables - that map sequence numbers to serialized event digests. + .pwes is named sub DB of partially witnessed key event escrowes + that each map pre + sequence number to serialized event digest. + these are for escrows of events with verified signatures but not + yet verified witness reciepts. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .pdes is named sub DB of partially delegated escrowed couples - that map digest to seal source couple that provides source - (delegator or issuer) event seal. Each couples is concatenations - of full qualified items, snu+dig of authorizing (delegating or - issuing) source event. - dgKey - Values are couples used to lookup source event in .kels sub DB - DB is keyed by identifier prefix plus digest of key event - Only one value per DB key is allowed - - .pwes is named sub DB of partially witnessed escrowed event tables - that map sequence numbers to serialized event digests. + .pdes is named sub DB of partially delegated key event escrows + that each map pre + sequence number to serialized event digest. This is + used in conjunction with .udes which escrows the associated seal + source couple. snKey - Values are digests used to lookup event in .evts sub DB + Values are digests used to lookup delegated event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed + .udes is named sub DB of unverified delegation seal source couple escrows + that map (pre, digest) of delegated event to delegating seal source + couple (sn, dig) that provides source delegator event seal. + Each couple is concatenation of fully qualified items, snu+dig + of delegating source event in which seal of delegated event appears. + dgKey + Values are serialized instances of CatCesrSuber as couples + (Seqner.qb64b, Saider.qb64b) used to lookup source event in delegator's + KEL. + DB is keyed by identifier prefix plus digest of key event + Only one value per DB key is allowed + Once escrow is accepted then delegation approval source seal couples + go into .aess database of authorizing event source seal couples + .uwes is named sub DB of unverified event indexed escrowed couples from witness signers. Witnesses are from witness list of latest establishment event for the receipted event. Each couple is concatenation of fully @@ -808,10 +839,6 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .fons is named subDB instance of MatterSuber that maps - (prefix, digest) e.g. dgKey to fn value (first seen ordinal number) of - the associated event. So one can lookup event digest, get its fn here - and then use fn to fetch event by fn from .fels. .states (subkey stts.) is named subDB instance of SerderSuber that maps a prefix to the latest keystate for that prefix. Used by ._kevers.db for read @@ -978,6 +1005,7 @@ def reopen(self, **kwa): self.evts = self.env.open_db(key=b'evts.') self.fels = self.env.open_db(key=b'fels.') + self.kels = self.env.open_db(key=b'kels.', dupsort=True) self.dtss = self.env.open_db(key=b'dtss.') self.aess = self.env.open_db(key=b'aess.') self.sigs = self.env.open_db(key=b'sigs.', dupsort=True) @@ -986,16 +1014,20 @@ def reopen(self, **kwa): self.ures = self.env.open_db(key=b'ures.', dupsort=True) self.vrcs = self.env.open_db(key=b'vrcs.', dupsort=True) self.vres = self.env.open_db(key=b'vres.', dupsort=True) - self.kels = self.env.open_db(key=b'kels.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) - self.pdes = self.env.open_db(key=b'pdes.') self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) + self.pdes = subing.OnIoDupSuber(db=self, subkey='pdes.') + self.udes = subing.CatCesrSuber(db=self, subkey='udes.', + klas=(coring.Seqner, coring.Saider)) self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) self.dels = self.env.open_db(key=b'dels.', dupsort=True) self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) self.qnfs = subing.IoSetSuber(db=self, subkey="qnfs.", dupsort=True) + # events as ordered by first seen ordinals + self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=core.Number) + self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) self.vers = subing.Suber(db=self, subkey="vers.") @@ -1010,8 +1042,6 @@ def reopen(self, **kwa): # delegable events escrows. events with local delegator that need approval self.delegables = subing.IoSetSuber(db=self, subkey='dees.') - # events as ordered by first seen ordinals - self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=core.Number) # Kever state made of KeyStateRecord key states # TODO: clean self.states = koming.Komer(db=self, @@ -1443,7 +1473,7 @@ def clean(self): cpydb.add(keys=keys, val=val) # Insecure raw imgs database copy. - for (key, val) in self.getAllItemIter(self.imgs): + for (key, val) in self.getTopItemIter(self.imgs): copy.imgs.setVal(key=key, val=val) # clone .habs habitat name prefix Komer subdb @@ -1507,11 +1537,18 @@ def clonePreIter(self, pre, fn=0): Returns iterator of first seen event messages with attachments for the identifier prefix pre starting at first seen order number, fn. Essentially a replay in first seen order with attachments + + Parameters: + pre is bytes of itdentifier prefix + fn is int fn to resume replay. Earliset is fn=0 + + Returns: + msgs (Iterator): over all items with pre starting at fn """ if hasattr(pre, 'encode'): pre = pre.encode("utf-8") - for fn, dig in self.getFelItemPreIter(pre, fn=fn): + for _, fn, dig in self.getFelItemPreIter(pre, fn=fn): try: msg = self.cloneEvtMsg(pre=pre, fn=fn, dig=dig) except Exception: @@ -1519,18 +1556,19 @@ def clonePreIter(self, pre, fn=0): yield msg - def cloneAllPreIter(self, key=b''): + def cloneAllPreIter(self): """ Returns iterator of first seen event messages with attachments for all - identifier prefixes starting at key. If key == b'' then rstart at first + identifier prefixes starting at key. If key == b'' then start at first key in databse. Use key to resume replay. Essentially a replay in first seen order with attachments of entire set of FELs. - Parameters: - key (bytes): fnKey(pre, fn) + Returns: + msgs (Iterator): over all items in db + """ - for pre, fn, dig in self.getFelItemAllPreIter(key=key): + for pre, fn, dig in self.getFelItemAllPreIter(): try: msg = self.cloneEvtMsg(pre=pre, fn=fn, dig=dig) except Exception: @@ -1628,7 +1666,7 @@ def cloneDelegation(self, kever): for dmsg in self.clonePreIter(pre=kever.delpre, fn=0): yield dmsg - def findAnchoringSealEvent(self, pre, seal, sn=0): + def fetchAllSealingEventByEventSeal(self, pre, seal, sn=0): """ Search through a KEL for the event that contains a specific anchored SealEvent type of provided seal but in dict form and is also fully @@ -1652,21 +1690,61 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): for evt in self.getEvtPreIter(pre=pre, sn=sn): # includes disputed & superseded srdr = serdering.SerderKERI(raw=evt.tobytes()) - for eseal in srdr.seals or []: + for eseal in srdr.seals or []: # or [] for seals 'a' field missing if tuple(eseal) == eventing.SealEvent._fields: eseal = eventing.SealEvent(**eseal) # convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): return srdr return None + # use alias here until can change everywhere for backwards compatibility + findAnchoringSealEvent = fetchAllSealingEventByEventSeal # alias - def findAnchoringSeal(self, pre, seal, sn=0): + + def fetchLastSealingEventByEventSeal(self, pre, seal, sn=0): """ - Search through a KEL for the event that contains an anchored - Seal with same Seal type as provided seal but in dict form. - Searchs from sn forward (default = 0). Only searches last event at any - sn therefore does not search any disputed or superseded events. - Returns the Serder of the first event with the anchored Seal seal, + Search through a KEL for the last event at any sn but that contains a + specific anchored event seal of namedtuple SealEvent type that matches + the provided seal in dict form and is also fully witnessed. + Searchs from provided sn forward (default = 0). + Searches only last events in KEL of pre so does not include disputed + and/or superseded events. + + Returns: + srdr (Serder): instance of the first event with the matching + anchoring SealEvent seal, + None if not found + + Parameters: + pre (bytes|str): identifier of the KEL to search + seal (dict): dict form of Seal of any type SealEvent to find in anchored + seals list of each event + sn (int): beginning sn to search + + """ + if tuple(seal) != eventing.SealEvent._fields: # wrong type of seal + return None + + seal = eventing.SealEvent(**seal) #convert to namedtuple + + for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # no disputed or superseded + srdr = serdering.SerderKERI(raw=evt.tobytes()) + for eseal in srdr.seals or []: # or [] for seals 'a' field missing + if tuple(eseal) == eventing.SealEvent._fields: + eseal = eventing.SealEvent(**eseal) # convert to namedtuple + if seal == eseal and self.fullyWitnessed(srdr): + return srdr + return None + + + + def fetchLastSealingEventBySeal(self, pre, seal, sn=0): + """Only searches last event at any sn therefore does not search + any disputed or superseded events. + Search through last event at each sn in KEL for the event that contains + an anchored Seal with same Seal type as provided seal but in dict form. + Searchs from sn forward (default = 0). + Returns the Serder of the first found event with the anchored Seal seal, None if not found Parameters: @@ -1681,7 +1759,7 @@ def findAnchoringSeal(self, pre, seal, sn=0): for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # only last evt at sn srdr = serdering.SerderKERI(raw=evt.tobytes()) - for eseal in srdr.seals or []: + for eseal in srdr.seals or []: # or [] for seals 'a' field missing if tuple(eseal) == Seal._fields: # same type of seal eseal = Seal(**eseal) #convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): @@ -1834,7 +1912,6 @@ def getEvtPreIter(self, pre, sn=0): for dig in self.getKelIter(pre, sn=sn): try: - dgkey = dbing.dgKey(pre, dig) # get message if not (raw := self.getEvt(key=dgkey)): raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) @@ -1920,13 +1997,13 @@ def appendFe(self, pre, val): pre is bytes identifier prefix for event val is event digest """ - return self.appendOrdValPre(db=self.fels, pre=pre, val=val) + return self.appendOnVal(db=self.fels, key=pre, val=val) def getFelItemPreIter(self, pre, fn=0): """ - Returns iterator of all (fn, dig) duples in first seen order for all events - with same prefix, pre, in database. Items are sorted by fnKey(pre, fn) - where fn is first seen order number int. + Returns iterator of all (pre, fn, dig) triples in first seen order for + all events with same prefix, pre, in database. Items are sorted by + fnKey(pre, fn) where fn is first seen order number int. Returns a First Seen Event Log FEL. Returned items are duples of (fn, dig): Where fn is first seen order number int and dig is event digest for lookup in .evts sub db. @@ -1936,11 +2013,14 @@ def getFelItemPreIter(self, pre, fn=0): Parameters: pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 + + Returns: + items (Iterator[(pre, fn, val)]): over all items starting at pre, on """ - return self.getAllOrdItemPreIter(db=self.fels, pre=pre, on=fn) + return self.getOnItemIter(db=self.fels, key=pre, on=fn) - def getFelItemAllPreIter(self, key=b''): + def getFelItemAllPreIter(self): """ Returns iterator of all (pre, fn, dig) triples in first seen order for all events for all prefixes in database. Items are sorted by @@ -1956,7 +2036,8 @@ def getFelItemAllPreIter(self, key=b''): key is key location in db to resume replay, If empty then start at first key in database """ - return self.getAllOrdItemAllPreIter(db=self.fels, key=key) + #return self.getAllOnItemAllPreIter(db=self.fels, key=key) + return self.getOnItemIter(db=self.fels, key=b'') def putDts(self, key, val): """ @@ -2207,7 +2288,7 @@ def putUres(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ures, key, vals) + return self.putIoDupVals(self.ures, key, vals) def addUre(self, key, val): """ @@ -2218,7 +2299,7 @@ def addUre(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ures, key, val) + return self.addIoDupVal(self.ures, key, val) def getUres(self, key): """ @@ -2228,7 +2309,7 @@ def getUres(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ures, key) + return self.getIoDupVals(self.ures, key) def getUresIter(self, key): """ @@ -2238,7 +2319,7 @@ def getUresIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.ures, key) + return self.getIoDupValsIter(self.ures, key) def getUreLast(self, key): """ @@ -2248,23 +2329,10 @@ def getUreLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ures, key) + return self.getIoDupValLast(self.ures, key) - def getUreItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event triple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is triple dig+pre+cig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.ures, key, skip) - def getUreItemsNextIter(self, key=b'', skip=True): + def getUreItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event triple items at next @@ -2276,7 +2344,8 @@ def getUreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ures, key, skip) + return self.getTopIoDupItemIter(self.ures, key) + #return self.getIoDupItemsNextIter(self.ures, key, skip) def cntUres(self, key): """ @@ -2284,7 +2353,7 @@ def cntUres(self, key): Return count of receipt triplets at key Returns zero if no entry at key """ - return self.cntIoVals(self.ures, key) + return self.cntIoDupVals(self.ures, key) def delUres(self, key): """ @@ -2292,7 +2361,7 @@ def delUres(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.ures, key) + return self.delIoDupVals(self.ures, key) def delUre(self, key, val): """ @@ -2304,7 +2373,7 @@ def delUre(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ures, key, val) + return self.delIoDupVal(self.ures, key, val) def putVrcs(self, key, vals): """ @@ -2374,7 +2443,7 @@ def putVres(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.vres, key, vals) + return self.putIoDupVals(self.vres, key, vals) def addVre(self, key, val): """ @@ -2385,7 +2454,7 @@ def addVre(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.vres, key, val) + return self.addIoDupVal(self.vres, key, val) def getVres(self, key): """ @@ -2395,7 +2464,7 @@ def getVres(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.vres, key) + return self.getIoDupVals(self.vres, key) def getVresIter(self, key): """ @@ -2405,7 +2474,7 @@ def getVresIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.vres, key) + return self.getIoDupValsIter(self.vres, key) def getVreLast(self, key): """ @@ -2415,23 +2484,9 @@ def getVreLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.vres, key) + return self.getIoDupValLast(self.vres, key) - def getVreItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event quintuple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is Quinlet is edig + spre + ssnu + sdig +sig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.vres, key, skip) - - def getVreItemsNextIter(self, key=b'', skip=True): + def getVreItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event quintuple items at next @@ -2443,7 +2498,8 @@ def getVreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.vres, key, skip) + return self.getTopIoDupItemIter(self.vres, key) + #return self.getIoDupItemsNextIter(self.vres, key, skip) def cntVres(self, key): """ @@ -2451,7 +2507,7 @@ def cntVres(self, key): Return count of receipt quinlets at key Returns zero if no entry at key """ - return self.cntIoVals(self.vres, key) + return self.cntIoDupVals(self.vres, key) def delVres(self, key): """ @@ -2459,7 +2515,7 @@ def delVres(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.vres, key) + return self.delIoDupVals(self.vres, key) def delVre(self, key, val): """ @@ -2471,7 +2527,7 @@ def delVre(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.vres, key, val) + return self.delIoDupVal(self.vres, key, val) def putKes(self, key, vals): """ @@ -2481,7 +2537,7 @@ def putKes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.kels, key, vals) + return self.putIoDupVals(self.kels, key, vals) def addKe(self, key, val): """ @@ -2491,7 +2547,7 @@ def addKe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.kels, key, val) + return self.addIoDupVal(self.kels, key, val) def getKes(self, key): """ @@ -2500,7 +2556,7 @@ def getKes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.kels, key) + return self.getIoDupVals(self.kels, key) def getKeLast(self, key): """ @@ -2509,7 +2565,7 @@ def getKeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.kels, key) + return self.getIoDupValLast(self.kels, key) def cntKes(self, key): """ @@ -2517,7 +2573,7 @@ def cntKes(self, key): Return count of dup key event dig val at key Returns zero if no entry at key """ - return self.cntIoVals(self.kels, key) + return self.cntIoDupVals(self.kels, key) def delKes(self, key): """ @@ -2525,7 +2581,7 @@ def delKes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.kels, key) + return self.delIoDupVals(self.kels, key) def getKelIter(self, pre, sn=0): @@ -2549,7 +2605,10 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAllPreIter(self.kels, pre, on=sn) + + return (self.getOnIoDupValIter(self.kels, pre, on=sn)) + + #return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) def getKelBackIter(self, pre, sn=0): @@ -2573,7 +2632,7 @@ def getKelBackIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAllPreBackIter(self.kels, pre, sn) + return self.getOnIoDupValBackIter(self.kels, pre, sn) def getKelLastIter(self, pre, sn=0): @@ -2597,7 +2656,7 @@ def getKelLastIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValLastAllPreIter(self.kels, pre, on=sn) + return self.getOnIoDupLastValIter(self.kels, pre, on=sn) def putPses(self, key, vals): @@ -2608,7 +2667,7 @@ def putPses(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.pses, key, vals) + return self.putIoDupVals(self.pses, key, vals) def addPse(self, key, val): """ @@ -2618,7 +2677,7 @@ def addPse(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.pses, key, val) + return self.addIoDupVal(self.pses, key, val) def getPses(self, key): """ @@ -2627,7 +2686,7 @@ def getPses(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.pses, key) + return self.getIoDupVals(self.pses, key) def getPsesIter(self, key): """ @@ -2636,7 +2695,7 @@ def getPsesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.pses, key) + return self.getIoDupValsIter(self.pses, key) def getPseLast(self, key): """ @@ -2645,21 +2704,9 @@ def getPseLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.pses, key) - - def getPseItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.pses, key, skip) + return self.getIoDupValLast(self.pses, key) - def getPseItemsNextIter(self, key=b'', skip=True): + def getPseItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event dig items at next key after key. @@ -2669,7 +2716,8 @@ def getPseItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.pses, key, skip) + return self.getTopIoDupItemIter(self.pses, key) + #return self.getIoDupItemsNextIter(self.pses, key, skip) def cntPses(self, key): """ @@ -2677,7 +2725,7 @@ def cntPses(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.pses, key) + return self.cntIoDupVals(self.pses, key) def delPses(self, key): """ @@ -2685,7 +2733,7 @@ def delPses(self, key): Deletes all values at key in db. Returns True If key exists in db Else False """ - return self.delIoVals(self.pses, key) + return self.delIoDupVals(self.pses, key) def delPse(self, key, val): """ @@ -2697,42 +2745,8 @@ def delPse(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.pses, key, val) - - def putPde(self, key, val): - """ - Use dgKey() - Write serialized event source couple to key (snu+dig) - Does not overwrite existing val if any - Returns True If val successfully written Else False - Returns False if key already exists - """ - return self.putVal(self.pdes, key, val) - - def setPde(self, key, val): - """ - Use dgKey() - Write serialized seal source couple to key (snu+dig) - Overwrites existing val if any - Returns True If val successfully written Else False - """ - return self.setVal(self.pdes, key, val) - - def getPde(self, key): - """ - Use dgKey() - Return seal source couple at key - Returns None if no entry at key - """ - return self.getVal(self.pdes, key) + return self.delIoDupVal(self.pses, key, val) - def delPde(self, key): - """ - Use dgKey() - Deletes value at key. - Returns True If key exists in database Else False - """ - return self.delVal(self.pdes, key) def putPwes(self, key, vals): """ @@ -2742,7 +2756,7 @@ def putPwes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.pwes, key, vals) + return self.putIoDupVals(self.pwes, key, vals) def addPwe(self, key, val): """ @@ -2752,7 +2766,7 @@ def addPwe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.pwes, key, val) + return self.addIoDupVal(self.pwes, key, val) def getPwes(self, key): """ @@ -2761,7 +2775,7 @@ def getPwes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.pwes, key) + return self.getIoDupVals(self.pwes, key) def getPwesIter(self, key): """ @@ -2770,7 +2784,7 @@ def getPwesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.pwes, key) + return self.getIoDupValsIter(self.pwes, key) def getPweLast(self, key): """ @@ -2779,21 +2793,9 @@ def getPweLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.pwes, key) + return self.getIoDupValLast(self.pwes, key) - def getPweItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial witnessed escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.pwes, key, skip) - - def getPweItemsNextIter(self, key=b'', skip=True): + def getPweItemIter(self, key=b''): """ Use sgKey() Return iterator of partial witnessed escrowed event dig items at next key after key. @@ -2803,7 +2805,20 @@ def getPweItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.pwes, key, skip) + return self.getTopIoDupItemIter(self.pwes, key) + #return self.getIoDupItemsNextIter(self.pwes, key, skip) + + #def getPweIoDupItemIter(self, key=b''): + #""" + #Use sgKey() + #Return iterator of partial witnessed escrowed event dig items at next key after key. + #Items is (key, val) where proem has already been stripped from val + #If key is b'' empty then returns dup items at first key. + #If skip is False and key is not b'' empty then returns dup items at key + #Raises StopIteration Error when empty + #Duplicates are retrieved in insertion order. + #""" + #return self.getTopIoDupItemIter(self.pwes, key) def cntPwes(self, key): """ @@ -2811,7 +2826,7 @@ def cntPwes(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.pwes, key) + return self.cntIoDupVals(self.pwes, key) def delPwes(self, key): """ @@ -2819,7 +2834,7 @@ def delPwes(self, key): Deletes all values at key in db. Returns True If key exists in db Else False """ - return self.delIoVals(self.pwes, key) + return self.delIoDupVals(self.pwes, key) def delPwe(self, key, val): """ @@ -2831,7 +2846,7 @@ def delPwe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.pwes, key, val) + return self.delIoDupVal(self.pwes, key, val) def putUwes(self, key, vals): """ @@ -2842,7 +2857,7 @@ def putUwes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.uwes, key, vals) + return self.putIoDupVals(self.uwes, key, vals) def addUwe(self, key, val): """ @@ -2853,7 +2868,7 @@ def addUwe(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.uwes, key, val) + return self.addIoDupVal(self.uwes, key, val) def getUwes(self, key): """ @@ -2863,7 +2878,7 @@ def getUwes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.uwes, key) + return self.getIoDupVals(self.uwes, key) def getUwesIter(self, key): """ @@ -2873,7 +2888,7 @@ def getUwesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.uwes, key) + return self.getIoDupValsIter(self.uwes, key) def getUweLast(self, key): """ @@ -2883,23 +2898,9 @@ def getUweLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.uwes, key) - - def getUweItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed receipt couple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is couple edig+wig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.uwes, key, skip) + return self.getIoDupValLast(self.uwes, key) - def getUweItemsNextIter(self, key=b'', skip=True): + def getUweItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed receipt couple items at next @@ -2911,7 +2912,8 @@ def getUweItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.uwes, key, skip) + return self.getTopIoDupItemIter(self.uwes, key) + #return self.getIoDupItemsNextIter(self.uwes, key, skip) def cntUwes(self, key): """ @@ -2919,7 +2921,7 @@ def cntUwes(self, key): Return count of receipt couples at key Returns zero if no entry at key """ - return self.cntIoVals(self.uwes, key) + return self.cntIoDupVals(self.uwes, key) def delUwes(self, key): """ @@ -2927,7 +2929,7 @@ def delUwes(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.uwes, key) + return self.delIoDupVals(self.uwes, key) def delUwe(self, key, val): """ @@ -2939,7 +2941,7 @@ def delUwe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.uwes, key, val) + return self.delIoDupVal(self.uwes, key, val) def putOoes(self, key, vals): """ @@ -2949,7 +2951,7 @@ def putOoes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ooes, key, vals) + return self.putIoDupVals(self.ooes, key, vals) def addOoe(self, key, val): """ @@ -2959,7 +2961,7 @@ def addOoe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ooes, key, val) + return self.addIoDupVal(self.ooes, key, val) def getOoes(self, key): """ @@ -2968,7 +2970,7 @@ def getOoes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ooes, key) + return self.getIoDupVals(self.ooes, key) def getOoeLast(self, key): """ @@ -2977,21 +2979,9 @@ def getOoeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ooes, key) + return self.getIoDupValLast(self.ooes, key) - def getOoeItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of out of order escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.ooes, key, skip) - - def getOoeItemsNextIter(self, key=b'', skip=True): + def getOoeItemIter(self, key=b''): """ Use sgKey() Return iterator of out of order escrowed event dig items at next key after key. @@ -3001,7 +2991,8 @@ def getOoeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ooes, key, skip) + return self.getTopIoDupItemIter(self.ooes, key) + #return self.getIoDupItemsNextIter(self.ooes, key, skip) def cntOoes(self, key): """ @@ -3009,7 +3000,7 @@ def cntOoes(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.ooes, key) + return self.cntIoDupVals(self.ooes, key) def delOoes(self, key): """ @@ -3017,7 +3008,7 @@ def delOoes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.ooes, key) + return self.delIoDupVals(self.ooes, key) def delOoe(self, key, val): """ @@ -3030,7 +3021,8 @@ def delOoe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ooes, key, val) + return self.delIoDupVal(self.ooes, key, val) + def putDes(self, key, vals): """ @@ -3040,7 +3032,7 @@ def putDes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.dels, key, vals) + return self.putIoDupVals(self.dels, key, vals) def addDe(self, key, val): """ @@ -3050,7 +3042,7 @@ def addDe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.dels, key, val) + return self.addIoDupVal(self.dels, key, val) def getDes(self, key): """ @@ -3059,7 +3051,7 @@ def getDes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.dels, key) + return self.getIoDupVals(self.dels, key) def getDeLast(self, key): """ @@ -3069,7 +3061,7 @@ def getDeLast(self, key): Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.dels, key) + return self.getIoDupValLast(self.dels, key) def cntDes(self, key): """ @@ -3077,7 +3069,7 @@ def cntDes(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.dels, key) + return self.cntIoDupVals(self.dels, key) def delDes(self, key): """ @@ -3085,9 +3077,9 @@ def delDes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.dels, key) + return self.delIoDupVals(self.dels, key) - def getDelIter(self, pre): + def getDelItemIter(self, pre): """ Returns iterator of all dup vals in insertion order for any entries with same prefix across all sequence numbers including gaps. @@ -3104,7 +3096,8 @@ def getDelIter(self, pre): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAnyPreIter(self.dels, pre) + return self.getTopIoDupItemIter(self.dels, pre) + #return self.getOnIoDupValsAnyPreIter(self.dels, pre) def putLdes(self, key, vals): """ @@ -3114,7 +3107,7 @@ def putLdes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ldes, key, vals) + return self.putIoDupVals(self.ldes, key, vals) def addLde(self, key, val): """ @@ -3124,7 +3117,7 @@ def addLde(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ldes, key, val) + return self.addIoDupVal(self.ldes, key, val) def getLdes(self, key): """ @@ -3133,7 +3126,7 @@ def getLdes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ldes, key) + return self.getIoDupVals(self.ldes, key) def getLdeLast(self, key): """ @@ -3142,21 +3135,9 @@ def getLdeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ldes, key) - - def getLdeItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of likely duplicitous escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.ldes, key, skip) + return self.getIoDupValLast(self.ldes, key) - def getLdeItemsNextIter(self, key=b'', skip=True): + def getLdeItemIter(self, key=b''): """ Use sgKey() Return iterator of likely duplicitous escrowed event dig items at next key after key. @@ -3166,7 +3147,8 @@ def getLdeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ldes, key, skip) + return self.getTopIoDupItemIter(self.ldes, key) + #return self.getIoDupItemsNextIter(self.ldes, key, skip) def cntLdes(self, key): """ @@ -3174,7 +3156,7 @@ def cntLdes(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.ldes, key) + return self.cntIoDupVals(self.ldes, key) def delLdes(self, key): """ @@ -3182,7 +3164,7 @@ def delLdes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.ldes, key) + return self.delIoDupVals(self.ldes, key) def delLde(self, key, val): """ @@ -3195,7 +3177,7 @@ def delLde(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ldes, key, val) + return self.delIoDupVal(self.ldes, key, val) class BaserDoer(doing.Doer): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 1dc1208b..5787c5ac 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -65,6 +65,51 @@ SuffixSize = 32 # does not include trailing separator MaxSuffix = int("f"*(SuffixSize), 16) +def onKey(top, on, *, sep=b'.'): + """ + Returns: + onkey (bytes): key formed by joining top key and hex str conversion of + int ordinal number on with sep character. + + Parameters: + top (str | bytes): top key prefix to be joined with hex version of on using sep + on (int): ordinal number to be converted to 32 hex bytes + sep (bytes): separator character for join + + """ + if hasattr(top, "encode"): + top = top.encode("utf-8") # convert str to bytes + return (b'%s%s%032x' % (top, sep, on)) + + +def snKey(pre, sn): + """ + Returns: + snkey (bytes): key formed by joining pre and hex str conversion of int + sequence ordinal number sn with sep character b".". + + Parameters: + pre (str | bytes): key prefix to be joined with hex version of on using + b"." sep + sn (int): sequence number to be converted to 32 hex bytes + """ + return onKey(pre, sn, sep=b'.') + + +def fnKey(pre, fn): + """ + Returns: + fnkey (bytes): key formed by joining pre and hex str conversion of int + first seen ordinal number fn with sep character b".". + + Parameters: + pre (str | bytes): key prefix to be joined with hex version of on using + b"." sep + fn (int): first seen ordinal number to be converted to 32 hex bytes + """ + return onKey(pre, fn, sep=b'.') + + def dgKey(pre, dig): """ Returns bytes DB key from concatenation of '.' with qualified Base64 prefix @@ -78,20 +123,6 @@ def dgKey(pre, dig): return (b'%s.%s' % (pre, dig)) -def onKey(pre, sn): - """ - Returns bytes DB key from concatenation with '.' of qualified Base64 prefix - bytes pre and int ordinal number of event, such as sequence number or first - seen order number. - """ - if hasattr(pre, "encode"): - pre = pre.encode("utf-8") # convert str to bytes - return (b'%s.%032x' % (pre, sn)) - -snKey = onKey # alias so intent is clear, sn vs fn -fnKey = onKey # alias so intent is clear, sn vs fn - - def dtKey(pre, dts): """ Returns bytes DB key from concatenation of '|' qualified Base64 prefix @@ -107,7 +138,8 @@ def dtKey(pre, dts): dts = dts.encode("utf-8") # convert str to bytes return (b'%s|%s' % (pre, dts)) - +# ToDo right split so key prefix could be top of key space with more than one +# part def splitKey(key, sep=b'.'): """ Returns duple of pre and either dig or on, sn, fn str or dts datetime str by @@ -127,13 +159,13 @@ def splitKey(key, sep=b'.'): else: if hasattr(sep, 'encode'): # make sep match bytes or str sep = sep.encode("utf-8") - splits = key.split(sep) + splits = key.rsplit(sep, 1) if len(splits) != 2: - raise ValueError("Unsplittable key = {}".format(key)) + raise ValueError(f"Unsplittable {key=} at {sep=}.") return tuple(splits) -def splitKeyON(key): +def splitOnKey(key, *, sep=b'.'): """ Returns list of pre and int on from key Accepts either bytes or str key @@ -141,12 +173,17 @@ def splitKeyON(key): """ if isinstance(key, memoryview): key = bytes(key) - pre, on = splitKey(key) + top, on = splitKey(key, sep=sep) on = int(on, 16) - return (pre, on) + return (top, on) -splitKeySN = splitKeyON # alias so intent is clear, sn vs fn -splitKeyFN = splitKeyON # alias so intent is clear, sn vs fn + +splitSnKey = splitOnKey # alias so intent is clear, sn vs fn +splitFnKey = splitOnKey # alias so intent is clear, sn vs fn + +splitKeyON = splitOnKey # backwards compatible alias +splitKeySN = splitSnKey # backwards compatible alias +splitKeyFN = splitFnKey # backwards compatible alias def splitKeyDT(key): @@ -485,7 +522,9 @@ def setVal(self, db, key, val): """ Write serialized bytes val to location key in db Overwrites existing val if any - Returns True If val successfully written Else False + Returns: + result (bool): True If val successfully written + False otherwise Parameters: db is opened named sub db with dupsort=False @@ -550,41 +589,9 @@ def cnt(self, db): return count - def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): - """ - Returns iterator of item duple (key, val), at each key over all - keys in db. If split is true then the key is split at sep and instead - of returing duple it results tuple with one entry for each key split - as well as the value. - - Works for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database - split (bool): True means split key at sep before returning - sep (bytes): separator char for key + def getTopItemIter(self, db, top=b''): """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db - - for key, val in cursor.iternext(): # return key, val at cursor - if split: - splits = bytes(key).split(sep) - splits.append(val) - else: - splits = (bytes(key), val) - yield tuple(splits) - - - def getTopItemIter(self, db, key=b''): - """ - Iterates over branch of db given by key + Iterates over branch of db given by top key Returns: items (abc.Iterator): iterator of (full key, val) tuples over a @@ -599,22 +606,24 @@ def getTopItemIter(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is - empty then gets all items in database + empty then gets all items in database. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any + if cursor.set_range(top): # move to val at key >= key if any for ckey, cval in cursor.iternext(): # get key, val at cursor ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch + if not ckey.startswith(top): # prev entry if any last in branch break # done yield (ckey, cval) # another entry in branch startswith key return # done raises StopIteration - def delTopVal(self, db, key=b''): + def delTopVal(self, db, top=b''): """ Deletes all values in branch of db given top key. @@ -624,9 +633,9 @@ def delTopVal(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is - empty then gets all items in database + empty then deletes all items in database Works for both dupsort==False and dupsort==True Because cursor.iternext() advances cursor after returning item its safe @@ -640,123 +649,213 @@ def delTopVal(self, db, key=b''): with self.env.begin(db=db, write=True, buffers=True) as txn: result = False cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any + if cursor.set_range(top): # move to val at key >= key if any ckey, cval = cursor.item() while ckey: # end of database key == b'' ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch + if not ckey.startswith(top): # prev entry if any last in branch break # done result = cursor.delete() or result # delete moves cursor to next item ckey, cval = cursor.item() # cursor now at next item after deleted return result - # For subdbs with no duplicate values allowed at each key. (dupsort==False) - # and use keys with ordinal as monotonically increasing number part - # such as sn or fn - def appendOrdValPre(self, db, pre, val): + # For subdbs the use keys with trailing part the is monotonically + # ordinal number serialized as 32 hex bytes + # used in OnSuberBase + def appendOnVal(self, db, key, val, *, sep=b'.'): """ - Appends val in order after last previous key with same pre in db. - Returns ordinal number in, on, of appended entry. Appended on is 1 greater - than previous latest on. - Uses onKey(pre, on) for entries. + Appends val in order after last previous onkey in db where + onkey has same given key prefix but with different serialized on suffix + attached with sep. + Returns ordinal number on, of appended entry. Appended on is 1 greater + than previous latest on at key. + Uses onKey(key, on) for entries. - Append val to end of db entries with same pre but with on incremented by - 1 relative to last preexisting entry at pre. + Works with either dupsort==True or False since always creates new full + key. + + Append val to end of db entries with same key but with on incremented by + 1 relative to last preexisting entry at key. + + Returns: + on (int): ordinal number of newly appended val Parameters: - db is opened named sub db with dupsort=False - pre is bytes identifier prefix for event - val is event digest + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split """ # set key with fn at max and then walk backwards to find last entry at pre # if any otherwise zeroth entry at pre - key = onKey(pre, MaxON) + onkey = onKey(key, MaxON, sep=sep) with self.env.begin(db=db, write=True, buffers=True) as txn: on = 0 # unless other cases match then zeroth entry at pre cursor = txn.cursor() - if not cursor.set_range(key): # max is past end of database + if not cursor.set_range(onkey): # max is past end of database # so either empty database or last is earlier pre or # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max - ckey = cursor.key() - cpre, cn = splitKeyON(ckey) - if cpre == pre: # last is last entry for same pre + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) + if ckey == key: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre - ckey = cursor.key() - cpre, cn = splitKeyON(ckey) - if cpre == pre: # last entry for pre is already at max - raise ValueError("Number part of key {} exceeds maximum" - " size.".format(ckey)) + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) + if ckey == key: # last entry for pre is already at max + raise ValueError(f"Number part {cn=} for key part {ckey=}" + f"exceeds maximum size.") else: # later pre so backup one entry # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre - ckey = cursor.key() - cpre, cn = splitKeyON(ckey) - if cpre == pre: # last entry at pre + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) + if ckey == key: # last entry at pre on = cn + 1 # increment - key = onKey(pre, on) + onkey = onKey(key, on, sep=sep) - if not cursor.put(key, val, overwrite=False): - raise ValueError("Failed appending {} at {}.".format(val, key)) + if not cursor.put(onkey, val, overwrite=False): + raise ValueError(f"Failed appending {val=} at {key=}.") return on + # used in OnSuberBase + def delOnVal(self, db, key, on=0, *, sep=b'.'): + """ + Deletes value at onkey consisting of key + sep + serialized on in db. + Returns True If key exists in database Else False - def getAllOrdItemPreIter(self, db, pre, on=0): + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which to delete + sep (bytes): separator character for split """ - Returns iterator of duple item, (on, dig), at each key over all ordinal - numbered keys with same prefix, pre, in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Returned items are duples of (on, dig) where on is ordinal number int - and dig is event digest for lookup in .evts sub db. + with self.env.begin(db=db, write=True, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return (txn.delete(onkey)) # when empty deletes whole db + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - Raises StopIteration Error when empty. + # used in OnSuberBase + def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns (int): count of of all ordinal keyed vals with key + but different on tail in db starting at ordinal number on of key. + Full key is composed of top+sep+ + When dupsort==true then duplicates are included in count since .iternext + includes duplicates. + when key is empty then counts whole db Parameters: - db is opened named sub db with dupsort=False - pre is bytes of itdentifier prefix - on is int ordinal number to resume replay + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate count + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - if not cursor.set_range(key): # moves to val at key >= key - return # no values end of db + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + count = 0 + if not cursor.set_range(onkey): # moves to val at key >= key + return count # no values end of db + + for ckey in cursor.iternext(values=False): # get key only at cursor + try: + ckey, cn = splitOnKey(ckey, sep=sep) + except ValueError as ex: # not splittable key + break - for key, val in cursor.iternext(): # get key, val at cursor - cpre, cn = splitKeyON(key) - if cpre != pre: # prev is now the last event for pre + if key and ckey != key: # prev is now the last event for pre break # done - yield (cn, val) # (on, dig) of event + count = count+1 + return count - def getAllOrdItemAllPreIter(self, db, key=b''): + # used in OnSuberBase + def getOnValIter(self, db, key=b'', on=0, *, sep=b'.'): """ - Returns iterator of triple item, (pre, on, dig), at each key over all - ordinal numbered keys for all prefixes in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Each returned item is triple (pre, on, dig) where pre is identifier prefix, - on is ordinal number int and dig is event digest for lookup in .evts sub db. + Returns iterator of the val at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db Raises StopIteration Error when empty. + Returns: + items (Iterator[bytes]): val with same + key but increments of on beginning with on + Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for (key, on, val) in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + yield (val) + + # used in OnSuberBase + def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val with same + key but increments of on beginning with on + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty + onkey = key + if not cursor.set_range(onkey): # moves to val at key >= onkey + return # no values end of db raises StopIteration + + for ckey, cval in cursor.iternext(): # get key, val at cursor + ckey, cn = splitOnKey(ckey, sep=sep) + if key and not ckey == key: + break + yield (ckey, cn, cval) - for key, val in cursor.iternext(): # return key, val at cursor - cpre, cn = splitKeyON(key) - yield (cpre, cn, val) # (pre, on, dig) of event + # ToDo + # getOnItemBackIter symmetric with getOnItemIter + # getOnValBackIter symmetric with getOnValIter + # IoSet insertion order in val so can have effective dups but with + # dupsort==False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent # effective duplicate key but with (dupsort==False). Actual key uses hidden # key suffix ordinal to provide insertion ordering of value members of set @@ -779,7 +878,7 @@ def putIoSetVals(self, db, key, vals, *, sep=b'.'): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False key (bytes): Apparent effective key - vals (abc.Iterable): serialized values to add to set of vals at key + vals (Iterable): serialized values to add to set of vals at key """ result = False @@ -810,8 +909,8 @@ def putIoSetVals(self, db, key, vals, *, sep=b'.'): def addIoSetVal(self, db, key, val, *, sep=b'.'): """ - Add val to insertion ordered set of values all with the same apparent - effective key if val not already in set of vals at key. + Add val idempotently to insertion ordered set of values all with the + same apparent effective key if val not already in set of vals at key. A Uses hidden ordinal key suffix for insertion ordering. The suffix is appended and stripped transparently. @@ -870,58 +969,6 @@ def setIoSetVals(self, db, key, vals, *, sep=b'.'): return result - def appendIoSetVal(self, db, key, val, *, sep=b'.'): - """ - Append val to insertion ordered set of values all with the same apparent - effective key. Assumes val is not already in set. - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. - - Returns: - ion (int): hidden insertion ordering ordinal of appended val - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - val (bytes): value to append - """ - ion = 0 # default is zeroth insertion at key - iokey = suffix(key, ion=MaxSuffix, sep=sep) # make iokey at max and walk back - with self.env.begin(db=db, write=True, buffers=True) as txn: - cursor = txn.cursor() # create cursor to walk back - if not cursor.set_range(iokey): # max is past end of database - # Three possibilities for max past end of database - # 1. last entry in db is for same key - # 2. last entry in db is for other key before key - # 3. database is empty - if cursor.last(): # not 3. empty db, so either 1. or 2. - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # 1. last is last entry for same key - ion = cion + 1 # so set ion to the increment of cion - else: # max is not past end of database - # Two possibilities for max not past end of databseso - # 1. cursor at max entry at key - # 2. other key after key with entry in database - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # 1. last entry for key is already at max - raise ValueError("Number part of key {} at maximum" - " size.".format(ckey)) - else: # 2. other key after key so backup one entry - # Two possibilities: 1. no prior entry 2. prior entry - if cursor.prev(): # prev entry, maybe same or earlier pre - # 2. prior entry with two possiblities: - # 1. same key - # 2. other key before key - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # prior (last) entry at key - ion = cion + 1 # so set ion to the increment of cion - - iokey = suffix(key, ion=ion, sep=sep) - if not cursor.put(iokey, val, overwrite=False): - raise ValueError("Failed appending {} at {}.".format(val, key)) - - return ion - def getIoSetVals(self, db, key, *, ion=0, sep=b'.'): """ @@ -1115,77 +1162,29 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return False - def getIoSetItems(self, db, key, *, ion=0, sep=b'.'): + def getTopIoSetItemIter(self, db, top=b'', *, sep=b'.'): """ Returns: - items (list): list of tuples (iokey, val) of entries in set of with - same apparent effective key. iokey includes the ordinal key suffix - Uses hidden ordinal key suffix for insertion ordering. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - ion (int): starting ordinal value, default 0 - - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - items = [] - iokey = suffix(key, ion, sep=sep) # start ion th value for key zeroth default - cursor = txn.cursor() - if cursor.set_range(iokey): # move to val at key >= iokey if any - for iokey, val in cursor.iternext(): # get iokey, val at cursor - ckey, cion = unsuffix(iokey, sep=sep) - if ckey != key: # prev entry if any was the last entry for key - break # done - items.append((iokey, val)) # another entry at key - return items + items (Iterator[(key,val)]): iterator of tuples (key, val) where + key is apparent key with hidden insertion ordering suffixe removed + from effective key. + Iterates over top branch of insertion ordered set values where each + effective key has trailing hidden suffix of serialization of insertion + ordering ordinal. - - def getIoSetItemsIter(self, db, key, *, ion=0, sep=b'.'): - """ - Returns: - items (abc.Iterator): iterator over insertion ordered set of values - at same apparent effective key where each iteration returns tuple - (iokey, val). iokey includes the ordinal key suffix. Uses hidden ordinal key suffix for insertion ordering. Raises StopIteration Error when empty. Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - ion (int): starting ordinal value, default 0 - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - iokey = suffix(key, ion, sep=sep) # start ion th value for key zeroth default - cursor = txn.cursor() - if cursor.set_range(iokey): # move to val at key >= iokey if any - for iokey, val in cursor.iternext(): # get key, val at cursor - ckey, cion = unsuffix(iokey, sep=sep) - if ckey != key: # prev entry if any was the last entry for key - break # done - yield (iokey, val) # another entry at key - return # done raises StopIteration - - - def delIoSetIokey(self, db, iokey): + top (bytes): top key in db. When top is empty then every item in db. + sep (bytes): sep character for attached io suffix """ - Deletes val at at actual iokey that includes ordinal key suffix. + for iokey, val in self.getTopItemIter(db=db, top=top): + key, ion = splitOnKey(iokey, sep=sep) + yield (key, val) - Returns: - result (bool): True if val was deleted at iokey. False otherwise - if no val at iokey - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - iokey (bytes): actual key with ordinal key suffix - """ - with self.env.begin(db=db, write=True, buffers=True) as txn: - try: - return txn.delete(iokey) - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") # For subdbs that support duplicates at each key (dupsort==True) @@ -1338,31 +1337,6 @@ def cntVals(self, db, key): return count - def cntValsAllPre(self, db, pre, on=0): - """ - Returns (int): count of of all vals with same pre in key but different - on in key in db starting at ordinal number on of pre - - Does not count dups - - Parameters: - db is opened named sub db - pre is bytes of key within sub db's keyspace pre.on - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - count = 0 - if not cursor.set_range(key): # moves to val at key >= key - return count # no values end of db - - for val in cursor.iternext(values=False): # get key, val at cursor - cpre, cn = splitKeyON(val) - if cpre != pre: # prev is now the last event for pre - break # done - count = count+1 - - return count def delVals(self, db, key, val=b''): @@ -1385,8 +1359,10 @@ def delVals(self, db, key, val=b''): # For subdbs that support insertion order preserving duplicates at each key. - # dupsort==True and prepends and strips io val proem - def putIoVals(self, db, key, vals): + # IoDup class IoVals IoItems + # dupsort==True and prepends and strips io val proem to each value. + # because dupsort==True values are limited to 511 bytes including proem + def putIoDupVals(self, db, key, vals): """ Write each entry from list of bytes vals to key in db in insertion order Adds to existing values at key if any @@ -1395,11 +1371,13 @@ def putIoVals(self, db, key, vals): Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order + all values that makes lexocographic order that same as insertion order. + Duplicates are ordered as a pair of key plus value so prepending proem to each value changes duplicate ordering. Proem is 33 characters long. With 32 character hex string followed by '.' for essentiall unlimited number of values which will be limited by memory. + With prepended proem ordinal must explicity check for duplicate values before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). @@ -1411,7 +1389,7 @@ def putIoVals(self, db, key, vals): """ result = False - dups = set(self.getIoVals(db, key)) #get preexisting dups if any + dups = set(self.getIoDupVals(db, key)) #get preexisting dups if any with self.env.begin(db=db, write=True, buffers=True) as txn: idx = 0 cursor = txn.cursor() @@ -1432,7 +1410,7 @@ def putIoVals(self, db, key, vals): return result - def addIoVal(self, db, key, val): + def addIoDupVal(self, db, key, val): """ Add val bytes as dup in insertion order to key in db Adds to existing values at key if any @@ -1440,32 +1418,47 @@ def addIoVal(self, db, key, val): Actual value written include prepended proem ordinal Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). Parameters: db is opened named sub db with dupsort=False key is bytes of key within sub db's keyspace val is bytes of value to be written """ - return self.putIoVals(db, key, [val]) + return self.putIoDupVals(db, key, [val]) - def getIoVals(self, db, key): + def getIoDupVals(self, db, key): """ Return list of duplicate values at key in db in insertion order Returns empty list if no entry at key Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True @@ -1485,18 +1478,26 @@ def getIoVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoValsIter(self, db, key): + def getIoDupValsIter(self, db, key): """ Return iterator of all duplicate values at key in db in insertion order Raises StopIteration Error when no remaining dup items = empty. Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True @@ -1515,13 +1516,26 @@ def getIoValsIter(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoValLast(self, db, key): + def getIoDupValLast(self, db, key): """ Return last added dup value at key in db in insertion order Returns None no entry at key Removes prepended proem ordinal from val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1540,98 +1554,23 @@ def getIoValLast(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoItemsNext(self, db, key=b"", skip=True): - """ - Return list of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key == b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Returns empty list if no entries at next key after key - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - - Assumes DB opened with dupsort=True - - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty string - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass - """ - - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - items = [] - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - # slice off prepended ordering prefix on value in item - items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] - return items - - - def getIoItemsNextIter(self, db, key=b"", skip=True): - """ - Return iterator of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Raises StopIteration Error when no remaining dup items = empty. - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - - Assumes DB opened with dupsort=True - - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass - """ - - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - for key, val in cursor.iternext_dup(keys=True): - yield (key, val[33:]) # slice off prepended ordering prefix - - - def cntIoVals(self, db, key): - """ - Return count of dup values at key in db, or zero otherwise - Assumes DB opened with dupsort=True - - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace + def delIoDupVals(self, db, key): """ + Deletes all values at key in db if key present. + Returns True If key exists - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - count = 0 - try: - if cursor.set_key(key): # moves to first_dup - count = cursor.count() - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - return count + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. - def delIoVals(self, db, key): - """ - Deletes all values at key in db if key present. - Returns True If key exists + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). Parameters: db is opened named sub db with dupsort=True @@ -1646,7 +1585,7 @@ def delIoVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def delIoVal(self, db, key, val): + def delIoDupVal(self, db, key, val): """ Deletes dup io val at key in db. Performs strip search to find match. Strips proems and then searches. @@ -1666,8 +1605,7 @@ def delIoVal(self, db, key, val): The problem is that escrow is not fixed buts stuffs gets added and deleted which just adds to the value of the proem. 2**16 is an impossibly large number so the proem will not max out practically. But its not - and elegant solution. So maybe escrows need to use a different approach. - But really didn't want to add another database just for escrows. + an elegant solution. Parameters: db is opened named sub db with dupsort=False @@ -1688,153 +1626,333 @@ def delIoVal(self, db, key, val): return False - def getIoValsAllPreIter(self, db, pre, on=0): + def cntIoDupVals(self, db, key): """ - Returns iterator of all dup vals in insertion order for all entries - with same prefix across all ordinal numbers in increasing order - without gaps between ordinal numbers - starting with on, default 0. Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + Return count of dup values at key in db, or zero otherwise + Assumes DB opened with dupsort=True - Raises StopIteration Error when empty. + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. - Duplicates are retrieved in insertion order. + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). Parameters: db is opened named sub db with dupsort=True - pre (bytes | str): of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to begin iteration at + key is bytes of key within sub db's keyspace """ + with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - # slice off prepended ordering prefix - yield val[33:] - key = snKey(pre, cnt:=cnt+1) - - - def getIoValsAllPreBackIter(self, db, pre, on=0): - """ - Returns iterator of all dup vals in insertion order for all entries - with same prefix across all sequence numbers in decreasing order without gaps - between ordinals at a given pre. - Starting with on (default = 0) as begining ordinal number or sequence number. - Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + count = 0 + try: + if cursor.set_key(key): # moves to first_dup + count = cursor.count() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + return count + + +# used in IoDupSuber.getItemIter + def getTopIoDupItemIter(self, db, top=b''): + """ + Iterates over top branch of db given by key of IoDup items where each value + has 33 byte insertion ordinal number proem (prefixed) with separator. + Automagically removes (strips) proem before returning items. + + Assumes DB opened with dupsort=True + + Returns: + items (abc.Iterator): iterator of (full key, val) tuples of all + dup items over a branch of the db given by top key where returned + full key is full database key for val not truncated top key. + Item is (key, val) with proem stripped from val stored in db. + If key = b'' then returns list of dup items for all keys in db. + + + Because cursor.iternext() advances cursor after returning item its safe + to delete the item within the iteration loop. curson.iternext() works + for both dupsort==False and dupsort==True Raises StopIteration Error when empty. - Duplicates are retrieved in insertion order. + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + top (bytes): truncated top key, a key space prefix to get all the items + from multiple branches of the key space. If top key is + empty then gets all items in database + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + for top, val in self.getTopItemIter(db=db, top=top): + val = val[33:] # strip proem + yield (top, val) + + + # methods for OnIoDup that combines IoDup value proem with On ordinal numbered + # trailing prefix + # this is so we do the proem add and strip here not in some higher level class + # like suber + + # used in OnIoDupSuber + def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): + """ + Appends val in order after last previous key with same pre in db where + full key has key prefix and serialized on suffix attached with sep and + value has ordinal proem prefixed. + Returns ordinal number in, on, of appended entry. Appended on is 1 greater + than previous latest on at pre. + Uses onKey(pre, on) for entries. + + Works with either dupsort==True or False since always creates new full + key. + + Append val to end of db entries with same pre but with on incremented by + 1 relative to last preexisting entry at pre. + + Returns: + on (int): ordinal number of newly appended val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of identifier prefix prepended to sn in key - within sub db's keyspace - on (int): is ordinal number to begin iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = snKey(pre, cnt := on) - # set_key returns True if exact key else false - while cursor.set_key(key): # moves to first_dup if valid key - for val in cursor.iternext_dup(): - # slice off prepended ordering prefix - yield val[33:] - key = snKey(pre, cnt:=cnt-1) - - - def getIoValLastAllPreIter(self, db, pre, on=0): - """ - Returns iterator of last only of dup vals of each key in insertion order - for all entries with same prefix across all sequence numbers in increasing order - without gaps starting with on (default = 0). Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + val = (b'%032x.' % (0)) + val # prepend ordering proem + return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) + + + # used in OnIoDupSuber + def getOnIoDupValIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db Raises StopIteration Error when empty. - Duplicates are retrieved in insertion order. + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnIoDupItemIter(db=db, key=key, on=on, sep=sep): + yield (val) + + + # used in OnIoDupSuber + def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + when key is empty then retrieves whole db + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to being iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + val = val[33:] # strip proem + yield (key, on, val) + + + def getOnIoDupLastValIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator of val of last insertion ordered duplicate at each + key over all ordinal numbered keys with same full key + of key + sep + on in db. Values are sorted by onKey(key, on) where on + is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + Returns: + val (Iterator[bytes]): last dup val at each onkey + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnIoDupLastItemIter(db=db, key=key, on=on, sep=sep): + yield (val) + + + def getOnIoDupLastItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator of triples (key, on, val), of last insertion ordered + duplicate at each key over all ordinal numbered keys with same full key + of key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - if cursor.last_dup(): # move to last_dup - yield cursor.value()[33:] # slice off prepended ordering prefix - key = snKey(pre, cnt:=cnt+1) + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty + onkey = key + + if not cursor.set_range(onkey): # # moves to first_dup at key>=onkey + return # no values end of db raises StopIteration + + while cursor.last_dup(): # move to last_dup at current ckey + onkey, cval = cursor.item() # get ckey cval of last dup + ckey, on = splitOnKey(onkey, sep=sep) # get key on + if key and not ckey == key: + break + + yield (ckey, on, cval[33:]) # slice off prepended ordering proem + onkey = onKey(ckey, on+1) + if not cursor.set_range(onkey): # # moves to first_dup at key>=onkey + return # no values end of db raises StopIteration + + def getOnIoDupValBackIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator going backwards of values, + of insertion ordered item at each key over all ordinal numbered keys + with same full key of key + sep + on in db. + Values are sorted by onKey(key, on) where on is ordinal number int and + key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Backwards means decreasing numerical value of duplicate proem, for each on, + decreasing numerical value on for each key and decresing lexocogrphic + order of each key prefix. - def getIoValsAnyPreIter(self, db, pre, on=0): + Returned items are vals + + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + val (Iterator[bytes]): at key including duplicates in backwards order + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ - Returns iterator of all dup vals in insertion order for any entries - with same prefix across all ordinal numbers in order including gaps - between ordinals at a given pre. Staring with on (default = 0). - Stops when pre is different. + for key, on, val in self.getOnIoDupItemBackIter(db=db, key=key, on=on, sep=sep): + yield (val) - Duplicates that may be deleted such as duplicitous event logs need - to be able to iterate across gaps in ordinal number. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + def getOnIoDupItemBackIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator going backwards of triples (key, on, val), + of insertion ordered item at each key over all ordinal numbered keys + with same full key of key + sep + on in db. + Values are sorted by onKey(key, on) where on is ordinal number int and + key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Backwards means decreasing numerical value of duplicate proem, for each on, + decreasing numerical value on for each key and decresing lexocogrphic + order of each key prefix. + + Returned items are triples of (key, on, val) + + when key is empty then retrieves whole db Raises StopIteration Error when empty. - Duplicates are retrieved in insertion order. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): beginning ordinal number to start iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_range(key): # moves to first dup of key >= key - key = cursor.key() # actual key - front, back = bytes(key).split(sep=b'.', maxsplit=1) - if front != pre: # set range may skip pre if none - break - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering prefix - cnt = int(back, 16) - key = snKey(pre, cnt:=cnt+1) + if not cursor.last(): # pre-position cursor at last dup of last key + return # empty database so raise StopIteration + + if key: # not empty so attempt to position at starting key not last + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + if cursor.set_range(onkey): # found key >= onkey + ckey, cn = splitOnKey(cursor.key(), sep=sep) + if ckey == key: # onkey in db + cursor.last_dup() # start at its last dup + else: # get closest key < onkey + if not cursor.prev(): # last dup of previous key + return # no earlier keys to designated start + + # cursor should now be correctly positioned for start either at + # last dup of either last key or onkey + for onkey, cval in cursor.iterprev(): # iterate backwards + ckey, on = splitOnKey(onkey, sep=sep) + if key and ckey != key: + return + yield (ckey, on, cval[33:]) + + + + # ToDo do we need a replay last backwards? + diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 19b8432f..36cdbc0e 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -79,7 +79,7 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): extype (Type[Exception]): the expected exception type if the message should remain in escrow """ - for (typ, pre, aid, ion), saider in self.escrowdb.getIoItemIter(keys=(typ,)): + for (typ, pre, aid), saider in self.escrowdb.getItemIter(keys=(typ, '')): try: tsgs = eventing.fetchTsgs(db=self.tigerdb, saider=saider) @@ -117,20 +117,20 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) except Exception as ex: # other error so remove from reply escrow - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: logger.error("Kevery unescrowed due to error: %s", ex.args[0]) else: # unescrow succeded - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow only + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow logger.info("Kevery unescrow succeeded for txn state=%s", serder.said) logger.debug(f"event=\n{serder.pretty()}\n") except Exception as ex: # log diagnostics errors etc - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.removeState(saider) if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) @@ -170,7 +170,7 @@ def escrowStateNotice(self, *, typ, pre, aid, serder, saider, dater, cigars=None for cigar in cigars: # process each couple to verify sig and write to db self.cigardb.put(keys=keys, vals=[(cigar.verfer, cigar)]) - return self.escrowdb.put(keys=(typ, pre, aid), vals=[saider]) # overwrite + return self.escrowdb.put(keys=(typ, pre, aid), vals=[saider]) # does not overwrite def updateReply(self, aid, serder, saider, dater): """ diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index f69fd8b1..54a33696 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -73,23 +73,43 @@ def __init__(self, db: dbing.LMDBer, *, self.sep = sep if sep is not None else self.Sep - def _tokey(self, keys: Union[str, bytes, memoryview, Iterable]): + def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], + topive: bool=False): """ - Converts key to key str with proper separators and returns key bytes. - If key is already str then returns. Else If key is iterable (non-str) - of strs then joins with separator converts to bytes and returns + Converts keys to key bytes with proper separators and returns key bytes. + If keys is already str or bytes or memoryview then returns key bytes. + Else If keys is iterable (non-str) of strs or bytes then joins with + separator converts to key bytes and returns. When keys is iterable and + topive is True then enables partial key from top branch of key space given + by partial keys by appending separator to end of partial key + + Returns: + key (bytes): each element of keys is joined by .sep. If top then last + char of key is also .sep Parameters: - keys (Union[str, bytes, Iterable]): str, bytes, or Iterable of str. + keys (str | bytes | memoryview | Iterable[str | bytes]): db key or + Iterable of (str | bytes) to form key. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ + if hasattr(keys, "encode"): # str + return keys.encode("utf-8") if isinstance(keys, memoryview): # memoryview of bytes return bytes(keys) # return bytes - if hasattr(keys, "encode"): # str - return keys.encode("utf-8") # convert to bytes elif hasattr(keys, "decode"): # bytes - return keys # return as is - return (self.sep.join(keys).encode("utf-8")) # iterable so join + return keys + if topive and keys[-1]: # topive and keys is not already partial + keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep + return (self.sep.join(key.decode() if hasattr(key, "decode") else key + for key in keys).encode("utf-8")) + def _tokeys(self, key: Union[str, bytes, memoryview]): @@ -110,7 +130,34 @@ def _tokeys(self, key: Union[str, bytes, memoryview]): def getItemIter(self, keys: Union[str, Iterable]=b""): + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + + Returns: + items (Iterator): of (key, val) tuples over the all the items in + subdb whose key startswith key made from keys. Keys may be keyspace + prefix to return branches of key space. When keys is empty then + returns all items in subdb + + Parameters: + keys (Iterator): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to get all the items from + multiple branches of the key space. If keys is empty then gets + all items in database. + """ + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): + yield (self._tokeys(key), self.deserializer(val)) + + + def getFullItemIter(self, keys: Union[str, Iterable]=b""): + """Iterator over items in .db that returns full items with subclass + specific special hidden parts shown for debugging or testing. + Returns: items (Iterator): of (key, val) tuples over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace @@ -124,7 +171,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): yield (self._tokeys(key), self.deserializer(val)) @@ -328,7 +375,7 @@ def trim(self, keys: Union[str, Iterable]=b""): Returns: result (bool): True if key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys))) + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys))) def cntAll(self): @@ -549,109 +596,55 @@ def rem(self, keys: Union[str, Iterable], val=None): key=self._tokey(keys), sep=self.sep) + #def remIokey(self, iokeys: str | bytes | memoryview | Iterable): + #""" + #Removes entries at iokeys - def getItemIter(self, keys: Union[str, Iterable]=b""): - """Get items iterator - Returns: - items (Iterator): of (key, val) tuples over the all the items in - subdb whose effective key startswith key made from keys. - Keys may be keyspace prefix in order to return branches of key space. - When keys is empty then returns all items in subdb. - Returned key in each item has ordinal suffix removed. - - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. + #Parameters: + #iokeys (str | bytes | memoryview | Iterable): of key str or + #tuple of key strs to be combined in order to form key - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self.deserializer(val)) + #Returns: + #result (bool): True if key exists so delete successful. False otherwise + #""" + #return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItem(self, keys: Union[str, Iterable]): - """ - Gets (iokeys, val) ioitems list at key made from keys where key is - apparent effective key and ioitems all have same apparent effective key - Parameters: - keys (Iterable): of key strs to be combined in order to form key + def getItemIter(self, keys: Union[str, Iterable]=b"", *, topive=False): + """Get items iterator Returns: - ioitems (Iterable): each item in list is tuple (iokeys, val) where each - iokeys is actual key tuple including hidden suffix and - each val is str - empty list if no entry at keys - - - """ - return ([(self._tokeys(iokey), self.deserializer(val)) for iokey, val - in self.db.getIoSetItems(db=self.sdb, - key=self._tokey(keys), - sep=self.sep)]) - - - def getIoSetItemIter(self, keys: Union[str, Iterable]): - """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - - Returns: - ioitems (Iterator): each item iterated is tuple (iokeys, val) where - each iokeys is actual keys tuple including hidden suffix and - each val is str - empty list if no entry at keys. - Raises StopIteration when done - - """ - for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, - key=self._tokey(keys), - sep=self.sep): - yield (self._tokeys(iokey), self.deserializer(val)) - - - def getIoItemIter(self, keys: Union[str, Iterable]=b""): - """ - Returns: - items (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith effective key made from keys. - Keys may be keyspace prefix to return branches of key space. + items (Iterator): of (key, val) tuples over the all the items in + subdb whose effective key startswith key made from keys. + Keys may be keyspace prefix in order to return branches of key space. When keys is empty then returns all items in subdb. - + Returned key in each item has ordinal suffix removed. Parameters: keys (Iterable): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from + a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + """ + for iokey, val in self.db.getTopIoSetItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive), + sep=self.sep.encode()): yield (self._tokeys(iokey), self.deserializer(val)) - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): - """ - Removes entry at iokeys - - Parameters: - iokeys (tuple): of key str or tuple of key strs to be combined in - order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - - class DupKomer(KomerBase): """ diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index ec4ae778..214cf5ed 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -36,13 +36,46 @@ DupSuber CesrDupSuber +Class Architecture + +Suber is simple lexographic database with only one value per key +OnSuber is simple lexographic database where trailing part of key is serialized + ordinal number so that the ordering within each key prefix is monotonically + increasing numeric + +The term 'set' of values means that no value may appear more than once in the set. +Sets support idempotent adds and puts to db. This means one can add or put the same +(key, val) pair multiple times and not change the db. + +DupSuber provides set of lexicographic ordered values at each key. Each value has + a limited size (key + value <= 511 byes). The set is performant. Good for indices. + +IoDupSuber provides set of insertion ordered values at each key. Each value has + a limited size (key + value <= 511 byes). The set is less perfromant than DupSuber + but more performant than IoSetSuber. Good for insertion ordered indices + +IoSetSuber proves set of insertion ordered values at each key. Value size is not limited + Good for any insertion ordered set where size may be too large for IoDupSuber + +OnIoDupSuber provides set of insertion ordered values where the where trailing + part of key is serialized ordinal number so that the ordering within each + key prefix is monotonically increasing numeric. + +Each of these base types for managing the key space may be mixed with other +Classes that provide different types of values these include. + +Cesr +CatCesr +Serder +etc. + """ from typing import Type, Union from collections.abc import Iterable, Iterator from .. import help -from ..help.helping import nonStringIterable +from ..help.helping import nonStringIterable, Reb64 from .. import core from ..core import coring, scheming, serdering from . import dbing @@ -60,8 +93,8 @@ class SuberBase(): db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes - verify (bool): some subclasses want to re-verify when deser from db - default false + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ Sep = '.' # separator for combining key iterables @@ -80,6 +113,8 @@ def __init__(self, db: dbing.LMDBer, *, each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ super(SuberBase, self).__init__() # for multi inheritance self.db = db @@ -89,24 +124,30 @@ def __init__(self, db: dbing.LMDBer, *, - def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], - top: bool=False): + def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], + topive: bool=False): """ - Converts keys to key str with proper separators and returns key bytes. - If keys is already str then returns. Else If keys is iterable (non-str) - of strs then joins with separator converts to bytes and returns. - top allows partial key from top branch of key space given by partial keys + Converts keys to key bytes with proper separators and returns key bytes. + If keys is already str or bytes or memoryview then returns key bytes. + Else If keys is iterable (non-str) of strs or bytes then joins with + separator converts to key bytes and returns. When keys is iterable and + topive is True then enables partial key from top branch of key space given + by partial keys by appending separator to end of partial key Returns: - key (bytes): each element of keys is joined by .sep. If top then last - char of key is also .sep + key (bytes): each element of keys is joined by .sep. If topive then + last char of key is .sep Parameters: - keys (Union[str, bytes, Iterable]): str, bytes, or Iterable of str. - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep. + keys (str | bytes | memoryview | Iterable[str | bytes]): db key or + Iterable of (str | bytes) to form key. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. False means treat as full branch in key space. Resultant key - does not end in .sep + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of topive value """ if hasattr(keys, "encode"): # str @@ -115,19 +156,22 @@ def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys - return (self.sep.join(keys).encode("utf-8")) + if topive and keys[-1]: # topive and keys is not already partial + keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep + return (self.sep.join(key.decode() if hasattr(key, "decode") else key + for key in keys).encode("utf-8")) - def _tokeys(self, key: Union[str, bytes, memoryview]): + def _tokeys(self, key: str | bytes | memoryview): """ Converts key bytes to keys tuple of strs by decoding and then splitting - at separator. + at separator .sep. Returns: - keys (iterable): of str + keys (tuple[str]): makes tuple by splitting key at sep Parameters: - key (Union[str, bytes]): str or bytes. + key (str | bytes | memoryview): db key. """ if isinstance(key, memoryview): # memoryview of bytes @@ -137,72 +181,160 @@ def _tokeys(self, key: Union[str, bytes, memoryview]): return tuple(key.split(self.sep)) - def _ser(self, val: Union[str, memoryview, bytes]): + def _ser(self, val: str | bytes | memoryview): """ Serialize value to bytes to store in db Parameters: - val (Union[str, memoryview, bytes]): encodable as bytes + val (str | bytes | memoryview): encodable as bytes """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # return bytes return (val.encode("utf-8") if hasattr(val, "encode") else val) - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: bytes | memoryview): """ Deserialize val to str Parameters: - val (Union[str, memoryview, bytes]): decodable as str + val (bytes | memoryview): decodable as str """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes return (val.decode("utf-8") if hasattr(val, "decode") else val) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def trim(self, keys: str|bytes|memoryview|Iterable=b"", *, topive=False): + """ + Removes all entries whose keys startswith keys. Enables removal of whole + branches of db key space. To ensure that proper separation of a branch + include empty string as last key in keys. For example ("a","") deletes + 'a.1'and 'a.2' but not 'ab' + + Parameters: + keys (Iteratabke[str | bytes | memoryview]): of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then trims all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + + Returns: + result (bool): True if val at key exists so delete successful. False otherwise """ + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, topive=topive))) + + + def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]="", + *, topive=False): + """Iterator over items in .db that returns full items with subclass + specific special hidden parts shown for debugging or testing. + Returns: - items (Iterator): if (key, val) tuples over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb. + This is meant to return full parts of items in both keyspace and + valuespace which may be useful in debugging or testing. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. + keys (str|bytes|memoryview|Iteratable[str | bytes | memoryview]): + of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): yield (self._tokeys(key), self._des(val)) - def trim(self, keys: Union[str, Iterable]=b""): - """ - Removes all entries whose keys startswith keys. Enables removal of whole - branches of db key space. To ensure that proper separation of a branch - include empty string as last key in keys. For example ("a","") deletes - 'a.1'and 'a.2' but not 'ab' + def getItemIter(self, keys: str|bytes|memoryview|Iterable="", + *, topive=False): + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + + Returns: + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb + + Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str|bytes|memoryview|Iterable[str|bytes|memoryview]): of key + parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value - Returns: - result (bool): True if key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys))) + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): + yield (self._tokeys(key), self._des(val)) class Suber(SuberBase): """ - Sub DB of LMDBer. Subclass of SuberBase + Subclass of SuberBase with no LMDB duplicates (i.e. multiple values at same key). """ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'docs.', dupsort: bool=False, **kwa): """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key @@ -278,10 +410,318 @@ def rem(self, keys: Union[str, Iterable]): return(self.db.delVal(db=self.sdb, key=self._tokey(keys))) +class OnSuberBase(SuberBase): + """ + Subclass of SuberBase that adds methods for keys with exposed key part suffix + that is 32 byte serializaton of monotonically increasing ordinal number on + such as sn or fn. + Each key consistes of top key joined with .sep to ordinal suffix + Works with dupsort==True or False + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Default False + sep (str): separator to convert keys iterator to key bytes for db key + Default '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(OnSuberBase, self).__init__(*pa, **kwa) + + + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): + """ + Returns: + on (int): ordinal number of newly appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization + on (int): ordinal number used with onKey(key,on) to form key. + """ + return (self.db.appendOnVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) + + + def remOn(self, keys: str | bytes | memoryview, on: int=0): + """ + Returns + result (bool): True if onkey made from key+sep+serialized on is + found in database so value is removed + Removes all duplicates if any at onkey. + False otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + """ + return (self.db.delOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode())) + + + def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): + """ + Returns + cnt (int): count of of all ordinal suffix keyed vals with same + key prefix but different on in onkey in db starting at ordinal + number on where key is formed with onKey(key,on). Count at + each onkey includes duplicates if any. + + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form top key + When keys is empty then counts whole database including + duplicates if any. + on (int): ordinal number used with onKey(key,on) to form key. + """ + return (self.db.cntOnVals(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode())) + + + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[bytes]): of val with same key but increments of + on >= on i.e. all key.on beginning with on + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form actual key + When keys is empty then retrieves whole database including + duplicates if any + on (int): ordinal number used with onKey(pre,on) to form key at at + which to initiate retrieval + sep (bytes): separator character for split + """ + for val in (self.db.getOnValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val with same + key but increments of on >= on i.e. all key.on beginning with on + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form actual key + When keys is empty then retrieves whole database including + duplicates if any + on (int): ordinal number used with onKey(pre,on) to form key at at + which to initiate retrieval + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + +class OnSuber(OnSuberBase, Suber): + """ + Subclass of OnSuberBase andSuber that adds methods for keys with ordinal + numbered suffixes. + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(OnSuber, self).__init__(*pa, **kwa) + + +class B64SuberBase(SuberBase): + """ + Base Class whose values are Iterables of Base64 str or bytes that are stored + in db as .sep joined Base64 bytes. Separator character must not be valid + Base64 character so the split will work unambiguously. + + Automatically joins and splits along separator to Iterable (tuple) of Base64 + + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + Must not be Base64 character. + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + + """ + super(B64SuberBase, self).__init__(*pa, **kwa) + if Reb64.match(self.sep.encode()): + raise ValueError("Invalid sep={self.sep}, must not be Base64 char.") + + + def _toval(self, vals: str|bytes|memoryview|Iterable[str|bytes|memoryview]): + """ + Converts vals to val bytes with proper separators and returns val bytes. + If vals is already str or bytes or memoryview then returns val bytes. + Else If vals is iterable (non-str) of strs or bytes or memoryview then + joins with .sep and converts to val bytes and returns. + + Returns: + val (bytes): each element of vals is joined by .sep. + + Parameters: + vals (str | bytes | memoryview | Iterable[str | bytes]): db val or + Iterable of (str | bytes | memoryview) to form val. + Note, join of bytes sep works with memoryview. + + """ + if hasattr(vals, "encode"): # str + val = vals.encode("utf-8") + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + if isinstance(vals, memoryview): # memoryview of bytes + val = bytes(vals) # return bytes + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + elif hasattr(vals, "decode"): # bytes + val = vals + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + vals = tuple(v.encode() if hasattr(v, "encode") else v for v in vals) # make bytes + for val in vals: + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return (self.sep.encode().join(vals)) + + #return (self.sep.join(val.decode() if hasattr(val, "decode") else val + #for val in vals).encode("utf-8")) + + + def _tovals(self, val: bytes | memoryview): + """ + Converts val bytes to vals tuple of strs by decoding and then splitting + at separator .sep. + + Returns: + vals (tuple[str]): makes tuple by splitting val at .sep + + Parameters: + val (bytes | memoryview): db Base64 val. + + """ + if isinstance(val, memoryview): # memoryview of bytes + val = bytes(val) + if hasattr(val, "decode"): # bytes + val = val.decode("utf-8") # convert to str + return tuple(val.split(self.sep)) + + + def _ser(self, val: Union[Iterable, str, bytes]): + """ + Serialize val to bytes to store in db + When val is Iterable then joins each elements with .sep returns val bytes + + Returns: + val (bytes): .sep join of each Base64 bytes in val + + Parameters: + val (Union[Iterable, bytes]): of Base64 bytes + + """ + if not nonStringIterable(val): # not iterable + val = (val, ) # make iterable + return (self._toval(val)) + + + def _des(self, val: memoryview | bytes): + """ + Converts val bytes to vals tuple of subclass instances by deserializing + .qb64b concatenation in order of each instance in .klas + + Returns: + vals (tuple): subclass instances + + Parameters: + val (Union[bytes, memoryview]): of concatenation of .qb64b + + """ + return self._tovals(val) + + +class B64Suber(B64SuberBase, Suber): + """ + Subclass of B64SuberBase and Suber that serializes and deserializes values + as .sep joined strings of Base64 components. + + .sep must not be Base64 character. + + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + Must not be Base64 character. + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(B64Suber, self).__init__(*pa, **kwa) + + class CesrSuberBase(SuberBase): """ - Sub class of Suber where data is CESR encode/decode ducktyped subclass + Sub class of SuberBase where data is CESR encode/decode ducktyped subclass instance such as Matter, Indexer, Counter with .qb64b property when provided as fully qualified serialization Automatically serializes and deserializes from qb64b to/from CESR instance @@ -290,11 +730,20 @@ class CesrSuberBase(SuberBase): def __init__(self, *pa, klas: Type[coring.Matter] = coring.Matter, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + Parameters: klas (Type[coring.Matter]): Class reference to subclass of Matter or Indexer or Counter or any ducktyped class of Matter + """ super(CesrSuberBase, self).__init__(*pa, **kwa) self.klas = klas @@ -309,15 +758,15 @@ def _ser(self, val: coring.Matter): return val.qb64b - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: memoryview | bytes): """ Deserialize val to str Parameters: - val (Union[str, memoryview, bytes]): convertable to coring.matter + val (memoryview | bytes): convertable to coring.matter """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes - return self.klas(qb64b=val) + return self.klas(qb64b=val) # qb64b parameter accepts str class CesrSuber(CesrSuberBase, Suber): @@ -333,15 +782,49 @@ class CesrSuber(CesrSuberBase, Suber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False klas (Type[coring.Matter]): Class reference to subclass of Matter or Indexer or Counter or any ducktyped class of Matter """ super(CesrSuber, self).__init__(*pa, **kwa) +class CesrOnSuber(CesrSuberBase, OnSuberBase, Suber): + """ + Subclass of CesrSuberBase, OnSuberBase, and Suber that adds methods for + keys with ordinal numbered suffixes and values that are Cesr serializations + of Matter subclass ducktypes. + + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(CesrOnSuber, self).__init__(*pa, **kwa) + + class CatCesrSuberBase(CesrSuberBase): """ Base Class whose values stored in db are a concatenation of the .qb64b property @@ -360,7 +843,7 @@ class CatCesrSuberBase(CesrSuberBase): def __init__(self, *pa, klas: Iterable = None, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -368,8 +851,10 @@ def __init__(self, *pa, klas: Iterable = None, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ if klas is None: @@ -377,19 +862,18 @@ def __init__(self, *pa, klas: Iterable = None, **kwa): if not nonStringIterable(klas): # not iterable klas = (klas, ) # make it so super(CatCesrSuberBase, self).__init__(*pa, klas=klas, **kwa) - # self.klas = klas def _ser(self, val: Union[Iterable, coring.Matter]): """ Serialize val to bytes to store in db - Concatenates .qb64b of each instance in objs and returns val bytes + Concatenates .qb64b of each instance in val and returns val bytes Returns: - val (bytes): concatenation of .qb64b of each object instance in vals + cat (bytes): concatenation of .qb64b of each object instance in vals Parameters: - subs (Union[Iterable, coring.Matter]): of subclass instances. + val (Union[Iterable, coring.Matter]): of subclass instances. """ if not nonStringIterable(val): # not iterable @@ -397,7 +881,7 @@ def _ser(self, val: Union[Iterable, coring.Matter]): return (b''.join(obj.qb64b for obj in val)) - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: memoryview | bytes | bytearray): """ Converts val bytes to vals tuple of subclass instances by deserializing .qb64b concatenation in order of each instance in .klas @@ -422,7 +906,7 @@ class CatCesrSuber(CatCesrSuberBase, Suber): such as Matter, Indexer, Counter Automatically serializes and deserializes from qb64b to/from CESR instances - Attributes: + Attributes: db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes @@ -432,7 +916,7 @@ class CatCesrSuber(CatCesrSuberBase, Suber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -440,8 +924,10 @@ def __init__(self, *pa, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(CatCesrSuber, self).__init__(*pa, **kwa) @@ -474,7 +960,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str='docs.', dupsort: bool=False, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -482,37 +968,49 @@ def __init__(self, db: dbing.LMDBer, *, each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(IoSetSuber, self).__init__(db=db, subkey=subkey, dupsort=False, **kwa) - def put(self, keys: Union[str, Iterable], vals: Iterable): + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Puts all vals at effective key made from keys and hidden ordinal suffix. that are not already in set of vals at key. Does not overwrite. Parameters: - keys (Iterable): of key strs to be combined in order to form key - vals (Iterable): of str serializations + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): of str serializations Returns: result (bool): True If successful, False otherwise. """ + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putIoSetVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals], sep=self.sep)) - def add(self, keys: Union[str, Iterable], val: Union[bytes, str, memoryview]): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): """ - Add val to vals at effective key made from keys and hidden ordinal suffix. - that is not already in set of vals at key. Does not overwrite. + Add val idempotently to vals at effective key made from keys and hidden + ordinal suffix. Idempotent means that added value is not already in set + of vals at key. Does not overwrite or add same value at same key more + than once. Parameters: - keys (Iterable): of key strs to be combined in order to form key - val (Union[bytes, str]): serialization + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str | bytes | memoryview): serialization Returns: result (bool): True means unique value added among duplications, @@ -525,7 +1023,8 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str, memoryview]): sep=self.sep)) - def pin(self, keys: Union[str, Iterable], vals: Iterable): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Pins (sets) vals at effective key made from keys and hidden ordinal suffix. Overwrites. Removes all pre-existing vals that share same effective keys @@ -539,15 +1038,15 @@ def pin(self, keys: Union[str, Iterable], vals: Iterable): result (bool): True If successful, False otherwise. """ - key = self._tokey(keys) - self.db.delIoSetVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.setIoSetVals(db=self.sdb, - key=key, + key=self._tokey(keys), vals=[self._ser(val) for val in vals], sep=self.sep)) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ Gets vals set list at key made from effective keys @@ -565,24 +1064,7 @@ def get(self, keys: Union[str, Iterable]): sep=self.sep)]) - def getLast(self, keys: Union[str, Iterable]): - """ - Gets last val inserted at effecive key made from keys and hidden ordinal - suffix. - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - - Returns: - val (str): value str, None if no entry at keys - - """ - val = self.db.getIoSetValLast(db=self.sdb, key=self._tokey(keys)) - return (self._des(val) if val is not None else val) - - - - def getIter(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets vals iterator at effecive key made from keys and hidden ordinal suffix. All vals in set of vals that share same effecive key are retrieved in @@ -601,32 +1083,37 @@ def getIter(self, keys: Union[str, Iterable]): yield self._des(val) - - def cnt(self, keys: Union[str, Iterable]): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ - Return count of values at effective key made from keys and hidden ordinal - suffix. Zero otherwise + Gets last val inserted at effecive key made from keys and hidden ordinal + suffix. Parameters: keys (Iterable): of key strs to be combined in order to form key + + Returns: + val (str): value str, None if no entry at keys + """ - return (self.db.cntIoSetVals(db=self.sdb, - key=self._tokey(keys), - sep=self.sep)) + val = self.db.getIoSetValLast(db=self.sdb, key=self._tokey(keys)) + return (self._des(val) if val is not None else val) - def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b''): + + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): """ Removes entry at effective key made from keys and hidden ordinal suffix that matches val if any. Otherwise deletes all values at effective key. Parameters: keys (Iterable): of key strs to be combined in order to form key - val (str): value at key to delete - if val is empty then remove all values at key + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key Returns: - result (bool): True if effective key with val exists so delete successful. + result (bool): True if effective key with val exists so rem successful. False otherwise """ @@ -641,108 +1128,54 @@ def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b'' sep=self.sep) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ - Return iterator over all the items in top branch defined by keys where - keys may be truncation of full branch. - - Returns: - items (Iterator): of (key, val) tuples over the all the items in - subdb whose effective key startswith key made from keys. - Keys may be keyspace prefix in order to return branches of key space. - When keys is empty then returns all items in subdb. - Returned key in each item has ordinal suffix removed. - - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self._des(val)) - - - def getIoSetItem(self, keys: Union[str, Iterable]): - """ - Gets (iokeys, val) ioitems list at key made from keys where key is - apparent effective key and ioitems all have same apparent effective key + Return count of values at effective key made from keys and hidden ordinal + suffix. Zero otherwise Parameters: keys (Iterable): of key strs to be combined in order to form key - - Returns: - ioitems (Iterable): each item in list is tuple (iokeys, val) where each - iokeys is actual key tuple including hidden suffix and - each val is str - empty list if no entry at keys - - """ - return ([(self._tokeys(iokey), self._des(val)) for iokey, val in - self.db.getIoSetItemsIter(db=self.sdb, - key=self._tokey(keys), - sep=self.sep)]) - - - def getIoSetItemIter(self, keys: Union[str, Iterable]): """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form key + return (self.db.cntIoSetVals(db=self.sdb, + key=self._tokey(keys), + sep=self.sep)) - Returns: - ioitems (Iterator): each item iterated is tuple (iokeys, val) where - each iokeys is actual keys tuple including hidden suffix and - each val is str - empty list if no entry at keys. - Raises StopIteration when done + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", + *, topive=False): """ - for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, - key=self._tokey(keys), - sep=self.sep): - yield (self._tokeys(iokey), self._des(val)) - + Return iterator over all the items in top branch defined by keys where + keys may be truncation of full branch. - def getIoItemIter(self, keys: Union[str, Iterable]=b""): - """ Returns: - items (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith effective key made from keys. - Keys may be keyspace prefix to return branches of key space. + items (Iterator): of (key, val) tuples over the all the items in + subdb whose effective key startswith key made from keys. + Keys may be keyspace prefix in order to return branches of key space. When keys is empty then returns all items in subdb. - + Returned key in each item has ordinal suffix removed. Parameters: keys (Iterable): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from + a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(iokey), self._des(val)) - - - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): - """ - Removes entry at keys - - Parameters: - iokeys (Iterable): of key str or tuple of key strs to be combined - in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + """ + for key, val in self.db.getTopIoSetItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive), sep=self.sep.encode()): + yield (self._tokeys(key), self._des(val)) class CesrIoSetSuber(CesrSuberBase, IoSetSuber): @@ -778,7 +1211,7 @@ class CesrIoSetSuber(CesrSuberBase, IoSetSuber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -786,6 +1219,8 @@ def __init__(self, *pa, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False klas (Type[coring.Matter]): Class reference to subclass of Matter or Indexer or Counter or any ducktyped class of Matter @@ -828,16 +1263,18 @@ class CatCesrIoSetSuber(CatCesrSuberBase, IoSetSuber): """ def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key - False (default) means do not enable duplicates at - each key + False (default) means do not enable duplicates at + each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(CatCesrIoSetSuber, self).__init__(*pa, **kwa) @@ -898,7 +1335,8 @@ def get(self, keys: Union[str, Iterable]): if val is not None else None) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", + *, topive=False): """ Returns: iterator (Iteratore: tuple (key, val) over the all the items in @@ -907,13 +1345,21 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): returns all items in subdb Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split yield (ikeys, self.klas(qb64b=bytes(val), @@ -1014,11 +1460,11 @@ def get(self, keys: Union[str, Iterable], decrypter: core.Decrypter = None): - def getItemIter(self, keys: Union[str, Iterable]=b"", - decrypter: core.Decrypter = None): + def getItemIter(self, keys: str|bytes|memoryview|Iterable= "", + decrypter: core.Decrypter = None, *, topive=False): """ Returns: - iterator (Iterator): of tuples (key, val) over the all the items in + items (Iterator): of tuples (key, val) over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace prefix to return branches of key space. When keys is empty then returns all items in subdb @@ -1027,13 +1473,21 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", db was encrypted and so decrypts before converting to Signer. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split if decrypter: @@ -1043,13 +1497,11 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", yield (ikeys, self.klas(qb64b=bytes(val), transferable=verfer.transferable)) - -class SerderSuber(Suber): +class SerderSuberBase(SuberBase): """ - Sub class of Suber where data is serialized Serder Subclass instance + Sub class of SuberBase where data is serialized Serder Subclass instance given by .klas Automatically serializes and deserializes using .klas Serder methods - """ def __init__(self, *pa, @@ -1059,197 +1511,153 @@ def __init__(self, *pa, Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False - Parameters: + Overridden Parameters: klas (Type[serdering.Serder]): Class reference to subclass of Serder """ - super(SerderSuber, self).__init__(*pa, **kwa) + super(SerderSuberBase, self).__init__(*pa, **kwa) self.klas = klas - def put(self, keys: Union[str, Iterable], val: serdering.SerderKERI): + def _ser(self, val: serdering.Serder): """ - Puts val at key made from keys. Does not overwrite - + Serialize value to bytes to store in db Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Serder): instance - - Returns: - result (bool): True If successful, False otherwise, such as key - already in database. + val (serdering.Serder): instance Serder subclass like SerderKERI """ - return (self.db.putVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) + return val.raw - def pin(self, keys: Union[str, Iterable], val: serdering.SerderKERI): + def _des(self, val: (str | memoryview | bytes)): """ - Pins (sets) val at key made from keys. Overwrites. - + Deserialize val to str Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Serder): instance - - Returns: - result (bool): True If successful. False otherwise. - """ - return (self.db.setVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) - - - def get(self, keys: Union[str, Iterable]): + val (Union[str, memoryview, bytes]): convertable to coring.matter """ - Gets Serder at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key + if isinstance(val, memoryview): # memoryview is always bytes + val = bytes(val) # convert to bytes + elif hasattr(val, "encode"): # str + val = val.encode() # convert to bytes + return self.klas(raw=val, verify=self.verify) - Returns: - Serder: - None: if no entry at keys - Usage: - Use walrus operator to catch and raise missing entry - if (srder := mydb.get(keys)) is None: - raise ExceptionHere - use srdr here +class SerderSuber(SerderSuberBase, Suber): + """ + Sub class of SerderSuberBase, Suber where data is serialized Serder Subclass + instance given by .klas + Automatically serializes and deserializes using .klas Serder methods + """ + def __init__(self, *pa, **kwa): """ - val = self.db.getVal(db=self.sdb, key=self._tokey(keys)) - return self.klas(raw=bytes(val), verify=self.verify) if val is not None else None - - - def rem(self, keys: Union[str, Iterable]): + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[serdering.Serder]): Class reference to subclass of Serder """ - Removes entry at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key + super(SerderSuber, self).__init__(*pa, **kwa) - Returns: - result (bool): True if key exists so delete successful. False otherwise - """ - return(self.db.delVal(db=self.sdb, key=self._tokey(keys))) +class SerderIoSetSuber(SerderSuberBase, IoSetSuber): + """ + Sub class of SerderSuberBase and IoSetSuber that allows multiple Serder + instances to be stored at the same db key in insertion order. + Example use case would be an escrow where the key is a sequence number + based index (such as snKey). - def getItemIter(self, keys: Union[str, Iterable]=b""): - """ - Returns: - iterator (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + Sub class of SerderSuberBase where data is serialized Serder Subclass instance + given by .klas + Automatically serializes and deserializes using .klas Serder methods - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. + Extends IoSetSuber so that all IoSetSuber methods now work with Serder + subclass for each val. - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield self._tokeys(iokey), self.klas(raw=bytes(val), verify=self.verify) + IoSetSuber stores at each effective key a set of distinct values that + share that same effective key where each member of the set is retrieved in + insertion order (dupsort==False) + The methods allows an Iterable (set valued) of Iterables of separation subclass + instances to be stored at a given effective key in insertion order. + Actual keys include a hidden ordinal key suffix that tracks insertion order. + The suffix is appended and stripped transparently from the keys. The set of + items with duplicate effective keys are retrieved in insertion order when + iterating or as a list of the set elements. The actual iokey for any item + includes the ordinal suffix. -class SchemerSuber(Suber): - """ - Sub class of Suber where data is serialized Schemer instance - Automatically serializes and deserializes using Schemer methods + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + klas (Iterable): of Class references to subclasses of CESR compatible + , each of to Type[coring.Matter etc] """ def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key - """ - super(SchemerSuber, self).__init__(*pa, **kwa) - - def put(self, keys: Union[str, Iterable], val: scheming.Schemer): - """ - Puts val at key made from keys. Does not overwrite - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Schemer): instance - - Returns: - result (bool): True If successful, False otherwise, such as key - already in database. - """ - return (self.db.putVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) - - def pin(self, keys: Union[str, Iterable], val: scheming.Schemer): - """ - Pins (sets) val at key made from keys. Overwrites. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Schemer): instance - - Returns: - result (bool): True If successful. False otherwise. - """ - return (self.db.setVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[serdering.Serder]): Class reference to subclass of Serder - def get(self, keys: Union[str, Iterable]): """ - Gets Serder at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key + super(SerderIoSetSuber, self).__init__(*pa, **kwa) - Returns: - Schemer: - None: if no entry at keys - Usage: - Use walrus operator to catch and raise missing entry - if (srder := mydb.get(keys)) is None: - raise ExceptionHere - use srdr here - """ - val = self.db.getVal(db=self.sdb, key=self._tokey(keys)) - return scheming.Schemer(raw=bytes(val)) if val is not None else None - def rem(self, keys: Union[str, Iterable]): - """ - Removes entry at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - """ - return self.db.delVal(db=self.sdb, key=self._tokey(keys)) +class SchemerSuber(SerderSuberBase, Suber): + """ + Sub class of SerderSuberBase and Suber where data is serialized Schemer instance + Schemer ser/des is ducktype of Serder using .raw + Automatically serializes and deserializes using Schemer methods + """ - def getItemIter(self, keys: Union[str, Iterable]=b""): + def __init__(self, *pa, + klas: Type[ scheming.Schemer] = scheming.Schemer, + **kwa): """ - Returns: - iterator (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb - - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[scheming.Schemer]): Class reference to ducktyped subclass + of Serder + Overridden Parameters: + klas (Type[scheming.Schemer]): Class reference to ducktyped subclass + of Serder intercepts passed in klas and forces it to Schemer """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield self._tokeys(iokey), scheming.Schemer(raw=bytes(val)) + if not issubclass(klas, scheming.Schemer): + raise TypeError(f"Invalid {klas=}, not subclass of {scheming.Schemer}.") + super(SchemerSuber, self).__init__(*pa, klas=klas, **kwa) class DupSuber(SuberBase): @@ -1273,7 +1681,8 @@ def __init__(self, db: Type[dbing.LMDBer], *, super(DupSuber, self).__init__(db=db, subkey=subkey, dupsort=True, **kwa) - def put(self, keys: Union[str, Iterable], vals: list): + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Puts all vals at key made from keys. Does not overwrite. Adds to existing dup values at key if any. Duplicate means another entry at the same key @@ -1282,8 +1691,10 @@ def put(self, keys: Union[str, Iterable], vals: list): unless it is a unique value for that key. Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): str or bytes of each value to be written at key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): str or bytes of each + value to be written at key Returns: result (bool): True If successful, False otherwise. @@ -1291,12 +1702,15 @@ def put(self, keys: Union[str, Iterable], vals: list): Apparently always returns True (how .put works with dupsort=True) """ + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals])) - def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview ): """ Add val to vals at key made from keys. Does not overwrite. Adds to existing dup values at key if any. Duplicate means another entry at the same key @@ -1305,8 +1719,8 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): unless it is a unique value for that key. Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Union[str, bytes]): value + keys (str | bytes | memoryview | Iterable): of key strs to be combined in order to form key + val (str | bytes | memoryview): value Returns: result (bool): True means unique value among duplications, @@ -1318,14 +1732,16 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): val=self._ser(val))) - def pin(self, keys: Union[str, Iterable], vals: list): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Pins (sets) vals at key made from keys. Overwrites. Removes all pre-existing dup vals and replaces them with vals Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): str or bytes values + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): str or bytes values Returns: result (bool): True If successful, False otherwise. @@ -1333,18 +1749,21 @@ def pin(self, keys: Union[str, Iterable], vals: list): """ key = self._tokey(keys) self.db.delVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putVals(db=self.sdb, key=key, vals=[self._ser(val) for val in vals])) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ Gets dup vals list at key made from keys Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key Returns: vals (list): each item in list is str @@ -1355,7 +1774,7 @@ def get(self, keys: Union[str, Iterable]): self.db.getValsIter(db=self.sdb, key=self._tokey(keys))] - def getLast(self, keys: Union[str, Iterable]): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ Gets last dup val at key made from keys @@ -1370,14 +1789,15 @@ def getLast(self, keys: Union[str, Iterable]): return self._des(val) if val is not None else val - def getIter(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets dup vals iterator at key made from keys Duplicates are retrieved in lexocographic order not insertion order. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key Returns: iterator: vals each of str. Raises StopIteration when done @@ -1387,17 +1807,19 @@ def getIter(self, keys: Union[str, Iterable]): yield self._des(val) - def cnt(self, keys: Union[str, Iterable]): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ Return count of dup values at key made from keys, zero otherwise Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key """ return (self.db.cntVals(db=self.sdb, key=self._tokey(keys))) - def rem(self, keys: Union[str, Iterable], val=b''): + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): """ Removes entry at keys @@ -1410,181 +1832,438 @@ def rem(self, keys: Union[str, Iterable], val=b''): result (bool): True if key exists so delete successful. False otherwise """ - return (self.db.delVals(db=self.sdb, key=self._tokey(keys), val=self._ser(val))) + if val: + return (self.db.delVals(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val))) + else: + return (self.db.delVals(db=self.sdb, + key=self._tokey(keys))) -class CesrDupSuber(DupSuber): + +class CesrDupSuber(CesrSuberBase, DupSuber): """ - Sub class of DupSuber that supports multiple entries at each key (duplicates) - with dupsort==True, where data where data is Matter.qb64b property - which is a fully qualified serialization of matter subclass instance - Automatically serializes and deserializes from qb64b to/from Matter instances + Sub class of DupSuber whose values are CESR ducktypes of Matter subclasses. + serialized to and deserialied from val instance .qb64b property + which is a fully qualified serialization. + Automatically serializes and deserializes from qb64b to/from Matter ducktyped + instances + DupSuber supports multiple entries at each key (duplicates) with dupsort==True Do not use if serialized value is greater than 511 bytes. This is a limitation of dupsort==True sub dbs in LMDB """ - def __init__(self, *pa, klas: Type[coring.Matter] = coring.Matter, **kwa): + def __init__(self, *pa, **kwa): """ - Parameters: - db (dbing.LMDBer): base db - subkey (str): LMDB sub database key - klas (Type[coring.Matter]): Class reference to subclass of Matter + Inherited Parameters: + """ super(CesrDupSuber, self).__init__(*pa, **kwa) - self.klas = klas - def put(self, keys: Union[str, Iterable], vals: list): +class IoDupSuber(DupSuber): + """ + Sub class of DupSuber that supports Insertion Ordering (IoDup) of duplicates + By automagically prepending and stripping ordinal proem to/from each + duplicate value at a given key. + + IoDupSuber supports insertion ordered multiple entries at each key + (duplicates) with dupsort==True + + Do not use if serialized length key + proem + value, is greater than 511 bytes. + This is a limitation of dupsort==True sub dbs in LMDB + + IoDupSuber may be more performant then IoSetSuber for values that are indices + to other sub dbs that fit the size constraint because LMDB support for + duplicates is more space efficient and code performant. + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + def __init__(self, *pa, **kwa): """ - Puts all vals at key made from keys. Does not overwrite. Adds to existing - dup values at key if any. Duplicate means another entry at the same key - but the entry is still a unique value. Duplicates are inserted in - lexocographic order not insertion order. Lmdb does not insert a duplicate - unless it is a unique value for that key. + Inherited Parameters: + + """ + super(IoDupSuber, self).__init__(*pa, **kwa) + + + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): + """ + Puts all vals idempotently at key made from keys in insertion order using + hidden ordinal proem. Idempotently means do not put any val in vals that is + already in dup vals at key. Does not overwrite. Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): instances of coring.Matter (subclass) + keys (Iterable): of key strs to be combined in order to form key + vals (Iterable): of str serializations Returns: result (bool): True If successful, False otherwise. - Apparently always returns True (how .put works with dupsort=True) - """ - return (self.db.putVals(db=self.sdb, - key=self._tokey(keys), - vals=[val.qb64b for val in vals])) + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable + return (self.db.putIoDupVals(db=self.sdb, + key=self._tokey(keys), + vals=[self._ser(val) for val in vals])) - def add(self, keys: Union[str, Iterable], val: coring.Matter): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): """ - Add val to vals at key made from keys. Does not overwrite. Adds to existing - dup values at key if any. Duplicate means another entry at the same key - but the entry is still a unique value. Duplicates are inserted in - lexocographic order not insertion order. Lmdb does not insert a duplicate - unless it is a unique value for that key. + Add val idempotently at key made from keys in insertion order using hidden + ordinal proem. Idempotently means do not add val that is already in + dup vals at key. Does not overwrite. Parameters: - keys (tuple): of key strs to be combined in order to form key - val (coring.Matter): instance (subclass) + keys (Iterable): of key strs to be combined in order to form key + val (str | bytes | memoryview): serialization Returns: - result (bool): True means unique value among duplications, - False means duplicte of same value already exists. + result (bool): True means unique value added among duplications, + False means duplicate of same value already exists. """ - return (self.db.addVal(db=self.sdb, - key=self._tokey(keys), - val=val.qb64b)) + return (self.db.addIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val))) - def pin(self, keys: Union[str, Iterable], vals: list): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ - Pins (sets) vals at key made from keys. Overwrites. Removes all - pre-existing dup vals and replaces them with vals + Pins (sets) vals at key made from keys in insertion order using hidden + ordinal proem. Overwrites. Removes all pre-existing vals that share + same keys and replaces them with vals Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): instances of coring.Matter (subclass) + keys (Iterable): of key strs to be combined in order to form key + vals (Iterable): str serializations Returns: result (bool): True If successful, False otherwise. """ key = self._tokey(keys) - self.db.delVals(db=self.sdb, key=key) # delete all values - return (self.db.putVals(db=self.sdb, - key=key, - vals=[val.qb64b for val in vals])) + self.db.delIoDupVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable + return self.db.putIoDupVals(db=self.sdb, + key=key, + vals=[self._ser(val) for val in vals]) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ - Gets dup vals list at key made from keys + Gets vals dup list in insertion order using key made from keys and + hidden ordinal proem on dups. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (Iterable): of key strs to be combined in order to form key Returns: - vals (list): each item in list is instance of self.klas + vals (Iterable): each item in list is str empty list if no entry at keys """ - return [self.klas(qb64b=bytes(val)) for val in - self.db.getValsIter(db=self.sdb, key=self._tokey(keys))] + return ([self._des(val) for val in + self.db.getIoDupVals(db=self.sdb, key=self._tokey(keys))]) - def getLast(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ - Gets last dup val at key made from keys + Gets vals dup iterator in insertion order using key made from keys and + hidden ordinal proem on dups. + All vals in dups that share same key are retrieved in insertion order. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key parts Returns: - val (str): instance of self.klas else None if no value at key + vals (Iterator): str values. Raises StopIteration when done """ - val = self.db.getValLast(db=self.sdb, key=self._tokey(keys)) - if val is not None: - val = self.klas(qb64b=bytes(val)) - return val + for val in self.db.getIoDupValsIter(db=self.sdb, + key=self._tokey(keys)): + yield self._des(val) + def getLast(self, keys: str | bytes | memoryview | Iterable): + """ + Gets last val inserted at key made from keys in insertion order using + hidden ordinal proem. + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + + Returns: + val (str): value str, None if no entry at keys - def getIter(self, keys: Union[str, Iterable]): """ - Gets dup vals iterator at key made from keys + val = self.db.getIoDupValLast(db=self.sdb, key=self._tokey(keys)) + return (self._des(val) if val is not None else val) - Duplicates are retrieved in lexocographic order not insertion order. + + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): + """ + Removes entry at key made from keys and dup val that matches val if any, + notwithstanding hidden ordinal proem. Otherwise deletes all dup values + at key if any. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key Returns: - iterator: vals each of self.klas. Raises StopIteration when done + result (bool): True if key with dup val exists so rem successful. + False otherwise """ - for val in self.db.getValsIter(db=self.sdb, key=self._tokey(keys)): - yield self.klas(qb64b=bytes(val)) + if val: + return self.db.delIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val)) + else: + return self.db.delIoDupVals(db=self.sdb, key=self._tokey(keys)) - def rem(self, keys: Union[str, Iterable], val=None): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ - Removes entry at keys + Return count of dup values at key made from keys with hidden ordinal + proem. Zero otherwise Parameters: - keys (tuple): of key strs to be combined in order to form key - val (coring.Matter): instance of coring.Matter subclass dup val - at key to delete - if val is None then remove all values at key + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + """ + return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) + + + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", + *, topive=False): + """ + Return iterator over all the items including dup items for all keys + in top branch defined by keys where keys may be truncation of full branch. Returns: - result (bool): True if key exists so delete successful. False otherwise + items (Iterator): of (key, val) tuples over the all the items in + subdb whose key startswith key made from keys and val has its hidden + dup ordinal proem removed. + Keys may be keyspace prefix in order to return branches of key space. + When keys is empty then returns all items in subdb. + + Parameters: + keys (str | bytes | memoryview | Iterable): key or key parts that + may be a truncation of a full keys tuple in in order to address + all the items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - if val is not None: - val = val.qb64b - else: - val = b'' - return (self.db.delVals(db=self.sdb, key=self._tokey(keys), val=val)) + for key, val in self.db.getTopIoDupItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): + yield (self._tokeys(key), self._des(val)) - def getItemIter(self, keys: Union[str, Iterable]=b""): + +class OnIoDupSuber(OnSuberBase, IoDupSuber): + """ + Sub class of IoDupSuber and OnSuberBase that supports Insertion Ordering + (IoDup) of duplicates but where the trailing part of the key space is + a serialized monotonically increasing ordinal number. This is useful for + escrows of key events etc where duplicates of likely events are maintained + in insertion order. + Insertion order is maintained by automagically prepending and stripping an + ordinal ordering proem to/from each duplicate value at a given key. + + OnIoDupSuber adds the convenience methods from OnSuberBase to IoDupSuber for + those cases where the keyspace has a trailing ordinal part. + + There are two ordinals, one in the key space and a hidden one in the duplicate + data value space. + + OnIoDupSuber supports insertion ordered multiple entries at each key + (duplicates) with dupsort==True + + Do not use if serialized length key + proem + value, is greater than 511 bytes. + This is a limitation of dupsort==True sub dbs in LMDB + + OnIoDupSuber may be more performant then IoSetSuber for values that are indices + to other sub dbs that fit the size constraint because LMDB support for + duplicates is more space efficient and code performant. + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + + """ + super(OnIoDupSuber, self).__init__(*pa, **kwa) + + + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): """ Returns: - iterator (Iteratore: tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + on (int): ordinal number of newly appended val Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.appendOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) + + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + val (Iterator[bytes]): deserialized val of of each + onkey + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[(top keys, on, val)]): triples of (onkeys, on int, + deserialized val) + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + def getOnLastIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + last (Iterator[bytes]): deserialized last duplicate val of of each + onkey + + Parameters: + keys (str | bytes | memoryview | iterator): top keys as prefix to be + combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupLastValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + + def getOnLastItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + items (Iterator[(top keys, on, val)]): triples of (keys, on int, + deserialized val) last duplicate item as each onkey where onkey + is the key+serialized on + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupLastItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + def getOnBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + val (Iterator[bytes]): deserialized val of of each + onkey in reverse order + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValBackIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[(top keys, on, val)]): triples of (onkeys, on int, + deserialized val) in reverse order + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(key), self.klas(qb64b=bytes(val))) + for keys, on, val in (self.db.getOnIoDupItemBackIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 0d46f9f1..e696baba 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -23,7 +23,7 @@ from ..core.signing import (Salter,) from ..core.eventing import SealEvent, ample, TraitDex, verifySigs from ..db import basing, dbing -from ..db.dbing import dgKey, snKey +from ..db.dbing import dgKey, snKey, splitSnKey from ..help import helping from ..kering import (MissingWitnessSignatureError, Version, MissingAnchorError, ValidationError, OutOfOrderError, LikelyDuplicitousError) @@ -1172,7 +1172,7 @@ def vcState(self, vci): status (Serder): transaction event state notification message """ digs = [] - for _, dig in self.reger.getTelItemPreIter(pre=vci.encode("utf-8")): + for _, _, dig in self.reger.getTelItemPreIter(pre=vci.encode("utf-8")): digs.append(dig) if len(digs) == 0: @@ -2031,9 +2031,10 @@ def processEscrowOutOfOrders(self): 5. Remove event digest from oots if processed successfully or a non-out-of-order event occurs. """ - for (pre, snb, digb) in self.reger.getOotItemIter(): + for key, digb in self.reger.getOotItemIter(): # (pre, snb, digb) in self.reger.getOotItemIter() try: - sn = int(snb, 16) + #sn = int(snb, 16) + pre, sn = splitSnKey(key) dgkey = dgKey(pre, digb) traw = self.reger.getTvt(dgkey) if traw is None: @@ -2099,8 +2100,9 @@ def processEscrowAnchorless(self): 6. Remove event digest from oots if processed successfully or a non-anchorless event occurs. """ - for (pre, snb, digb) in self.reger.getTaeItemIter(): - sn = int(snb, 16) + for key, digb in self.reger.getTaeItemIter(): #(pre, snb, digb) in self.reger.getTaeItemIter() + pre, sn = splitSnKey(key) + #sn = int(snb, 16) try: dgkey = dgKey(pre, digb) traw = self.reger.getTvt(dgkey) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index cf2ed805..c2fccfda 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -508,7 +508,7 @@ def clonePreIter(self, pre, fn=0): if hasattr(pre, 'encode'): pre = pre.encode("utf-8") - for fn, dig in self.getTelItemPreIter(pre, fn=fn): + for _, fn, dig in self.getTelItemPreIter(pre, fn=fn): msg = self.cloneTvt(pre, dig) yield msg @@ -671,7 +671,7 @@ def getTelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOrdItemPreIter(db=self.tels, pre=pre, on=fn) + return self.getOnItemIter(db=self.tels, key=pre, on=fn) def cntTels(self, pre, fn=0): """ @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntValsAllPre(db=self.tels, pre=pre, on=fn) + return self.cntOnVals(db=self.tels, key=pre, on=fn) def getTibs(self, key): """ @@ -809,7 +809,8 @@ def getTaeItemIter(self): Return iterator of all items in .taes """ - return self.getAllItemIter(self.taes, split=True) + return self.getTopItemIter(self.taes) + def delTae(self, key): """ @@ -852,7 +853,8 @@ def getOotItemIter(self): Return iterator of all items in .taes """ - return self.getAllItemIter(self.oots, split=True) + return self.getTopItemIter(self.oots) + def delOot(self, key): """ @@ -907,7 +909,7 @@ def putBaks(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.baks, key, vals) + return self.putIoDupVals(self.baks, key, vals) def addBak(self, key, val): @@ -918,7 +920,7 @@ def addBak(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.baks, key, val) + return self.addIoDupVal(self.baks, key, val) def getBaks(self, key): @@ -928,7 +930,7 @@ def getBaks(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.baks, key) + return self.getIoDupVals(self.baks, key) def getBaksIter(self, key): @@ -938,7 +940,7 @@ def getBaksIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.baks, key) + return self.getIoDupValsIter(self.baks, key) def cntBaks(self, key): """ @@ -946,7 +948,7 @@ def cntBaks(self, key): Return count of backer prefixes at key Returns zero if no entry at key """ - return self.cntIoVals(self.baks, key) + return self.cntIoDupVals(self.baks, key) def delBaks(self, key): @@ -955,7 +957,7 @@ def delBaks(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.baks, key) + return self.delIoDupVals(self.baks, key) def delBak(self, key, val): @@ -968,7 +970,7 @@ def delBak(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.baks, key, val) + return self.delIoDupVal(self.baks, key, val) def buildProof(prefixer, seqner, diger, sigers): diff --git a/tests/app/test_forwarding.py b/tests/app/test_forwarding.py index 6f3523cd..0fa1b3c5 100644 --- a/tests/app/test_forwarding.py +++ b/tests/app/test_forwarding.py @@ -68,7 +68,7 @@ def test_postman(seeder): doist.exit() msgs = [] - for _, topic, msg in mbx.cloneTopicIter(topic=recpHab.pre + "/echo", fn=0): + for _, topic, msg in mbx.cloneTopicIter(topic=recpHab.pre + "/echo"): msgs.append(msg) assert len(msgs) == 1 diff --git a/tests/app/test_storing.py b/tests/app/test_storing.py index d37eeae3..2abafbc7 100644 --- a/tests/app/test_storing.py +++ b/tests/app/test_storing.py @@ -9,7 +9,7 @@ from keri.app import keeping from keri.core import coring, serdering -from keri.db import dbing, basing +from keri.db import dbing, basing, subing from keri.peer import exchanging from keri.app.storing import Mailboxer @@ -28,7 +28,8 @@ def test_mailboxing(): assert mber.env.path() == mber.path assert os.path.exists(mber.path) - assert isinstance(mber.tpcs, lmdb._Database) + #assert isinstance(mber.tpcs, lmdb._Database) + assert isinstance(mber.tpcs, subing.OnSuber) mber.close(clear=True) assert not os.path.exists(mber.path) @@ -85,7 +86,7 @@ def test_mailboxing(): mber.storeMsg(topic=dest.qb64b, msg=exn.raw) msgs = [] - for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=0): + for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b): msgs.append((fn, msg)) assert(len(msgs)) == 10 @@ -95,18 +96,18 @@ def test_mailboxing(): d = exn.ked["a"] assert d["b"] == idx - msgs = [] - for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=10): - msgs.append(msg) + #msgs = [] + #for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=10): + #msgs.append(msg) - assert(len(msgs)) == 0 + #assert(len(msgs)) == 0 - msgs = [] - for tn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=4): - msgs.append((tn, msg)) + #msgs = [] + #for tn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=4): + #msgs.append((tn, msg)) - assert(len(msgs)) == 6 - assert msgs[0][0] == 4 + #assert(len(msgs)) == 6 + #assert msgs[0][0] == 4 diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index d3f419fa..7808bdc1 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -352,39 +352,77 @@ def test_partial_signed_escrow(): def test_missing_delegator_escrow(): """ Test missing delegator escrow + + bod is the delegator + del is the delegate + wat is the watcher """ - # bob is the delegator del is bob's delegate bobSalt = core.Salter(raw=b'0123456789abcdef').qb64 delSalt = core.Salter(raw=b'abcdef0123456789').qb64 + watSalt = core.Salter(raw=b'wxyzabcdefghijkl').qb64 psr = parsing.Parser() - with basing.openDB(name="bob") as bobDB, \ - keeping.openKS(name="bob") as bobKS, \ - basing.openDB(name="del") as delDB, \ - keeping.openKS(name="del") as delKS: + with (basing.openDB(name="bob") as bobDB, + keeping.openKS(name="bob") as bobKS, + basing.openDB(name="del") as delDB, + keeping.openKS(name="del") as delKS, + basing.openDB(name="wat") as watDB, \ + keeping.openKS(name="wat") as watKS ): # Init key pair managers bobMgr = keeping.Manager(ks=bobKS, salt=bobSalt) delMgr = keeping.Manager(ks=delKS, salt=delSalt) + watMgr = keeping.Manager(ks=watKS, salt=watSalt) # Init Keverys bobKvy = eventing.Kevery(db=bobDB) delKvy = eventing.Kevery(db=delDB) + watKvy = eventing.Kevery(db=watDB) + + # Setup Wat with own inception event + verfers, digers = watMgr.incept(stem='wat', temp=True) # algo default salty and rooted + + watSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + watPre = watSrdr.pre + watMgr.move(old=verfers[0].qb64, new=watPre) # move key pair label to prefix + # Setup wat's prefixes so wat's KEL will be Kever.locallyOwned() + watDB.prefixes.add(watPre) + assert watPre in watDB.prefixes + # setup wat's on kel + sigers = watMgr.sign(ser=watSrdr.raw, verfers=verfers) + msg = bytearray(watSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + watIcpMsg = msg # save for later - # Setup Bob by creating inception event + # apply msg to wats's Kevery + psr.parse(ims=bytearray(watIcpMsg), kvy=watKvy, local=True) + watK = watKvy.kevers[watPre] + assert watK.prefixer.qb64 == watPre + assert watK.serder.said == watSrdr.said + + # Setup Bob with own inception event verfers, digers = bobMgr.incept(stem='bob', temp=True) # algo default salty and rooted bobSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], ndigs=[diger.qb64 for diger in digers], code=coring.MtrDex.Blake3_256) - bobPre = bobSrdr.ked["i"] - + bobPre = bobSrdr.pre bobMgr.move(old=verfers[0].qb64, new=bobPre) # move key pair label to prefix + # Setup Bob's prefixes so bob's KEL will be Kever.locallyOwned() and + # Del's KEL will be Kever.locallyDelegated() + bobDB.prefixes.add(bobPre) + assert bobPre in bobDB.prefixes sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) - msg = bytearray(bobSrdr.raw) counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) @@ -395,28 +433,32 @@ def test_missing_delegator_escrow(): bobIcpMsg = msg # save for later # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(bobIcpMsg), kvy=bobKvy, local=True) bobK = bobKvy.kevers[bobPre] assert bobK.prefixer.qb64 == bobPre assert bobK.serder.said == bobSrdr.said + assert bobK.sn == 0 # apply msg to del's Kevery so he knows about the AID - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) + psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) assert bobK.prefixer.qb64 in delKvy.kevers + delBobK = bobKvy.kevers[bobPre] # bobs kever in dels kevery + assert delBobK.sn == 0 - # Setup Del's inception event assuming that Bob's next event will be an ixn delegating event + # Setup Del's inception event assuming that Bob's next event will be + # an ixn delegating event verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted - delSrdr = eventing.delcept(keys=[verfer.qb64 for verfer in verfers], delpre=bobPre, ndigs=[diger.qb64 for diger in digers]) - delPre = delSrdr.ked["i"] - + delPre = delSrdr.pre delMgr.move(old=verfers[0].qb64, new=delPre) # move key pair label to prefix + # Setup Del's prefixes so Del's KEL will be Kever.locallyOwned() + delDB.prefixes.add(delPre) + assert delPre in delDB.prefixes - # Now create delegating event + # Now create delegating event for Bob seal = eventing.SealEvent(i=delPre, s=delSrdr.ked["s"], d=delSrdr.said) @@ -433,14 +475,14 @@ def test_missing_delegator_escrow(): msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - bobIxnMsg = msg + bobIxnMsg1 = msg # delegating event with attachments # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(bobIxnMsg1), kvy=bobKvy, local=True) assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 1 - # now create msg with Del's delegated inception event + # now create Del's delegated inception event msg sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) @@ -455,9 +497,12 @@ def test_missing_delegator_escrow(): seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) msg.extend(bobSrdr.saidb) + delIcpMsg = msg # apply Del's delegated inception event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) + # because the attachment includes valid source seal then the Delegables + # escrow is bypassed and is validated and shows up in AES + psr.parse(ims=bytearray(delIcpMsg), kvy=bobKvy, local=True) # bobKvy.process(ims=bytearray(msg)) # process local copy of msg assert delPre in bobKvy.kevers # successfully validated bobDelK = bobKvy.kevers[delPre] # delK in bobs kevery @@ -467,53 +512,50 @@ def test_missing_delegator_escrow(): assert couple == seqner.qb64b + bobSrdr.saidb # apply Del's inception msg to Del's Kevery - # Dels event will fail but will add to its escrow - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + # Because locallyOwned by delegate event does not validate delegation + # and ignores the attached source seal + psr.parse(ims=bytearray(delIcpMsg), kvy=delKvy, local=True) + assert delPre in delKvy.kevers + delK = delKvy.kevers[delPre] + # no AES entry for del's own delegated event when locallyOwned + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + + # apply Del's delegated inception event message to wats's Kevery as remote + # because the attachment includes valid source seal but wat does not + # yet have Bob's delegating event entry. The event goes into partial + # delegated event escrow + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - - # verify Kevery process partials escrow is idempotent to previously escrowed events - # assuming not stale but nothing else has changed - delKvy.processEscrowPartialSigs() - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + assert escrows[0] == delSrdr.said # escrow entry for event + + # Now apply Bob's incept to wat's kvy and process escrow + psr.parse(ims=bytearray(bobIcpMsg), kvy=watKvy, local=False) + assert bobPre in watKvy.kevers + watBobK = watKvy.kevers[bobPre] + assert watBobK.sn == 0 + watKvy.processEscrows() + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + assert escrows[0] == delSrdr.said # escrow entry for event - # apply Bob's inception to Dels' Kvy - psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(bobIcpMsg)) # process remote copy of msg - assert bobPre in delKvy.kevers # message accepted - delKvy.processEscrowPartialSigs() # process escrow - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - - # apply Bob's delegating interaction to Dels' Kvy - psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(bobIxnMsg)) # process remote copy of msg - delKvy.processEscrowPartialSigs() # process escrows - assert delPre in delKvy.kevers # event removed from escrow - delK = delKvy.kevers[delPre] - assert delK.delegated - assert delK.serder.said == delSrdr.said - couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + # Now apply Bob's ixn to wat's kvy and process escrow + psr.parse(ims=bytearray(bobIxnMsg1), kvy=watKvy, local=False) + watKvy.processEscrows() + escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + assert len(escrows) == 0 + assert watBobK.sn == 1 + + assert delPre in watKvy.kevers # successfully validated + watDelK = watKvy.kevers[delPre] # delK in wats kevery + assert watDelK.delegated + assert watDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = watKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) assert couple == seqner.qb64b + bobSrdr.saidb - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 0 - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow is None # delegated inception delegation couple # Setup Del rotation event verfers, digers = delMgr.rotate(pre=delPre, temp=True) @@ -542,19 +584,25 @@ def test_missing_delegator_escrow(): for siger in sigers: msg.extend(siger.qb64b) - # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + bobIxnMsg2 = msg + + # apply bobs IXN msg to bob's Kevery + psr.parse(ims=bytearray(bobIxnMsg2), kvy=bobKvy, local=True) assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 2 # apply msg to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg - assert delKvy.kevers[bobPre].serder.said == bobSrdr.said + psr.parse(ims=bytearray(bobIxnMsg2), kvy=delKvy, local=True) + assert delBobK.serder.said == bobSrdr.said + assert delBobK.sn == 2 + + # apply msg to wat's Kevery + psr.parse(ims=bytearray(bobIxnMsg2), kvy=watKvy, local=True) + assert watBobK.serder.said == bobSrdr.said + assert watBobK.sn == 2 # now create msg from Del's delegated rotation event sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) - msg = bytearray(delSrdr.raw) counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) @@ -568,22 +616,28 @@ def test_missing_delegator_escrow(): msg.extend(seqner.qb64b) msg.extend(bobSrdr.saidb) + delRotMsg = msg + # apply Del's delegated Rotation event message to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + psr.parse(ims=bytearray(delRotMsg), kvy=delKvy, local=True) assert delK.delegated assert delK.serder.said == delSrdr.said - couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - assert couple == seqner.qb64b + bobSrdr.saidb + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) # apply Del's delegated Rotation event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(delRotMsg), kvy=bobKvy, local=True) assert bobDelK.delegated assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) assert couple == seqner.qb64b + bobSrdr.saidb + # apply Del's delegated Rotation event message to wats's Kevery + psr.parse(ims=bytearray(delRotMsg), kvy=watKvy, local=True) + assert watDelK.delegated + assert watDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = watKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + assert not os.path.exists(delKS.path) @@ -1425,5 +1479,6 @@ def test_unverified_trans_receipt_escrow(): if __name__ == "__main__": - test_unverified_receipt_escrow() + #test_unverified_receipt_escrow() + test_missing_delegator_escrow() diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index c50cee10..2b799a98 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -4907,6 +4907,40 @@ def test_load_event(mockHelpingNowUTC): # Endorse Tee's inception event with Wan's Hab just so we have non-trans receipts evt = eventing.loadEvent(wanHab.db, teeHab.pre, teeHab.pre) + #assert evt == {'ked': {'a': [], + #'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], + #'bt': '1', + #'c': [], + #'d': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', + #'di': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'i': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', + #'k': ['DLDlVl1H2Q138A5tftVRpyy834ejsY33BB71kXLRNP2h'], + #'kt': '1', + #'n': ['EBTtZqMkJOO4nf3cCt6SdezwkoCKtx2fGUKHeFApj_Yx'], + #'nt': '1', + #'s': '0', + #'t': 'dip', + #'v': 'KERI10JSON00018d_'}, + #'receipts': {'nontransferable': [{'prefix': 'BEXrSXVksXpnfno_Di6RBX2Lsr9VWRAihjLhowfjNOQQ', + #'signature': '0BCQOeNT3mwAHxh6mYU9K_B2VmbtjJh7_8115k4JrBPR3c4' + #'3jUSO197H2J73vWMi61qzOovNkSWQbnRx3NFnrk8I'}], + #'transferable': [{'prefix': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'said': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'sequence': '0AAAAAAAAAAAAAAAAAAAAAAA', + #'signature': 'AADGbcmUNw_SX7OVNX-PQYl41UZx_pgJXHOoMWrcfmCDGgkc1-' + #'MqXJjMD9S9moJ-lpPL9-AiXgITemMZL_QYGzIA'}]}, + #'signatures': [{'index': 0, + #'signature': 'AAC1-NTntZ0xkgHwooNcKxe9G4XC-rgkSryVz0B_QrZR2kkv4IKi7DMkfMBd4Eck-' + #'2NAi0DMuZeXnlvch6ZP0coO'}], + #'source_seal': {'said': 'EF7pHYN6XABC9znRdzprt5frW-MMry9rfrCI-_t5Y8VD', + #'sequence': 1}, + #'stored': True, + #'timestamp': '2021-01-01T00:00:00.000000+00:00', + #'witness_signatures': [{'index': 0, + #'signature': 'AABPMW3J1iZyMC-elPOkdIhddhZB_BJYHTdYv5SxcrOfJL_5igDVB6zKD' + #'AQiTj_cNa7oP-l6xSRRxwlHDwqgSwcB'}], + #'witnesses': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP']} + # no source seal in load assert evt == {'ked': {'a': [], 'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], 'bt': '1', @@ -4932,8 +4966,6 @@ def test_load_event(mockHelpingNowUTC): 'signatures': [{'index': 0, 'signature': 'AAC1-NTntZ0xkgHwooNcKxe9G4XC-rgkSryVz0B_QrZR2kkv4IKi7DMkfMBd4Eck-' '2NAi0DMuZeXnlvch6ZP0coO'}], - 'source_seal': {'said': 'EF7pHYN6XABC9znRdzprt5frW-MMry9rfrCI-_t5Y8VD', - 'sequence': 1}, 'stored': True, 'timestamp': '2021-01-01T00:00:00.000000+00:00', 'witness_signatures': [{'index': 0, @@ -4947,7 +4979,7 @@ def test_load_event(mockHelpingNowUTC): if __name__ == "__main__": # pytest.main(['-vv', 'test_eventing.py::test_keyeventfuncs']) #test_process_manual() - #test_keyeventsequence_0() + test_keyeventsequence_0() #test_process_transferable() #test_messagize() test_direct_mode() diff --git a/tests/core/test_kevery.py b/tests/core/test_kevery.py index 97b6dd7d..f740cc67 100644 --- a/tests/core/test_kevery.py +++ b/tests/core/test_kevery.py @@ -435,3 +435,4 @@ def test_stale_event_receipts(): if __name__ == "__main__": test_kevery() + test_stale_event_receipts() diff --git a/tests/core/test_replay.py b/tests/core/test_replay.py index 4902f66f..48fcf4db 100644 --- a/tests/core/test_replay.py +++ b/tests/core/test_replay.py @@ -238,6 +238,7 @@ def test_replay(): # get disjoints receipts (vrcs) from Deb of Cam's events by processing Deb's cues debCamVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 assert debCamVrcs == (b'{"v":"KERI10JSON000091_","t":"rct","d":"EBp-SQb9fTgeoQkIkOd2xegv' b'Xy3epjOskiPrf6JDIEuj","i":"EBp-SQb9fTgeoQkIkOd2xegvXy3epjOskiPrf' b'6JDIEuj","s":"0"}-FABELfp9ZhqQCGov3wPRLa6vn5VkIQjug2sb2QD17T-TIp' @@ -265,6 +266,7 @@ def test_replay(): # get disjoints receipts (rcts) from Bev of Deb's events by processing Bevs's cues bevMsgs = bevHab.processCues(bevKevery.cues) + assert len(bevKevery.cues) == 0 assert bevMsgs == (b'{"v":"KERI10JSON0000fd_","t":"icp","d":"EBXqe7Xzsw2aolT09Ouh5Zw9' b'kNn2sgoHmo4zCn7Q7ZSC","i":"BAqph4mAWcf7mkIgk1Xrpvr7dWT7YvHIam_hq' b'UAT2rqw","s":"0","kt":"1","k":["BAqph4mAWcf7mkIgk1Xrpvr7dWT7YvHI' @@ -312,6 +314,7 @@ def test_replay(): # get disjoints receipts (vrcs) from Deb of Bev's events by processing Deb's cues debBevVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 assert debBevVrcs == (b'{"v":"KERI10JSON000091_","t":"rct","d":"EBXqe7Xzsw2aolT09Ouh5Zw9' b'kNn2sgoHmo4zCn7Q7ZSC","i":"BAqph4mAWcf7mkIgk1Xrpvr7dWT7YvHIam_hq' b'UAT2rqw","s":"0"}-FABELfp9ZhqQCGov3wPRLa6vn5VkIQjug2sb2QD17T-TIp' @@ -558,6 +561,7 @@ def test_replay_all(): # get disjoints receipts (vrcs) from Cam of Deb's events by processing Cam's cues camMsgs = camHab.processCues(camKevery.cues) + assert len(camKevery.cues) == 0 # Play camMsgs to Deb # create non-local kevery for Deb to process msgs from Cam @@ -572,6 +576,7 @@ def test_replay_all(): # get disjoints receipts (vrcs) from Deb of Cam's events by processing Deb's cues debCamVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 # Play disjoints debCamVrcs to Cam parsing.Parser().parseOne(ims=bytearray(debCamVrcs), kvy=camKevery) @@ -590,6 +595,7 @@ def test_replay_all(): # get disjoints receipts (rcts) from Bev of Deb's events by processing Bevs's cues bevMsgs = bevHab.processCues(bevKevery.cues) + assert len(bevKevery.cues) == 0 # Play bevMsgs to Deb parsing.Parser().parse(ims=bytearray(bevMsgs), kvy=debKevery) @@ -600,6 +606,7 @@ def test_replay_all(): # get disjoints receipts (vrcs) from Deb of Bev's events by processing Deb's cues debBevVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 # Play disjoints debBevVrcs to Bev parsing.Parser().parseOne(ims=bytearray(debBevVrcs), kvy=bevKevery) diff --git a/tests/core/test_scheming.py b/tests/core/test_scheming.py index a79bac99..09c4df8e 100644 --- a/tests/core/test_scheming.py +++ b/tests/core/test_scheming.py @@ -172,6 +172,7 @@ def test_json_schema_dict(): sce = Schemer(sed=sed, code=MtrDex.Blake3_256) said = 'ENQKl3r1Z6HiLXOD-050aVvKziCWJtXWg3vY2FWUGSxG' assert sce.said == said + payload = b'{"a": {"b": 123, "c": "2018-11-13T20:20:39+00:00"}}' assert sce.verify(payload) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 16a0819e..659403c2 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -24,6 +24,7 @@ from keri.db import basing from keri.db import dbing +from keri.db import subing from keri.db.basing import openDB, Baser, KeyStateRecord from keri.db.dbing import (dgKey, onKey, snKey) from keri.db.dbing import openLMDB @@ -281,11 +282,11 @@ def test_baser(): # replay preB events in database items = [item for item in db.getFelItemPreIter(preB)] - assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] + assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 items = [item for item in db.getFelItemPreIter(preB, fn=3)] - assert items == [(3, digX), (4, digY)] + assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 items = [item for item in db.getFelItemPreIter(preB, fn=5)] @@ -490,100 +491,27 @@ def test_baser(): assert db.putUres(key=cKey, vals=cVals) assert db.putUres(key=dKey, vals=dVals) - # Test getUreItemsNext( key=b"") - # aVals - items = db.getUreItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUreItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUreItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getUreItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getUreItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getUreItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUreItemsNextIter()] + items = [item for item in db.getUreItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals + vals = [bytes(val) for key, val in items] + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUreItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - items = [item for item in db.getUreItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getUreItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getUreItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delUre(ikey, val) == True # bVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -593,7 +521,7 @@ def test_baser(): assert db.delUre(ikey, val) == True # cVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -603,7 +531,7 @@ def test_baser(): assert db.delUre(ikey, val) == True # dVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -612,10 +540,6 @@ def test_baser(): for key, val in items: assert db.delUre(ikey, val) == True - # none - items = [item for item in db.getUreItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Validator (transferable) Receipts # test .vrcs sub db methods dgkey @@ -697,100 +621,26 @@ def test_baser(): assert db.putVres(key=cKey, vals=cVals) assert db.putVres(key=dKey, vals=dVals) - # Test getVreItemsNext( key=b"") - # aVals - items = db.getVreItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getVreItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getVreItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getVreItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getVreItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getVreItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getVreItemsNextIter()] + items = [item for item in db.getVreItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getVreItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getVreItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getVreItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getVreItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delVre(ikey, val) == True - # bVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -800,7 +650,7 @@ def test_baser(): assert db.delVre(ikey, val) == True # cVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -810,7 +660,7 @@ def test_baser(): assert db.delVre(ikey, val) == True # dVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -819,10 +669,6 @@ def test_baser(): for key, val in items: assert db.delVre(ikey, val) == True - # none - items = [item for item in db.getVreItemsNext(key=ikey)] - assert items == [] # empty - assert not items # test .kels insertion order dup methods. dup vals are insertion order @@ -882,100 +728,28 @@ def test_baser(): assert db.putPses(key=cKey, vals=cVals) assert db.putPses(key=dKey, vals=dVals) - # Test getPseItemsNext( key=b"") - # aVals - items = db.getPseItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPseItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPseItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getPseItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getPseItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getPseItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getPseItemsNextIter()] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getPseItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getPseItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getPseItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - items = [item for item in db.getPseItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getPseItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delPse(ikey, val) == True + # bVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -985,7 +759,7 @@ def test_baser(): assert db.delPse(ikey, val) == True # cVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -995,7 +769,7 @@ def test_baser(): assert db.delPse(ikey, val) == True # dVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1004,33 +778,45 @@ def test_baser(): for key, val in items: assert db.delPse(ikey, val) == True - # none - items = [item for item in db.getPseItemsNext(key=ikey)] - assert items == [] # empty - assert not items - # Test .pdes partial delegated escrow seal source couples + # Test .udes partial delegated escrow seal source couples key = dgKey(preb, digb) assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") - # test .pdes sub db methods + # test .pdes SerderIoSetSuber methods + assert isinstance(db.pdes, subing.OnIoDupSuber) + + + # test .udes CatCesrSuber sub db methods + assert isinstance(db.udes, subing.CatCesrSuber) + assert db.udes.klas == (coring.Seqner, coring.Saider) + ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' sdig2 = b'EBYYJRCCpAGO7WjjsLhtHVR37Pawv67kveIFUPvt38x0' val1 = ssnu1 + sdig1 + tuple1 = (coring.Seqner(qb64b=ssnu1), coring.Saider(qb64b=sdig1)) val2 = ssnu2 + sdig2 + tuple2 = (coring.Seqner(qb64b=ssnu2), coring.Saider(qb64b=sdig2)) + + + assert db.udes.get(keys=key) == None + assert db.udes.rem(keys=key) == False + assert db.udes.put(keys=key, val=tuple1) == True + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val1 + assert db.udes.put(keys=key, val=tuple2) == False + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val1 + assert db.udes.pin(keys=key, val=tuple2) == True + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val2 + assert db.udes.rem(keys=key) == True + assert db.udes.get(keys=key) == None + + - assert db.getPde(key) == None - assert db.delPde(key) == False - assert db.putPde(key, val1) == True - assert db.getPde(key) == val1 - assert db.putPde(key, val2) == False - assert db.getPde(key) == val1 - assert db.setPde(key, val2) == True - assert db.getPde(key) == val2 - assert db.delPde(key) == True - assert db.getPde(key) == None # Partially Witnessed Escrow Events # test .pwes insertion order dup methods. dup vals are insertion order @@ -1069,100 +855,26 @@ def test_baser(): assert db.putPwes(key=cKey, vals=cVals) assert db.putPwes(key=dKey, vals=dVals) - # Test getPweItemsNext( key=b"") - # aVals - items = db.getPweItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPweItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPweItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getPweItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getPweItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getPweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getPweItemsNextIter()] + items = [item for item in db.getPweItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getPweItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getPweItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getPweItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getPweItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delPwe(ikey, val) == True - # bVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1172,7 +884,7 @@ def test_baser(): assert db.delPwe(ikey, val) == True # cVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1182,7 +894,7 @@ def test_baser(): assert db.delPwe(ikey, val) == True # dVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1191,10 +903,6 @@ def test_baser(): for key, val in items: assert db.delPwe(ikey, val) == True - # none - items = [item for item in db.getPweItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Unverified Witness Receipt Escrows # test .uwes insertion order dup methods. dup vals are insertion order @@ -1233,100 +941,26 @@ def test_baser(): assert db.putUwes(key=cKey, vals=cVals) assert db.putUwes(key=dKey, vals=dVals) - # Test getUweItemsNext( key=b"") - # aVals - items = db.getUweItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUweItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUweItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getUweItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getUweItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getUweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUweItemsNextIter()] + items = [item for item in db.getUweItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUweItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getUweItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getUweItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getUweItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delUwe(ikey, val) == True - # bVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1336,7 +970,7 @@ def test_baser(): assert db.delUwe(ikey, val) == True # cVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1346,7 +980,7 @@ def test_baser(): assert db.delUwe(ikey, val) == True # dVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1355,10 +989,7 @@ def test_baser(): for key, val in items: assert db.delUwe(ikey, val) == True - # none - items = [item for item in db.getUweItemsNext(key=ikey)] - assert items == [] # empty - assert not items + # test .ooes insertion order dup methods. dup vals are insertion order key = b'A' @@ -1395,100 +1026,25 @@ def test_baser(): assert db.putOoes(key=cKey, vals=cVals) assert db.putOoes(key=dKey, vals=dVals) - # Test getOoeItemsNext( key=b"") - # aVals - items = db.getOoeItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getOoeItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getOoeItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getOoeItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getOoeItemsNext(key=ikey) - assert items == [] # empty - assert not items - # Test getOoeItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getOoeItemsNextIter()] + items = [item for item in db.getOoeItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getOoeItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getOoeItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getOoeItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getOoeItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delOoe(ikey, val) == True - # bVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1498,7 +1054,7 @@ def test_baser(): assert db.delOoe(ikey, val) == True # cVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1508,7 +1064,7 @@ def test_baser(): assert db.delOoe(ikey, val) == True # dVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1517,12 +1073,6 @@ def test_baser(): for key, val in items: assert db.delOoe(ikey, val) == True - # none - items = [item for item in db.getOoeItemsNext(key=ikey)] - assert items == [] # empty - assert not items - - # test .dels insertion order dup methods. dup vals are insertion order key = b'A' vals = [b"z", b"m", b"x", b"a"] @@ -1575,100 +1125,26 @@ def test_baser(): assert db.putLdes(key=cKey, vals=cVals) assert db.putLdes(key=dKey, vals=dVals) - # Test getOoeItemsNext( key=b"") - # aVals - items = db.getLdeItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getLdeItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getLdeItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getLdeItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getLdeItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getLdeItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getLdeItemsNextIter()] + items = [item for item in db.getLdeItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getLdeItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getLdeItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getLdeItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getLdeItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delLde(ikey, val) == True - # bVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1678,7 +1154,7 @@ def test_baser(): assert db.delLde(ikey, val) == True # cVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1688,7 +1164,7 @@ def test_baser(): assert db.delLde(ikey, val) == True # dVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1697,13 +1173,6 @@ def test_baser(): for key, val in items: assert db.delLde(ikey, val) == True - # none - items = [item for item in db.getLdeItemsNext(key=ikey)] - assert items == [] # empty - assert not items - - - assert not os.path.exists(db.path) """ End Test """ @@ -1910,7 +1379,7 @@ def test_fetchkeldel(): assert vals == lastvals - # test getDelIter + # test getDelItemIter preb = 'BTmuupUhPx5_yZ-Wk1x4ejhccWzwEHHzq7K0gzQPYGGw'.encode("utf-8") sn = 1 # do not start at zero key = snKey(preb, sn) @@ -1931,8 +1400,8 @@ def test_fetchkeldel(): for val in vals2: assert db.addDe(key, val) == True - vals = [bytes(val) for val in db.getDelIter(preb)] allvals = vals0 + vals1 + vals2 + vals = [bytes(val) for key, val in db.getDelItemIter(preb)] assert vals == allvals assert not os.path.exists(db.path) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index c88cca4f..9c2348c5 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -17,7 +17,7 @@ from keri.db import dbing from keri.db.dbing import clearDatabaserDir, openLMDB from keri.db.dbing import (dgKey, onKey, fnKey, snKey, dtKey, splitKey, - splitKeyON, splitKeyFN, splitKeySN, splitKeyDT) + splitOnKey, splitKeyFN, splitSnKey, splitKeyDT) from keri.db.dbing import LMDBer @@ -35,11 +35,41 @@ def test_key_funcs(): sn = 3 dts = b'2021-02-13T19:16:50.750302+00:00' + + # test onKey generator of key from top key and trailing ordinal number + assert onKey(pre, 0) == pre + b'.' + b"%032x" % 0 + assert onKey(pre, 1) == pre + b'.' + b"%032x" % 1 + assert onKey(pre, 2) == pre + b'.' + b"%032x" % 2 + assert onKey(pre, 3) == pre + b'.' + b"%032x" % 3 + assert onKey(pre, 4) == pre + b'.' + b"%032x" % 4 + + assert onKey(pre, 0, sep=b'|') == pre + b'|' + b"%032x" % 0 + assert onKey(pre, 4, sep=b'|') == pre + b'|' + b"%032x" % 4 + + assert (onkey := onKey(top=pre, on=0)) == pre + b'.' + b"%032x" % 0 + assert splitKey(key=onkey) == (pre, b"%032x" % 0) + assert splitOnKey(onkey) == (pre, 0) + assert (onkey := onKey(top=pre, on=1)) == pre + b'.' + b"%032x" % 1 + assert splitKey(key=onkey) == (pre, b"%032x" % 1) + assert splitOnKey(onkey) == (pre, 1) + assert (onkey := onKey(top=pre, on=15)) == pre + b'.' + b"%032x" % 15 + assert splitKey(key=onkey) == (pre, b"%032x" % 15) + assert splitOnKey(onkey) == (pre, 15) + + assert (onkey := onKey(top=pre, on=0, sep=b'|')) == pre + b'|' + b"%032x" % 0 + assert splitKey(key=onkey, sep=b'|') == (pre, b"%032x" % 0) + assert splitOnKey(onkey, sep=b'|') == (pre, 0) + assert (onkey := onKey(top=pre, on=15, sep=b'|')) == pre + b'|' + b"%032x" % 15 + assert splitKey(key=onkey, sep=b'|') == (pre, b"%032x" % 15) + assert splitOnKey(onkey, sep=b'|') == (pre, 15) + + + # test snKey assert snKey(pre, sn) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.00000000000000000000000000000003') assert splitKey(snKey(pre, sn)) == (pre, b'%032x' % sn) - assert splitKeySN(snKey(pre, sn)) == (pre, sn) + assert splitSnKey(snKey(pre, sn)) == (pre, sn) assert dgKey(pre, dig) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') @@ -63,7 +93,7 @@ def test_key_funcs(): b'.00000000000000000000000000000003') assert splitKey(snKey(pre, sn).decode("utf-8")) == (pre, '%032x' % sn) - assert splitKeySN(snKey(pre, sn).decode("utf-8")) == (pre, sn) + assert splitSnKey(snKey(pre, sn).decode("utf-8")) == (pre, sn) assert dgKey(pre, dig) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') @@ -83,8 +113,7 @@ def test_key_funcs(): with pytest.raises(ValueError): splitKey(pre) - with pytest.raises(ValueError): - splitKey(dgKey(pre, dgKey(pre, dig))) + assert splitKey(dgKey(pre, dgKey(pre, dig))) # rsplit on gets trailing part # memoryview # Bytes @@ -95,7 +124,7 @@ def test_key_funcs(): key = memoryview(snKey(pre, sn)) assert splitKey(key) == (pre, b'%032x' % sn) - assert splitKeySN(key) == (pre, sn) + assert splitSnKey(key) == (pre, sn) key = memoryview(dgKey(pre, dig)) assert splitKey(key) == (pre, dig) @@ -263,7 +292,7 @@ def test_lmdber(): assert dber.delVal(db, key) == True assert dber.getVal(db, key) == None - # Test getAllItemIter(self, db, key=b'', split=True, sep=b'.') + # Test getTopItemIter key = b"a.1" val = b"wow" assert dber.putVal(db, key, val) == True @@ -273,12 +302,12 @@ def test_lmdber(): key = b"b.1" val = b"woo" assert dber.putVal(db, key, val) == True - assert [(bytes(pre), bytes(num), bytes(val)) for pre, num, val - in dber.getAllItemIter(db=db)] == [(b'a', b'1', b'wow'), - (b'a', b'2', b'wee'), - (b'b', b'1', b'woo')] + assert [(bytes(key), bytes(val)) for key, val + in dber.getTopItemIter(db=db)] == [(b'a.1', b'wow'), + (b'a.2', b'wee'), + (b'b.1', b'woo')] - assert dber.delTopVal(db, key=b"a.") + assert dber.delTopVal(db, top=b"a.") items = [ (key, bytes(val)) for key, val in dber.getTopItemIter(db=db )] assert items == [(b'b.1', b'woo')] @@ -288,6 +317,7 @@ def test_lmdber(): preA = b'BBKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' preB = b'EH7Oq9oxCgYa-nnNLvwhp9sFZpALILlRYyB-6n4WDi7w' preC = b'EIDA1n-WiBA0A8YOqnKrB-wWQYYC49i5zY_qrIZIicQg' + preD = b'EAYC49i5zY_qrIZIicQgIDA1n-WiBA0A8YOqnKrB-wWQ' keyA0 = onKey(preA, 0) @@ -319,10 +349,10 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None - # test appendOrdValPre + # test appendOnValPre # empty database assert dber.getVal(db, keyB0) == None - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -330,7 +360,7 @@ def test_lmdber(): # earlier pre in database only assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -339,7 +369,7 @@ def test_lmdber(): # earlier and later pre in db but not same pre assert dber.getVal(db, keyA0) == digA assert dber.putVal(db, keyC0, val=digC) == True - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -349,13 +379,13 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None assert dber.getVal(db, keyC0) == digC - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU # earlier pre and later pre and earlier entry for same pre assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOrdValPre(db, preB, digV) + on = dber.appendOnVal(db, preB, digV) assert on == 1 assert dber.getVal(db, keyB1) == digV @@ -365,51 +395,126 @@ def test_lmdber(): assert dber.delVal(db, keyC0) == True assert dber.getVal(db, keyC0) == None # another value for preB - on = dber.appendOrdValPre(db, preB, digW) + on = dber.appendOnVal(db, preB, digW) assert on == 2 assert dber.getVal(db, keyB2) == digW # yet another value for preB - on = dber.appendOrdValPre(db, preB, digX) + on = dber.appendOnVal(db, preB, digX) assert on == 3 assert dber.getVal(db, keyB3) == digX # yet another value for preB - on = dber.appendOrdValPre(db, preB, digY ) + on = dber.appendOnVal(db, preB, digY ) assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntValsAllPre(db, preB) == 5 + assert dber.appendOnVal(db, preD, digY ) == 0 + + assert dber.cntOnVals(db, key=preB) == 5 + assert dber.cntOnVals(db, key=b'') == 6 # all keys + assert dber.cntOnVals(db) == 6 # all keys - # replay preB events in database - items = [item for item in dber.getAllOrdItemPreIter(db, preB)] - assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] + # iter replay + # replay preB event items in database + items = [item for item in dber.getOnItemIter(db, preB)] + assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), + (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 - items = [item for item in dber.getAllOrdItemPreIter(db, preB, on=3)] - assert items == [(3, digX), (4, digY)] + items = [item for item in dber.getOnItemIter(db, preB, on=3)] + assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 - items = [item for item in dber.getAllOrdItemPreIter(db, preB, on=5)] + items = [item for item in dber.getOnItemIter(db, preB, on=5)] assert items == [] # replay all events in database with pre events before and after assert dber.putVal(db, keyA0, val=digA) == True assert dber.putVal(db, keyC0, val=digC) == True - items = [item for item in dber.getAllOrdItemAllPreIter(db)] - assert items == [(preA, 0, digA), (preB, 0, digU), (preB, 1, digV), - (preB, 2, digW), (preB, 3, digX), (preB, 4, digY), + items = [item for item in dber.getOnItemIter(db, key=b'')] + assert items == [(preA, 0, digA), + (preD, 0, digY), + (preB, 0, digU), + (preB, 1, digV), + (preB, 2, digW), + (preB, 3, digX), + (preB, 4, digY), (preC, 0, digC)] + items = [item for item in dber.getOnItemIter(db)] + assert items == [(preA, 0, digA), + (preD, 0, digY), + (preB, 0, digU), + (preB, 1, digV), + (preB, 2, digW), + (preB, 3, digX), + (preB, 4, digY), + (preC, 0, digC)] # resume replay all starting at preB on=2 - items = [item for item in dber.getAllOrdItemAllPreIter(db, key=keyB2)] - assert items == [(preB, 2, digW), (preB, 3, digX), (preB, 4, digY), - (preC, 0, digC)] + top, on = splitOnKey(keyB2) + items = [item for item in dber.getOnItemIter(db, key=top, on=on)] + assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] # resume replay all starting at preC on=1 - items = [item for item in dber.getAllOrdItemAllPreIter(db, key=onKey(preC, 1))] + items = [item for item in dber.getOnItemIter(db, key=preC, on=1)] assert items == [] + # val replay + # replay preB event vals in database + vals = [val for val in dber.getOnValIter(db, preB)] + assert vals == [digU, digV, digW, digX, digY] + + # resume replay preB events at on = 3 + vals = [val for val in dber.getOnValIter(db, preB, on=3)] + assert vals == [digX, digY] + + # resume replay preB events at on = 5 + vals = [val for val in dber.getOnValIter(db, preB, on=5)] + assert vals == [] + + vals = [val for val in dber.getOnValIter(db, key=b'')] + assert vals == [digA, + digY, + digU, + digV, + digW, + digX, + digY, + digC] + + vals = [val for val in dber.getOnValIter(db)] + assert vals == [digA, + digY, + digU, + digV, + digW, + digX, + digY, + digC] + + # resume replay all starting at preB on=2 + top, on = splitOnKey(keyB2) + vals = [val for val in dber.getOnValIter(db, key=top, on=on)] + assert vals == [digW, digX, digY] + + # resume replay all starting at preC on=1 + vals = [val for val in dber.getOnValIter(db, key=preC, on=1)] + assert vals == [] + + + # test delOnVal + assert dber.delOnVal(db, key=preB) # default on=0 + assert not dber.delOnVal(db, key=preB, on=0) + assert dber.delOnVal(db, key=preB, on=1) + assert not dber.delOnVal(db, key=preB, on=1) + + items = [item for item in dber.getOnItemIter(db, key=preB)] + assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] + + with pytest.raises(KeyError): + assert dber.delOnVal(db, key=b'') # empty key + # test Vals dup methods. dup vals are lexocographic key = b'A' @@ -445,104 +550,65 @@ def test_lmdber(): vals = [b"z", b"m", b"x", b"a"] db = dber.env.open_db(key=b'peep.', dupsort=True) - assert dber.getIoVals(db, key) == [] - assert dber.getIoValLast(db, key) == None - assert dber.cntIoVals(db, key) == 0 - assert dber.delIoVals(db, key) == False - assert dber.putIoVals(db, key, vals) == True - assert dber.getIoVals(db, key) == vals # preserved insertion order - assert dber.cntIoVals(db, key) == len(vals) == 4 - assert dber.getIoValLast(db, key) == vals[-1] - assert dber.putIoVals(db, key, vals=[b'a']) == False # duplicate - assert dber.getIoVals(db, key) == vals # no change - assert dber.addIoVal(db, key, val=b'b') == True - assert dber.addIoVal(db, key, val=b'a') == False - assert dber.getIoVals(db, key) == [b"z", b"m", b"x", b"a", b"b"] - assert [val for val in dber.getIoValsIter(db, key)] == [b"z", b"m", b"x", b"a", b'b'] - assert dber.delIoVals(db, key) == True - assert dber.getIoVals(db, key) == [] - assert dber.putIoVals(db, key, vals) == True + assert dber.getIoDupVals(db, key) == [] + assert dber.getIoDupValLast(db, key) == None + assert dber.cntIoDupVals(db, key) == 0 + assert dber.delIoDupVals(db, key) == False + assert dber.putIoDupVals(db, key, vals) == True + assert dber.getIoDupVals(db, key) == vals # preserved insertion order + assert dber.cntIoDupVals(db, key) == len(vals) == 4 + assert dber.getIoDupValLast(db, key) == vals[-1] + assert dber.putIoDupVals(db, key, vals=[b'a']) == False # duplicate + assert dber.getIoDupVals(db, key) == vals # no change + assert dber.addIoDupVal(db, key, val=b'b') == True + assert dber.addIoDupVal(db, key, val=b'a') == False + assert dber.getIoDupVals(db, key) == [b"z", b"m", b"x", b"a", b"b"] + assert [val for val in dber.getIoDupValsIter(db, key)] == [b"z", b"m", b"x", b"a", b'b'] + assert dber.delIoDupVals(db, key) == True + assert dber.getIoDupVals(db, key) == [] + assert dber.putIoDupVals(db, key, vals) == True for val in vals: - assert dber.delIoVal(db, key, val) - assert dber.getIoVals(db, key) == [] - assert dber.putIoVals(db, key, vals) == True + assert dber.delIoDupVal(db, key, val) + assert dber.getIoDupVals(db, key) == [] + assert dber.putIoDupVals(db, key, vals) == True for val in sorted(vals): - assert dber.delIoVal(db, key, val) - assert dber.getIoVals(db, key) == [] + assert dber.delIoDupVal(db, key, val) + assert dber.getIoDupVals(db, key) == [] #delete and add in odd order - assert dber.putIoVals(db, key, vals) == True - assert dber.delIoVal(db, key, vals[2]) - assert dber.addIoVal(db, key, b'w') - assert dber.delIoVal(db, key, vals[0]) - assert dber.addIoVal(db, key, b'e') - assert dber.getIoVals(db, key) == [b'm', b'a', b'w', b'e'] - - # Test getIoValsAllPreIter(self, db, pre) - vals0 = [b"gamma", b"beta"] - sn = 0 - key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True + assert dber.putIoDupVals(db, key, vals) == True + assert dber.delIoDupVal(db, key, vals[2]) + assert dber.addIoDupVal(db, key, b'w') + assert dber.delIoDupVal(db, key, vals[0]) + assert dber.addIoDupVal(db, key, b'e') + assert dber.getIoDupVals(db, key) == [b'm', b'a', b'w', b'e'] - vals1 = [b"mary", b"peter", b"john", b"paul"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True - - vals2 = [b"dog", b"cat", b"bird"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoValsAllPreIter(db, pre)] - allvals = vals0 + vals1 + vals2 - assert vals == allvals - - # Test getIoValsLastAllPreIter(self, db, pre) - pre = b'BAejWzwQPYGGwTmuupUhPx5_yZ-Wk1xEHHzq7K0gzhcc' - vals0 = [b"gamma", b"beta"] - sn = 0 - key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True - - vals1 = [b"mary", b"peter", b"john", b"paul"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True - - vals2 = [b"dog", b"cat", b"bird"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True - - vals = [bytes(val) for val in dber.getIoValLastAllPreIter(db, pre)] - lastvals = [vals0[-1], vals1[-1], vals2[-1]] - assert vals == lastvals - - # Test getIoValsAnyPreIter(self, db, pre) + # Test TopIoDupItemIter(self, db, pre) pre = b'BBPYGGwTmuupUhPx5_yZ-Wk1x4ejWzwEHHzq7K0gzhcc' vals0 = [b"gamma", b"beta"] sn = 1 # not start at zero key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True + assert dber.addIoDupVal(db, key, vals0[0]) == True + assert dber.addIoDupVal(db, key, vals0[1]) == True vals1 = [b"mary", b"peter", b"john", b"paul"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True + assert dber.putIoDupVals(db, key, vals1) == True vals2 = [b"dog", b"cat", b"bird"] sn += 2 # gap key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True + assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 + vals = [bytes(val) for key, val in dber.getTopIoDupItemIter(db, pre)] + # dber.getTopIoDupItemIter() assert vals == allvals - # Setup Tests for getIoItemsNext and getIoItemsNextIter + + + # Setup Tests for TopIoDupItemsIter edb = dber.env.open_db(key=b'escrow.', dupsort=True) aKey = snKey(pre=b'A', sn=1) aVals = [b"z", b"m", b"x"] @@ -553,138 +619,370 @@ def test_lmdber(): dKey = snKey(pre=b'A', sn=7) dVals = [b"k", b"b"] - assert dber.putIoVals(edb, key=aKey, vals=aVals) - assert dber.putIoVals(edb, key=bKey, vals=bVals) - assert dber.putIoVals(edb, key=cKey, vals=cVals) - assert dber.putIoVals(edb, key=dKey, vals=dVals) - - # Test getIoItemsNext(self, db, key=b"") - # aVals - items = dber.getIoItemsNext(edb) # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = dber.getIoItemsNext(edb, key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = dber.getIoItemsNext(edb, key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = dber.getIoItemsNext(edb, key=b'', skip=False) # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = dber.getIoItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = dber.getIoItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = dber.getIoItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = dber.getIoItemsNext(edb, key=ikey) - assert items == [] # empty + + # Test variousItemIter + assert dber.putIoDupVals(edb, key=aKey, vals=aVals) + assert dber.putIoDupVals(edb, key=bKey, vals=bVals) + assert dber.putIoDupVals(edb, key=cKey, vals=cVals) + assert dber.putIoDupVals(edb, key=dKey, vals=dVals) + + # test getTopIoDupItemIter + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x'), + (b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z'), + (b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n'), + (b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=aKey)] + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x')] + + # dups at bKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=bKey)] + assert items == [(b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z')] + + # dups at cKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=cKey)] + assert items == [(b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n')] + + # dups at dKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=dKey)] + assert items == [(b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + # dups at missing key + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=b"B.")] assert not items - # Test getIoItemsNextIter(self, db, key=b"") - # get dups at first key in database - # aVals - items = [item for item in dber.getIoItemsNextIter(edb)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in dber.getIoItemsNextIter(edb, key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in dber.getIoItemsNextIter(edb, key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in dber.getIoItemsNextIter(edb, key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True - - # bVals - items = [item for item in dber.getIoItemsNextIter(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True - - # cVals - items = [item for item in dber.getIoItemsNextIter(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True - - # dVals - items = [item for item in dber.getIoItemsNext(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True - - # none - items = [item for item in dber.getIoItemsNext(edb, key=ikey)] - assert items == [] # empty + + # test getIoDupItemIter + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x'), + (b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z'), + (b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n'), + (b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=aKey)] + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x')] + # dups at bKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=bKey)] + assert items == [(b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z')] + + # dups at cKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=cKey)] + assert items == [(b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n')] + + # dups at dKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=dKey)] + assert items == [(b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + # dups at missing key + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=b"B.")] assert not items + + # test OnIoDup methods + ldb = dber.env.open_db(key=b'log.', dupsort=True) + # first pre + sn = 0 + key = snKey(preA, sn) + valsA0 = [b"echo", b"bravo"] + itemsA0 = [ + (preA, sn, valsA0[0]), + (preA, sn, valsA0[1]) + ] + assert dber.addIoDupVal(ldb, key, valsA0[0]) == True + assert dber.addIoDupVal(ldb, key, valsA0[1]) == True + + sn += 1 + key = snKey(preA, sn) + valsA1 = [b"sue", b"bob", b"val", b"zoe"] + itemsA1 = [ + (preA, sn, valsA1[0]), + (preA, sn, valsA1[1]), + (preA, sn, valsA1[2]), + (preA, sn, valsA1[3]), + ] + assert dber.putIoDupVals(ldb, key, valsA1) == True + + sn += 1 + key = snKey(preA, sn) + valsA2 = [b"fish", b"bat", b"snail"] + itemsA2 = [ + (preA, sn, valsA2[0]), + (preA, sn, valsA2[1]), + (preA, sn, valsA2[2]), + ] + assert dber.putIoDupVals(ldb, key, valsA2) == True + + # second pre + sn = 0 + key = snKey(preB, sn) + valsB0 = [b"gamma", b"beta"] + itemsB0 = [ + (preB, sn, valsB0[0]), + (preB, sn, valsB0[1]) + ] + assert dber.addIoDupVal(ldb, key, valsB0[0]) == True + assert dber.addIoDupVal(ldb, key, valsB0[1]) == True + + sn += 1 + key = snKey(preB, sn) + valsB1 = [b"mary", b"peter", b"john", b"paul"] + itemsB1 = [ + (preB, sn, valsB1[0]), + (preB, sn, valsB1[1]), + (preB, sn, valsB1[2]), + (preB, sn, valsB1[3]), + ] + assert dber.putIoDupVals(ldb, key, valsB1) == True + + + sn += 1 + key = snKey(preB, sn) + valsB2 = [b"dog", b"cat", b"bird"] + itemsB2 = [ + (preB, sn, valsB2[0]), + (preB, sn, valsB2[1]), + (preB, sn, valsB2[2]), + ] + assert dber.putIoDupVals(ldb, key, valsB2) == True + + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preA)] + lastitems = [itemsA0[-1], itemsA1[-1], itemsA2[-1]] + assert items == lastitems + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preA, on=1)] + lastitems = [itemsA1[-1], itemsA2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preA)] + lastvals = [valsA0[-1], valsA1[-1], valsA2[-1]] + assert vals == lastvals + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preA, on=1)] + lastvals = [valsA1[-1], valsA2[-1]] + assert vals == lastvals + + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preB)] + lastitems = [itemsB0[-1], itemsB1[-1], itemsB2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preB)] + lastvals = [valsB0[-1], valsB1[-1], valsB2[-1]] + assert vals == lastvals + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb)] + lastitems = [itemsA0[-1], itemsA1[-1], itemsA2[-1], itemsB0[-1], itemsB1[-1], itemsB2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb)] + lastvals = [valsA0[-1], valsA1[-1], valsA2[-1], valsB0[-1], valsB1[-1], valsB2[-1]] + assert vals == lastvals + + # test back iter + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preB, on=3)] + assert items ==[(preB, 2, b'bird'), + (preB, 2, b'cat'), + (preB, 2, b'dog'), + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma')] + + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preB, on=3)] + assert vals ==[ + b'bird', + b'cat', + b'dog', + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preB, on=1)] + assert items ==[ + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma') + ] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preB, on=1)] + assert vals ==[ + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preA, on=5)] + assert items ==[(preA, 2, b'snail'), + (preA, 2, b'bat'), + (preA, 2, b'fish'), + (preA, 1, b'zoe'), + (preA, 1, b'val'), + (preA, 1, b'bob'), + (preA, 1, b'sue'), + (preA, 0, b'bravo'), + (preA, 0, b'echo')] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preA, on=5)] + assert vals ==[ + b'snail', + b'bat', + b'fish', + b'zoe', + b'val', + b'bob', + b'sue', + b'bravo', + b'echo' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preA, on=0)] + assert items ==[ + (preA, 0, b'bravo'), + (preA, 0, b'echo')] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preA, on=0)] + assert vals ==[ + b'bravo', + b'echo' + ] + + # all items from last to first + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb)] + assert items ==[ + (preB, 2, b'bird'), + (preB, 2, b'cat'), + (preB, 2, b'dog'), + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma'), + (preA, 2, b'snail'), + (preA, 2, b'bat'), + (preA, 2, b'fish'), + (preA, 1, b'zoe'), + (preA, 1, b'val'), + (preA, 1, b'bob'), + (preA, 1, b'sue'), + (preA, 0, b'bravo'), + (preA, 0, b'echo'), + ] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb)] + assert vals ==[ + b'bird', + b'cat', + b'dog', + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma', + b'snail', + b'bat', + b'fish', + b'zoe', + b'val', + b'bob', + b'sue', + b'bravo', + b'echo' + ] + + + # test OnIoDup methods + key = b'Z' + assert 0 == dber.appendOnIoDupVal(ldb, key, val=b'k') + assert 1 == dber.appendOnIoDupVal(ldb, key, val=b'l') + assert 2 == dber.appendOnIoDupVal(ldb, key, val=b'm') + assert 3 == dber.appendOnIoDupVal(ldb, key, val=b'n') + + assert dber.cntOnVals(ldb, key) == 4 + + vals = [ bytes(val) for val in dber.getOnIoDupValIter(ldb, key=key)] + assert vals == [b'k', b'l', b'm', b'n'] + + vals = [ bytes(val) for val in dber.getOnIoDupValIter(ldb, key=key, on=2)] + assert vals == [ b'm', b'n'] + + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] + assert items == [(b'Z', 0, b'k'), + (b'Z', 1, b'l'), + (b'Z', 2, b'm'), + (b'Z', 3, b'n')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key, on=2)] + assert items == [ + (b'Z', 2, b'm'), + (b'Z', 3, b'n')] + + # test back iter + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=3)] + assert items ==[(b'Z', 3, b'n'), + (b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=4)] + assert items ==[(b'Z', 3, b'n'), + (b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=2)] + assert items == [(b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + + # test IoSetVals insertion order set of vals methods. key0 = b'ABC.ZYX' key1 = b'DEF.WVU' @@ -766,23 +1064,6 @@ def test_lmdber(): assert dber.getIoSetVals(db, key1) == vals1 assert dber.getIoSetVals(db, key2) == vals2 - assert dber.appendIoSetVal(db, key1, val=b"k") == 4 - assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] - - assert dber.getIoSetItems(db, key0) == [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - (b'ABC.ZYX.00000000000000000000000000000001', b'm'), - (b'ABC.ZYX.00000000000000000000000000000002', b'x'), - (b'ABC.ZYX.00000000000000000000000000000003', b'a')] - - assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemsIter(db, key0)] == - [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - (b'ABC.ZYX.00000000000000000000000000000001', b'm'), - (b'ABC.ZYX.00000000000000000000000000000002', b'x'), - (b'ABC.ZYX.00000000000000000000000000000003', b'a')]) - - for iokey, val in dber.getIoSetItemsIter(db, key0): - assert dber.delIoSetIokey(db, iokey) - assert dber.getIoSetVals(db, key0) == [] vals3 = [b"q", b"e"] assert dber.setIoSetVals(db, key2, vals3) @@ -804,17 +1085,12 @@ def test_lmdber(): dber.putIoSetVals(db, empty_key, [some_value]) dber.addIoSetVal(db, empty_key, some_value) dber.setIoSetVals(db, empty_key, [some_value]) - dber.appendIoSetVal(db, empty_key, some_value) dber.getIoSetVals(db, empty_key) [_ for _ in dber.getIoSetValsIter(db, empty_key)] dber.getIoSetValLast(db, empty_key) dber.cntIoSetVals(db, empty_key) dber.delIoSetVals(db, empty_key) dber.delIoSetVal(db, empty_key, some_value) - dber.getIoSetItems(db, empty_key) - dber.getIoSetItemsIter(db, empty_key) - with pytest.raises(KeyError): - dber.delIoSetIokey(db, empty_key) with pytest.raises(KeyError): dber.putVals(db, empty_key, [some_value]) with pytest.raises(KeyError): @@ -830,21 +1106,21 @@ def test_lmdber(): with pytest.raises(KeyError): dber.delVals(db, empty_key) with pytest.raises(KeyError): - dber.putIoVals(db, empty_key, [some_value]) + dber.putIoDupVals(db, empty_key, [some_value]) with pytest.raises(KeyError): - dber.addIoVal(db, empty_key, some_value) + dber.addIoDupVal(db, empty_key, some_value) with pytest.raises(KeyError): - dber.getIoVals(db, empty_key) + dber.getIoDupVals(db, empty_key) with pytest.raises(KeyError): - [_ for _ in dber.getIoValsIter(db, empty_key)] + [_ for _ in dber.getIoDupValsIter(db, empty_key)] with pytest.raises(KeyError): - dber.getIoValLast(db, empty_key) + dber.getIoDupValLast(db, empty_key) with pytest.raises(KeyError): - dber.cntIoVals(db, empty_key) + dber.cntIoDupVals(db, empty_key) with pytest.raises(KeyError): - dber.delIoVals(db, empty_key) + dber.delIoDupVals(db, empty_key) with pytest.raises(KeyError): - dber.delIoVal(db, empty_key, some_value) + dber.delIoDupVal(db, empty_key, some_value) assert not os.path.exists(dber.path) @@ -853,4 +1129,6 @@ def test_lmdber(): if __name__ == "__main__": test_key_funcs() + test_suffix() test_lmdber() + test_opendatabaser() diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index ae1975e3..ecc58ff9 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -758,7 +758,7 @@ def __iter__(self): loc = locDB.get(keys=(end.eid, scheme)) assert loc == wit3loc - # test IoItem methods + ## test IoItem methods iokeys0 = [f'{cid0}.witness.00000000000000000000000000000000'.encode("utf-8"), f'{cid0}.witness.00000000000000000000000000000001'.encode("utf-8"), f'{cid0}.witness.00000000000000000000000000000002'.encode("utf-8")] @@ -773,20 +773,13 @@ def __iter__(self): 'witness', '00000000000000000000000000000002')] - + # test getItemIter i = 0 - for iokeys, end in endDB.getIoSetItem(keys=keys0): + for keys, end in endDB.getItemIter(keys=keys0): assert end == ends[i] - assert iokeys == iokeys0[i] + assert keys == keys0 i += 1 - i = 0 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0): - assert end == ends[i] - assert iokeys == iokeys0[i] - i += 1 - - # test getAllItemIter ends = ends + [wit3end] i = 0 for keys, end in endDB.getItemIter(): @@ -829,27 +822,29 @@ def __iter__(self): '00000000000000000000000000000000')] i = 0 - for iokeys, end in endDB.getIoItemIter(keys=(cid0, "")): + for iokeys, end in endDB.getFullItemIter(keys=(cid0, "")): assert end == ends[i] assert iokeys == iokeysall[i] i += 1 - i = 0 - for iokeys, end in endDB.getIoItemIter(): - assert end == ends[i] - assert iokeys == iokeysall[i] - i += 1 - assert endDB.remIokey(iokeys) - - assert endDB.cnt(keys0) == 0 - assert endDB.cnt(keys1) == 0 + #for iokeys, val in endDB.getFullItemIter(): + #assert endDB.remIokey(iokeys=iokeys) + #assert endDB.cnt(keys=keys0) == 0 + #assert endDB.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) assert not db.opened if __name__ == "__main__": - test_dup_komer() + test_kom_happy_path() test_kom_get_item_iter() + test_put_invalid_dataclass() + test_get_invalid_dataclass() + test_not_found_entity() + test_serialization() + test_custom_serialization() + test_deserialization() + test_dup_komer() test_ioset_komer() diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 4cee7586..6b97d3c5 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -12,7 +12,7 @@ from keri import help from keri import core -from keri.core import coring, eventing, serdering, indexing +from keri.core import coring, eventing, serdering, indexing, scheming from keri.db import dbing, subing from keri.app import keeping @@ -31,56 +31,85 @@ def test_suber(): assert db.name == "test" assert db.opened - sdb = subing.Suber(db=db, subkey='bags.') - assert isinstance(sdb, subing.Suber) - assert not sdb.sdb.flags()["dupsort"] + suber = subing.Suber(db=db, subkey='bags.') + assert isinstance(suber, subing.Suber) + assert not suber.sdb.flags()["dupsort"] sue = "Hello sailer!" keys = ("test_key", "0001") - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue kip = "Hey gorgeous!" - result = sdb.put(keys=keys, val=kip) + result = suber.put(keys=keys, val=kip) assert not result - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual == sue - result = sdb.pin(keys=keys, val=kip) + result = suber.pin(keys=keys, val=kip) assert result - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual == kip + suber.rem(keys) + actual = suber.get(keys=keys) + assert actual is None + + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) + assert actual == sue + + # test with keys as tuple of bytes + keys = (b"test_key", b"0001") + suber.rem(keys) + actual = suber.get(keys=keys) + assert actual is None + + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) + assert actual == sue + + # test with keys as mixed tuple of bytes + keys = (b"test_key", "0001") + suber.rem(keys) + actual = suber.get(keys=keys) + assert actual is None + + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) + assert actual == sue + + # test with keys as string not tuple keys = "keystr" bob = "Shove off!" - sdb.put(keys=keys, val=bob) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=bob) + actual = suber.get(keys=keys) assert actual == bob - sdb.rem(keys) + suber.rem(keys) - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual is None liz = "May life is insane." keys = ("test_key", "0002") - sdb.put(keys=keys, val=liz) - actual = sdb.get(("not_found", "0002")) + suber.put(keys=keys, val=liz) + actual = suber.get(("not_found", "0002")) assert actual is None w = "Blue dog" @@ -88,26 +117,26 @@ def test_suber(): y = "Red apple" z = "White snow" - sdb = subing.Suber(db=db, subkey='pugs.') - assert isinstance(sdb, subing.Suber) + suber = subing.Suber(db=db, subkey='pugs.') + assert isinstance(suber, subing.Suber) - sdb.put(keys=("a","1"), val=w) - sdb.put(keys=("a","2"), val=x) - sdb.put(keys=("a","3"), val=y) - sdb.put(keys=("a","4"), val=z) + suber.put(keys=("a","1"), val=w) + suber.put(keys=("a","2"), val=x) + suber.put(keys=("a","3"), val=y) + suber.put(keys=("a","4"), val=z) - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), (('a', '4'), z)] - sdb.put(keys=("b","1"), val=w) - sdb.put(keys=("b","2"), val=x) - sdb.put(keys=("bc","3"), val=y) - sdb.put(keys=("ac","4"), val=z) + suber.put(keys=("b","1"), val=w) + suber.put(keys=("b","2"), val=x) + suber.put(keys=("bc","3"), val=y) + suber.put(keys=("ac","4"), val=z) - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -117,20 +146,36 @@ def test_suber(): (('b', '2'), 'Green tree'), (('bc', '3'), 'Red apple')] + + # test with top keys for partial tree topkeys = ("b","") # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=topkeys)] assert items == [(('b', '1'), w), (('b', '2'), x)] topkeys = ("a","") # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=topkeys)] + assert items == [(('a', '1'), w), + (('a', '2'), x), + (('a', '3'), y), + (('a', '4'), z)] + + # test with top parameter + keys = ("b", ) # last element empty to force trailing separator + items = [(keys, val) for keys, val in suber.getItemIter(keys=keys, topive=True)] + assert items == [(('b', '1'), w), + (('b', '2'), x)] + + keys = ("a", ) # last element empty to force trailing separator + items = [(keys, val) for keys, val in suber.getItemIter(keys=keys, topive=True)] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), (('a', '4'), z)] - assert sdb.trim(keys=("b", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + # Test trim + assert suber.trim(keys=("b", "")) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -138,20 +183,271 @@ def test_suber(): (('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim(keys=("a", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim(keys=("a", "")) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim() - items = [(keys, val) for keys, val in sdb.getItemIter()] + # Test trim with top parameters + suber.put(keys=("a","1"), val=w) + suber.put(keys=("a","2"), val=x) + suber.put(keys=("a","3"), val=y) + suber.put(keys=("a","4"), val=z) + suber.put(keys=("b","1"), val=w) + suber.put(keys=("b","2"), val=x) + + assert suber.trim(keys=("b",), topive=True) + items = [(keys, val) for keys, val in suber.getItemIter()] + assert items == [(('a', '1'), 'Blue dog'), + (('a', '2'), 'Green tree'), + (('a', '3'), 'Red apple'), + (('a', '4'), 'White snow'), + (('ac', '4'), 'White snow'), + (('bc', '3'), 'Red apple')] + + assert suber.trim(keys=("a",), topive=True) + items = [(keys, val) for keys, val in suber.getItemIter()] + assert items == [(('ac', '4'), 'White snow'), + (('bc', '3'), 'Red apple')] + + assert suber.trim() + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [] - assert not sdb.trim() + assert not suber.trim() assert not os.path.exists(db.path) assert not db.opened +def test_on_suber(): + """ + Test OnSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.OnSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.OnSuber) + assert not onsuber.sdb.flags()["dupsort"] + + w = "Blue dog" + x = "Green tree" + y = "Red apple" + z = "White snow" + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + # test getOnIter + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == ['Red apple', + 'White snow'] + + + assert 0 == onsuber.appendOn(keys=("b",), val=w) + assert 1 == onsuber.appendOn(keys=("b",), val=x) + assert 0 == onsuber.appendOn(keys=("bc",), val=y) + assert 0 == onsuber.appendOn(keys=("ac",), val=z) + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('ac', '00000000000000000000000000000000'), 'White snow'), + (('b', '00000000000000000000000000000000'), 'Blue dog'), + (('b', '00000000000000000000000000000001'), 'Green tree'), + (('bc', '00000000000000000000000000000000'), 'Red apple')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + items = [item for item in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + assert onsuber.remOn(keys='a', on=1) + assert not onsuber.remOn(keys='a', on=1) + assert onsuber.remOn(keys='a', on=3) + assert not onsuber.remOn(keys='a', on=3) + + assert onsuber.cntOn(keys=("a",)) == 2 + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 2, 'Red apple')] + + + assert not os.path.exists(db.path) + assert not db.opened + + +def test_B64_suber(): + """ + Test B64Suber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + # Test Single klas + buber = subing.B64Suber(db=db, subkey='bags.') # default klas is [Matter] + assert isinstance(buber, subing.B64Suber) + assert not buber.sdb.flags()["dupsort"] + + vals0 = ("alpha", "beta") + vals1 = ("gamma", ) + + keys0 = ("cat", "dog") + assert buber.put(keys=keys0, val=vals0) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + + assert buber.rem(keys0) + assert not buber.get(keys=keys0) + + assert buber.put(keys=keys0, val=vals0) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + + assert not buber.put(keys=keys0, val=vals1) + + assert buber.pin(keys=keys0, val=vals1) + actuals = buber.get(keys=keys0) + assert actuals == vals1 + + assert buber.rem(keys0) + assert not buber.get(keys=keys0) + + # test with vals as non Iterable or put but Iterable on get + assert buber.put(keys=keys0, val="gamma") + actuals = buber.get(keys=keys0) + assert actuals == vals1 + + assert buber.rem(keys0) + + # test val as already joined string + with pytest.raises(ValueError): + assert buber.put(keys=keys0, val="alpha.beta") + + # test bytes for val + assert buber.put(keys=keys0, val=(b"alpha", b"beta")) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + assert buber.rem(keys0) + + # test with keys as string not tuple + keys1 = "{}.{}".format("bird", "fish") + + assert buber.put(keys=keys1, val=vals1) + actuals = buber.get(keys=keys1) + assert actuals == vals1 + + assert buber.rem(keys1) + assert not buber.get(keys=keys1) + + # test missing entry at keys + badkey = "badkey" + assert not buber.get(badkey) + + # test iteritems + assert buber.put(keys0, vals0) + assert buber.put(keys1, vals1) + + items = [ items for items in buber.getItemIter()] + assert items == [(('bird', 'fish'), ('gamma',)), (('cat', 'dog'), ('alpha', 'beta'))] + + buber.put(keys=("b","1"), val=vals0) + buber.put(keys=("b","2"), val=vals1) + buber.put(keys=("c","1"), val=vals0) + buber.put(keys=("c","2"), val=vals1) + + topkeys = ("b","") # last element empty to force trailing separator + items = [items for items in buber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), ('alpha', 'beta')), + (('b', '2'), ('gamma',))] + + assert not os.path.exists(db.path) + assert not db.opened + """Done Test""" + + def test_dup_suber(): """ Test DubSuber LMDBer sub database class @@ -162,54 +458,54 @@ def test_dup_suber(): assert db.name == "test" assert db.opened - sdb = subing.DupSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.DupSuber) - assert sdb.sdb.flags()["dupsort"] + dupber = subing.DupSuber(db=db, subkey='bags.') + assert isinstance(dupber, subing.DupSuber) + assert dupber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - sdb.put(keys=keys0, vals=[sue, sal]) - actual = sdb.get(keys=keys0) + dupber.put(keys=keys0, vals=[sue, sal]) + actual = dupber.get(keys=keys0) assert actual == [sue, sal] # lexicographic order - assert sdb.cnt(keys0) == 2 + assert dupber.cnt(keys0) == 2 - sdb.rem(keys0) - actual = sdb.get(keys=keys0) + dupber.rem(keys0) + actual = dupber.get(keys=keys0) assert not actual assert actual == [] - assert sdb.cnt(keys0) == 0 + assert dupber.cnt(keys0) == 0 - sdb.put(keys=keys0, vals=[sal, sue]) - actual = sdb.get(keys=keys0) + dupber.put(keys=keys0, vals=[sal, sue]) + actual = dupber.get(keys=keys0) assert actual == [sue, sal] # lexicographic order - actual = sdb.getLast(keys=keys0) + actual = dupber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = dupber.add(keys=keys0, val=sam) assert result - actual = sdb.get(keys=keys0) + actual = dupber.get(keys=keys0) assert actual == [sam, sue, sal] # lexicographic order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = dupber.pin(keys=keys0, vals=[zoe, zia]) assert result - actual = sdb.get(keys=keys0) + actual = dupber.get(keys=keys0) assert actual == [zia, zoe] # lexi order - sdb.put(keys=keys1, vals=[sal, sue, sam]) - actual = sdb.get(keys=keys1) + dupber.put(keys=keys1, vals=[sal, sue, sam]) + actual = dupber.get(keys=keys1) assert actual == [sam, sue, sal] # lexicographic order - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(dupber.getIter(keys=keys1)): assert val == actual[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in dupber.getItemIter()] assert items == [(('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0001'), 'See ya later.'), (('test_key', '0002'), 'A real charmer!'), @@ -217,9 +513,9 @@ def test_dup_suber(): (('test_key', '0002'), 'Not my type.')] - assert sdb.put(keys=("test", "blue"), vals=[sal, sue, sam]) + assert dupber.put(keys=("test", "blue"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in dupber.getItemIter(keys=topkeys)] assert items == [(('test', 'blue'), 'A real charmer!'), (('test', 'blue'), 'Hello sailer!'), (('test', 'blue'), 'Not my type.')] @@ -227,32 +523,492 @@ def test_dup_suber(): # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - sdb.put(keys=keys2, vals=[bob]) - actual = sdb.get(keys=keys2) + dupber.put(keys=keys2, vals=[bob]) + actual = dupber.get(keys=keys2) assert actual == [bob] - assert sdb.cnt(keys2) == 1 - sdb.rem(keys2) - actual = sdb.get(keys=keys2) + assert dupber.cnt(keys2) == 1 + dupber.rem(keys2) + actual = dupber.get(keys=keys2) assert actual == [] - assert sdb.cnt(keys2) == 0 + assert dupber.cnt(keys2) == 0 - sdb.put(keys=keys2, vals=[bob]) - actual = sdb.get(keys=keys2) + dupber.put(keys=keys2, vals=[bob]) + actual = dupber.get(keys=keys2) assert actual == [bob] bil = "Go away." - sdb.pin(keys=keys2, vals=[bil]) - actual = sdb.get(keys=keys2) + dupber.pin(keys=keys2, vals=[bil]) + actual = dupber.get(keys=keys2) assert actual == [bil] - sdb.add(keys=keys2, val=bob) - actual = sdb.get(keys=keys2) + dupber.add(keys=keys2, val=bob) + actual = dupber.get(keys=keys2) assert actual == [bil, bob] + + # test cntpre + vals = ["hi", "me", "my"] + for i, val in enumerate(vals): + assert dupber.put(keys=dbing.onKey("bob", i), vals=val) + + vals = ["bye", "guy", "gal"] + for i, val in enumerate(vals): + assert dupber.put(keys=dbing.onKey("bob", i), vals=val) + + items = [(keys, val) for keys, val in dupber.getItemIter(keys="bob")] + assert items == [ + (('bob', '00000000000000000000000000000000'), 'bye'), + (('bob', '00000000000000000000000000000000'), 'hi'), + (('bob', '00000000000000000000000000000001'), 'guy'), + (('bob', '00000000000000000000000000000001'), 'me'), + (('bob', '00000000000000000000000000000002'), 'gal'), + (('bob', '00000000000000000000000000000002'), 'my') + ] + + assert dupber.cnt(keys=dbing.onKey("bob", 1)) == 2 + + vals = [val for val in dupber.getIter(keys=dbing.onKey("bob", 2))] + assert vals == ["gal", "my"] + + + assert not os.path.exists(db.path) + assert not db.opened + + +def test_iodup_suber(): + """ + Test IoDupSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + ioduber = subing.IoDupSuber(db=db, subkey='bags.') + assert isinstance(ioduber, subing.IoDupSuber) + assert ioduber.sdb.flags()["dupsort"] + + sue = "Hello sailer!" + sal = "Not my type." + + keys0 = ("test_key", "0001") + keys1 = ("test_key", "0002") + + sue = "Hello sailer!" + sal = "Not my type." + + keys0 = ("test_key", "0001") + keys1 = ("test_key", "0002") + + assert ioduber.put(keys=keys0, vals=[sal, sue]) + actuals = ioduber.get(keys=keys0) + assert actuals == [sal, sue] # insertion order not lexicographic + assert ioduber.cnt(keys0) == 2 + actual = ioduber.getLast(keys=keys0) + assert actual == sue + + assert ioduber.rem(keys0) + actuals = ioduber.get(keys=keys0) + assert not actuals + assert actuals == [] + assert ioduber.cnt(keys0) == 0 + + assert ioduber.put(keys=keys0, vals=[sue, sal]) + actuals = ioduber.get(keys=keys0) + assert actuals == [sue, sal] # insertion order + actual = ioduber.getLast(keys=keys0) + assert actual == sal + + sam = "A real charmer!" + result = ioduber.add(keys=keys0, val=sam) + assert result + actuals = ioduber.get(keys=keys0) + assert actuals == [sue, sal, sam] # insertion order + + zoe = "See ya later." + zia = "Hey gorgeous!" + + result = ioduber.pin(keys=keys0, vals=[zoe, zia]) + assert result + actuals = ioduber.get(keys=keys0) + assert actuals == [zoe, zia] # insertion order + + assert ioduber.put(keys=keys1, vals=[sal, sue, sam]) + actuals = ioduber.get(keys=keys1) + assert actuals == [sal, sue, sam] + + for i, val in enumerate(ioduber.getIter(keys=keys1)): + assert val == actuals[i] + + items = [(keys, val) for keys, val in ioduber.getItemIter()] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] + + items = list(ioduber.getFullItemIter()) + assert items == [(('test_key', '0001'), '00000000000000000000000000000000.See ya later.'), + (('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!'), + (('test_key', '0002'), '00000000000000000000000000000000.Not my type.'), + (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), + (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] + + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] + + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!')] + + + + + # Test with top keys + assert ioduber.put(keys=("test", "pop"), vals=[sal, sue, sam]) + topkeys = ("test", "") + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=topkeys)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # test with top parameter + keys = ("test", ) + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys, topive=True)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # IoItems + items = list(ioduber.getFullItemIter(keys=topkeys)) + assert items == [(('test', 'pop'), '00000000000000000000000000000000.Not my type.'), + (('test', 'pop'), '00000000000000000000000000000001.Hello sailer!'), + (('test', 'pop'), '00000000000000000000000000000002.A real charmer!')] + + # test remove with a specific val + assert ioduber.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in ioduber.getItemIter()] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!'), + (('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'A real charmer!')] + + assert ioduber.trim(keys=("test", "")) + items = [(keys, val) for keys, val in ioduber.getItemIter()] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'A real charmer!')] + + assert ioduber.cnt(keys=keys0) == 2 + assert ioduber.cnt(keys=keys1) == 2 + + # test with keys as string not tuple + keys2 = "keystr" + bob = "Shove off!" + assert ioduber.put(keys=keys2, vals=[bob]) + actuals = ioduber.get(keys=keys2) + assert actuals == [bob] + assert ioduber.cnt(keys2) == 1 + assert ioduber.rem(keys2) + actuals = ioduber.get(keys=keys2) + assert actuals == [] + assert ioduber.cnt(keys2) == 0 + + assert ioduber.put(keys=keys2, vals=[bob]) + actuals = ioduber.get(keys=keys2) + assert actuals == [bob] + + bil = "Go away." + assert ioduber.pin(keys=keys2, vals=[bil]) + actuals = ioduber.get(keys=keys2) + assert actuals == [bil] + + assert ioduber.add(keys=keys2, val=bob) + actuals = ioduber.get(keys=keys2) + assert actuals == [bil, bob] + + # Test trim + assert ioduber.trim() # default trims whole database + assert ioduber.put(keys=keys1, vals=[bob, bil]) + assert ioduber.get(keys=keys1) == [bob, bil] + + + assert not os.path.exists(db.path) + assert not db.opened + + +def test_on_iodup_suber(): + """ + Test OnIoDupSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.OnIoDupSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.OnIoDupSuber) + assert onsuber.sdb.flags()["dupsort"] + + w = "Blue dog" + x = "Green tree" + y = "Red apple" + z = "White snow" + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + + # test getOnIter + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == ['Red apple', + 'White snow'] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + # test add duplicates + assert onsuber.add(keys=dbing.onKey("b", 0), val=w) + assert onsuber.add(keys=dbing.onKey("b", 1), val=x) + assert onsuber.add(keys=dbing.onKey("bc", 0), val=y) + assert onsuber.add(keys=dbing.onKey("ac", 0), val=z) + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('ac', '00000000000000000000000000000000'), 'White snow'), + (('b', '00000000000000000000000000000000'), 'Blue dog'), + (('b', '00000000000000000000000000000001'), 'Green tree'), + (('bc', '00000000000000000000000000000000'), 'Red apple')] + + + # test getOnItemIter getOnIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + vals = [val for val in onsuber.getOnIter(keys='b')] + assert vals == ['Blue dog', 'Green tree'] + + vals = [val for val in onsuber.getOnIter(keys=('b', ))] + assert vals == ['Blue dog', 'Green tree'] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + vals = [val for val in onsuber.getOnIter(keys=('b', ""))] + assert vals == [] + + items = [item for item in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + vals = [val for val in onsuber.getOnIter(keys='')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + + # test with duplicates + assert onsuber.add(keys=dbing.onKey("a", 0), val=z) + assert onsuber.add(keys=dbing.onKey("a", 1), val=y) + assert onsuber.add(keys=dbing.onKey("a", 2), val=x) + assert onsuber.add(keys=dbing.onKey("a", 3), val=w) + + assert onsuber.cntOn(keys=("a",)) == 8 + assert onsuber.cntOn(keys=("a",), on=2) == 4 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter(keys=("a", ""))] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000000'), 'White snow'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000001'), 'Red apple'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000002'), 'Green tree'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('a', '00000000000000000000000000000003'), 'Blue dog')] + + # test getOnItemIter getOnIter getOnItemBackIter getOnBackIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 0, 'White snow'), + (('a',), 1, 'Green tree'), + (('a',), 1, 'Red apple'), + (('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 3, 'White snow'), + (('a',), 3, 'Blue dog')] + + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'White snow', + 'Green tree', + 'Red apple', + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog', + ] + + + items = [item for item in onsuber.getOnItemBackIter(keys='a', on=4)] + assert items == [(('a',), 3, 'Blue dog'), + (('a',), 3, 'White snow'), + (('a',), 2, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 1, 'Red apple'), + (('a',), 1, 'Green tree'), + (('a',), 0, 'White snow'), + (('a',), 0, 'Blue dog')] + + + vals = [val for val in onsuber.getOnBackIter(keys='a', on=4)] + assert vals == ['Blue dog', + 'White snow', + 'Green tree', + 'Red apple', + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog'] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items ==[(('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 3, 'White snow'), + (('a',), 3, 'Blue dog')] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == [ + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog', + ] + + items = [item for item in onsuber.getOnItemBackIter(keys='a', on=1)] + assert items ==[(('a',), 1, 'Red apple'), + (('a',), 1, 'Green tree'), + (('a',), 0, 'White snow'), + (('a',), 0, 'Blue dog')] + + + vals = [val for val in onsuber.getOnBackIter(keys='a', on=1)] + assert vals == ['Red apple', + 'Green tree', + 'White snow', + 'Blue dog'] + + # test append with duplicates + assert 4 == onsuber.appendOn(keys=("a",), val=x) + assert onsuber.cntOn(keys=("a",)) == 9 + + # test remove + assert onsuber.remOn(keys='a', on=1) + assert not onsuber.remOn(keys='a', on=1) + assert onsuber.remOn(keys='a', on=3) + assert not onsuber.remOn(keys='a', on=3) + + assert onsuber.cntOn(keys=("a",)) == 5 + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 0, 'White snow'), + (('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 4, 'Green tree')] + + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == [ + 'Blue dog', + 'White snow', + 'Red apple', + 'Green tree', + 'Green tree', + ] + + assert not os.path.exists(db.path) assert not db.opened + def test_ioset_suber(): """ Test IoSetSuber LMDBer sub database class @@ -263,9 +1019,9 @@ def test_ioset_suber(): assert db.name == "test" assert db.opened - sdb = subing.IoSetSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.IoSetSuber) - assert not sdb.sdb.flags()["dupsort"] + iosuber = subing.IoSetSuber(db=db, subkey='bags.') + assert isinstance(iosuber, subing.IoSetSuber) + assert not iosuber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." @@ -273,85 +1029,95 @@ def test_ioset_suber(): keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - assert sdb.put(keys=keys0, vals=[sal, sue]) - actuals = sdb.get(keys=keys0) + assert iosuber.put(keys=keys0, vals=[sal, sue]) + actuals = iosuber.get(keys=keys0) assert actuals == [sal, sue] # insertion order not lexicographic - assert sdb.cnt(keys0) == 2 - actual = sdb.getLast(keys=keys0) + assert iosuber.cnt(keys0) == 2 + actual = iosuber.getLast(keys=keys0) assert actual == sue - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert iosuber.rem(keys0) + actuals = iosuber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert iosuber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[sue, sal]) - actuals = sdb.get(keys=keys0) + assert iosuber.put(keys=keys0, vals=[sue, sal]) + actuals = iosuber.get(keys=keys0) assert actuals == [sue, sal] # insertion order - actual = sdb.getLast(keys=keys0) + actual = iosuber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = iosuber.add(keys=keys0, val=sam) assert result - actuals = sdb.get(keys=keys0) + actuals = iosuber.get(keys=keys0) assert actuals == [sue, sal, sam] # insertion order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = iosuber.pin(keys=keys0, vals=[zoe, zia]) assert result - actuals = sdb.get(keys=keys0) + actuals = iosuber.get(keys=keys0) assert actuals == [zoe, zia] # insertion order - assert sdb.put(keys=keys1, vals=[sal, sue, sam]) - actuals = sdb.get(keys=keys1) + assert iosuber.put(keys=keys1, vals=[sal, sue, sam]) + actuals = iosuber.get(keys=keys1) assert actuals == [sal, sue, sam] - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(iosuber.getIter(keys=keys1)): assert val == actuals[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'Hello sailer!'), (('test_key', '0002'), 'A real charmer!')] - - items = list(sdb.getIoItemIter()) + items = list(iosuber.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = sdb.getIoSetItem(keys=keys1) - assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), - (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] - assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), - (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!')] + + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] + - assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) + # Test with top keys + assert iosuber.put(keys=("test", "pop"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=topkeys)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # test with top parameter + keys = ("test", ) + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys, topive=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] - items = list(sdb.getIoItemIter(keys=topkeys)) + # IoItems + items = list(iosuber.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] # test remove with a specific val - assert sdb.rem(keys=("test_key", "0002"), val=sue) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert iosuber.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!'), @@ -360,45 +1126,45 @@ def test_ioset_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - assert sdb.trim(keys=("test", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert iosuber.trim(keys=("test", "")) + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getIoItemIter(): - assert sdb.remIokey(iokeys=iokeys) - - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert iosuber.put(keys=keys2, vals=[bob]) + actuals = iosuber.get(keys=keys2) assert actuals == [bob] - assert sdb.cnt(keys2) == 1 - assert sdb.rem(keys2) - actuals = sdb.get(keys=keys2) + assert iosuber.cnt(keys2) == 1 + assert iosuber.rem(keys2) + actuals = iosuber.get(keys=keys2) assert actuals == [] - assert sdb.cnt(keys2) == 0 + assert iosuber.cnt(keys2) == 0 - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert iosuber.put(keys=keys2, vals=[bob]) + actuals = iosuber.get(keys=keys2) assert actuals == [bob] bil = "Go away." - assert sdb.pin(keys=keys2, vals=[bil]) - actuals = sdb.get(keys=keys2) + assert iosuber.pin(keys=keys2, vals=[bil]) + actuals = iosuber.get(keys=keys2) assert actuals == [bil] - assert sdb.add(keys=keys2, val=bob) - actuals = sdb.get(keys=keys2) + assert iosuber.add(keys=keys2, val=bob) + actuals = iosuber.get(keys=keys2) assert actuals == [bil, bob] + # Test trim and append + assert iosuber.trim() # default trims whole database + assert iosuber.put(keys=keys1, vals=[bob, bil]) + assert iosuber.get(keys=keys1) == [bob, bil] + assert not os.path.exists(db.path) assert not db.opened @@ -414,10 +1180,10 @@ def test_cesr_ioset_suber(): assert db.name == "test" assert db.opened - sdb = subing.CesrIoSetSuber(db=db, subkey='bags.', klas=coring.Saider) - assert isinstance(sdb, subing.CesrIoSetSuber) - assert issubclass(sdb.klas, coring.Saider) - assert not sdb.sdb.flags()["dupsort"] + cisuber = subing.CesrIoSetSuber(db=db, subkey='bags.', klas=coring.Saider) + assert isinstance(cisuber, subing.CesrIoSetSuber) + assert issubclass(cisuber.klas, coring.Saider) + assert not cisuber.sdb.flags()["dupsort"] seqner0 = coring.Seqner(sn=20) seq0 = seqner0.qb64 @@ -491,63 +1257,63 @@ def test_cesr_ioset_suber(): said2 = saider2.qb64 assert said2 == 'EJxOaEsBSObrcmrsnlfHOdVAowGhUBKoE2Ce3TZ4Mhgu' - assert sdb.put(keys=keys0, vals=[saider1, saider0]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.put(keys=keys0, vals=[saider1, saider0]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said1, said0] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said0 - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert cisuber.rem(keys0) + actuals = cisuber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert cisuber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[saider0, saider1]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.put(keys=keys0, vals=[saider0, saider1]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said0, said1] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said1 - assert sdb.add(keys=keys0, val=saider2) - assert sdb.cnt(keys0) == 3 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.add(keys=keys0, val=saider2) + assert cisuber.cnt(keys0) == 3 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 3 sers = [actual.qb64 for actual in actuals] assert sers == [said0, said1, said2] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said2 - assert sdb.pin(keys=keys0, vals=[saider1, saider2]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.pin(keys=keys0, vals=[saider1, saider2]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said1, said2] - assert sdb.put(keys=keys1, vals=[saider2, saider1, saider0]) - assert sdb.cnt(keys1) == 3 - actuals = sdb.get(keys=keys1) # insertion order not lexicographic + assert cisuber.put(keys=keys1, vals=[saider2, saider1, saider0]) + assert cisuber.cnt(keys1) == 3 + actuals = cisuber.get(keys=keys1) # insertion order not lexicographic assert len(actuals) == 3 sers = [actual.qb64 for actual in actuals] assert sers == [said2, said1, said0] - assert sdb.rem(keys=keys1, val=saider1) - assert sdb.cnt(keys1) == 2 - actuals = sdb.get(keys=keys1) # insertion order not lexicographic + assert cisuber.rem(keys=keys1, val=saider1) + assert cisuber.cnt(keys1) == 2 + actuals = cisuber.get(keys=keys1) # insertion order not lexicographic sers = [actual.qb64 for actual in actuals] assert sers == [said2, said0] - sers = [val.qb64 for val in sdb.getIter(keys=keys1)] + sers = [val.qb64 for val in cisuber.getIter(keys=keys1)] assert sers == [said2, said0] - items = [(keys, val.qb64) for keys, val in sdb.getItemIter()] + items = [(keys, val.qb64) for keys, val in cisuber.getItemIter()] assert items == [ (keys1, said2), (keys1, said0), @@ -556,7 +1322,7 @@ def test_cesr_ioset_suber(): ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoItemIter()] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getFullItemIter()] assert items == [ ((*keys1, '00000000000000000000000000000000'), said2), ((*keys1, '00000000000000000000000000000002'), said0), @@ -564,38 +1330,30 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItem(keys=keys1)] - assert items == [ - ((*keys1, '00000000000000000000000000000000'), said2), - ((*keys1, '00000000000000000000000000000002'), said0), - ] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getItemIter(keys=keys0)] + assert items == [(keys0, said1), (keys0, said2)] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getItemIter(keys=keys1)] assert items == [ - ((*keys0, '00000000000000000000000000000000'), said1), - ((*keys0, '00000000000000000000000000000001'), said2), + (keys1, said2), + (keys1, said0), ] topkeys = (seq1, "") - items = [(keys, val.qb64) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val.qb64) for keys, val in cisuber.getItemIter(keys=topkeys)] assert items == [ (keys1, said2), (keys1, said0), ] topkeys = (seq0, "") - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoItemIter(keys=topkeys)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getFullItemIter(keys=topkeys)] assert items == [ ((*keys0, '00000000000000000000000000000000'), said1), ((*keys0, '00000000000000000000000000000001'), said2), ] - for iokeys, val in sdb.getIoItemIter(): - assert sdb.remIokey(iokeys=iokeys) - - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) @@ -612,73 +1370,188 @@ def test_serder_suber(): assert db.name == "test" assert db.opened - sdb = subing.SerderSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.SerderSuber) - assert not sdb.sdb.flags()["dupsort"] + serber = subing.SerderSuber(db=db, subkey='bags.') + assert isinstance(serber, subing.SerderSuber) + assert not serber.sdb.flags()["dupsort"] + assert serber.klas == serdering.SerderKERI pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" srdr0 = eventing.incept(keys=[pre]) keys = (pre, srdr0.said) - sdb.put(keys=keys, val=srdr0) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr0) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - sdb.rem(keys) - actual = sdb.get(keys=keys) + assert serber.rem(keys) + actual = serber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=srdr0) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr0) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) - result = sdb.put(keys=keys, val=srdr1) + result = serber.put(keys=keys, val=srdr1) assert not result + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - result = sdb.pin(keys=keys, val=srdr1) + result = serber.pin(keys=keys, val=srdr1) assert result - actual = sdb.get(keys=keys) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said # test with keys as string not tuple keys = "{}.{}".format(pre, srdr1.said) - sdb.put(keys=keys, val=srdr1) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr1) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said - sdb.rem(keys) - actual = sdb.get(keys=keys) + assert serber.rem(keys) + actual = serber.get(keys=keys) assert actual is None # test missing entry at keys badkey = "badkey" - actual = sdb.get(badkey) + actual = serber.get(badkey) assert actual is None # test iteritems - sdb = subing.SerderSuber(db=db, subkey='pugs.') - assert isinstance(sdb, subing.SerderSuber) - sdb.put(keys=("a","1"), val=srdr0) - sdb.put(keys=("a","2"), val=srdr1) + serber = subing.SerderSuber(db=db, subkey='pugs.') + assert isinstance(serber, subing.SerderSuber) + serber.put(keys=("a","1"), val=srdr0) + serber.put(keys=("a","2"), val=srdr1) - items = [(keys, srdr.said) for keys, srdr in sdb.getItemIter()] + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter()] assert items == [(('a', '1'), srdr0.said), (('a', '2'), srdr1.said)] - assert sdb.put(keys=("b","1"), val=srdr0) - assert sdb.put(keys=("b","2"), val=srdr1) - assert sdb.put(keys=("bc","1"), val=srdr0) + assert serber.put(keys=("b","1"), val=srdr0) + assert serber.put(keys=("b","2"), val=srdr1) + assert serber.put(keys=("bc","1"), val=srdr0) + + topkeys = ("b", "") # append empty str to force trailing .sep + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), srdr0.said), + (('b', '2'), srdr1.said)] + + assert not os.path.exists(db.path) + assert not db.opened + +def test_serder_ioset_suber(): + """ + Test SerderIoSetSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + serber = subing.SerderIoSetSuber(db=db, subkey='bags.') + assert isinstance(serber, subing.SerderIoSetSuber) + assert not serber.sdb.flags()["dupsort"] + assert serber.klas == serdering.SerderKERI + + pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + srdr0 = eventing.incept(keys=[pre]) + + keys = (pre, srdr0.said) + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr0.said + + assert serber.rem(keys) + actuals = serber.get(keys=keys) + assert not actuals # empty list + + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr0.said + + srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) + + result = serber.put(keys=keys, vals=(srdr1, )) + assert result + actuals = serber.get(keys=keys) + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr0.said + assert isinstance(actuals[1], serdering.SerderKERI) + assert actuals[1].said == srdr1.said + + + result = serber.pin(keys=keys, vals=(srdr1, )) + assert result + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr1.said + + + # test rem a specific value + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr1.said + assert isinstance(actuals[1], serdering.SerderKERI) + assert actuals[1].said == srdr0.said + + assert serber.rem(keys=keys, val=srdr1) + actuals = serber.get(keys=keys) + assert len(actuals) == 1 + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr0.said + + + # test with keys as string not tuple + keys = "{}.{}".format(pre, srdr1.said) + + serber.put(keys=keys, vals=(srdr1, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr1.said + + assert serber.rem(keys) + actuals = serber.get(keys=keys) + assert not actuals + + # test missing entry at keys + badkey = "badkey" + actuals = serber.get(badkey) + assert not actuals + + # test iteritems + serber = subing.SerderIoSetSuber(db=db, subkey='pugs.') + assert isinstance(serber, subing.SerderIoSetSuber) + serber.put(keys=("a","1"), vals=(srdr0, )) + serber.put(keys=("a","1"), vals=(srdr1, )) + serber.put(keys=("a","2"), vals=(srdr1, )) + + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter()] + assert items == [ + (('a', '1'), srdr0.said), + (('a', '1'), srdr1.said), + (('a', '2'), srdr1.said), + ] + + assert serber.put(keys=("b","1"), vals=(srdr0, )) + assert serber.put(keys=("b","2"), vals=(srdr1, )) + assert serber.put(keys=("bc","1"), vals=(srdr0, )) topkeys = ("b", "") # append empty str to force trailing .sep - items = [(keys, srdr.said) for keys, srdr in sdb.getItemIter(keys=topkeys)] + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter(keys=topkeys)] assert items == [(('b', '1'), srdr0.said), (('b', '2'), srdr1.said)] @@ -686,6 +1559,105 @@ def test_serder_suber(): assert not db.opened +def test_schemer_suber(): + """ + Test SchemerSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + scmber = subing.SchemerSuber(db=db, subkey='bags.') + assert isinstance(scmber, subing.SchemerSuber) + assert not scmber.sdb.flags()["dupsort"] + assert scmber.klas == scheming.Schemer + + pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + + raw0 = (b'{"$id":"EMRvS7lGxc1eDleXBkvSHkFs8vUrslRcla6UXOJdcczw","$schema":"http://json' + b'-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"str' + b'ing"},"b":{"type":"number"},"c":{"type":"string","format":"date-time"}}}') + + scmr0 = scheming.Schemer(raw=raw0) + + keys = (pre, scmr0.said) + scmber.put(keys=keys, val=scmr0) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + assert scmber.rem(keys) + actual = scmber.get(keys=keys) + assert actual is None + + scmber.put(keys=keys, val=scmr0) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + raw1 = (b'{"$id":"ENQKl3r1Z6HiLXOD-050aVvKziCWJtXWg3vY2FWUGSxG","$schema":"http://json' + b'-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"obj' + b'ect","properties":{"b":{"type":"number"},"c":{"type":"string","format":"date' + b'-time"}}}}}') + scmr1 = scheming.Schemer(raw=raw1) + + result = scmber.put(keys=keys, val=scmr1) + assert not result + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + result = scmber.pin(keys=keys, val=scmr1) + assert result + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr1.said + + # test with keys as string not tuple + keys = "{}.{}".format(pre, scmr1.said) + + scmber.put(keys=keys, val=scmr1) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr1.said + + assert scmber.rem(keys) + actual = scmber.get(keys=keys) + assert actual is None + + # test missing entry at keys + badkey = "badkey" + actual = scmber.get(badkey) + assert actual is None + + # test iteritems + scmber = subing.SchemerSuber(db=db, subkey='pugs.') + assert isinstance(scmber, subing.SchemerSuber) + scmber.put(keys=("a","1"), val=scmr0) + scmber.put(keys=("a","2"), val=scmr1) + + items = [(keys, srdr.said) for keys, srdr in scmber.getItemIter()] + assert items == [(('a', '1'), scmr0.said), + (('a', '2'), scmr1.said)] + + assert scmber.put(keys=("b","1"), val=scmr0) + assert scmber.put(keys=("b","2"), val=scmr1) + assert scmber.put(keys=("bc","1"), val=scmr0) + + topkeys = ("b", "") # append empty str to force trailing .sep + items = [(keys, srdr.said) for keys, srdr in scmber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), scmr0.said), + (('b', '2'), scmr1.said)] + + with pytest.raises(TypeError): + scmber = subing.SchemerSuber(db=db, subkey='bags.', klas=coring.Matter) + + + assert not os.path.exists(db.path) + assert not db.opened + def test_cesr_suber(): """ @@ -829,10 +1801,117 @@ def test_cesr_suber(): assert not db.opened """Done Test""" +def test_cesr_on_suber(): + """ + Test CesrOnSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.CesrOnSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.CesrOnSuber) + assert not onsuber.sdb.flags()["dupsort"] + + prew = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + w = coring.Matter(qb64=prew) + prex = "BHHzqZWzwE-Wk7K0gzQPYGGwTmuupUhPx5_y1x4ejhcc" + x = coring.Matter(qb64=prex) + prey = "EHHzqZWzwE-Wk7K0gzQPYGGwTmuupUhPx5_y1x4ejhcc" + y = coring.Matter(qb64=prey) + prez = "EAPYGGwTmuupWzwEHHzq7K0gzUhPx5_yZ-Wk1x4ejhcc" + z = coring.Matter(qb64=prez) + + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val.qb64) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), w.qb64), + (('a', '00000000000000000000000000000001'), x.qb64), + (('a', '00000000000000000000000000000002'), y.qb64), + (('a', '00000000000000000000000000000003'), z.qb64)] + + # test getOnItemIter + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, y.qb64), + (('a',), 3, z.qb64)] + + assert 0 == onsuber.appendOn(keys=("ac",), val=z) + assert 0 == onsuber.appendOn(keys=("b",), val=w) + assert 1 == onsuber.appendOn(keys=("b",), val=x) + assert 0 == onsuber.appendOn(keys=("bc",), val=y) + + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val.qb64) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), w.qb64), + (('a', '00000000000000000000000000000001'), x.qb64), + (('a', '00000000000000000000000000000002'), y.qb64), + (('a', '00000000000000000000000000000003'), z.qb64), + (('ac', '00000000000000000000000000000000'), z.qb64), + (('b', '00000000000000000000000000000000'), w.qb64), + (('b', '00000000000000000000000000000001'), x.qb64), + (('bc', '00000000000000000000000000000000'), y.qb64)] + + # test getOnItemIter + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, w.qb64), + (('b',), 1, x.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, w.qb64), + (('b',), 1, x.qb64)] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64), + (('ac',), 0, z.qb64), + (('b',), 0, w.qb64), + (('b',), 1, x.qb64), + (('bc',), 0, y.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter()] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64), + (('ac',), 0, z.qb64), + (('b',), 0, w.qb64), + (('b',), 1, x.qb64), + (('bc',), 0, y.qb64)] + + assert not os.path.exists(db.path) + assert not db.opened -def test_cat_suber(): + + +def test_cat_cesr_suber(): """ - Test CatSuber LMDBer sub database class + Test CatCesrSuber LMDBer sub database class """ with dbing.openLMDB() as db: @@ -967,7 +2046,7 @@ def test_cat_suber(): """Done Test""" -def test_cat__cesr_ioset_suber(): +def test_cat_cesr_ioset_suber(): """ Test CatIoSetSuber LMDBer sub database class """ @@ -1113,7 +2192,7 @@ def test_cat__cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoItemIter()] + for iokeys, vals in sdb.getFullItemIter()] assert items == [ (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), @@ -1122,15 +2201,15 @@ def test_cat__cesr_ioset_suber(): (keys2 + ('00000000000000000000000000000001', ), [sqr2.qb64, dgr2.qb64]) ] - items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItem(keys=keys1)] - assert items == [(keys1 + ('00000000000000000000000000000000', ), [sqr2.qb64, dgr2.qb64])] + items = [(keys, [val.qb64 for val in vals]) + for keys, vals in sdb.getItemIter(keys=keys1)] + assert items == [(keys1, [sqr2.qb64, dgr2.qb64])] - items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItemIter(keys=keys0)] + items = [(keys, [val.qb64 for val in vals]) + for keys, vals in sdb.getItemIter(keys=keys0)] assert items == [ - (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), - (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), + (keys0, [sqr0.qb64, dgr0.qb64]), + (keys0, [sqr1.qb64, dgr1.qb64]), ] @@ -1143,19 +2222,13 @@ def test_cat__cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoItemIter(keys=topkeys)] + for iokeys, vals in sdb.getFullItemIter(keys=topkeys)] assert items == [ (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), ] - for iokeys, val in sdb.getIoItemIter(): - assert sdb.remIokey(iokeys=iokeys) - - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 - assert sdb.cnt(keys=keys2) == 0 # Try Siger Indexer Subclass sdb = subing.CatCesrIoSetSuber(db=db, subkey='pigs.', klas=(indexing.Siger, )) @@ -1637,11 +2710,21 @@ def test_crypt_signer_suber(): if __name__ == "__main__": - test_cesr_ioset_suber() - test_serder_suber() + test_suber() + test_on_suber() + test_B64_suber() + test_dup_suber() + test_iodup_suber() + test_on_iodup_suber() + test_ioset_suber() + test_cat_cesr_suber() test_cesr_suber() - test_cat_suber() - test_cat__cesr_ioset_suber() + test_cesr_on_suber() + test_cesr_ioset_suber() + test_cat_cesr_ioset_suber() test_cesr_dup_suber() + test_serder_suber() + test_serder_ioset_suber() + test_schemer_suber() test_signer_suber() test_crypt_signer_suber() diff --git a/tests/vdr/test_viring.py b/tests/vdr/test_viring.py index 8ec90e47..2e98dda6 100644 --- a/tests/vdr/test_viring.py +++ b/tests/vdr/test_viring.py @@ -231,7 +231,7 @@ def test_issuer(): assert issuer.putTel(snKey(vcdig, sn + 2), val=idig.qb64b) is True assert issuer.putTel(snKey(vcdig, sn + 3), val=rdig.qb64b) is True - result = [(sn, dig) for sn, dig in issuer.getTelItemPreIter(vcdig)] + result = [(sn, dig) for _, sn, dig in issuer.getTelItemPreIter(vcdig)] assert result == [(0, idig.qb64b), (1, rdig.qb64b), (2, idig.qb64b), (3, rdig.qb64b)] bak1 = b'BA1Q98kT0HRn9R62lY-LufjjKdbCeL1mqu9arTgOmbqI'