From 540945bb27d3aceb463981ebc04ce0a364a7fb8f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 14 Aug 2024 11:57:13 -0600 Subject: [PATCH 01/78] regress without delegation kel walk in event processing since already handled in escrow when source seal not provided --- src/keri/core/eventing.py | 75 ++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 5443e7d8..f098691b 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1561,16 +1561,16 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + local=local, + delseqner=delseqner, + delsaider=delsaider) self.delpre = delpre # may be None self.delegated = True if self.delpre else False @@ -1920,16 +1920,16 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # returned sigers and wigers are verified signatures # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=tholder, - wigers=wigers, - toader=toader, - wits=wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=tholder, + wigers=wigers, + toader=toader, + wits=wits, + local=local, + delseqner=delseqner, + delsaider=delsaider) @@ -1985,18 +1985,18 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=self.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local) + sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers=sigers, + verfers=self.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + local=local) # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, first=True if not check else False) # First seen accepted # validates so update state @@ -2314,19 +2314,19 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"for event={serder.ked}.") if delpre: - if not (delseqner and delsaider): - seal = dict(i=serder.ked["i"], s=serder.snh, d=serder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) + #if not (delseqner and delsaider): + #seal = dict(i=serder.ked["i"], s=serder.snh, d=serder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) self.validateDelegation(serder, sigers=sigers, wigers=wigers, wits=wits, local=local, delpre=delpre, - delseqner=delseqner, delsaider=delsaider) + ) # delseqner=delseqner, delsaider=delsaider - return sigers, wigers, delpre, delseqner, delsaider + return (sigers, wigers, delpre) # delseqner, delsaider def exposeds(self, sigers): @@ -2834,6 +2834,7 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, self.db.putEvt(dgkey, serder.raw) # idempotent (maybe already excrowed) # update event source + if seqner and saider: # delegation for authorized delegated or issued event couple = seqner.qb64b + saider.qb64b self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal From 4b6e02d3aa2c9482c80ca1773ba6d1a468d57f18 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 15 Aug 2024 18:13:16 -0600 Subject: [PATCH 02/78] fixed escrow call in Kever.validateDelegation --- src/keri/core/eventing.py | 50 +++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index f098691b..f6571ab3 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1579,7 +1579,8 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen # returns fn == None if already logged fn log is non idempotent - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, + wits=wits, first=True if not check else False, seqner=delseqner, saider=delsaider, firner=firner, dater=dater, local=local) @@ -1935,8 +1936,10 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # .valSigWigsDel above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, - first=True if not check else False, seqner=delseqner, saider=delsaider, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, + wits=wits, + first=True if not check else False, + seqner=delseqner, saider=delsaider, firner=firner, dater=dater, local=local) # nxt and signatures verify so update state @@ -2600,15 +2603,21 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, if self.locallyOwned() or self.locallyWitnessed(wits=wits): return - if self.kevers is None or delpre not in self.kevers: # drop event + if self.kevers is None or delpre not in self.kevers: # missing delegator # ToDo XXXX cue a trigger to get the KEL of the delegator - # the processDelegableEvent should also cue a trigger to get KEL + # the processPSEvent should also cue a trigger to get KEL # of delegator if still missing when processing escrow later. - self.escrowDelegableEvent(serder=serder, sigers=sigers, - wigers=wigers,local=local) - raise MissingDelegableApprovalError(f"Missing Kever for delegator" - f" = {delpre} of event" - f" = {serder.ked}.") + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + local=local) + raise MissingDelegationError(f"Missing KEL of delegator " + f"{delpre} of evt = {serder.ked}.") + + + #self.escrowDelegableEvent(serder=serder, sigers=sigers, + #wigers=wigers,local=local) + #raise MissingDelegableApprovalError(f"Missing Kever for delegator" + #f" = {delpre} of event" + #f" = {serder.ked}.") dkever = self.kevers[delpre] if dkever.doNotDelegate: # drop event if delegation not allowed @@ -2621,7 +2630,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # Once fully receipted, cue in Kevery will then trigger cue to approve # delegation - if delseqner is None or delsaider is None: + if delseqner is None or delsaider is None: # missing delegation seal ref if self.locallyOwned(delpre): # local delegator so escrow. # Won't get to here if not local and locallyOwned(delpre) because # valSigsWigsDel will send nonlocal sourced delegable event to @@ -2834,11 +2843,22 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, self.db.putEvt(dgkey, serder.raw) # idempotent (maybe already excrowed) # update event source - - if seqner and saider: # delegation for authorized delegated or issued event + # delegation for authorized delegated or issued event + # when seqner and saider are provided they are only assured to be valid + # kever for event if kel is delegated and not locallyOwned + # and not locallyWitnessed as the validateDelegation is short circuited + # for non delegated kels, local controllers, and local witnesses. + # These checks prevent ddos via malicious source seal attachments. + # MUST NOT setAes if not delegated or locallyOwned or locallyWitnessed + if (self.delpre and not self.locallyOwned() + and not self.locallyWitnessed(wits=wits) and seqner and saider): couple = seqner.qb64b + saider.qb64b self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + #if seqner and saider: + #couple = seqner.qb64b + saider.qb64b + #self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + if esr := self.db.esrs.get(keys=dgkeys): # preexisting esr if local and not esr.local: # local overwrites prexisting remote esr.local = local @@ -5290,6 +5310,10 @@ def processEscrowPartialSigs(self): else: delpre = eserder.ked["di"] + # consider using here instead todo XXXX + # Kever.fetchDelegatingEvent(delegator=delpre, + # serder=eserder) + # need Kever reference for eserder.pre seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) if srdr is not None: From c983c6f4e5e9572be4a524e16871128032ad023d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 18 Aug 2024 15:28:27 -0600 Subject: [PATCH 03/78] refactoring prep for fixing delegation --- src/keri/core/eventing.py | 235 ++++++++++++++++++++++++++++---------- src/keri/db/basing.py | 66 ++++++----- 2 files changed, 213 insertions(+), 88 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index f6571ab3..857cbdc4 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2397,6 +2397,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, Location Seal is from Delegate's establishment event Assumes state setup + self is last accepted event if any or yet to be accepted event, + serder is received event + Parameters: serder (SerderKERI): instance of delegated event serder sigers (list[Siger]): of Siger instances of indexed controller sigs of @@ -2416,7 +2419,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, If this event is not delegated ignored Returns: - (str | None): qb64 delegator prefix or None if not delegated + None Process Logic: A delegative event is processed differently for each of four different @@ -2525,17 +2528,20 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, events signed by compromised keys. The result of superseded recovery is that the KEL is forked at the sn of the superseding event. All events in the superseded branch of the fork still exist but, by virtue of being superseded, - are disputed. The set of superseding events in the superseding fork forms the authoritative - branch of the KEL. All the already seen superseded events in the superseded fork + are disputed. The set of superseding events in the superseding fork forms + the authoritative branch of the KEL. + + All the already seen superseded events in the superseded fork still remain in the KEL and may be viewed in order of their original acceptance because the database stores all accepted events in order of acceptance and denotes this order using the first seen ordinal number, fn. - The fn is not the same as the sn (sequence number). + + Recall that the fn is not the same as the sn (sequence number). Each event accepted into a KEL has a unique fn but multiple events due to recovery forks may share the same sn. - Superseding Rules for Recovery at given SN (sequence number) + Superseding Rules for Recovery at given 'sn' (sequence number) A0. Any rotation event may supersede an interaction event at the same sn. where that interaction event is not before any other rotation event. @@ -2589,6 +2595,19 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, recovery is no longer possible because the delegatee would no longer control the privete keys needed to verifiably sign a recovery rotation. + + Repair of approval soruce seal couple in 'aes' database on recursive + climb the kel tree. Once an event has been accepted into its kel. + Later adding a source seal couple to 'aes' should then be OK from a + security perspective since its only making discovery less expensive. + + When malicious source seal couple is received but event is validly + delegated and the delegation source seal is repaired then need to replace + malicious source seal couple with repaired seal so repaired seal not + malicous seal gets written to 'aes' db. When the event is valid but + non-delegated then need to nullify malicous source seal couple so it + does not get written to 'aes' datable + """ if not delpre: # not delegable so no delegation validation needed return @@ -2604,20 +2623,18 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, return if self.kevers is None or delpre not in self.kevers: # missing delegator - # ToDo XXXX cue a trigger to get the KEL of the delegator - # the processPSEvent should also cue a trigger to get KEL + # ToDo XXXX cue a trigger to get the KEL of the delegator. This may + # require OOBIing with the delegator. + # The processPSEvent should also cue a trigger to get KEL # of delegator if still missing when processing escrow later. self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) + if delseqner and delsaider: # save in case not attached later + self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"Missing KEL of delegator " f"{delpre} of evt = {serder.ked}.") - #self.escrowDelegableEvent(serder=serder, sigers=sigers, - #wigers=wigers,local=local) - #raise MissingDelegableApprovalError(f"Missing Kever for delegator" - #f" = {delpre} of event" - #f" = {serder.ked}.") dkever = self.kevers[delpre] if dkever.doNotDelegate: # drop event if delegation not allowed @@ -2631,7 +2648,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # delegation if delseqner is None or delsaider is None: # missing delegation seal ref - if self.locallyOwned(delpre): # local delegator so escrow. + if self.locallyOwned(delpre): # local delegator so escrow delegable. # Won't get to here if not local and locallyOwned(delpre) because # valSigsWigsDel will send nonlocal sourced delegable event to # misfit escrow first. Mistfit escrow must first @@ -2642,8 +2659,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, f" delegation by {delpre} of" f"event = {serder.ked}.") - else: # not local delegator so escrow + else: # not local delegator so escrow PSEvent self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) + # since delseqner or delsaider is None there is no PACouple to escrow here + #if delseqner and delsaider: # save in case not attached later + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"No delegation seal for delegator " f"{delpre} of evt = {serder.ked}.") @@ -2653,9 +2673,10 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # get the dig of the delegating event. Using getKeLast ensures delegating # event has not already been superceded key = snKey(pre=delpre, sn=ssn) # database key - raw = self.db.getKeLast(key) # get dig of delegating event + raw = self.db.getKeLast(key) # get dig of delegating event as index last - if raw is None: # no delegating event at key pre, sn + if raw is None: # no index to delegating event at key pre, sn + # Have to wait until delegating event at sn shows up in kel # ToDo XXXX process this cue of query to fetch delegating event from # delegator self.cues.push(dict(kin="query", q=dict(pre=delpre, @@ -2667,7 +2688,8 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, #sn = validateSN(sn=serder.snh, inceptive=inceptive) sn = Number(num=serder.sn).validate(inceptive=inceptive).sn self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + if delseqner and delsaider: # save in case not attached later + self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError("No delegating event from {} at {} for " "evt = {}.".format(delpre, delsaider.qb64, @@ -2676,24 +2698,24 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # get the delegating event from dig ddig = bytes(raw) key = dgKey(pre=delpre, dig=ddig) # database key - raw = self.db.getEvt(key) - if raw is None: # drop event + raw = self.db.getEvt(key) # get actual last event + if raw is None: # drop event should never happen unless database is broken raise ValidationError("Missing delegation from {} at event dig = {} for evt = {}." "".format(delpre, ddig, serder.ked)) - dserder = serdering.SerderKERI(raw=bytes(raw)) # delegating event - + dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event - # compare digests to make sure they match here + # compare digests to make sure they match here. + # should never fail unless database broken if not dserder.compare(said=delsaider.qb64): # drop event raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." "".format(delpre, ddig, serder.ked)) found = False # find event seal of delegated event in delegating data + # purported delegating event from source seal couple # XXXX ToDo need to change logic here to support native CESR seals not just dicts # for JSON, CBOR, MGPK - # may want to try harder here by walking KEL for dseal in dserder.seals: # find delegating seal anchor if tuple(dseal) == SealEvent._fields: seal = SealEvent(**dseal) @@ -2703,29 +2725,32 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, found = True break - if not found: # drop event + if not found: # drop event but may want to try harder here by walking KEL + # source seal couple is bad could be malicious so need to nullify as + # also walk KEL to find if any. raise ValidationError(f"Missing delegation seal in designated event" f"from {delpre} in {dserder.seals} for " f"evt = {serder.ked}.") - # Found anchor so can assume delegation successful unless its one of - # the superseding condidtions. - # Assumes database is reverified each bootup chain-of-custody of disc broken. + # Found anchor so can confirm delegation successful unless its one of + # the superseding conditions. Valid if not superseding drt of drt. + # Assumes database is reverified each bootup or otherwise when + # chain-of-custody of disc has been broken. # Rule for non-supeding delegated rotation of rotation. - # Returning delegator indicates success and eventually results acceptance + # Returning delegator indicates success and eventually results in acceptance # via Kever.logEvent which also writes the delgating event source couple to # db.aess so we can find it later if ((serder.ilk == Ilks.dip) or # delegated inception (serder.sner.num == self.sner.num + 1) or # inorder event (serder.sner.num == self.sner.num and # superseding event - self.ilk == Ilks.ixn and # superseded is ixn + self.ilk == Ilks.ixn and # superseded is ixn and serder.ilk == Ilks.drt)): # recovery rotation superseding ixn - return # delegator # indicates delegation valid with return of delegator + return # indicates delegation valid # get to here means drt rotation superseding another drt rotation # Kever.logEvent saves authorizer (delegator) seal source couple in - # db.aess data base so can use it here to recusively look up delegating - # events + # db.aess (.getAes, .setAes etc) data base so can use it here to + # recusively look up delegating events # set up recursive search for superseding delegations serfn = serder # new potentially superseding delegated event i.e. serf new @@ -2767,34 +2792,110 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # repeat - def fetchDelegatingEvent(self, delegator, serder): - """Returns delegating event by delegator of delegated event given by - serder otherwise raises ValidationError. + def fetchDelegatingEvent(self, delpre, serder, *, + delseqner=None, delsaider=None, eager=False): + """Returns delegating event of delegator given by its aid delpre of + delegated event given by serder otherwise raises ValidationError. + Assumes serder is already delegated event + Returns: + dserder (SerderKERI): key event with delegating seal + Parameters: - delegator (str): qb64 of identifier prefix of delegator + delpre (str): qb64 of identifier prefix of delegator serder (SerderKERI): delegated serder - - """ - dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key - if (couple := self.db.getAes(dgkey)): # delegation source couple + delseqner (Seqner | None): instance of delegating event sequence number. + delsaider (Saider | None): instance of of delegating event digest. + eager (bool): True means do more expensive KEL walk instead of escrow + False means do not do expensive KEL walk now + + Assumes db.aes source seal couple of delegating event cannot be written + unless that event has been accepted (first seen) into delegator's KEL + + ToDo XXXX + Use the db.fons database to lookup delegating events to ensure that + lookup only uses delegating events that were indeed accepted (first seen) + at some point even though they may now be superseded. + + Looking up in the db.evts from dig in db.aes could be malicious escrows + of delegating events?? But a malicious escrow of delegating event would + only write source seal couple to db.pdes not db.aes + + When escrowing .escrowPACouple, the delegating seal source couple goes + in db.pdes indexed by delegating event pre,dig + + Delegating event may have been superceded but delegated event validator + does not know it yet because db.aes keeps original delegating event + source seal from before is was superseded. + + Therefore lookup of delegating event needs to find delegating event via + db.fons and not last key event in .kels which would only return the + superseding event. + + When walking delegation tree a given delegating event may have + been superceded by another delegating event in the same delegator + KEL. This method does not distinguish between superseding and + superseded so can't assume that the delegating event is the last + event at its sn, i.e. the superseding event. + So this method must use db.fons to lookup to ensure that delegating + event was accepted (first seen) even if it has subsequently been + superseded. + This assumes that the delegating source seal in the AES db could + not have been written into the delegate's db.aes unless the delegating + event had been validated and accepted (first seen) into the + delegator's kel even though the delegating event may have later + been superseded at the time of this fetch. + + + Fetch delegating event has two cases: + 1. Delegated event has yet to be accepted (not seen) so can't repair + aes db because aes db must not be written until delegated event has + been accepted (first seen) bu .logEvent. + So must indicate the newly found dserder generates new source couple. + Returned dserder has the info to generate source couple but + does not indicate that a new one is needed. + + 2. Delegated event has been accepted so can repair aes source couple entry + when necessary. + + Need to indicate both if delegated event has been accepted and when + aes has been repaired. + seen=True (delegated event has been accepted first seen) + repair=True found new couple so repair externally if seen == False + or else repair just happened. + + When eager == False then may return None which means to escrow event and + try later with eager = True such as in escrow processor. + """ + dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate + if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate seqner, saider = deSourceCouple(couple) - dgkey = dgKey(pre=delegator, dig=saider) # event at its said - # get event by dig not by sn at last event because may have been superceded + deldig = saider.qb64 # dig of delegating event + # extra careful double check that .aes is valid by getting + # fner = first seen Number instance index + if not self.db.fons.get(keys=(delpre, deldig)): # None + raise ValidationError(f"Invalid delagation authorizing source " + f"seal couple for {serder.ked}") + + + dgkey = dgKey(pre=delpre, dig=deldig) # event at its said if not (raw := self.db.getEvt(dgkey)): # database broken this should never happen so do not supersede + # ToDo XXXX should repair by deleting the erroneous aes entry and + # returning found one raise ValidationError(f"Missing delegation event for {serder.ked}") - dserder = serdering.SerderKERI(raw=bytes(raw)) # original delegating event i.e. boss original + # original delegating event i.e. boss original + dserder = serdering.SerderKERI(raw=bytes(raw)) - else: #try to find seal the hard way + else: #try to find seal the hard way by walking the delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict - if not (dserder := self.db.findAnchoringSealEvent(pre=serder.delpre, seal=seal)): + if not (dserder := self.db.findAnchoringSealEvent(pre=delpre, seal=seal)): # database broken this should never happen so do not supersede raise ValidationError(f"Missing delegation source seal for {serder.ked}") - return dserder + return dserder # get actually found delseqner, delsaider from dserder def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, @@ -2850,7 +2951,7 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, # for non delegated kels, local controllers, and local witnesses. # These checks prevent ddos via malicious source seal attachments. # MUST NOT setAes if not delegated or locallyOwned or locallyWitnessed - if (self.delpre and not self.locallyOwned() + if (self.delpre and not serder.ilk == Ilks.ixn and not self.locallyOwned() and not self.locallyWitnessed(wits=wits) and seqner and saider): couple = seqner.qb64b + saider.qb64b self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal @@ -4697,15 +4798,17 @@ def fetchEstEvent(self, pre, sn): found = False while not found: - dig = bytes(self.db.getKeLast(key=snKey(pre, sn))) + dig = self.db.getKeLast(key=snKey(pre, sn)) if not dig: return None # retrieve event by dig - raw = bytes(self.db.getEvt(key=dgKey(pre=pre, dig=dig))) + dig = bytes(dig) + raw = self.db.getEvt(key=dgKey(pre=pre, dig=dig)) if not raw: return None + raw = bytes(raw) serder = serdering.SerderKERI(raw=raw) # deserialize event raw if serder.ked["t"] in (Ilks.icp, Ilks.dip, Ilks.rot, Ilks.drt): return serder # establishment event so return @@ -5291,8 +5394,9 @@ def processEscrowPartialSigs(self): raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - if not wigs: # empty list - # wigs maybe empty while waiting for first witness signature + if not wigs: # empty list wigs witness sigs not wits + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature # which may not arrive until some time after event is fully signed # so just log for debugging but do not unescrow by raising # ValidationError @@ -5309,11 +5413,6 @@ def processEscrowPartialSigs(self): delpre = self.kevers[eserder.pre].delpre else: delpre = eserder.ked["di"] - - # consider using here instead todo XXXX - # Kever.fetchDelegatingEvent(delegator=delpre, - # serder=eserder) - # need Kever reference for eserder.pre seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) if srdr is not None: @@ -5351,6 +5450,7 @@ def processEscrowPartialSigs(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + self.db.delPde(dgkey) # remove escrow if any if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): self.cues.push(dict(kin="psUnescrow", serder=eserder)) @@ -5469,11 +5569,12 @@ def processEscrowPartialWigs(self): raise ValidationError("Missing escrowed evt sigs at " "dig = {}.".format(bytes(edig))) - # get wigs + # get witness signatures (wigs not wits) wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs if not wigs: # empty list - # wigs maybe empty while waiting for first witness signature + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature # which may not arrive until some time after event is fully signed # so just log for debugging but do not unescrow by raising # ValidationError @@ -5492,6 +5593,18 @@ def processEscrowPartialWigs(self): couple = self.db.getPde(dgKey(pre, bytes(edig))) if couple is not None: delseqner, delsaider = deSourceCouple(couple) + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre + else: + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + couple = delseqner.qb64b + delsaider.qb64b + self.db.putPde(dgkey, couple) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=delseqner, delsaider=delsaider, local=esr.local) @@ -5513,14 +5626,15 @@ def processEscrowPartialWigs(self): # validation will be successful as event would not be in # partially witnessed escrow unless they had already validated - except MissingWitnessSignatureError as ex: - # still waiting on missing witness sigs + except (MissingWitnessSignatureError, MissingDelegationError) as ex: + # still waiting on missing witness sigs or delegation if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + self.db.delPde(dgkey) # remove escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) else: @@ -5531,6 +5645,7 @@ def processEscrowPartialWigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + self.db.delPde(dgkey) # remove escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) logger.debug(f"event=\n{eserder.pretty()}\n") diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 2fc0b8e5..7c7b3f58 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -611,6 +611,37 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event Only one value per DB key is allowed + .kels is named sub DB of key event log tables that map sequence numbers + to serialized event digests. + Uses sequence number or sn. + snKey + Values are digests used to lookup event in .evts sub DB + DB is keyed by identifier prefix plus sequence number of key event + More than one value per DB key is allowed + + .fels is named sub DB of first seen event log table (FEL) of digests + that indexes events in first 'seen' accepted order for replay and + cloning of event log. + Uses first seen order number or fn. + fnKey + DB is keyed by identifier prefix plus monotonically increasing first + seen order number fn. + Value is digest of serialized event used to lookup event in .evts sub DB + Only one value per DB key is allowed. + Provides append only ordering of accepted first seen events. + + .fons is named subDB CesrSuber + Uses digest + dgKey + Maps prefix and digest to fn value (first seen ordinal number) of + the associated event. So one used pre and event digest, get its fn here + and then use fn to fetch event from .evnts by fn from .fels. + This ensures that any event looked up this way was first seen at + some point in time even if later superseded by a recovery rotation. + Whereas direct lookup in .evts could be escrowed events that may + never have been accepted as first seen. + CesrSuber(db=self, subkey='fons.', klas=core.Number) + .esrs is named sub DB instance of Komer of EventSourceRecord dgKey DB is keyed by identifier prefix plus digest (said) of serialized event @@ -636,8 +667,9 @@ class Baser(dbing.LMDBer): Escrow processing determines if and how to promote event source to local and then reprocess - .delegables is named sub DB instance of CesrIoSetSuber for delegable escrows - subkey "dees." + .delegables is named sub DB instance of CesrIoSetSuber for delegable event + escrows of key event with local delegator that need approval. + subkey "dees." delegable event escrows snKey DB is keyed by event controller prefix plus sn of serialized event where sn is 32 char hex string with leading zeros @@ -648,18 +680,6 @@ class Baser(dbing.LMDBer): source for a delegable event of a local delegator must first pass through the misfit escrow and get promoted to local source. - # delegable events escrows. events with local delegator that need approval - self.delegables = subing.CesrIoSetSuber(db=self, subkey='dees.', klas=coring.Diger) - - .fels is named sub DB of first seen event log table (FEL) of digests - that indexes events in first 'seen' accepted order for replay and - cloning of event log. Only one value per DB key is allowed. - Provides append only ordering of accepted first seen events. - Uses first seen order number or fn. - fnKey - DB is keyed by identifier prefix plus monotonically increasing first - seen order number fn. - Value is digest of serialized event used to lookup event in .evts sub DB .dtss is named sub DB of datetime stamp strings in ISO 8601 format of the datetime when the event was first escrosed and then later first @@ -740,13 +760,6 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .kels is named sub DB of key event log tables that map sequence numbers - to serialized event digests. - snKey - Values are digests used to lookup event in .evts sub DB - DB is keyed by identifier prefix plus sequence number of key event - More than one value per DB key is allowed - .pses is named sub DB of partially signed escrowed event tables that map sequence numbers to serialized event digests. snKey @@ -808,10 +821,6 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .fons is named subDB instance of MatterSuber that maps - (prefix, digest) e.g. dgKey to fn value (first seen ordinal number) of - the associated event. So one can lookup event digest, get its fn here - and then use fn to fetch event by fn from .fels. .states (subkey stts.) is named subDB instance of SerderSuber that maps a prefix to the latest keystate for that prefix. Used by ._kevers.db for read @@ -978,6 +987,7 @@ def reopen(self, **kwa): self.evts = self.env.open_db(key=b'evts.') self.fels = self.env.open_db(key=b'fels.') + self.kels = self.env.open_db(key=b'kels.', dupsort=True) self.dtss = self.env.open_db(key=b'dtss.') self.aess = self.env.open_db(key=b'aess.') self.sigs = self.env.open_db(key=b'sigs.', dupsort=True) @@ -986,7 +996,6 @@ def reopen(self, **kwa): self.ures = self.env.open_db(key=b'ures.', dupsort=True) self.vrcs = self.env.open_db(key=b'vrcs.', dupsort=True) self.vres = self.env.open_db(key=b'vres.', dupsort=True) - self.kels = self.env.open_db(key=b'kels.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) self.pdes = self.env.open_db(key=b'pdes.') self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) @@ -996,6 +1005,9 @@ def reopen(self, **kwa): self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) self.qnfs = self.env.open_db(key=b'qnfs.', dupsort=True) + # events as ordered by first seen ordinals + self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=core.Number) + self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) self.vers = subing.Suber(db=self, subkey="vers.") @@ -1010,8 +1022,6 @@ def reopen(self, **kwa): # delegable events escrows. events with local delegator that need approval self.delegables = subing.IoSetSuber(db=self, subkey='dees.') - # events as ordered by first seen ordinals - self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=core.Number) # Kever state made of KeyStateRecord key states # TODO: clean self.states = koming.Komer(db=self, From 01809fb2e6565cb4d7b2b18d45a155eec4600605 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 20 Aug 2024 07:58:44 -0600 Subject: [PATCH 04/78] refactor delegation seal escrow to catcesrsuber --- src/keri/core/eventing.py | 69 ++++++++++++++++----------- src/keri/db/basing.py | 98 ++++++++++++++++++++++----------------- src/keri/db/subing.py | 1 + tests/core/test_escrow.py | 29 ++++++++---- tests/core/test_kevery.py | 1 + tests/db/test_basing.py | 52 ++++++++++++++++----- tests/db/test_subing.py | 30 ++++++++++++ 7 files changed, 192 insertions(+), 88 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 857cbdc4..23a94b81 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2820,10 +2820,10 @@ def fetchDelegatingEvent(self, delpre, serder, *, Looking up in the db.evts from dig in db.aes could be malicious escrows of delegating events?? But a malicious escrow of delegating event would - only write source seal couple to db.pdes not db.aes + only write source seal couple to db.udes not db.aes When escrowing .escrowPACouple, the delegating seal source couple goes - in db.pdes indexed by delegating event pre,dig + in db.udes indexed by delegating event pre,dig Delegating event may have been superceded but delegated event validator does not know it yet because db.aes keeps original delegating event @@ -3026,8 +3026,9 @@ def escrowMFEvent(self, serder, sigers, wigers=None, if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent res = self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed @@ -3132,8 +3133,9 @@ def escrowPACouple(self, serder, seqner, saider, local=True): """ local = True if local else False # ignored since not escrowing serder here dgkey = dgKey(serder.preb, serder.saidb) - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent logger.debug("Kever state: Escrowed source couple for partially signed " "or delegated event = %s\n", serder.ked) @@ -3163,8 +3165,9 @@ def escrowPWEvent(self, serder, wigers, sigers=None, if sigers: self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.putEvt(dgkey, serder.raw) # update event source @@ -4851,8 +4854,9 @@ def escrowMFEvent(self, serder, sigers, wigers=None, if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) # log escrowed logger.debug("Kevery process: escrowed misfit event=\n%s", @@ -4891,8 +4895,9 @@ def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None, l if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.addOoe(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed logger.debug("Kevery process: escrowed out of order event=\n%s", @@ -5405,9 +5410,11 @@ def processEscrowPartialSigs(self): # seal source (delegator issuer if any) delseqner = delsaider = None - couple = self.db.getPde(dgkey) - if couple is not None: - delseqner, delsaider = deSourceCouple(couple) + if (couple := self.db.udes.get(keys=dgkey)): + delseqner, delsaider = couple + #couple = self.db.getUde(dgkey) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): if eserder.pre in self.kevers: delpre = self.kevers[eserder.pre].delpre @@ -5418,8 +5425,10 @@ def processEscrowPartialSigs(self): if srdr is not None: delseqner = coring.Seqner(sn=srdr.sn) delsaider = coring.Saider(qb64=srdr.said) - couple = delseqner.qb64b + delsaider.qb64b - self.db.putPde(dgkey, couple) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] @@ -5450,7 +5459,8 @@ def processEscrowPartialSigs(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): self.cues.push(dict(kin="psUnescrow", serder=eserder)) @@ -5465,7 +5475,8 @@ def processEscrowPartialSigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): self.cues.push(dict(kin="psUnescrow", serder=eserder)) @@ -5590,9 +5601,11 @@ def processEscrowPartialWigs(self): # seal source (delegator issuer if any) delseqner = delsaider = None - couple = self.db.getPde(dgKey(pre, bytes(edig))) - if couple is not None: - delseqner, delsaider = deSourceCouple(couple) + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple + #couple = self.db.getUde(dgKey(pre, bytes(edig))) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): if eserder.pre in self.kevers: delpre = self.kevers[eserder.pre].delpre @@ -5603,8 +5616,10 @@ def processEscrowPartialWigs(self): if srdr is not None: delseqner = coring.Seqner(sn=srdr.sn) delsaider = coring.Saider(qb64=srdr.said) - couple = delseqner.qb64b + delsaider.qb64b - self.db.putPde(dgkey, couple) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=delseqner, delsaider=delsaider, local=esr.local) @@ -5634,7 +5649,8 @@ def processEscrowPartialWigs(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) else: @@ -5645,7 +5661,8 @@ def processEscrowPartialWigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) logger.debug(f"event=\n{eserder.pretty()}\n") diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 7c7b3f58..63a45606 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -760,7 +760,15 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .pses is named sub DB of partially signed escrowed event tables + .pses is named sub DB of partially signed event escrows + that map sequence numbers to serialized event digests. + snKey + Values are digests used to lookup event in .evts sub DB + DB is keyed by identifier prefix plus sequence number of key event + More than one value per DB key is allowed + + + .pwes is named sub DB of partially witnessed event escrowes that map sequence numbers to serialized event digests. snKey Values are digests used to lookup event in .evts sub DB @@ -777,12 +785,15 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of key event Only one value per DB key is allowed - .pwes is named sub DB of partially witnessed escrowed event tables - that map sequence numbers to serialized event digests. - snKey - Values are digests used to lookup event in .evts sub DB - DB is keyed by identifier prefix plus sequence number of key event - More than one value per DB key is allowed + .udes is named sub DB of unverified delegation seal source couple escrows + that map (pre, digest) of delegated event to delegating seal source + couple (sn, dig) that provides source delegator event seal. + Each couple is concatenation of fully qualified items, snu+dig + of delegating source event in which seal of delegated event appears. + dgKey + Values are couples used to lookup source event in .kels sub DB + DB is keyed by identifier prefix plus digest of key event + Only one value per DB key is allowed .uwes is named sub DB of unverified event indexed escrowed couples from witness signers. Witnesses are from witness list of latest establishment @@ -997,7 +1008,10 @@ def reopen(self, **kwa): self.vrcs = self.env.open_db(key=b'vrcs.', dupsort=True) self.vres = self.env.open_db(key=b'vres.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) - self.pdes = self.env.open_db(key=b'pdes.') + #self.pdes = self.env.open_db(key=b'pdes.') + #self.udes = self.env.open_db(key=b'udes.') + self.udes = subing.CatCesrSuber(db=self, subkey='udes.', + klas=(coring.Seqner, coring.Saider)) self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) @@ -2705,40 +2719,40 @@ def delPse(self, key, val): """ return self.delIoVal(self.pses, key, val) - def putPde(self, key, val): - """ - Use dgKey() - Write serialized event source couple to key (snu+dig) - Does not overwrite existing val if any - Returns True If val successfully written Else False - Returns False if key already exists - """ - return self.putVal(self.pdes, key, val) - - def setPde(self, key, val): - """ - Use dgKey() - Write serialized seal source couple to key (snu+dig) - Overwrites existing val if any - Returns True If val successfully written Else False - """ - return self.setVal(self.pdes, key, val) - - def getPde(self, key): - """ - Use dgKey() - Return seal source couple at key - Returns None if no entry at key - """ - return self.getVal(self.pdes, key) - - def delPde(self, key): - """ - Use dgKey() - Deletes value at key. - Returns True If key exists in database Else False - """ - return self.delVal(self.pdes, key) + #def putUde(self, key, val): + #""" + #Use dgKey() + #Write serialized event source couple to key (snu+dig) + #Does not overwrite existing val if any + #Returns True If val successfully written Else False + #Returns False if key already exists + #""" + #return self.putVal(self.udes, key, val) + + #def setUde(self, key, val): + #""" + #Use dgKey() + #Write serialized seal source couple to key (snu+dig) + #Overwrites existing val if any + #Returns True If val successfully written Else False + #""" + #return self.setVal(self.udes, key, val) + + #def getUde(self, key): + #""" + #Use dgKey() + #Return seal source couple at key + #Returns None if no entry at key + #""" + #return self.getVal(self.udes, key) + + #def delUde(self, key): + #""" + #Use dgKey() + #Deletes value at key. + #Returns True If key exists in database Else False + #""" + #return self.delVal(self.udes, key) def putPwes(self, key, vals): """ diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index ec4ae778..9232bb87 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -115,6 +115,7 @@ def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys + keys = (key.decode() if hasattr(key, "decode") else key for key in keys) return (self.sep.join(keys).encode("utf-8")) diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index d3f419fa..3b96e727 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -474,8 +474,12 @@ def test_missing_delegator_escrow(): escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 1 assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow entry for event + escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb + # verify Kevery process partials escrow is idempotent to previously escrowed events # assuming not stale but nothing else has changed @@ -484,8 +488,11 @@ def test_missing_delegator_escrow(): escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 1 assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow entry for event + escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb # apply Bob's inception to Dels' Kvy psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) @@ -496,8 +503,11 @@ def test_missing_delegator_escrow(): escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 1 assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + #escrow entry for event + escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb # apply Bob's delegating interaction to Dels' Kvy psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) @@ -512,8 +522,11 @@ def test_missing_delegator_escrow(): escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) assert len(escrows) == 0 - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow is None # delegated inception delegation couple + #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + #assert escrow is None # delegated inception delegation couple + #escrow entry for event delegated inception delegation couple + escrow = self.db.udes.get(keys=(delPre, delSrdr.said)) + assert escrow is None # Setup Del rotation event verfers, digers = delMgr.rotate(pre=delPre, temp=True) diff --git a/tests/core/test_kevery.py b/tests/core/test_kevery.py index 97b6dd7d..f740cc67 100644 --- a/tests/core/test_kevery.py +++ b/tests/core/test_kevery.py @@ -435,3 +435,4 @@ def test_stale_event_receipts(): if __name__ == "__main__": test_kevery() + test_stale_event_receipts() diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 16a0819e..17872eb7 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -1009,28 +1009,56 @@ def test_baser(): assert items == [] # empty assert not items - # Test .pdes partial delegated escrow seal source couples + # Test .udes partial delegated escrow seal source couples key = dgKey(preb, digb) assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") - # test .pdes sub db methods + # test .udes sub db methods + #ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' + #sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' + #ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' + #sdig2 = b'EBYYJRCCpAGO7WjjsLhtHVR37Pawv67kveIFUPvt38x0' + #val1 = ssnu1 + sdig1 + #val2 = ssnu2 + sdig2 + + #assert db.getUde(key) == None + #assert db.delUde(key) == False + #assert db.putUde(key, val1) == True + #assert db.getUde(key) == val1 + #assert db.putUde(key, val2) == False + #assert db.getUde(key) == val1 + #assert db.setUde(key, val2) == True + #assert db.getUde(key) == val2 + #assert db.delUde(key) == True + #assert db.getUde(key) == None + + # test .udes catcesrsuber sub db methods ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' sdig2 = b'EBYYJRCCpAGO7WjjsLhtHVR37Pawv67kveIFUPvt38x0' val1 = ssnu1 + sdig1 + tuple1 = (coring.Seqner(qb64b=ssnu1), coring.Saider(qb64b=sdig1)) val2 = ssnu2 + sdig2 + tuple2 = (coring.Seqner(qb64b=ssnu2), coring.Saider(qb64b=sdig2)) + + + assert db.udes.get(keys=key) == None + assert db.udes.rem(keys=key) == False + assert db.udes.put(keys=key, val=tuple1) == True + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val1 + assert db.udes.put(keys=key, val=tuple2) == False + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val1 + assert db.udes.pin(keys=key, val=tuple2) == True + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val2 + assert db.udes.rem(keys=key) == True + assert db.udes.get(keys=key) == None + + - assert db.getPde(key) == None - assert db.delPde(key) == False - assert db.putPde(key, val1) == True - assert db.getPde(key) == val1 - assert db.putPde(key, val2) == False - assert db.getPde(key) == val1 - assert db.setPde(key, val2) == True - assert db.getPde(key) == val2 - assert db.delPde(key) == True - assert db.getPde(key) == None # Partially Witnessed Escrow Events # test .pwes insertion order dup methods. dup vals are insertion order diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 4cee7586..c046695f 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -61,6 +61,35 @@ def test_suber(): actual = sdb.get(keys=keys) assert actual == kip + sdb.rem(keys) + actual = sdb.get(keys=keys) + assert actual is None + + sdb.put(keys=keys, val=sue) + actual = sdb.get(keys=keys) + assert actual == sue + + # test with keys as tuple of bytes + keys = (b"test_key", b"0001") + sdb.rem(keys) + actual = sdb.get(keys=keys) + assert actual is None + + sdb.put(keys=keys, val=sue) + actual = sdb.get(keys=keys) + assert actual == sue + + # test with keys as mixed tuple of bytes + keys = (b"test_key", "0001") + sdb.rem(keys) + actual = sdb.get(keys=keys) + assert actual is None + + sdb.put(keys=keys, val=sue) + actual = sdb.get(keys=keys) + assert actual == sue + + # test with keys as string not tuple keys = "keystr" @@ -1637,6 +1666,7 @@ def test_crypt_signer_suber(): if __name__ == "__main__": + test_suber() test_cesr_ioset_suber() test_serder_suber() test_cesr_suber() From 42a3ad6d0938743062d7978d5fa2e23efd3b5630 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 21 Aug 2024 20:34:47 -0600 Subject: [PATCH 05/78] Refactor SerderSuber so can create SerderIoSetSuber with SerderSuberBase so mro works. Create SerderIoSetSuber with unit tests. --- src/keri/db/subing.py | 244 +++++++++++++++++++++++++--------------- tests/db/test_subing.py | 162 +++++++++++++++++++++----- 2 files changed, 285 insertions(+), 121 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 9232bb87..71515313 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -60,8 +60,8 @@ class SuberBase(): db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes - verify (bool): some subclasses want to re-verify when deser from db - default false + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ Sep = '.' # separator for combining key iterables @@ -80,6 +80,8 @@ def __init__(self, db: dbing.LMDBer, *, each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ super(SuberBase, self).__init__() # for multi inheritance self.db = db @@ -115,8 +117,9 @@ def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys - keys = (key.decode() if hasattr(key, "decode") else key for key in keys) - return (self.sep.join(keys).encode("utf-8")) + #keys = (key.decode() if hasattr(key, "decode") else key for key in keys) + return (self.sep.join(key.decode() if hasattr(key, "decode") else key + for key in keys).encode("utf-8")) def _tokeys(self, key: Union[str, bytes, memoryview]): @@ -204,6 +207,17 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'docs.', dupsort: bool=False, **kwa): """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key @@ -282,7 +296,7 @@ def rem(self, keys: Union[str, Iterable]): class CesrSuberBase(SuberBase): """ - Sub class of Suber where data is CESR encode/decode ducktyped subclass + Sub class of SuberBase where data is CESR encode/decode ducktyped subclass instance such as Matter, Indexer, Counter with .qb64b property when provided as fully qualified serialization Automatically serializes and deserializes from qb64b to/from CESR instance @@ -291,11 +305,20 @@ class CesrSuberBase(SuberBase): def __init__(self, *pa, klas: Type[coring.Matter] = coring.Matter, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + Parameters: klas (Type[coring.Matter]): Class reference to subclass of Matter or Indexer or Counter or any ducktyped class of Matter + """ super(CesrSuberBase, self).__init__(*pa, **kwa) self.klas = klas @@ -318,7 +341,7 @@ def _des(self, val: Union[str, memoryview, bytes]): """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes - return self.klas(qb64b=val) + return self.klas(qb64b=val) # qb64b parameter accepts str class CesrSuber(CesrSuberBase, Suber): @@ -334,9 +357,16 @@ class CesrSuber(CesrSuberBase, Suber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False klas (Type[coring.Matter]): Class reference to subclass of Matter or Indexer or Counter or any ducktyped class of Matter """ @@ -361,7 +391,7 @@ class CatCesrSuberBase(CesrSuberBase): def __init__(self, *pa, klas: Iterable = None, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -369,8 +399,10 @@ def __init__(self, *pa, klas: Iterable = None, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ if klas is None: @@ -378,7 +410,6 @@ def __init__(self, *pa, klas: Iterable = None, **kwa): if not nonStringIterable(klas): # not iterable klas = (klas, ) # make it so super(CatCesrSuberBase, self).__init__(*pa, klas=klas, **kwa) - # self.klas = klas def _ser(self, val: Union[Iterable, coring.Matter]): @@ -423,7 +454,7 @@ class CatCesrSuber(CatCesrSuberBase, Suber): such as Matter, Indexer, Counter Automatically serializes and deserializes from qb64b to/from CESR instances - Attributes: + Attributes: db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes @@ -433,7 +464,7 @@ class CatCesrSuber(CatCesrSuberBase, Suber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -441,8 +472,10 @@ def __init__(self, *pa, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(CatCesrSuber, self).__init__(*pa, **kwa) @@ -475,7 +508,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str='docs.', dupsort: bool=False, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -483,6 +516,10 @@ def __init__(self, db: dbing.LMDBer, *, each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(IoSetSuber, self).__init__(db=db, subkey=subkey, dupsort=False, **kwa) @@ -500,6 +537,8 @@ def put(self, keys: Union[str, Iterable], vals: Iterable): result (bool): True If successful, False otherwise. """ + #if not nonStringIterable(vals): # not iterable + #vals = (vals, ) # make iterable return (self.db.putIoSetVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals], @@ -542,6 +581,8 @@ def pin(self, keys: Union[str, Iterable], vals: Iterable): """ key = self._tokey(keys) self.db.delIoSetVals(db=self.sdb, key=key) # delete all values + #if not nonStringIterable(vals): # not iterable + #vals = (vals, ) # make iterable return (self.db.setIoSetVals(db=self.sdb, key=key, vals=[self._ser(val) for val in vals], @@ -779,7 +820,7 @@ class CesrIoSetSuber(CesrSuberBase, IoSetSuber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -787,6 +828,8 @@ def __init__(self, *pa, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False klas (Type[coring.Matter]): Class reference to subclass of Matter or Indexer or Counter or any ducktyped class of Matter @@ -829,16 +872,18 @@ class CatCesrIoSetSuber(CatCesrSuberBase, IoSetSuber): """ def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key - False (default) means do not enable duplicates at - each key + False (default) means do not enable duplicates at + each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(CatCesrIoSetSuber, self).__init__(*pa, **kwa) @@ -1044,13 +1089,11 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", yield (ikeys, self.klas(qb64b=bytes(val), transferable=verfer.transferable)) - -class SerderSuber(Suber): +class SerderSuberBase(SuberBase): """ - Sub class of Suber where data is serialized Serder Subclass instance + Sub class of SuberBase where data is serialized Serder Subclass instance given by .klas Automatically serializes and deserializes using .klas Serder methods - """ def __init__(self, *pa, @@ -1060,99 +1103,120 @@ def __init__(self, *pa, Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False - Parameters: + Overridden Parameters: klas (Type[serdering.Serder]): Class reference to subclass of Serder """ - super(SerderSuber, self).__init__(*pa, **kwa) + super(SerderSuberBase, self).__init__(*pa, **kwa) self.klas = klas - def put(self, keys: Union[str, Iterable], val: serdering.SerderKERI): + def _ser(self, val: serdering.Serder): """ - Puts val at key made from keys. Does not overwrite - + Serialize value to bytes to store in db Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Serder): instance - - Returns: - result (bool): True If successful, False otherwise, such as key - already in database. + val (serdering.Serder): instance Serder subclass like SerderKERI """ - return (self.db.putVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) + return val.raw - def pin(self, keys: Union[str, Iterable], val: serdering.SerderKERI): + def _des(self, val: (str | memoryview | bytes)): """ - Pins (sets) val at key made from keys. Overwrites. - + Deserialize val to str Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Serder): instance - - Returns: - result (bool): True If successful. False otherwise. + val (Union[str, memoryview, bytes]): convertable to coring.matter """ - return (self.db.setVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) + if isinstance(val, memoryview): # memoryview is always bytes + val = bytes(val) # convert to bytes + elif hasattr(val, "encode"): # str + val = val.encode() # convert to bytes + return self.klas(raw=val, verify=self.verify) - def get(self, keys: Union[str, Iterable]): +class SerderSuber(SerderSuberBase, Suber): + """ + Sub class of SerderSuberBase, Suber where data is serialized Serder Subclass + instance given by .klas + Automatically serializes and deserializes using .klas Serder methods + """ + + def __init__(self, *pa, **kwa): """ - Gets Serder at keys + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[serdering.Serder]): Class reference to subclass of Serder + """ + super(SerderSuber, self).__init__(*pa, **kwa) - Parameters: - keys (tuple): of key strs to be combined in order to form key - Returns: - Serder: - None: if no entry at keys +class SerderIoSetSuber(SerderSuberBase, IoSetSuber): + """ + Sub class of SerderSuberBase and IoSetSuber that allows multiple Serder + instances to be stored at the same db key in insertion order. + Example use case would be an escrow where the key is a sequence number + based index (such as snKey). - Usage: - Use walrus operator to catch and raise missing entry - if (srder := mydb.get(keys)) is None: - raise ExceptionHere - use srdr here + Sub class of SerderSuberBase where data is serialized Serder Subclass instance + given by .klas + Automatically serializes and deserializes using .klas Serder methods - """ - val = self.db.getVal(db=self.sdb, key=self._tokey(keys)) - return self.klas(raw=bytes(val), verify=self.verify) if val is not None else None + Extends IoSetSuber so that all IoSetSuber methods now work with Serder + subclass for each val. + IoSetSuber stores at each effective key a set of distinct values that + share that same effective key where each member of the set is retrieved in + insertion order (dupsort==False) + The methods allows an Iterable (set valued) of Iterables of separation subclass + instances to be stored at a given effective key in insertion order. - def rem(self, keys: Union[str, Iterable]): - """ - Removes entry at keys + Actual keys include a hidden ordinal key suffix that tracks insertion order. + The suffix is appended and stripped transparently from the keys. The set of + items with duplicate effective keys are retrieved in insertion order when + iterating or as a list of the set elements. The actual iokey for any item + includes the ordinal suffix. - Parameters: - keys (tuple): of key strs to be combined in order to form key + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + klas (Iterable): of Class references to subclasses of CESR compatible + , each of to Type[coring.Matter etc] - Returns: - result (bool): True if key exists so delete successful. False otherwise - """ - return(self.db.delVal(db=self.sdb, key=self._tokey(keys))) + """ + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[serdering.Serder]): Class reference to subclass of Serder - def getItemIter(self, keys: Union[str, Iterable]=b""): """ - Returns: - iterator (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + super(SerderIoSetSuber, self).__init__(*pa, **kwa) - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield self._tokeys(iokey), self.klas(raw=bytes(val), verify=self.verify) class SchemerSuber(Suber): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index c046695f..bd6e10e0 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -641,79 +641,176 @@ def test_serder_suber(): assert db.name == "test" assert db.opened - sdb = subing.SerderSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.SerderSuber) - assert not sdb.sdb.flags()["dupsort"] + serber = subing.SerderSuber(db=db, subkey='bags.') + assert isinstance(serber, subing.SerderSuber) + assert not serber.sdb.flags()["dupsort"] + assert serber.klas == serdering.SerderKERI pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" srdr0 = eventing.incept(keys=[pre]) keys = (pre, srdr0.said) - sdb.put(keys=keys, val=srdr0) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr0) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - sdb.rem(keys) - actual = sdb.get(keys=keys) + serber.rem(keys) + actual = serber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=srdr0) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr0) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) - result = sdb.put(keys=keys, val=srdr1) + result = serber.put(keys=keys, val=srdr1) assert not result + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - result = sdb.pin(keys=keys, val=srdr1) + result = serber.pin(keys=keys, val=srdr1) assert result - actual = sdb.get(keys=keys) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said # test with keys as string not tuple keys = "{}.{}".format(pre, srdr1.said) - sdb.put(keys=keys, val=srdr1) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr1) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said - sdb.rem(keys) - actual = sdb.get(keys=keys) + serber.rem(keys) + actual = serber.get(keys=keys) assert actual is None # test missing entry at keys badkey = "badkey" - actual = sdb.get(badkey) + actual = serber.get(badkey) assert actual is None # test iteritems - sdb = subing.SerderSuber(db=db, subkey='pugs.') - assert isinstance(sdb, subing.SerderSuber) - sdb.put(keys=("a","1"), val=srdr0) - sdb.put(keys=("a","2"), val=srdr1) + serber = subing.SerderSuber(db=db, subkey='pugs.') + assert isinstance(serber, subing.SerderSuber) + serber.put(keys=("a","1"), val=srdr0) + serber.put(keys=("a","2"), val=srdr1) - items = [(keys, srdr.said) for keys, srdr in sdb.getItemIter()] + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter()] assert items == [(('a', '1'), srdr0.said), (('a', '2'), srdr1.said)] - assert sdb.put(keys=("b","1"), val=srdr0) - assert sdb.put(keys=("b","2"), val=srdr1) - assert sdb.put(keys=("bc","1"), val=srdr0) + assert serber.put(keys=("b","1"), val=srdr0) + assert serber.put(keys=("b","2"), val=srdr1) + assert serber.put(keys=("bc","1"), val=srdr0) topkeys = ("b", "") # append empty str to force trailing .sep - items = [(keys, srdr.said) for keys, srdr in sdb.getItemIter(keys=topkeys)] + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter(keys=topkeys)] assert items == [(('b', '1'), srdr0.said), (('b', '2'), srdr1.said)] assert not os.path.exists(db.path) assert not db.opened +def test_serder_ioset_suber(): + """ + Test SerderIoSetSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + serber = subing.SerderIoSetSuber(db=db, subkey='bags.') + assert isinstance(serber, subing.SerderIoSetSuber) + assert not serber.sdb.flags()["dupsort"] + assert serber.klas == serdering.SerderKERI + + pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + srdr0 = eventing.incept(keys=[pre]) + + keys = (pre, srdr0.said) + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr0.said + + serber.rem(keys) + actuals = serber.get(keys=keys) + assert not actuals # empty list + + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr0.said + + srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) + result = serber.put(keys=keys, vals=(srdr1, )) + assert result + actuals = serber.get(keys=keys) + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr0.said + assert isinstance(actuals[1], serdering.SerderKERI) + assert actuals[1].said == srdr1.said + + + result = serber.pin(keys=keys, vals=(srdr1, )) + assert result + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr1.said + + # test with keys as string not tuple + keys = "{}.{}".format(pre, srdr1.said) + + serber.put(keys=keys, vals=(srdr1, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr1.said + + serber.rem(keys) + actuals = serber.get(keys=keys) + assert not actuals + + # test missing entry at keys + badkey = "badkey" + actuals = serber.get(badkey) + assert not actuals + + # test iteritems + serber = subing.SerderIoSetSuber(db=db, subkey='pugs.') + assert isinstance(serber, subing.SerderIoSetSuber) + serber.put(keys=("a","1"), vals=(srdr0, )) + serber.put(keys=("a","1"), vals=(srdr1, )) + serber.put(keys=("a","2"), vals=(srdr1, )) + + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter()] + assert items == [ + (('a', '1'), srdr0.said), + (('a', '1'), srdr1.said), + (('a', '2'), srdr1.said), + ] + + assert serber.put(keys=("b","1"), vals=(srdr0, )) + assert serber.put(keys=("b","2"), vals=(srdr1, )) + assert serber.put(keys=("bc","1"), vals=(srdr0, )) + + topkeys = ("b", "") # append empty str to force trailing .sep + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), srdr0.said), + (('b', '2'), srdr1.said)] + + assert not os.path.exists(db.path) + assert not db.opened def test_cesr_suber(): @@ -996,7 +1093,7 @@ def test_cat_suber(): """Done Test""" -def test_cat__cesr_ioset_suber(): +def test_cat_cesr_ioset_suber(): """ Test CatIoSetSuber LMDBer sub database class """ @@ -1667,11 +1764,14 @@ def test_crypt_signer_suber(): if __name__ == "__main__": test_suber() - test_cesr_ioset_suber() - test_serder_suber() - test_cesr_suber() + test_dup_suber() + test_ioset_suber() test_cat_suber() - test_cat__cesr_ioset_suber() + test_cesr_suber() + test_cesr_ioset_suber() + test_cat_cesr_ioset_suber() test_cesr_dup_suber() + test_serder_suber() + test_serder_ioset_suber() test_signer_suber() test_crypt_signer_suber() From bfd041f2aa951a6e1ffea51df36460a378235b91 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 22 Aug 2024 13:38:06 -0600 Subject: [PATCH 06/78] more unit tests for SerderIoSetSuber --- src/keri/db/subing.py | 5 +++-- tests/db/test_subing.py | 25 +++++++++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 71515313..6a8f005c 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -664,8 +664,9 @@ def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b'' Parameters: keys (Iterable): of key strs to be combined in order to form key - val (str): value at key to delete - if val is empty then remove all values at key + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key Returns: result (bool): True if effective key with val exists so delete successful. diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index bd6e10e0..86e95606 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -655,7 +655,7 @@ def test_serder_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - serber.rem(keys) + assert serber.rem(keys) actual = serber.get(keys=keys) assert actual is None @@ -685,7 +685,7 @@ def test_serder_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said - serber.rem(keys) + assert serber.rem(keys) actual = serber.get(keys=keys) assert actual is None @@ -741,7 +741,7 @@ def test_serder_ioset_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - serber.rem(keys) + assert serber.rem(keys) actuals = serber.get(keys=keys) assert not actuals # empty list @@ -752,6 +752,7 @@ def test_serder_ioset_suber(): assert actual.said == srdr0.said srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) + result = serber.put(keys=keys, vals=(srdr1, )) assert result actuals = serber.get(keys=keys) @@ -768,6 +769,22 @@ def test_serder_ioset_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said + + # test rem a specific value + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr1.said + assert isinstance(actuals[1], serdering.SerderKERI) + assert actuals[1].said == srdr0.said + + assert serber.rem(keys=keys, val=srdr1) + actuals = serber.get(keys=keys) + assert len(actuals) == 1 + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr0.said + + # test with keys as string not tuple keys = "{}.{}".format(pre, srdr1.said) @@ -777,7 +794,7 @@ def test_serder_ioset_suber(): assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said - serber.rem(keys) + assert serber.rem(keys) actuals = serber.get(keys=keys) assert not actuals From 341e3af4045fdef811ca82e1721673bb19508873 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 22 Aug 2024 14:17:29 -0600 Subject: [PATCH 07/78] added basing.pdes excrow of type IoSetSuber --- src/keri/db/basing.py | 40 ++++++++++++++++++++++------------------ tests/db/test_basing.py | 28 ++++++++-------------------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 63a45606..4bd0c5bf 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -606,21 +606,24 @@ class Baser(dbing.LMDBer): kevers (dict): Kever instances indexed by identifier prefix qb64 prefixes (OrderedSet): local prefixes corresponding to habitats for this db - .evts is named sub DB whose values are serialized events + .evts is named sub DB whose values are serialized key events dgKey DB is keyed by identifier prefix plus digest of serialized event Only one value per DB key is allowed - .kels is named sub DB of key event log tables that map sequence numbers - to serialized event digests. + .kels is named sub DB of key event logs as indices that map sequence numbers + to serialized key event digests. + Actual serialized key events are stored in .evts by SAID digest Uses sequence number or sn. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .fels is named sub DB of first seen event log table (FEL) of digests - that indexes events in first 'seen' accepted order for replay and + .fels is named sub DB of first seen event logs (FEL) as indices that map + first seen ordinal number to digests. + Actual serialized key events are stored in .evts by SAID digest + This indexes events in first 'seen' accepted order for replay and cloning of event log. Uses first seen order number or fn. fnKey @@ -760,7 +763,7 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .pses is named sub DB of partially signed event escrows + .pses is named sub DB of partially signed key event escrows that map sequence numbers to serialized event digests. snKey Values are digests used to lookup event in .evts sub DB @@ -768,22 +771,21 @@ class Baser(dbing.LMDBer): More than one value per DB key is allowed - .pwes is named sub DB of partially witnessed event escrowes + .pwes is named sub DB of partially witnessed key event escrowes that map sequence numbers to serialized event digests. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .pdes is named sub DB of partially delegated escrowed couples - that map digest to seal source couple that provides source - (delegator or issuer) event seal. Each couples is concatenations - of full qualified items, snu+dig of authorizing (delegating or - issuing) source event. - dgKey - Values are couples used to lookup source event in .kels sub DB - DB is keyed by identifier prefix plus digest of key event - Only one value per DB key is allowed + .pdes is named sub DB of partially delegated key event escrows + that map sequence numbers to serialized event digests. This is + used in conjunction with .udes which escrows the associated seal + source couple. + snKey + Values are digests used to lookup delegated event in .evts sub DB + DB is keyed by identifier prefix plus sequence number of key event + More than one value per DB key is allowed .udes is named sub DB of unverified delegation seal source couple escrows that map (pre, digest) of delegated event to delegating seal source @@ -791,7 +793,9 @@ class Baser(dbing.LMDBer): Each couple is concatenation of fully qualified items, snu+dig of delegating source event in which seal of delegated event appears. dgKey - Values are couples used to lookup source event in .kels sub DB + Values are serialized instances of CatCesrSuber as couples + (Seqner.qb64b, Saider.qb64b) used to lookup source event in delegator's + KEL. DB is keyed by identifier prefix plus digest of key event Only one value per DB key is allowed @@ -1009,7 +1013,7 @@ def reopen(self, **kwa): self.vres = self.env.open_db(key=b'vres.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) #self.pdes = self.env.open_db(key=b'pdes.') - #self.udes = self.env.open_db(key=b'udes.') + self.pdes = subing.IoSetSuber(db=self, subkey='pdes.') self.udes = subing.CatCesrSuber(db=self, subkey='udes.', klas=(coring.Seqner, coring.Saider)) self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 17872eb7..a3d8b98b 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -24,6 +24,7 @@ from keri.db import basing from keri.db import dbing +from keri.db import subing from keri.db.basing import openDB, Baser, KeyStateRecord from keri.db.dbing import (dgKey, onKey, snKey) from keri.db.dbing import openLMDB @@ -1013,26 +1014,13 @@ def test_baser(): key = dgKey(preb, digb) assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") - # test .udes sub db methods - #ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' - #sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' - #ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' - #sdig2 = b'EBYYJRCCpAGO7WjjsLhtHVR37Pawv67kveIFUPvt38x0' - #val1 = ssnu1 + sdig1 - #val2 = ssnu2 + sdig2 - - #assert db.getUde(key) == None - #assert db.delUde(key) == False - #assert db.putUde(key, val1) == True - #assert db.getUde(key) == val1 - #assert db.putUde(key, val2) == False - #assert db.getUde(key) == val1 - #assert db.setUde(key, val2) == True - #assert db.getUde(key) == val2 - #assert db.delUde(key) == True - #assert db.getUde(key) == None - - # test .udes catcesrsuber sub db methods + # test .pdes SerderIoSetSuber methods + assert isinstance(db.pdes, subing.IoSetSuber) + + # test .udes CatCesrSuber sub db methods + assert isinstance(db.udes, subing.CatCesrSuber) + assert db.udes.klas == (coring.Seqner, coring.Saider) + ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' From eb53a57eb69b3f52c8323983cd3ef537e865302d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Aug 2024 14:05:41 -0600 Subject: [PATCH 08/78] More refactor clean up of subing --- src/keri/core/eventing.py | 174 +++++++++++++ src/keri/db/basing.py | 34 --- src/keri/db/dbing.py | 132 +++++----- src/keri/db/subing.py | 499 ++++++++++++++++++++------------------ src/keri/vdr/viring.py | 2 +- tests/db/test_dbing.py | 2 +- tests/db/test_subing.py | 145 ++++++++--- 7 files changed, 619 insertions(+), 369 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 23a94b81..a412c37d 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5671,6 +5671,180 @@ def processEscrowPartialWigs(self): break key = ekey # setup next while iteration, with key after ekey + def processEscrowPartialDels(self): + """ + Process delgated events escrowed by Kever that were only partially fulfilled + due to missing or unverified delegation seals from delegators. + Events only make into this escrow after fully signed and if witnessed + fully witnessed. + + db.pdes is an instance of subing.IoSetSuber and uses instance methods + for access to the underlying database. + Escrowed items in .pdes are indexed in database table keyed by prefix and + sequence number with multiple entries at same key held in insertion order. + This allows FIFO processing of escrowed events with same prefix and sn. + + Value in each .pdes entry is dgkey (SAID) of event stored in db.evts where + db.evts holds SerderKERI.raw of event. + + Steps: + Each pass (walk index table) + For each prefix,sn + For each escrow item dup at prefix,sn: + Get Event + Get and Attach Signatures + Get and Attach Witness Signatures + Process event as if it came in over the wire + If successful then remove from escrow table + """ + + key = ekey = b'' # both start same. when not same means escrows found + while True: # break when done + for ekey, edig in self.db.getPweItemsNextIter(key=key): + try: + pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) + + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) + + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) + + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) + + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + + # get sigs + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) + + # get witness signatures (wigs not wits) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + + if not wigs: # empty list + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) + + # raise ValidationError("Missing escrowed evt wigs at " + # "dig = {}.".format(bytes(edig))) + + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple + #couple = self.db.getUde(dgKey(pre, bytes(edig))) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre + else: + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, local=esr.local) + + # If process does NOT validate wigs then process will attempt + # to re-escrow and then raise MissingWitnessSignatureError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pwe escrow is called by + # Kever.self.escrowPWEvent Which calls + # self.db.addPwe(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pwe db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except (MissingWitnessSignatureError, MissingDelegationError) as ex: + # still waiting on missing witness sigs or delegation + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + + def processEscrowUnverWitness(self): """ Process escrowed unverified event receipts from witness receiptors diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 4bd0c5bf..5a193788 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2723,40 +2723,6 @@ def delPse(self, key, val): """ return self.delIoVal(self.pses, key, val) - #def putUde(self, key, val): - #""" - #Use dgKey() - #Write serialized event source couple to key (snu+dig) - #Does not overwrite existing val if any - #Returns True If val successfully written Else False - #Returns False if key already exists - #""" - #return self.putVal(self.udes, key, val) - - #def setUde(self, key, val): - #""" - #Use dgKey() - #Write serialized seal source couple to key (snu+dig) - #Overwrites existing val if any - #Returns True If val successfully written Else False - #""" - #return self.setVal(self.udes, key, val) - - #def getUde(self, key): - #""" - #Use dgKey() - #Return seal source couple at key - #Returns None if no entry at key - #""" - #return self.getVal(self.udes, key) - - #def delUde(self, key): - #""" - #Use dgKey() - #Deletes value at key. - #Returns True If key exists in database Else False - #""" - #return self.delVal(self.udes, key) def putPwes(self, key, vals): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 1dc1208b..5d29d242 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -652,8 +652,39 @@ def delTopVal(self, db, key=b''): # For subdbs with no duplicate values allowed at each key. (dupsort==False) - # and use keys with ordinal as monotonically increasing number part - # such as sn or fn + # and use keys with suffic ordinal that is monotonically increasing number part + # such as fn where no duplicates allowed at a given (pre, on) + + def cntAllOrdValsPre(self, db, pre, on=0): + """ + Returns (int): count of of all ordinal keyed vals with same pre in key + but different on in key in db starting at ordinal number on of pre. + For db with dupsort=False but ordinal number suffix in each key + + Parameters: + db is opened named sub db + pre is bytes of key within sub db's keyspace pre.on + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = onKey(pre, on) # start replay at this enty 0 is earliest + count = 0 + if not cursor.set_range(key): # moves to val at key >= key + return count # no values end of db + + for key in cursor.iternext(values=False): # get key only at cursor + try: + cpre, cn = splitKeyON(key) + except ValueError as ex: # not splittable key + break + + if cpre != pre: # prev is now the last event for pre + break # done + count = count+1 + + return count + + def appendOrdValPre(self, db, pre, val): """ Appends val in order after last previous key with same pre in db. @@ -757,6 +788,9 @@ def getAllOrdItemAllPreIter(self, db, key=b''): yield (cpre, cn, val) # (pre, on, dig) of event + + # IoSet insertion order in val so can have effective dups but with + # dupsort = False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent # effective duplicate key but with (dupsort==False). Actual key uses hidden # key suffix ordinal to provide insertion ordering of value members of set @@ -810,8 +844,8 @@ def putIoSetVals(self, db, key, vals, *, sep=b'.'): def addIoSetVal(self, db, key, val, *, sep=b'.'): """ - Add val to insertion ordered set of values all with the same apparent - effective key if val not already in set of vals at key. + Add val idempotently to insertion ordered set of values all with the + same apparent effective key if val not already in set of vals at key. A Uses hidden ordinal key suffix for insertion ordering. The suffix is appended and stripped transparently. @@ -845,38 +879,18 @@ def addIoSetVal(self, db, key, val, *, sep=b'.'): return cursor.put(iokey, val, dupdata=False, overwrite=False) - def setIoSetVals(self, db, key, vals, *, sep=b'.'): - """ - Erase all vals at key and then add unique vals as insertion ordered set of - values all with the same apparent effective key. - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. - - Returns: - result (bool): True is added to set. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - vals (abc.Iterable): serialized values to add to set of vals at key - """ - self.delIoSetVals(db=db, key=key, sep=sep) - result = False - vals = oset(vals) # make set - with self.env.begin(db=db, write=True, buffers=True) as txn: - for i, val in enumerate(vals): - iokey = suffix(key, i, sep=sep) # ion is at add on amount - result = txn.put(iokey, val, dupdata=False, overwrite=True) or result - return result - - def appendIoSetVal(self, db, key, val, *, sep=b'.'): """ - Append val to insertion ordered set of values all with the same apparent - effective key. Assumes val is not already in set. + Append val non-idempotently to insertion ordered set of values all with + the same apparent effective key. If val already in set at key then + after append there will be multiple entries in database with val at key + each with different insertion order (iokey). Uses hidden ordinal key suffix for insertion ordering. The suffix is appended and stripped transparently. + Works by walking backward to find last iokey for key instead of reading + all vals for ioky. + Returns: ion (int): hidden insertion ordering ordinal of appended val @@ -923,6 +937,33 @@ def appendIoSetVal(self, db, key, val, *, sep=b'.'): return ion + + def setIoSetVals(self, db, key, vals, *, sep=b'.'): + """ + Erase all vals at key and then add unique vals as insertion ordered set of + values all with the same apparent effective key. + Uses hidden ordinal key suffix for insertion ordering. + The suffix is appended and stripped transparently. + + Returns: + result (bool): True is added to set. + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): Apparent effective key + vals (abc.Iterable): serialized values to add to set of vals at key + """ + self.delIoSetVals(db=db, key=key, sep=sep) + result = False + vals = oset(vals) # make set + with self.env.begin(db=db, write=True, buffers=True) as txn: + for i, val in enumerate(vals): + iokey = suffix(key, i, sep=sep) # ion is at add on amount + result = txn.put(iokey, val, dupdata=False, overwrite=True) or result + return result + + + def getIoSetVals(self, db, key, *, ion=0, sep=b'.'): """ Returns: @@ -1338,31 +1379,6 @@ def cntVals(self, db, key): return count - def cntValsAllPre(self, db, pre, on=0): - """ - Returns (int): count of of all vals with same pre in key but different - on in key in db starting at ordinal number on of pre - - Does not count dups - - Parameters: - db is opened named sub db - pre is bytes of key within sub db's keyspace pre.on - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - count = 0 - if not cursor.set_range(key): # moves to val at key >= key - return count # no values end of db - - for val in cursor.iternext(values=False): # get key, val at cursor - cpre, cn = splitKeyON(val) - if cpre != pre: # prev is now the last event for pre - break # done - count = count+1 - - return count def delVals(self, db, key, val=b''): @@ -1385,7 +1401,9 @@ def delVals(self, db, key, val=b''): # For subdbs that support insertion order preserving duplicates at each key. - # dupsort==True and prepends and strips io val proem + # IoDup class IoVals IoItems + # dupsort==True and prepends and strips io val proem to each value. + # because dupsort==True values are limited to 511 bytes including proem def putIoVals(self, db, key, vals): """ Write each entry from list of bytes vals to key in db in insertion order diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 6a8f005c..54a428f9 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -91,24 +91,30 @@ def __init__(self, db: dbing.LMDBer, *, - def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], + def _tokey(self, keys: str | bytes | memoryview | Iterable[str | bytes], top: bool=False): """ - Converts keys to key str with proper separators and returns key bytes. - If keys is already str then returns. Else If keys is iterable (non-str) - of strs then joins with separator converts to bytes and returns. - top allows partial key from top branch of key space given by partial keys + Converts keys to key bytes with proper separators and returns key bytes. + If keys is already str or bytes then returns key bytes. + Else If keys is iterable (non-str) of strs or bytes then joins with + separator converts to key bytes and returns. When keys is iterable and + top is True then enables partial key from top branch of key space given + by partial keys by appending separator to end of partial key Returns: key (bytes): each element of keys is joined by .sep. If top then last char of key is also .sep Parameters: - keys (Union[str, bytes, Iterable]): str, bytes, or Iterable of str. + keys (str | bytes | memoryview | Iterable[str | bytes]): db key or + Iterable of (str | bytes) to form key. top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep. + key space given by partial keys. Resultant key ends in .sep + character. False means treat as full branch in key space. Resultant key - does not end in .sep + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ if hasattr(keys, "encode"): # str @@ -117,21 +123,22 @@ def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys - #keys = (key.decode() if hasattr(key, "decode") else key for key in keys) + if top and keys[-1]: # top and keys is not already partial + keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep return (self.sep.join(key.decode() if hasattr(key, "decode") else key for key in keys).encode("utf-8")) - def _tokeys(self, key: Union[str, bytes, memoryview]): + def _tokeys(self, key: str | bytes | memoryview): """ Converts key bytes to keys tuple of strs by decoding and then splitting - at separator. + at separator .sep. Returns: - keys (iterable): of str + keys (tuple[str]): makes tuple by splitting key at sep Parameters: - key (Union[str, bytes]): str or bytes. + key (str | bytes | memoryview): db key. """ if isinstance(key, memoryview): # memoryview of bytes @@ -141,48 +148,62 @@ def _tokeys(self, key: Union[str, bytes, memoryview]): return tuple(key.split(self.sep)) - def _ser(self, val: Union[str, memoryview, bytes]): + def _ser(self, val: str | bytes | memoryview): """ Serialize value to bytes to store in db Parameters: - val (Union[str, memoryview, bytes]): encodable as bytes + val (str | bytes | memoryview): encodable as bytes """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # return bytes return (val.encode("utf-8") if hasattr(val, "encode") else val) - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: str | bytes | memoryview): """ Deserialize val to str Parameters: - val (Union[str, memoryview, bytes]): decodable as str + val (str | bytes | memoryview): decodable as str """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes return (val.decode("utf-8") if hasattr(val, "decode") else val) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): """ Returns: - items (Iterator): if (key, val) tuples over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. + keys (Iterator[str | bytes | memoryview]): of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): yield (self._tokeys(key), self._des(val)) - def trim(self, keys: Union[str, Iterable]=b""): + def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): """ Removes all entries whose keys startswith keys. Enables removal of whole branches of db key space. To ensure that proper separation of a branch @@ -190,17 +211,31 @@ def trim(self, keys: Union[str, Iterable]=b""): 'a.1'and 'a.2' but not 'ab' Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (Iterator[str | bytes | memoryview]): of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + Returns: - result (bool): True if key exists so delete successful. False otherwise + result (bool): True if val at key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys))) + return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys, top=top))) class Suber(SuberBase): """ - Sub DB of LMDBer. Subclass of SuberBase + Subclass of SuberBase with no LMDB duplicates (i.e. multiple values at same key). """ def __init__(self, db: dbing.LMDBer, *, @@ -212,7 +247,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key False (default) means do not enable duplicates at - each key + each key. Set to False sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' verify (bool): True means reverify when ._des from db when applicable @@ -294,6 +329,100 @@ def rem(self, keys: Union[str, Iterable]): + + +class OrdSuber(Suber): + """ + Subclass of Suber that adds methods for keys with ordinal numbered suffixes. + Each key consistes of pre joined with .sep to ordinal suffix + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(Suber, self).__init__(*pa, **kwa) + + + def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): + """ + Returns + cnt (int): count of of all ordinal suffix keyed vals with same pre + in key but different on in key in db starting at ordinal number + on of pre where key is formed with onKey(pre,on) + Does not count dups at same on for a given pre, only + unique on at a given pre. + + Parameters: + pre (str | bytes | memoryview): prefix to to be combined with on + to form key + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.cntAllOrdValsPre(db=self.sdb, pre=self._tokey(pre), on=on)) + + # appendOrdPre + + #def appendOrdValPre(self, db, pre, val): + #""" + #Appends val in order after last previous key with same pre in db. + #Returns ordinal number in, on, of appended entry. Appended on is 1 greater + #than previous latest on. + #Uses onKey(pre, on) for entries. + + #Append val to end of db entries with same pre but with on incremented by + #1 relative to last preexisting entry at pre. + + #Parameters: + #db is opened named sub db with dupsort=False + #pre is bytes identifier prefix for event + #val is event digest + #""" + + # getAllOrdItemPreIter + #def getAllOrdItemPreIter(self, db, pre, on=0): + #""" + #Returns iterator of duple item, (on, dig), at each key over all ordinal + #numbered keys with same prefix, pre, in db. Values are sorted by + #onKey(pre, on) where on is ordinal number int. + #Returned items are duples of (on, dig) where on is ordinal number int + #and dig is event digest for lookup in .evts sub db. + + #Raises StopIteration Error when empty. + + #Parameters: + #db is opened named sub db with dupsort=False + #pre is bytes of itdentifier prefix + #on is int ordinal number to resume replay + #""" + + + #def getAllOrdItemAllPreIter(self, db, key=b''): + #""" + #Returns iterator of triple item, (pre, on, dig), at each key over all + #ordinal numbered keys for all prefixes in db. Values are sorted by + #onKey(pre, on) where on is ordinal number int. + #Each returned item is triple (pre, on, dig) where pre is identifier prefix, + #on is ordinal number int and dig is event digest for lookup in .evts sub db. + + #Raises StopIteration Error when empty. + + #Parameters: + #db is opened named sub db with dupsort=False + #key is key location in db to resume replay, + #If empty then start at first key in database + #""" + + class CesrSuberBase(SuberBase): """ Sub class of SuberBase where data is CESR encode/decode ducktyped subclass @@ -524,7 +653,8 @@ def __init__(self, db: dbing.LMDBer, *, super(IoSetSuber, self).__init__(db=db, subkey=subkey, dupsort=False, **kwa) - def put(self, keys: Union[str, Iterable], vals: Iterable): + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Puts all vals at effective key made from keys and hidden ordinal suffix. that are not already in set of vals at key. Does not overwrite. @@ -537,18 +667,21 @@ def put(self, keys: Union[str, Iterable], vals: Iterable): result (bool): True If successful, False otherwise. """ - #if not nonStringIterable(vals): # not iterable - #vals = (vals, ) # make iterable + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putIoSetVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals], sep=self.sep)) - def add(self, keys: Union[str, Iterable], val: Union[bytes, str, memoryview]): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): """ - Add val to vals at effective key made from keys and hidden ordinal suffix. - that is not already in set of vals at key. Does not overwrite. + Add val idempotently to vals at effective key made from keys and hidden + ordinal suffix. Idempotent means that added value is not already in set + of vals at key. Does not overwrite or add same value at same key more + than once. Parameters: keys (Iterable): of key strs to be combined in order to form key @@ -564,8 +697,12 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str, memoryview]): val=self._ser(val), sep=self.sep)) + #def append(self, keys: str | bytes | memoryview | Iterable, + # val: str | bytes | memoryview): + - def pin(self, keys: Union[str, Iterable], vals: Iterable): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Pins (sets) vals at effective key made from keys and hidden ordinal suffix. Overwrites. Removes all pre-existing vals that share same effective keys @@ -581,15 +718,15 @@ def pin(self, keys: Union[str, Iterable], vals: Iterable): """ key = self._tokey(keys) self.db.delIoSetVals(db=self.sdb, key=key) # delete all values - #if not nonStringIterable(vals): # not iterable - #vals = (vals, ) # make iterable + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.setIoSetVals(db=self.sdb, key=key, vals=[self._ser(val) for val in vals], sep=self.sep)) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ Gets vals set list at key made from effective keys @@ -607,7 +744,7 @@ def get(self, keys: Union[str, Iterable]): sep=self.sep)]) - def getLast(self, keys: Union[str, Iterable]): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ Gets last val inserted at effecive key made from keys and hidden ordinal suffix. @@ -624,7 +761,7 @@ def getLast(self, keys: Union[str, Iterable]): - def getIter(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets vals iterator at effecive key made from keys and hidden ordinal suffix. All vals in set of vals that share same effecive key are retrieved in @@ -644,7 +781,7 @@ def getIter(self, keys: Union[str, Iterable]): - def cnt(self, keys: Union[str, Iterable]): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ Return count of values at effective key made from keys and hidden ordinal suffix. Zero otherwise @@ -657,7 +794,8 @@ def cnt(self, keys: Union[str, Iterable]): sep=self.sep)) - def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b''): + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): """ Removes entry at effective key made from keys and hidden ordinal suffix that matches val if any. Otherwise deletes all values at effective key. @@ -684,7 +822,8 @@ def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b'' sep=self.sep) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + *, top=False): """ Return iterator over all the items in top branch defined by keys where keys may be truncation of full branch. @@ -698,18 +837,28 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): Parameters: keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from + a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for iokey, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): key, ion = dbing.unsuffix(iokey, sep=self.sep) yield (self._tokeys(key), self._des(val)) - def getIoSetItem(self, keys: Union[str, Iterable]): + def getIoSetItem(self, keys: str | bytes | memoryview | Iterable): """ Gets (iokeys, val) ioitems list at key made from keys where key is apparent effective key and ioitems all have same apparent effective key @@ -730,7 +879,7 @@ def getIoSetItem(self, keys: Union[str, Iterable]): sep=self.sep)]) - def getIoSetItemIter(self, keys: Union[str, Iterable]): + def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key @@ -752,7 +901,7 @@ def getIoSetItemIter(self, keys: Union[str, Iterable]): yield (self._tokeys(iokey), self._des(val)) - def getIoItemIter(self, keys: Union[str, Iterable]=b""): + def getIoItemIter(self, keys: str | bytes | memoryview | Iterable = b""): """ Returns: items (Iterator): tuple (key, val) over the all the items in @@ -773,13 +922,13 @@ def getIoItemIter(self, keys: Union[str, Iterable]=b""): yield (self._tokeys(iokey), self._des(val)) - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): + def remIokey(self, iokeys: str | bytes | memoryview | Iterable): """ - Removes entry at keys + Removes entries at iokeys Parameters: - iokeys (Iterable): of key str or tuple of key strs to be combined - in order to form key + iokeys (str | bytes | memoryview | Iterable): of key str or + tuple of key strs to be combined in order to form key Returns: result (bool): True if key exists so delete successful. False otherwise @@ -1339,7 +1488,8 @@ def __init__(self, db: Type[dbing.LMDBer], *, super(DupSuber, self).__init__(db=db, subkey=subkey, dupsort=True, **kwa) - def put(self, keys: Union[str, Iterable], vals: list): + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Puts all vals at key made from keys. Does not overwrite. Adds to existing dup values at key if any. Duplicate means another entry at the same key @@ -1348,8 +1498,10 @@ def put(self, keys: Union[str, Iterable], vals: list): unless it is a unique value for that key. Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): str or bytes of each value to be written at key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): str or bytes of each + value to be written at key Returns: result (bool): True If successful, False otherwise. @@ -1357,12 +1509,15 @@ def put(self, keys: Union[str, Iterable], vals: list): Apparently always returns True (how .put works with dupsort=True) """ + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals])) - def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview ): """ Add val to vals at key made from keys. Does not overwrite. Adds to existing dup values at key if any. Duplicate means another entry at the same key @@ -1371,8 +1526,8 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): unless it is a unique value for that key. Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Union[str, bytes]): value + keys (str | bytes | memoryview | Iterable): of key strs to be combined in order to form key + val (str | bytes | memoryview): value Returns: result (bool): True means unique value among duplications, @@ -1384,14 +1539,16 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): val=self._ser(val))) - def pin(self, keys: Union[str, Iterable], vals: list): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Pins (sets) vals at key made from keys. Overwrites. Removes all pre-existing dup vals and replaces them with vals Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): str or bytes values + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): str or bytes values Returns: result (bool): True If successful, False otherwise. @@ -1399,18 +1556,21 @@ def pin(self, keys: Union[str, Iterable], vals: list): """ key = self._tokey(keys) self.db.delVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putVals(db=self.sdb, key=key, vals=[self._ser(val) for val in vals])) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ Gets dup vals list at key made from keys Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key Returns: vals (list): each item in list is str @@ -1421,7 +1581,7 @@ def get(self, keys: Union[str, Iterable]): self.db.getValsIter(db=self.sdb, key=self._tokey(keys))] - def getLast(self, keys: Union[str, Iterable]): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ Gets last dup val at key made from keys @@ -1436,14 +1596,15 @@ def getLast(self, keys: Union[str, Iterable]): return self._des(val) if val is not None else val - def getIter(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets dup vals iterator at key made from keys Duplicates are retrieved in lexocographic order not insertion order. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key Returns: iterator: vals each of str. Raises StopIteration when done @@ -1453,17 +1614,19 @@ def getIter(self, keys: Union[str, Iterable]): yield self._des(val) - def cnt(self, keys: Union[str, Iterable]): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ Return count of dup values at key made from keys, zero otherwise Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key """ return (self.db.cntVals(db=self.sdb, key=self._tokey(keys))) - def rem(self, keys: Union[str, Iterable], val=b''): + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): """ Removes entry at keys @@ -1476,181 +1639,33 @@ def rem(self, keys: Union[str, Iterable], val=b''): result (bool): True if key exists so delete successful. False otherwise """ - return (self.db.delVals(db=self.sdb, key=self._tokey(keys), val=self._ser(val))) + if val: + return (self.db.delVals(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val))) + else: + return (self.db.delVals(db=self.sdb, + key=self._tokey(keys))) + -class CesrDupSuber(DupSuber): +class CesrDupSuber(CesrSuberBase, DupSuber): """ - Sub class of DupSuber that supports multiple entries at each key (duplicates) - with dupsort==True, where data where data is Matter.qb64b property - which is a fully qualified serialization of matter subclass instance - Automatically serializes and deserializes from qb64b to/from Matter instances + Sub class of DupSuber whose values are CESR ducktypes of Matter subclasses. + serialized to and deserialied from val instance .qb64b property + which is a fully qualified serialization. + Automatically serializes and deserializes from qb64b to/from Matter ducktyped + instances + DupSuber supports multiple entries at each key (duplicates) with dupsort==True Do not use if serialized value is greater than 511 bytes. This is a limitation of dupsort==True sub dbs in LMDB """ - def __init__(self, *pa, klas: Type[coring.Matter] = coring.Matter, **kwa): - """ - Parameters: - db (dbing.LMDBer): base db - subkey (str): LMDB sub database key - klas (Type[coring.Matter]): Class reference to subclass of Matter - """ - super(CesrDupSuber, self).__init__(*pa, **kwa) - self.klas = klas - - - def put(self, keys: Union[str, Iterable], vals: list): - """ - Puts all vals at key made from keys. Does not overwrite. Adds to existing - dup values at key if any. Duplicate means another entry at the same key - but the entry is still a unique value. Duplicates are inserted in - lexocographic order not insertion order. Lmdb does not insert a duplicate - unless it is a unique value for that key. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): instances of coring.Matter (subclass) - - Returns: - result (bool): True If successful, False otherwise. - - Apparently always returns True (how .put works with dupsort=True) - - """ - return (self.db.putVals(db=self.sdb, - key=self._tokey(keys), - vals=[val.qb64b for val in vals])) - - - def add(self, keys: Union[str, Iterable], val: coring.Matter): - """ - Add val to vals at key made from keys. Does not overwrite. Adds to existing - dup values at key if any. Duplicate means another entry at the same key - but the entry is still a unique value. Duplicates are inserted in - lexocographic order not insertion order. Lmdb does not insert a duplicate - unless it is a unique value for that key. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (coring.Matter): instance (subclass) - - Returns: - result (bool): True means unique value among duplications, - False means duplicte of same value already exists. - - """ - return (self.db.addVal(db=self.sdb, - key=self._tokey(keys), - val=val.qb64b)) - - - def pin(self, keys: Union[str, Iterable], vals: list): - """ - Pins (sets) vals at key made from keys. Overwrites. Removes all - pre-existing dup vals and replaces them with vals - - Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): instances of coring.Matter (subclass) - - Returns: - result (bool): True If successful, False otherwise. - - """ - key = self._tokey(keys) - self.db.delVals(db=self.sdb, key=key) # delete all values - return (self.db.putVals(db=self.sdb, - key=key, - vals=[val.qb64b for val in vals])) - - - def get(self, keys: Union[str, Iterable]): - """ - Gets dup vals list at key made from keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - vals (list): each item in list is instance of self.klas - empty list if no entry at keys - - """ - return [self.klas(qb64b=bytes(val)) for val in - self.db.getValsIter(db=self.sdb, key=self._tokey(keys))] - - - def getLast(self, keys: Union[str, Iterable]): - """ - Gets last dup val at key made from keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - val (str): instance of self.klas else None if no value at key - - """ - val = self.db.getValLast(db=self.sdb, key=self._tokey(keys)) - if val is not None: - val = self.klas(qb64b=bytes(val)) - return val - - - - def getIter(self, keys: Union[str, Iterable]): - """ - Gets dup vals iterator at key made from keys - - Duplicates are retrieved in lexocographic order not insertion order. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - iterator: vals each of self.klas. Raises StopIteration when done - - """ - for val in self.db.getValsIter(db=self.sdb, key=self._tokey(keys)): - yield self.klas(qb64b=bytes(val)) - - - def rem(self, keys: Union[str, Iterable], val=None): - """ - Removes entry at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (coring.Matter): instance of coring.Matter subclass dup val - at key to delete - if val is None then remove all values at key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - + def __init__(self, *pa, **kwa): """ - if val is not None: - val = val.qb64b - else: - val = b'' - return (self.db.delVals(db=self.sdb, key=self._tokey(keys), val=val)) - + Inherited Parameters: - def getItemIter(self, keys: Union[str, Iterable]=b""): """ - Returns: - iterator (Iteratore: tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + super(CesrDupSuber, self).__init__(*pa, **kwa) - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. - """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(key), self.klas(qb64b=bytes(val))) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index cf2ed805..b8b0e8a7 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntValsAllPre(db=self.tels, pre=pre, on=fn) + return self.cntAllOrdValsPre(db=self.tels, pre=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index c88cca4f..f56cb113 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -377,7 +377,7 @@ def test_lmdber(): assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntValsAllPre(db, preB) == 5 + assert dber.cntAllOrdValsPre(db, preB) == 5 # replay preB events in database items = [item for item in dber.getAllOrdItemPreIter(db, preB)] diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 86e95606..8dd9e994 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -146,6 +146,8 @@ def test_suber(): (('b', '2'), 'Green tree'), (('bc', '3'), 'Red apple')] + + # test with top keys for partial tree topkeys = ("b","") # last element empty to force trailing separator items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] assert items == [(('b', '1'), w), @@ -158,6 +160,20 @@ def test_suber(): (('a', '3'), y), (('a', '4'), z)] + # test with top parameter + keys = ("b", ) # last element empty to force trailing separator + items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + assert items == [(('b', '1'), w), + (('b', '2'), x)] + + keys = ("a", ) # last element empty to force trailing separator + items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + assert items == [(('a', '1'), w), + (('a', '2'), x), + (('a', '3'), y), + (('a', '4'), z)] + + # Test trim assert sdb.trim(keys=("b", "")) items = [(keys, val) for keys, val in sdb.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), @@ -171,6 +187,28 @@ def test_suber(): items = [(keys, val) for keys, val in sdb.getItemIter()] assert items == [(('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] + # Test trim with top parameters + sdb.put(keys=("a","1"), val=w) + sdb.put(keys=("a","2"), val=x) + sdb.put(keys=("a","3"), val=y) + sdb.put(keys=("a","4"), val=z) + sdb.put(keys=("b","1"), val=w) + sdb.put(keys=("b","2"), val=x) + + assert sdb.trim(keys=("b",), top=True) + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('a', '1'), 'Blue dog'), + (('a', '2'), 'Green tree'), + (('a', '3'), 'Red apple'), + (('a', '4'), 'White snow'), + (('ac', '4'), 'White snow'), + (('bc', '3'), 'Red apple')] + + assert sdb.trim(keys=("a",), top=True) + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('ac', '4'), 'White snow'), + (('bc', '3'), 'Red apple')] + assert sdb.trim() items = [(keys, val) for keys, val in sdb.getItemIter()] assert items == [] @@ -191,54 +229,54 @@ def test_dup_suber(): assert db.name == "test" assert db.opened - sdb = subing.DupSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.DupSuber) - assert sdb.sdb.flags()["dupsort"] + dupber = subing.DupSuber(db=db, subkey='bags.') + assert isinstance(dupber, subing.DupSuber) + assert dupber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - sdb.put(keys=keys0, vals=[sue, sal]) - actual = sdb.get(keys=keys0) + dupber.put(keys=keys0, vals=[sue, sal]) + actual = dupber.get(keys=keys0) assert actual == [sue, sal] # lexicographic order - assert sdb.cnt(keys0) == 2 + assert dupber.cnt(keys0) == 2 - sdb.rem(keys0) - actual = sdb.get(keys=keys0) + dupber.rem(keys0) + actual = dupber.get(keys=keys0) assert not actual assert actual == [] - assert sdb.cnt(keys0) == 0 + assert dupber.cnt(keys0) == 0 - sdb.put(keys=keys0, vals=[sal, sue]) - actual = sdb.get(keys=keys0) + dupber.put(keys=keys0, vals=[sal, sue]) + actual = dupber.get(keys=keys0) assert actual == [sue, sal] # lexicographic order - actual = sdb.getLast(keys=keys0) + actual = dupber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = dupber.add(keys=keys0, val=sam) assert result - actual = sdb.get(keys=keys0) + actual = dupber.get(keys=keys0) assert actual == [sam, sue, sal] # lexicographic order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = dupber.pin(keys=keys0, vals=[zoe, zia]) assert result - actual = sdb.get(keys=keys0) + actual = dupber.get(keys=keys0) assert actual == [zia, zoe] # lexi order - sdb.put(keys=keys1, vals=[sal, sue, sam]) - actual = sdb.get(keys=keys1) + dupber.put(keys=keys1, vals=[sal, sue, sam]) + actual = dupber.get(keys=keys1) assert actual == [sam, sue, sal] # lexicographic order - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(dupber.getIter(keys=keys1)): assert val == actual[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in dupber.getItemIter()] assert items == [(('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0001'), 'See ya later.'), (('test_key', '0002'), 'A real charmer!'), @@ -246,9 +284,9 @@ def test_dup_suber(): (('test_key', '0002'), 'Not my type.')] - assert sdb.put(keys=("test", "blue"), vals=[sal, sue, sam]) + assert dupber.put(keys=("test", "blue"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in dupber.getItemIter(keys=topkeys)] assert items == [(('test', 'blue'), 'A real charmer!'), (('test', 'blue'), 'Hello sailer!'), (('test', 'blue'), 'Not my type.')] @@ -256,28 +294,54 @@ def test_dup_suber(): # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - sdb.put(keys=keys2, vals=[bob]) - actual = sdb.get(keys=keys2) + dupber.put(keys=keys2, vals=[bob]) + actual = dupber.get(keys=keys2) assert actual == [bob] - assert sdb.cnt(keys2) == 1 - sdb.rem(keys2) - actual = sdb.get(keys=keys2) + assert dupber.cnt(keys2) == 1 + dupber.rem(keys2) + actual = dupber.get(keys=keys2) assert actual == [] - assert sdb.cnt(keys2) == 0 + assert dupber.cnt(keys2) == 0 - sdb.put(keys=keys2, vals=[bob]) - actual = sdb.get(keys=keys2) + dupber.put(keys=keys2, vals=[bob]) + actual = dupber.get(keys=keys2) assert actual == [bob] bil = "Go away." - sdb.pin(keys=keys2, vals=[bil]) - actual = sdb.get(keys=keys2) + dupber.pin(keys=keys2, vals=[bil]) + actual = dupber.get(keys=keys2) assert actual == [bil] - sdb.add(keys=keys2, val=bob) - actual = sdb.get(keys=keys2) + dupber.add(keys=keys2, val=bob) + actual = dupber.get(keys=keys2) assert actual == [bil, bob] + + # test cntpre + vals = ["hi", "me", "my"] + for i, val in enumerate(vals): + assert dupber.put(keys=dbing.onKey("bob", i), vals=val) + + vals = ["bye", "guy", "gal"] + for i, val in enumerate(vals): + assert dupber.put(keys=dbing.onKey("bob", i), vals=val) + + items = [(keys, val) for keys, val in dupber.getItemIter(keys="bob")] + assert items == [ + (('bob', '00000000000000000000000000000000'), 'bye'), + (('bob', '00000000000000000000000000000000'), 'hi'), + (('bob', '00000000000000000000000000000001'), 'guy'), + (('bob', '00000000000000000000000000000001'), 'me'), + (('bob', '00000000000000000000000000000002'), 'gal'), + (('bob', '00000000000000000000000000000002'), 'my') + ] + + assert dupber.cnt(keys=dbing.onKey("bob", 1)) == 2 + + vals = [val for val in dupber.getIter(keys=dbing.onKey("bob", 2))] + assert vals == ["gal", "my"] + + assert not os.path.exists(db.path) assert not db.opened @@ -350,6 +414,8 @@ def test_ioset_suber(): (('test_key', '0002'), 'A real charmer!')] + + items = list(sdb.getIoItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), @@ -366,6 +432,9 @@ def test_ioset_suber(): assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + + + # Test with top keys assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) topkeys = ("test", "") items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] @@ -373,6 +442,14 @@ def test_ioset_suber(): (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] + # test with top parameter + keys = ("test", ) + items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # IoItems items = list(sdb.getIoItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), From 696ec3553518ccad6fbb7cb323303cc5b8bbcf33 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Aug 2024 15:36:08 -0600 Subject: [PATCH 09/78] refactor the IoDup methods in dbing to name them IoDup consistently --- src/keri/db/basing.py | 178 ++++++++++++++++++++-------------------- src/keri/db/dbing.py | 143 ++++++++++++++++++++++++++------ src/keri/db/subing.py | 60 ++++++++++++++ src/keri/vdr/viring.py | 14 ++-- tests/db/test_dbing.py | 144 ++++++++++++++++---------------- tests/db/test_subing.py | 163 ++++++++++++++++++++++++++++++++++++ 6 files changed, 507 insertions(+), 195 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 5a193788..adda4da9 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2231,7 +2231,7 @@ def putUres(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ures, key, vals) + return self.putIoDupVals(self.ures, key, vals) def addUre(self, key, val): """ @@ -2242,7 +2242,7 @@ def addUre(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ures, key, val) + return self.addIoDupVal(self.ures, key, val) def getUres(self, key): """ @@ -2252,7 +2252,7 @@ def getUres(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ures, key) + return self.getIoDupVals(self.ures, key) def getUresIter(self, key): """ @@ -2262,7 +2262,7 @@ def getUresIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.ures, key) + return self.getIoDupValsIter(self.ures, key) def getUreLast(self, key): """ @@ -2272,7 +2272,7 @@ def getUreLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ures, key) + return self.getIoDupValLast(self.ures, key) def getUreItemsNext(self, key=b'', skip=True): """ @@ -2286,7 +2286,7 @@ def getUreItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.ures, key, skip) + return self.getIoDupItemsNext(self.ures, key, skip) def getUreItemsNextIter(self, key=b'', skip=True): """ @@ -2300,7 +2300,7 @@ def getUreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ures, key, skip) + return self.getIoDupItemsNextIter(self.ures, key, skip) def cntUres(self, key): """ @@ -2308,7 +2308,7 @@ def cntUres(self, key): Return count of receipt triplets at key Returns zero if no entry at key """ - return self.cntIoVals(self.ures, key) + return self.cntIoDupVals(self.ures, key) def delUres(self, key): """ @@ -2316,7 +2316,7 @@ def delUres(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.ures, key) + return self.delIoDupVals(self.ures, key) def delUre(self, key, val): """ @@ -2328,7 +2328,7 @@ def delUre(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ures, key, val) + return self.delIoDupVal(self.ures, key, val) def putVrcs(self, key, vals): """ @@ -2398,7 +2398,7 @@ def putVres(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.vres, key, vals) + return self.putIoDupVals(self.vres, key, vals) def addVre(self, key, val): """ @@ -2409,7 +2409,7 @@ def addVre(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.vres, key, val) + return self.addIoDupVal(self.vres, key, val) def getVres(self, key): """ @@ -2419,7 +2419,7 @@ def getVres(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.vres, key) + return self.getIoDupVals(self.vres, key) def getVresIter(self, key): """ @@ -2429,7 +2429,7 @@ def getVresIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.vres, key) + return self.getIoDupValsIter(self.vres, key) def getVreLast(self, key): """ @@ -2439,7 +2439,7 @@ def getVreLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.vres, key) + return self.getIoDupValLast(self.vres, key) def getVreItemsNext(self, key=b'', skip=True): """ @@ -2453,7 +2453,7 @@ def getVreItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.vres, key, skip) + return self.getIoDupItemsNext(self.vres, key, skip) def getVreItemsNextIter(self, key=b'', skip=True): """ @@ -2467,7 +2467,7 @@ def getVreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.vres, key, skip) + return self.getIoDupItemsNextIter(self.vres, key, skip) def cntVres(self, key): """ @@ -2475,7 +2475,7 @@ def cntVres(self, key): Return count of receipt quinlets at key Returns zero if no entry at key """ - return self.cntIoVals(self.vres, key) + return self.cntIoDupVals(self.vres, key) def delVres(self, key): """ @@ -2483,7 +2483,7 @@ def delVres(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.vres, key) + return self.delIoDupVals(self.vres, key) def delVre(self, key, val): """ @@ -2495,7 +2495,7 @@ def delVre(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.vres, key, val) + return self.delIoDupVal(self.vres, key, val) def putKes(self, key, vals): """ @@ -2505,7 +2505,7 @@ def putKes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.kels, key, vals) + return self.putIoDupVals(self.kels, key, vals) def addKe(self, key, val): """ @@ -2515,7 +2515,7 @@ def addKe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.kels, key, val) + return self.addIoDupVal(self.kels, key, val) def getKes(self, key): """ @@ -2524,7 +2524,7 @@ def getKes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.kels, key) + return self.getIoDupVals(self.kels, key) def getKeLast(self, key): """ @@ -2533,7 +2533,7 @@ def getKeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.kels, key) + return self.getIoDupValLast(self.kels, key) def cntKes(self, key): """ @@ -2541,7 +2541,7 @@ def cntKes(self, key): Return count of dup key event dig val at key Returns zero if no entry at key """ - return self.cntIoVals(self.kels, key) + return self.cntIoDupVals(self.kels, key) def delKes(self, key): """ @@ -2549,7 +2549,7 @@ def delKes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.kels, key) + return self.delIoDupVals(self.kels, key) def getKelIter(self, pre, sn=0): @@ -2632,7 +2632,7 @@ def putPses(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.pses, key, vals) + return self.putIoDupVals(self.pses, key, vals) def addPse(self, key, val): """ @@ -2642,7 +2642,7 @@ def addPse(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.pses, key, val) + return self.addIoDupVal(self.pses, key, val) def getPses(self, key): """ @@ -2651,7 +2651,7 @@ def getPses(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.pses, key) + return self.getIoDupVals(self.pses, key) def getPsesIter(self, key): """ @@ -2660,7 +2660,7 @@ def getPsesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.pses, key) + return self.getIoDupValsIter(self.pses, key) def getPseLast(self, key): """ @@ -2669,7 +2669,7 @@ def getPseLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.pses, key) + return self.getIoDupValLast(self.pses, key) def getPseItemsNext(self, key=b'', skip=True): """ @@ -2681,7 +2681,7 @@ def getPseItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.pses, key, skip) + return self.getIoDupItemsNext(self.pses, key, skip) def getPseItemsNextIter(self, key=b'', skip=True): """ @@ -2693,7 +2693,7 @@ def getPseItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.pses, key, skip) + return self.getIoDupItemsNextIter(self.pses, key, skip) def cntPses(self, key): """ @@ -2701,7 +2701,7 @@ def cntPses(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.pses, key) + return self.cntIoDupVals(self.pses, key) def delPses(self, key): """ @@ -2709,7 +2709,7 @@ def delPses(self, key): Deletes all values at key in db. Returns True If key exists in db Else False """ - return self.delIoVals(self.pses, key) + return self.delIoDupVals(self.pses, key) def delPse(self, key, val): """ @@ -2721,7 +2721,7 @@ def delPse(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.pses, key, val) + return self.delIoDupVal(self.pses, key, val) def putPwes(self, key, vals): @@ -2732,7 +2732,7 @@ def putPwes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.pwes, key, vals) + return self.putIoDupVals(self.pwes, key, vals) def addPwe(self, key, val): """ @@ -2742,7 +2742,7 @@ def addPwe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.pwes, key, val) + return self.addIoDupVal(self.pwes, key, val) def getPwes(self, key): """ @@ -2751,7 +2751,7 @@ def getPwes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.pwes, key) + return self.getIoDupVals(self.pwes, key) def getPwesIter(self, key): """ @@ -2760,7 +2760,7 @@ def getPwesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.pwes, key) + return self.getIoDupValsIter(self.pwes, key) def getPweLast(self, key): """ @@ -2769,7 +2769,7 @@ def getPweLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.pwes, key) + return self.getIoDupValLast(self.pwes, key) def getPweItemsNext(self, key=b'', skip=True): """ @@ -2781,7 +2781,7 @@ def getPweItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.pwes, key, skip) + return self.getIoDupItemsNext(self.pwes, key, skip) def getPweItemsNextIter(self, key=b'', skip=True): """ @@ -2793,7 +2793,7 @@ def getPweItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.pwes, key, skip) + return self.getIoDupItemsNextIter(self.pwes, key, skip) def cntPwes(self, key): """ @@ -2801,7 +2801,7 @@ def cntPwes(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.pwes, key) + return self.cntIoDupVals(self.pwes, key) def delPwes(self, key): """ @@ -2809,7 +2809,7 @@ def delPwes(self, key): Deletes all values at key in db. Returns True If key exists in db Else False """ - return self.delIoVals(self.pwes, key) + return self.delIoDupVals(self.pwes, key) def delPwe(self, key, val): """ @@ -2821,7 +2821,7 @@ def delPwe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.pwes, key, val) + return self.delIoDupVal(self.pwes, key, val) def putUwes(self, key, vals): """ @@ -2832,7 +2832,7 @@ def putUwes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.uwes, key, vals) + return self.putIoDupVals(self.uwes, key, vals) def addUwe(self, key, val): """ @@ -2843,7 +2843,7 @@ def addUwe(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.uwes, key, val) + return self.addIoDupVal(self.uwes, key, val) def getUwes(self, key): """ @@ -2853,7 +2853,7 @@ def getUwes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.uwes, key) + return self.getIoDupVals(self.uwes, key) def getUwesIter(self, key): """ @@ -2863,7 +2863,7 @@ def getUwesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.uwes, key) + return self.getIoDupValsIter(self.uwes, key) def getUweLast(self, key): """ @@ -2873,7 +2873,7 @@ def getUweLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.uwes, key) + return self.getIoDupValLast(self.uwes, key) def getUweItemsNext(self, key=b'', skip=True): """ @@ -2887,7 +2887,7 @@ def getUweItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.uwes, key, skip) + return self.getIoDupItemsNext(self.uwes, key, skip) def getUweItemsNextIter(self, key=b'', skip=True): """ @@ -2901,7 +2901,7 @@ def getUweItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.uwes, key, skip) + return self.getIoDupItemsNextIter(self.uwes, key, skip) def cntUwes(self, key): """ @@ -2909,7 +2909,7 @@ def cntUwes(self, key): Return count of receipt couples at key Returns zero if no entry at key """ - return self.cntIoVals(self.uwes, key) + return self.cntIoDupVals(self.uwes, key) def delUwes(self, key): """ @@ -2917,7 +2917,7 @@ def delUwes(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.uwes, key) + return self.delIoDupVals(self.uwes, key) def delUwe(self, key, val): """ @@ -2929,7 +2929,7 @@ def delUwe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.uwes, key, val) + return self.delIoDupVal(self.uwes, key, val) def putOoes(self, key, vals): """ @@ -2939,7 +2939,7 @@ def putOoes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ooes, key, vals) + return self.putIoDupVals(self.ooes, key, vals) def addOoe(self, key, val): """ @@ -2949,7 +2949,7 @@ def addOoe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ooes, key, val) + return self.addIoDupVal(self.ooes, key, val) def getOoes(self, key): """ @@ -2958,7 +2958,7 @@ def getOoes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ooes, key) + return self.getIoDupVals(self.ooes, key) def getOoeLast(self, key): """ @@ -2967,7 +2967,7 @@ def getOoeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ooes, key) + return self.getIoDupValLast(self.ooes, key) def getOoeItemsNext(self, key=b'', skip=True): """ @@ -2979,7 +2979,7 @@ def getOoeItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.ooes, key, skip) + return self.getIoDupItemsNext(self.ooes, key, skip) def getOoeItemsNextIter(self, key=b'', skip=True): """ @@ -2991,7 +2991,7 @@ def getOoeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ooes, key, skip) + return self.getIoDupItemsNextIter(self.ooes, key, skip) def cntOoes(self, key): """ @@ -2999,7 +2999,7 @@ def cntOoes(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.ooes, key) + return self.cntIoDupVals(self.ooes, key) def delOoes(self, key): """ @@ -3007,7 +3007,7 @@ def delOoes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.ooes, key) + return self.delIoDupVals(self.ooes, key) def delOoe(self, key, val): """ @@ -3020,7 +3020,7 @@ def delOoe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ooes, key, val) + return self.delIoDupVal(self.ooes, key, val) def putQnfs(self, key, vals): """ @@ -3030,7 +3030,7 @@ def putQnfs(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.qnfs, key, vals) + return self.putIoDupVals(self.qnfs, key, vals) def addQnf(self, key, val): """ @@ -3040,7 +3040,7 @@ def addQnf(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.qnfs, key, val) + return self.addIoDupVal(self.qnfs, key, val) def getQnfs(self, key): """ @@ -3049,7 +3049,7 @@ def getQnfs(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.qnfs, key) + return self.getIoDupVals(self.qnfs, key) def getQnfLast(self, key): """ @@ -3058,7 +3058,7 @@ def getQnfLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.qnfs, key) + return self.getIoDupValLast(self.qnfs, key) def getQnfItemsNext(self, key=b'', skip=True): """ @@ -3070,7 +3070,7 @@ def getQnfItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.qnfs, key, skip) + return self.getIoDupItemsNext(self.qnfs, key, skip) def getQnfItemsNextIter(self, key=b'', skip=True): """ @@ -3082,7 +3082,7 @@ def getQnfItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.qnfs, key, skip) + return self.getIoDupItemsNextIter(self.qnfs, key, skip) def cntQnfs(self, key): """ @@ -3090,7 +3090,7 @@ def cntQnfs(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.qnfs, key) + return self.cntIoDupVals(self.qnfs, key) def delQnfs(self, key): """ @@ -3098,7 +3098,7 @@ def delQnfs(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.qnfs, key) + return self.delIoDupVals(self.qnfs, key) def delQnf(self, key, val): """ @@ -3111,7 +3111,7 @@ def delQnf(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.qnfs, key, val) + return self.delIoDupVal(self.qnfs, key, val) def putDes(self, key, vals): """ @@ -3121,7 +3121,7 @@ def putDes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.dels, key, vals) + return self.putIoDupVals(self.dels, key, vals) def addDe(self, key, val): """ @@ -3131,7 +3131,7 @@ def addDe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.dels, key, val) + return self.addIoDupVal(self.dels, key, val) def getDes(self, key): """ @@ -3140,7 +3140,7 @@ def getDes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.dels, key) + return self.getIoDupVals(self.dels, key) def getDeLast(self, key): """ @@ -3150,7 +3150,7 @@ def getDeLast(self, key): Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.dels, key) + return self.getIoDupValLast(self.dels, key) def cntDes(self, key): """ @@ -3158,7 +3158,7 @@ def cntDes(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.dels, key) + return self.cntIoDupVals(self.dels, key) def delDes(self, key): """ @@ -3166,7 +3166,7 @@ def delDes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.dels, key) + return self.delIoDupVals(self.dels, key) def getDelIter(self, pre): """ @@ -3195,7 +3195,7 @@ def putLdes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ldes, key, vals) + return self.putIoDupVals(self.ldes, key, vals) def addLde(self, key, val): """ @@ -3205,7 +3205,7 @@ def addLde(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ldes, key, val) + return self.addIoDupVal(self.ldes, key, val) def getLdes(self, key): """ @@ -3214,7 +3214,7 @@ def getLdes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ldes, key) + return self.getIoDupVals(self.ldes, key) def getLdeLast(self, key): """ @@ -3223,7 +3223,7 @@ def getLdeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ldes, key) + return self.getIoDupValLast(self.ldes, key) def getLdeItemsNext(self, key=b'', skip=True): """ @@ -3235,7 +3235,7 @@ def getLdeItemsNext(self, key=b'', skip=True): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoItemsNext(self.ldes, key, skip) + return self.getIoDupItemsNext(self.ldes, key, skip) def getLdeItemsNextIter(self, key=b'', skip=True): """ @@ -3247,7 +3247,7 @@ def getLdeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ldes, key, skip) + return self.getIoDupItemsNextIter(self.ldes, key, skip) def cntLdes(self, key): """ @@ -3255,7 +3255,7 @@ def cntLdes(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.ldes, key) + return self.cntIoDupVals(self.ldes, key) def delLdes(self, key): """ @@ -3263,7 +3263,7 @@ def delLdes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.ldes, key) + return self.delIoDupVals(self.ldes, key) def delLde(self, key, val): """ @@ -3276,7 +3276,7 @@ def delLde(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ldes, key, val) + return self.delIoDupVal(self.ldes, key, val) class BaserDoer(doing.Doer): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 5d29d242..f5fbed83 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1404,7 +1404,7 @@ def delVals(self, db, key, val=b''): # IoDup class IoVals IoItems # dupsort==True and prepends and strips io val proem to each value. # because dupsort==True values are limited to 511 bytes including proem - def putIoVals(self, db, key, vals): + def putIoDupVals(self, db, key, vals): """ Write each entry from list of bytes vals to key in db in insertion order Adds to existing values at key if any @@ -1413,11 +1413,13 @@ def putIoVals(self, db, key, vals): Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order + all values that makes lexocographic order that same as insertion order. + Duplicates are ordered as a pair of key plus value so prepending proem to each value changes duplicate ordering. Proem is 33 characters long. With 32 character hex string followed by '.' for essentiall unlimited number of values which will be limited by memory. + With prepended proem ordinal must explicity check for duplicate values before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). @@ -1429,7 +1431,7 @@ def putIoVals(self, db, key, vals): """ result = False - dups = set(self.getIoVals(db, key)) #get preexisting dups if any + dups = set(self.getIoDupVals(db, key)) #get preexisting dups if any with self.env.begin(db=db, write=True, buffers=True) as txn: idx = 0 cursor = txn.cursor() @@ -1450,7 +1452,7 @@ def putIoVals(self, db, key, vals): return result - def addIoVal(self, db, key, val): + def addIoDupVal(self, db, key, val): """ Add val bytes as dup in insertion order to key in db Adds to existing values at key if any @@ -1458,32 +1460,47 @@ def addIoVal(self, db, key, val): Actual value written include prepended proem ordinal Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). Parameters: db is opened named sub db with dupsort=False key is bytes of key within sub db's keyspace val is bytes of value to be written """ - return self.putIoVals(db, key, [val]) + return self.putIoDupVals(db, key, [val]) - def getIoVals(self, db, key): + def getIoDupVals(self, db, key): """ Return list of duplicate values at key in db in insertion order Returns empty list if no entry at key Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True @@ -1503,18 +1520,26 @@ def getIoVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoValsIter(self, db, key): + def getIoDupValsIter(self, db, key): """ Return iterator of all duplicate values at key in db in insertion order Raises StopIteration Error when no remaining dup items = empty. Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True @@ -1533,13 +1558,26 @@ def getIoValsIter(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoValLast(self, db, key): + def getIoDupValLast(self, db, key): """ Return last added dup value at key in db in insertion order Returns None no entry at key Removes prepended proem ordinal from val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1558,7 +1596,7 @@ def getIoValLast(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoItemsNext(self, db, key=b"", skip=True): + def getIoDupItemsNext(self, db, key=b"", skip=True): """ Return list of all dup items at next key after key in db in insertion order. Item is (key, val) with proem stripped from val stored in db. @@ -1572,6 +1610,19 @@ def getIoItemsNext(self, db, key=b"", skip=True): Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace or empty string @@ -1592,7 +1643,7 @@ def getIoItemsNext(self, db, key=b"", skip=True): return items - def getIoItemsNextIter(self, db, key=b"", skip=True): + def getIoDupItemsNextIter(self, db, key=b"", skip=True): """ Return iterator of all dup items at next key after key in db in insertion order. Item is (key, val) with proem stripped from val stored in db. @@ -1606,6 +1657,19 @@ def getIoItemsNextIter(self, db, key=b"", skip=True): Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace or empty @@ -1624,11 +1688,24 @@ def getIoItemsNextIter(self, db, key=b"", skip=True): yield (key, val[33:]) # slice off prepended ordering prefix - def cntIoVals(self, db, key): + def cntIoDupVals(self, db, key): """ Return count of dup values at key in db, or zero otherwise Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1646,11 +1723,24 @@ def cntIoVals(self, db, key): return count - def delIoVals(self, db, key): + def delIoDupVals(self, db, key): """ Deletes all values at key in db if key present. Returns True If key exists + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1664,7 +1754,7 @@ def delIoVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def delIoVal(self, db, key, val): + def delIoDupVal(self, db, key, val): """ Deletes dup io val at key in db. Performs strip search to find match. Strips proems and then searches. @@ -1684,8 +1774,7 @@ def delIoVal(self, db, key, val): The problem is that escrow is not fixed buts stuffs gets added and deleted which just adds to the value of the proem. 2**16 is an impossibly large number so the proem will not max out practically. But its not - and elegant solution. So maybe escrows need to use a different approach. - But really didn't want to add another database just for escrows. + an elegant solution. Parameters: db is opened named sub db with dupsort=False diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 54a428f9..bf8755e5 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1374,6 +1374,10 @@ class SchemerSuber(Suber): Sub class of Suber where data is serialized Schemer instance Automatically serializes and deserializes using Schemer methods + ToDo XXXX make this a subclass of SerderSuber since from a ser des interface + Schemer is duck type of Serder, Then can get rid of the redundant put, add, + pin, get etc definitions + """ def __init__(self, *pa, **kwa): @@ -1669,3 +1673,59 @@ def __init__(self, *pa, **kwa): super(CesrDupSuber, self).__init__(*pa, **kwa) +class IoDupSuber(DupSuber): + """ + Sub class of DupSuber that supports Insertion Ordering (IoDup) of duplicates + By automagically prepending and stripping ordinal proem to/from each + duplicate value at a given key. + + IoDupSuber supports insertion ordered multiple entries at each key + (duplicates) with dupsort==True + + Do not use if serialized length key + proem + value, is greater than 511 bytes. + This is a limitation of dupsort==True sub dbs in LMDB + + IoDupSuber may be more performant then IoSetSuber for values that are indices + to other sub dbs that fit the size constraint because LMDB support for + duplicates is more space efficient and code performant. + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + + """ + super(IoDupSuber, self).__init__(*pa, **kwa) + + + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): + """ + Puts all vals at effective key made from keys and hidden ordinal suffix. + that are not already in set of vals at key. Does not overwrite. + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + vals (Iterable): of str serializations + + Returns: + result (bool): True If successful, False otherwise. + + """ + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable + return (self.db.putIoDupVals(db=self.sdb, + key=self._tokey(keys), + vals=[self._ser(val) for val in vals])) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index b8b0e8a7..5a5076e9 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -907,7 +907,7 @@ def putBaks(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.baks, key, vals) + return self.putIoDupVals(self.baks, key, vals) def addBak(self, key, val): @@ -918,7 +918,7 @@ def addBak(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.baks, key, val) + return self.addIoDupVal(self.baks, key, val) def getBaks(self, key): @@ -928,7 +928,7 @@ def getBaks(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.baks, key) + return self.getIoDupVals(self.baks, key) def getBaksIter(self, key): @@ -938,7 +938,7 @@ def getBaksIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.baks, key) + return self.getIoDupValsIter(self.baks, key) def cntBaks(self, key): """ @@ -946,7 +946,7 @@ def cntBaks(self, key): Return count of backer prefixes at key Returns zero if no entry at key """ - return self.cntIoVals(self.baks, key) + return self.cntIoDupVals(self.baks, key) def delBaks(self, key): @@ -955,7 +955,7 @@ def delBaks(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.baks, key) + return self.delIoDupVals(self.baks, key) def delBak(self, key, val): @@ -968,7 +968,7 @@ def delBak(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.baks, key, val) + return self.delIoDupVal(self.baks, key, val) def buildProof(prefixer, seqner, diger, sigers): diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index f56cb113..fb7357a9 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -445,54 +445,54 @@ def test_lmdber(): vals = [b"z", b"m", b"x", b"a"] db = dber.env.open_db(key=b'peep.', dupsort=True) - assert dber.getIoVals(db, key) == [] - assert dber.getIoValLast(db, key) == None - assert dber.cntIoVals(db, key) == 0 - assert dber.delIoVals(db, key) == False - assert dber.putIoVals(db, key, vals) == True - assert dber.getIoVals(db, key) == vals # preserved insertion order - assert dber.cntIoVals(db, key) == len(vals) == 4 - assert dber.getIoValLast(db, key) == vals[-1] - assert dber.putIoVals(db, key, vals=[b'a']) == False # duplicate - assert dber.getIoVals(db, key) == vals # no change - assert dber.addIoVal(db, key, val=b'b') == True - assert dber.addIoVal(db, key, val=b'a') == False - assert dber.getIoVals(db, key) == [b"z", b"m", b"x", b"a", b"b"] - assert [val for val in dber.getIoValsIter(db, key)] == [b"z", b"m", b"x", b"a", b'b'] - assert dber.delIoVals(db, key) == True - assert dber.getIoVals(db, key) == [] - assert dber.putIoVals(db, key, vals) == True + assert dber.getIoDupVals(db, key) == [] + assert dber.getIoDupValLast(db, key) == None + assert dber.cntIoDupVals(db, key) == 0 + assert dber.delIoDupVals(db, key) == False + assert dber.putIoDupVals(db, key, vals) == True + assert dber.getIoDupVals(db, key) == vals # preserved insertion order + assert dber.cntIoDupVals(db, key) == len(vals) == 4 + assert dber.getIoDupValLast(db, key) == vals[-1] + assert dber.putIoDupVals(db, key, vals=[b'a']) == False # duplicate + assert dber.getIoDupVals(db, key) == vals # no change + assert dber.addIoDupVal(db, key, val=b'b') == True + assert dber.addIoDupVal(db, key, val=b'a') == False + assert dber.getIoDupVals(db, key) == [b"z", b"m", b"x", b"a", b"b"] + assert [val for val in dber.getIoDupValsIter(db, key)] == [b"z", b"m", b"x", b"a", b'b'] + assert dber.delIoDupVals(db, key) == True + assert dber.getIoDupVals(db, key) == [] + assert dber.putIoDupVals(db, key, vals) == True for val in vals: - assert dber.delIoVal(db, key, val) - assert dber.getIoVals(db, key) == [] - assert dber.putIoVals(db, key, vals) == True + assert dber.delIoDupVal(db, key, val) + assert dber.getIoDupVals(db, key) == [] + assert dber.putIoDupVals(db, key, vals) == True for val in sorted(vals): - assert dber.delIoVal(db, key, val) - assert dber.getIoVals(db, key) == [] + assert dber.delIoDupVal(db, key, val) + assert dber.getIoDupVals(db, key) == [] #delete and add in odd order - assert dber.putIoVals(db, key, vals) == True - assert dber.delIoVal(db, key, vals[2]) - assert dber.addIoVal(db, key, b'w') - assert dber.delIoVal(db, key, vals[0]) - assert dber.addIoVal(db, key, b'e') - assert dber.getIoVals(db, key) == [b'm', b'a', b'w', b'e'] + assert dber.putIoDupVals(db, key, vals) == True + assert dber.delIoDupVal(db, key, vals[2]) + assert dber.addIoDupVal(db, key, b'w') + assert dber.delIoDupVal(db, key, vals[0]) + assert dber.addIoDupVal(db, key, b'e') + assert dber.getIoDupVals(db, key) == [b'm', b'a', b'w', b'e'] # Test getIoValsAllPreIter(self, db, pre) vals0 = [b"gamma", b"beta"] sn = 0 key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True + assert dber.addIoDupVal(db, key, vals0[0]) == True + assert dber.addIoDupVal(db, key, vals0[1]) == True vals1 = [b"mary", b"peter", b"john", b"paul"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True + assert dber.putIoDupVals(db, key, vals1) == True vals2 = [b"dog", b"cat", b"bird"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True + assert dber.putIoDupVals(db, key, vals2) == True vals = [bytes(val) for val in dber.getIoValsAllPreIter(db, pre)] allvals = vals0 + vals1 + vals2 @@ -503,18 +503,18 @@ def test_lmdber(): vals0 = [b"gamma", b"beta"] sn = 0 key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True + assert dber.addIoDupVal(db, key, vals0[0]) == True + assert dber.addIoDupVal(db, key, vals0[1]) == True vals1 = [b"mary", b"peter", b"john", b"paul"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True + assert dber.putIoDupVals(db, key, vals1) == True vals2 = [b"dog", b"cat", b"bird"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True + assert dber.putIoDupVals(db, key, vals2) == True vals = [bytes(val) for val in dber.getIoValLastAllPreIter(db, pre)] lastvals = [vals0[-1], vals1[-1], vals2[-1]] @@ -525,18 +525,18 @@ def test_lmdber(): vals0 = [b"gamma", b"beta"] sn = 1 # not start at zero key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True + assert dber.addIoDupVal(db, key, vals0[0]) == True + assert dber.addIoDupVal(db, key, vals0[1]) == True vals1 = [b"mary", b"peter", b"john", b"paul"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True + assert dber.putIoDupVals(db, key, vals1) == True vals2 = [b"dog", b"cat", b"bird"] sn += 2 # gap key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True + assert dber.putIoDupVals(db, key, vals2) == True vals = [bytes(val) for val in dber.getIoValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 @@ -553,35 +553,35 @@ def test_lmdber(): dKey = snKey(pre=b'A', sn=7) dVals = [b"k", b"b"] - assert dber.putIoVals(edb, key=aKey, vals=aVals) - assert dber.putIoVals(edb, key=bKey, vals=bVals) - assert dber.putIoVals(edb, key=cKey, vals=cVals) - assert dber.putIoVals(edb, key=dKey, vals=dVals) + assert dber.putIoDupVals(edb, key=aKey, vals=aVals) + assert dber.putIoDupVals(edb, key=bKey, vals=bVals) + assert dber.putIoDupVals(edb, key=cKey, vals=cVals) + assert dber.putIoDupVals(edb, key=dKey, vals=dVals) # Test getIoItemsNext(self, db, key=b"") # aVals - items = dber.getIoItemsNext(edb) # get first key in database + items = dber.getIoDupItemsNext(edb) # get first key in database assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = dber.getIoItemsNext(edb, key=aKey, skip=False) # get aKey in database + items = dber.getIoDupItemsNext(edb, key=aKey, skip=False) # get aKey in database assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = dber.getIoItemsNext(edb, key=aKey) # get bKey in database + items = dber.getIoDupItemsNext(edb, key=aKey) # get bKey in database assert items # not empty ikey = items[0][0] assert ikey == bKey vals = [val for key, val in items] assert vals == bVals - items = dber.getIoItemsNext(edb, key=b'', skip=False) # get first key in database + items = dber.getIoDupItemsNext(edb, key=b'', skip=False) # get first key in database assert items # not empty ikey = items[0][0] assert ikey == aKey @@ -589,7 +589,7 @@ def test_lmdber(): assert vals == aVals # bVals - items = dber.getIoItemsNext(edb, key=ikey) + items = dber.getIoDupItemsNext(edb, key=ikey) assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -597,7 +597,7 @@ def test_lmdber(): assert vals == bVals # cVals - items = dber.getIoItemsNext(edb, key=ikey) + items = dber.getIoDupItemsNext(edb, key=ikey) assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -605,7 +605,7 @@ def test_lmdber(): assert vals == cVals # dVals - items = dber.getIoItemsNext(edb, key=ikey) + items = dber.getIoDupItemsNext(edb, key=ikey) assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -613,75 +613,75 @@ def test_lmdber(): assert vals == dVals # none - items = dber.getIoItemsNext(edb, key=ikey) + items = dber.getIoDupItemsNext(edb, key=ikey) assert items == [] # empty assert not items # Test getIoItemsNextIter(self, db, key=b"") # get dups at first key in database # aVals - items = [item for item in dber.getIoItemsNextIter(edb)] + items = [item for item in dber.getIoDupItemsNextIter(edb)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in dber.getIoItemsNextIter(edb, key=aKey, skip=False)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey, skip=False)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in dber.getIoItemsNextIter(edb, key=aKey)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey vals = [val for key, val in items] assert vals == bVals - items = [item for item in dber.getIoItemsNextIter(edb, key=b'', skip=False)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=b'', skip=False)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True + assert dber.delIoDupVal(edb, ikey, val) == True # bVals - items = [item for item in dber.getIoItemsNextIter(edb, key=ikey)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] assert items # not empty ikey = items[0][0] assert ikey == bKey vals = [val for key, val in items] assert vals == bVals for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True + assert dber.delIoDupVal(edb, ikey, val) == True # cVals - items = [item for item in dber.getIoItemsNextIter(edb, key=ikey)] + items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] assert items # not empty ikey = items[0][0] assert ikey == cKey vals = [val for key, val in items] assert vals == cVals for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True + assert dber.delIoDupVal(edb, ikey, val) == True # dVals - items = [item for item in dber.getIoItemsNext(edb, key=ikey)] + items = [item for item in dber.getIoDupItemsNext(edb, key=ikey)] assert items # not empty ikey = items[0][0] assert ikey == dKey vals = [val for key, val in items] assert vals == dVals for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True + assert dber.delIoDupVal(edb, ikey, val) == True # none - items = [item for item in dber.getIoItemsNext(edb, key=ikey)] + items = [item for item in dber.getIoDupItemsNext(edb, key=ikey)] assert items == [] # empty assert not items @@ -830,21 +830,21 @@ def test_lmdber(): with pytest.raises(KeyError): dber.delVals(db, empty_key) with pytest.raises(KeyError): - dber.putIoVals(db, empty_key, [some_value]) + dber.putIoDupVals(db, empty_key, [some_value]) with pytest.raises(KeyError): - dber.addIoVal(db, empty_key, some_value) + dber.addIoDupVal(db, empty_key, some_value) with pytest.raises(KeyError): - dber.getIoVals(db, empty_key) + dber.getIoDupVals(db, empty_key) with pytest.raises(KeyError): - [_ for _ in dber.getIoValsIter(db, empty_key)] + [_ for _ in dber.getIoDupValsIter(db, empty_key)] with pytest.raises(KeyError): - dber.getIoValLast(db, empty_key) + dber.getIoDupValLast(db, empty_key) with pytest.raises(KeyError): - dber.cntIoVals(db, empty_key) + dber.cntIoDupVals(db, empty_key) with pytest.raises(KeyError): - dber.delIoVals(db, empty_key) + dber.delIoDupVals(db, empty_key) with pytest.raises(KeyError): - dber.delIoVal(db, empty_key, some_value) + dber.delIoDupVal(db, empty_key, some_value) assert not os.path.exists(dber.path) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 8dd9e994..8cd2a312 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -346,6 +346,169 @@ def test_dup_suber(): assert not db.opened +def test_iodup_suber(): + """ + Test IoDupSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + sdb = subing.IoSetSuber(db=db, subkey='bags.') + assert isinstance(sdb, subing.IoSetSuber) + assert not sdb.sdb.flags()["dupsort"] + + sue = "Hello sailer!" + sal = "Not my type." + + keys0 = ("test_key", "0001") + keys1 = ("test_key", "0002") + + assert sdb.put(keys=keys0, vals=[sal, sue]) + actuals = sdb.get(keys=keys0) + assert actuals == [sal, sue] # insertion order not lexicographic + assert sdb.cnt(keys0) == 2 + actual = sdb.getLast(keys=keys0) + assert actual == sue + + assert sdb.rem(keys0) + actuals = sdb.get(keys=keys0) + assert not actuals + assert actuals == [] + assert sdb.cnt(keys0) == 0 + + assert sdb.put(keys=keys0, vals=[sue, sal]) + actuals = sdb.get(keys=keys0) + assert actuals == [sue, sal] # insertion order + actual = sdb.getLast(keys=keys0) + assert actual == sal + + sam = "A real charmer!" + result = sdb.add(keys=keys0, val=sam) + assert result + actuals = sdb.get(keys=keys0) + assert actuals == [sue, sal, sam] # insertion order + + zoe = "See ya later." + zia = "Hey gorgeous!" + + result = sdb.pin(keys=keys0, vals=[zoe, zia]) + assert result + actuals = sdb.get(keys=keys0) + assert actuals == [zoe, zia] # insertion order + + assert sdb.put(keys=keys1, vals=[sal, sue, sam]) + actuals = sdb.get(keys=keys1) + assert actuals == [sal, sue, sam] + + for i, val in enumerate(sdb.getIter(keys=keys1)): + assert val == actuals[i] + + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] + + + + + items = list(sdb.getIoItemIter()) + assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), + (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), + (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), + (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), + (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + + items = sdb.getIoSetItem(keys=keys1) + assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), + (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), + (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + + items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), + (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + + + + # Test with top keys + assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) + topkeys = ("test", "") + items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # test with top parameter + keys = ("test", ) + items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # IoItems + items = list(sdb.getIoItemIter(keys=topkeys)) + assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), + (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), + (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] + + # test remove with a specific val + assert sdb.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!'), + (('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'A real charmer!')] + + assert sdb.trim(keys=("test", "")) + items = [(keys, val) for keys, val in sdb.getItemIter()] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'A real charmer!')] + + for iokeys, val in sdb.getIoItemIter(): + assert sdb.remIokey(iokeys=iokeys) + + assert sdb.cnt(keys=keys0) == 0 + assert sdb.cnt(keys=keys1) == 0 + + + # test with keys as string not tuple + keys2 = "keystr" + bob = "Shove off!" + assert sdb.put(keys=keys2, vals=[bob]) + actuals = sdb.get(keys=keys2) + assert actuals == [bob] + assert sdb.cnt(keys2) == 1 + assert sdb.rem(keys2) + actuals = sdb.get(keys=keys2) + assert actuals == [] + assert sdb.cnt(keys2) == 0 + + assert sdb.put(keys=keys2, vals=[bob]) + actuals = sdb.get(keys=keys2) + assert actuals == [bob] + + bil = "Go away." + assert sdb.pin(keys=keys2, vals=[bil]) + actuals = sdb.get(keys=keys2) + assert actuals == [bil] + + assert sdb.add(keys=keys2, val=bob) + actuals = sdb.get(keys=keys2) + assert actuals == [bil, bob] + + assert not os.path.exists(db.path) + assert not db.opened + + def test_ioset_suber(): """ Test IoSetSuber LMDBer sub database class From c87128d0958f560b0ed44b6b8370fbfa2d4f72c8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Aug 2024 19:15:00 -0600 Subject: [PATCH 10/78] more refactoring clean up technical debt in subing and koming --- src/keri/core/routing.py | 2 +- src/keri/db/dbing.py | 228 ++++++++++++++++---------------- src/keri/db/escrowing.py | 2 +- src/keri/db/koming.py | 48 ++++--- src/keri/db/subing.py | 272 ++++++++++++++++++++++++++++----------- tests/db/test_koming.py | 13 +- tests/db/test_subing.py | 30 +++-- 7 files changed, 372 insertions(+), 223 deletions(-) diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 57d1346a..4f67c9ce 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -470,7 +470,7 @@ def processEscrowReply(self): quadruple (prefixer, seqner, diger, siger) """ - for (route, ion), saider in self.db.rpes.getIoItemIter(): + for (route, ion), saider in self.db.rpes.getFullItemIter(): try: tsgs = eventing.fetchTsgs(db=self.db.ssgs, saider=saider) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index f5fbed83..af146ad7 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1520,11 +1520,11 @@ def getIoDupVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoDupValsIter(self, db, key): + def getIoDupValLast(self, db, key): """ - Return iterator of all duplicate values at key in db in insertion order - Raises StopIteration Error when no remaining dup items = empty. - Removes prepended proem ordinal from each val before returning + Return last added dup value at key in db in insertion order + Returns None no entry at key + Removes prepended proem ordinal from val before returning Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. @@ -1540,7 +1540,6 @@ def getIoDupValsIter(self, db, key): before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). - Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1548,22 +1547,21 @@ def getIoDupValsIter(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - vals = [] + val = None try: - if cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering proem + if cursor.set_key(key): # move to first_dup + if cursor.last_dup(): # move to last_dup + val = cursor.value()[33:] # slice off prepended ordering proem + return val except lmdb.BadValsizeError as ex: raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoDupValLast(self, db, key): + def delIoDupVals(self, db, key): """ - Return last added dup value at key in db in insertion order - Returns None no entry at key - Removes prepended proem ordinal from val before returning - Assumes DB opened with dupsort=True + Deletes all values at key in db if key present. + Returns True If key exists Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to @@ -1583,78 +1581,58 @@ def getIoDupValLast(self, db, key): key is bytes of key within sub db's keyspace """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - val = None + with self.env.begin(db=db, write=True, buffers=True) as txn: try: - if cursor.set_key(key): # move to first_dup - if cursor.last_dup(): # move to last_dup - val = cursor.value()[33:] # slice off prepended ordering proem - return val + return (txn.delete(key)) except lmdb.BadValsizeError as ex: raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoDupItemsNext(self, db, key=b"", skip=True): + def delIoDupVal(self, db, key, val): """ - Return list of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key == b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Returns empty list if no entries at next key after key - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - + Deletes dup io val at key in db. Performs strip search to find match. + Strips proems and then searches. + Returns True if delete else False if val not present Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - + all values that makes lexocographic order that same as insertion order Duplicates are ordered as a pair of key plus value so prepending proem to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited + With 32 character hex string followed by '.' for essentially unlimited number of values which will be limited by memory. - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Does a linear search so not very efficient when not deleting from the front. + This is hack for supporting escrow which needs to delete individual dup. + The problem is that escrow is not fixed buts stuffs gets added and + deleted which just adds to the value of the proem. 2**16 is an impossibly + large number so the proem will not max out practically. But its not + an elegant solution. Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty string - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass + db is opened named sub db with dupsort=False + key is bytes of key within sub db's keyspace + val is bytes of value to be deleted without intersion ordering proem """ - with self.env.begin(db=db, write=False, buffers=True) as txn: + with self.env.begin(db=db, write=True, buffers=True) as txn: cursor = txn.cursor() - items = [] - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - # slice off prepended ordering prefix on value in item - items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] - return items + try: + if cursor.set_key(key): # move to first_dup + for proval in cursor.iternext_dup(): # value with proem + if val == proval[33:]: # strip of proem + return cursor.delete() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + return False - def getIoDupItemsNextIter(self, db, key=b"", skip=True): + def cntIoDupVals(self, db, key): """ - Return iterator of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Raises StopIteration Error when no remaining dup items = empty. - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - + Return count of dup values at key in db, or zero otherwise Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. @@ -1672,25 +1650,27 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): Parameters: db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass + key is bytes of key within sub db's keyspace """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - for key, val in cursor.iternext_dup(keys=True): - yield (key, val[33:]) # slice off prepended ordering prefix + count = 0 + try: + if cursor.set_key(key): # moves to first_dup + count = cursor.count() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + return count - def cntIoDupVals(self, db, key): + + def getIoDupValsIter(self, db, key): """ - Return count of dup values at key in db, or zero otherwise + Return iterator of all duplicate values at key in db in insertion order + Raises StopIteration Error when no remaining dup items = empty. + Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. @@ -1706,6 +1686,7 @@ def cntIoDupVals(self, db, key): before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1713,20 +1694,30 @@ def cntIoDupVals(self, db, key): with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - count = 0 + vals = [] try: if cursor.set_key(key): # moves to first_dup - count = cursor.count() + for val in cursor.iternext_dup(): + yield val[33:] # slice off prepended ordering proem except lmdb.BadValsizeError as ex: raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - return count - def delIoDupVals(self, db, key): + + def getIoDupItemsNext(self, db, key=b"", skip=True): """ - Deletes all values at key in db if key present. - Returns True If key exists + Return list of all dup items at next key after key in db in insertion order. + Item is (key, val) with proem stripped from val stored in db. + If key == b'' then returns list of dup items at first key in db. + If skip is False and key is not empty then returns dup items at key + Returns empty list if no entries at next key after key + + If key is empty then gets io items (key, io value) at first key in db + Use the return key from items as next key for next call to function in + order to iterate through the database + + Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to @@ -1743,56 +1734,67 @@ def delIoDupVals(self, db, key): Parameters: db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace + key is bytes of key within sub db's keyspace or empty string + skip is Boolean If True skips to next key if key is not empty string + Othewise don't skip for first pass """ - with self.env.begin(db=db, write=True, buffers=True) as txn: - try: - return (txn.delete(key)) - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + items = [] + if cursor.set_range(key): # moves to first_dup at key + found = True + if skip and key and cursor.key() == key: # skip to next key + found = cursor.next_nodup() # skip to next key not dup if any + if found: + # slice off prepended ordering prefix on value in item + items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] + return items - def delIoDupVal(self, db, key, val): + def getIoDupItemsNextIter(self, db, key=b"", skip=True): """ - Deletes dup io val at key in db. Performs strip search to find match. - Strips proems and then searches. - Returns True if delete else False if val not present + Return iterator of all dup items at next key after key in db in insertion order. + Item is (key, val) with proem stripped from val stored in db. + If key = b'' then returns list of dup items at first key in db. + If skip is False and key is not empty then returns dup items at key + Raises StopIteration Error when no remaining dup items = empty. + + If key is empty then gets io items (key, io value) at first key in db + Use the return key from items as next key for next call to function in + order to iterate through the database + Assumes DB opened with dupsort=True Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order + all values that makes lexocographic order that same as insertion order. + Duplicates are ordered as a pair of key plus value so prepending proem to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentially unlimited + With 32 character hex string followed by '.' for essentiall unlimited number of values which will be limited by memory. - Does a linear search so not very efficient when not deleting from the front. - This is hack for supporting escrow which needs to delete individual dup. - The problem is that escrow is not fixed buts stuffs gets added and - deleted which just adds to the value of the proem. 2**16 is an impossibly - large number so the proem will not max out practically. But its not - an elegant solution. + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). Parameters: - db is opened named sub db with dupsort=False - key is bytes of key within sub db's keyspace - val is bytes of value to be deleted without intersion ordering proem + db is opened named sub db with dupsort=True + key is bytes of key within sub db's keyspace or empty + skip is Boolean If True skips to next key if key is not empty string + Othewise don't skip for first pass """ - with self.env.begin(db=db, write=True, buffers=True) as txn: + with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - try: - if cursor.set_key(key): # move to first_dup - for proval in cursor.iternext_dup(): # value with proem - if val == proval[33:]: # strip of proem - return cursor.delete() - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - return False + if cursor.set_range(key): # moves to first_dup at key + found = True + if skip and key and cursor.key() == key: # skip to next key + found = cursor.next_nodup() # skip to next key not dup if any + if found: + for key, val in cursor.iternext_dup(keys=True): + yield (key, val[33:]) # slice off prepended ordering prefix def getIoValsAllPreIter(self, db, pre, on=0): diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 19b8432f..1a4ef028 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -79,7 +79,7 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): extype (Type[Exception]): the expected exception type if the message should remain in escrow """ - for (typ, pre, aid, ion), saider in self.escrowdb.getIoItemIter(keys=(typ,)): + for (typ, pre, aid, ion), saider in self.escrowdb.getFullItemIter(keys=(typ,)): try: tsgs = eventing.fetchTsgs(db=self.tigerdb, saider=saider) diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index f69fd8b1..b73310da 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -110,7 +110,34 @@ def _tokeys(self, key: Union[str, bytes, memoryview]): def getItemIter(self, keys: Union[str, Iterable]=b""): + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + + Returns: + items (Iterator): of (key, val) tuples over the all the items in + subdb whose key startswith key made from keys. Keys may be keyspace + prefix to return branches of key space. When keys is empty then + returns all items in subdb + + Parameters: + keys (Iterator): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to get all the items from + multiple branches of the key space. If keys is empty then gets + all items in database. + """ + for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + yield (self._tokeys(key), self.deserializer(val)) + + + def getFullItemIter(self, keys: Union[str, Iterable]=b""): + """Iterator over items in .db that returns full items with subclass + specific special hidden parts shown for debugging or testing. + Returns: items (Iterator): of (key, val) tuples over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace @@ -616,27 +643,6 @@ def getIoSetItemIter(self, keys: Union[str, Iterable]): yield (self._tokeys(iokey), self.deserializer(val)) - def getIoItemIter(self, keys: Union[str, Iterable]=b""): - """ - Returns: - items (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith effective key made from keys. - Keys may be keyspace prefix to return branches of key space. - When keys is empty then returns all items in subdb. - - - Parameters: - keys (Iterable): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(iokey), self.deserializer(val)) - - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): """ Removes entry at iokeys diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index bf8755e5..4c124cda 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -172,13 +172,57 @@ def _des(self, val: str | bytes | memoryview): def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", *, top=False): - """ + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + Returns: items (Iterator[tuple[key,val]]): (key, val) tuples of each item over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace prefix to return branches of key space. When keys is empty then returns all items in subdb + + + Parameters: + keys (Iterator[str | bytes | memoryview]): of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + """ + for key, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): + yield (self._tokeys(key), self._des(val)) + + + def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): + """Iterator over items in .db that returns full items with subclass + specific special hidden parts shown for debugging or testing. + + Returns: + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb. + This is meant to return full parts of items in both keyspace and + valuespace which may be useful in debugging or testing. + Parameters: keys (Iterator[str | bytes | memoryview]): of key parts that may be a truncation of a full keys tuple in in order to address all the @@ -660,8 +704,9 @@ def put(self, keys: str | bytes | memoryview | Iterable, that are not already in set of vals at key. Does not overwrite. Parameters: - keys (Iterable): of key strs to be combined in order to form key - vals (Iterable): of str serializations + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): of str serializations Returns: result (bool): True If successful, False otherwise. @@ -684,8 +729,9 @@ def add(self, keys: str | bytes | memoryview | Iterable, than once. Parameters: - keys (Iterable): of key strs to be combined in order to form key - val (Union[bytes, str]): serialization + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str | bytes | memoryview): serialization Returns: result (bool): True means unique value added among duplications, @@ -697,8 +743,34 @@ def add(self, keys: str | bytes | memoryview | Iterable, val=self._ser(val), sep=self.sep)) - #def append(self, keys: str | bytes | memoryview | Iterable, - # val: str | bytes | memoryview): + + def append(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): + """Append val non-idempotently to insertion ordered set of values all with + the same apparent effective key. If val already in set at key then + after append there will be multiple entries in database with val at key + each with different insertion order (iokey). + Uses hidden ordinal key suffix for insertion ordering. + The suffix is appended and stripped transparently. + + Works by walking backward to find last iokey for key instead of reading + all vals for iokey. + + Returns: + ion (int): hidden insertion ordering ordinal of appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str | bytes | memoryview): serialization + + + """ + return (self.db.appendIoSetVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep)) + def pin(self, keys: str | bytes | memoryview | Iterable, @@ -716,12 +788,10 @@ def pin(self, keys: str | bytes | memoryview | Iterable, result (bool): True If successful, False otherwise. """ - key = self._tokey(keys) - self.db.delIoSetVals(db=self.sdb, key=key) # delete all values if not nonStringIterable(vals): # not iterable vals = (vals, ) # make iterable return (self.db.setIoSetVals(db=self.sdb, - key=key, + key=self._tokey(keys), vals=[self._ser(val) for val in vals], sep=self.sep)) @@ -744,23 +814,6 @@ def get(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)]) - def getLast(self, keys: str | bytes | memoryview | Iterable): - """ - Gets last val inserted at effecive key made from keys and hidden ordinal - suffix. - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - - Returns: - val (str): value str, None if no entry at keys - - """ - val = self.db.getIoSetValLast(db=self.sdb, key=self._tokey(keys)) - return (self._des(val) if val is not None else val) - - - def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets vals iterator at effecive key made from keys and hidden ordinal suffix. @@ -780,18 +833,21 @@ def getIter(self, keys: str | bytes | memoryview | Iterable): yield self._des(val) - - def cnt(self, keys: str | bytes | memoryview | Iterable): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ - Return count of values at effective key made from keys and hidden ordinal - suffix. Zero otherwise + Gets last val inserted at effecive key made from keys and hidden ordinal + suffix. Parameters: keys (Iterable): of key strs to be combined in order to form key + + Returns: + val (str): value str, None if no entry at keys + """ - return (self.db.cntIoSetVals(db=self.sdb, - key=self._tokey(keys), - sep=self.sep)) + val = self.db.getIoSetValLast(db=self.sdb, key=self._tokey(keys)) + return (self._des(val) if val is not None else val) + def rem(self, keys: str | bytes | memoryview | Iterable, @@ -822,6 +878,35 @@ def rem(self, keys: str | bytes | memoryview | Iterable, sep=self.sep) + def remIokey(self, iokeys: str | bytes | memoryview | Iterable): + """ + Removes entries at iokeys + + Parameters: + iokeys (str | bytes | memoryview | Iterable): of key str or + tuple of key strs to be combined in order to form key + + Returns: + result (bool): True if key exists so delete successful. False otherwise + + """ + return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) + + + def cnt(self, keys: str | bytes | memoryview | Iterable): + """ + Return count of values at effective key made from keys and hidden ordinal + suffix. Zero otherwise + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + """ + return (self.db.cntIoSetVals(db=self.sdb, + key=self._tokey(keys), + sep=self.sep)) + + + def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", *, top=False): """ @@ -901,42 +986,6 @@ def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable): yield (self._tokeys(iokey), self._des(val)) - def getIoItemIter(self, keys: str | bytes | memoryview | Iterable = b""): - """ - Returns: - items (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith effective key made from keys. - Keys may be keyspace prefix to return branches of key space. - When keys is empty then returns all items in subdb. - - - Parameters: - keys (Iterable): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(iokey), self._des(val)) - - - def remIokey(self, iokeys: str | bytes | memoryview | Iterable): - """ - Removes entries at iokeys - - Parameters: - iokeys (str | bytes | memoryview | Iterable): of key str or - tuple of key strs to be combined in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - - class CesrIoSetSuber(CesrSuberBase, IoSetSuber): """ Subclass of CesrSuber and IoSetSuber. @@ -1713,8 +1762,9 @@ def __init__(self, *pa, **kwa): def put(self, keys: str | bytes | memoryview | Iterable, vals: str | bytes | memoryview | Iterable): """ - Puts all vals at effective key made from keys and hidden ordinal suffix. - that are not already in set of vals at key. Does not overwrite. + Puts all vals idempotently at key made from keys in insertion order using + hidden ordinal proem. Idempotently means do not put any val in vals that is + already in dup vals at key. Does not overwrite. Parameters: keys (Iterable): of key strs to be combined in order to form key @@ -1729,3 +1779,81 @@ def put(self, keys: str | bytes | memoryview | Iterable, return (self.db.putIoDupVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals])) + + + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): + """ + Add val idempotently at key made from keys in insertion order using hidden + ordinal proem. Idempotently means do not add val that is already in + dup vals at key. Does not overwrite. + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + val (Union[bytes, str]): serialization + + Returns: + result (bool): True means unique value added among duplications, + False means duplicate of same value already exists. + + """ + return (self.db.addIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val))) + + + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): + """ + Pins (sets) vals at key made from keys in insertion order using hidden + ordinal proem. Overwrites. Removes all pre-existing vals that share + same keys and replaces them with vals + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + vals (Iterable): str serializations + + Returns: + result (bool): True If successful, False otherwise. + + """ + key = self._tokey(keys) + self.db.delIoDupVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable + return self.db.putIoDupVals(db=self.sdb, + key=keys, + vals=[self._ser(val) for val in vals]) + + + def get(self, keys: str | bytes | memoryview | Iterable): + """ + Gets vals set list at key made from keys in insertion order using + hidden ordinal proem. + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + + Returns: + vals (Iterable): each item in list is str + empty list if no entry at keys + + """ + return ([self._des(val) for val in + self.db.getIoDupVals(db=self.sdb, key=self._tokey(keys))]) + + + def getLast(self, keys: str | bytes | memoryview | Iterable): + """ + Gets last val inserted at key made from keys in insertion order using + hidden ordinal proem. + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + + Returns: + val (str): value str, None if no entry at keys + + """ + val = self.db.getIoDupValLast(db=self.sdb, key=self._tokey(keys)) + return (self._des(val) if val is not None else val) diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index ae1975e3..fb2e9f1d 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -829,13 +829,13 @@ def __iter__(self): '00000000000000000000000000000000')] i = 0 - for iokeys, end in endDB.getIoItemIter(keys=(cid0, "")): + for iokeys, end in endDB.getFullItemIter(keys=(cid0, "")): assert end == ends[i] assert iokeys == iokeysall[i] i += 1 i = 0 - for iokeys, end in endDB.getIoItemIter(): + for iokeys, end in endDB.getFullItemIter(): assert end == ends[i] assert iokeys == iokeysall[i] i += 1 @@ -850,6 +850,13 @@ def __iter__(self): if __name__ == "__main__": - test_dup_komer() + test_kom_happy_path() test_kom_get_item_iter() + test_put_invalid_dataclass() + test_get_invalid_dataclass() + test_not_found_entity() + test_serialization() + test_custom_serialization() + test_deserialization() + test_dup_komer() test_ioset_komer() diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 8cd2a312..f68723da 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -416,7 +416,7 @@ def test_iodup_suber(): - items = list(sdb.getIoItemIter()) + items = list(sdb.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), @@ -450,7 +450,7 @@ def test_iodup_suber(): (('test', 'pop'), 'A real charmer!')] # IoItems - items = list(sdb.getIoItemIter(keys=topkeys)) + items = list(sdb.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] @@ -473,7 +473,7 @@ def test_iodup_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getIoItemIter(): + for iokeys, val in sdb.getFullItemIter(): assert sdb.remIokey(iokeys=iokeys) assert sdb.cnt(keys=keys0) == 0 @@ -579,7 +579,7 @@ def test_ioset_suber(): - items = list(sdb.getIoItemIter()) + items = list(sdb.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), @@ -613,7 +613,7 @@ def test_ioset_suber(): (('test', 'pop'), 'A real charmer!')] # IoItems - items = list(sdb.getIoItemIter(keys=topkeys)) + items = list(sdb.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] @@ -636,7 +636,7 @@ def test_ioset_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getIoItemIter(): + for iokeys, val in sdb.getFullItemIter(): assert sdb.remIokey(iokeys=iokeys) assert sdb.cnt(keys=keys0) == 0 @@ -668,6 +668,12 @@ def test_ioset_suber(): actuals = sdb.get(keys=keys2) assert actuals == [bil, bob] + # ToDo test with append non idempotent + #assert dber.putIoSetVals(db, key1, vals1) == True + #assert dber.getIoSetVals(db, key1) == vals1 + #assert dber.appendIoSetVal(db, key1, val=b"k") == 4 + #assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] + assert not os.path.exists(db.path) assert not db.opened @@ -825,7 +831,7 @@ def test_cesr_ioset_suber(): ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoItemIter()] + items = [(iokeys, val.qb64) for iokeys, val in sdb.getFullItemIter()] assert items == [ ((*keys1, '00000000000000000000000000000000'), said2), ((*keys1, '00000000000000000000000000000002'), said0), @@ -854,13 +860,13 @@ def test_cesr_ioset_suber(): ] topkeys = (seq0, "") - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoItemIter(keys=topkeys)] + items = [(iokeys, val.qb64) for iokeys, val in sdb.getFullItemIter(keys=topkeys)] assert items == [ ((*keys0, '00000000000000000000000000000000'), said1), ((*keys0, '00000000000000000000000000000001'), said2), ] - for iokeys, val in sdb.getIoItemIter(): + for iokeys, val in sdb.getFullItemIter(): assert sdb.remIokey(iokeys=iokeys) assert sdb.cnt(keys=keys0) == 0 @@ -1496,7 +1502,7 @@ def test_cat_cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoItemIter()] + for iokeys, vals in sdb.getFullItemIter()] assert items == [ (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), @@ -1526,14 +1532,14 @@ def test_cat_cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoItemIter(keys=topkeys)] + for iokeys, vals in sdb.getFullItemIter(keys=topkeys)] assert items == [ (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), ] - for iokeys, val in sdb.getIoItemIter(): + for iokeys, val in sdb.getFullItemIter(): assert sdb.remIokey(iokeys=iokeys) assert sdb.cnt(keys=keys0) == 0 From cf7a71512b0a25bd5690a9c3842bfcd11668dc43 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 26 Aug 2024 20:13:40 -0600 Subject: [PATCH 11/78] even more refactoring cleanup --- src/keri/db/koming.py | 60 +++++++++++++++------------- src/keri/db/subing.py | 86 ++++++++++++++++++++++------------------- tests/db/test_koming.py | 12 ++++++ tests/db/test_subing.py | 6 +++ 4 files changed, 98 insertions(+), 66 deletions(-) diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index b73310da..ca0459a2 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -576,36 +576,29 @@ def rem(self, keys: Union[str, Iterable], val=None): key=self._tokey(keys), sep=self.sep) - - def getItemIter(self, keys: Union[str, Iterable]=b""): - """Get items iterator - Returns: - items (Iterator): of (key, val) tuples over the all the items in - subdb whose effective key startswith key made from keys. - Keys may be keyspace prefix in order to return branches of key space. - When keys is empty then returns all items in subdb. - Returned key in each item has ordinal suffix removed. + def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): + """ + Removes entry at iokeys Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. + iokeys (tuple): of key str or tuple of key strs to be combined in + order to form key + + Returns: + result (bool): True if key exists so delete successful. False otherwise """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self.deserializer(val)) + return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItem(self, keys: Union[str, Iterable]): + def getIoSetItem(self, keys: Union[str, Iterable], *, ion=0): """ Gets (iokeys, val) ioitems list at key made from keys where key is apparent effective key and ioitems all have same apparent effective key Parameters: keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 Returns: ioitems (Iterable): each item in list is tuple (iokeys, val) where each @@ -618,16 +611,18 @@ def getIoSetItem(self, keys: Union[str, Iterable]): return ([(self._tokeys(iokey), self.deserializer(val)) for iokey, val in self.db.getIoSetItems(db=self.sdb, key=self._tokey(keys), + ion=ion, sep=self.sep)]) - def getIoSetItemIter(self, keys: Union[str, Iterable]): + def getIoSetItemIter(self, keys: Union[str, Iterable], *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 Returns: ioitems (Iterator): each item iterated is tuple (iokeys, val) where @@ -639,23 +634,34 @@ def getIoSetItemIter(self, keys: Union[str, Iterable]): """ for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, key=self._tokey(keys), + ion=ion, sep=self.sep): yield (self._tokeys(iokey), self.deserializer(val)) - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): - """ - Removes entry at iokeys - Parameters: - iokeys (tuple): of key str or tuple of key strs to be combined in - order to form key + def getItemIter(self, keys: Union[str, Iterable]=b""): + """Get items iterator Returns: - result (bool): True if key exists so delete successful. False otherwise + items (Iterator): of (key, val) tuples over the all the items in + subdb whose effective key startswith key made from keys. + Keys may be keyspace prefix in order to return branches of key space. + When keys is empty then returns all items in subdb. + Returned key in each item has ordinal suffix removed. + + Parameters: + keys (Iterator): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to get all the items from + multiple branches of the key space. If keys is empty then gets + all items in database. Append "" to end of keys Iterable to + ensure get properly separated top branch key. """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) + for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + key, ion = dbing.unsuffix(iokey, sep=self.sep) + yield (self._tokeys(key), self.deserializer(val)) + diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 4c124cda..827208fe 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -906,50 +906,15 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)) - - def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", - *, top=False): - """ - Return iterator over all the items in top branch defined by keys where - keys may be truncation of full branch. - - Returns: - items (Iterator): of (key, val) tuples over the all the items in - subdb whose effective key startswith key made from keys. - Keys may be keyspace prefix in order to return branches of key space. - When keys is empty then returns all items in subdb. - Returned key in each item has ordinal suffix removed. - - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to address all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. - Either append "" to end of keys Iterable to ensure get properly - separated top branch key or use top=True. - - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self._des(val)) - - - def getIoSetItem(self, keys: str | bytes | memoryview | Iterable): + def getIoSetItem(self, keys: str | bytes | memoryview | Iterable, + *, ion=0): """ Gets (iokeys, val) ioitems list at key made from keys where key is apparent effective key and ioitems all have same apparent effective key Parameters: keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 Returns: ioitems (Iterable): each item in list is tuple (iokeys, val) where each @@ -961,16 +926,19 @@ def getIoSetItem(self, keys: str | bytes | memoryview | Iterable): return ([(self._tokeys(iokey), self._des(val)) for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, key=self._tokey(keys), + ion=ion, sep=self.sep)]) - def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable): + def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable, + *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 Returns: ioitems (Iterator): each item iterated is tuple (iokeys, val) where @@ -982,10 +950,50 @@ def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable): """ for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, key=self._tokey(keys), + ion=ion, sep=self.sep): yield (self._tokeys(iokey), self._des(val)) + + def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + *, top=False): + """ + Return iterator over all the items in top branch defined by keys where + keys may be truncation of full branch. + + Returns: + items (Iterator): of (key, val) tuples over the all the items in + subdb whose effective key startswith key made from keys. + Keys may be keyspace prefix in order to return branches of key space. + When keys is empty then returns all items in subdb. + Returned key in each item has ordinal suffix removed. + + Parameters: + keys (Iterator): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to address all the items from + multiple branches of the key space. If keys is empty then gets + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + """ + for iokey, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): + key, ion = dbing.unsuffix(iokey, sep=self.sep) + yield (self._tokeys(key), self._des(val)) + + + + class CesrIoSetSuber(CesrSuberBase, IoSetSuber): """ Subclass of CesrSuber and IoSetSuber. diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index fb2e9f1d..b5877fab 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -786,6 +786,18 @@ def __iter__(self): assert iokeys == iokeys0[i] i += 1 + i = 1 + for iokeys, end in endDB.getIoSetItem(keys=keys0, ion=i): + assert end == ends[i] + assert iokeys == iokeys0[i] + i += 1 + + i = 1 + for iokeys, end in endDB.getIoSetItemIter(keys=keys0, ion=i): + assert end == ends[i] + assert iokeys == iokeys0[i] + i += 1 + # test getAllItemIter ends = ends + [wit3end] i = 0 diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index f68723da..db8a5efd 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -595,6 +595,12 @@ def test_ioset_suber(): assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + items = sdb.getIoSetItem(keys=keys1, ion=1) + assert items ==[(('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), + (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + + items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0, ion=1)] + assert items == [(('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] # Test with top keys From 90216c328af57aea84371bddb243d168aac1696c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 27 Aug 2024 13:04:30 -0600 Subject: [PATCH 12/78] more clean up of subing --- src/keri/core/routing.py | 2 +- src/keri/db/dbing.py | 6 +- src/keri/db/escrowing.py | 2 +- src/keri/db/subing.py | 92 +++++++---------- tests/db/test_subing.py | 209 ++++++++++++++++++++------------------- 5 files changed, 144 insertions(+), 167 deletions(-) diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 4f67c9ce..1418272d 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -470,7 +470,7 @@ def processEscrowReply(self): quadruple (prefixer, seqner, diger, siger) """ - for (route, ion), saider in self.db.rpes.getFullItemIter(): + for (route,), saider in self.db.rpes.getItemIter(): try: tsgs = eventing.fetchTsgs(db=self.db.ssgs, saider=saider) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index af146ad7..131f6e2f 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -626,7 +626,7 @@ def delTopVal(self, db, key=b''): db (lmdb._Database): instance of named sub db with dupsort==False key (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is - empty then gets all items in database + empty then deletes all items in database Works for both dupsort==False and dupsort==True Because cursor.iternext() advances cursor after returning item its safe @@ -1189,12 +1189,14 @@ def getIoSetItemsIter(self, db, key, *, ion=0, sep=b'.'): at same apparent effective key where each iteration returns tuple (iokey, val). iokey includes the ordinal key suffix. Uses hidden ordinal key suffix for insertion ordering. + Does not work with partial effective key. Raises StopIteration Error when empty. Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key + key (bytes): Apparent effective key. Does not work with partial key + that is not the full effective key sans the ion part. ion (int): starting ordinal value, default 0 """ with self.env.begin(db=db, write=False, buffers=True) as txn: diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 1a4ef028..dcf741d0 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -79,7 +79,7 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): extype (Type[Exception]): the expected exception type if the message should remain in escrow """ - for (typ, pre, aid, ion), saider in self.escrowdb.getFullItemIter(keys=(typ,)): + for (typ, pre, aid, ion), saider in self.escrowdb.getFullItemIter(keys=(typ, '')): try: tsgs = eventing.fetchTsgs(db=self.tigerdb, saider=saider) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 827208fe..fb6edbd6 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -170,32 +170,22 @@ def _des(self, val: str | bytes | memoryview): return (val.decode("utf-8") if hasattr(val, "decode") else val) - def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): - """Iterator over items in .db subclasses that do special hidden transforms - on either the keyspace or valuespace should override this method to hide - hidden parts from the returned items. For example, adding either - a hidden key space suffix or hidden val space proem to ensure insertion - order. Use getFullItemIter instead to return full items with hidden parts - shown for debugging or testing. - - Returns: - items (Iterator[tuple[key,val]]): (key, val) tuples of each item - over the all the items in subdb whose key startswith key made from - keys. Keys may be keyspace prefix to return branches of key space. - When keys is empty then returns all items in subdb - - + def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): + """ + Removes all entries whose keys startswith keys. Enables removal of whole + branches of db key space. To ensure that proper separation of a branch + include empty string as last key in keys. For example ("a","") deletes + 'a.1'and 'a.2' but not 'ab' Parameters: keys (Iterator[str | bytes | memoryview]): of key parts that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. - If keys is empty then gets all items in database. + If keys is empty then trims all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. - top (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. @@ -204,10 +194,11 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", When last item in keys is empty str then will treat as partial ending in sep regardless of top value + + Returns: + result (bool): True if val at key exists so delete successful. False otherwise """ - for key, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): - yield (self._tokeys(key), self._des(val)) + return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys, top=top))) def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", @@ -246,13 +237,22 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", yield (self._tokeys(key), self._des(val)) - def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): - """ - Removes all entries whose keys startswith keys. Enables removal of whole - branches of db key space. To ensure that proper separation of a branch - include empty string as last key in keys. For example ("a","") deletes - 'a.1'and 'a.2' but not 'ab' + def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + *, top=False): + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + + Returns: + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb + + Parameters: keys (Iterator[str | bytes | memoryview]): of key parts that may be @@ -262,6 +262,7 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. + top (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. @@ -270,11 +271,10 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", When last item in keys is empty str then will treat as partial ending in sep regardless of top value - - Returns: - result (bool): True if val at key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys, top=top))) + for key, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): + yield (self._tokeys(key), self._des(val)) class Suber(SuberBase): @@ -374,7 +374,6 @@ def rem(self, keys: Union[str, Iterable]): - class OrdSuber(Suber): """ Subclass of Suber that adds methods for keys with ordinal numbered suffixes. @@ -906,32 +905,7 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)) - def getIoSetItem(self, keys: str | bytes | memoryview | Iterable, - *, ion=0): - """ - Gets (iokeys, val) ioitems list at key made from keys where key is - apparent effective key and ioitems all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - ion (int): starting ordinal value, default 0 - - Returns: - ioitems (Iterable): each item in list is tuple (iokeys, val) where each - iokeys is actual key tuple including hidden suffix and - each val is str - empty list if no entry at keys - - """ - return ([(self._tokeys(iokey), self._des(val)) for iokey, val in - self.db.getIoSetItemsIter(db=self.sdb, - key=self._tokey(keys), - ion=ion, - sep=self.sep)]) - - - def getIoSetItemIter(self, keys: str | bytes | memoryview | Iterable, - *, ion=0): + def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index db8a5efd..2244c100 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -356,9 +356,9 @@ def test_iodup_suber(): assert db.name == "test" assert db.opened - sdb = subing.IoSetSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.IoSetSuber) - assert not sdb.sdb.flags()["dupsort"] + ioduber = subing.IoSetSuber(db=db, subkey='bags.') + assert isinstance(ioduber, subing.IoSetSuber) + assert not ioduber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." @@ -366,47 +366,47 @@ def test_iodup_suber(): keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - assert sdb.put(keys=keys0, vals=[sal, sue]) - actuals = sdb.get(keys=keys0) + assert ioduber.put(keys=keys0, vals=[sal, sue]) + actuals = ioduber.get(keys=keys0) assert actuals == [sal, sue] # insertion order not lexicographic - assert sdb.cnt(keys0) == 2 - actual = sdb.getLast(keys=keys0) + assert ioduber.cnt(keys0) == 2 + actual = ioduber.getLast(keys=keys0) assert actual == sue - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert ioduber.rem(keys0) + actuals = ioduber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert ioduber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[sue, sal]) - actuals = sdb.get(keys=keys0) + assert ioduber.put(keys=keys0, vals=[sue, sal]) + actuals = ioduber.get(keys=keys0) assert actuals == [sue, sal] # insertion order - actual = sdb.getLast(keys=keys0) + actual = ioduber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = ioduber.add(keys=keys0, val=sam) assert result - actuals = sdb.get(keys=keys0) + actuals = ioduber.get(keys=keys0) assert actuals == [sue, sal, sam] # insertion order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = ioduber.pin(keys=keys0, vals=[zoe, zia]) assert result - actuals = sdb.get(keys=keys0) + actuals = ioduber.get(keys=keys0) assert actuals == [zoe, zia] # insertion order - assert sdb.put(keys=keys1, vals=[sal, sue, sam]) - actuals = sdb.get(keys=keys1) + assert ioduber.put(keys=keys1, vals=[sal, sue, sam]) + actuals = ioduber.get(keys=keys1) assert actuals == [sal, sue, sam] - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(ioduber.getIter(keys=keys1)): assert val == actuals[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in ioduber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), @@ -416,48 +416,48 @@ def test_iodup_suber(): - items = list(sdb.getFullItemIter()) + items = list(ioduber.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = sdb.getIoSetItem(keys=keys1) + items = [(iokeys, val) for iokeys, val in ioduber.getIoSetItemIter(keys=keys1)] assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val) for iokeys, val in ioduber.getIoSetItemIter(keys=keys0)] assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] # Test with top keys - assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) + assert ioduber.put(keys=("test", "pop"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=topkeys)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] # test with top parameter keys = ("test", ) - items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys, top=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] # IoItems - items = list(sdb.getFullItemIter(keys=topkeys)) + items = list(ioduber.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] # test remove with a specific val - assert sdb.rem(keys=("test_key", "0002"), val=sue) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert ioduber.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in ioduber.getItemIter()] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!'), @@ -466,43 +466,43 @@ def test_iodup_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - assert sdb.trim(keys=("test", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert ioduber.trim(keys=("test", "")) + items = [(keys, val) for keys, val in ioduber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getFullItemIter(): - assert sdb.remIokey(iokeys=iokeys) + for iokeys, val in ioduber.getFullItemIter(): + assert ioduber.remIokey(iokeys=iokeys) - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 + assert ioduber.cnt(keys=keys0) == 0 + assert ioduber.cnt(keys=keys1) == 0 # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert ioduber.put(keys=keys2, vals=[bob]) + actuals = ioduber.get(keys=keys2) assert actuals == [bob] - assert sdb.cnt(keys2) == 1 - assert sdb.rem(keys2) - actuals = sdb.get(keys=keys2) + assert ioduber.cnt(keys2) == 1 + assert ioduber.rem(keys2) + actuals = ioduber.get(keys=keys2) assert actuals == [] - assert sdb.cnt(keys2) == 0 + assert ioduber.cnt(keys2) == 0 - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert ioduber.put(keys=keys2, vals=[bob]) + actuals = ioduber.get(keys=keys2) assert actuals == [bob] bil = "Go away." - assert sdb.pin(keys=keys2, vals=[bil]) - actuals = sdb.get(keys=keys2) + assert ioduber.pin(keys=keys2, vals=[bil]) + actuals = ioduber.get(keys=keys2) assert actuals == [bil] - assert sdb.add(keys=keys2, val=bob) - actuals = sdb.get(keys=keys2) + assert ioduber.add(keys=keys2, val=bob) + actuals = ioduber.get(keys=keys2) assert actuals == [bil, bob] assert not os.path.exists(db.path) @@ -519,9 +519,9 @@ def test_ioset_suber(): assert db.name == "test" assert db.opened - sdb = subing.IoSetSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.IoSetSuber) - assert not sdb.sdb.flags()["dupsort"] + iosuber = subing.IoSetSuber(db=db, subkey='bags.') + assert isinstance(iosuber, subing.IoSetSuber) + assert not iosuber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." @@ -529,47 +529,47 @@ def test_ioset_suber(): keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - assert sdb.put(keys=keys0, vals=[sal, sue]) - actuals = sdb.get(keys=keys0) + assert iosuber.put(keys=keys0, vals=[sal, sue]) + actuals = iosuber.get(keys=keys0) assert actuals == [sal, sue] # insertion order not lexicographic - assert sdb.cnt(keys0) == 2 - actual = sdb.getLast(keys=keys0) + assert iosuber.cnt(keys0) == 2 + actual = iosuber.getLast(keys=keys0) assert actual == sue - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert iosuber.rem(keys0) + actuals = iosuber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert iosuber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[sue, sal]) - actuals = sdb.get(keys=keys0) + assert iosuber.put(keys=keys0, vals=[sue, sal]) + actuals = iosuber.get(keys=keys0) assert actuals == [sue, sal] # insertion order - actual = sdb.getLast(keys=keys0) + actual = iosuber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = iosuber.add(keys=keys0, val=sam) assert result - actuals = sdb.get(keys=keys0) + actuals = iosuber.get(keys=keys0) assert actuals == [sue, sal, sam] # insertion order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = iosuber.pin(keys=keys0, vals=[zoe, zia]) assert result - actuals = sdb.get(keys=keys0) + actuals = iosuber.get(keys=keys0) assert actuals == [zoe, zia] # insertion order - assert sdb.put(keys=keys1, vals=[sal, sue, sam]) - actuals = sdb.get(keys=keys1) + assert iosuber.put(keys=keys1, vals=[sal, sue, sam]) + actuals = iosuber.get(keys=keys1) assert actuals == [sal, sue, sam] - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(iosuber.getIter(keys=keys1)): assert val == actuals[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), @@ -579,54 +579,54 @@ def test_ioset_suber(): - items = list(sdb.getFullItemIter()) + items = list(iosuber.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = sdb.getIoSetItem(keys=keys1) + items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys1)] assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys0)] assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] - items = sdb.getIoSetItem(keys=keys1, ion=1) + items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys1, ion=1)] assert items ==[(('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0, ion=1)] + items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys0, ion=1)] assert items == [(('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] # Test with top keys - assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) + assert iosuber.put(keys=("test", "pop"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=topkeys)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] # test with top parameter keys = ("test", ) - items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys, top=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] # IoItems - items = list(sdb.getFullItemIter(keys=topkeys)) + items = list(iosuber.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] # test remove with a specific val - assert sdb.rem(keys=("test_key", "0002"), val=sue) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert iosuber.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!'), @@ -635,50 +635,51 @@ def test_ioset_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - assert sdb.trim(keys=("test", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert iosuber.trim(keys=("test", "")) + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getFullItemIter(): - assert sdb.remIokey(iokeys=iokeys) + for iokeys, val in iosuber.getFullItemIter(): + assert iosuber.remIokey(iokeys=iokeys) - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 + assert iosuber.cnt(keys=keys0) == 0 + assert iosuber.cnt(keys=keys1) == 0 # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert iosuber.put(keys=keys2, vals=[bob]) + actuals = iosuber.get(keys=keys2) assert actuals == [bob] - assert sdb.cnt(keys2) == 1 - assert sdb.rem(keys2) - actuals = sdb.get(keys=keys2) + assert iosuber.cnt(keys2) == 1 + assert iosuber.rem(keys2) + actuals = iosuber.get(keys=keys2) assert actuals == [] - assert sdb.cnt(keys2) == 0 + assert iosuber.cnt(keys2) == 0 - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert iosuber.put(keys=keys2, vals=[bob]) + actuals = iosuber.get(keys=keys2) assert actuals == [bob] bil = "Go away." - assert sdb.pin(keys=keys2, vals=[bil]) - actuals = sdb.get(keys=keys2) + assert iosuber.pin(keys=keys2, vals=[bil]) + actuals = iosuber.get(keys=keys2) assert actuals == [bil] - assert sdb.add(keys=keys2, val=bob) - actuals = sdb.get(keys=keys2) + assert iosuber.add(keys=keys2, val=bob) + actuals = iosuber.get(keys=keys2) assert actuals == [bil, bob] # ToDo test with append non idempotent - #assert dber.putIoSetVals(db, key1, vals1) == True - #assert dber.getIoSetVals(db, key1) == vals1 - #assert dber.appendIoSetVal(db, key1, val=b"k") == 4 - #assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] + assert iosuber.trim() # default trims whole database + assert iosuber.put(keys=keys1, vals=[bob, bil]) + assert iosuber.get(keys=keys1) == [bob, bil] + assert iosuber.append(keys=keys1, val=bob) == 2 + assert iosuber.get(keys=keys1) == [bob, bil, bob] assert not os.path.exists(db.path) assert not db.opened @@ -845,7 +846,7 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItem(keys=keys1)] + items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItemIter(keys=keys1)] assert items == [ ((*keys1, '00000000000000000000000000000000'), said2), ((*keys1, '00000000000000000000000000000002'), said0), @@ -1518,7 +1519,7 @@ def test_cat_cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItem(keys=keys1)] + for iokeys, vals in sdb.getIoSetItemIter(keys=keys1)] assert items == [(keys1 + ('00000000000000000000000000000000', ), [sqr2.qb64, dgr2.qb64])] items = [(iokeys, [val.qb64 for val in vals]) From b512984f8d53f14551c888af2bbd9a7eb26c6eea Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 28 Aug 2024 13:20:25 -0600 Subject: [PATCH 13/78] even more refactoring of subing, dbing, basing --- src/keri/app/storing.py | 2 +- src/keri/core/eventing.py | 148 +------------------------------------- src/keri/db/basing.py | 20 ++++-- src/keri/db/dbing.py | 89 +++++++++++++++-------- src/keri/db/koming.py | 26 +------ src/keri/db/subing.py | 2 +- tests/db/test_dbing.py | 31 +++++--- tests/db/test_koming.py | 4 +- 8 files changed, 101 insertions(+), 221 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 297dc97e..b64cf6b6 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -125,7 +125,7 @@ def cloneTopicIter(self, topic, fn=0): if hasattr(topic, 'encode'): topic = topic.encode("utf-8") - for (key, dig) in self.getIoSetItemsIter(self.tpcs, key=topic, ion=fn): + for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): topic, ion = dbing.unsuffix(key) if msg := self.msgs.get(keys=dig): yield ion, topic, msg.encode("utf-8") diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index a412c37d..6b7d16a8 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5527,7 +5527,7 @@ def processEscrowPartialWigs(self): """ key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done + while True: # break when done getPweIoDupItemIter(key=b'') for ekey, edig in self.db.getPweItemsNextIter(key=key): try: pre, sn = splitKeySN(ekey) # get pre and sn from escrow item @@ -5698,151 +5698,7 @@ def processEscrowPartialDels(self): If successful then remove from escrow table """ - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.db.getPweItemsNextIter(key=key): - try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) - if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) - - # check date if expired then remove escrow. - dtb = self.db.getDts(dgkey) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) - - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) - - # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) - if eraw is None: - # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) - - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - - # get sigs - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs - if not sigs: # empty list - # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) - - # get witness signatures (wigs not wits) - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - - if not wigs: # empty list - # wigs maybe empty if not wits or if wits while waiting - # for first witness signature - # which may not arrive until some time after event is fully signed - # so just log for debugging but do not unescrow by raising - # ValidationError - logger.debug("Kevery unescrow wigs: No event wigs yet at." - "dig = %s", bytes(edig)) - - # raise ValidationError("Missing escrowed evt wigs at " - # "dig = {}.".format(bytes(edig))) - - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - - # seal source (delegator issuer if any) - delseqner = delsaider = None - if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): - delseqner, delsaider = couple - #couple = self.db.getUde(dgKey(pre, bytes(edig))) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) - - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) - - # If process does NOT validate wigs then process will attempt - # to re-escrow and then raise MissingWitnessSignatureError - # (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pwe escrow is called by - # Kever.self.escrowPWEvent Which calls - # self.db.addPwe(snKey(pre, sn), serder.digb) - # which in turn will NOT enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pwe db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. - # Assumes that controller signature validation and delegation - # validation will be successful as event would not be in - # partially witnessed escrow unless they had already validated - - except (MissingWitnessSignatureError, MissingDelegationError) as ex: - # still waiting on missing witness sigs or delegation - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) - - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) - - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") - - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey - + pass def processEscrowUnverWitness(self): diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index adda4da9..206506fe 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2573,7 +2573,7 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAllPreIter(self.kels, pre, on=sn) + return self.getIoDupValsAllPreIter(self.kels, pre, on=sn) def getKelBackIter(self, pre, sn=0): @@ -2597,7 +2597,7 @@ def getKelBackIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAllPreBackIter(self.kels, pre, sn) + return self.getIoDupValsAllPreBackIter(self.kels, pre, sn) def getKelLastIter(self, pre, sn=0): @@ -2621,7 +2621,7 @@ def getKelLastIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValLastAllPreIter(self.kels, pre, on=sn) + return self.getIoDupValLastAllPreIter(self.kels, pre, on=sn) def putPses(self, key, vals): @@ -2795,6 +2795,18 @@ def getPweItemsNextIter(self, key=b'', skip=True): """ return self.getIoDupItemsNextIter(self.pwes, key, skip) + def getPweIoDupItemIter(self, key=b''): + """ + Use sgKey() + Return iterator of partial witnessed escrowed event dig items at next key after key. + Items is (key, val) where proem has already been stripped from val + If key is b'' empty then returns dup items at first key. + If skip is False and key is not b'' empty then returns dup items at key + Raises StopIteration Error when empty + Duplicates are retrieved in insertion order. + """ + return self.getIoDupItemsIter(self.pwes, key) + def cntPwes(self, key): """ Use snKey() @@ -3185,7 +3197,7 @@ def getDelIter(self, pre): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAnyPreIter(self.dels, pre) + return self.getIoDupValsAnyPreIter(self.dels, pre) def putLdes(self, key, vals): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 131f6e2f..9f43f753 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1156,33 +1156,7 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return False - def getIoSetItems(self, db, key, *, ion=0, sep=b'.'): - """ - Returns: - items (list): list of tuples (iokey, val) of entries in set of with - same apparent effective key. iokey includes the ordinal key suffix - Uses hidden ordinal key suffix for insertion ordering. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - ion (int): starting ordinal value, default 0 - - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - items = [] - iokey = suffix(key, ion, sep=sep) # start ion th value for key zeroth default - cursor = txn.cursor() - if cursor.set_range(iokey): # move to val at key >= iokey if any - for iokey, val in cursor.iternext(): # get iokey, val at cursor - ckey, cion = unsuffix(iokey, sep=sep) - if ckey != key: # prev entry if any was the last entry for key - break # done - items.append((iokey, val)) # another entry at key - return items - - - def getIoSetItemsIter(self, db, key, *, ion=0, sep=b'.'): + def getIoSetItemIter(self, db, key, *, ion=0, sep=b'.'): """ Returns: items (abc.Iterator): iterator over insertion ordered set of values @@ -1754,6 +1728,7 @@ def getIoDupItemsNext(self, db, key=b"", skip=True): return items + def getIoDupItemsNextIter(self, db, key=b"", skip=True): """ Return iterator of all dup items at next key after key in db in insertion order. @@ -1799,7 +1774,59 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): yield (key, val[33:]) # slice off prepended ordering prefix - def getIoValsAllPreIter(self, db, pre, on=0): + def getIoDupItemIter(self, db, key=b''): + """ + Iterates over branch of db given by key of IoDup items where each value + has 33 byte insertion ordinal number proem (prefixed) with separator. + Automagically removes (strips) proem before returning items. + + Assumes DB opened with dupsort=True + + Returns: + items (abc.Iterator): iterator of (full key, val) tuples of all + dup items over a branch of the db given by top key where returned + full key is full database key for val not truncated top key. + Item is (key, val) with proem stripped from val stored in db. + If key = b'' then returns list of dup items for all keys in db. + + + Because cursor.iternext() advances cursor after returning item its safe + to delete the item within the iteration loop. curson.iternext() works + for both dupsort==False and dupsort==True + + Raises StopIteration Error when empty. + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): truncated top key, a key space prefix to get all the items + from multiple branches of the key space. If top key is + empty then gets all items in database + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + if cursor.set_range(key): # move to val at key >= key if any + for ckey, cval in cursor.iternext(): # get key, val first dup at cursor + ckey = bytes(ckey) + if not ckey.startswith(key): # prev entry if any last in branch + break # done + yield (ckey, cval[33:]) # slice off prepended ordering prefix + return # done raises StopIteration + + + def getIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries with same prefix across all ordinal numbers in increasing order @@ -1835,7 +1862,7 @@ def getIoValsAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt+1) - def getIoValsAllPreBackIter(self, db, pre, on=0): + def getIoDupValsAllPreBackIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries with same prefix across all sequence numbers in decreasing order without gaps @@ -1873,7 +1900,7 @@ def getIoValsAllPreBackIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt-1) - def getIoValLastAllPreIter(self, db, pre, on=0): + def getIoDupValLastAllPreIter(self, db, pre, on=0): """ Returns iterator of last only of dup vals of each key in insertion order for all entries with same prefix across all sequence numbers in increasing order @@ -1908,7 +1935,7 @@ def getIoValLastAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt+1) - def getIoValsAnyPreIter(self, db, pre, on=0): + def getIoDupValsAnyPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for any entries with same prefix across all ordinal numbers in order including gaps diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index ca0459a2..77f0001a 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -591,30 +591,6 @@ def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItem(self, keys: Union[str, Iterable], *, ion=0): - """ - Gets (iokeys, val) ioitems list at key made from keys where key is - apparent effective key and ioitems all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - ion (int): starting ordinal value, default 0 - - Returns: - ioitems (Iterable): each item in list is tuple (iokeys, val) where each - iokeys is actual key tuple including hidden suffix and - each val is str - empty list if no entry at keys - - - """ - return ([(self._tokeys(iokey), self.deserializer(val)) for iokey, val - in self.db.getIoSetItems(db=self.sdb, - key=self._tokey(keys), - ion=ion, - sep=self.sep)]) - - def getIoSetItemIter(self, keys: Union[str, Iterable], *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is @@ -632,7 +608,7 @@ def getIoSetItemIter(self, keys: Union[str, Iterable], *, ion=0): Raises StopIteration when done """ - for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, + for iokey, val in self.db.getIoSetItemIter(db=self.sdb, key=self._tokey(keys), ion=ion, sep=self.sep): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index fb6edbd6..6615d8d8 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -922,7 +922,7 @@ def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): Raises StopIteration when done """ - for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, + for iokey, val in self.db.getIoSetItemIter(db=self.sdb, key=self._tokey(keys), ion=ion, sep=self.sep): diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index fb7357a9..a12864fe 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -494,7 +494,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoValsAllPreIter(db, pre)] + vals = [bytes(val) for val in dber.getIoDupValsAllPreIter(db, pre)] allvals = vals0 + vals1 + vals2 assert vals == allvals @@ -516,7 +516,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoValLastAllPreIter(db, pre)] + vals = [bytes(val) for val in dber.getIoDupValLastAllPreIter(db, pre)] lastvals = [vals0[-1], vals1[-1], vals2[-1]] assert vals == lastvals @@ -538,7 +538,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoValsAnyPreIter(db, pre)] + vals = [bytes(val) for val in dber.getIoDupValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 assert vals == allvals @@ -553,11 +553,25 @@ def test_lmdber(): dKey = snKey(pre=b'A', sn=7) dVals = [b"k", b"b"] + + # Test getIoDupItemIter assert dber.putIoDupVals(edb, key=aKey, vals=aVals) assert dber.putIoDupVals(edb, key=bKey, vals=bVals) assert dber.putIoDupVals(edb, key=cKey, vals=cVals) assert dber.putIoDupVals(edb, key=dKey, vals=dVals) + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x'), + (b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z'), + (b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n'), + (b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + # Test getIoItemsNext(self, db, key=b"") # aVals items = dber.getIoDupItemsNext(edb) # get first key in database @@ -769,18 +783,14 @@ def test_lmdber(): assert dber.appendIoSetVal(db, key1, val=b"k") == 4 assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] - assert dber.getIoSetItems(db, key0) == [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - (b'ABC.ZYX.00000000000000000000000000000001', b'm'), - (b'ABC.ZYX.00000000000000000000000000000002', b'x'), - (b'ABC.ZYX.00000000000000000000000000000003', b'a')] - assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemsIter(db, key0)] == + assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemIter(db, key0)] == [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), (b'ABC.ZYX.00000000000000000000000000000001', b'm'), (b'ABC.ZYX.00000000000000000000000000000002', b'x'), (b'ABC.ZYX.00000000000000000000000000000003', b'a')]) - for iokey, val in dber.getIoSetItemsIter(db, key0): + for iokey, val in dber.getIoSetItemIter(db, key0): assert dber.delIoSetIokey(db, iokey) assert dber.getIoSetVals(db, key0) == [] @@ -811,8 +821,7 @@ def test_lmdber(): dber.cntIoSetVals(db, empty_key) dber.delIoSetVals(db, empty_key) dber.delIoSetVal(db, empty_key, some_value) - dber.getIoSetItems(db, empty_key) - dber.getIoSetItemsIter(db, empty_key) + dber.getIoSetItemIter(db, empty_key) with pytest.raises(KeyError): dber.delIoSetIokey(db, empty_key) with pytest.raises(KeyError): diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index b5877fab..1359bb28 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -775,7 +775,7 @@ def __iter__(self): i = 0 - for iokeys, end in endDB.getIoSetItem(keys=keys0): + for iokeys, end in endDB.getIoSetItemIter(keys=keys0): assert end == ends[i] assert iokeys == iokeys0[i] i += 1 @@ -787,7 +787,7 @@ def __iter__(self): i += 1 i = 1 - for iokeys, end in endDB.getIoSetItem(keys=keys0, ion=i): + for iokeys, end in endDB.getIoSetItemIter(keys=keys0, ion=i): assert end == ends[i] assert iokeys == iokeys0[i] i += 1 From 695189275d898682f7e44e1c4180ab2988a538c6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 28 Aug 2024 16:35:23 -0600 Subject: [PATCH 14/78] added IoDupSuber with unit tests --- src/keri/core/eventing.py | 254 +++++++++++++++++++------------------- src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 182 +++++++++++++++++---------- src/keri/db/subing.py | 133 +++++++++++++++++++- tests/db/test_dbing.py | 78 +++++++++++- tests/db/test_subing.py | 70 ++++++----- 6 files changed, 492 insertions(+), 227 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 6b7d16a8..c662095a 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5526,150 +5526,150 @@ def processEscrowPartialWigs(self): If successful then remove from escrow table """ - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done getPweIoDupItemIter(key=b'') - for ekey, edig in self.db.getPweItemsNextIter(key=key): - try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) - if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) - - # check date if expired then remove escrow. - dtb = self.db.getDts(dgkey) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) - - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) - - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + #key = ekey = b'' # both start same. when not same means escrows found + #while True: # break when done + for ekey, edig in self.db.getPweIoDupItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) + try: + pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) - # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) - if eraw is None: - # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) - # get sigs - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs - if not sigs: # empty list - # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) - # get witness signatures (wigs not wits) - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) - if not wigs: # empty list - # wigs maybe empty if not wits or if wits while waiting - # for first witness signature - # which may not arrive until some time after event is fully signed - # so just log for debugging but do not unescrow by raising - # ValidationError - logger.debug("Kevery unescrow wigs: No event wigs yet at." - "dig = %s", bytes(edig)) + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - # raise ValidationError("Missing escrowed evt wigs at " - # "dig = {}.".format(bytes(edig))) + # get sigs + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) - # seal source (delegator issuer if any) - delseqner = delsaider = None - if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): - delseqner, delsaider = couple - #couple = self.db.getUde(dgKey(pre, bytes(edig))) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + # get witness signatures (wigs not wits) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + if not wigs: # empty list + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) - # If process does NOT validate wigs then process will attempt - # to re-escrow and then raise MissingWitnessSignatureError - # (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pwe escrow is called by - # Kever.self.escrowPWEvent Which calls - # self.db.addPwe(snKey(pre, sn), serder.digb) - # which in turn will NOT enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pwe db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. - # Assumes that controller signature validation and delegation - # validation will be successful as event would not be in - # partially witnessed escrow unless they had already validated + # raise ValidationError("Missing escrowed evt wigs at " + # "dig = {}.".format(bytes(edig))) - except (MissingWitnessSignatureError, MissingDelegationError) as ex: - # still waiting on missing witness sigs or delegation - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple + #couple = self.db.getUde(dgKey(pre, bytes(edig))) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, local=esr.local) + + # If process does NOT validate wigs then process will attempt + # to re-escrow and then raise MissingWitnessSignatureError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pwe escrow is called by + # Kever.self.escrowPWEvent Which calls + # self.db.addPwe(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pwe db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except (MissingWitnessSignatureError, MissingDelegationError) as ex: + # still waiting on missing witness sigs or delegation + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") + + #if ekey == key: # still same so no escrows found on last while iteration + #break + #key = ekey # setup next while iteration, with key after ekey def processEscrowPartialDels(self): """ diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 206506fe..58e32048 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2805,7 +2805,7 @@ def getPweIoDupItemIter(self, key=b''): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsIter(self.pwes, key) + return self.getTopIoDupItemIter(self.pwes, key) def cntPwes(self, key): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 9f43f753..fbdedee1 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -601,7 +601,9 @@ def getTopItemIter(self, db, key=b''): db (lmdb._Database): instance of named sub db with dupsort==False key (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is - empty then gets all items in database + empty then gets all items in database. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() @@ -1156,6 +1158,26 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return False + def delIoSetIokey(self, db, iokey): + """ + Deletes val at at actual iokey that includes ordinal key suffix. + + Returns: + result (bool): True if val was deleted at iokey. False otherwise + if no val at iokey + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + iokey (bytes): actual key with ordinal key suffix + """ + with self.env.begin(db=db, write=True, buffers=True) as txn: + try: + return txn.delete(iokey) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + def getIoSetItemIter(self, db, key, *, ion=0, sep=b'.'): """ Returns: @@ -1185,25 +1207,6 @@ def getIoSetItemIter(self, db, key, *, ion=0, sep=b'.'): return # done raises StopIteration - def delIoSetIokey(self, db, iokey): - """ - Deletes val at at actual iokey that includes ordinal key suffix. - - Returns: - result (bool): True if val was deleted at iokey. False otherwise - if no val at iokey - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - iokey (bytes): actual key with ordinal key suffix - """ - with self.env.begin(db=db, write=True, buffers=True) as txn: - try: - return txn.delete(iokey) - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - # For subdbs that support duplicates at each key (dupsort==True) def putVals(self, db, key, vals): @@ -1496,6 +1499,44 @@ def getIoDupVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + def getIoDupValsIter(self, db, key): + """ + Return iterator of all duplicate values at key in db in insertion order + Raises StopIteration Error when no remaining dup items = empty. + Removes prepended proem ordinal from each val before returning + Assumes DB opened with dupsort=True + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + + + Parameters: + db is opened named sub db with dupsort=True + key is bytes of key within sub db's keyspace + """ + + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + vals = [] + try: + if cursor.set_key(key): # moves to first_dup + for val in cursor.iternext_dup(): + yield val[33:] # slice off prepended ordering proem + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + def getIoDupValLast(self, db, key): """ Return last added dup value at key in db in insertion order @@ -1642,43 +1683,8 @@ def cntIoDupVals(self, db, key): - def getIoDupValsIter(self, db, key): - """ - Return iterator of all duplicate values at key in db in insertion order - Raises StopIteration Error when no remaining dup items = empty. - Removes prepended proem ordinal from each val before returning - Assumes DB opened with dupsort=True - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace - """ - - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - vals = [] - try: - if cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering proem - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoDupItemsNext(self, db, key=b"", skip=True): @@ -1774,7 +1780,7 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): yield (key, val[33:]) # slice off prepended ordering prefix - def getIoDupItemIter(self, db, key=b''): + def getIoDupItemIter(self, db, key=b'', *, ion=0): """ Iterates over branch of db given by key of IoDup items where each value has 33 byte insertion ordinal number proem (prefixed) with separator. @@ -1783,11 +1789,11 @@ def getIoDupItemIter(self, db, key=b''): Assumes DB opened with dupsort=True Returns: - items (abc.Iterator): iterator of (full key, val) tuples of all - dup items over a branch of the db given by top key where returned - full key is full database key for val not truncated top key. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items for all keys in db. + ioitems (Iterator): each item iterated is tuple (keys, ioval) where + each keys is actual keys tuple and each ioval is dup that + includes prefixed insertion ordering proem + empty list if no entry at keys. + Raises StopIteration when done Because cursor.iternext() advances cursor after returning item its safe @@ -1801,6 +1807,9 @@ def getIoDupItemIter(self, db, key=b''): key (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is empty then gets all items in database + ion (int): starting ordinal value, default 0 + + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to @@ -1822,10 +1831,59 @@ def getIoDupItemIter(self, db, key=b''): ckey = bytes(ckey) if not ckey.startswith(key): # prev entry if any last in branch break # done - yield (ckey, cval[33:]) # slice off prepended ordering prefix + cion = int(bytes(cval[:32]), 16) # convert without sep '.' in [33] + if cion < ion: # cion cursor insertion order proem as int + continue # skip dups whose cion is below ion + yield (ckey, cval) # include proem on val return # done raises StopIteration + def getTopIoDupItemIter(self, db, key=b''): + """ + Iterates over top branch of db given by key of IoDup items where each value + has 33 byte insertion ordinal number proem (prefixed) with separator. + Automagically removes (strips) proem before returning items. + + Assumes DB opened with dupsort=True + + Returns: + items (abc.Iterator): iterator of (full key, val) tuples of all + dup items over a branch of the db given by top key where returned + full key is full database key for val not truncated top key. + Item is (key, val) with proem stripped from val stored in db. + If key = b'' then returns list of dup items for all keys in db. + + + Because cursor.iternext() advances cursor after returning item its safe + to delete the item within the iteration loop. curson.iternext() works + for both dupsort==False and dupsort==True + + Raises StopIteration Error when empty. + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): truncated top key, a key space prefix to get all the items + from multiple branches of the key space. If top key is + empty then gets all items in database + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + for key, val in self.getTopItemIter(db=db, key=key): + val = val[33:] # strip proem + yield (key, val) + + def getIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 6615d8d8..2415628d 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -221,6 +221,8 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", If keys is empty then gets all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. top (bool): True means treat as partial key tuple from top branch of @@ -261,6 +263,8 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", If keys is empty then gets all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. top (bool): True means treat as partial key tuple from top branch of @@ -862,7 +866,7 @@ def rem(self, keys: str | bytes | memoryview | Iterable, if val is empty then remove all values at key Returns: - result (bool): True if effective key with val exists so delete successful. + result (bool): True if effective key with val exists so rem successful. False otherwise """ @@ -950,6 +954,8 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. top (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep @@ -1804,14 +1810,14 @@ def pin(self, keys: str | bytes | memoryview | Iterable, if not nonStringIterable(vals): # not iterable vals = (vals, ) # make iterable return self.db.putIoDupVals(db=self.sdb, - key=keys, + key=key, vals=[self._ser(val) for val in vals]) def get(self, keys: str | bytes | memoryview | Iterable): """ - Gets vals set list at key made from keys in insertion order using - hidden ordinal proem. + Gets vals dup list in insertion order using key made from keys and + hidden ordinal proem on dups. Parameters: keys (Iterable): of key strs to be combined in order to form key @@ -1825,6 +1831,24 @@ def get(self, keys: str | bytes | memoryview | Iterable): self.db.getIoDupVals(db=self.sdb, key=self._tokey(keys))]) + def getIter(self, keys: str | bytes | memoryview | Iterable): + """ + Gets vals dup iterator in insertion order using key made from keys and + hidden ordinal proem on dups. + All vals in dups that share same key are retrieved in insertion order. + + Parameters: + keys (str | bytes | memoryview | Iterable): of key parts + + Returns: + vals (Iterator): str values. Raises StopIteration when done + + """ + for val in self.db.getIoDupValsIter(db=self.sdb, + key=self._tokey(keys)): + yield self._des(val) + + def getLast(self, keys: str | bytes | memoryview | Iterable): """ Gets last val inserted at key made from keys in insertion order using @@ -1839,3 +1863,104 @@ def getLast(self, keys: str | bytes | memoryview | Iterable): """ val = self.db.getIoDupValLast(db=self.sdb, key=self._tokey(keys)) return (self._des(val) if val is not None else val) + + + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): + """ + Removes entry at key made from keys and dup val that matches val if any, + notwithstanding hidden ordinal proem. Otherwise deletes all dup values + at key if any. + + Parameters: + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key + + Returns: + result (bool): True if key with dup val exists so rem successful. + False otherwise + + """ + if val: + return self.db.delIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val)) + else: + return self.db.delIoDupVals(db=self.sdb, key=self._tokey(keys)) + + + def cnt(self, keys: str | bytes | memoryview | Iterable): + """ + Return count of dup values at key made from keys with hidden ordinal + proem. Zero otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + """ + return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) + + + def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): + """ + Gets (iokeys, val) ioitems iterator at key made from keys where key is + apparent effective key and items all have same apparent effective key + + Parameters: + keys (Iterable): of key strs to be combined in order to form key + ion (int): starting ordinal value, default 0 + + Returns: + ioitems (Iterator): each item iterated is tuple (keys, ioval) where + each keys is actual keys tuple and each ioval is dup that + includes prefixed insertion ordering proem. + Empty list if no entry at keys. + Raises StopIteration when done + + """ + for key, ioval in self.db.getIoDupItemIter(db=self.sdb, + key=self._tokey(keys), + ion=ion): + yield (self._tokeys(key), self._des(ioval)) + + + def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + *, top=False): + """ + Return iterator over all the items including dup items for all keys + in top branch defined by keys where keys may be truncation of full branch. + + Returns: + items (Iterator): of (key, val) tuples over the all the items in + subdb whose key startswith key made from keys and val has its hidden + dup ordinal proem removed. + Keys may be keyspace prefix in order to return branches of key space. + When keys is empty then returns all items in subdb. + + Parameters: + keys (str | bytes | memoryview | Iterable): key or key parts that + may be a truncation of a full keys tuple in in order to address + all the items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + top (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + """ + for key, val in self.db.getTopItemIter(db=self.sdb, + key=self._tokey(keys, top=top)): + val = val[33:] # strip off proem + yield (self._tokeys(key), self._des(val)) + diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index a12864fe..9680da29 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -554,13 +554,14 @@ def test_lmdber(): dVals = [b"k", b"b"] - # Test getIoDupItemIter + # Test variousItemIter assert dber.putIoDupVals(edb, key=aKey, vals=aVals) assert dber.putIoDupVals(edb, key=bKey, vals=bVals) assert dber.putIoDupVals(edb, key=cKey, vals=cVals) assert dber.putIoDupVals(edb, key=dKey, vals=dVals) - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb)] # default all + # test getTopIoDupItemIter + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb)] # default all assert items == [(b'A.00000000000000000000000000000001', b'z'), (b'A.00000000000000000000000000000001', b'm'), (b'A.00000000000000000000000000000001', b'x'), @@ -572,6 +573,77 @@ def test_lmdber(): (b'A.00000000000000000000000000000007', b'k'), (b'A.00000000000000000000000000000007', b'b')] + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=aKey)] + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x')] + + # dups at bKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=bKey)] + assert items == [(b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z')] + + # dups at cKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=cKey)] + assert items == [(b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n')] + + # dups at dKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=dKey)] + assert items == [(b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + # dups at missing key + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=b"B.")] + assert not items + + + # test getIoDupItemIter + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000000.z'), + (b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), + (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000000.o'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000001.r'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z'), + (b'A.00000000000000000000000000000004', b'00000000000000000000000000000000.h'), + (b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n'), + (b'A.00000000000000000000000000000007', b'00000000000000000000000000000000.k'), + (b'A.00000000000000000000000000000007', b'00000000000000000000000000000001.b')] + + # all keys but ion is non zero + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, ion=1)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), + (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000001.r'), + (b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z'), + (b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n'), + (b'A.00000000000000000000000000000007', b'00000000000000000000000000000001.b')] + + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=aKey, ion=1)] + assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), + (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x')] + + # dups at bKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=bKey, ion=2)] + assert items == [(b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z')] + + # dups at cKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=cKey, ion=1)] + assert items ==[(b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n')] + + # dups at dKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=dKey, ion=3)] + assert not items + + # dups at missing key + items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=b"B.")] + assert not items + + # Test getIoItemsNext(self, db, key=b"") # aVals items = dber.getIoDupItemsNext(edb) # get first key in database @@ -699,6 +771,8 @@ def test_lmdber(): assert items == [] # empty assert not items + + # test IoSetVals insertion order set of vals methods. key0 = b'ABC.ZYX' key1 = b'DEF.WVU' diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 2244c100..ffca9aa9 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -356,9 +356,15 @@ def test_iodup_suber(): assert db.name == "test" assert db.opened - ioduber = subing.IoSetSuber(db=db, subkey='bags.') - assert isinstance(ioduber, subing.IoSetSuber) - assert not ioduber.sdb.flags()["dupsort"] + ioduber = subing.IoDupSuber(db=db, subkey='bags.') + assert isinstance(ioduber, subing.IoDupSuber) + assert ioduber.sdb.flags()["dupsort"] + + sue = "Hello sailer!" + sal = "Not my type." + + keys0 = ("test_key", "0001") + keys1 = ("test_key", "0002") sue = "Hello sailer!" sal = "Not my type." @@ -413,26 +419,28 @@ def test_iodup_suber(): (('test_key', '0002'), 'Hello sailer!'), (('test_key', '0002'), 'A real charmer!')] - - - items = list(ioduber.getFullItemIter()) - assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), - (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), - (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), - (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + assert items == [(('test_key', '0001'), '00000000000000000000000000000000.See ya later.'), + (('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!'), + (('test_key', '0002'), '00000000000000000000000000000000.Not my type.'), + (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), + (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] - items = [(iokeys, val) for iokeys, val in ioduber.getIoSetItemIter(keys=keys1)] - assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), - (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), '00000000000000000000000000000000.Not my type.'), + (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), + (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] - items = [(iokeys, val) for iokeys, val in ioduber.getIoSetItemIter(keys=keys0)] - assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), - (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), '00000000000000000000000000000000.See ya later.'), + (('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!')] + items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys1, ion=1)] + assert items ==[(('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), + (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] + items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys0, ion=1)] + assert items == [(('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!')] # Test with top keys assert ioduber.put(keys=("test", "pop"), vals=[sal, sue, sam]) @@ -451,9 +459,9 @@ def test_iodup_suber(): # IoItems items = list(ioduber.getFullItemIter(keys=topkeys)) - assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), - (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] + assert items == [(('test', 'pop'), '00000000000000000000000000000000.Not my type.'), + (('test', 'pop'), '00000000000000000000000000000001.Hello sailer!'), + (('test', 'pop'), '00000000000000000000000000000002.A real charmer!')] # test remove with a specific val assert ioduber.rem(keys=("test_key", "0002"), val=sue) @@ -473,12 +481,8 @@ def test_iodup_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in ioduber.getFullItemIter(): - assert ioduber.remIokey(iokeys=iokeys) - - assert ioduber.cnt(keys=keys0) == 0 - assert ioduber.cnt(keys=keys1) == 0 - + assert ioduber.cnt(keys=keys0) == 2 + assert ioduber.cnt(keys=keys1) == 2 # test with keys as string not tuple keys2 = "keystr" @@ -505,6 +509,12 @@ def test_iodup_suber(): actuals = ioduber.get(keys=keys2) assert actuals == [bil, bob] + # Test trim + assert ioduber.trim() # default trims whole database + assert ioduber.put(keys=keys1, vals=[bob, bil]) + assert ioduber.get(keys=keys1) == [bob, bil] + + assert not os.path.exists(db.path) assert not db.opened @@ -576,9 +586,6 @@ def test_ioset_suber(): (('test_key', '0002'), 'Hello sailer!'), (('test_key', '0002'), 'A real charmer!')] - - - items = list(iosuber.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), @@ -674,7 +681,7 @@ def test_ioset_suber(): actuals = iosuber.get(keys=keys2) assert actuals == [bil, bob] - # ToDo test with append non idempotent + # Test trim and append assert iosuber.trim() # default trims whole database assert iosuber.put(keys=keys1, vals=[bob, bil]) assert iosuber.get(keys=keys1) == [bob, bil] @@ -2035,6 +2042,7 @@ def test_crypt_signer_suber(): if __name__ == "__main__": test_suber() test_dup_suber() + test_iodup_suber() test_ioset_suber() test_cat_suber() test_cesr_suber() From 1ead2a24c54b7287fb4ecd8a1644bc2e527976e6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 28 Aug 2024 16:59:35 -0600 Subject: [PATCH 15/78] added comments todo --- src/keri/db/subing.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 2415628d..df3e8e6e 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1964,3 +1964,11 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", val = val[33:] # strip off proem yield (self._tokeys(key), self._des(val)) +# ToDo + # these seem like a mixture of OrdSuber and IoDupSuber since ord suber + # has onkey for its key and IoDupSuber has dupsort==True but OrdSuber does + # not. .kels uses these methods + # getIoDupValsAllPreIter(self, db, pre, on=0): + # getIoDupValsAllPreBackIter(self, db, pre, on=0): + # used by .dels but maybe someother method works as well + # getIoDupValsAnyPreIter(self, db, pre, on=0) From f3af009ae6d77187dede3dd08e2cede7bf440ab9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 29 Aug 2024 08:00:20 -0600 Subject: [PATCH 16/78] preliminary refactoring to revise Ordinal Numbered db keys --- src/keri/db/basing.py | 6 +- src/keri/db/dbing.py | 8 +- src/keri/db/subing.py | 204 ++++++++++++++++++++++------------------- src/keri/vdr/viring.py | 4 +- tests/db/test_dbing.py | 30 +++--- 5 files changed, 132 insertions(+), 120 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 58e32048..9278de2f 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1944,7 +1944,7 @@ def appendFe(self, pre, val): pre is bytes identifier prefix for event val is event digest """ - return self.appendOrdValPre(db=self.fels, pre=pre, val=val) + return self.appendOnValPre(db=self.fels, pre=pre, val=val) def getFelItemPreIter(self, pre, fn=0): """ @@ -1961,7 +1961,7 @@ def getFelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOrdItemPreIter(db=self.fels, pre=pre, on=fn) + return self.getAllOnItemPreIter(db=self.fels, pre=pre, on=fn) def getFelItemAllPreIter(self, key=b''): @@ -1980,7 +1980,7 @@ def getFelItemAllPreIter(self, key=b''): key is key location in db to resume replay, If empty then start at first key in database """ - return self.getAllOrdItemAllPreIter(db=self.fels, key=key) + return self.getAllOnItemAllPreIter(db=self.fels, key=key) def putDts(self, key, val): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index fbdedee1..71a76e75 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -657,7 +657,7 @@ def delTopVal(self, db, key=b''): # and use keys with suffic ordinal that is monotonically increasing number part # such as fn where no duplicates allowed at a given (pre, on) - def cntAllOrdValsPre(self, db, pre, on=0): + def cntAllOnValsPre(self, db, pre, on=0): """ Returns (int): count of of all ordinal keyed vals with same pre in key but different on in key in db starting at ordinal number on of pre. @@ -687,7 +687,7 @@ def cntAllOrdValsPre(self, db, pre, on=0): return count - def appendOrdValPre(self, db, pre, val): + def appendOnValPre(self, db, pre, val): """ Appends val in order after last previous key with same pre in db. Returns ordinal number in, on, of appended entry. Appended on is 1 greater @@ -737,7 +737,7 @@ def appendOrdValPre(self, db, pre, val): return on - def getAllOrdItemPreIter(self, db, pre, on=0): + def getAllOnItemPreIter(self, db, pre, on=0): """ Returns iterator of duple item, (on, dig), at each key over all ordinal numbered keys with same prefix, pre, in db. Values are sorted by @@ -765,7 +765,7 @@ def getAllOrdItemPreIter(self, db, pre, on=0): yield (cn, val) # (on, dig) of event - def getAllOrdItemAllPreIter(self, db, key=b''): + def getAllOnItemAllPreIter(self, db, key=b''): """ Returns iterator of triple item, (pre, on, dig), at each key over all ordinal numbered keys for all prefixes in db. Values are sorted by diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index df3e8e6e..3d9c012c 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -378,98 +378,6 @@ def rem(self, keys: Union[str, Iterable]): -class OrdSuber(Suber): - """ - Subclass of Suber that adds methods for keys with ordinal numbered suffixes. - Each key consistes of pre joined with .sep to ordinal suffix - - """ - - def __init__(self, *pa, **kwa): - """ - Inherited Parameters: - db (dbing.LMDBer): base db - subkey (str): LMDB sub database key - dupsort (bool): True means enable duplicates at each key - False (default) means do not enable duplicates at - each key. Set to False - sep (str): separator to convert keys iterator to key bytes for db key - default is self.Sep == '.' - verify (bool): True means reverify when ._des from db when applicable - False means do not reverify. Default False - """ - super(Suber, self).__init__(*pa, **kwa) - - - def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): - """ - Returns - cnt (int): count of of all ordinal suffix keyed vals with same pre - in key but different on in key in db starting at ordinal number - on of pre where key is formed with onKey(pre,on) - Does not count dups at same on for a given pre, only - unique on at a given pre. - - Parameters: - pre (str | bytes | memoryview): prefix to to be combined with on - to form key - on (int): ordinal number used with onKey(pre,on) to form key. - """ - return (self.db.cntAllOrdValsPre(db=self.sdb, pre=self._tokey(pre), on=on)) - - # appendOrdPre - - #def appendOrdValPre(self, db, pre, val): - #""" - #Appends val in order after last previous key with same pre in db. - #Returns ordinal number in, on, of appended entry. Appended on is 1 greater - #than previous latest on. - #Uses onKey(pre, on) for entries. - - #Append val to end of db entries with same pre but with on incremented by - #1 relative to last preexisting entry at pre. - - #Parameters: - #db is opened named sub db with dupsort=False - #pre is bytes identifier prefix for event - #val is event digest - #""" - - # getAllOrdItemPreIter - #def getAllOrdItemPreIter(self, db, pre, on=0): - #""" - #Returns iterator of duple item, (on, dig), at each key over all ordinal - #numbered keys with same prefix, pre, in db. Values are sorted by - #onKey(pre, on) where on is ordinal number int. - #Returned items are duples of (on, dig) where on is ordinal number int - #and dig is event digest for lookup in .evts sub db. - - #Raises StopIteration Error when empty. - - #Parameters: - #db is opened named sub db with dupsort=False - #pre is bytes of itdentifier prefix - #on is int ordinal number to resume replay - #""" - - - #def getAllOrdItemAllPreIter(self, db, key=b''): - #""" - #Returns iterator of triple item, (pre, on, dig), at each key over all - #ordinal numbered keys for all prefixes in db. Values are sorted by - #onKey(pre, on) where on is ordinal number int. - #Each returned item is triple (pre, on, dig) where pre is identifier prefix, - #on is ordinal number int and dig is event digest for lookup in .evts sub db. - - #Raises StopIteration Error when empty. - - #Parameters: - #db is opened named sub db with dupsort=False - #key is key location in db to resume replay, - #If empty then start at first key in database - #""" - - class CesrSuberBase(SuberBase): """ Sub class of SuberBase where data is CESR encode/decode ducktyped subclass @@ -1965,10 +1873,114 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", yield (self._tokeys(key), self._des(val)) # ToDo - # these seem like a mixture of OrdSuber and IoDupSuber since ord suber - # has onkey for its key and IoDupSuber has dupsort==True but OrdSuber does - # not. .kels uses these methods +# OnIoDupSuber methods instead of pre just use key prefix parts top val so +# can work with any key space that has last part has ordinal +# try refactoring these to use cursor.iternext cursor.iterpre cursor.next cursor.prev +# so they work across prefixes without having the extra burden doing repeated +# iter within a given pre dups. This way they could work with or without duplicates +# and then the IoDupIter could just strip the proem whereas OrdSuber would not care +# OnSuber + #Although these are similar to OrdSuber in that they both have ordinal + # not hidden as last part of keys. OnIoDup suber also has proem that + # must be prefixed and stripped so not able to mixin with each other + # in multiple inheritance + # used by .kels # getIoDupValsAllPreIter(self, db, pre, on=0): # getIoDupValsAllPreBackIter(self, db, pre, on=0): - # used by .dels but maybe someother method works as well + # getIoDupValLastAllPreIter(self.kels, pre, on=sn) + # used by .dels # getIoDupValsAnyPreIter(self, db, pre, on=0) + + + + +class OnSuber(Suber): + """ + Subclass of Suber that adds methods for keys with ordinal numbered suffixes. + Each key consistes of pre joined with .sep to ordinal suffix + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(OnSuber, self).__init__(*pa, **kwa) + + + def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): + """ + Returns + cnt (int): count of of all ordinal suffix keyed vals with same pre + in key but different on in key in db starting at ordinal number + on of pre where key is formed with onKey(pre,on) + Does not count dups at same on for a given pre, only + unique on at a given pre. + + Parameters: + pre (str | bytes | memoryview): prefix to to be combined with on + to form key + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.cntAllOnValsPre(db=self.sdb, pre=self._tokey(pre), on=on)) + + # appendOrdPre + + #def appendOrdValPre(self, db, pre, val): + #""" + #Appends val in order after last previous key with same pre in db. + #Returns ordinal number in, on, of appended entry. Appended on is 1 greater + #than previous latest on. + #Uses onKey(pre, on) for entries. + + #Append val to end of db entries with same pre but with on incremented by + #1 relative to last preexisting entry at pre. + + #Parameters: + #db is opened named sub db with dupsort=False + #pre is bytes identifier prefix for event + #val is event digest + #""" + + # getAllOrdItemPreIter + #def getAllOrdItemPreIter(self, db, pre, on=0): + #""" + #Returns iterator of duple item, (on, dig), at each key over all ordinal + #numbered keys with same prefix, pre, in db. Values are sorted by + #onKey(pre, on) where on is ordinal number int. + #Returned items are duples of (on, dig) where on is ordinal number int + #and dig is event digest for lookup in .evts sub db. + + #Raises StopIteration Error when empty. + + #Parameters: + #db is opened named sub db with dupsort=False + #pre is bytes of itdentifier prefix + #on is int ordinal number to resume replay + #""" + + + #def getAllOrdItemAllPreIter(self, db, key=b''): + #""" + #Returns iterator of triple item, (pre, on, dig), at each key over all + #ordinal numbered keys for all prefixes in db. Values are sorted by + #onKey(pre, on) where on is ordinal number int. + #Each returned item is triple (pre, on, dig) where pre is identifier prefix, + #on is ordinal number int and dig is event digest for lookup in .evts sub db. + + #Raises StopIteration Error when empty. + + #Parameters: + #db is opened named sub db with dupsort=False + #key is key location in db to resume replay, + #If empty then start at first key in database + #""" diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 5a5076e9..ebbf600d 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -671,7 +671,7 @@ def getTelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOrdItemPreIter(db=self.tels, pre=pre, on=fn) + return self.getAllOnItemPreIter(db=self.tels, pre=pre, on=fn) def cntTels(self, pre, fn=0): """ @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntAllOrdValsPre(db=self.tels, pre=pre, on=fn) + return self.cntAllOnValsPre(db=self.tels, pre=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 9680da29..7a825ab2 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -322,7 +322,7 @@ def test_lmdber(): # test appendOrdValPre # empty database assert dber.getVal(db, keyB0) == None - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnValPre(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -330,7 +330,7 @@ def test_lmdber(): # earlier pre in database only assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnValPre(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -339,7 +339,7 @@ def test_lmdber(): # earlier and later pre in db but not same pre assert dber.getVal(db, keyA0) == digA assert dber.putVal(db, keyC0, val=digC) == True - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnValPre(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -349,13 +349,13 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None assert dber.getVal(db, keyC0) == digC - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnValPre(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU # earlier pre and later pre and earlier entry for same pre assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOrdValPre(db, preB, digV) + on = dber.appendOnValPre(db, preB, digV) assert on == 1 assert dber.getVal(db, keyB1) == digV @@ -365,49 +365,49 @@ def test_lmdber(): assert dber.delVal(db, keyC0) == True assert dber.getVal(db, keyC0) == None # another value for preB - on = dber.appendOrdValPre(db, preB, digW) + on = dber.appendOnValPre(db, preB, digW) assert on == 2 assert dber.getVal(db, keyB2) == digW # yet another value for preB - on = dber.appendOrdValPre(db, preB, digX) + on = dber.appendOnValPre(db, preB, digX) assert on == 3 assert dber.getVal(db, keyB3) == digX # yet another value for preB - on = dber.appendOrdValPre(db, preB, digY ) + on = dber.appendOnValPre(db, preB, digY ) assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntAllOrdValsPre(db, preB) == 5 + assert dber.cntAllOnValsPre(db, preB) == 5 # replay preB events in database - items = [item for item in dber.getAllOrdItemPreIter(db, preB)] + items = [item for item in dber.getAllOnItemPreIter(db, preB)] assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] # resume replay preB events at on = 3 - items = [item for item in dber.getAllOrdItemPreIter(db, preB, on=3)] + items = [item for item in dber.getAllOnItemPreIter(db, preB, on=3)] assert items == [(3, digX), (4, digY)] # resume replay preB events at on = 5 - items = [item for item in dber.getAllOrdItemPreIter(db, preB, on=5)] + items = [item for item in dber.getAllOnItemPreIter(db, preB, on=5)] assert items == [] # replay all events in database with pre events before and after assert dber.putVal(db, keyA0, val=digA) == True assert dber.putVal(db, keyC0, val=digC) == True - items = [item for item in dber.getAllOrdItemAllPreIter(db)] + items = [item for item in dber.getAllOnItemAllPreIter(db)] assert items == [(preA, 0, digA), (preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY), (preC, 0, digC)] # resume replay all starting at preB on=2 - items = [item for item in dber.getAllOrdItemAllPreIter(db, key=keyB2)] + items = [item for item in dber.getAllOnItemAllPreIter(db, key=keyB2)] assert items == [(preB, 2, digW), (preB, 3, digX), (preB, 4, digY), (preC, 0, digC)] # resume replay all starting at preC on=1 - items = [item for item in dber.getAllOrdItemAllPreIter(db, key=onKey(preC, 1))] + items = [item for item in dber.getAllOnItemAllPreIter(db, key=onKey(preC, 1))] assert items == [] From c86a4c7758baa1e66e00404fc2fc2dde0fb0b4db Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 29 Aug 2024 08:15:33 -0600 Subject: [PATCH 17/78] some comments to remind what to refactor --- src/keri/db/dbing.py | 67 ++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 71a76e75..eacba003 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -107,7 +107,8 @@ def dtKey(pre, dts): dts = dts.encode("utf-8") # convert str to bytes return (b'%s|%s' % (pre, dts)) - +# ToDo right split so key prefix could be top of key space with more than one +# part def splitKey(key, sep=b'.'): """ Returns duple of pre and either dig or on, sn, fn str or dts datetime str by @@ -656,36 +657,8 @@ def delTopVal(self, db, key=b''): # For subdbs with no duplicate values allowed at each key. (dupsort==False) # and use keys with suffic ordinal that is monotonically increasing number part # such as fn where no duplicates allowed at a given (pre, on) - - def cntAllOnValsPre(self, db, pre, on=0): - """ - Returns (int): count of of all ordinal keyed vals with same pre in key - but different on in key in db starting at ordinal number on of pre. - For db with dupsort=False but ordinal number suffix in each key - - Parameters: - db is opened named sub db - pre is bytes of key within sub db's keyspace pre.on - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - count = 0 - if not cursor.set_range(key): # moves to val at key >= key - return count # no values end of db - - for key in cursor.iternext(values=False): # get key only at cursor - try: - cpre, cn = splitKeyON(key) - except ValueError as ex: # not splittable key - break - - if cpre != pre: # prev is now the last event for pre - break # done - count = count+1 - - return count - +# ToDo change pre to top so can use in suber for any top branch key space whose last +# part is ordinal number def appendOnValPre(self, db, pre, val): """ @@ -737,6 +710,38 @@ def appendOnValPre(self, db, pre, val): return on + def cntAllOnValsPre(self, db, pre, on=0): + """ + Returns (int): count of of all ordinal keyed vals with same pre in key + but different on in key in db starting at ordinal number on of pre. + For db with dupsort=False but ordinal number suffix in each key + + Parameters: + db is opened named sub db + pre is bytes of key within sub db's keyspace pre.on + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = onKey(pre, on) # start replay at this enty 0 is earliest + count = 0 + if not cursor.set_range(key): # moves to val at key >= key + return count # no values end of db + + for key in cursor.iternext(values=False): # get key only at cursor + try: + cpre, cn = splitKeyON(key) + except ValueError as ex: # not splittable key + break + + if cpre != pre: # prev is now the last event for pre + break # done + count = count+1 + + return count + + +#Returned item should be (top, on, val) + def getAllOnItemPreIter(self, db, pre, on=0): """ Returns iterator of duple item, (on, dig), at each key over all ordinal From eeadba63438b552a852aa6e8520e1438ff0cd198 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 29 Aug 2024 11:01:53 -0600 Subject: [PATCH 18/78] refactor paramters to top key to reflect more accurately its usage --- src/keri/app/connecting.py | 2 +- src/keri/db/dbing.py | 26 +++++++++++++------------- src/keri/db/koming.py | 8 ++++---- src/keri/db/subing.py | 16 ++++++++-------- tests/db/test_dbing.py | 7 ++++--- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/keri/app/connecting.py b/src/keri/app/connecting.py index 7ab81dd1..e6a2f545 100644 --- a/src/keri/app/connecting.py +++ b/src/keri/app/connecting.py @@ -201,7 +201,7 @@ def setImg(self, pre, typ, stream): stream (file): file-like stream of image data """ - self.hby.db.delTopVal(db=self.hby.db.imgs, key=pre.encode("utf-8")) + self.hby.db.delTopVal(db=self.hby.db.imgs, top=pre.encode("utf-8")) key = f"{pre}.content-type".encode("utf-8") self.hby.db.setVal(db=self.hby.db.imgs, key=key, val=typ.encode("utf-8")) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index eacba003..22100e3b 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -128,9 +128,9 @@ def splitKey(key, sep=b'.'): else: if hasattr(sep, 'encode'): # make sep match bytes or str sep = sep.encode("utf-8") - splits = key.split(sep) + splits = key.rsplit(sep, 1) if len(splits) != 2: - raise ValueError("Unsplittable key = {}".format(key)) + raise ValueError(f"Unsplittable {key=} at {sep=}.") return tuple(splits) @@ -550,7 +550,7 @@ def cnt(self, db): count += 1 return count - + # Suber subclasses don't need this since they always split keys int tuple to return def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): """ Returns iterator of item duple (key, val), at each key over all @@ -583,9 +583,9 @@ def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): yield tuple(splits) - def getTopItemIter(self, db, key=b''): + def getTopItemIter(self, db, top=b''): """ - Iterates over branch of db given by key + Iterates over branch of db given by top key Returns: items (abc.Iterator): iterator of (full key, val) tuples over a @@ -600,7 +600,7 @@ def getTopItemIter(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is empty then gets all items in database. In Python str.startswith('') always returns True so if branch @@ -608,16 +608,16 @@ def getTopItemIter(self, db, key=b''): """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any + if cursor.set_range(top): # move to val at key >= key if any for ckey, cval in cursor.iternext(): # get key, val at cursor ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch + if not ckey.startswith(top): # prev entry if any last in branch break # done yield (ckey, cval) # another entry in branch startswith key return # done raises StopIteration - def delTopVal(self, db, key=b''): + def delTopVal(self, db, top=b''): """ Deletes all values in branch of db given top key. @@ -627,7 +627,7 @@ def delTopVal(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is empty then deletes all items in database @@ -643,11 +643,11 @@ def delTopVal(self, db, key=b''): with self.env.begin(db=db, write=True, buffers=True) as txn: result = False cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any + if cursor.set_range(top): # move to val at key >= key if any ckey, cval = cursor.item() while ckey: # end of database key == b'' ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch + if not ckey.startswith(top): # prev entry if any last in branch break # done result = cursor.delete() or result # delete moves cursor to next item ckey, cval = cursor.item() # cursor now at next item after deleted @@ -1884,7 +1884,7 @@ def getTopIoDupItemIter(self, db, key=b''): before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). """ - for key, val in self.getTopItemIter(db=db, key=key): + for key, val in self.getTopItemIter(db=db, top=key): val = val[33:] # strip proem yield (key, val) diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index 77f0001a..05ac15a2 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -130,7 +130,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): yield (self._tokeys(key), self.deserializer(val)) @@ -151,7 +151,7 @@ def getFullItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): yield (self._tokeys(key), self.deserializer(val)) @@ -355,7 +355,7 @@ def trim(self, keys: Union[str, Iterable]=b""): Returns: result (bool): True if key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys))) + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys))) def cntAll(self): @@ -634,7 +634,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): ensure get properly separated top branch key. """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for iokey, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): key, ion = dbing.unsuffix(iokey, sep=self.sep) yield (self._tokeys(key), self.deserializer(val)) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 3d9c012c..df03c981 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -198,7 +198,7 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", Returns: result (bool): True if val at key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys, top=top))) + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, top=top))) def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", @@ -235,7 +235,7 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", """ for key, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): + top=self._tokey(keys, top=top)): yield (self._tokeys(key), self._des(val)) @@ -277,7 +277,7 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", """ for key, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): + top=self._tokey(keys, top=top)): yield (self._tokeys(key), self._des(val)) @@ -875,7 +875,7 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", """ for iokey, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): + top=self._tokey(keys, top=top)): key, ion = dbing.unsuffix(iokey, sep=self.sep) yield (self._tokeys(key), self._des(val)) @@ -1054,7 +1054,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split yield (ikeys, self.klas(qb64b=bytes(val), @@ -1174,7 +1174,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split if decrypter: @@ -1412,7 +1412,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for iokey, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): yield self._tokeys(iokey), scheming.Schemer(raw=bytes(val)) @@ -1868,7 +1868,7 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", """ for key, val in self.db.getTopItemIter(db=self.sdb, - key=self._tokey(keys, top=top)): + top=self._tokey(keys, top=top)): val = val[33:] # strip off proem yield (self._tokeys(key), self._des(val)) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 7a825ab2..cabf2b9a 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -83,8 +83,7 @@ def test_key_funcs(): with pytest.raises(ValueError): splitKey(pre) - with pytest.raises(ValueError): - splitKey(dgKey(pre, dgKey(pre, dig))) + assert splitKey(dgKey(pre, dgKey(pre, dig))) # rsplit on gets trailing part # memoryview # Bytes @@ -278,7 +277,7 @@ def test_lmdber(): (b'a', b'2', b'wee'), (b'b', b'1', b'woo')] - assert dber.delTopVal(db, key=b"a.") + assert dber.delTopVal(db, top=b"a.") items = [ (key, bytes(val)) for key, val in dber.getTopItemIter(db=db )] assert items == [(b'b.1', b'woo')] @@ -936,4 +935,6 @@ def test_lmdber(): if __name__ == "__main__": test_key_funcs() + test_suffix() test_lmdber() + test_opendatabaser() From a071a72f4c6b5d49d3423367d9480dcdcf66254e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 29 Aug 2024 14:48:29 -0600 Subject: [PATCH 19/78] refactored fnKey snKey splitOnKey etc --- src/keri/app/cli/commands/escrow.py | 8 +-- src/keri/core/eventing.py | 20 +++--- src/keri/db/dbing.py | 102 +++++++++++++++++++--------- src/keri/db/subing.py | 2 +- src/keri/vdr/viring.py | 2 +- tests/core/test_replay.py | 7 ++ tests/db/test_dbing.py | 40 +++++++++-- 7 files changed, 128 insertions(+), 53 deletions(-) diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow.py index 4602a678..014daf54 100644 --- a/src/keri/app/cli/commands/escrow.py +++ b/src/keri/app/cli/commands/escrow.py @@ -57,7 +57,7 @@ def escrows(tymth, tock=0.0, **opts): key = ekey = b'' # both start same. when not same means escrows found while True: for ekey, edig in hby.db.getOoeItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: oots.append(eventing.loadEvent(hby.db, pre, edig)) @@ -75,7 +75,7 @@ def escrows(tymth, tock=0.0, **opts): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done for ekey, edig in hby.db.getPweItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: pwes.append(eventing.loadEvent(hby.db, pre, edig)) @@ -93,7 +93,7 @@ def escrows(tymth, tock=0.0, **opts): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done for ekey, edig in hby.db.getPseItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: pses.append(eventing.loadEvent(hby.db, pre, edig)) @@ -111,7 +111,7 @@ def escrows(tymth, tock=0.0, **opts): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done for ekey, edig in hby.db.getLdeItemsNextIter(key=key): - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: ldes.append(eventing.loadEvent(hby.db, pre, edig)) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index c662095a..e7c59768 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -43,7 +43,7 @@ from ..db import basing, dbing from ..db.basing import KeyStateRecord, StateEERecord, OobiRecord -from ..db.dbing import dgKey, snKey, fnKey, splitKeySN, splitKey +from ..db.dbing import dgKey, snKey, fnKey, splitSnKey, splitKey logger = help.ogler.getLogger() @@ -3929,7 +3929,7 @@ def processAttachedReceiptCouples(self, serder, cigars, firner=None, local=None) # Only accept receipt if event is latest event at sn. Means its been # first seen and is the most recent first seen with that sn if firner: - ldig = self.db.getFe(key=fnKey(pre=pre, sn=firner.sn)) + ldig = self.db.getFe(key=fnKey(pre=pre, fn=firner.sn)) else: ldig = self.db.getKeLast(key=snKey(pre=pre, sn=sn)) # retrieve dig of last event at sn. @@ -4106,7 +4106,7 @@ def processAttachedReceiptQuadruples(self, serder, trqs, firner=None, local=None sn = serder.sn if firner: # retrieve last event by fn ordinal - ldig = self.db.getFe(key=fnKey(pre=pre, sn=firner.sn)) + ldig = self.db.getFe(key=fnKey(pre=pre, fn=firner.sn)) else: # Only accept receipt if for last seen version of receipted event at sn ldig = self.db.getKeLast(key=snKey(pre=pre, sn=sn)) # retrieve dig of last event at sn. @@ -5211,7 +5211,7 @@ def processEscrowOutOfOrders(self): while True: # break when done for ekey, edig in self.db.getOoeItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below @@ -5350,7 +5350,7 @@ def processEscrowPartialSigs(self): for ekey, edig in self.db.getPseItemsNextIter(key=key): eserder = None try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below @@ -5530,7 +5530,7 @@ def processEscrowPartialWigs(self): #while True: # break when done for ekey, edig in self.db.getPweIoDupItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below @@ -5755,7 +5755,7 @@ def processEscrowUnverWitness(self): while True: # break when done for ekey, ecouple in self.db.getUweItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow db key + pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key # get escrowed receipt's rdiger of receipted event and # wiger indexed signature of receipted event @@ -5875,7 +5875,7 @@ def processEscrowUnverNonTrans(self): while True: # break when done for ekey, etriplet in self.db.getUreItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item rsaider, sprefixer, cigar = deReceiptTriple(etriplet) cigar.verfer = Verfer(qb64b=sprefixer.qb64b) @@ -6388,7 +6388,7 @@ def processEscrowUnverTrans(self): while True: # break when done for ekey, equinlet in self.db.getVreItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) # check date if expired then remove escrow. @@ -6558,7 +6558,7 @@ def processEscrowDuplicitous(self): while True: # break when done for ekey, edig in self.db.getLdeItemsNextIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 22100e3b..6420af79 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -65,6 +65,51 @@ SuffixSize = 32 # does not include trailing separator MaxSuffix = int("f"*(SuffixSize), 16) +def onKey(top, on, *, sep=b'.'): + """ + Returns: + onkey (bytes): key formed by joining top key and hex str conversion of + int ordinal number on with sep character. + + Parameters: + top (str | bytes): top key prefix to be joined with hex version of on using sep + on (int): ordinal number to be converted to 32 hex bytes + sep (bytes): separator character for join + + """ + if hasattr(top, "encode"): + top = top.encode("utf-8") # convert str to bytes + return (b'%s%s%032x' % (top, sep, on)) + + +def snKey(pre, sn): + """ + Returns: + snkey (bytes): key formed by joining pre and hex str conversion of int + sequence ordinal number sn with sep character b".". + + Parameters: + pre (str | bytes): key prefix to be joined with hex version of on using + b"." sep + sn (int): sequence number to be converted to 32 hex bytes + """ + + return onKey(pre, sn, sep=b'.') + +def fnKey(pre, fn): + """ + Returns: + fnkey (bytes): key formed by joining pre and hex str conversion of int + first seen ordinal number fn with sep character b".". + + Parameters: + pre (str | bytes): key prefix to be joined with hex version of on using + b"." sep + fn (int): first seen ordinal number to be converted to 32 hex bytes + """ + return onKey(pre, fn, sep=b'.') + + def dgKey(pre, dig): """ Returns bytes DB key from concatenation of '.' with qualified Base64 prefix @@ -78,20 +123,6 @@ def dgKey(pre, dig): return (b'%s.%s' % (pre, dig)) -def onKey(pre, sn): - """ - Returns bytes DB key from concatenation with '.' of qualified Base64 prefix - bytes pre and int ordinal number of event, such as sequence number or first - seen order number. - """ - if hasattr(pre, "encode"): - pre = pre.encode("utf-8") # convert str to bytes - return (b'%s.%032x' % (pre, sn)) - -snKey = onKey # alias so intent is clear, sn vs fn -fnKey = onKey # alias so intent is clear, sn vs fn - - def dtKey(pre, dts): """ Returns bytes DB key from concatenation of '|' qualified Base64 prefix @@ -134,7 +165,7 @@ def splitKey(key, sep=b'.'): return tuple(splits) -def splitKeyON(key): +def splitOnKey(key, *, sep=b'.'): """ Returns list of pre and int on from key Accepts either bytes or str key @@ -142,12 +173,17 @@ def splitKeyON(key): """ if isinstance(key, memoryview): key = bytes(key) - pre, on = splitKey(key) + top, on = splitKey(key, sep=sep) on = int(on, 16) - return (pre, on) + return (top, on) -splitKeySN = splitKeyON # alias so intent is clear, sn vs fn -splitKeyFN = splitKeyON # alias so intent is clear, sn vs fn + +splitSnKey = splitOnKey # alias so intent is clear, sn vs fn +splitFnKey = splitOnKey # alias so intent is clear, sn vs fn + +splitKeyON = splitOnKey # backwards compatible alias +splitKeySN = splitSnKey # backwards compatible alias +splitKeyFN = splitFnKey # backwards compatible alias def splitKeyDT(key): @@ -686,12 +722,12 @@ def appendOnValPre(self, db, pre, val): # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max ckey = cursor.key() - cpre, cn = splitKeyON(ckey) + cpre, cn = splitOnKey(ckey) if cpre == pre: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre ckey = cursor.key() - cpre, cn = splitKeyON(ckey) + cpre, cn = splitOnKey(ckey) if cpre == pre: # last entry for pre is already at max raise ValueError("Number part of key {} exceeds maximum" " size.".format(ckey)) @@ -699,7 +735,7 @@ def appendOnValPre(self, db, pre, val): # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre ckey = cursor.key() - cpre, cn = splitKeyON(ckey) + cpre, cn = splitOnKey(ckey) if cpre == pre: # last entry at pre on = cn + 1 # increment @@ -710,30 +746,32 @@ def appendOnValPre(self, db, pre, val): return on - def cntAllOnValsPre(self, db, pre, on=0): + def cntAllOnValsPre(self, db, top, on=0, *, sep=b'.'): """ - Returns (int): count of of all ordinal keyed vals with same pre in key - but different on in key in db starting at ordinal number on of pre. - For db with dupsort=False but ordinal number suffix in each key + Returns (int): count of of all ordinal keyed vals with top key + but different on tail in db starting at ordinal number on of top. + Full key is composed of top+sep+ + When dupsort==true then duplicates are included in count Parameters: db is opened named sub db - pre is bytes of key within sub db's keyspace pre.on + top (bytes): top key within sub db's keyspace with trailing part on + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest + key = onKey(top, on) # start replay at this enty 0 is earliest count = 0 if not cursor.set_range(key): # moves to val at key >= key return count # no values end of db for key in cursor.iternext(values=False): # get key only at cursor try: - cpre, cn = splitKeyON(key) + cpre, cn = splitOnKey(key) except ValueError as ex: # not splittable key break - if cpre != pre: # prev is now the last event for pre + if cpre != top: # prev is now the last event for pre break # done count = count+1 @@ -764,7 +802,7 @@ def getAllOnItemPreIter(self, db, pre, on=0): return # no values end of db for key, val in cursor.iternext(): # get key, val at cursor - cpre, cn = splitKeyON(key) + cpre, cn = splitOnKey(key) if cpre != pre: # prev is now the last event for pre break # done yield (cn, val) # (on, dig) of event @@ -791,7 +829,7 @@ def getAllOnItemAllPreIter(self, db, key=b''): return # no values end of db for key, val in cursor.iternext(): # return key, val at cursor - cpre, cn = splitKeyON(key) + cpre, cn = splitOnKey(key) yield (cpre, cn, val) # (pre, on, dig) of event diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index df03c981..bd2c4463 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1931,7 +1931,7 @@ def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): to form key on (int): ordinal number used with onKey(pre,on) to form key. """ - return (self.db.cntAllOnValsPre(db=self.sdb, pre=self._tokey(pre), on=on)) + return (self.db.cntAllOnValsPre(db=self.sdb, top=self._tokey(pre), on=on)) # appendOrdPre diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index ebbf600d..a6489cee 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntAllOnValsPre(db=self.tels, pre=pre, on=fn) + return self.cntAllOnValsPre(db=self.tels, top=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/core/test_replay.py b/tests/core/test_replay.py index 70b5ecf0..f2aed2cb 100644 --- a/tests/core/test_replay.py +++ b/tests/core/test_replay.py @@ -232,6 +232,7 @@ def test_replay(): # get disjoints receipts (vrcs) from Deb of Cam's events by processing Deb's cues debCamVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 assert debCamVrcs == (b'{"v":"KERI10JSON000091_","t":"rct","d":"EBp-SQb9fTgeoQkIkOd2xegv' b'Xy3epjOskiPrf6JDIEuj","i":"EBp-SQb9fTgeoQkIkOd2xegvXy3epjOskiPrf' b'6JDIEuj","s":"0"}-FABELfp9ZhqQCGov3wPRLa6vn5VkIQjug2sb2QD17T-TIp' @@ -259,6 +260,7 @@ def test_replay(): # get disjoints receipts (rcts) from Bev of Deb's events by processing Bevs's cues bevMsgs = bevHab.processCues(bevKevery.cues) + assert len(bevKevery.cues) == 0 assert bevMsgs == (b'{"v":"KERI10JSON0000fd_","t":"icp","d":"EBXqe7Xzsw2aolT09Ouh5Zw9' b'kNn2sgoHmo4zCn7Q7ZSC","i":"BAqph4mAWcf7mkIgk1Xrpvr7dWT7YvHIam_hq' b'UAT2rqw","s":"0","kt":"1","k":["BAqph4mAWcf7mkIgk1Xrpvr7dWT7YvHI' @@ -306,6 +308,7 @@ def test_replay(): # get disjoints receipts (vrcs) from Deb of Bev's events by processing Deb's cues debBevVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 assert debBevVrcs == (b'{"v":"KERI10JSON000091_","t":"rct","d":"EBXqe7Xzsw2aolT09Ouh5Zw9' b'kNn2sgoHmo4zCn7Q7ZSC","i":"BAqph4mAWcf7mkIgk1Xrpvr7dWT7YvHIam_hq' b'UAT2rqw","s":"0"}-FABELfp9ZhqQCGov3wPRLa6vn5VkIQjug2sb2QD17T-TIp' @@ -552,6 +555,7 @@ def test_replay_all(): # get disjoints receipts (vrcs) from Cam of Deb's events by processing Cam's cues camMsgs = camHab.processCues(camKevery.cues) + assert len(camKevery.cues) == 0 # Play camMsgs to Deb # create non-local kevery for Deb to process msgs from Cam @@ -566,6 +570,7 @@ def test_replay_all(): # get disjoints receipts (vrcs) from Deb of Cam's events by processing Deb's cues debCamVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 # Play disjoints debCamVrcs to Cam parsing.Parser().parseOne(ims=bytearray(debCamVrcs), kvy=camKevery) @@ -584,6 +589,7 @@ def test_replay_all(): # get disjoints receipts (rcts) from Bev of Deb's events by processing Bevs's cues bevMsgs = bevHab.processCues(bevKevery.cues) + assert len(bevKevery.cues) == 0 # Play bevMsgs to Deb parsing.Parser().parse(ims=bytearray(bevMsgs), kvy=debKevery) @@ -594,6 +600,7 @@ def test_replay_all(): # get disjoints receipts (vrcs) from Deb of Bev's events by processing Deb's cues debBevVrcs = debHab.processCues(debKevery.cues) + assert len(debKevery.cues) == 0 # Play disjoints debBevVrcs to Bev parsing.Parser().parseOne(ims=bytearray(debBevVrcs), kvy=bevKevery) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index cabf2b9a..3d24c7a1 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -17,7 +17,7 @@ from keri.db import dbing from keri.db.dbing import clearDatabaserDir, openLMDB from keri.db.dbing import (dgKey, onKey, fnKey, snKey, dtKey, splitKey, - splitKeyON, splitKeyFN, splitKeySN, splitKeyDT) + splitOnKey, splitKeyFN, splitSnKey, splitKeyDT) from keri.db.dbing import LMDBer @@ -35,11 +35,41 @@ def test_key_funcs(): sn = 3 dts = b'2021-02-13T19:16:50.750302+00:00' + + # test onKey generator of key from top key and trailing ordinal number + assert onKey(pre, 0) == pre + b'.' + b"%032x" % 0 + assert onKey(pre, 1) == pre + b'.' + b"%032x" % 1 + assert onKey(pre, 2) == pre + b'.' + b"%032x" % 2 + assert onKey(pre, 3) == pre + b'.' + b"%032x" % 3 + assert onKey(pre, 4) == pre + b'.' + b"%032x" % 4 + + assert onKey(pre, 0, sep=b'|') == pre + b'|' + b"%032x" % 0 + assert onKey(pre, 4, sep=b'|') == pre + b'|' + b"%032x" % 4 + + assert (onkey := onKey(top=pre, on=0)) == pre + b'.' + b"%032x" % 0 + assert splitKey(key=onkey) == (pre, b"%032x" % 0) + assert splitOnKey(onkey) == (pre, 0) + assert (onkey := onKey(top=pre, on=1)) == pre + b'.' + b"%032x" % 1 + assert splitKey(key=onkey) == (pre, b"%032x" % 1) + assert splitOnKey(onkey) == (pre, 1) + assert (onkey := onKey(top=pre, on=15)) == pre + b'.' + b"%032x" % 15 + assert splitKey(key=onkey) == (pre, b"%032x" % 15) + assert splitOnKey(onkey) == (pre, 15) + + assert (onkey := onKey(top=pre, on=0, sep=b'|')) == pre + b'|' + b"%032x" % 0 + assert splitKey(key=onkey, sep=b'|') == (pre, b"%032x" % 0) + assert splitOnKey(onkey, sep=b'|') == (pre, 0) + assert (onkey := onKey(top=pre, on=15, sep=b'|')) == pre + b'|' + b"%032x" % 15 + assert splitKey(key=onkey, sep=b'|') == (pre, b"%032x" % 15) + assert splitOnKey(onkey, sep=b'|') == (pre, 15) + + + # test snKey assert snKey(pre, sn) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.00000000000000000000000000000003') assert splitKey(snKey(pre, sn)) == (pre, b'%032x' % sn) - assert splitKeySN(snKey(pre, sn)) == (pre, sn) + assert splitSnKey(snKey(pre, sn)) == (pre, sn) assert dgKey(pre, dig) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') @@ -63,7 +93,7 @@ def test_key_funcs(): b'.00000000000000000000000000000003') assert splitKey(snKey(pre, sn).decode("utf-8")) == (pre, '%032x' % sn) - assert splitKeySN(snKey(pre, sn).decode("utf-8")) == (pre, sn) + assert splitSnKey(snKey(pre, sn).decode("utf-8")) == (pre, sn) assert dgKey(pre, dig) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') @@ -94,7 +124,7 @@ def test_key_funcs(): key = memoryview(snKey(pre, sn)) assert splitKey(key) == (pre, b'%032x' % sn) - assert splitKeySN(key) == (pre, sn) + assert splitSnKey(key) == (pre, sn) key = memoryview(dgKey(pre, dig)) assert splitKey(key) == (pre, dig) @@ -376,7 +406,7 @@ def test_lmdber(): assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntAllOnValsPre(db, preB) == 5 + assert dber.cntAllOnValsPre(db, top=preB) == 5 # replay preB events in database items = [item for item in dber.getAllOnItemPreIter(db, preB)] From 30d0ac3f0edee00dac8949f75f03dc4965664563 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 30 Aug 2024 17:03:28 -0600 Subject: [PATCH 20/78] refactoring of ordinal db operations --- src/keri/db/basing.py | 4 +- src/keri/db/dbing.py | 125 +++++++++++++++++++++------------------ src/keri/db/subing.py | 2 +- src/keri/vdr/eventing.py | 2 +- src/keri/vdr/viring.py | 6 +- tests/db/test_basing.py | 4 +- tests/db/test_dbing.py | 13 ++-- tests/vdr/test_viring.py | 2 +- 8 files changed, 83 insertions(+), 75 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 9278de2f..3218d2d3 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1535,7 +1535,7 @@ def clonePreIter(self, pre, fn=0): if hasattr(pre, 'encode'): pre = pre.encode("utf-8") - for fn, dig in self.getFelItemPreIter(pre, fn=fn): + for _, fn, dig in self.getFelItemPreIter(pre, fn=fn): try: msg = self.cloneEvtMsg(pre=pre, fn=fn, dig=dig) except Exception: @@ -1961,7 +1961,7 @@ def getFelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOnItemPreIter(db=self.fels, pre=pre, on=fn) + return self.getTopOnItemIter(db=self.fels, top=pre, on=fn) def getFelItemAllPreIter(self, key=b''): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 6420af79..e6c8180f 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -93,9 +93,9 @@ def snKey(pre, sn): b"." sep sn (int): sequence number to be converted to 32 hex bytes """ - return onKey(pre, sn, sep=b'.') + def fnKey(pre, fn): """ Returns: @@ -696,6 +696,71 @@ def delTopVal(self, db, top=b''): # ToDo change pre to top so can use in suber for any top branch key space whose last # part is ordinal number + + def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): + """ + Returns (int): count of of all ordinal keyed vals with top key + but different on tail in db starting at ordinal number on of top. + Full key is composed of top+sep+ + When dupsort==true then duplicates are included in count since .iternext + includes duplicates. + + Parameters: + db (lmdbsubdb): named sub db of lmdb + top (bytes): top key within sub db's keyspace with trailing part on + on (int): ordinal number at which to initiate count + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + count = 0 + if not cursor.set_range(key): # moves to val at key >= key + return count # no values end of db + + for key in cursor.iternext(values=False): # get key only at cursor + try: + cpre, cn = splitOnKey(key, sep=sep) + except ValueError as ex: # not splittable key + break + + if cpre != top: # prev is now the last event for pre + break # done + count = count+1 + + return count + + + def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): + """ + Returns iterator of triples (top, on, val), at each key over all ordinal + numbered keys with same top keyin db. Values are sorted by + onKey(top, on) where on is ordinal number int. + Returned items are triples of (top, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + + Raises StopIteration Error when empty. + + Parameters: + db (subdb): named sub db in lmdb + top (bytes): top key within sub db's keyspace with trailing part on + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + if not cursor.set_range(key): # moves to val at key >= key + return # no values end of db + + for key, val in cursor.iternext(): # get key, val at cursor + cpre, cn = splitOnKey(key, sep=sep) + if cpre != top: # prev is now the last event for pre + break # done + yield (top, cn, val) + + def appendOnValPre(self, db, pre, val): """ Appends val in order after last previous key with same pre in db. @@ -746,66 +811,8 @@ def appendOnValPre(self, db, pre, val): return on - def cntAllOnValsPre(self, db, top, on=0, *, sep=b'.'): - """ - Returns (int): count of of all ordinal keyed vals with top key - but different on tail in db starting at ordinal number on of top. - Full key is composed of top+sep+ - When dupsort==true then duplicates are included in count - Parameters: - db is opened named sub db - top (bytes): top key within sub db's keyspace with trailing part on - sep (bytes): separator character for split - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(top, on) # start replay at this enty 0 is earliest - count = 0 - if not cursor.set_range(key): # moves to val at key >= key - return count # no values end of db - for key in cursor.iternext(values=False): # get key only at cursor - try: - cpre, cn = splitOnKey(key) - except ValueError as ex: # not splittable key - break - - if cpre != top: # prev is now the last event for pre - break # done - count = count+1 - - return count - - -#Returned item should be (top, on, val) - - def getAllOnItemPreIter(self, db, pre, on=0): - """ - Returns iterator of duple item, (on, dig), at each key over all ordinal - numbered keys with same prefix, pre, in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Returned items are duples of (on, dig) where on is ordinal number int - and dig is event digest for lookup in .evts sub db. - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - pre is bytes of itdentifier prefix - on is int ordinal number to resume replay - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - if not cursor.set_range(key): # moves to val at key >= key - return # no values end of db - - for key, val in cursor.iternext(): # get key, val at cursor - cpre, cn = splitOnKey(key) - if cpre != pre: # prev is now the last event for pre - break # done - yield (cn, val) # (on, dig) of event def getAllOnItemAllPreIter(self, db, key=b''): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index bd2c4463..6866f1b0 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1931,7 +1931,7 @@ def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): to form key on (int): ordinal number used with onKey(pre,on) to form key. """ - return (self.db.cntAllOnValsPre(db=self.sdb, top=self._tokey(pre), on=on)) + return (self.db.cntTopOnVals(db=self.sdb, top=self._tokey(pre), on=on)) # appendOrdPre diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 0d46f9f1..2cc89a23 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -1172,7 +1172,7 @@ def vcState(self, vci): status (Serder): transaction event state notification message """ digs = [] - for _, dig in self.reger.getTelItemPreIter(pre=vci.encode("utf-8")): + for _, _, dig in self.reger.getTelItemPreIter(pre=vci.encode("utf-8")): digs.append(dig) if len(digs) == 0: diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index a6489cee..3314ecc0 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -508,7 +508,7 @@ def clonePreIter(self, pre, fn=0): if hasattr(pre, 'encode'): pre = pre.encode("utf-8") - for fn, dig in self.getTelItemPreIter(pre, fn=fn): + for _, fn, dig in self.getTelItemPreIter(pre, fn=fn): msg = self.cloneTvt(pre, dig) yield msg @@ -671,7 +671,7 @@ def getTelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOnItemPreIter(db=self.tels, pre=pre, on=fn) + return self.getTopOnItemIter(db=self.tels, top=pre, on=fn) def cntTels(self, pre, fn=0): """ @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntAllOnValsPre(db=self.tels, top=pre, on=fn) + return self.cntTopOnVals(db=self.tels, top=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index a3d8b98b..825c039d 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -282,11 +282,11 @@ def test_baser(): # replay preB events in database items = [item for item in db.getFelItemPreIter(preB)] - assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] + assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 items = [item for item in db.getFelItemPreIter(preB, fn=3)] - assert items == [(3, digX), (4, digY)] + assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 items = [item for item in db.getFelItemPreIter(preB, fn=5)] diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 3d24c7a1..e6ff17bf 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -406,18 +406,19 @@ def test_lmdber(): assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntAllOnValsPre(db, top=preB) == 5 + assert dber.cntTopOnVals(db, top=preB) == 5 # replay preB events in database - items = [item for item in dber.getAllOnItemPreIter(db, preB)] - assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] + items = [item for item in dber.getTopOnItemIter(db, preB)] + assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), + (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 - items = [item for item in dber.getAllOnItemPreIter(db, preB, on=3)] - assert items == [(3, digX), (4, digY)] + items = [item for item in dber.getTopOnItemIter(db, preB, on=3)] + assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 - items = [item for item in dber.getAllOnItemPreIter(db, preB, on=5)] + items = [item for item in dber.getTopOnItemIter(db, preB, on=5)] assert items == [] # replay all events in database with pre events before and after diff --git a/tests/vdr/test_viring.py b/tests/vdr/test_viring.py index 8ec90e47..2e98dda6 100644 --- a/tests/vdr/test_viring.py +++ b/tests/vdr/test_viring.py @@ -231,7 +231,7 @@ def test_issuer(): assert issuer.putTel(snKey(vcdig, sn + 2), val=idig.qb64b) is True assert issuer.putTel(snKey(vcdig, sn + 3), val=rdig.qb64b) is True - result = [(sn, dig) for sn, dig in issuer.getTelItemPreIter(vcdig)] + result = [(sn, dig) for _, sn, dig in issuer.getTelItemPreIter(vcdig)] assert result == [(0, idig.qb64b), (1, rdig.qb64b), (2, idig.qb64b), (3, rdig.qb64b)] bak1 = b'BA1Q98kT0HRn9R62lY-LufjjKdbCeL1mqu9arTgOmbqI' From f8d42c3d69d888e7ab175b96baa81fe8adb68446 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 30 Aug 2024 19:45:08 -0600 Subject: [PATCH 21/78] some more refactoring --- src/keri/db/dbing.py | 164 +++++++++++++++++++++---------------------- 1 file changed, 81 insertions(+), 83 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index e6c8180f..c09a22f2 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -733,10 +733,10 @@ def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): """ - Returns iterator of triples (top, on, val), at each key over all ordinal + Returns iterator of triples (key, on, val), at each key over all ordinal numbered keys with same top keyin db. Values are sorted by - onKey(top, on) where on is ordinal number int. - Returned items are triples of (top, on, val) + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Returned items are triples of (key, on, val) When dupsort==true then duplicates are included in items since .iternext includes duplicates. @@ -750,15 +750,47 @@ def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + if top: # not empty + key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty top + key = top # get all values empty key is start of db if not cursor.set_range(key): # moves to val at key >= key return # no values end of db for key, val in cursor.iternext(): # get key, val at cursor - cpre, cn = splitOnKey(key, sep=sep) - if cpre != top: # prev is now the last event for pre - break # done - yield (top, cn, val) + ckey, cn = splitOnKey(key, sep=sep) + if not ckey.startswith(top): # prev is now the last event for pre + break # done + #if ckey != top: # prev is now the last event for pre + #break # done + yield (ckey, cn, val) + + + def getAllOnItemAllPreIter(self, db, key=b''): + """ + Returns iterator of triple item, (pre, on, dig), at each key over all + ordinal numbered keys for all prefixes in db. Values are sorted by + onKey(pre, on) where on is ordinal number int. + Each returned item is triple (pre, on, dig) where pre is identifier prefix, + on is ordinal number int and dig is event digest for lookup in .evts sub db. + + Raises StopIteration Error when empty. + + Parameters: + db is opened named sub db with dupsort=False + key is key location in db to resume replay, + If empty then start at first key in database + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + if not cursor.set_range(key): # moves to val at key >= key, first if empty + return # no values end of db + + for key, val in cursor.iternext(): # return key, val at cursor + cpre, cn = splitOnKey(key) + yield (cpre, cn, val) # (pre, on, dig) of event + + def appendOnValPre(self, db, pre, val): @@ -812,35 +844,6 @@ def appendOnValPre(self, db, pre, val): - - - - def getAllOnItemAllPreIter(self, db, key=b''): - """ - Returns iterator of triple item, (pre, on, dig), at each key over all - ordinal numbered keys for all prefixes in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Each returned item is triple (pre, on, dig) where pre is identifier prefix, - on is ordinal number int and dig is event digest for lookup in .evts sub db. - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db - - for key, val in cursor.iternext(): # return key, val at cursor - cpre, cn = splitOnKey(key) - yield (cpre, cn, val) # (pre, on, dig) of event - - - # IoSet insertion order in val so can have effective dups but with # dupsort = False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent @@ -1732,9 +1735,50 @@ def cntIoDupVals(self, db, key): return count + def getTopIoDupItemIter(self, db, key=b''): + """ + Iterates over top branch of db given by key of IoDup items where each value + has 33 byte insertion ordinal number proem (prefixed) with separator. + Automagically removes (strips) proem before returning items. + + Assumes DB opened with dupsort=True + + Returns: + items (abc.Iterator): iterator of (full key, val) tuples of all + dup items over a branch of the db given by top key where returned + full key is full database key for val not truncated top key. + Item is (key, val) with proem stripped from val stored in db. + If key = b'' then returns list of dup items for all keys in db. + + + Because cursor.iternext() advances cursor after returning item its safe + to delete the item within the iteration loop. curson.iternext() works + for both dupsort==False and dupsort==True + Raises StopIteration Error when empty. + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): truncated top key, a key space prefix to get all the items + from multiple branches of the key space. If top key is + empty then gets all items in database + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + for key, val in self.getTopItemIter(db=db, top=key): + val = val[33:] # strip proem + yield (key, val) def getIoDupItemsNext(self, db, key=b"", skip=True): @@ -1888,52 +1932,6 @@ def getIoDupItemIter(self, db, key=b'', *, ion=0): return # done raises StopIteration - def getTopIoDupItemIter(self, db, key=b''): - """ - Iterates over top branch of db given by key of IoDup items where each value - has 33 byte insertion ordinal number proem (prefixed) with separator. - Automagically removes (strips) proem before returning items. - - Assumes DB opened with dupsort=True - - Returns: - items (abc.Iterator): iterator of (full key, val) tuples of all - dup items over a branch of the db given by top key where returned - full key is full database key for val not truncated top key. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items for all keys in db. - - - Because cursor.iternext() advances cursor after returning item its safe - to delete the item within the iteration loop. curson.iternext() works - for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items - from multiple branches of the key space. If top key is - empty then gets all items in database - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - """ - for key, val in self.getTopItemIter(db=db, top=key): - val = val[33:] # strip proem - yield (key, val) - - def getIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries From 2b497b8b175e63cd22de067bec1e99453d05a468 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 09:19:39 -0600 Subject: [PATCH 22/78] refactor out the old getAllOnItemAllpre --- src/keri/app/cli/commands/migrate.py | 2 +- src/keri/app/habbing.py | 4 ++-- src/keri/db/basing.py | 32 +++++++++++++++++++--------- src/keri/db/dbing.py | 31 +-------------------------- tests/db/test_dbing.py | 11 +++++----- 5 files changed, 31 insertions(+), 49 deletions(-) diff --git a/src/keri/app/cli/commands/migrate.py b/src/keri/app/cli/commands/migrate.py index 36d3b3c0..87858779 100644 --- a/src/keri/app/cli/commands/migrate.py +++ b/src/keri/app/cli/commands/migrate.py @@ -148,7 +148,7 @@ def migrateKeys(db): digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", klas=(coring.Prefixer, coring.Seqner)) - for pre, fn, dig in db.getFelItemAllPreIter(key=b''): + for pre, fn, dig in db.getFelItemAllPreIter(): dgkey = dbing.dgKey(pre, dig) # get message if not (raw := db.getEvt(key=dgkey)): raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index d9431aa4..ba295922 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -1538,7 +1538,7 @@ def replay(self, pre=None, fn=0): return msgs - def replayAll(self, key=b''): + def replayAll(self): """ Returns replay of FEL first seen event log for all pre starting at key @@ -1547,7 +1547,7 @@ def replayAll(self, key=b''): """ msgs = bytearray() - for msg in self.db.cloneAllPreIter(key=key): + for msg in self.db.cloneAllPreIter(): msgs.extend(msg) return msgs diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 3218d2d3..029abab3 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1531,6 +1531,13 @@ def clonePreIter(self, pre, fn=0): Returns iterator of first seen event messages with attachments for the identifier prefix pre starting at first seen order number, fn. Essentially a replay in first seen order with attachments + + Parameters: + pre is bytes of itdentifier prefix + fn is int fn to resume replay. Earliset is fn=0 + + Returns: + msgs (Iterator): over all items with pre starting at fn """ if hasattr(pre, 'encode'): pre = pre.encode("utf-8") @@ -1543,18 +1550,19 @@ def clonePreIter(self, pre, fn=0): yield msg - def cloneAllPreIter(self, key=b''): + def cloneAllPreIter(self): """ Returns iterator of first seen event messages with attachments for all - identifier prefixes starting at key. If key == b'' then rstart at first + identifier prefixes starting at key. If key == b'' then start at first key in databse. Use key to resume replay. Essentially a replay in first seen order with attachments of entire set of FELs. - Parameters: - key (bytes): fnKey(pre, fn) + Returns: + msgs (Iterator): over all items in db + """ - for pre, fn, dig in self.getFelItemAllPreIter(key=key): + for pre, fn, dig in self.getFelItemAllPreIter(): try: msg = self.cloneEvtMsg(pre=pre, fn=fn, dig=dig) except Exception: @@ -1948,9 +1956,9 @@ def appendFe(self, pre, val): def getFelItemPreIter(self, pre, fn=0): """ - Returns iterator of all (fn, dig) duples in first seen order for all events - with same prefix, pre, in database. Items are sorted by fnKey(pre, fn) - where fn is first seen order number int. + Returns iterator of all (pre, fn, dig) triples in first seen order for + all events with same prefix, pre, in database. Items are sorted by + fnKey(pre, fn) where fn is first seen order number int. Returns a First Seen Event Log FEL. Returned items are duples of (fn, dig): Where fn is first seen order number int and dig is event digest for lookup in .evts sub db. @@ -1960,11 +1968,14 @@ def getFelItemPreIter(self, pre, fn=0): Parameters: pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 + + Returns: + items (Iterator[(pre, fn, val)]): over all items starting at pre, on """ return self.getTopOnItemIter(db=self.fels, top=pre, on=fn) - def getFelItemAllPreIter(self, key=b''): + def getFelItemAllPreIter(self): """ Returns iterator of all (pre, fn, dig) triples in first seen order for all events for all prefixes in database. Items are sorted by @@ -1980,7 +1991,8 @@ def getFelItemAllPreIter(self, key=b''): key is key location in db to resume replay, If empty then start at first key in database """ - return self.getAllOnItemAllPreIter(db=self.fels, key=key) + #return self.getAllOnItemAllPreIter(db=self.fels, key=key) + return self.getTopOnItemIter(db=self.fels, top=b'') def putDts(self, key, val): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index c09a22f2..903f4402 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -731,7 +731,7 @@ def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): return count - def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): + def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): """ Returns iterator of triples (key, on, val), at each key over all ordinal numbered keys with same top keyin db. Values are sorted by @@ -761,38 +761,9 @@ def getTopOnItemIter(self, db, top, on=0, *, sep=b'.'): ckey, cn = splitOnKey(key, sep=sep) if not ckey.startswith(top): # prev is now the last event for pre break # done - #if ckey != top: # prev is now the last event for pre - #break # done yield (ckey, cn, val) - def getAllOnItemAllPreIter(self, db, key=b''): - """ - Returns iterator of triple item, (pre, on, dig), at each key over all - ordinal numbered keys for all prefixes in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Each returned item is triple (pre, on, dig) where pre is identifier prefix, - on is ordinal number int and dig is event digest for lookup in .evts sub db. - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db - - for key, val in cursor.iternext(): # return key, val at cursor - cpre, cn = splitOnKey(key) - yield (cpre, cn, val) # (pre, on, dig) of event - - - - def appendOnValPre(self, db, pre, val): """ Appends val in order after last previous key with same pre in db. diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index e6ff17bf..0f51ae5d 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -425,19 +425,18 @@ def test_lmdber(): assert dber.putVal(db, keyA0, val=digA) == True assert dber.putVal(db, keyC0, val=digC) == True - items = [item for item in dber.getAllOnItemAllPreIter(db)] + items = [item for item in dber.getTopOnItemIter(db, top=b'')] assert items == [(preA, 0, digA), (preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY), (preC, 0, digC)] - # resume replay all starting at preB on=2 - items = [item for item in dber.getAllOnItemAllPreIter(db, key=keyB2)] - assert items == [(preB, 2, digW), (preB, 3, digX), (preB, 4, digY), - (preC, 0, digC)] + top, on = splitOnKey(keyB2) + items = [item for item in dber.getTopOnItemIter(db, top=top, on=on)] + assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] # resume replay all starting at preC on=1 - items = [item for item in dber.getAllOnItemAllPreIter(db, key=onKey(preC, 1))] + items = [item for item in dber.getTopOnItemIter(db, top=preC, on=1)] assert items == [] From f95ac48e07e782d38a418c6361ed322eb5e47930 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 17:45:10 -0600 Subject: [PATCH 23/78] create OnSuber with partial unit tests --- src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 47 +++--- src/keri/db/subing.py | 341 ++++++++++++++++++++++------------------ tests/db/test_dbing.py | 16 +- tests/db/test_subing.py | 200 +++++++++++++++-------- 5 files changed, 360 insertions(+), 246 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 029abab3..140c4909 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1952,7 +1952,7 @@ def appendFe(self, pre, val): pre is bytes identifier prefix for event val is event digest """ - return self.appendOnValPre(db=self.fels, pre=pre, val=val) + return self.appendTopOnVal(db=self.fels, top=pre, val=val) def getFelItemPreIter(self, pre, fn=0): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 903f4402..73430e86 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -690,12 +690,8 @@ def delTopVal(self, db, top=b''): return result - # For subdbs with no duplicate values allowed at each key. (dupsort==False) - # and use keys with suffic ordinal that is monotonically increasing number part - # such as fn where no duplicates allowed at a given (pre, on) -# ToDo change pre to top so can use in suber for any top branch key space whose last -# part is ordinal number - + # For subdbs the use keys with trailing part the is monotonically + # ordinal number serialized as 32 hex bytes def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): """ @@ -742,6 +738,9 @@ def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): Raises StopIteration Error when empty. + Returns: + items (Iterator[(top, on, val)]): triples of top key, on int, val + Parameters: db (subdb): named sub db in lmdb top (bytes): top key within sub db's keyspace with trailing part on @@ -763,25 +762,33 @@ def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): break # done yield (ckey, cn, val) + # only valid for dupsort==False such as fn where only one entry per ordinal + # is allowed - def appendOnValPre(self, db, pre, val): + def appendTopOnVal(self, db, top, val, *, sep=b'.'): """ Appends val in order after last previous key with same pre in db. Returns ordinal number in, on, of appended entry. Appended on is 1 greater - than previous latest on. + than previous latest on at pre. Uses onKey(pre, on) for entries. + Assumes dupsort==False so no duplicates at a given onKey. + Append val to end of db entries with same pre but with on incremented by 1 relative to last preexisting entry at pre. + Returns: + on (int): ordinal number of newly appended val + Parameters: - db is opened named sub db with dupsort=False - pre is bytes identifier prefix for event - val is event digest + db (subdb): named sub db in lmdb + top (bytes): top key within sub db's keyspace with trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split """ # set key with fn at max and then walk backwards to find last entry at pre # if any otherwise zeroth entry at pre - key = onKey(pre, MaxON) + key = onKey(top, MaxON, sep=sep) with self.env.begin(db=db, write=True, buffers=True) as txn: on = 0 # unless other cases match then zeroth entry at pre cursor = txn.cursor() @@ -790,24 +797,24 @@ def appendOnValPre(self, db, pre, val): # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max ckey = cursor.key() - cpre, cn = splitOnKey(ckey) - if cpre == pre: # last is last entry for same pre + cpre, cn = splitOnKey(ckey, sep=sep) + if cpre == top: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre ckey = cursor.key() - cpre, cn = splitOnKey(ckey) - if cpre == pre: # last entry for pre is already at max + cpre, cn = splitOnKey(ckey, sep=sep) + if cpre == top: # last entry for pre is already at max raise ValueError("Number part of key {} exceeds maximum" " size.".format(ckey)) else: # later pre so backup one entry # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre ckey = cursor.key() - cpre, cn = splitOnKey(ckey) - if cpre == pre: # last entry at pre + cpre, cn = splitOnKey(ckey, sep=sep) + if cpre == top: # last entry at pre on = cn + 1 # increment - key = onKey(pre, on) + key = onKey(top, on, sep=sep) if not cursor.put(key, val, overwrite=False): raise ValueError("Failed appending {} at {}.".format(val, key)) @@ -839,7 +846,7 @@ def putIoSetVals(self, db, key, vals, *, sep=b'.'): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False key (bytes): Apparent effective key - vals (abc.Iterable): serialized values to add to set of vals at key + vals (Iterable): serialized values to add to set of vals at key """ result = False diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 6866f1b0..3f9bb0ff 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -91,14 +91,14 @@ def __init__(self, db: dbing.LMDBer, *, - def _tokey(self, keys: str | bytes | memoryview | Iterable[str | bytes], - top: bool=False): + def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], + topive: bool=False): """ Converts keys to key bytes with proper separators and returns key bytes. If keys is already str or bytes then returns key bytes. Else If keys is iterable (non-str) of strs or bytes then joins with separator converts to key bytes and returns. When keys is iterable and - top is True then enables partial key from top branch of key space given + topive is True then enables partial key from top branch of key space given by partial keys by appending separator to end of partial key Returns: @@ -108,7 +108,7 @@ def _tokey(self, keys: str | bytes | memoryview | Iterable[str | bytes], Parameters: keys (str | bytes | memoryview | Iterable[str | bytes]): db key or Iterable of (str | bytes) to form key. - top (bool): True means treat as partial key tuple from top branch of + topive (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. False means treat as full branch in key space. Resultant key @@ -123,7 +123,7 @@ def _tokey(self, keys: str | bytes | memoryview | Iterable[str | bytes], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys - if top and keys[-1]: # top and keys is not already partial + if topive and keys[-1]: # topive and keys is not already partial keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep return (self.sep.join(key.decode() if hasattr(key, "decode") else key for key in keys).encode("utf-8")) @@ -170,8 +170,7 @@ def _des(self, val: str | bytes | memoryview): return (val.decode("utf-8") if hasattr(val, "decode") else val) - def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): + def trim(self, keys: str|bytes|memoryview|Iterable=b"", *, topive=False): """ Removes all entries whose keys startswith keys. Enables removal of whole branches of db key space. To ensure that proper separation of a branch @@ -179,14 +178,14 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", 'a.1'and 'a.2' but not 'ab' Parameters: - keys (Iterator[str | bytes | memoryview]): of key parts that may be + keys (Iteratabke[str | bytes | memoryview]): of key parts that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then trims all items in database. Either append "" to end of keys Iterable to ensure get properly separated top branch key or use top=True. - top (bool): True means treat as partial key tuple from top branch of + topive (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. False means treat as full branch in key space. Resultant key @@ -198,11 +197,11 @@ def trim(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", Returns: result (bool): True if val at key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, top=top))) + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, topive=topive))) def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): + *, topive=False): """Iterator over items in .db that returns full items with subclass specific special hidden parts shown for debugging or testing. @@ -215,7 +214,8 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", valuespace which may be useful in debugging or testing. Parameters: - keys (Iterator[str | bytes | memoryview]): of key parts that may be + keys (str|bytes|memoryview|Iteratable[str | bytes | memoryview]): + of key parts that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets all items in database. @@ -225,7 +225,7 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", key is empty string it matches all keys in db with startswith. - top (bool): True means treat as partial key tuple from top branch of + topive (bool): True means treat as partial key tuple from top branch of key space given by partial keys. Resultant key ends in .sep character. False means treat as full branch in key space. Resultant key @@ -235,12 +235,12 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", """ for key, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, top=top)): + top=self._tokey(keys, topive=topive)): yield (self._tokeys(key), self._des(val)) - def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", - *, top=False): + def getItemIter(self, keys: str|bytes|memoryview|Iterable=b"", + *, topive=False): """Iterator over items in .db subclasses that do special hidden transforms on either the keyspace or valuespace should override this method to hide hidden parts from the returned items. For example, adding either @@ -257,7 +257,8 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", Parameters: - keys (Iterator[str | bytes | memoryview]): of key parts that may be + keys (str|bytes|memoryview|Iterable[str|bytes|memoryview]): of key + parts that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets all items in database. @@ -267,17 +268,17 @@ def getItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", key is empty string it matches all keys in db with startswith. - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ for key, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, top=top)): + top=self._tokey(keys, topive=topive)): yield (self._tokeys(key), self._des(val)) @@ -376,6 +377,111 @@ def rem(self, keys: Union[str, Iterable]): return(self.db.delVal(db=self.sdb, key=self._tokey(keys))) +class OnSuberBase(SuberBase): + """ + Subclass of SuberBase that adds methods for keys with exposed key part suffix + that is 32 byte serializaton of monotonically increasing ordinal number on + such as sn or fn. + Each key consistes of top key joined with .sep to ordinal suffix + Works with dupsort==True or False + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Default False + sep (str): separator to convert keys iterator to key bytes for db key + Default '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(OnSuberBase, self).__init__(*pa, **kwa) + + + def cntOn(self, keys: str | bytes | memoryview, on: int=0): + """ + Returns + cnt (int): count of of all ordinal suffix keyed vals with same top + in key but different on in key in db starting at ordinal number + on of pre where key is formed with onKey(pre,on) + Does not count dups at same on for a given pre, only + unique on at a given pre. + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form top key + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.cntTopOnVals(db=self.sdb, + top=self._tokey(keys), + on=on, + sep=self.sep.encode())) + + def getOnItemIter(self, keys: str | bytes | memoryview | Iterable, on: int=0): + """ + Returns + items (Iterator[(top keys, on, val)]): triples of (top keys, on int, + deserialized val) + + Parameters: + keys (str | bytes | memoryview | iterator): top keys as prefix to be + combined with serialized on suffix and sep to form key + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getTopOnItemIter(db=self.sdb, + top=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + +class OnSuber(OnSuberBase, Suber): + """ + Subclass of Suber that adds methods for keys with ordinal numbered suffixes. + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(OnSuber, self).__init__(*pa, **kwa) + + + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): + """ + Returns: + on (int): ordinal number of newly appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.appendTopOnVal(db=self.sdb, + top=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) + class CesrSuberBase(SuberBase): @@ -843,7 +949,7 @@ def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", - *, top=False): + *, topive=False): """ Return iterator over all the items in top branch defined by keys where keys may be truncation of full branch. @@ -856,7 +962,7 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", Returned key in each item has ordinal suffix removed. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets all items in database. @@ -865,17 +971,17 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", In Python str.startswith('') always returns True so if branch key is empty string it matches all keys in db with startswith. - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ for iokey, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, top=top)): + top=self._tokey(keys, topive=topive)): key, ion = dbing.unsuffix(iokey, sep=self.sep) yield (self._tokeys(key), self._des(val)) @@ -1039,7 +1145,8 @@ def get(self, keys: Union[str, Iterable]): if val is not None else None) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str | bytes | memoryview | Iterable =b"", + *, topive=False): """ Returns: iterator (Iteratore: tuple (key, val) over the all the items in @@ -1048,13 +1155,21 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): returns all items in subdb Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split yield (ikeys, self.klas(qb64b=bytes(val), @@ -1155,11 +1270,11 @@ def get(self, keys: Union[str, Iterable], decrypter: core.Decrypter = None): - def getItemIter(self, keys: Union[str, Iterable]=b"", - decrypter: core.Decrypter = None): + def getItemIter(self, keys: str|bytes|memoryview|Iterable=b"", + decrypter: core.Decrypter = None, *, topive=False): """ Returns: - iterator (Iterator): of tuples (key, val) over the all the items in + items (Iterator): of tuples (key, val) over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace prefix to return branches of key space. When keys is empty then returns all items in subdb @@ -1168,13 +1283,21 @@ def getItemIter(self, keys: Union[str, Iterable]=b"", db was encrypted and so decrypts before converting to Signer. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split if decrypter: @@ -1397,7 +1520,8 @@ def rem(self, keys: Union[str, Iterable]): """ return self.db.delVal(db=self.sdb, key=self._tokey(keys)) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str | bytes | memoryview | Iterable =b"", + *, topive=False): """ Returns: iterator (Iterator): tuple (key, val) over the all the items in @@ -1406,13 +1530,22 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): returns all items in subdb Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (str | bytes | memoryview | Iterable): tuple of bytes or + strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): + for iokey, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): yield self._tokeys(iokey), scheming.Schemer(raw=bytes(val)) @@ -1686,7 +1819,7 @@ def add(self, keys: str | bytes | memoryview | Iterable, Parameters: keys (Iterable): of key strs to be combined in order to form key - val (Union[bytes, str]): serialization + val (str | bytes | memoryview): serialization Returns: result (bool): True means unique value added among duplications, @@ -1818,7 +1951,8 @@ def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): apparent effective key and items all have same apparent effective key Parameters: - keys (Iterable): of key strs to be combined in order to form key + keys (str|bytes|memoryview|Iterable): of key strs to be combined + in order to form key ion (int): starting ordinal value, default 0 Returns: @@ -1836,7 +1970,7 @@ def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", - *, top=False): + *, topive=False): """ Return iterator over all the items including dup items for all keys in top branch defined by keys where keys may be truncation of full branch. @@ -1858,17 +1992,17 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", In Python str.startswith('') always returns True so if branch key is empty string it matches all keys in db with startswith. - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ for key, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, top=top)): + top=self._tokey(keys, topive=topive)): val = val[33:] # strip off proem yield (self._tokeys(key), self._des(val)) @@ -1891,96 +2025,3 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", # used by .dels # getIoDupValsAnyPreIter(self, db, pre, on=0) - - - -class OnSuber(Suber): - """ - Subclass of Suber that adds methods for keys with ordinal numbered suffixes. - Each key consistes of pre joined with .sep to ordinal suffix - - """ - - def __init__(self, *pa, **kwa): - """ - Inherited Parameters: - db (dbing.LMDBer): base db - subkey (str): LMDB sub database key - dupsort (bool): True means enable duplicates at each key - False (default) means do not enable duplicates at - each key. Set to False - sep (str): separator to convert keys iterator to key bytes for db key - default is self.Sep == '.' - verify (bool): True means reverify when ._des from db when applicable - False means do not reverify. Default False - """ - super(OnSuber, self).__init__(*pa, **kwa) - - - def cntOrdPre(self, pre: str | bytes | memoryview, on: int=0): - """ - Returns - cnt (int): count of of all ordinal suffix keyed vals with same pre - in key but different on in key in db starting at ordinal number - on of pre where key is formed with onKey(pre,on) - Does not count dups at same on for a given pre, only - unique on at a given pre. - - Parameters: - pre (str | bytes | memoryview): prefix to to be combined with on - to form key - on (int): ordinal number used with onKey(pre,on) to form key. - """ - return (self.db.cntTopOnVals(db=self.sdb, top=self._tokey(pre), on=on)) - - # appendOrdPre - - #def appendOrdValPre(self, db, pre, val): - #""" - #Appends val in order after last previous key with same pre in db. - #Returns ordinal number in, on, of appended entry. Appended on is 1 greater - #than previous latest on. - #Uses onKey(pre, on) for entries. - - #Append val to end of db entries with same pre but with on incremented by - #1 relative to last preexisting entry at pre. - - #Parameters: - #db is opened named sub db with dupsort=False - #pre is bytes identifier prefix for event - #val is event digest - #""" - - # getAllOrdItemPreIter - #def getAllOrdItemPreIter(self, db, pre, on=0): - #""" - #Returns iterator of duple item, (on, dig), at each key over all ordinal - #numbered keys with same prefix, pre, in db. Values are sorted by - #onKey(pre, on) where on is ordinal number int. - #Returned items are duples of (on, dig) where on is ordinal number int - #and dig is event digest for lookup in .evts sub db. - - #Raises StopIteration Error when empty. - - #Parameters: - #db is opened named sub db with dupsort=False - #pre is bytes of itdentifier prefix - #on is int ordinal number to resume replay - #""" - - - #def getAllOrdItemAllPreIter(self, db, key=b''): - #""" - #Returns iterator of triple item, (pre, on, dig), at each key over all - #ordinal numbered keys for all prefixes in db. Values are sorted by - #onKey(pre, on) where on is ordinal number int. - #Each returned item is triple (pre, on, dig) where pre is identifier prefix, - #on is ordinal number int and dig is event digest for lookup in .evts sub db. - - #Raises StopIteration Error when empty. - - #Parameters: - #db is opened named sub db with dupsort=False - #key is key location in db to resume replay, - #If empty then start at first key in database - #""" diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 0f51ae5d..dbafab59 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -351,7 +351,7 @@ def test_lmdber(): # test appendOrdValPre # empty database assert dber.getVal(db, keyB0) == None - on = dber.appendOnValPre(db, preB, digU) + on = dber.appendTopOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -359,7 +359,7 @@ def test_lmdber(): # earlier pre in database only assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOnValPre(db, preB, digU) + on = dber.appendTopOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -368,7 +368,7 @@ def test_lmdber(): # earlier and later pre in db but not same pre assert dber.getVal(db, keyA0) == digA assert dber.putVal(db, keyC0, val=digC) == True - on = dber.appendOnValPre(db, preB, digU) + on = dber.appendTopOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -378,13 +378,13 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None assert dber.getVal(db, keyC0) == digC - on = dber.appendOnValPre(db, preB, digU) + on = dber.appendTopOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU # earlier pre and later pre and earlier entry for same pre assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOnValPre(db, preB, digV) + on = dber.appendTopOnVal(db, preB, digV) assert on == 1 assert dber.getVal(db, keyB1) == digV @@ -394,15 +394,15 @@ def test_lmdber(): assert dber.delVal(db, keyC0) == True assert dber.getVal(db, keyC0) == None # another value for preB - on = dber.appendOnValPre(db, preB, digW) + on = dber.appendTopOnVal(db, preB, digW) assert on == 2 assert dber.getVal(db, keyB2) == digW # yet another value for preB - on = dber.appendOnValPre(db, preB, digX) + on = dber.appendTopOnVal(db, preB, digX) assert on == 3 assert dber.getVal(db, keyB3) == digX # yet another value for preB - on = dber.appendOnValPre(db, preB, digY ) + on = dber.appendTopOnVal(db, preB, digY ) assert on == 4 assert dber.getVal(db, keyB4) == digY diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index ffca9aa9..f49fa6e7 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -31,62 +31,62 @@ def test_suber(): assert db.name == "test" assert db.opened - sdb = subing.Suber(db=db, subkey='bags.') - assert isinstance(sdb, subing.Suber) - assert not sdb.sdb.flags()["dupsort"] + suber = subing.Suber(db=db, subkey='bags.') + assert isinstance(suber, subing.Suber) + assert not suber.sdb.flags()["dupsort"] sue = "Hello sailer!" keys = ("test_key", "0001") - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue kip = "Hey gorgeous!" - result = sdb.put(keys=keys, val=kip) + result = suber.put(keys=keys, val=kip) assert not result - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual == sue - result = sdb.pin(keys=keys, val=kip) + result = suber.pin(keys=keys, val=kip) assert result - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual == kip - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue # test with keys as tuple of bytes keys = (b"test_key", b"0001") - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue # test with keys as mixed tuple of bytes keys = (b"test_key", "0001") - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue @@ -95,21 +95,21 @@ def test_suber(): bob = "Shove off!" - sdb.put(keys=keys, val=bob) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=bob) + actual = suber.get(keys=keys) assert actual == bob - sdb.rem(keys) + suber.rem(keys) - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual is None liz = "May life is insane." keys = ("test_key", "0002") - sdb.put(keys=keys, val=liz) - actual = sdb.get(("not_found", "0002")) + suber.put(keys=keys, val=liz) + actual = suber.get(("not_found", "0002")) assert actual is None w = "Blue dog" @@ -117,26 +117,26 @@ def test_suber(): y = "Red apple" z = "White snow" - sdb = subing.Suber(db=db, subkey='pugs.') - assert isinstance(sdb, subing.Suber) + suber = subing.Suber(db=db, subkey='pugs.') + assert isinstance(suber, subing.Suber) - sdb.put(keys=("a","1"), val=w) - sdb.put(keys=("a","2"), val=x) - sdb.put(keys=("a","3"), val=y) - sdb.put(keys=("a","4"), val=z) + suber.put(keys=("a","1"), val=w) + suber.put(keys=("a","2"), val=x) + suber.put(keys=("a","3"), val=y) + suber.put(keys=("a","4"), val=z) - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), (('a', '4'), z)] - sdb.put(keys=("b","1"), val=w) - sdb.put(keys=("b","2"), val=x) - sdb.put(keys=("bc","3"), val=y) - sdb.put(keys=("ac","4"), val=z) + suber.put(keys=("b","1"), val=w) + suber.put(keys=("b","2"), val=x) + suber.put(keys=("bc","3"), val=y) + suber.put(keys=("ac","4"), val=z) - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -149,12 +149,12 @@ def test_suber(): # test with top keys for partial tree topkeys = ("b","") # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=topkeys)] assert items == [(('b', '1'), w), (('b', '2'), x)] topkeys = ("a","") # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=topkeys)] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), @@ -162,20 +162,20 @@ def test_suber(): # test with top parameter keys = ("b", ) # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=keys, topive=True)] assert items == [(('b', '1'), w), (('b', '2'), x)] keys = ("a", ) # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=keys, topive=True)] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), (('a', '4'), z)] # Test trim - assert sdb.trim(keys=("b", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim(keys=("b", "")) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -183,20 +183,20 @@ def test_suber(): (('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim(keys=("a", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim(keys=("a", "")) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] # Test trim with top parameters - sdb.put(keys=("a","1"), val=w) - sdb.put(keys=("a","2"), val=x) - sdb.put(keys=("a","3"), val=y) - sdb.put(keys=("a","4"), val=z) - sdb.put(keys=("b","1"), val=w) - sdb.put(keys=("b","2"), val=x) - - assert sdb.trim(keys=("b",), top=True) - items = [(keys, val) for keys, val in sdb.getItemIter()] + suber.put(keys=("a","1"), val=w) + suber.put(keys=("a","2"), val=x) + suber.put(keys=("a","3"), val=y) + suber.put(keys=("a","4"), val=z) + suber.put(keys=("b","1"), val=w) + suber.put(keys=("b","2"), val=x) + + assert suber.trim(keys=("b",), topive=True) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -204,21 +204,86 @@ def test_suber(): (('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim(keys=("a",), top=True) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim(keys=("a",), topive=True) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim() - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim() + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [] - assert not sdb.trim() + assert not suber.trim() assert not os.path.exists(db.path) assert not db.opened +def test_on_suber(): + """ + Test OnSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.OnSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.OnSuber) + assert not onsuber.sdb.flags()["dupsort"] + + w = "Blue dog" + x = "Green tree" + y = "Red apple" + z = "White snow" + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + + # test getOnItemIter + + assert 0 == onsuber.appendOn(keys=("b",), val=w) + assert 1 == onsuber.appendOn(keys=("b",), val=x) + assert 0 == onsuber.appendOn(keys=("bc",), val=y) + assert 0 == onsuber.appendOn(keys=("ac",), val=z) + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('ac', '00000000000000000000000000000000'), 'White snow'), + (('b', '00000000000000000000000000000000'), 'Blue dog'), + (('b', '00000000000000000000000000000001'), 'Green tree'), + (('bc', '00000000000000000000000000000000'), 'Red apple')] + + # test getOnItemIter + + + + assert not os.path.exists(db.path) + assert not db.opened + + + def test_dup_suber(): """ Test DubSuber LMDBer sub database class @@ -452,7 +517,7 @@ def test_iodup_suber(): # test with top parameter keys = ("test", ) - items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys, topive=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] @@ -620,7 +685,7 @@ def test_ioset_suber(): # test with top parameter keys = ("test", ) - items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys, top=True)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys, topive=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] @@ -2041,6 +2106,7 @@ def test_crypt_signer_suber(): if __name__ == "__main__": test_suber() + test_on_suber() test_dup_suber() test_iodup_suber() test_ioset_suber() From f884ba26a1f4aafada08fede20dc99abf62f33fd Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 22:15:38 -0600 Subject: [PATCH 24/78] clean up fix texts call signatures --- src/keri/db/basing.py | 4 ++-- src/keri/db/dbing.py | 49 ++++++++++++++++++++++------------------- src/keri/db/subing.py | 26 +++++++++++----------- src/keri/vdr/viring.py | 4 ++-- tests/db/test_dbing.py | 38 ++++++++++++++++++++++++-------- tests/db/test_subing.py | 31 +++++++++++++++++++++++++- 6 files changed, 102 insertions(+), 50 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 140c4909..d39fd641 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1972,7 +1972,7 @@ def getFelItemPreIter(self, pre, fn=0): Returns: items (Iterator[(pre, fn, val)]): over all items starting at pre, on """ - return self.getTopOnItemIter(db=self.fels, top=pre, on=fn) + return self.getOnItemIter(db=self.fels, key=pre, on=fn) def getFelItemAllPreIter(self): @@ -1992,7 +1992,7 @@ def getFelItemAllPreIter(self): first key in database """ #return self.getAllOnItemAllPreIter(db=self.fels, key=key) - return self.getTopOnItemIter(db=self.fels, top=b'') + return self.getOnItemIter(db=self.fels, key=b'') def putDts(self, key, val): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 73430e86..8637e474 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -693,7 +693,7 @@ def delTopVal(self, db, top=b''): # For subdbs the use keys with trailing part the is monotonically # ordinal number serialized as 32 hex bytes - def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): + def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): """ Returns (int): count of of all ordinal keyed vals with top key but different on tail in db starting at ordinal number on of top. @@ -703,34 +703,37 @@ def cntTopOnVals(self, db, top, on=0, *, sep=b'.'): Parameters: db (lmdbsubdb): named sub db of lmdb - top (bytes): top key within sub db's keyspace with trailing part on + key (bytes): key within sub db's keyspace plus trailing part on on (int): ordinal number at which to initiate count sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key count = 0 - if not cursor.set_range(key): # moves to val at key >= key + if not cursor.set_range(onkey): # moves to val at key >= key return count # no values end of db - for key in cursor.iternext(values=False): # get key only at cursor + for ckey in cursor.iternext(values=False): # get key only at cursor try: - cpre, cn = splitOnKey(key, sep=sep) + ckey, cn = splitOnKey(ckey, sep=sep) except ValueError as ex: # not splittable key break - if cpre != top: # prev is now the last event for pre + if key and ckey != key: # prev is now the last event for pre break # done count = count+1 return count - def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): + def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): """ Returns iterator of triples (key, on, val), at each key over all ordinal - numbered keys with same top keyin db. Values are sorted by + numbered keys with same key + sep + on in db. Values are sorted by onKey(key, on) where on is ordinal number int and key is prefix sans on. Returned items are triples of (key, on, val) When dupsort==true then duplicates are included in items since .iternext @@ -739,28 +742,28 @@ def getTopOnItemIter(self, db, top=b'', on=0, *, sep=b'.'): Raises StopIteration Error when empty. Returns: - items (Iterator[(top, on, val)]): triples of top key, on int, val + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: db (subdb): named sub db in lmdb - top (bytes): top key within sub db's keyspace with trailing part on + key (bytes): key within sub db's keyspace plus trailing part on on (int): ordinal number at which to initiate retrieval sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if top: # not empty - key = onKey(top, on, sep=sep) # start replay at this enty 0 is earliest - else: # empty top - key = top # get all values empty key is start of db - if not cursor.set_range(key): # moves to val at key >= key - return # no values end of db - - for key, val in cursor.iternext(): # get key, val at cursor - ckey, cn = splitOnKey(key, sep=sep) - if not ckey.startswith(top): # prev is now the last event for pre - break # done - yield (ckey, cn, val) + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty + onkey = key + if not cursor.set_range(onkey): # moves to val at key >= onkey + return # no values end of db raises StopIteration + + for ckey, cval in cursor.iternext(): # get key, val at cursor + ckey, cn = splitOnKey(ckey, sep=sep) + if key and not ckey == key: + break + yield (ckey, cn, cval) # only valid for dupsort==False such as fn where only one entry per ordinal # is allowed diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 3f9bb0ff..ea3a8a55 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -200,7 +200,7 @@ def trim(self, keys: str|bytes|memoryview|Iterable=b"", *, topive=False): return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, topive=topive))) - def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", + def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]="", *, topive=False): """Iterator over items in .db that returns full items with subclass specific special hidden parts shown for debugging or testing. @@ -239,7 +239,7 @@ def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]=b"", yield (self._tokeys(key), self._des(val)) - def getItemIter(self, keys: str|bytes|memoryview|Iterable=b"", + def getItemIter(self, keys: str|bytes|memoryview|Iterable="", *, topive=False): """Iterator over items in .db subclasses that do special hidden transforms on either the keyspace or valuespace should override this method to hide @@ -403,7 +403,7 @@ def __init__(self, *pa, **kwa): super(OnSuberBase, self).__init__(*pa, **kwa) - def cntOn(self, keys: str | bytes | memoryview, on: int=0): + def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): """ Returns cnt (int): count of of all ordinal suffix keyed vals with same top @@ -417,12 +417,12 @@ def cntOn(self, keys: str | bytes | memoryview, on: int=0): combined with serialized on suffix and sep to form top key on (int): ordinal number used with onKey(pre,on) to form key. """ - return (self.db.cntTopOnVals(db=self.sdb, - top=self._tokey(keys), + return (self.db.cntOnVals(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())) - def getOnItemIter(self, keys: str | bytes | memoryview | Iterable, on: int=0): + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns items (Iterator[(top keys, on, val)]): triples of (top keys, on int, @@ -434,8 +434,8 @@ def getOnItemIter(self, keys: str | bytes | memoryview | Iterable, on: int=0): on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ - for keys, on, val in (self.db.getTopOnItemIter(db=self.sdb, - top=self._tokey(keys), on=on, sep=self.sep.encode())): + for keys, on, val in (self.db.getOnItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): yield (self._tokeys(keys), on, self._des(val)) @@ -948,7 +948,7 @@ def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): - def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ Return iterator over all the items in top branch defined by keys where @@ -1145,7 +1145,7 @@ def get(self, keys: Union[str, Iterable]): if val is not None else None) - def getItemIter(self, keys: str | bytes | memoryview | Iterable =b"", + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ Returns: @@ -1270,7 +1270,7 @@ def get(self, keys: Union[str, Iterable], decrypter: core.Decrypter = None): - def getItemIter(self, keys: str|bytes|memoryview|Iterable=b"", + def getItemIter(self, keys: str|bytes|memoryview|Iterable= "", decrypter: core.Decrypter = None, *, topive=False): """ Returns: @@ -1520,7 +1520,7 @@ def rem(self, keys: Union[str, Iterable]): """ return self.db.delVal(db=self.sdb, key=self._tokey(keys)) - def getItemIter(self, keys: str | bytes | memoryview | Iterable =b"", + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ Returns: @@ -1969,7 +1969,7 @@ def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): yield (self._tokeys(key), self._des(ioval)) - def getItemIter(self, keys: str | bytes | memoryview | Iterable = b"", + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ Return iterator over all the items including dup items for all keys diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 3314ecc0..11123c57 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -671,7 +671,7 @@ def getTelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getTopOnItemIter(db=self.tels, top=pre, on=fn) + return self.getOnItemIter(db=self.tels, key=pre, on=fn) def cntTels(self, pre, fn=0): """ @@ -685,7 +685,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntTopOnVals(db=self.tels, top=pre, on=fn) + return self.cntOnVals(db=self.tels, key=pre, on=fn) def getTibs(self, key): """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index dbafab59..68cb272c 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -317,6 +317,7 @@ def test_lmdber(): preA = b'BBKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' preB = b'EH7Oq9oxCgYa-nnNLvwhp9sFZpALILlRYyB-6n4WDi7w' preC = b'EIDA1n-WiBA0A8YOqnKrB-wWQYYC49i5zY_qrIZIicQg' + preD = b'EAYC49i5zY_qrIZIicQgIDA1n-WiBA0A8YOqnKrB-wWQ' keyA0 = onKey(preA, 0) @@ -406,37 +407,56 @@ def test_lmdber(): assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntTopOnVals(db, top=preB) == 5 + assert dber.appendTopOnVal(db, preD, digY ) == 0 + + assert dber.cntOnVals(db, key=preB) == 5 + assert dber.cntOnVals(db, key=b'') == 6 # all keys + assert dber.cntOnVals(db) == 6 # all keys # replay preB events in database - items = [item for item in dber.getTopOnItemIter(db, preB)] + items = [item for item in dber.getOnItemIter(db, preB)] assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 - items = [item for item in dber.getTopOnItemIter(db, preB, on=3)] + items = [item for item in dber.getOnItemIter(db, preB, on=3)] assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 - items = [item for item in dber.getTopOnItemIter(db, preB, on=5)] + items = [item for item in dber.getOnItemIter(db, preB, on=5)] assert items == [] # replay all events in database with pre events before and after assert dber.putVal(db, keyA0, val=digA) == True assert dber.putVal(db, keyC0, val=digC) == True - items = [item for item in dber.getTopOnItemIter(db, top=b'')] - assert items == [(preA, 0, digA), (preB, 0, digU), (preB, 1, digV), - (preB, 2, digW), (preB, 3, digX), (preB, 4, digY), + items = [item for item in dber.getOnItemIter(db, key=b'')] + assert items == [(preA, 0, digA), + (preD, 0, digY), + (preB, 0, digU), + (preB, 1, digV), + (preB, 2, digW), + (preB, 3, digX), + (preB, 4, digY), + (preC, 0, digC)] + + items = [item for item in dber.getOnItemIter(db)] + assert items == [(preA, 0, digA), + (preD, 0, digY), + (preB, 0, digU), + (preB, 1, digV), + (preB, 2, digW), + (preB, 3, digX), + (preB, 4, digY), (preC, 0, digC)] # resume replay all starting at preB on=2 top, on = splitOnKey(keyB2) - items = [item for item in dber.getTopOnItemIter(db, top=top, on=on)] + items = [item for item in dber.getOnItemIter(db, key=top, on=on)] assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] # resume replay all starting at preC on=1 - items = [item for item in dber.getTopOnItemIter(db, top=preC, on=1)] + items = [item for item in dber.getOnItemIter(db, key=preC, on=1)] assert items == [] diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index f49fa6e7..2c285a8f 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -254,8 +254,17 @@ def test_on_suber(): (('a', '00000000000000000000000000000002'), 'Red apple'), (('a', '00000000000000000000000000000003'), 'White snow')] - # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + assert 0 == onsuber.appendOn(keys=("b",), val=w) assert 1 == onsuber.appendOn(keys=("b",), val=x) @@ -264,6 +273,7 @@ def test_on_suber(): assert onsuber.cntOn(keys=("b",)) == 2 assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 items = [(keys, val) for keys, val in onsuber.getItemIter()] assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), @@ -276,7 +286,26 @@ def test_on_suber(): (('bc', '00000000000000000000000000000000'), 'Red apple')] # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + items = [item for item in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] assert not os.path.exists(db.path) From af1da8fdd62648fa8a3b45b61c78c078f350f961 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 22:27:51 -0600 Subject: [PATCH 25/78] more clean up refactoring --- src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 30 +++++++++++++++--------------- src/keri/db/subing.py | 4 ++-- tests/db/test_dbing.py | 18 +++++++++--------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d39fd641..0fc69cfc 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1952,7 +1952,7 @@ def appendFe(self, pre, val): pre is bytes identifier prefix for event val is event digest """ - return self.appendTopOnVal(db=self.fels, top=pre, val=val) + return self.appendOnVal(db=self.fels, key=pre, val=val) def getFelItemPreIter(self, pre, fn=0): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 8637e474..27bcc4ed 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -768,7 +768,7 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): # only valid for dupsort==False such as fn where only one entry per ordinal # is allowed - def appendTopOnVal(self, db, top, val, *, sep=b'.'): + def appendOnVal(self, db, key, val, *, sep=b'.'): """ Appends val in order after last previous key with same pre in db. Returns ordinal number in, on, of appended entry. Appended on is 1 greater @@ -785,42 +785,42 @@ def appendTopOnVal(self, db, top, val, *, sep=b'.'): Parameters: db (subdb): named sub db in lmdb - top (bytes): top key within sub db's keyspace with trailing part on + key (bytes): key within sub db's keyspace plus trailing part on val (bytes): serialized value to append sep (bytes): separator character for split """ # set key with fn at max and then walk backwards to find last entry at pre # if any otherwise zeroth entry at pre - key = onKey(top, MaxON, sep=sep) + onkey = onKey(key, MaxON, sep=sep) with self.env.begin(db=db, write=True, buffers=True) as txn: on = 0 # unless other cases match then zeroth entry at pre cursor = txn.cursor() - if not cursor.set_range(key): # max is past end of database + if not cursor.set_range(onkey): # max is past end of database # so either empty database or last is earlier pre or # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max ckey = cursor.key() - cpre, cn = splitOnKey(ckey, sep=sep) - if cpre == top: # last is last entry for same pre + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre ckey = cursor.key() - cpre, cn = splitOnKey(ckey, sep=sep) - if cpre == top: # last entry for pre is already at max - raise ValueError("Number part of key {} exceeds maximum" - " size.".format(ckey)) + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last entry for pre is already at max + raise ValueError(f"Number part {cn=} for key part {ckey=}" + f"exceeds maximum size.") else: # later pre so backup one entry # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre ckey = cursor.key() - cpre, cn = splitOnKey(ckey, sep=sep) - if cpre == top: # last entry at pre + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last entry at pre on = cn + 1 # increment - key = onKey(top, on, sep=sep) + onkey = onKey(key, on, sep=sep) - if not cursor.put(key, val, overwrite=False): - raise ValueError("Failed appending {} at {}.".format(val, key)) + if not cursor.put(onkey, val, overwrite=False): + raise ValueError(f"Failed appending {val=} at {key=}.") return on diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index ea3a8a55..db56269f 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -477,8 +477,8 @@ def appendOn(self, keys: str | bytes | memoryview, val (str | bytes | memoryview): serialization on (int): ordinal number used with onKey(pre,on) to form key. """ - return (self.db.appendTopOnVal(db=self.sdb, - top=self._tokey(keys), + return (self.db.appendOnVal(db=self.sdb, + key=self._tokey(keys), val=self._ser(val), sep=self.sep.encode())) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 68cb272c..73701ac2 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -352,7 +352,7 @@ def test_lmdber(): # test appendOrdValPre # empty database assert dber.getVal(db, keyB0) == None - on = dber.appendTopOnVal(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -360,7 +360,7 @@ def test_lmdber(): # earlier pre in database only assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendTopOnVal(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -369,7 +369,7 @@ def test_lmdber(): # earlier and later pre in db but not same pre assert dber.getVal(db, keyA0) == digA assert dber.putVal(db, keyC0, val=digC) == True - on = dber.appendTopOnVal(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -379,13 +379,13 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None assert dber.getVal(db, keyC0) == digC - on = dber.appendTopOnVal(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU # earlier pre and later pre and earlier entry for same pre assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendTopOnVal(db, preB, digV) + on = dber.appendOnVal(db, preB, digV) assert on == 1 assert dber.getVal(db, keyB1) == digV @@ -395,19 +395,19 @@ def test_lmdber(): assert dber.delVal(db, keyC0) == True assert dber.getVal(db, keyC0) == None # another value for preB - on = dber.appendTopOnVal(db, preB, digW) + on = dber.appendOnVal(db, preB, digW) assert on == 2 assert dber.getVal(db, keyB2) == digW # yet another value for preB - on = dber.appendTopOnVal(db, preB, digX) + on = dber.appendOnVal(db, preB, digX) assert on == 3 assert dber.getVal(db, keyB3) == digX # yet another value for preB - on = dber.appendTopOnVal(db, preB, digY ) + on = dber.appendOnVal(db, preB, digY ) assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.appendTopOnVal(db, preD, digY ) == 0 + assert dber.appendOnVal(db, preD, digY ) == 0 assert dber.cntOnVals(db, key=preB) == 5 assert dber.cntOnVals(db, key=b'') == 6 # all keys From 058ceef990c628ff7458a44fc9653ff68963ec25 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 31 Aug 2024 22:29:30 -0600 Subject: [PATCH 26/78] added test vector --- tests/db/test_subing.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 2c285a8f..549a6de9 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -307,6 +307,16 @@ def test_on_suber(): (('b',), 1, 'Green tree'), (('bc',), 0, 'Red apple')] + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + assert not os.path.exists(db.path) assert not db.opened From 1d5bfca96b3106f7895a6f2fd2b4907d04c345a2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 17:28:45 -0600 Subject: [PATCH 27/78] tests for OnIoDupSuber --- src/keri/db/dbing.py | 13 ++-- src/keri/db/subing.py | 83 ++++++++++++++++++----- tests/db/test_subing.py | 145 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 20 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 27bcc4ed..524a3762 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -700,10 +700,12 @@ def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): Full key is composed of top+sep+ When dupsort==true then duplicates are included in count since .iternext includes duplicates. + when key is empty then retrieves whole db Parameters: db (lmdbsubdb): named sub db of lmdb key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db on (int): ordinal number at which to initiate count sep (bytes): separator character for split """ @@ -738,6 +740,7 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): Returned items are triples of (key, on, val) When dupsort==true then duplicates are included in items since .iternext includes duplicates. + when key is empty then retrieves whole db Raises StopIteration Error when empty. @@ -747,6 +750,7 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): Parameters: db (subdb): named sub db in lmdb key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db on (int): ordinal number at which to initiate retrieval sep (bytes): separator character for split """ @@ -765,17 +769,18 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): break yield (ckey, cn, cval) - # only valid for dupsort==False such as fn where only one entry per ordinal - # is allowed + def appendOnVal(self, db, key, val, *, sep=b'.'): """ - Appends val in order after last previous key with same pre in db. + Appends val in order after last previous key with same pre in db where + full key has key prefix and serialized on suffix attached with sep. Returns ordinal number in, on, of appended entry. Appended on is 1 greater than previous latest on at pre. Uses onKey(pre, on) for entries. - Assumes dupsort==False so no duplicates at a given onKey. + Works with either dupsort==True or False since always creates new full + key. Append val to end of db entries with same pre but with on incremented by 1 relative to last preexisting entry at pre. diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index db56269f..1f1c679c 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -403,6 +403,25 @@ def __init__(self, *pa, **kwa): super(OnSuberBase, self).__init__(*pa, **kwa) + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): + """ + Returns: + on (int): ordinal number of newly appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.appendOnVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) + + + def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): """ Returns @@ -415,6 +434,7 @@ def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): Parameters: keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form top key + When keys is empty then counts whole database including duplicates on (int): ordinal number used with onKey(pre,on) to form key. """ return (self.db.cntOnVals(db=self.sdb, @@ -431,6 +451,7 @@ def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): Parameters: keys (str | bytes | memoryview | iterator): top keys as prefix to be combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ @@ -465,22 +486,6 @@ def __init__(self, *pa, **kwa): super(OnSuber, self).__init__(*pa, **kwa) - def appendOn(self, keys: str | bytes | memoryview, - val: str | bytes | memoryview): - """ - Returns: - on (int): ordinal number of newly appended val - - Parameters: - keys (str | bytes | memoryview | Iterable): top keys as prefix to be - combined with serialized on suffix and sep to form key - val (str | bytes | memoryview): serialization - on (int): ordinal number used with onKey(pre,on) to form key. - """ - return (self.db.appendOnVal(db=self.sdb, - key=self._tokey(keys), - val=self._ser(val), - sep=self.sep.encode())) @@ -2025,3 +2030,49 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", # used by .dels # getIoDupValsAnyPreIter(self, db, pre, on=0) + +class OnIoDupSuber(OnSuberBase, DupSuber): + """ + Sub class of IoDupSuber and OnSuberBase that supports Insertion Ordering + (IoDup) of duplicates but where the trailing part of the key space is + a serialized monotonically increasing ordinal number. This is useful for + escrows of key events etc where duplicates of likely events are maintained + in insertion order. + Insertion order is maintained by automagically prepending and stripping an + ordinal ordering proem to/from each duplicate value at a given key. + + OnIoDupSuber adds the convenience methods from OnSuberBase to IoDupSuber for + those cases where the keyspace has a trailing ordinal part. + + There are two ordinals, one in the key space and a hidden one in the duplicate + data value space. + + OnIoDupSuber supports insertion ordered multiple entries at each key + (duplicates) with dupsort==True + + Do not use if serialized length key + proem + value, is greater than 511 bytes. + This is a limitation of dupsort==True sub dbs in LMDB + + OnIoDupSuber may be more performant then IoSetSuber for values that are indices + to other sub dbs that fit the size constraint because LMDB support for + duplicates is more space efficient and code performant. + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + + """ + super(OnIoDupSuber, self).__init__(*pa, **kwa) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 549a6de9..a9f67703 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -623,6 +623,150 @@ def test_iodup_suber(): assert not db.opened +def test_oniodup_suber(): + """ + Test OnIoDupSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.OnIoDupSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.OnIoDupSuber) + assert onsuber.sdb.flags()["dupsort"] + + w = "Blue dog" + x = "Green tree" + y = "Red apple" + z = "White snow" + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + # test add duplicates + assert onsuber.add(keys=dbing.onKey("b", 0), val=w) + assert onsuber.add(keys=dbing.onKey("b", 1), val=x) + assert onsuber.add(keys=dbing.onKey("bc", 0), val=y) + assert onsuber.add(keys=dbing.onKey("ac", 0), val=z) + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('ac', '00000000000000000000000000000000'), 'White snow'), + (('b', '00000000000000000000000000000000'), 'Blue dog'), + (('b', '00000000000000000000000000000001'), 'Green tree'), + (('bc', '00000000000000000000000000000000'), 'Red apple')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + items = [item for item in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + # test with duplicates + assert onsuber.add(keys=dbing.onKey("a", 0), val=z) + assert onsuber.add(keys=dbing.onKey("a", 1), val=y) + assert onsuber.add(keys=dbing.onKey("a", 2), val=x) + assert onsuber.add(keys=dbing.onKey("a", 3), val=w) + + assert onsuber.cntOn(keys=("a",)) == 8 + assert onsuber.cntOn(keys=("a",), on=2) == 4 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter(keys=("a", ""))] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000000'), 'White snow'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000001'), 'Red apple'), + (('a', '00000000000000000000000000000002'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'Blue dog'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 0, 'White snow'), + (('a',), 1, 'Green tree'), + (('a',), 1, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'Blue dog'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'Blue dog'), + (('a',), 3, 'White snow')] + + # test append with duplicates + assert 4 == onsuber.appendOn(keys=("a",), val=x) + assert onsuber.cntOn(keys=("a",)) == 9 + + assert not os.path.exists(db.path) + assert not db.opened + + + def test_ioset_suber(): """ Test IoSetSuber LMDBer sub database class @@ -2148,6 +2292,7 @@ def test_crypt_signer_suber(): test_on_suber() test_dup_suber() test_iodup_suber() + test_oniodup_suber() test_ioset_suber() test_cat_suber() test_cesr_suber() From db25724d8b8dd84693e39225f8a60b5b7342e437 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 19:00:16 -0600 Subject: [PATCH 28/78] fixed class hierarchy for OnIoDupSuber and unit tests refactored dbing support methods more consistent --- src/keri/db/dbing.py | 198 +++++++++++++++++++++++++++------------- src/keri/db/subing.py | 70 ++++++++++---- tests/db/test_dbing.py | 29 ++++-- tests/db/test_subing.py | 21 +++-- 4 files changed, 219 insertions(+), 99 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 524a3762..3c876f87 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1721,7 +1721,67 @@ def cntIoDupVals(self, db, key): return count - def getTopIoDupItemIter(self, db, key=b''): +# used in IoDupSuber + def getIoDupItemIter(self, db, key=b'', *, ion=0): + """ + Iterates over branch of db given by key of IoDup items where each value + has 33 byte insertion ordinal number proem (prefixed) with separator. + Automagically removes (strips) proem before returning items. + + Assumes DB opened with dupsort=True + + Returns: + ioitems (Iterator): each item iterated is tuple (keys, ioval) where + each keys is actual keys tuple and each ioval is dup that + includes prefixed insertion ordering proem + empty list if no entry at keys. + Raises StopIteration when done + + + Because cursor.iternext() advances cursor after returning item its safe + to delete the item within the iteration loop. curson.iternext() works + for both dupsort==False and dupsort==True + + Raises StopIteration Error when empty. + + Parameters: + db (lmdb._Database): instance of named sub db with dupsort==False + key (bytes): truncated top key, a key space prefix to get all the items + from multiple branches of the key space. If top key is + empty then gets all items in database + ion (int): starting ordinal value, default 0 + + + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + if cursor.set_range(key): # move to val at key >= key if any + for ckey, cval in cursor.iternext(): # get key, val first dup at cursor + ckey = bytes(ckey) + if not ckey.startswith(key): # prev entry if any last in branch + break # done + cion = int(bytes(cval[:32]), 16) # convert without sep '.' in [33] + if cion < ion: # cion cursor insertion order proem as int + continue # skip dups whose cion is below ion + yield (ckey, cval) # include proem on val + return # done raises StopIteration + + +# used in IoDupSuber.getItemIter + def getTopIoDupItemIter(self, db, top=b''): """ Iterates over top branch of db given by key of IoDup items where each value has 33 byte insertion ordinal number proem (prefixed) with separator. @@ -1745,7 +1805,7 @@ def getTopIoDupItemIter(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is empty then gets all items in database @@ -1762,11 +1822,12 @@ def getTopIoDupItemIter(self, db, key=b''): before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). """ - for key, val in self.getTopItemIter(db=db, top=key): + for top, val in self.getTopItemIter(db=db, top=top): val = val[33:] # strip proem - yield (key, val) - + yield (top, val) +# not needed in IoDupSuber since not used anywhere always uses the Iter version +# below. confirm this in general and refactor any that do use to use the iter verio def getIoDupItemsNext(self, db, key=b"", skip=True): """ Return list of all dup items at next key after key in db in insertion order. @@ -1813,7 +1874,8 @@ def getIoDupItemsNext(self, db, key=b"", skip=True): items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] return items - +# don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter +# which in turen can be replaced with IoDupSuber.getItemIter def getIoDupItemsNextIter(self, db, key=b"", skip=True): """ @@ -1859,64 +1921,7 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): for key, val in cursor.iternext_dup(keys=True): yield (key, val[33:]) # slice off prepended ordering prefix - - def getIoDupItemIter(self, db, key=b'', *, ion=0): - """ - Iterates over branch of db given by key of IoDup items where each value - has 33 byte insertion ordinal number proem (prefixed) with separator. - Automagically removes (strips) proem before returning items. - - Assumes DB opened with dupsort=True - - Returns: - ioitems (Iterator): each item iterated is tuple (keys, ioval) where - each keys is actual keys tuple and each ioval is dup that - includes prefixed insertion ordering proem - empty list if no entry at keys. - Raises StopIteration when done - - - Because cursor.iternext() advances cursor after returning item its safe - to delete the item within the iteration loop. curson.iternext() works - for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items - from multiple branches of the key space. If top key is - empty then gets all items in database - ion (int): starting ordinal value, default 0 - - - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any - for ckey, cval in cursor.iternext(): # get key, val first dup at cursor - ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch - break # done - cion = int(bytes(cval[:32]), 16) # convert without sep '.' in [33] - if cion < ion: # cion cursor insertion order proem as int - continue # skip dups whose cion is below ion - yield (ckey, cval) # include proem on val - return # done raises StopIteration - +# Should not need this in OnIoDupSuber because can replace with get getIoDupItemIter def getIoDupValsAllPreIter(self, db, pre, on=0): """ @@ -2026,7 +2031,7 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): yield cursor.value()[33:] # slice off prepended ordering prefix key = snKey(pre, cnt:=cnt+1) - +# used by .dels getIoDupItemIter def getIoDupValsAnyPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for any entries @@ -2068,3 +2073,66 @@ def getIoDupValsAnyPreIter(self, db, pre, on=0): yield val[33:] # slice off prepended ordering prefix cnt = int(back, 16) key = snKey(pre, cnt:=cnt+1) + + + # methods for OnIoDup that combines IoDup value proem with On ordinal numbered + # trailing prefix + # this is so we do the proem add and strip here not in some higher level class + # like suber + + # used in OnIoDupSuber + def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): + """ + Appends val in order after last previous key with same pre in db where + full key has key prefix and serialized on suffix attached with sep and + value has ordinal proem prefixed. + Returns ordinal number in, on, of appended entry. Appended on is 1 greater + than previous latest on at pre. + Uses onKey(pre, on) for entries. + + Works with either dupsort==True or False since always creates new full + key. + + Append val to end of db entries with same pre but with on incremented by + 1 relative to last preexisting entry at pre. + + Returns: + on (int): ordinal number of newly appended val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split + """ + val = (b'%032x.' % (0)) + val # prepend ordering proem + return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) + + # used in OnIoDupSuber + def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + val = val[33:] # strip proem + yield (key, on, val) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 1f1c679c..7031796c 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1950,14 +1950,16 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) - def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): + def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable = '', + *, ion=0): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: keys (str|bytes|memoryview|Iterable): of key strs to be combined - in order to form key + in order to form key. When keys is empty then retrieves + all items in database. ion (int): starting ordinal value, default 0 Returns: @@ -2006,9 +2008,9 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, + + for key, val in self.db.getTopIoDupItemIter(db=self.sdb, top=self._tokey(keys, topive=topive)): - val = val[33:] # strip off proem yield (self._tokeys(key), self._des(val)) # ToDo @@ -2018,20 +2020,16 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", # so they work across prefixes without having the extra burden doing repeated # iter within a given pre dups. This way they could work with or without duplicates # and then the IoDupIter could just strip the proem whereas OrdSuber would not care -# OnSuber - #Although these are similar to OrdSuber in that they both have ordinal - # not hidden as last part of keys. OnIoDup suber also has proem that - # must be prefixed and stripped so not able to mixin with each other - # in multiple inheritance - # used by .kels - # getIoDupValsAllPreIter(self, db, pre, on=0): - # getIoDupValsAllPreBackIter(self, db, pre, on=0): - # getIoDupValLastAllPreIter(self.kels, pre, on=sn) - # used by .dels - # getIoDupValsAnyPreIter(self, db, pre, on=0) - - -class OnIoDupSuber(OnSuberBase, DupSuber): + +# used by .kels + # getIoDupValsAllPreIter(self, db, pre, on=0): + # getIoDupValsAllPreBackIter(self, db, pre, on=0): + # getIoDupValLastAllPreIter(self.kels, pre, on=sn) +# used by .dels + # getIoDupValsAnyPreIter(self, db, pre, on=0) + + +class OnIoDupSuber(OnSuberBase, IoDupSuber): """ Sub class of IoDupSuber and OnSuberBase that supports Insertion Ordering (IoDup) of duplicates but where the trailing part of the key space is @@ -2076,3 +2074,39 @@ def __init__(self, *pa, **kwa): """ super(OnIoDupSuber, self).__init__(*pa, **kwa) + + + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): + """ + Returns: + on (int): ordinal number of newly appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization + on (int): ordinal number used with onKey(pre,on) to form key. + """ + return (self.db.appendOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) + + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + items (Iterator[(top keys, on, val)]): triples of (top keys, on int, + deserialized val) + + Parameters: + keys (str | bytes | memoryview | iterator): top keys as prefix to be + combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 73701ac2..c9d22623 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -349,7 +349,7 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None - # test appendOrdValPre + # test appendOnValPre # empty database assert dber.getVal(db, keyB0) == None on = dber.appendOnVal(db, preB, digU) @@ -526,6 +526,8 @@ def test_lmdber(): assert dber.addIoDupVal(db, key, b'e') assert dber.getIoDupVals(db, key) == [b'm', b'a', b'w', b'e'] + + # Test getIoValsAllPreIter(self, db, pre) vals0 = [b"gamma", b"beta"] sn = 0 @@ -623,29 +625,29 @@ def test_lmdber(): (b'A.00000000000000000000000000000007', b'b')] # dups at aKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=aKey)] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=aKey)] assert items == [(b'A.00000000000000000000000000000001', b'z'), (b'A.00000000000000000000000000000001', b'm'), (b'A.00000000000000000000000000000001', b'x')] # dups at bKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=bKey)] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=bKey)] assert items == [(b'A.00000000000000000000000000000002', b'o'), (b'A.00000000000000000000000000000002', b'r'), (b'A.00000000000000000000000000000002', b'z')] # dups at cKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=cKey)] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=cKey)] assert items == [(b'A.00000000000000000000000000000004', b'h'), (b'A.00000000000000000000000000000004', b'n')] # dups at dKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=dKey)] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=dKey)] assert items == [(b'A.00000000000000000000000000000007', b'k'), (b'A.00000000000000000000000000000007', b'b')] # dups at missing key - items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, key=b"B.")] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=b"B.")] assert not items @@ -820,6 +822,21 @@ def test_lmdber(): assert items == [] # empty assert not items + # test OnIoDup methods + key = b'Z' + assert 0 == dber.appendOnIoDupVal(edb, key, val=b'k') + assert 1 == dber.appendOnIoDupVal(edb, key, val=b'l') + assert 2 == dber.appendOnIoDupVal(edb, key, val=b'm') + assert 3 == dber.appendOnIoDupVal(edb, key, val=b'n') + + assert dber.cntOnVals(edb, key) == 4 + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(edb, key=key)] + assert items == [(b'Z', 0, b'k'), + (b'Z', 1, b'l'), + (b'Z', 2, b'm'), + (b'Z', 3, b'n')] + # test IoSetVals insertion order set of vals methods. diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index a9f67703..a3db90b1 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -736,10 +736,10 @@ def test_oniodup_suber(): (('a', '00000000000000000000000000000000'), 'White snow'), (('a', '00000000000000000000000000000001'), 'Green tree'), (('a', '00000000000000000000000000000001'), 'Red apple'), - (('a', '00000000000000000000000000000002'), 'Green tree'), (('a', '00000000000000000000000000000002'), 'Red apple'), - (('a', '00000000000000000000000000000003'), 'Blue dog'), - (('a', '00000000000000000000000000000003'), 'White snow')] + (('a', '00000000000000000000000000000002'), 'Green tree'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('a', '00000000000000000000000000000003'), 'Blue dog')] # test getOnItemIter items = [item for item in onsuber.getOnItemIter(keys='a')] @@ -747,16 +747,17 @@ def test_oniodup_suber(): (('a',), 0, 'White snow'), (('a',), 1, 'Green tree'), (('a',), 1, 'Red apple'), - (('a',), 2, 'Green tree'), (('a',), 2, 'Red apple'), - (('a',), 3, 'Blue dog'), - (('a',), 3, 'White snow')] + (('a',), 2, 'Green tree'), + (('a',), 3, 'White snow'), + (('a',), 3, 'Blue dog')] items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] - assert items == [(('a',), 2, 'Green tree'), - (('a',), 2, 'Red apple'), - (('a',), 3, 'Blue dog'), - (('a',), 3, 'White snow')] + assert items ==[(('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 3, 'White snow'), + (('a',), 3, 'Blue dog')] + # test append with duplicates assert 4 == onsuber.appendOn(keys=("a",), val=x) From d55ecaf394e33cf7c6fd7b2a003465778954f30e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 19:03:21 -0600 Subject: [PATCH 29/78] added test vector --- tests/db/test_dbing.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index c9d22623..7a8c3afe 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -837,6 +837,11 @@ def test_lmdber(): (b'Z', 2, b'm'), (b'Z', 3, b'n')] + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(edb, key=key, on=2)] + assert items == [ + (b'Z', 2, b'm'), + (b'Z', 3, b'n')] + # test IoSetVals insertion order set of vals methods. From 2fc53a6326d2a13a86c43ca8dac6bb36c0ee0278 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 19:16:30 -0600 Subject: [PATCH 30/78] some clean up --- src/keri/db/dbing.py | 138 ++++++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 3c876f87..a002bf46 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1921,6 +1921,71 @@ def getIoDupItemsNextIter(self, db, key=b"", skip=True): for key, val in cursor.iternext_dup(keys=True): yield (key, val[33:]) # slice off prepended ordering prefix + + # methods for OnIoDup that combines IoDup value proem with On ordinal numbered + # trailing prefix + # this is so we do the proem add and strip here not in some higher level class + # like suber + + # used in OnIoDupSuber + def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): + """ + Appends val in order after last previous key with same pre in db where + full key has key prefix and serialized on suffix attached with sep and + value has ordinal proem prefixed. + Returns ordinal number in, on, of appended entry. Appended on is 1 greater + than previous latest on at pre. + Uses onKey(pre, on) for entries. + + Works with either dupsort==True or False since always creates new full + key. + + Append val to end of db entries with same pre but with on incremented by + 1 relative to last preexisting entry at pre. + + Returns: + on (int): ordinal number of newly appended val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split + """ + val = (b'%032x.' % (0)) + val # prepend ordering proem + return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) + + # used in OnIoDupSuber + def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + val = val[33:] # strip proem + yield (key, on, val) + + + # Should not need this in OnIoDupSuber because can replace with get getIoDupItemIter def getIoDupValsAllPreIter(self, db, pre, on=0): @@ -1996,12 +2061,12 @@ def getIoDupValsAllPreBackIter(self, db, pre, on=0): yield val[33:] key = snKey(pre, cnt:=cnt-1) - +# need to fix this so it is not stopped by gaps in on def getIoDupValLastAllPreIter(self, db, pre, on=0): """ Returns iterator of last only of dup vals of each key in insertion order - for all entries with same prefix across all sequence numbers in increasing order - without gaps starting with on (default = 0). Stops if gap or different pre. + for all entries with same key across all sequence numbers in increasing order + without gaps starting with on (default = 0). Stops if gap or different key. Assumes that key is combination of prefix and sequence number given by .snKey(). Removes prepended proem ordinal from each val before returning @@ -2028,10 +2093,11 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=on) while cursor.set_key(key): # moves to first_dup if cursor.last_dup(): # move to last_dup - yield cursor.value()[33:] # slice off prepended ordering prefix + yield cursor.value()[33:] # slice off prepended ordering proem key = snKey(pre, cnt:=cnt+1) -# used by .dels getIoDupItemIter +# used by .dels should be able to replace with self.getTopIoDupItemIter which +# gets used by IoDupSuber.getIoDupItemIter def getIoDupValsAnyPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for any entries @@ -2074,65 +2140,3 @@ def getIoDupValsAnyPreIter(self, db, pre, on=0): cnt = int(back, 16) key = snKey(pre, cnt:=cnt+1) - - # methods for OnIoDup that combines IoDup value proem with On ordinal numbered - # trailing prefix - # this is so we do the proem add and strip here not in some higher level class - # like suber - - # used in OnIoDupSuber - def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): - """ - Appends val in order after last previous key with same pre in db where - full key has key prefix and serialized on suffix attached with sep and - value has ordinal proem prefixed. - Returns ordinal number in, on, of appended entry. Appended on is 1 greater - than previous latest on at pre. - Uses onKey(pre, on) for entries. - - Works with either dupsort==True or False since always creates new full - key. - - Append val to end of db entries with same pre but with on incremented by - 1 relative to last preexisting entry at pre. - - Returns: - on (int): ordinal number of newly appended val - - Parameters: - db (subdb): named sub db in lmdb - key (bytes): key within sub db's keyspace plus trailing part on - val (bytes): serialized value to append - sep (bytes): separator character for split - """ - val = (b'%032x.' % (0)) + val # prepend ordering proem - return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) - - # used in OnIoDupSuber - def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): - """ - Returns iterator of triples (key, on, val), at each key over all ordinal - numbered keys with same key + sep + on in db. Values are sorted by - onKey(key, on) where on is ordinal number int and key is prefix sans on. - Values duplicates are sorted internally by hidden prefixed insertion order - proem ordinal - Returned items are triples of (key, on, val) - When dupsort==true then duplicates are included in items since .iternext - includes duplicates. - when key is empty then retrieves whole db - - Raises StopIteration Error when empty. - - Returns: - items (Iterator[(key, on, val)]): triples of key, on, val - - Parameters: - db (subdb): named sub db in lmdb - key (bytes): key within sub db's keyspace plus trailing part on - when key is empty then retrieves whole db - on (int): ordinal number at which to initiate retrieval - sep (bytes): separator character for split - """ - for key, on, val in self.getOnItemIter(db=db, key=key, on=on, sep=sep): - val = val[33:] # strip proem - yield (key, on, val) From 84315d6db10a8ce40ec65664c2d261d0c1ac04d3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 1 Sep 2024 20:19:15 -0600 Subject: [PATCH 31/78] more clean up refactoring --- src/keri/db/basing.py | 6 +++--- src/keri/db/dbing.py | 34 ++++++++++++++++++++++++++-------- tests/db/test_dbing.py | 4 ++-- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 0fc69cfc..c90ef387 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2585,7 +2585,7 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoDupValsAllPreIter(self.kels, pre, on=sn) + return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) def getKelBackIter(self, pre, sn=0): @@ -2609,7 +2609,7 @@ def getKelBackIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoDupValsAllPreBackIter(self.kels, pre, sn) + return self.getOnIoDupValsAllPreBackIter(self.kels, pre, sn) def getKelLastIter(self, pre, sn=0): @@ -3209,7 +3209,7 @@ def getDelIter(self, pre): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoDupValsAnyPreIter(self.dels, pre) + return self.getOnIoDupValsAnyPreIter(self.dels, pre) def putLdes(self, key, vals): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index a002bf46..8421303c 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1986,9 +1986,10 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): -# Should not need this in OnIoDupSuber because can replace with get getIoDupItemIter - def getIoDupValsAllPreIter(self, db, pre, on=0): +# without gaps is used for replay with on to provide partial replay + + def getOnIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries with same prefix across all ordinal numbers in increasing order @@ -2024,7 +2025,12 @@ def getIoDupValsAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt+1) - def getIoDupValsAllPreBackIter(self, db, pre, on=0): + + # ToDo need unit tests in dbing Used to replay backwards all duplicate + # values starting at on + # need to fix this so it is not stopped by gaps or if gap raises error as + # gap is normally a problem for replay so maybe a parameter to raise error on gap + def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries with same prefix across all sequence numbers in decreasing order without gaps @@ -2061,7 +2067,9 @@ def getIoDupValsAllPreBackIter(self, db, pre, on=0): yield val[33:] key = snKey(pre, cnt:=cnt-1) -# need to fix this so it is not stopped by gaps in on +# Used to replay forward all last duplicate values starting at on +# need to fix this so it is not stopped by gaps or if gap raises error as +# gap is normally a problem for replay so maybe a parameter to raise error on gap def getIoDupValLastAllPreIter(self, db, pre, on=0): """ Returns iterator of last only of dup vals of each key in insertion order @@ -2096,9 +2104,20 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): yield cursor.value()[33:] # slice off prepended ordering proem key = snKey(pre, cnt:=cnt+1) -# used by .dels should be able to replace with self.getTopIoDupItemIter which -# gets used by IoDupSuber.getIoDupItemIter - def getIoDupValsAnyPreIter(self, db, pre, on=0): + + +# to do do we need a replay last backwards? + + # used by .dels do we need "on" parameter? if not then should be able to + # replace with self.getTopIoDupItemIter which gets used by + # IoDupSuber.getIoDupItemIter which crosses gaps. + # duplicitous items in escrow may have gaps but do we need to have an on start + # of duplicity escrow processing? + + # use this method to inform how to cross gaps in other replays above with parameter + # to raise error if detect gap when should not be one for event logs vs escrows + + def getOnIoDupValsAnyPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for any entries with same prefix across all ordinal numbers in order including gaps @@ -2139,4 +2158,3 @@ def getIoDupValsAnyPreIter(self, db, pre, on=0): yield val[33:] # slice off prepended ordering prefix cnt = int(back, 16) key = snKey(pre, cnt:=cnt+1) - diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 7a8c3afe..a0ef7043 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -545,7 +545,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoDupValsAllPreIter(db, pre)] + vals = [bytes(val) for val in dber.getOnIoDupValsAllPreIter(db, pre)] allvals = vals0 + vals1 + vals2 assert vals == allvals @@ -589,7 +589,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoDupValsAnyPreIter(db, pre)] + vals = [bytes(val) for val in dber.getOnIoDupValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 assert vals == allvals From 047b54704a85485121787d27589d2b22189c48c1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 09:13:20 -0600 Subject: [PATCH 32/78] some more refactoring simplifying --- src/keri/db/basing.py | 5 +++-- src/keri/db/dbing.py | 17 +++++++++++------ tests/db/test_basing.py | 4 ++-- tests/db/test_dbing.py | 3 ++- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index c90ef387..fc132f52 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -3192,7 +3192,7 @@ def delDes(self, key): """ return self.delIoDupVals(self.dels, key) - def getDelIter(self, pre): + def getDelItemIter(self, pre): """ Returns iterator of all dup vals in insertion order for any entries with same prefix across all sequence numbers including gaps. @@ -3209,7 +3209,8 @@ def getDelIter(self, pre): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getOnIoDupValsAnyPreIter(self.dels, pre) + return self.getTopIoDupItemIter(self.dels, pre) + #return self.getOnIoDupValsAnyPreIter(self.dels, pre) def putLdes(self, key, vals): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 8421303c..9895b522 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1722,6 +1722,8 @@ def cntIoDupVals(self, db, key): # used in IoDupSuber +# do we really need this? Do we really need to access a subset of duplicates by +# their ion? Is not yet used so now is the time to remove this def getIoDupItemIter(self, db, key=b'', *, ion=0): """ Iterates over branch of db given by key of IoDup items where each value @@ -1988,6 +1990,8 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): # without gaps is used for replay with on to provide partial replay +# Consider using ItemIter instead of just replaying vals since ItemIter already +# works with either dupsort==True or False def getOnIoDupValsAllPreIter(self, db, pre, on=0): """ @@ -2030,6 +2034,10 @@ def getOnIoDupValsAllPreIter(self, db, pre, on=0): # values starting at on # need to fix this so it is not stopped by gaps or if gap raises error as # gap is normally a problem for replay so maybe a parameter to raise error on gap + + # Consider creating BackItemIter instead of just replaying vals since + # ItemIter already works with either dupsort==True or False + # so if we create TopBackItemIter then we can use that everywhere def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries @@ -2106,16 +2114,13 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): -# to do do we need a replay last backwards? +# ToDo do we need a replay last backwards? + - # used by .dels do we need "on" parameter? if not then should be able to - # replace with self.getTopIoDupItemIter which gets used by - # IoDupSuber.getIoDupItemIter which crosses gaps. - # duplicitous items in escrow may have gaps but do we need to have an on start - # of duplicity escrow processing? # use this method to inform how to cross gaps in other replays above with parameter # to raise error if detect gap when should not be one for event logs vs escrows + # then remove this method since not used otherwise def getOnIoDupValsAnyPreIter(self, db, pre, on=0): """ diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 825c039d..fc02d3ec 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -1926,7 +1926,7 @@ def test_fetchkeldel(): assert vals == lastvals - # test getDelIter + # test getDelItemIter preb = 'BTmuupUhPx5_yZ-Wk1x4ejhccWzwEHHzq7K0gzQPYGGw'.encode("utf-8") sn = 1 # do not start at zero key = snKey(preb, sn) @@ -1947,8 +1947,8 @@ def test_fetchkeldel(): for val in vals2: assert db.addDe(key, val) == True - vals = [bytes(val) for val in db.getDelIter(preb)] allvals = vals0 + vals1 + vals2 + vals = [bytes(val) for key, val in db.getDelItemIter(preb)] assert vals == allvals assert not os.path.exists(db.path) diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index a0ef7043..609c0894 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -589,8 +589,9 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getOnIoDupValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 + vals = [bytes(val) for key, val in dber.getTopIoDupItemIter(db, pre)] + # dber.getTopIoDupItemIter() assert vals == allvals # Setup Tests for getIoItemsNext and getIoItemsNextIter From 030d00633796a926a4192330cd1485155779c852 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 13:15:05 -0600 Subject: [PATCH 33/78] refactoring to getIoSetItemIter and getIoDupItemIter are symmetrical --- src/keri/app/storing.py | 18 ++++-- src/keri/db/dbing.py | 35 +++++----- src/keri/db/koming.py | 16 ++--- src/keri/db/subing.py | 20 +++--- tests/app/test_forwarding.py | 2 +- tests/app/test_storing.py | 20 +++--- tests/db/test_dbing.py | 20 +++--- tests/db/test_koming.py | 21 +----- tests/db/test_subing.py | 121 ++++++++++++++++------------------- 9 files changed, 128 insertions(+), 145 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index b64cf6b6..6b5d5441 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -121,14 +121,24 @@ def cloneTopicIter(self, topic, fn=0): Returns iterator of first seen exn messages with attachments for the identifier prefix pre starting at first seen order number, fn. + ToDo looks like misuse of IoSet this should not be IoSet but simply + Ordinal Numbered db. since should not be using hidden ion has not + hidden. + """ if hasattr(topic, 'encode'): topic = topic.encode("utf-8") - for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): - topic, ion = dbing.unsuffix(key) - if msg := self.msgs.get(keys=dig): - yield ion, topic, msg.encode("utf-8") + for ion, (topic, dig) in enumerate(self.getIoSetItemIter(self.tpcs, topic)): + if ion >= fn: + if msg := self.msgs.get(keys=dig): + yield ion, topic, msg.encode("utf-8") + + #for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): + #topic, ion = dbing.unsuffix(key) + #if msg := self.msgs.get(keys=dig): + #yield ion, topic, msg.encode("utf-8") + class Respondant(doing.DoDoer): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 9895b522..aa846fee 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1197,7 +1197,7 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return False - def delIoSetIokey(self, db, iokey): + def delIoSetIokey(self, db, iokey): # change this to key, ion """ Deletes val at at actual iokey that includes ordinal key suffix. @@ -1217,33 +1217,28 @@ def delIoSetIokey(self, db, iokey): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoSetItemIter(self, db, key, *, ion=0, sep=b'.'): + def getIoSetItemIter(self, db, top=b'', *, sep=b'.'): """ Returns: - items (abc.Iterator): iterator over insertion ordered set of values - at same apparent effective key where each iteration returns tuple - (iokey, val). iokey includes the ordinal key suffix. + items (Iterator[(key,val)]): iterator of tuples (key, val) where + key is apparent key with hidden insertion ordering suffixe removed + from effective key. + Iterates over top branch of insertion ordered set values where each + effective key has trailing hidden suffix of serialization of insertion + ordering ordinal. + Uses hidden ordinal key suffix for insertion ordering. - Does not work with partial effective key. Raises StopIteration Error when empty. Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key. Does not work with partial key - that is not the full effective key sans the ion part. - ion (int): starting ordinal value, default 0 + top (bytes): top key in db. When top is empty then every item in db. + sep (bytes): sep character for attached io suffix """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - iokey = suffix(key, ion, sep=sep) # start ion th value for key zeroth default - cursor = txn.cursor() - if cursor.set_range(iokey): # move to val at key >= iokey if any - for iokey, val in cursor.iternext(): # get key, val at cursor - ckey, cion = unsuffix(iokey, sep=sep) - if ckey != key: # prev entry if any was the last entry for key - break # done - yield (iokey, val) # another entry at key - return # done raises StopIteration + for iokey, val in self.getTopItemIter(db=db, top=top): + key, ion = splitOnKey(iokey, sep=sep) + yield (key, val) @@ -1685,6 +1680,8 @@ def delIoDupVal(self, db, key, val): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return False + #def delIoDupIonVal(self, db, key, ion): # to match delIoSetIoKey + def cntIoDupVals(self, db, key): """ diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index 05ac15a2..e3cda6a5 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -591,27 +591,27 @@ def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItemIter(self, keys: Union[str, Iterable], *, ion=0): + def getIoSetItemIter(self, keys: Union[str, Iterable] = ''): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: - keys (Iterable): of key strs to be combined in order to form key - ion (int): starting ordinal value, default 0 + keys (Iterable): of key strs to be combined in order to form apparent + key without insertion ordering suffix. When keys + is empty then retrieves all items in db. Returns: - ioitems (Iterator): each item iterated is tuple (iokeys, val) where - each iokeys is actual keys tuple including hidden suffix and + items (Iterator): each item iterated is tuple (keys, val) where + each keys is apparent keys tuple without hidden suffix and each val is str empty list if no entry at keys. Raises StopIteration when done """ for iokey, val in self.db.getIoSetItemIter(db=self.sdb, - key=self._tokey(keys), - ion=ion, - sep=self.sep): + top=self._tokey(keys), + sep=self.sep.encode()): yield (self._tokeys(iokey), self.deserializer(val)) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 7031796c..86c76a8a 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -928,28 +928,28 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)) - def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable, *, ion=0): + def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable = ''): """ Gets (iokeys, val) ioitems iterator at key made from keys where key is apparent effective key and items all have same apparent effective key Parameters: - keys (Iterable): of key strs to be combined in order to form key - ion (int): starting ordinal value, default 0 + keys (Iterable): of key strs to be combined in order to form apparent + key. When key is empty then retrieves all items in db. + Returns: - ioitems (Iterator): each item iterated is tuple (iokeys, val) where - each iokeys is actual keys tuple including hidden suffix and + items (Iterator): each item iterated is tuple (keys, val) where + each keys is apparent keys tuple without hidden suffix and each val is str empty list if no entry at keys. Raises StopIteration when done """ - for iokey, val in self.db.getIoSetItemIter(db=self.sdb, - key=self._tokey(keys), - ion=ion, - sep=self.sep): - yield (self._tokeys(iokey), self._des(val)) + for key, val in self.db.getIoSetItemIter(db=self.sdb, + top=self._tokey(keys), + sep=self.sep.encode()): + yield (self._tokeys(key), self._des(val)) diff --git a/tests/app/test_forwarding.py b/tests/app/test_forwarding.py index 6f3523cd..0fa1b3c5 100644 --- a/tests/app/test_forwarding.py +++ b/tests/app/test_forwarding.py @@ -68,7 +68,7 @@ def test_postman(seeder): doist.exit() msgs = [] - for _, topic, msg in mbx.cloneTopicIter(topic=recpHab.pre + "/echo", fn=0): + for _, topic, msg in mbx.cloneTopicIter(topic=recpHab.pre + "/echo"): msgs.append(msg) assert len(msgs) == 1 diff --git a/tests/app/test_storing.py b/tests/app/test_storing.py index d37eeae3..f0aebdd4 100644 --- a/tests/app/test_storing.py +++ b/tests/app/test_storing.py @@ -85,7 +85,7 @@ def test_mailboxing(): mber.storeMsg(topic=dest.qb64b, msg=exn.raw) msgs = [] - for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=0): + for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b): msgs.append((fn, msg)) assert(len(msgs)) == 10 @@ -95,18 +95,18 @@ def test_mailboxing(): d = exn.ked["a"] assert d["b"] == idx - msgs = [] - for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=10): - msgs.append(msg) + #msgs = [] + #for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=10): + #msgs.append(msg) - assert(len(msgs)) == 0 + #assert(len(msgs)) == 0 - msgs = [] - for tn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=4): - msgs.append((tn, msg)) + #msgs = [] + #for tn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=4): + #msgs.append((tn, msg)) - assert(len(msgs)) == 6 - assert msgs[0][0] == 4 + #assert(len(msgs)) == 6 + #assert msgs[0][0] == 4 diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 609c0894..e48f706c 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -930,15 +930,15 @@ def test_lmdber(): assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] - assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemIter(db, key0)] == - [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - (b'ABC.ZYX.00000000000000000000000000000001', b'm'), - (b'ABC.ZYX.00000000000000000000000000000002', b'x'), - (b'ABC.ZYX.00000000000000000000000000000003', b'a')]) - - for iokey, val in dber.getIoSetItemIter(db, key0): - assert dber.delIoSetIokey(db, iokey) - assert dber.getIoSetVals(db, key0) == [] + #assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemIter(db, key0)] == + #[(b'ABC.ZYX.00000000000000000000000000000000', b'z'), + #(b'ABC.ZYX.00000000000000000000000000000001', b'm'), + #(b'ABC.ZYX.00000000000000000000000000000002', b'x'), + #(b'ABC.ZYX.00000000000000000000000000000003', b'a')]) + + #for iokey, val in dber.getIoSetItemIter(db, key0): + #assert dber.delIoSetIokey(db, iokey) + #assert dber.getIoSetVals(db, key0) == [] vals3 = [b"q", b"e"] assert dber.setIoSetVals(db, key2, vals3) @@ -967,7 +967,7 @@ def test_lmdber(): dber.cntIoSetVals(db, empty_key) dber.delIoSetVals(db, empty_key) dber.delIoSetVal(db, empty_key, some_value) - dber.getIoSetItemIter(db, empty_key) + #dber.getIoSetItemIter(db, empty_key) with pytest.raises(KeyError): dber.delIoSetIokey(db, empty_key) with pytest.raises(KeyError): diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index 1359bb28..86c305a6 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -758,7 +758,7 @@ def __iter__(self): loc = locDB.get(keys=(end.eid, scheme)) assert loc == wit3loc - # test IoItem methods + ## test IoItem methods iokeys0 = [f'{cid0}.witness.00000000000000000000000000000000'.encode("utf-8"), f'{cid0}.witness.00000000000000000000000000000001'.encode("utf-8"), f'{cid0}.witness.00000000000000000000000000000002'.encode("utf-8")] @@ -775,28 +775,13 @@ def __iter__(self): i = 0 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0): + for keys, end in endDB.getIoSetItemIter(keys=keys0): assert end == ends[i] - assert iokeys == iokeys0[i] + assert keys == keys0 i += 1 - i = 0 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0): - assert end == ends[i] - assert iokeys == iokeys0[i] - i += 1 - i = 1 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0, ion=i): - assert end == ends[i] - assert iokeys == iokeys0[i] - i += 1 - i = 1 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0, ion=i): - assert end == ends[i] - assert iokeys == iokeys0[i] - i += 1 # test getAllItemIter ends = ends + [wit3end] diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index a3db90b1..7d7dc16f 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -842,21 +842,15 @@ def test_ioset_suber(): (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys1)] - assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), - (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys0)] - assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), - (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] - - items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys1, ion=1)] - assert items ==[(('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] + items = [(keys, val) for keys, val in iosuber.getIoSetItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!')] - items = [(iokeys, val) for iokeys, val in iosuber.getIoSetItemIter(keys=keys0, ion=1)] - assert items == [(('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + items = [(keys, val) for keys, val in iosuber.getIoSetItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] # Test with top keys @@ -952,10 +946,10 @@ def test_cesr_ioset_suber(): assert db.name == "test" assert db.opened - sdb = subing.CesrIoSetSuber(db=db, subkey='bags.', klas=coring.Saider) - assert isinstance(sdb, subing.CesrIoSetSuber) - assert issubclass(sdb.klas, coring.Saider) - assert not sdb.sdb.flags()["dupsort"] + cisuber = subing.CesrIoSetSuber(db=db, subkey='bags.', klas=coring.Saider) + assert isinstance(cisuber, subing.CesrIoSetSuber) + assert issubclass(cisuber.klas, coring.Saider) + assert not cisuber.sdb.flags()["dupsort"] seqner0 = coring.Seqner(sn=20) seq0 = seqner0.qb64 @@ -1029,63 +1023,63 @@ def test_cesr_ioset_suber(): said2 = saider2.qb64 assert said2 == 'EJxOaEsBSObrcmrsnlfHOdVAowGhUBKoE2Ce3TZ4Mhgu' - assert sdb.put(keys=keys0, vals=[saider1, saider0]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.put(keys=keys0, vals=[saider1, saider0]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said1, said0] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said0 - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert cisuber.rem(keys0) + actuals = cisuber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert cisuber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[saider0, saider1]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.put(keys=keys0, vals=[saider0, saider1]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said0, said1] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said1 - assert sdb.add(keys=keys0, val=saider2) - assert sdb.cnt(keys0) == 3 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.add(keys=keys0, val=saider2) + assert cisuber.cnt(keys0) == 3 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 3 sers = [actual.qb64 for actual in actuals] assert sers == [said0, said1, said2] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said2 - assert sdb.pin(keys=keys0, vals=[saider1, saider2]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.pin(keys=keys0, vals=[saider1, saider2]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said1, said2] - assert sdb.put(keys=keys1, vals=[saider2, saider1, saider0]) - assert sdb.cnt(keys1) == 3 - actuals = sdb.get(keys=keys1) # insertion order not lexicographic + assert cisuber.put(keys=keys1, vals=[saider2, saider1, saider0]) + assert cisuber.cnt(keys1) == 3 + actuals = cisuber.get(keys=keys1) # insertion order not lexicographic assert len(actuals) == 3 sers = [actual.qb64 for actual in actuals] assert sers == [said2, said1, said0] - assert sdb.rem(keys=keys1, val=saider1) - assert sdb.cnt(keys1) == 2 - actuals = sdb.get(keys=keys1) # insertion order not lexicographic + assert cisuber.rem(keys=keys1, val=saider1) + assert cisuber.cnt(keys1) == 2 + actuals = cisuber.get(keys=keys1) # insertion order not lexicographic sers = [actual.qb64 for actual in actuals] assert sers == [said2, said0] - sers = [val.qb64 for val in sdb.getIter(keys=keys1)] + sers = [val.qb64 for val in cisuber.getIter(keys=keys1)] assert sers == [said2, said0] - items = [(keys, val.qb64) for keys, val in sdb.getItemIter()] + items = [(keys, val.qb64) for keys, val in cisuber.getItemIter()] assert items == [ (keys1, said2), (keys1, said0), @@ -1094,7 +1088,7 @@ def test_cesr_ioset_suber(): ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getFullItemIter()] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getFullItemIter()] assert items == [ ((*keys1, '00000000000000000000000000000000'), said2), ((*keys1, '00000000000000000000000000000002'), said0), @@ -1102,38 +1096,35 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItemIter(keys=keys1)] - assert items == [ - ((*keys1, '00000000000000000000000000000000'), said2), - ((*keys1, '00000000000000000000000000000002'), said0), - ] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getIoSetItemIter(keys=keys0)] + assert items == [(keys0, said1), (keys0, said2)] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getIoSetItemIter(keys=keys1)] assert items == [ - ((*keys0, '00000000000000000000000000000000'), said1), - ((*keys0, '00000000000000000000000000000001'), said2), + (keys1, said2), + (keys1, said0), ] topkeys = (seq1, "") - items = [(keys, val.qb64) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val.qb64) for keys, val in cisuber.getItemIter(keys=topkeys)] assert items == [ (keys1, said2), (keys1, said0), ] topkeys = (seq0, "") - items = [(iokeys, val.qb64) for iokeys, val in sdb.getFullItemIter(keys=topkeys)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getFullItemIter(keys=topkeys)] assert items == [ ((*keys0, '00000000000000000000000000000000'), said1), ((*keys0, '00000000000000000000000000000001'), said2), ] - for iokeys, val in sdb.getFullItemIter(): - assert sdb.remIokey(iokeys=iokeys) + for iokeys, val in cisuber.getFullItemIter(): + assert cisuber.remIokey(iokeys=iokeys) - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 + assert cisuber.cnt(keys=keys0) == 0 + assert cisuber.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) @@ -1774,15 +1765,15 @@ def test_cat_cesr_ioset_suber(): (keys2 + ('00000000000000000000000000000001', ), [sqr2.qb64, dgr2.qb64]) ] - items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItemIter(keys=keys1)] - assert items == [(keys1 + ('00000000000000000000000000000000', ), [sqr2.qb64, dgr2.qb64])] + items = [(keys, [val.qb64 for val in vals]) + for keys, vals in sdb.getIoSetItemIter(keys=keys1)] + assert items == [(keys1, [sqr2.qb64, dgr2.qb64])] - items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItemIter(keys=keys0)] + items = [(keys, [val.qb64 for val in vals]) + for keys, vals in sdb.getIoSetItemIter(keys=keys0)] assert items == [ - (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), - (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), + (keys0, [sqr0.qb64, dgr0.qb64]), + (keys0, [sqr1.qb64, dgr1.qb64]), ] From 6b11ac18ebad28698cc970cada802a1eaed0e24a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 14:29:47 -0600 Subject: [PATCH 34/78] refactored out getIoSetItemIter and getIoDupItemIter they are now same functionality as getItemIter so no longer neede. --- src/keri/app/storing.py | 2 +- src/keri/db/dbing.py | 63 +-------------------------- src/keri/db/koming.py | 96 +++++++++++++++++++++-------------------- src/keri/db/subing.py | 73 +++++++++---------------------- tests/db/test_dbing.py | 58 ++++++++++++------------- tests/db/test_koming.py | 8 +--- tests/db/test_subing.py | 32 ++++++-------- 7 files changed, 116 insertions(+), 216 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 6b5d5441..c003f39e 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -129,7 +129,7 @@ def cloneTopicIter(self, topic, fn=0): if hasattr(topic, 'encode'): topic = topic.encode("utf-8") - for ion, (topic, dig) in enumerate(self.getIoSetItemIter(self.tpcs, topic)): + for ion, (topic, dig) in enumerate(self.getTopIoSetItemIter(self.tpcs, topic)): if ion >= fn: if msg := self.msgs.get(keys=dig): yield ion, topic, msg.encode("utf-8") diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index aa846fee..27962159 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1217,7 +1217,7 @@ def delIoSetIokey(self, db, iokey): # change this to key, ion " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoSetItemIter(self, db, top=b'', *, sep=b'.'): + def getTopIoSetItemIter(self, db, top=b'', *, sep=b'.'): """ Returns: items (Iterator[(key,val)]): iterator of tuples (key, val) where @@ -1718,67 +1718,6 @@ def cntIoDupVals(self, db, key): return count -# used in IoDupSuber -# do we really need this? Do we really need to access a subset of duplicates by -# their ion? Is not yet used so now is the time to remove this - def getIoDupItemIter(self, db, key=b'', *, ion=0): - """ - Iterates over branch of db given by key of IoDup items where each value - has 33 byte insertion ordinal number proem (prefixed) with separator. - Automagically removes (strips) proem before returning items. - - Assumes DB opened with dupsort=True - - Returns: - ioitems (Iterator): each item iterated is tuple (keys, ioval) where - each keys is actual keys tuple and each ioval is dup that - includes prefixed insertion ordering proem - empty list if no entry at keys. - Raises StopIteration when done - - - Because cursor.iternext() advances cursor after returning item its safe - to delete the item within the iteration loop. curson.iternext() works - for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items - from multiple branches of the key space. If top key is - empty then gets all items in database - ion (int): starting ordinal value, default 0 - - - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any - for ckey, cval in cursor.iternext(): # get key, val first dup at cursor - ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch - break # done - cion = int(bytes(cval[:32]), 16) # convert without sep '.' in [33] - if cion < ion: # cion cursor insertion order proem as int - continue # skip dups whose cion is below ion - yield (ckey, cval) # include proem on val - return # done raises StopIteration - - # used in IoDupSuber.getItemIter def getTopIoDupItemIter(self, db, top=b''): """ diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index e3cda6a5..1354163c 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -73,23 +73,43 @@ def __init__(self, db: dbing.LMDBer, *, self.sep = sep if sep is not None else self.Sep - def _tokey(self, keys: Union[str, bytes, memoryview, Iterable]): + def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], + topive: bool=False): """ - Converts key to key str with proper separators and returns key bytes. - If key is already str then returns. Else If key is iterable (non-str) - of strs then joins with separator converts to bytes and returns + Converts keys to key bytes with proper separators and returns key bytes. + If keys is already str or bytes or memoryview then returns key bytes. + Else If keys is iterable (non-str) of strs or bytes then joins with + separator converts to key bytes and returns. When keys is iterable and + topive is True then enables partial key from top branch of key space given + by partial keys by appending separator to end of partial key + + Returns: + key (bytes): each element of keys is joined by .sep. If top then last + char of key is also .sep Parameters: - keys (Union[str, bytes, Iterable]): str, bytes, or Iterable of str. + keys (str | bytes | memoryview | Iterable[str | bytes]): db key or + Iterable of (str | bytes) to form key. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ + if hasattr(keys, "encode"): # str + return keys.encode("utf-8") if isinstance(keys, memoryview): # memoryview of bytes return bytes(keys) # return bytes - if hasattr(keys, "encode"): # str - return keys.encode("utf-8") # convert to bytes elif hasattr(keys, "decode"): # bytes - return keys # return as is - return (self.sep.join(keys).encode("utf-8")) # iterable so join + return keys + if topive and keys[-1]: # topive and keys is not already partial + keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep + return (self.sep.join(key.decode() if hasattr(key, "decode") else key + for key in keys).encode("utf-8")) + def _tokeys(self, key: Union[str, bytes, memoryview]): @@ -591,33 +611,7 @@ def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItemIter(self, keys: Union[str, Iterable] = ''): - """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form apparent - key without insertion ordering suffix. When keys - is empty then retrieves all items in db. - - Returns: - items (Iterator): each item iterated is tuple (keys, val) where - each keys is apparent keys tuple without hidden suffix and - each val is str - empty list if no entry at keys. - Raises StopIteration when done - - """ - for iokey, val in self.db.getIoSetItemIter(db=self.sdb, - top=self._tokey(keys), - sep=self.sep.encode()): - yield (self._tokeys(iokey), self.deserializer(val)) - - - - - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: Union[str, Iterable]=b"", *, topive=False): """Get items iterator Returns: items (Iterator): of (key, val) tuples over the all the items in @@ -627,17 +621,27 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): Returned key in each item has ordinal suffix removed. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from + keys (Iterable): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self.deserializer(val)) - + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + """ + for iokey, val in self.db.getTopIoSetItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive), + sep=self.sep.encode()): + yield (self._tokeys(iokey), self.deserializer(val)) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 86c76a8a..9379ca3a 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -95,7 +95,7 @@ def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], topive: bool=False): """ Converts keys to key bytes with proper separators and returns key bytes. - If keys is already str or bytes then returns key bytes. + If keys is already str or bytes or memoryview then returns key bytes. Else If keys is iterable (non-str) of strs or bytes then joins with separator converts to key bytes and returns. When keys is iterable and topive is True then enables partial key from top branch of key space given @@ -928,31 +928,6 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): sep=self.sep)) - def getIoSetItemIter(self, keys: str|bytes|memoryview|Iterable = ''): - """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form apparent - key. When key is empty then retrieves all items in db. - - - Returns: - items (Iterator): each item iterated is tuple (keys, val) where - each keys is apparent keys tuple without hidden suffix and - each val is str - empty list if no entry at keys. - Raises StopIteration when done - - """ - for key, val in self.db.getIoSetItemIter(db=self.sdb, - top=self._tokey(keys), - sep=self.sep.encode()): - yield (self._tokeys(key), self._des(val)) - - - def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ @@ -985,14 +960,11 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", partial ending in sep regardless of top value """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, topive=topive)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) + for key, val in self.db.getTopIoSetItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive), sep=self.sep.encode()): yield (self._tokeys(key), self._des(val)) - - class CesrIoSetSuber(CesrSuberBase, IoSetSuber): """ Subclass of CesrSuber and IoSetSuber. @@ -1950,30 +1922,27 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) - def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable = '', - *, ion=0): - """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key + #def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable = ''): + #""" + #Gets (iokeys, val) ioitems iterator at key made from keys where key is + #apparent effective key and items all have same apparent effective key - Parameters: - keys (str|bytes|memoryview|Iterable): of key strs to be combined - in order to form key. When keys is empty then retrieves - all items in database. - ion (int): starting ordinal value, default 0 + #Parameters: + #keys (str|bytes|memoryview|Iterable): of key strs to be combined + #in order to form key. When keys is empty then retrieves + #all items in database. - Returns: - ioitems (Iterator): each item iterated is tuple (keys, ioval) where - each keys is actual keys tuple and each ioval is dup that - includes prefixed insertion ordering proem. - Empty list if no entry at keys. - Raises StopIteration when done + #Returns: + #items (Iterator): each item iterated is tuple (keys, val) where + #each keys is actual keys tuple and each val is dup without + #prefixed insertion ordering proem. + #Empty list if no entry at keys. + #Raises StopIteration when done - """ - for key, ioval in self.db.getIoDupItemIter(db=self.sdb, - key=self._tokey(keys), - ion=ion): - yield (self._tokeys(key), self._des(ioval)) + #""" + #for key, val in self.db.getIoDupItemIter(db=self.sdb, + #key=self._tokey(keys)): + #yield (self._tokeys(key), self._des(val)) def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index e48f706c..caae1bb5 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -653,46 +653,42 @@ def test_lmdber(): # test getIoDupItemIter - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb)] # default all - assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000000.z'), - (b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), - (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000000.o'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000001.r'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z'), - (b'A.00000000000000000000000000000004', b'00000000000000000000000000000000.h'), - (b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n'), - (b'A.00000000000000000000000000000007', b'00000000000000000000000000000000.k'), - (b'A.00000000000000000000000000000007', b'00000000000000000000000000000001.b')] - - # all keys but ion is non zero - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, ion=1)] # default all - assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), - (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000001.r'), - (b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z'), - (b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n'), - (b'A.00000000000000000000000000000007', b'00000000000000000000000000000001.b')] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x'), + (b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z'), + (b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n'), + (b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] - # dups at aKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=aKey, ion=1)] - assert items == [(b'A.00000000000000000000000000000001', b'00000000000000000000000000000001.m'), - (b'A.00000000000000000000000000000001', b'00000000000000000000000000000002.x')] + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=aKey)] + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x')] # dups at bKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=bKey, ion=2)] - assert items == [(b'A.00000000000000000000000000000002', b'00000000000000000000000000000002.z')] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=bKey)] + assert items == [(b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z')] # dups at cKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=cKey, ion=1)] - assert items ==[(b'A.00000000000000000000000000000004', b'00000000000000000000000000000001.n')] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=cKey)] + assert items == [(b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n')] # dups at dKey - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=dKey, ion=3)] - assert not items + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=dKey)] + assert items == [(b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] # dups at missing key - items = [(ikey, bytes(ival)) for ikey, ival in dber.getIoDupItemIter(edb, key=b"B.")] + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=b"B.")] assert not items diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index 86c305a6..76a7e83a 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -773,17 +773,13 @@ def __iter__(self): 'witness', '00000000000000000000000000000002')] - + # test getItemIter i = 0 - for keys, end in endDB.getIoSetItemIter(keys=keys0): + for keys, end in endDB.getItemIter(keys=keys0): assert end == ends[i] assert keys == keys0 i += 1 - - - - # test getAllItemIter ends = ends + [wit3end] i = 0 for keys, end in endDB.getItemIter(): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 7d7dc16f..64b599a8 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -530,21 +530,17 @@ def test_iodup_suber(): (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] - items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys1)] - assert items == [(('test_key', '0002'), '00000000000000000000000000000000.Not my type.'), - (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), - (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] + + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!')] - items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys0)] - assert items == [(('test_key', '0001'), '00000000000000000000000000000000.See ya later.'), - (('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!')] - items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys1, ion=1)] - assert items ==[(('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), - (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] - items = [(keys, ioval) for keys, ioval in ioduber.getIoDupItemIter(keys=keys0, ion=1)] - assert items == [(('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!')] # Test with top keys assert ioduber.put(keys=("test", "pop"), vals=[sal, sue, sam]) @@ -843,11 +839,11 @@ def test_ioset_suber(): (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(keys, val) for keys, val in iosuber.getIoSetItemIter(keys=keys0)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys0)] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!')] - items = [(keys, val) for keys, val in iosuber.getIoSetItemIter(keys=keys1)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys1)] assert items == [(('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'Hello sailer!'), (('test_key', '0002'), 'A real charmer!')] @@ -1096,10 +1092,10 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - items = [(iokeys, val.qb64) for iokeys, val in cisuber.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getItemIter(keys=keys0)] assert items == [(keys0, said1), (keys0, said2)] - items = [(iokeys, val.qb64) for iokeys, val in cisuber.getIoSetItemIter(keys=keys1)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getItemIter(keys=keys1)] assert items == [ (keys1, said2), (keys1, said0), @@ -1766,11 +1762,11 @@ def test_cat_cesr_ioset_suber(): ] items = [(keys, [val.qb64 for val in vals]) - for keys, vals in sdb.getIoSetItemIter(keys=keys1)] + for keys, vals in sdb.getItemIter(keys=keys1)] assert items == [(keys1, [sqr2.qb64, dgr2.qb64])] items = [(keys, [val.qb64 for val in vals]) - for keys, vals in sdb.getIoSetItemIter(keys=keys0)] + for keys, vals in sdb.getItemIter(keys=keys0)] assert items == [ (keys0, [sqr0.qb64, dgr0.qb64]), (keys0, [sqr1.qb64, dgr1.qb64]), From d3cd1fccaf21782760dc9269fe2029db2609e18f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 15:21:19 -0600 Subject: [PATCH 35/78] more refactoring. --- src/keri/db/dbing.py | 9 +++++++-- src/keri/db/koming.py | 21 +++++++++++---------- src/keri/db/subing.py | 5 +++-- tests/db/test_koming.py | 13 ++++--------- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 27962159..354bcd99 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -919,7 +919,7 @@ def addIoSetVal(self, db, key, val, *, sep=b'.'): iokey = suffix(key, ion, sep=sep) # ion is at add on amount return cursor.put(iokey, val, dupdata=False, overwrite=False) - + # deprecated since violates set property of each item in a set is unique def appendIoSetVal(self, db, key, val, *, sep=b'.'): """ Append val non-idempotently to insertion ordered set of values all with @@ -1196,10 +1196,15 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return cursor.delete() # delete also moves to next so doubly moved return False - + # deprecated since violates set property of each item in a set is unique + # this is needed to remove non-idempotently appended items which violate + # set property def delIoSetIokey(self, db, iokey): # change this to key, ion """ Deletes val at at actual iokey that includes ordinal key suffix. + Need this to delete value that is non-idempotently added with + appendIoSetVal where multiple of same value exist at same key because + delIoSetVal will only delete the first one in list. Returns: result (bool): True if val was deleted at iokey. False otherwise diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index 1354163c..54a33696 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -596,19 +596,20 @@ def rem(self, keys: Union[str, Iterable], val=None): key=self._tokey(keys), sep=self.sep) - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): - """ - Removes entry at iokeys + #def remIokey(self, iokeys: str | bytes | memoryview | Iterable): + #""" + #Removes entries at iokeys - Parameters: - iokeys (tuple): of key str or tuple of key strs to be combined in - order to form key + #Parameters: + #iokeys (str | bytes | memoryview | Iterable): of key str or + #tuple of key strs to be combined in order to form key - Returns: - result (bool): True if key exists so delete successful. False otherwise + #Returns: + #result (bool): True if key exists so delete successful. False otherwise + + #""" + #return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) def getItemIter(self, keys: Union[str, Iterable]=b"", *, topive=False): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 9379ca3a..87ba4190 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -765,7 +765,7 @@ def add(self, keys: str | bytes | memoryview | Iterable, val=self._ser(val), sep=self.sep)) - + # deprecated since violates set property of each item in a set is unique def append(self, keys: str | bytes | memoryview | Iterable, val: str | bytes | memoryview): """Append val non-idempotently to insertion ordered set of values all with @@ -899,7 +899,8 @@ def rem(self, keys: str | bytes | memoryview | Iterable, key=self._tokey(keys), sep=self.sep) - + # deprecated since violates set property of each item in a set is unique + # this is to be able to remove non idempotent append def remIokey(self, iokeys: str | bytes | memoryview | Iterable): """ Removes entries at iokeys diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index 76a7e83a..ecc58ff9 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -827,16 +827,11 @@ def __iter__(self): assert iokeys == iokeysall[i] i += 1 - i = 0 - for iokeys, end in endDB.getFullItemIter(): - assert end == ends[i] - assert iokeys == iokeysall[i] - i += 1 - assert endDB.remIokey(iokeys) - - assert endDB.cnt(keys0) == 0 - assert endDB.cnt(keys1) == 0 + #for iokeys, val in endDB.getFullItemIter(): + #assert endDB.remIokey(iokeys=iokeys) + #assert endDB.cnt(keys=keys0) == 0 + #assert endDB.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) assert not db.opened From 03e26c6cdb258915c858098b86bce8e19de90637 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 16:05:03 -0600 Subject: [PATCH 36/78] created CesrOnSuber subclass with unit test --- src/keri/db/escrowing.py | 2 +- src/keri/db/subing.py | 27 ++++++++++ tests/db/test_subing.py | 112 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index dcf741d0..082833cb 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -170,7 +170,7 @@ def escrowStateNotice(self, *, typ, pre, aid, serder, saider, dater, cigars=None for cigar in cigars: # process each couple to verify sig and write to db self.cigardb.put(keys=keys, vals=[(cigar.verfer, cigar)]) - return self.escrowdb.put(keys=(typ, pre, aid), vals=[saider]) # overwrite + return self.escrowdb.put(keys=(typ, pre, aid), vals=[saider]) # does not overwrite def updateReply(self, aid, serder, saider, dater): """ diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 87ba4190..5242bf95 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -568,6 +568,33 @@ def __init__(self, *pa, **kwa): super(CesrSuber, self).__init__(*pa, **kwa) +class CesrOnSuber(CesrSuberBase, OnSuberBase, Suber): + """ + Subclass of CesrSuberBase, OnSuberBase, and Suber that adds methods for + keys with ordinal numbered suffixes and values that are Cesr serializations + of Matter subclass ducktypes. + + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(CesrOnSuber, self).__init__(*pa, **kwa) + + class CatCesrSuberBase(CesrSuberBase): """ Base Class whose values stored in db are a concatenation of the .qb64b property diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 64b599a8..a10a7a53 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -619,7 +619,7 @@ def test_iodup_suber(): assert not db.opened -def test_oniodup_suber(): +def test_on_iodup_suber(): """ Test OnIoDupSuber LMDBer sub database class """ @@ -1468,6 +1468,113 @@ def test_cesr_suber(): assert not db.opened """Done Test""" +def test_cesr_on_suber(): + """ + Test CesrOnSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.CesrOnSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.CesrOnSuber) + assert not onsuber.sdb.flags()["dupsort"] + + prew = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + w = coring.Matter(qb64=prew) + prex = "BHHzqZWzwE-Wk7K0gzQPYGGwTmuupUhPx5_y1x4ejhcc" + x = coring.Matter(qb64=prex) + prey = "EHHzqZWzwE-Wk7K0gzQPYGGwTmuupUhPx5_y1x4ejhcc" + y = coring.Matter(qb64=prey) + prez = "EAPYGGwTmuupWzwEHHzq7K0gzUhPx5_yZ-Wk1x4ejhcc" + z = coring.Matter(qb64=prez) + + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val.qb64) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), w.qb64), + (('a', '00000000000000000000000000000001'), x.qb64), + (('a', '00000000000000000000000000000002'), y.qb64), + (('a', '00000000000000000000000000000003'), z.qb64)] + + # test getOnItemIter + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, y.qb64), + (('a',), 3, z.qb64)] + + assert 0 == onsuber.appendOn(keys=("ac",), val=z) + assert 0 == onsuber.appendOn(keys=("b",), val=w) + assert 1 == onsuber.appendOn(keys=("b",), val=x) + assert 0 == onsuber.appendOn(keys=("bc",), val=y) + + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val.qb64) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), w.qb64), + (('a', '00000000000000000000000000000001'), x.qb64), + (('a', '00000000000000000000000000000002'), y.qb64), + (('a', '00000000000000000000000000000003'), z.qb64), + (('ac', '00000000000000000000000000000000'), z.qb64), + (('b', '00000000000000000000000000000000'), w.qb64), + (('b', '00000000000000000000000000000001'), x.qb64), + (('bc', '00000000000000000000000000000000'), y.qb64)] + + # test getOnItemIter + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, w.qb64), + (('b',), 1, x.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, w.qb64), + (('b',), 1, x.qb64)] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64), + (('ac',), 0, z.qb64), + (('b',), 0, w.qb64), + (('b',), 1, x.qb64), + (('bc',), 0, y.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter()] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64), + (('ac',), 0, z.qb64), + (('b',), 0, w.qb64), + (('b',), 1, x.qb64), + (('bc',), 0, y.qb64)] + + assert not os.path.exists(db.path) + assert not db.opened + + def test_cat_suber(): """ @@ -2280,10 +2387,11 @@ def test_crypt_signer_suber(): test_on_suber() test_dup_suber() test_iodup_suber() - test_oniodup_suber() + test_on_iodup_suber() test_ioset_suber() test_cat_suber() test_cesr_suber() + test_cesr_on_suber() test_cesr_ioset_suber() test_cat_cesr_ioset_suber() test_cesr_dup_suber() From bb4811ea2582d59e36e6aaf2d21a09925ccbb1c4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 17:48:05 -0600 Subject: [PATCH 37/78] more clean up refactoring. Created delOnVal method and corresponding method in OnSuberBase --- src/keri/db/dbing.py | 149 +++++++++++++++++++++++---------------- src/keri/db/escrowing.py | 6 +- src/keri/db/subing.py | 38 +++++++--- tests/db/test_dbing.py | 12 ++++ tests/db/test_subing.py | 25 +++++++ 5 files changed, 156 insertions(+), 74 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 354bcd99..d322eec4 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -693,14 +693,98 @@ def delTopVal(self, db, top=b''): # For subdbs the use keys with trailing part the is monotonically # ordinal number serialized as 32 hex bytes + + + def appendOnVal(self, db, key, val, *, sep=b'.'): + """ + Appends val in order after last previous onkey in db where + onkey has same given key prefix but with different serialized on suffix + attached with sep. + Returns ordinal number on, of appended entry. Appended on is 1 greater + than previous latest on at key. + Uses onKey(key, on) for entries. + + Works with either dupsort==True or False since always creates new full + key. + + Append val to end of db entries with same key but with on incremented by + 1 relative to last preexisting entry at key. + + Returns: + on (int): ordinal number of newly appended val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split + """ + # set key with fn at max and then walk backwards to find last entry at pre + # if any otherwise zeroth entry at pre + onkey = onKey(key, MaxON, sep=sep) + with self.env.begin(db=db, write=True, buffers=True) as txn: + on = 0 # unless other cases match then zeroth entry at pre + cursor = txn.cursor() + if not cursor.set_range(onkey): # max is past end of database + # so either empty database or last is earlier pre or + # last is last entry at same pre + if cursor.last(): # not empty db. last entry earlier than max + ckey = cursor.key() + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last is last entry for same pre + on = cn + 1 # increment + else: # not past end so not empty either later pre or max entry at pre + ckey = cursor.key() + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last entry for pre is already at max + raise ValueError(f"Number part {cn=} for key part {ckey=}" + f"exceeds maximum size.") + else: # later pre so backup one entry + # either no entry before last or earlier pre with entry + if cursor.prev(): # prev entry, maybe same or earlier pre + ckey = cursor.key() + ckey, cn = splitOnKey(ckey, sep=sep) + if ckey == key: # last entry at pre + on = cn + 1 # increment + + onkey = onKey(key, on, sep=sep) + + if not cursor.put(onkey, val, overwrite=False): + raise ValueError(f"Failed appending {val=} at {key=}.") + return on + + + def delOnVal(self, db, key, on=0, *, sep=b'.'): + """ + Deletes value at onkey consisting of key + sep + serialized on in db. + Returns True If key exists in database Else False + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which to delete + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=True, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return (txn.delete(onkey)) # when empty deletes whole db + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): """ - Returns (int): count of of all ordinal keyed vals with top key - but different on tail in db starting at ordinal number on of top. + Returns (int): count of of all ordinal keyed vals with key + but different on tail in db starting at ordinal number on of key. Full key is composed of top+sep+ When dupsort==true then duplicates are included in count since .iternext includes duplicates. - when key is empty then retrieves whole db + when key is empty then counts whole db Parameters: db (lmdbsubdb): named sub db of lmdb @@ -771,65 +855,6 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): - def appendOnVal(self, db, key, val, *, sep=b'.'): - """ - Appends val in order after last previous key with same pre in db where - full key has key prefix and serialized on suffix attached with sep. - Returns ordinal number in, on, of appended entry. Appended on is 1 greater - than previous latest on at pre. - Uses onKey(pre, on) for entries. - - Works with either dupsort==True or False since always creates new full - key. - - Append val to end of db entries with same pre but with on incremented by - 1 relative to last preexisting entry at pre. - - Returns: - on (int): ordinal number of newly appended val - - Parameters: - db (subdb): named sub db in lmdb - key (bytes): key within sub db's keyspace plus trailing part on - val (bytes): serialized value to append - sep (bytes): separator character for split - """ - # set key with fn at max and then walk backwards to find last entry at pre - # if any otherwise zeroth entry at pre - onkey = onKey(key, MaxON, sep=sep) - with self.env.begin(db=db, write=True, buffers=True) as txn: - on = 0 # unless other cases match then zeroth entry at pre - cursor = txn.cursor() - if not cursor.set_range(onkey): # max is past end of database - # so either empty database or last is earlier pre or - # last is last entry at same pre - if cursor.last(): # not empty db. last entry earlier than max - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) - if ckey == key: # last is last entry for same pre - on = cn + 1 # increment - else: # not past end so not empty either later pre or max entry at pre - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) - if ckey == key: # last entry for pre is already at max - raise ValueError(f"Number part {cn=} for key part {ckey=}" - f"exceeds maximum size.") - else: # later pre so backup one entry - # either no entry before last or earlier pre with entry - if cursor.prev(): # prev entry, maybe same or earlier pre - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) - if ckey == key: # last entry at pre - on = cn + 1 # increment - - onkey = onKey(key, on, sep=sep) - - if not cursor.put(onkey, val, overwrite=False): - raise ValueError(f"Failed appending {val=} at {key=}.") - return on - - - # IoSet insertion order in val so can have effective dups but with # dupsort = False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 082833cb..3a4267a9 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -117,20 +117,20 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) except Exception as ex: # other error so remove from reply escrow - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: logger.error("Kevery unescrowed due to error: %s", ex.args[0]) else: # unescrow succeded - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow only + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow only self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) logger.info("Kevery unescrow succeeded for txn state=%s", serder.said) logger.debug(f"event=\n{serder.pretty()}\n") except Exception as ex: # log diagnostics errors etc - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) self.removeState(saider) if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 5242bf95..0cefb1be 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -413,7 +413,7 @@ def appendOn(self, keys: str | bytes | memoryview, keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form key val (str | bytes | memoryview): serialization - on (int): ordinal number used with onKey(pre,on) to form key. + on (int): ordinal number used with onKey(key,on) to form key. """ return (self.db.appendOnVal(db=self.sdb, key=self._tokey(keys), @@ -421,21 +421,40 @@ def appendOn(self, keys: str | bytes | memoryview, sep=self.sep.encode())) + def remOn(self, keys: str | bytes | memoryview, on: int=0): + """ + Returns + result (bool): True if onkey made from key+sep+serialized on is + found in database so value is removed + Removes all duplicates if any at onkey. + False otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + """ + return (self.db.delOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode())) + def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): """ Returns - cnt (int): count of of all ordinal suffix keyed vals with same top - in key but different on in key in db starting at ordinal number - on of pre where key is formed with onKey(pre,on) - Does not count dups at same on for a given pre, only - unique on at a given pre. + cnt (int): count of of all ordinal suffix keyed vals with same + key prefix but different on in onkey in db starting at ordinal + number on where key is formed with onKey(key,on). Count at + each onkey includes duplicates if any. + Parameters: keys (str | bytes | memoryview | Iterable): top keys as prefix to be combined with serialized on suffix and sep to form top key - When keys is empty then counts whole database including duplicates - on (int): ordinal number used with onKey(pre,on) to form key. + When keys is empty then counts whole database including + duplicates if any. + on (int): ordinal number used with onKey(key,on) to form key. """ return (self.db.cntOnVals(db=self.sdb, key=self._tokey(keys), @@ -451,7 +470,8 @@ def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): Parameters: keys (str | bytes | memoryview | iterator): top keys as prefix to be combined with serialized on suffix and sep to form key - When keys is empty then retrieves whole database including duplicates + When keys is empty then retrieves whole database including + duplicates if any on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index caae1bb5..b75cf6b3 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -459,6 +459,18 @@ def test_lmdber(): items = [item for item in dber.getOnItemIter(db, key=preC, on=1)] assert items == [] + # test delOnVal + assert dber.delOnVal(db, key=preB) # default on=0 + assert not dber.delOnVal(db, key=preB, on=0) + assert dber.delOnVal(db, key=preB, on=1) + assert not dber.delOnVal(db, key=preB, on=1) + + items = [item for item in dber.getOnItemIter(db, key=preB)] + assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] + + with pytest.raises(KeyError): + assert dber.delOnVal(db, key=b'') # empty key + # test Vals dup methods. dup vals are lexocographic key = b'A' diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index a10a7a53..698bc7f2 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -317,6 +317,17 @@ def test_on_suber(): (('b',), 1, 'Green tree'), (('bc',), 0, 'Red apple')] + assert onsuber.remOn(keys='a', on=1) + assert not onsuber.remOn(keys='a', on=1) + assert onsuber.remOn(keys='a', on=3) + assert not onsuber.remOn(keys='a', on=3) + + assert onsuber.cntOn(keys=("a",)) == 2 + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 2, 'Red apple')] + assert not os.path.exists(db.path) assert not db.opened @@ -759,6 +770,20 @@ def test_on_iodup_suber(): assert 4 == onsuber.appendOn(keys=("a",), val=x) assert onsuber.cntOn(keys=("a",)) == 9 + assert onsuber.remOn(keys='a', on=1) + assert not onsuber.remOn(keys='a', on=1) + assert onsuber.remOn(keys='a', on=3) + assert not onsuber.remOn(keys='a', on=3) + + assert onsuber.cntOn(keys=("a",)) == 5 + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 0, 'White snow'), + (('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 4, 'Green tree')] + assert not os.path.exists(db.path) assert not db.opened From 6e67288dd0ff571e9fed0df8cd7595e88b2e2781 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 2 Sep 2024 17:56:39 -0600 Subject: [PATCH 38/78] final refactor of broker escrowing --- src/keri/db/escrowing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index 3a4267a9..36cdbc0e 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -79,7 +79,7 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): extype (Type[Exception]): the expected exception type if the message should remain in escrow """ - for (typ, pre, aid, ion), saider in self.escrowdb.getFullItemIter(keys=(typ, '')): + for (typ, pre, aid), saider in self.escrowdb.getItemIter(keys=(typ, '')): try: tsgs = eventing.fetchTsgs(db=self.tigerdb, saider=saider) @@ -117,20 +117,20 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): logger.exception("Kevery unescrow attempt failed: %s", ex.args[0]) except Exception as ex: # other error so remove from reply escrow - self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: logger.error("Kevery unescrowed due to error: %s", ex.args[0]) else: # unescrow succeded - self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow only self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow logger.info("Kevery unescrow succeeded for txn state=%s", serder.said) logger.debug(f"event=\n{serder.pretty()}\n") except Exception as ex: # log diagnostics errors etc - self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.removeState(saider) if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) From caa534f02a26c0714fc4f1571e721564c45f1316 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 14:20:19 -0600 Subject: [PATCH 39/78] refactored mailbox to use OnSuber --- src/keri/app/storing.py | 135 ++++++++++++++++++++++++++------------ src/keri/db/dbing.py | 9 ++- src/keri/db/subing.py | 13 ++-- tests/app/test_storing.py | 5 +- 4 files changed, 109 insertions(+), 53 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index c003f39e..073b365c 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -34,6 +34,17 @@ def __init__(self, name="mbx", headDirPath=None, reopen=True, **kwa): perm: reopen: kwa: + + Mailboxer uses two dbs for mailbox messages these are .tpcs and .msgs. + The message index is in .tpcs (topics). + Each .tpcs index key consists of topic.on where topic is bytes + identifier or prefix/topic for message and on is serialized + ordinal number to orders the appearance of a topic message. + Eash .tpcs val is the digest of the message. + The message itself is stored in .msgs where the key is the msg digest + and the value is the serialized messag itself. + Multiple messages can share the same topic but with a different ordinal. + """ self.tpcs = None self.msgs = None @@ -48,91 +59,131 @@ def reopen(self, **kwa): """ super(Mailboxer, self).reopen(**kwa) - self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True) + #self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True) + self.tpcs = subing.OnSuber(db=self, subkey='tpcs.') self.msgs = subing.Suber(db=self, subkey='msgs.') # key states return self.env - def delTopic(self, key): - """ - Use snKey() - Deletes value at key. - Returns True If key exists in database Else False + def delTopic(self, key, on=0): + """Removes topic index from .tpcs without deleting message from .msgs + + Returns: + result (boo): True if full key consisting of key and serialized on + exists in database so removed + False otherwise (not removed) """ - return self.delIoSetVals(self.tpcs, key) + #return self.delIoSetVals(self.tpcs, key) + return self.tpcs.remOn(keys=key, on=on) def appendToTopic(self, topic, val): - """ - Return first seen order number int, fn, of appended entry. - Computes fn as next fn after last entry. + """Appends val to end of db entries with same topic but with on + incremented by 1 relative to last preexisting entry at topic. - Append val to end of db entries with same topic but with fn incremented by - 1 relative to last preexisting entry at pre. + Returns: + on (int): order number int, on, of appended entry. + Computes on as next on after last entry. Parameters: - topic is bytes identifier prefix/topic for message - val is event digest + topic (bytes): topic identifier for message + val (bytes): msg digest """ - return self.appendIoSetVal(db=self.tpcs, key=topic, val=val) + #return self.appendIoSetVal(db=self.tpcs, key=topic, val=val) + return self.tpcs.appendOn(key=topic, val=val) + def getTopicMsgs(self, topic, fn=0): """ Returns: - ioset (oset): the insertion ordered set of values at same apparent - effective key. - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. + msgs (Iterable[bytes]): belonging to topic indices with same topic but all + on >= fn i.e. all topic.on beginning with fn Parameters: - topic (Option(bytes|str)): Apparent effective key - fn (int) starting index + topic (Option(bytes|str)): key prefix combined with serialized on + to form full actual key. When key is empty then retrieves + whole database. + fn (int): starting index ordinal number used with onKey(pre,on) + to form key at at which to initiate retrieval """ - if hasattr(topic, "encode"): - topic = topic.encode("utf-8") - - digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn) msgs = [] - for dig in digs: + for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn): if msg := self.msgs.get(keys=dig): - msgs.append(msg.encode("utf-8")) + msgs.append(msg.encode()) # want bytes not str return msgs + #if hasattr(topic, "encode"): + #topic = topic.encode("utf-8") + + #digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn) + #msgs = [] + #for dig in digs: + #if msg := self.msgs.get(keys=dig): + #msgs.append(msg.encode("utf-8")) + #return msgs + def storeMsg(self, topic, msg): """ - Add exn event to mailbox of dest identifier + Add exn event to mailbox topic and on that is 1 greater than last msg + at topic. + + Returns: + result (bool): True if msg successfully stored and indexed at topic + False otherwise Parameters: - msg (bytes): - topic (qb64b): + topic (str | bytes): topic (Option(bytes|str)): key prefix combined + with serialized on to form full actual key. + msg (bytes): serialized message """ - if hasattr(topic, "encode"): - topic = topic.encode("utf-8") - if hasattr(msg, "encode"): msg = msg.encode("utf-8") digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b - self.appendToTopic(topic=topic, val=digb) + on = self.tpcs.appendOn(keys=topic, val=digb) return self.msgs.pin(keys=digb, val=msg) + #if hasattr(topic, "encode"): + #topic = topic.encode("utf-8") + + #if hasattr(msg, "encode"): + #msg = msg.encode("utf-8") + + #digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b + #self.appendToTopic(topic=topic, val=digb) + #return self.msgs.pin(keys=digb, val=msg) + + def cloneTopicIter(self, topic, fn=0): """ - Returns iterator of first seen exn messages with attachments for the - identifier prefix pre starting at first seen order number, fn. + Returns: + triple (Iterator[(on, topic, msg): iterator of messages at topic + beginning with ordinal fn. + + topic (Option(bytes|str)): key prefix combined with serialized on + to form full actual key. When key is empty then retrieves + whole database. + fn (int): starting index ordinal number used with onKey(pre,on) + to form key at at which to initiate retrieval + + ToDo looks like misuse of IoSet this should not be IoSet but simply Ordinal Numbered db. since should not be using hidden ion has not hidden. """ - if hasattr(topic, 'encode'): - topic = topic.encode("utf-8") + for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn): + if msg := self.msgs.get(keys=dig): + yield (on, topic, msg.encode("utf-8")) + + #if hasattr(topic, 'encode'): + #topic = topic.encode("utf-8") - for ion, (topic, dig) in enumerate(self.getTopIoSetItemIter(self.tpcs, topic)): - if ion >= fn: - if msg := self.msgs.get(keys=dig): - yield ion, topic, msg.encode("utf-8") + #for ion, (topic, dig) in enumerate(self.getTopIoSetItemIter(self.tpcs, topic)): + #if ion >= fn: + #if msg := self.msgs.get(keys=dig): + #yield ion, topic, msg.encode("utf-8") #for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): #topic, ion = dbing.unsuffix(key) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index d322eec4..fece6423 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -522,7 +522,9 @@ def setVal(self, db, key, val): """ Write serialized bytes val to location key in db Overwrites existing val if any - Returns True If val successfully written Else False + Returns: + result (bool): True If val successfully written + False otherwise Parameters: db is opened named sub db with dupsort=False @@ -829,7 +831,8 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): Raises StopIteration Error when empty. Returns: - items (Iterator[(key, on, val)]): triples of key, on, val + items (Iterator[(key, on, val)]): triples of key, on, val with same + key but increments of on beginning with on Parameters: db (subdb): named sub db in lmdb @@ -856,7 +859,7 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): # IoSet insertion order in val so can have effective dups but with - # dupsort = False so val not limited to 511 bytes + # dupsort==False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent # effective duplicate key but with (dupsort==False). Actual key uses hidden # key suffix ordinal to provide insertion ordering of value members of set diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 0cefb1be..fa36ef8b 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -463,16 +463,17 @@ def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ - Returns - items (Iterator[(top keys, on, val)]): triples of (top keys, on int, - deserialized val) + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val with same + key but increments of on >= on i.e. all key.on beginning with on Parameters: - keys (str | bytes | memoryview | iterator): top keys as prefix to be - combined with serialized on suffix and sep to form key + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form actual key When keys is empty then retrieves whole database including duplicates if any - on (int): ordinal number used with onKey(pre,on) to form key. + on (int): ordinal number used with onKey(pre,on) to form key at at + which to initiate retrieval sep (bytes): separator character for split """ for keys, on, val in (self.db.getOnItemIter(db=self.sdb, diff --git a/tests/app/test_storing.py b/tests/app/test_storing.py index f0aebdd4..2abafbc7 100644 --- a/tests/app/test_storing.py +++ b/tests/app/test_storing.py @@ -9,7 +9,7 @@ from keri.app import keeping from keri.core import coring, serdering -from keri.db import dbing, basing +from keri.db import dbing, basing, subing from keri.peer import exchanging from keri.app.storing import Mailboxer @@ -28,7 +28,8 @@ def test_mailboxing(): assert mber.env.path() == mber.path assert os.path.exists(mber.path) - assert isinstance(mber.tpcs, lmdb._Database) + #assert isinstance(mber.tpcs, lmdb._Database) + assert isinstance(mber.tpcs, subing.OnSuber) mber.close(clear=True) assert not os.path.exists(mber.path) From 00ad0c26f290005abf4168c60d44f2dfb6f1ff4e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 14:53:51 -0600 Subject: [PATCH 40/78] Removed obsolete methods Comments on todo things --- src/keri/db/dbing.py | 109 +++++++--------------------------------- src/keri/db/subing.py | 45 ----------------- tests/db/test_dbing.py | 17 ------- tests/db/test_subing.py | 18 ------- 4 files changed, 17 insertions(+), 172 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index fece6423..8094c086 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -589,6 +589,7 @@ def cnt(self, db): return count # Suber subclasses don't need this since they always split keys int tuple to return + # replace this everywhere with getTopItemIter def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): """ Returns iterator of item duple (key, val), at each key over all @@ -947,65 +948,6 @@ def addIoSetVal(self, db, key, val, *, sep=b'.'): iokey = suffix(key, ion, sep=sep) # ion is at add on amount return cursor.put(iokey, val, dupdata=False, overwrite=False) - # deprecated since violates set property of each item in a set is unique - def appendIoSetVal(self, db, key, val, *, sep=b'.'): - """ - Append val non-idempotently to insertion ordered set of values all with - the same apparent effective key. If val already in set at key then - after append there will be multiple entries in database with val at key - each with different insertion order (iokey). - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. - - Works by walking backward to find last iokey for key instead of reading - all vals for ioky. - - Returns: - ion (int): hidden insertion ordering ordinal of appended val - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - val (bytes): value to append - """ - ion = 0 # default is zeroth insertion at key - iokey = suffix(key, ion=MaxSuffix, sep=sep) # make iokey at max and walk back - with self.env.begin(db=db, write=True, buffers=True) as txn: - cursor = txn.cursor() # create cursor to walk back - if not cursor.set_range(iokey): # max is past end of database - # Three possibilities for max past end of database - # 1. last entry in db is for same key - # 2. last entry in db is for other key before key - # 3. database is empty - if cursor.last(): # not 3. empty db, so either 1. or 2. - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # 1. last is last entry for same key - ion = cion + 1 # so set ion to the increment of cion - else: # max is not past end of database - # Two possibilities for max not past end of databseso - # 1. cursor at max entry at key - # 2. other key after key with entry in database - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # 1. last entry for key is already at max - raise ValueError("Number part of key {} at maximum" - " size.".format(ckey)) - else: # 2. other key after key so backup one entry - # Two possibilities: 1. no prior entry 2. prior entry - if cursor.prev(): # prev entry, maybe same or earlier pre - # 2. prior entry with two possiblities: - # 1. same key - # 2. other key before key - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # prior (last) entry at key - ion = cion + 1 # so set ion to the increment of cion - - iokey = suffix(key, ion=ion, sep=sep) - if not cursor.put(iokey, val, overwrite=False): - raise ValueError("Failed appending {} at {}.".format(val, key)) - - return ion - - def setIoSetVals(self, db, key, vals, *, sep=b'.'): """ @@ -1224,31 +1166,6 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return cursor.delete() # delete also moves to next so doubly moved return False - # deprecated since violates set property of each item in a set is unique - # this is needed to remove non-idempotently appended items which violate - # set property - def delIoSetIokey(self, db, iokey): # change this to key, ion - """ - Deletes val at at actual iokey that includes ordinal key suffix. - Need this to delete value that is non-idempotently added with - appendIoSetVal where multiple of same value exist at same key because - delIoSetVal will only delete the first one in list. - - Returns: - result (bool): True if val was deleted at iokey. False otherwise - if no val at iokey - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - iokey (bytes): actual key with ordinal key suffix - """ - with self.env.begin(db=db, write=True, buffers=True) as txn: - try: - return txn.delete(iokey) - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getTopIoSetItemIter(self, db, top=b'', *, sep=b'.'): """ @@ -1713,8 +1630,6 @@ def delIoDupVal(self, db, key, val): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return False - #def delIoDupIonVal(self, db, key, ion): # to match delIoSetIoKey - def cntIoDupVals(self, db, key): """ @@ -1960,7 +1875,7 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): # without gaps is used for replay with on to provide partial replay # Consider using ItemIter instead of just replaying vals since ItemIter already -# works with either dupsort==True or False +# works with either dupsort==True or False. also skips gaps. def getOnIoDupValsAllPreIter(self, db, pre, on=0): """ @@ -1998,15 +1913,23 @@ def getOnIoDupValsAllPreIter(self, db, pre, on=0): key = snKey(pre, cnt:=cnt+1) + # Create multiple methods + # getTopItemBackIter symmetric with getTopItemIter + # getTopIoSetItemBackIter symmetric getTopIoSetItemIter + # getTopIoDupItemBackIter symmetric with getTopIoDupItemIter + # getOnItemBackIter symmetric with getOnItemIter + + + # instead of just replaying vals replay items since + # getTopItemIter already works with either dupsort==True or False + # so if we create getTopBackItemIter then we can use that everywhere # ToDo need unit tests in dbing Used to replay backwards all duplicate # values starting at on # need to fix this so it is not stopped by gaps or if gap raises error as # gap is normally a problem for replay so maybe a parameter to raise error on gap - # Consider creating BackItemIter instead of just replaying vals since - # ItemIter already works with either dupsort==True or False - # so if we create TopBackItemIter then we can use that everywhere + def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries @@ -2044,6 +1967,7 @@ def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): yield val[33:] key = snKey(pre, cnt:=cnt-1) +# Last is special so need method. # Used to replay forward all last duplicate values starting at on # need to fix this so it is not stopped by gaps or if gap raises error as # gap is normally a problem for replay so maybe a parameter to raise error on gap @@ -2086,8 +2010,9 @@ def getIoDupValLastAllPreIter(self, db, pre, on=0): # ToDo do we need a replay last backwards? - - # use this method to inform how to cross gaps in other replays above with parameter + # not used anymore. + # before deleting this method use it inform how to cross gaps in other + # replays above with parameter such as replay last above. # to raise error if detect gap when should not be one for event logs vs escrows # then remove this method since not used otherwise diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index fa36ef8b..a136701b 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -813,35 +813,6 @@ def add(self, keys: str | bytes | memoryview | Iterable, val=self._ser(val), sep=self.sep)) - # deprecated since violates set property of each item in a set is unique - def append(self, keys: str | bytes | memoryview | Iterable, - val: str | bytes | memoryview): - """Append val non-idempotently to insertion ordered set of values all with - the same apparent effective key. If val already in set at key then - after append there will be multiple entries in database with val at key - each with different insertion order (iokey). - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. - - Works by walking backward to find last iokey for key instead of reading - all vals for iokey. - - Returns: - ion (int): hidden insertion ordering ordinal of appended val - - Parameters: - keys (str | bytes | memoryview | Iterable): of key parts to be - combined in order to form key - val (str | bytes | memoryview): serialization - - - """ - return (self.db.appendIoSetVal(db=self.sdb, - key=self._tokey(keys), - val=self._ser(val), - sep=self.sep)) - - def pin(self, keys: str | bytes | memoryview | Iterable, vals: str | bytes | memoryview | Iterable): @@ -947,22 +918,6 @@ def rem(self, keys: str | bytes | memoryview | Iterable, key=self._tokey(keys), sep=self.sep) - # deprecated since violates set property of each item in a set is unique - # this is to be able to remove non idempotent append - def remIokey(self, iokeys: str | bytes | memoryview | Iterable): - """ - Removes entries at iokeys - - Parameters: - iokeys (str | bytes | memoryview | Iterable): of key str or - tuple of key strs to be combined in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def cnt(self, keys: str | bytes | memoryview | Iterable): """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index b75cf6b3..b52e980e 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -934,19 +934,6 @@ def test_lmdber(): assert dber.getIoSetVals(db, key1) == vals1 assert dber.getIoSetVals(db, key2) == vals2 - assert dber.appendIoSetVal(db, key1, val=b"k") == 4 - assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] - - - #assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemIter(db, key0)] == - #[(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - #(b'ABC.ZYX.00000000000000000000000000000001', b'm'), - #(b'ABC.ZYX.00000000000000000000000000000002', b'x'), - #(b'ABC.ZYX.00000000000000000000000000000003', b'a')]) - - #for iokey, val in dber.getIoSetItemIter(db, key0): - #assert dber.delIoSetIokey(db, iokey) - #assert dber.getIoSetVals(db, key0) == [] vals3 = [b"q", b"e"] assert dber.setIoSetVals(db, key2, vals3) @@ -968,16 +955,12 @@ def test_lmdber(): dber.putIoSetVals(db, empty_key, [some_value]) dber.addIoSetVal(db, empty_key, some_value) dber.setIoSetVals(db, empty_key, [some_value]) - dber.appendIoSetVal(db, empty_key, some_value) dber.getIoSetVals(db, empty_key) [_ for _ in dber.getIoSetValsIter(db, empty_key)] dber.getIoSetValLast(db, empty_key) dber.cntIoSetVals(db, empty_key) dber.delIoSetVals(db, empty_key) dber.delIoSetVal(db, empty_key, some_value) - #dber.getIoSetItemIter(db, empty_key) - with pytest.raises(KeyError): - dber.delIoSetIokey(db, empty_key) with pytest.raises(KeyError): dber.putVals(db, empty_key, [some_value]) with pytest.raises(KeyError): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 698bc7f2..1a042462 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -913,11 +913,6 @@ def test_ioset_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in iosuber.getFullItemIter(): - assert iosuber.remIokey(iokeys=iokeys) - - assert iosuber.cnt(keys=keys0) == 0 - assert iosuber.cnt(keys=keys1) == 0 # test with keys as string not tuple @@ -949,8 +944,6 @@ def test_ioset_suber(): assert iosuber.trim() # default trims whole database assert iosuber.put(keys=keys1, vals=[bob, bil]) assert iosuber.get(keys=keys1) == [bob, bil] - assert iosuber.append(keys=keys1, val=bob) == 2 - assert iosuber.get(keys=keys1) == [bob, bil, bob] assert not os.path.exists(db.path) assert not db.opened @@ -1141,11 +1134,6 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - for iokeys, val in cisuber.getFullItemIter(): - assert cisuber.remIokey(iokeys=iokeys) - - assert cisuber.cnt(keys=keys0) == 0 - assert cisuber.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) @@ -1921,12 +1909,6 @@ def test_cat_cesr_ioset_suber(): (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), ] - for iokeys, val in sdb.getFullItemIter(): - assert sdb.remIokey(iokeys=iokeys) - - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 - assert sdb.cnt(keys=keys2) == 0 # Try Siger Indexer Subclass sdb = subing.CatCesrIoSetSuber(db=db, subkey='pigs.', klas=(indexing.Siger, )) From cf0407ed0b7c75845d8eeedc6df03f05913551a0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 16:10:53 -0600 Subject: [PATCH 41/78] more refactoring removing unused methods dbing basing --- src/keri/app/notifying.py | 3 +- src/keri/db/basing.py | 103 +-------- src/keri/db/dbing.py | 123 ++++------- src/keri/db/subing.py | 31 +++ src/keri/vdr/eventing.py | 12 +- src/keri/vdr/viring.py | 6 +- tests/db/test_basing.py | 441 +------------------------------------- tests/db/test_dbing.py | 83 +------ 8 files changed, 96 insertions(+), 706 deletions(-) diff --git a/src/keri/app/notifying.py b/src/keri/app/notifying.py index 4ad03d22..09ac24db 100644 --- a/src/keri/app/notifying.py +++ b/src/keri/app/notifying.py @@ -184,7 +184,8 @@ def getItemIter(self, keys: Union[str, Iterable] = b""): each entry in db """ - for key, val in self.db.getAllItemIter(db=self.sdb, key=self._tokey(keys), split=False): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys)): yield self._tokeys(key), self.klas(raw=bytes(val)) def cntAll(self): diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index fc132f52..750f6211 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1467,7 +1467,7 @@ def clean(self): cpydb.add(keys=keys, val=val) # Insecure raw imgs database copy. - for (key, val) in self.getAllItemIter(self.imgs): + for (key, val) in self.getTopItemIter(self.imgs): copy.imgs.setVal(key=key, val=val) # clone .habs habitat name prefix Komer subdb @@ -2286,19 +2286,6 @@ def getUreLast(self, key): """ return self.getIoDupValLast(self.ures, key) - def getUreItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event triple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is triple dig+pre+cig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.ures, key, skip) def getUreItemsNextIter(self, key=b'', skip=True): """ @@ -2453,20 +2440,6 @@ def getVreLast(self, key): """ return self.getIoDupValLast(self.vres, key) - def getVreItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event quintuple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is Quinlet is edig + spre + ssnu + sdig +sig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.vres, key, skip) - def getVreItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -2683,18 +2656,6 @@ def getPseLast(self, key): """ return self.getIoDupValLast(self.pses, key) - def getPseItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.pses, key, skip) - def getPseItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -2783,18 +2744,6 @@ def getPweLast(self, key): """ return self.getIoDupValLast(self.pwes, key) - def getPweItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial witnessed escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.pwes, key, skip) - def getPweItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -2899,20 +2848,6 @@ def getUweLast(self, key): """ return self.getIoDupValLast(self.uwes, key) - def getUweItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed receipt couple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is couple edig+wig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.uwes, key, skip) - def getUweItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -2993,18 +2928,6 @@ def getOoeLast(self, key): """ return self.getIoDupValLast(self.ooes, key) - def getOoeItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of out of order escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.ooes, key, skip) - def getOoeItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -3084,18 +3007,6 @@ def getQnfLast(self, key): """ return self.getIoDupValLast(self.qnfs, key) - def getQnfItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of out of order escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.qnfs, key, skip) - def getQnfItemsNextIter(self, key=b'', skip=True): """ Use sgKey() @@ -3250,18 +3161,6 @@ def getLdeLast(self, key): """ return self.getIoDupValLast(self.ldes, key) - def getLdeItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of likely duplicitous escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNext(self.ldes, key, skip) - def getLdeItemsNextIter(self, key=b'', skip=True): """ Use sgKey() diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 8094c086..5f3a1704 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -588,39 +588,6 @@ def cnt(self, db): count += 1 return count - # Suber subclasses don't need this since they always split keys int tuple to return - # replace this everywhere with getTopItemIter - def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): - """ - Returns iterator of item duple (key, val), at each key over all - keys in db. If split is true then the key is split at sep and instead - of returing duple it results tuple with one entry for each key split - as well as the value. - - Works for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database - split (bool): True means split key at sep before returning - sep (bytes): separator char for key - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db - - for key, val in cursor.iternext(): # return key, val at cursor - if split: - splits = bytes(key).split(sep) - splits.append(val) - else: - splits = (bytes(key), val) - yield tuple(splits) - def getTopItemIter(self, db, top=b''): """ @@ -1714,51 +1681,51 @@ def getTopIoDupItemIter(self, db, top=b''): # not needed in IoDupSuber since not used anywhere always uses the Iter version # below. confirm this in general and refactor any that do use to use the iter verio - def getIoDupItemsNext(self, db, key=b"", skip=True): - """ - Return list of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key == b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Returns empty list if no entries at next key after key - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - - Assumes DB opened with dupsort=True - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty string - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass - """ - - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - items = [] - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - # slice off prepended ordering prefix on value in item - items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] - return items + #def getIoDupItemsNext(self, db, key=b"", skip=True): + #""" + #Return list of all dup items at next key after key in db in insertion order. + #Item is (key, val) with proem stripped from val stored in db. + #If key == b'' then returns list of dup items at first key in db. + #If skip is False and key is not empty then returns dup items at key + #Returns empty list if no entries at next key after key + + #If key is empty then gets io items (key, io value) at first key in db + #Use the return key from items as next key for next call to function in + #order to iterate through the database + + #Assumes DB opened with dupsort=True + + #Duplicates at a given key preserve insertion order of duplicate. + #Because lmdb is lexocographic an insertion ordering proem is prepended to + #all values that makes lexocographic order that same as insertion order. + + #Duplicates are ordered as a pair of key plus value so prepending proem + #to each value changes duplicate ordering. Proem is 33 characters long. + #With 32 character hex string followed by '.' for essentiall unlimited + #number of values which will be limited by memory. + + #With prepended proem ordinal must explicity check for duplicate values + #before insertion. Uses a python set for the duplicate inclusion test. + #Set inclusion scales with O(1) whereas list inclusion scales with O(n). + + #Parameters: + #db is opened named sub db with dupsort=True + #key is bytes of key within sub db's keyspace or empty string + #skip is Boolean If True skips to next key if key is not empty string + #Othewise don't skip for first pass + #""" + + #with self.env.begin(db=db, write=False, buffers=True) as txn: + #cursor = txn.cursor() + #items = [] + #if cursor.set_range(key): # moves to first_dup at key + #found = True + #if skip and key and cursor.key() == key: # skip to next key + #found = cursor.next_nodup() # skip to next key not dup if any + #if found: + ## slice off prepended ordering prefix on value in item + #items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] + #return items # don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter # which in turen can be replaced with IoDupSuber.getItemIter diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index a136701b..09c1bb8e 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -36,6 +36,37 @@ DupSuber CesrDupSuber +Class Architecture + +Suber is simple lexographic database with only one value per key +OnSuber is simple lexographic database where trailing part of key is serialized + ordinal number so that the ordering within each key prefix is monotonically + increasing numeric + +The term 'set' of values means that no value may appear more than once in the set. + +DupSuber provides set of lexicographic ordered values at each key. Each value has + a limited size (key + value <= 511 byes). The set is performant. Good for indices. + +IoDupSuber provides set of insertion ordered values at each key. Each value has + a limited size (key + value <= 511 byes). The set is less perfromant than DupSuber + but more performant than IoSetSuber. Good for insertion ordered indices + +IoSetSuber proves set of insertion ordered values at each key. Value size is not limited + Good for any insertion ordered set where size may be too large for IoDupSuber + +OnIoDupSuber provides set of insertion ordered values where the where trailing + part of key is serialized ordinal number so that the ordering within each + key prefix is monotonically increasing numeric. + +Each of these base types for managing the key space may be mixed with other +Classes that provide different types of values these include. + +Cesr +CatCesr +Serder +etc. + """ from typing import Type, Union diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 2cc89a23..e696baba 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -23,7 +23,7 @@ from ..core.signing import (Salter,) from ..core.eventing import SealEvent, ample, TraitDex, verifySigs from ..db import basing, dbing -from ..db.dbing import dgKey, snKey +from ..db.dbing import dgKey, snKey, splitSnKey from ..help import helping from ..kering import (MissingWitnessSignatureError, Version, MissingAnchorError, ValidationError, OutOfOrderError, LikelyDuplicitousError) @@ -2031,9 +2031,10 @@ def processEscrowOutOfOrders(self): 5. Remove event digest from oots if processed successfully or a non-out-of-order event occurs. """ - for (pre, snb, digb) in self.reger.getOotItemIter(): + for key, digb in self.reger.getOotItemIter(): # (pre, snb, digb) in self.reger.getOotItemIter() try: - sn = int(snb, 16) + #sn = int(snb, 16) + pre, sn = splitSnKey(key) dgkey = dgKey(pre, digb) traw = self.reger.getTvt(dgkey) if traw is None: @@ -2099,8 +2100,9 @@ def processEscrowAnchorless(self): 6. Remove event digest from oots if processed successfully or a non-anchorless event occurs. """ - for (pre, snb, digb) in self.reger.getTaeItemIter(): - sn = int(snb, 16) + for key, digb in self.reger.getTaeItemIter(): #(pre, snb, digb) in self.reger.getTaeItemIter() + pre, sn = splitSnKey(key) + #sn = int(snb, 16) try: dgkey = dgKey(pre, digb) traw = self.reger.getTvt(dgkey) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 11123c57..c2fccfda 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -809,7 +809,8 @@ def getTaeItemIter(self): Return iterator of all items in .taes """ - return self.getAllItemIter(self.taes, split=True) + return self.getTopItemIter(self.taes) + def delTae(self, key): """ @@ -852,7 +853,8 @@ def getOotItemIter(self): Return iterator of all items in .taes """ - return self.getAllItemIter(self.oots, split=True) + return self.getTopItemIter(self.oots) + def delOot(self, key): """ diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index fc02d3ec..29a498a4 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -491,64 +491,6 @@ def test_baser(): assert db.putUres(key=cKey, vals=cVals) assert db.putUres(key=dKey, vals=dVals) - # Test getUreItemsNext( key=b"") - # aVals - items = db.getUreItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUreItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUreItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getUreItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getUreItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getUreItemsNextIter(key=b"") # get dups at first key in database @@ -613,10 +555,6 @@ def test_baser(): for key, val in items: assert db.delUre(ikey, val) == True - # none - items = [item for item in db.getUreItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Validator (transferable) Receipts # test .vrcs sub db methods dgkey @@ -698,64 +636,6 @@ def test_baser(): assert db.putVres(key=cKey, vals=cVals) assert db.putVres(key=dKey, vals=dVals) - # Test getVreItemsNext( key=b"") - # aVals - items = db.getVreItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getVreItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getVreItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getVreItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getVreItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getVreItemsNextIter(key=b"") # get dups at first key in database @@ -820,10 +700,6 @@ def test_baser(): for key, val in items: assert db.delVre(ikey, val) == True - # none - items = [item for item in db.getVreItemsNext(key=ikey)] - assert items == [] # empty - assert not items # test .kels insertion order dup methods. dup vals are insertion order @@ -883,64 +759,6 @@ def test_baser(): assert db.putPses(key=cKey, vals=cVals) assert db.putPses(key=dKey, vals=dVals) - # Test getPseItemsNext( key=b"") - # aVals - items = db.getPseItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPseItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPseItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getPseItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getPseItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getPseItemsNextIter(key=b"") # get dups at first key in database @@ -1005,10 +823,6 @@ def test_baser(): for key, val in items: assert db.delPse(ikey, val) == True - # none - items = [item for item in db.getPseItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Test .udes partial delegated escrow seal source couples key = dgKey(preb, digb) @@ -1085,64 +899,6 @@ def test_baser(): assert db.putPwes(key=cKey, vals=cVals) assert db.putPwes(key=dKey, vals=dVals) - # Test getPweItemsNext( key=b"") - # aVals - items = db.getPweItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPweItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPweItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getPweItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getPweItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getPweItemsNextIter(key=b"") # get dups at first key in database @@ -1207,10 +963,6 @@ def test_baser(): for key, val in items: assert db.delPwe(ikey, val) == True - # none - items = [item for item in db.getPweItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Unverified Witness Receipt Escrows # test .uwes insertion order dup methods. dup vals are insertion order @@ -1249,64 +1001,6 @@ def test_baser(): assert db.putUwes(key=cKey, vals=cVals) assert db.putUwes(key=dKey, vals=dVals) - # Test getUweItemsNext( key=b"") - # aVals - items = db.getUweItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUweItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUweItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getUweItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getUweItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getUweItemsNextIter(key=b"") # get dups at first key in database @@ -1371,10 +1065,7 @@ def test_baser(): for key, val in items: assert db.delUwe(ikey, val) == True - # none - items = [item for item in db.getUweItemsNext(key=ikey)] - assert items == [] # empty - assert not items + # test .ooes insertion order dup methods. dup vals are insertion order key = b'A' @@ -1411,65 +1102,6 @@ def test_baser(): assert db.putOoes(key=cKey, vals=cVals) assert db.putOoes(key=dKey, vals=dVals) - # Test getOoeItemsNext( key=b"") - # aVals - items = db.getOoeItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getOoeItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getOoeItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getOoeItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getOoeItemsNext(key=ikey) - assert items == [] # empty - assert not items - # Test getOoeItemsNextIter(key=b"") # get dups at first key in database # aVals @@ -1533,12 +1165,6 @@ def test_baser(): for key, val in items: assert db.delOoe(ikey, val) == True - # none - items = [item for item in db.getOoeItemsNext(key=ikey)] - assert items == [] # empty - assert not items - - # test .dels insertion order dup methods. dup vals are insertion order key = b'A' vals = [b"z", b"m", b"x", b"a"] @@ -1591,64 +1217,6 @@ def test_baser(): assert db.putLdes(key=cKey, vals=cVals) assert db.putLdes(key=dKey, vals=dVals) - # Test getOoeItemsNext( key=b"") - # aVals - items = db.getLdeItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getLdeItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getLdeItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getLdeItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getLdeItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getLdeItemsNextIter(key=b"") # get dups at first key in database @@ -1713,13 +1281,6 @@ def test_baser(): for key, val in items: assert db.delLde(ikey, val) == True - # none - items = [item for item in db.getLdeItemsNext(key=ikey)] - assert items == [] # empty - assert not items - - - assert not os.path.exists(db.path) """ End Test """ diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index b52e980e..e7f14a88 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -292,7 +292,7 @@ def test_lmdber(): assert dber.delVal(db, key) == True assert dber.getVal(db, key) == None - # Test getAllItemIter(self, db, key=b'', split=True, sep=b'.') + # Test getTopItemIter key = b"a.1" val = b"wow" assert dber.putVal(db, key, val) == True @@ -302,10 +302,10 @@ def test_lmdber(): key = b"b.1" val = b"woo" assert dber.putVal(db, key, val) == True - assert [(bytes(pre), bytes(num), bytes(val)) for pre, num, val - in dber.getAllItemIter(db=db)] == [(b'a', b'1', b'wow'), - (b'a', b'2', b'wee'), - (b'b', b'1', b'woo')] + assert [(bytes(key), bytes(val)) for key, val + in dber.getTopItemIter(db=db)] == [(b'a.1', b'wow'), + (b'a.2', b'wee'), + (b'b.1', b'woo')] assert dber.delTopVal(db, top=b"a.") items = [ (key, bytes(val)) for key, val in dber.getTopItemIter(db=db )] @@ -704,65 +704,6 @@ def test_lmdber(): assert not items - # Test getIoItemsNext(self, db, key=b"") - # aVals - items = dber.getIoDupItemsNext(edb) # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = dber.getIoDupItemsNext(edb, key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = dber.getIoDupItemsNext(edb, key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = dber.getIoDupItemsNext(edb, key=b'', skip=False) # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = dber.getIoDupItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = dber.getIoDupItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = dber.getIoDupItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = dber.getIoDupItemsNext(edb, key=ikey) - assert items == [] # empty - assert not items - # Test getIoItemsNextIter(self, db, key=b"") # get dups at first key in database # aVals @@ -816,20 +757,6 @@ def test_lmdber(): for key, val in items: assert dber.delIoDupVal(edb, ikey, val) == True - # dVals - items = [item for item in dber.getIoDupItemsNext(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - for key, val in items: - assert dber.delIoDupVal(edb, ikey, val) == True - - # none - items = [item for item in dber.getIoDupItemsNext(edb, key=ikey)] - assert items == [] # empty - assert not items # test OnIoDup methods key = b'Z' From a460caa87f4cbed669816ac79c368364af8d2b79 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 17:01:10 -0600 Subject: [PATCH 42/78] more refactoring --- src/keri/core/eventing.py | 2 +- src/keri/db/basing.py | 5 +++-- src/keri/db/dbing.py | 47 --------------------------------------- tests/db/test_basing.py | 45 ++++++------------------------------- 4 files changed, 11 insertions(+), 88 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index e7c59768..1200b7e3 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5873,7 +5873,7 @@ def processEscrowUnverNonTrans(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, etriplet in self.db.getUreItemsNextIter(key=key): + for ekey, etriplet in self.db.getUreItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item rsaider, sprefixer, cigar = deReceiptTriple(etriplet) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 750f6211..27f36377 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2287,7 +2287,7 @@ def getUreLast(self, key): return self.getIoDupValLast(self.ures, key) - def getUreItemsNextIter(self, key=b'', skip=True): + def getUreItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event triple items at next @@ -2299,7 +2299,8 @@ def getUreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.ures, key, skip) + return self.getTopIoDupItemIter(self.ures, key) + #return self.getIoDupItemsNextIter(self.ures, key, skip) def cntUres(self, key): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 5f3a1704..12477384 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1679,53 +1679,6 @@ def getTopIoDupItemIter(self, db, top=b''): val = val[33:] # strip proem yield (top, val) -# not needed in IoDupSuber since not used anywhere always uses the Iter version -# below. confirm this in general and refactor any that do use to use the iter verio - #def getIoDupItemsNext(self, db, key=b"", skip=True): - #""" - #Return list of all dup items at next key after key in db in insertion order. - #Item is (key, val) with proem stripped from val stored in db. - #If key == b'' then returns list of dup items at first key in db. - #If skip is False and key is not empty then returns dup items at key - #Returns empty list if no entries at next key after key - - #If key is empty then gets io items (key, io value) at first key in db - #Use the return key from items as next key for next call to function in - #order to iterate through the database - - #Assumes DB opened with dupsort=True - - #Duplicates at a given key preserve insertion order of duplicate. - #Because lmdb is lexocographic an insertion ordering proem is prepended to - #all values that makes lexocographic order that same as insertion order. - - #Duplicates are ordered as a pair of key plus value so prepending proem - #to each value changes duplicate ordering. Proem is 33 characters long. - #With 32 character hex string followed by '.' for essentiall unlimited - #number of values which will be limited by memory. - - #With prepended proem ordinal must explicity check for duplicate values - #before insertion. Uses a python set for the duplicate inclusion test. - #Set inclusion scales with O(1) whereas list inclusion scales with O(n). - - #Parameters: - #db is opened named sub db with dupsort=True - #key is bytes of key within sub db's keyspace or empty string - #skip is Boolean If True skips to next key if key is not empty string - #Othewise don't skip for first pass - #""" - - #with self.env.begin(db=db, write=False, buffers=True) as txn: - #cursor = txn.cursor() - #items = [] - #if cursor.set_range(key): # moves to first_dup at key - #found = True - #if skip and key and cursor.key() == key: # skip to next key - #found = cursor.next_nodup() # skip to next key not dup if any - #if found: - ## slice off prepended ordering prefix on value in item - #items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] - #return items # don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter # which in turen can be replaced with IoDupSuber.getItemIter diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 29a498a4..5655838e 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -495,38 +495,23 @@ def test_baser(): # Test getUreItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUreItemsNextIter()] + items = [item for item in db.getUreItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals + vals = [bytes(val) for key, val in items] + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUreItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - items = [item for item in db.getUreItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getUreItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getUreItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delUre(ikey, val) == True # bVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -536,7 +521,7 @@ def test_baser(): assert db.delUre(ikey, val) == True # cVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -546,7 +531,7 @@ def test_baser(): assert db.delUre(ikey, val) == True # dVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -647,13 +632,6 @@ def test_baser(): vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getVreItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - items = [item for item in db.getVreItemsNextIter(key=aKey)] assert items # not empty ikey = items[0][0] @@ -661,15 +639,6 @@ def test_baser(): vals = [val for key, val in items] assert vals == bVals - items = [item for item in db.getVreItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delVre(ikey, val) == True - # bVals items = [item for item in db.getVreItemsNextIter(key=ikey)] assert items # not empty From 752614a4c6edfc113f1e7141669ebf401653726c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 18:57:13 -0600 Subject: [PATCH 43/78] refactored out getIoDupItemsNextIter --- src/keri/app/cli/commands/escrow.py | 8 +- src/keri/core/eventing.py | 14 +-- src/keri/db/basing.py | 57 +++++----- src/keri/db/dbing.py | 86 +++++++-------- tests/db/test_basing.py | 156 +++++++--------------------- tests/db/test_dbing.py | 104 +++++++++---------- 6 files changed, 177 insertions(+), 248 deletions(-) diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow.py index 014daf54..277ad4b7 100644 --- a/src/keri/app/cli/commands/escrow.py +++ b/src/keri/app/cli/commands/escrow.py @@ -56,7 +56,7 @@ def escrows(tymth, tock=0.0, **opts): oots = list() key = ekey = b'' # both start same. when not same means escrows found while True: - for ekey, edig in hby.db.getOoeItemsNextIter(key=key): + for ekey, edig in hby.db.getOoeItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: @@ -74,7 +74,7 @@ def escrows(tymth, tock=0.0, **opts): pwes = list() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in hby.db.getPweItemsNextIter(key=key): + for ekey, edig in hby.db.getPweItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: @@ -92,7 +92,7 @@ def escrows(tymth, tock=0.0, **opts): pses = list() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in hby.db.getPseItemsNextIter(key=key): + for ekey, edig in hby.db.getPseItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: @@ -110,7 +110,7 @@ def escrows(tymth, tock=0.0, **opts): ldes = list() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in hby.db.getLdeItemsNextIter(key=key): + for ekey, edig in hby.db.getLdeItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 1200b7e3..580bc1f9 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5209,7 +5209,7 @@ def processEscrowOutOfOrders(self): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getOoeItemsNextIter(key=key): + for ekey, edig in self.db.getOoeItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) @@ -5347,7 +5347,7 @@ def processEscrowPartialSigs(self): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getPseItemsNextIter(key=key): + for ekey, edig in self.db.getPseItemIter(key=key): eserder = None try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item @@ -5528,7 +5528,7 @@ def processEscrowPartialWigs(self): #key = ekey = b'' # both start same. when not same means escrows found #while True: # break when done - for ekey, edig in self.db.getPweIoDupItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) + for ekey, edig in self.db.getPweItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) @@ -5753,7 +5753,7 @@ def processEscrowUnverWitness(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, ecouple in self.db.getUweItemsNextIter(key=key): + for ekey, ecouple in self.db.getUweItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key @@ -6139,7 +6139,7 @@ def processQueryNotFound(self): pre = b'' sn = 0 while True: # break when done - for ekey, edig in self.db.getQnfItemsNextIter(key=key): + for ekey, edig in self.db.getQnfItemIter(key=key): try: pre, _ = splitKey(ekey) # get pre and sn from escrow item # check date if expired then remove escrow. @@ -6386,7 +6386,7 @@ def processEscrowUnverTrans(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, equinlet in self.db.getVreItemsNextIter(key=key): + for ekey, equinlet in self.db.getVreItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) @@ -6556,7 +6556,7 @@ def processEscrowDuplicitous(self): """ key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getLdeItemsNextIter(key=key): + for ekey, edig in self.db.getLdeItemIter(key=key): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 27f36377..0aff8f1a 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2441,7 +2441,7 @@ def getVreLast(self, key): """ return self.getIoDupValLast(self.vres, key) - def getVreItemsNextIter(self, key=b'', skip=True): + def getVreItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event quintuple items at next @@ -2453,7 +2453,8 @@ def getVreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.vres, key, skip) + return self.getTopIoDupItemIter(self.vres, key) + #return self.getIoDupItemsNextIter(self.vres, key, skip) def cntVres(self, key): """ @@ -2657,7 +2658,7 @@ def getPseLast(self, key): """ return self.getIoDupValLast(self.pses, key) - def getPseItemsNextIter(self, key=b'', skip=True): + def getPseItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event dig items at next key after key. @@ -2667,7 +2668,8 @@ def getPseItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.pses, key, skip) + return self.getTopIoDupItemIter(self.pses, key) + #return self.getIoDupItemsNextIter(self.pses, key, skip) def cntPses(self, key): """ @@ -2745,19 +2747,7 @@ def getPweLast(self, key): """ return self.getIoDupValLast(self.pwes, key) - def getPweItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() - Return iterator of partial witnessed escrowed event dig items at next key after key. - Items is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoDupItemsNextIter(self.pwes, key, skip) - - def getPweIoDupItemIter(self, key=b''): + def getPweItemIter(self, key=b''): """ Use sgKey() Return iterator of partial witnessed escrowed event dig items at next key after key. @@ -2768,6 +2758,19 @@ def getPweIoDupItemIter(self, key=b''): Duplicates are retrieved in insertion order. """ return self.getTopIoDupItemIter(self.pwes, key) + #return self.getIoDupItemsNextIter(self.pwes, key, skip) + + #def getPweIoDupItemIter(self, key=b''): + #""" + #Use sgKey() + #Return iterator of partial witnessed escrowed event dig items at next key after key. + #Items is (key, val) where proem has already been stripped from val + #If key is b'' empty then returns dup items at first key. + #If skip is False and key is not b'' empty then returns dup items at key + #Raises StopIteration Error when empty + #Duplicates are retrieved in insertion order. + #""" + #return self.getTopIoDupItemIter(self.pwes, key) def cntPwes(self, key): """ @@ -2849,7 +2852,7 @@ def getUweLast(self, key): """ return self.getIoDupValLast(self.uwes, key) - def getUweItemsNextIter(self, key=b'', skip=True): + def getUweItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed receipt couple items at next @@ -2861,7 +2864,8 @@ def getUweItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.uwes, key, skip) + return self.getTopIoDupItemIter(self.uwes, key) + #return self.getIoDupItemsNextIter(self.uwes, key, skip) def cntUwes(self, key): """ @@ -2929,7 +2933,7 @@ def getOoeLast(self, key): """ return self.getIoDupValLast(self.ooes, key) - def getOoeItemsNextIter(self, key=b'', skip=True): + def getOoeItemIter(self, key=b''): """ Use sgKey() Return iterator of out of order escrowed event dig items at next key after key. @@ -2939,7 +2943,8 @@ def getOoeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.ooes, key, skip) + return self.getTopIoDupItemIter(self.ooes, key) + #return self.getIoDupItemsNextIter(self.ooes, key, skip) def cntOoes(self, key): """ @@ -3008,7 +3013,7 @@ def getQnfLast(self, key): """ return self.getIoDupValLast(self.qnfs, key) - def getQnfItemsNextIter(self, key=b'', skip=True): + def getQnfItemIter(self, key=b''): """ Use sgKey() Return iterator of out of order escrowed event dig items at next key after key. @@ -3018,7 +3023,8 @@ def getQnfItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.qnfs, key, skip) + return self.getTopIoDupItemIter(self.qnfs, key) + #return self.getIoDupItemsNextIter(self.qnfs, key, skip) def cntQnfs(self, key): """ @@ -3162,7 +3168,7 @@ def getLdeLast(self, key): """ return self.getIoDupValLast(self.ldes, key) - def getLdeItemsNextIter(self, key=b'', skip=True): + def getLdeItemIter(self, key=b''): """ Use sgKey() Return iterator of likely duplicitous escrowed event dig items at next key after key. @@ -3172,7 +3178,8 @@ def getLdeItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoDupItemsNextIter(self.ldes, key, skip) + return self.getTopIoDupItemIter(self.ldes, key) + #return self.getIoDupItemsNextIter(self.ldes, key, skip) def cntLdes(self, key): """ diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 12477384..7786285a 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1683,49 +1683,49 @@ def getTopIoDupItemIter(self, db, top=b''): # don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter # which in turen can be replaced with IoDupSuber.getItemIter - def getIoDupItemsNextIter(self, db, key=b"", skip=True): - """ - Return iterator of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Raises StopIteration Error when no remaining dup items = empty. - - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database - - Assumes DB opened with dupsort=True - - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order. - - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentiall unlimited - number of values which will be limited by memory. - - With prepended proem ordinal must explicity check for duplicate values - before insertion. Uses a python set for the duplicate inclusion test. - Set inclusion scales with O(1) whereas list inclusion scales with O(n). - - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass - """ - - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - for key, val in cursor.iternext_dup(keys=True): - yield (key, val[33:]) # slice off prepended ordering prefix + #def getIoDupItemsNextIter(self, db, key=b"", skip=True): + #""" + #Return iterator of all dup items at next key after key in db in insertion order. + #Item is (key, val) with proem stripped from val stored in db. + #If key = b'' then returns list of dup items at first key in db. + #If skip is False and key is not empty then returns dup items at key + #Raises StopIteration Error when no remaining dup items = empty. + + #If key is empty then gets io items (key, io value) at first key in db + #Use the return key from items as next key for next call to function in + #order to iterate through the database + + #Assumes DB opened with dupsort=True + + #Duplicates at a given key preserve insertion order of duplicate. + #Because lmdb is lexocographic an insertion ordering proem is prepended to + #all values that makes lexocographic order that same as insertion order. + + #Duplicates are ordered as a pair of key plus value so prepending proem + #to each value changes duplicate ordering. Proem is 33 characters long. + #With 32 character hex string followed by '.' for essentiall unlimited + #number of values which will be limited by memory. + + #With prepended proem ordinal must explicity check for duplicate values + #before insertion. Uses a python set for the duplicate inclusion test. + #Set inclusion scales with O(1) whereas list inclusion scales with O(n). + + #Parameters: + #db is opened named sub db with dupsort=True + #key is bytes of key within sub db's keyspace or empty + #skip is Boolean If True skips to next key if key is not empty string + #Othewise don't skip for first pass + #""" + + #with self.env.begin(db=db, write=False, buffers=True) as txn: + #cursor = txn.cursor() + #if cursor.set_range(key): # moves to first_dup at key + #found = True + #if skip and key and cursor.key() == key: # skip to next key + #found = cursor.next_nodup() # skip to next key not dup if any + #if found: + #for key, val in cursor.iternext_dup(keys=True): + #yield (key, val[33:]) # slice off prepended ordering prefix # methods for OnIoDup that combines IoDup value proem with On ordinal numbered diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index 5655838e..f8971f7b 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -625,22 +625,22 @@ def test_baser(): # Test getVreItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getVreItemsNextIter()] + items = [item for item in db.getVreItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getVreItemsNextIter(key=aKey)] + items = [item for item in db.getVreItemIter(key=aKey)] assert items # not empty ikey = items[0][0] - assert ikey == bKey + assert ikey == aKey vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals # bVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -650,7 +650,7 @@ def test_baser(): assert db.delVre(ikey, val) == True # cVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -660,7 +660,7 @@ def test_baser(): assert db.delVre(ikey, val) == True # dVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -732,38 +732,24 @@ def test_baser(): # Test getPseItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getPseItemsNextIter()] + items = [item for item in db.getPseItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getPseItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getPseItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - items = [item for item in db.getPseItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getPseItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delPse(ikey, val) == True + # bVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -773,7 +759,7 @@ def test_baser(): assert db.delPse(ikey, val) == True # cVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -783,7 +769,7 @@ def test_baser(): assert db.delPse(ikey, val) == True # dVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -872,38 +858,22 @@ def test_baser(): # Test getPweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getPweItemsNextIter()] + items = [item for item in db.getPweItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getPweItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getPweItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getPweItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getPweItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delPwe(ikey, val) == True # bVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -913,7 +883,7 @@ def test_baser(): assert db.delPwe(ikey, val) == True # cVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -923,7 +893,7 @@ def test_baser(): assert db.delPwe(ikey, val) == True # dVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -974,38 +944,22 @@ def test_baser(): # Test getUweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUweItemsNextIter()] + items = [item for item in db.getUweItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUweItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getUweItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getUweItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getUweItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delUwe(ikey, val) == True - # bVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1015,7 +969,7 @@ def test_baser(): assert db.delUwe(ikey, val) == True # cVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1025,7 +979,7 @@ def test_baser(): assert db.delUwe(ikey, val) == True # dVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1074,38 +1028,22 @@ def test_baser(): # Test getOoeItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getOoeItemsNextIter()] + items = [item for item in db.getOoeItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getOoeItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getOoeItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getOoeItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getOoeItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delOoe(ikey, val) == True # bVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1115,7 +1053,7 @@ def test_baser(): assert db.delOoe(ikey, val) == True # cVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1125,7 +1063,7 @@ def test_baser(): assert db.delOoe(ikey, val) == True # dVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1190,38 +1128,22 @@ def test_baser(): # Test getLdeItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getLdeItemsNextIter()] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getLdeItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getLdeItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getLdeItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getLdeItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getLdeItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delLde(ikey, val) == True # bVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1231,7 +1153,7 @@ def test_baser(): assert db.delLde(ikey, val) == True # cVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1241,7 +1163,7 @@ def test_baser(): assert db.delLde(ikey, val) == True # dVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index e7f14a88..48be99de 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -704,58 +704,58 @@ def test_lmdber(): assert not items - # Test getIoItemsNextIter(self, db, key=b"") - # get dups at first key in database - # aVals - items = [item for item in dber.getIoDupItemsNextIter(edb)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in dber.getIoDupItemsNextIter(edb, key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert dber.delIoDupVal(edb, ikey, val) == True - - # bVals - items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - for key, val in items: - assert dber.delIoDupVal(edb, ikey, val) == True - - # cVals - items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - for key, val in items: - assert dber.delIoDupVal(edb, ikey, val) == True + ## Test getIoItemsNextIter(self, db, key=b"") + ## get dups at first key in database + ## aVals + #items = [item for item in dber.getIoDupItemsNextIter(edb)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == aKey + #vals = [val for key, val in items] + #assert vals == aVals + + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey, skip=False)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == aKey + #vals = [val for key, val in items] + #assert vals == aVals + + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == bKey + #vals = [val for key, val in items] + #assert vals == bVals + + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=b'', skip=False)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == aKey + #vals = [val for key, val in items] + #assert vals == aVals + #for key, val in items: + #assert dber.delIoDupVal(edb, ikey, val) == True + + ## bVals + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == bKey + #vals = [val for key, val in items] + #assert vals == bVals + #for key, val in items: + #assert dber.delIoDupVal(edb, ikey, val) == True + + ## cVals + #items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] + #assert items # not empty + #ikey = items[0][0] + #assert ikey == cKey + #vals = [val for key, val in items] + #assert vals == cVals + #for key, val in items: + #assert dber.delIoDupVal(edb, ikey, val) == True # test OnIoDup methods From 502a7b7841adafb6c7fbd6b14683f0e15c19aac6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 3 Sep 2024 18:58:00 -0600 Subject: [PATCH 44/78] clean up --- src/keri/db/dbing.py | 48 -------------------------------------------- 1 file changed, 48 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 7786285a..21b6380e 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1680,54 +1680,6 @@ def getTopIoDupItemIter(self, db, top=b''): yield (top, val) -# don't need this in IoDupSuber because can replace with self.getTopIoDupItemIter -# which in turen can be replaced with IoDupSuber.getItemIter - - #def getIoDupItemsNextIter(self, db, key=b"", skip=True): - #""" - #Return iterator of all dup items at next key after key in db in insertion order. - #Item is (key, val) with proem stripped from val stored in db. - #If key = b'' then returns list of dup items at first key in db. - #If skip is False and key is not empty then returns dup items at key - #Raises StopIteration Error when no remaining dup items = empty. - - #If key is empty then gets io items (key, io value) at first key in db - #Use the return key from items as next key for next call to function in - #order to iterate through the database - - #Assumes DB opened with dupsort=True - - #Duplicates at a given key preserve insertion order of duplicate. - #Because lmdb is lexocographic an insertion ordering proem is prepended to - #all values that makes lexocographic order that same as insertion order. - - #Duplicates are ordered as a pair of key plus value so prepending proem - #to each value changes duplicate ordering. Proem is 33 characters long. - #With 32 character hex string followed by '.' for essentiall unlimited - #number of values which will be limited by memory. - - #With prepended proem ordinal must explicity check for duplicate values - #before insertion. Uses a python set for the duplicate inclusion test. - #Set inclusion scales with O(1) whereas list inclusion scales with O(n). - - #Parameters: - #db is opened named sub db with dupsort=True - #key is bytes of key within sub db's keyspace or empty - #skip is Boolean If True skips to next key if key is not empty string - #Othewise don't skip for first pass - #""" - - #with self.env.begin(db=db, write=False, buffers=True) as txn: - #cursor = txn.cursor() - #if cursor.set_range(key): # moves to first_dup at key - #found = True - #if skip and key and cursor.key() == key: # skip to next key - #found = cursor.next_nodup() # skip to next key not dup if any - #if found: - #for key, val in cursor.iternext_dup(keys=True): - #yield (key, val[33:]) # slice off prepended ordering prefix - - # methods for OnIoDup that combines IoDup value proem with On ordinal numbered # trailing prefix # this is so we do the proem add and strip here not in some higher level class From 180e4db83526f85da04e7e6397aaa07517340ee8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 10:16:17 -0600 Subject: [PATCH 45/78] created methods for OnValIterators to better support event logs --- src/keri/db/basing.py | 5 ++- src/keri/db/dbing.py | 67 ++++++++++++++++++++++++++++++++++++++--- src/keri/db/subing.py | 40 ++++++++++++++++++++++++ tests/db/test_dbing.py | 52 +++++++++++++++++++++++++++++++- tests/db/test_subing.py | 23 ++++++++++++++ 5 files changed, 180 insertions(+), 7 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 0aff8f1a..983d8c8b 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2560,7 +2560,10 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) + for key, on, val in self.getOnIoDupItemIter(self.kels, pre, on=sn): + yield val + + #return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) def getKelBackIter(self, pre, sn=0): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 21b6380e..47a84655 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -662,9 +662,7 @@ def delTopVal(self, db, top=b''): # For subdbs the use keys with trailing part the is monotonically # ordinal number serialized as 32 hex bytes - - - + # used in OnSuberBase def appendOnVal(self, db, key, val, *, sep=b'.'): """ Appends val in order after last previous onkey in db where @@ -723,7 +721,7 @@ def appendOnVal(self, db, key, val, *, sep=b'.'): raise ValueError(f"Failed appending {val=} at {key=}.") return on - + # used in OnSuberBase def delOnVal(self, db, key, on=0, *, sep=b'.'): """ Deletes value at onkey consisting of key + sep + serialized on in db. @@ -746,7 +744,7 @@ def delOnVal(self, db, key, on=0, *, sep=b'.'): raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - + # used in OnSuberBase def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): """ Returns (int): count of of all ordinal keyed vals with key @@ -785,7 +783,34 @@ def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): return count + # used in OnSuberBase + def getOnValIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of the val at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + Returns: + items (Iterator[bytes]): val with same + key but increments of on beginning with on + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for (key, on, val) in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + yield (val) + + # used in OnSuberBase def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): """ Returns iterator of triples (key, on, val), at each key over all ordinal @@ -1713,6 +1738,36 @@ def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): val = (b'%032x.' % (0)) + val # prepend ordering proem return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) + + # used in OnIoDupSuber + def getOnIoDupValIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnIoDupItemIter(db=db, key=key, on=on, sep=sep): + yield (val) + + # used in OnIoDupSuber def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): """ @@ -1749,6 +1804,8 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): # Consider using ItemIter instead of just replaying vals since ItemIter already # works with either dupsort==True or False. also skips gaps. +# replace with getOnIoDupItemIter which iterates over all pre starting at given on + def getOnIoDupValsAllPreIter(self, db, pre, on=0): """ Returns iterator of all dup vals in insertion order for all entries diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 09c1bb8e..b32cf984 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -492,6 +492,27 @@ def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): on=on, sep=self.sep.encode())) + + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[bytes]): of val with same key but increments of + on >= on i.e. all key.on beginning with on + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form actual key + When keys is empty then retrieves whole database including + duplicates if any + on (int): ordinal number used with onKey(pre,on) to form key at at + which to initiate retrieval + sep (bytes): separator character for split + """ + for val in (self.db.getOnValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns: @@ -2098,6 +2119,25 @@ def appendOn(self, keys: str | bytes | memoryview, sep=self.sep.encode())) + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + items (Iterator[(top keys, on, val)]): triples of (top keys, on int, + deserialized val) + + Parameters: + keys (str | bytes | memoryview | iterator): top keys as prefix to be + combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 48be99de..8603eeb4 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -413,7 +413,8 @@ def test_lmdber(): assert dber.cntOnVals(db, key=b'') == 6 # all keys assert dber.cntOnVals(db) == 6 # all keys - # replay preB events in database + # iter replay + # replay preB event items in database items = [item for item in dber.getOnItemIter(db, preB)] assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY)] @@ -459,6 +460,49 @@ def test_lmdber(): items = [item for item in dber.getOnItemIter(db, key=preC, on=1)] assert items == [] + # val replay + # replay preB event vals in database + vals = [val for val in dber.getOnValIter(db, preB)] + assert vals == [digU, digV, digW, digX, digY] + + # resume replay preB events at on = 3 + vals = [val for val in dber.getOnValIter(db, preB, on=3)] + assert vals == [digX, digY] + + # resume replay preB events at on = 5 + vals = [val for val in dber.getOnValIter(db, preB, on=5)] + assert vals == [] + + vals = [val for val in dber.getOnValIter(db, key=b'')] + assert vals == [digA, + digY, + digU, + digV, + digW, + digX, + digY, + digC] + + vals = [val for val in dber.getOnValIter(db)] + assert vals == [digA, + digY, + digU, + digV, + digW, + digX, + digY, + digC] + + # resume replay all starting at preB on=2 + top, on = splitOnKey(keyB2) + vals = [val for val in dber.getOnValIter(db, key=top, on=on)] + assert vals == [digW, digX, digY] + + # resume replay all starting at preC on=1 + vals = [val for val in dber.getOnValIter(db, key=preC, on=1)] + assert vals == [] + + # test delOnVal assert dber.delOnVal(db, key=preB) # default on=0 assert not dber.delOnVal(db, key=preB, on=0) @@ -779,6 +823,12 @@ def test_lmdber(): (b'Z', 3, b'n')] + vals = [ bytes(val) for val in dber.getOnIoDupValIter(edb, key=key)] + assert vals == [b'k', b'l', b'm', b'n'] + + vals = [ bytes(val) for val in dber.getOnIoDupValIter(edb, key=key, on=2)] + assert vals == [ b'm', b'n'] + # test IoSetVals insertion order set of vals methods. key0 = b'ABC.ZYX' diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 1a042462..7a508113 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -255,6 +255,12 @@ def test_on_suber(): (('a', '00000000000000000000000000000003'), 'White snow')] # test getOnItemIter + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + items = [item for item in onsuber.getOnItemIter(keys='a')] assert items == [(('a',), 0, 'Blue dog'), (('a',), 1, 'Green tree'), @@ -265,6 +271,23 @@ def test_on_suber(): assert items == [(('a',), 2, 'Red apple'), (('a',), 3, 'White snow')] + # test getOnIter + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == ['Red apple', + 'White snow'] + assert 0 == onsuber.appendOn(keys=("b",), val=w) assert 1 == onsuber.appendOn(keys=("b",), val=x) From 84751b3bbb1cec06b661cef7cc043e6c562f576d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 10:28:36 -0600 Subject: [PATCH 46/78] refactor getKelIter to use new methods --- src/keri/db/basing.py | 4 ++-- src/keri/db/dbing.py | 43 ------------------------------------------ tests/db/test_dbing.py | 4 ---- 3 files changed, 2 insertions(+), 49 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 983d8c8b..3e560cb5 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2560,8 +2560,8 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - for key, on, val in self.getOnIoDupItemIter(self.kels, pre, on=sn): - yield val + + return (self.getOnIoDupValIter(self.kels, pre, on=sn)) #return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 47a84655..a106c7bf 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1799,49 +1799,6 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): - -# without gaps is used for replay with on to provide partial replay -# Consider using ItemIter instead of just replaying vals since ItemIter already -# works with either dupsort==True or False. also skips gaps. - -# replace with getOnIoDupItemIter which iterates over all pre starting at given on - - def getOnIoDupValsAllPreIter(self, db, pre, on=0): - """ - Returns iterator of all dup vals in insertion order for all entries - with same prefix across all ordinal numbers in increasing order - without gaps between ordinal numbers - starting with on, default 0. Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning - - Raises StopIteration Error when empty. - - Duplicates are retrieved in insertion order. - - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. - - Parameters: - db is opened named sub db with dupsort=True - pre (bytes | str): of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to begin iteration at - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - # slice off prepended ordering prefix - yield val[33:] - key = snKey(pre, cnt:=cnt+1) - - # Create multiple methods # getTopItemBackIter symmetric with getTopItemIter # getTopIoSetItemBackIter symmetric getTopIoSetItemIter diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 8603eeb4..b84bff02 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -601,10 +601,6 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getOnIoDupValsAllPreIter(db, pre)] - allvals = vals0 + vals1 + vals2 - assert vals == allvals - # Test getIoValsLastAllPreIter(self, db, pre) pre = b'BAejWzwQPYGGwTmuupUhPx5_yZ-Wk1xEHHzq7K0gzhcc' vals0 = [b"gamma", b"beta"] From 6bc1f8a4bcac9409199c06e4b00956c035a24d1d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 10:30:28 -0600 Subject: [PATCH 47/78] some clean up --- src/keri/db/dbing.py | 83 +++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index a106c7bf..a7242b40 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1798,12 +1798,53 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): yield (key, on, val) + # Last is special so need method. + # Used to replay forward all last duplicate values starting at on + # need to fix this so it is not stopped by gaps or if gap raises error as + # gap is normally a problem for replay so maybe a parameter to raise error on gap + def getIoDupValLastAllPreIter(self, db, pre, on=0): + """ + Returns iterator of last only of dup vals of each key in insertion order + for all entries with same key across all sequence numbers in increasing order + without gaps starting with on (default = 0). Stops if gap or different key. + Assumes that key is combination of prefix and sequence number given + by .snKey(). + Removes prepended proem ordinal from each val before returning + + Raises StopIteration Error when empty. + + Duplicates are retrieved in insertion order. + + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order + Duplicates are ordered as a pair of key plus value so prepending prefix + to each value changes duplicate ordering. Proem is 17 characters long. + With 16 character hex string followed by '.'. + + + Parameters: + db is opened named sub db with dupsort=True + pre is bytes of itdentifier prefix prepended to sn in key + within sub db's keyspace + on (int): ordinal number to being iteration + """ + with self.env.begin(db=db, write=False, buffers=True) as txn: + cursor = txn.cursor() + key = snKey(pre, cnt:=on) + while cursor.set_key(key): # moves to first_dup + if cursor.last_dup(): # move to last_dup + yield cursor.value()[33:] # slice off prepended ordering proem + key = snKey(pre, cnt:=cnt+1) + + + # Create Two methods + # getOnItemBackIter symmetric with getOnItemIter + # getOnIoDupItemBackIter symmetric with getOnIoDupItemIter - # Create multiple methods # getTopItemBackIter symmetric with getTopItemIter # getTopIoSetItemBackIter symmetric getTopIoSetItemIter # getTopIoDupItemBackIter symmetric with getTopIoDupItemIter - # getOnItemBackIter symmetric with getOnItemIter + # instead of just replaying vals replay items since @@ -1853,44 +1894,6 @@ def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): yield val[33:] key = snKey(pre, cnt:=cnt-1) -# Last is special so need method. -# Used to replay forward all last duplicate values starting at on -# need to fix this so it is not stopped by gaps or if gap raises error as -# gap is normally a problem for replay so maybe a parameter to raise error on gap - def getIoDupValLastAllPreIter(self, db, pre, on=0): - """ - Returns iterator of last only of dup vals of each key in insertion order - for all entries with same key across all sequence numbers in increasing order - without gaps starting with on (default = 0). Stops if gap or different key. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning - - Raises StopIteration Error when empty. - - Duplicates are retrieved in insertion order. - - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. - - - Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to being iteration - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - if cursor.last_dup(): # move to last_dup - yield cursor.value()[33:] # slice off prepended ordering proem - key = snKey(pre, cnt:=cnt+1) - # ToDo do we need a replay last backwards? From f2b30337c9433aae2c79fb756f936658ff7be3a6 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 11:53:44 -0600 Subject: [PATCH 48/78] remove commented out code --- src/keri/app/storing.py | 36 ------------------------------------ src/keri/db/subing.py | 23 ----------------------- 2 files changed, 59 deletions(-) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 073b365c..80cf0f2f 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -58,8 +58,6 @@ def reopen(self, **kwa): :return: """ super(Mailboxer, self).reopen(**kwa) - - #self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True) self.tpcs = subing.OnSuber(db=self, subkey='tpcs.') self.msgs = subing.Suber(db=self, subkey='msgs.') # key states @@ -73,7 +71,6 @@ def delTopic(self, key, on=0): exists in database so removed False otherwise (not removed) """ - #return self.delIoSetVals(self.tpcs, key) return self.tpcs.remOn(keys=key, on=on) def appendToTopic(self, topic, val): @@ -88,7 +85,6 @@ def appendToTopic(self, topic, val): topic (bytes): topic identifier for message val (bytes): msg digest """ - #return self.appendIoSetVal(db=self.tpcs, key=topic, val=val) return self.tpcs.appendOn(key=topic, val=val) @@ -111,15 +107,6 @@ def getTopicMsgs(self, topic, fn=0): msgs.append(msg.encode()) # want bytes not str return msgs - #if hasattr(topic, "encode"): - #topic = topic.encode("utf-8") - - #digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn) - #msgs = [] - #for dig in digs: - #if msg := self.msgs.get(keys=dig): - #msgs.append(msg.encode("utf-8")) - #return msgs def storeMsg(self, topic, msg): """ @@ -143,16 +130,6 @@ def storeMsg(self, topic, msg): on = self.tpcs.appendOn(keys=topic, val=digb) return self.msgs.pin(keys=digb, val=msg) - #if hasattr(topic, "encode"): - #topic = topic.encode("utf-8") - - #if hasattr(msg, "encode"): - #msg = msg.encode("utf-8") - - #digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b - #self.appendToTopic(topic=topic, val=digb) - #return self.msgs.pin(keys=digb, val=msg) - def cloneTopicIter(self, topic, fn=0): """ @@ -177,19 +154,6 @@ def cloneTopicIter(self, topic, fn=0): if msg := self.msgs.get(keys=dig): yield (on, topic, msg.encode("utf-8")) - #if hasattr(topic, 'encode'): - #topic = topic.encode("utf-8") - - #for ion, (topic, dig) in enumerate(self.getTopIoSetItemIter(self.tpcs, topic)): - #if ion >= fn: - #if msg := self.msgs.get(keys=dig): - #yield ion, topic, msg.encode("utf-8") - - #for (key, dig) in self.getIoSetItemIter(self.tpcs, key=topic, ion=fn): - #topic, ion = dbing.unsuffix(key) - #if msg := self.msgs.get(keys=dig): - #yield ion, topic, msg.encode("utf-8") - class Respondant(doing.DoDoer): diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index b32cf984..30d5d107 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1978,29 +1978,6 @@ def cnt(self, keys: str | bytes | memoryview | Iterable): return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) - #def getIoDupItemIter(self, keys: str|bytes|memoryview|Iterable = ''): - #""" - #Gets (iokeys, val) ioitems iterator at key made from keys where key is - #apparent effective key and items all have same apparent effective key - - #Parameters: - #keys (str|bytes|memoryview|Iterable): of key strs to be combined - #in order to form key. When keys is empty then retrieves - #all items in database. - - #Returns: - #items (Iterator): each item iterated is tuple (keys, val) where - #each keys is actual keys tuple and each val is dup without - #prefixed insertion ordering proem. - #Empty list if no entry at keys. - #Raises StopIteration when done - - #""" - #for key, val in self.db.getIoDupItemIter(db=self.sdb, - #key=self._tokey(keys)): - #yield (self._tokeys(key), self._des(val)) - - def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", *, topive=False): """ From 624da64f20ed1bd1a3f6dee77174f1ad39b7e7b9 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 11:54:21 -0600 Subject: [PATCH 49/78] clean up --- src/keri/db/basing.py | 2 +- src/keri/db/dbing.py | 2 +- tests/db/test_dbing.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 3e560cb5..4f6895bd 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2611,7 +2611,7 @@ def getKelLastIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoDupValLastAllPreIter(self.kels, pre, on=sn) + return self.getOnIoDupValLastAllPreIter(self.kels, pre, on=sn) def putPses(self, key, vals): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index a7242b40..52fe068c 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1802,7 +1802,7 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): # Used to replay forward all last duplicate values starting at on # need to fix this so it is not stopped by gaps or if gap raises error as # gap is normally a problem for replay so maybe a parameter to raise error on gap - def getIoDupValLastAllPreIter(self, db, pre, on=0): + def getOnIoDupValLastAllPreIter(self, db, pre, on=0): """ Returns iterator of last only of dup vals of each key in insertion order for all entries with same key across all sequence numbers in increasing order diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index b84bff02..8ea3cfc7 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -619,7 +619,7 @@ def test_lmdber(): key = snKey(pre, sn) assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoDupValLastAllPreIter(db, pre)] + vals = [bytes(val) for val in dber.getOnIoDupValLastAllPreIter(db, pre)] lastvals = [vals0[-1], vals1[-1], vals2[-1]] assert vals == lastvals From 6c5c5cccbab13c31d9bb0cc3a6c766dc24770d40 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 17:15:47 -0600 Subject: [PATCH 50/78] refactored backIter methods fixed bug --- src/keri/core/eventing.py | 2 + src/keri/db/basing.py | 4 +- src/keri/db/dbing.py | 254 +++++++++++++------------ src/keri/db/subing.py | 67 ++++--- tests/core/test_eventing.py | 2 +- tests/db/test_dbing.py | 367 +++++++++++++++++++++++++----------- tests/db/test_subing.py | 43 +++++ 7 files changed, 490 insertions(+), 249 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 580bc1f9..d07caf2b 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -3245,6 +3245,8 @@ def fetchPriorDigers(self, sn: int | None = None) -> list | None: pre = self.prefixer.qb64 if sn is None: sn = self.lastEst.s - 1 + if sn < 0: + return None for digb in self.db.getKelBackIter(pre, sn): dgkey = dgKey(pre, digb) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 4f6895bd..69557fed 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -2587,7 +2587,7 @@ def getKelBackIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getOnIoDupValsAllPreBackIter(self.kels, pre, sn) + return self.getOnIoDupValBackIter(self.kels, pre, sn) def getKelLastIter(self, pre, sn=0): @@ -2611,7 +2611,7 @@ def getKelLastIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getOnIoDupValLastAllPreIter(self.kels, pre, on=sn) + return self.getOnIoDupLastValIter(self.kels, pre, on=sn) def putPses(self, key, vals): diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 52fe068c..57a3f491 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -697,21 +697,21 @@ def appendOnVal(self, db, key, val, *, sep=b'.'): # so either empty database or last is earlier pre or # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) if ckey == key: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) if ckey == key: # last entry for pre is already at max raise ValueError(f"Number part {cn=} for key part {ckey=}" f"exceeds maximum size.") else: # later pre so backup one entry # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre - ckey = cursor.key() - ckey, cn = splitOnKey(ckey, sep=sep) + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) if ckey == key: # last entry at pre on = cn + 1 # increment @@ -849,6 +849,9 @@ def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): break yield (ckey, cn, cval) + # ToDo + # getOnItemBackIter symmetric with getOnItemIter + # getOnValBackIter symmetric with getOnValIter # IoSet insertion order in val so can have effective dups but with @@ -1777,8 +1780,6 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): Values duplicates are sorted internally by hidden prefixed insertion order proem ordinal Returned items are triples of (key, on, val) - When dupsort==true then duplicates are included in items since .iternext - includes duplicates. when key is empty then retrieves whole db Raises StopIteration Error when empty. @@ -1798,151 +1799,166 @@ def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): yield (key, on, val) - # Last is special so need method. - # Used to replay forward all last duplicate values starting at on - # need to fix this so it is not stopped by gaps or if gap raises error as - # gap is normally a problem for replay so maybe a parameter to raise error on gap - def getOnIoDupValLastAllPreIter(self, db, pre, on=0): - """ - Returns iterator of last only of dup vals of each key in insertion order - for all entries with same key across all sequence numbers in increasing order - without gaps starting with on (default = 0). Stops if gap or different key. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + def getOnIoDupLastValIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator of val of last insertion ordered duplicate at each + key over all ordinal numbered keys with same full key + of key + sep + on in db. Values are sorted by onKey(key, on) where on + is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + + when key is empty then retrieves whole db Raises StopIteration Error when empty. + Returns: + val (Iterator[bytes]): last dup val at each onkey - Duplicates are retrieved in insertion order. + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnIoDupLastItemIter(db=db, key=key, on=on, sep=sep): + yield (val) - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + def getOnIoDupLastItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator of triples (key, on, val), of last insertion ordered + duplicate at each key over all ordinal numbered keys with same full key + of key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to being iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - if cursor.last_dup(): # move to last_dup - yield cursor.value()[33:] # slice off prepended ordering proem - key = snKey(pre, cnt:=cnt+1) + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty + onkey = key + if not cursor.set_range(onkey): # # moves to first_dup at key>=onkey + return # no values end of db raises StopIteration - # Create Two methods - # getOnItemBackIter symmetric with getOnItemIter - # getOnIoDupItemBackIter symmetric with getOnIoDupItemIter + while cursor.last_dup(): # move to last_dup at current ckey + onkey, cval = cursor.item() # get ckey cval of last dup + ckey, on = splitOnKey(onkey, sep=sep) # get key on + if key and not ckey == key: + break - # getTopItemBackIter symmetric with getTopItemIter - # getTopIoSetItemBackIter symmetric getTopIoSetItemIter - # getTopIoDupItemBackIter symmetric with getTopIoDupItemIter + yield (ckey, on, cval[33:]) # slice off prepended ordering proem + onkey = onKey(ckey, on+1) + if not cursor.set_range(onkey): # # moves to first_dup at key>=onkey + return # no values end of db raises StopIteration - # instead of just replaying vals replay items since - # getTopItemIter already works with either dupsort==True or False - # so if we create getTopBackItemIter then we can use that everywhere - # ToDo need unit tests in dbing Used to replay backwards all duplicate - # values starting at on - # need to fix this so it is not stopped by gaps or if gap raises error as - # gap is normally a problem for replay so maybe a parameter to raise error on gap + # getOnIoDupItemBackIter symmetric with getOnIoDupItemIter + # getOnIoDupValBackIter symmetric with getOnIoDupValIter - def getOnIoDupValsAllPreBackIter(self, db, pre, on=0): - """ - Returns iterator of all dup vals in insertion order for all entries - with same prefix across all sequence numbers in decreasing order without gaps - between ordinals at a given pre. - Starting with on (default = 0) as begining ordinal number or sequence number. - Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + def getOnIoDupValBackIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator going backwards of values, + of insertion ordered item at each key over all ordinal numbered keys + with same full key of key + sep + on in db. + Values are sorted by onKey(key, on) where on is ordinal number int and + key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Backwards means decreasing numerical value of duplicate proem, for each on, + decreasing numerical value on for each key and decresing lexocogrphic + order of each key prefix. - Raises StopIteration Error when empty. + Returned items are vals + + when key is empty then retrieves whole db - Duplicates are retrieved in insertion order. + Raises StopIteration Error when empty. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + Returns: + val (Iterator[bytes]): at key including duplicates in backwards order Parameters: - db is opened named sub db with dupsort=True - pre is bytes of identifier prefix prepended to sn in key - within sub db's keyspace - on (int): is ordinal number to begin iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = snKey(pre, cnt := on) - # set_key returns True if exact key else false - while cursor.set_key(key): # moves to first_dup if valid key - for val in cursor.iternext_dup(): - # slice off prepended ordering prefix - yield val[33:] - key = snKey(pre, cnt:=cnt-1) - - - -# ToDo do we need a replay last backwards? - + for key, on, val in self.getOnIoDupItemBackIter(db=db, key=key, on=on, sep=sep): + yield (val) - # not used anymore. - # before deleting this method use it inform how to cross gaps in other - # replays above with parameter such as replay last above. - # to raise error if detect gap when should not be one for event logs vs escrows - # then remove this method since not used otherwise - def getOnIoDupValsAnyPreIter(self, db, pre, on=0): - """ - Returns iterator of all dup vals in insertion order for any entries - with same prefix across all ordinal numbers in order including gaps - between ordinals at a given pre. Staring with on (default = 0). - Stops when pre is different. + def getOnIoDupItemBackIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator going backwards of triples (key, on, val), + of insertion ordered item at each key over all ordinal numbered keys + with same full key of key + sep + on in db. + Values are sorted by onKey(key, on) where on is ordinal number int and + key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Backwards means decreasing numerical value of duplicate proem, for each on, + decreasing numerical value on for each key and decresing lexocogrphic + order of each key prefix. - Duplicates that may be deleted such as duplicitous event logs need - to be able to iterate across gaps in ordinal number. + Returned items are triples of (key, on, val) - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + when key is empty then retrieves whole db Raises StopIteration Error when empty. - Duplicates are retrieved in insertion order. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): beginning ordinal number to start iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_range(key): # moves to first dup of key >= key - key = cursor.key() # actual key - front, back = bytes(key).split(sep=b'.', maxsplit=1) - if front != pre: # set range may skip pre if none - break - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering prefix - cnt = int(back, 16) - key = snKey(pre, cnt:=cnt+1) + if not cursor.last(): # pre-position cursor at last dup of last key + return # empty database so raise StopIteration + + if key: # not empty so attempt to position at starting key not last + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + if cursor.set_range(onkey): # found key >= onkey + ckey, cn = splitOnKey(cursor.key(), sep=sep) + if ckey == key: # onkey in db + cursor.last_dup() # start at its last dup + else: # get closest key < onkey + if not cursor.prev(): # last dup of previous key + return # no earlier keys to designated start + + # cursor should now be correctly positioned for start either at + # last dup of either last key or onkey + for onkey, cval in cursor.iterprev(): # iterate backwards + ckey, on = splitOnKey(onkey, sep=sep) + if key and ckey != key: + return + yield (ckey, on, cval[33:]) + + + + # ToDo do we need a replay last backwards? + diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 30d5d107..48ec914e 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -2015,21 +2015,6 @@ def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", top=self._tokey(keys, topive=topive)): yield (self._tokeys(key), self._des(val)) -# ToDo -# OnIoDupSuber methods instead of pre just use key prefix parts top val so -# can work with any key space that has last part has ordinal -# try refactoring these to use cursor.iternext cursor.iterpre cursor.next cursor.prev -# so they work across prefixes without having the extra burden doing repeated -# iter within a given pre dups. This way they could work with or without duplicates -# and then the IoDupIter could just strip the proem whereas OrdSuber would not care - -# used by .kels - # getIoDupValsAllPreIter(self, db, pre, on=0): - # getIoDupValsAllPreBackIter(self, db, pre, on=0): - # getIoDupValLastAllPreIter(self.kels, pre, on=sn) -# used by .dels - # getIoDupValsAnyPreIter(self, db, pre, on=0) - class OnIoDupSuber(OnSuberBase, IoDupSuber): """ @@ -2099,9 +2084,45 @@ def appendOn(self, keys: str | bytes | memoryview, def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns - items (Iterator[(top keys, on, val)]): triples of (top keys, on int, + val (Iterator[bytes]): deserialized val of of each + onkey + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[(top keys, on, val)]): triples of (onkeys, on int, deserialized val) + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + def getOnLastIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + last (Iterator[bytes]): deserialized last duplicate val of of each + onkey + Parameters: keys (str | bytes | memoryview | iterator): top keys as prefix to be combined with serialized on suffix and sep to form key @@ -2109,25 +2130,27 @@ def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ - for val in (self.db.getOnIoDupValIter(db=self.sdb, + for val in (self.db.getOnIoDupLastValIter(db=self.sdb, key=self._tokey(keys), on=on, sep=self.sep.encode())): yield (self._des(val)) - def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + def getOnLastItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns - items (Iterator[(top keys, on, val)]): triples of (top keys, on int, - deserialized val) + items (Iterator[(top keys, on, val)]): triples of (keys, on int, + deserialized val) last duplicate item as each onkey where onkey + is the key+serialized on Parameters: - keys (str | bytes | memoryview | iterator): top keys as prefix to be + keys (str | bytes | memoryview | iterator): keys as prefix to be combined with serialized on suffix and sep to form key When keys is empty then retrieves whole database including duplicates on (int): ordinal number used with onKey(pre,on) to form key. sep (bytes): separator character for split """ - for keys, on, val in (self.db.getOnIoDupItemIter(db=self.sdb, + for keys, on, val in (self.db.getOnIoDupLastItemIter(db=self.sdb, key=self._tokey(keys), on=on, sep=self.sep.encode())): yield (self._tokeys(keys), on, self._des(val)) + diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index c50cee10..a4474d54 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -4947,7 +4947,7 @@ def test_load_event(mockHelpingNowUTC): if __name__ == "__main__": # pytest.main(['-vv', 'test_eventing.py::test_keyeventfuncs']) #test_process_manual() - #test_keyeventsequence_0() + test_keyeventsequence_0() #test_process_transferable() #test_messagize() test_direct_mode() diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 8ea3cfc7..9c2348c5 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -583,47 +583,7 @@ def test_lmdber(): assert dber.getIoDupVals(db, key) == [b'm', b'a', b'w', b'e'] - - # Test getIoValsAllPreIter(self, db, pre) - vals0 = [b"gamma", b"beta"] - sn = 0 - key = snKey(pre, sn) - assert dber.addIoDupVal(db, key, vals0[0]) == True - assert dber.addIoDupVal(db, key, vals0[1]) == True - - vals1 = [b"mary", b"peter", b"john", b"paul"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoDupVals(db, key, vals1) == True - - vals2 = [b"dog", b"cat", b"bird"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoDupVals(db, key, vals2) == True - - # Test getIoValsLastAllPreIter(self, db, pre) - pre = b'BAejWzwQPYGGwTmuupUhPx5_yZ-Wk1xEHHzq7K0gzhcc' - vals0 = [b"gamma", b"beta"] - sn = 0 - key = snKey(pre, sn) - assert dber.addIoDupVal(db, key, vals0[0]) == True - assert dber.addIoDupVal(db, key, vals0[1]) == True - - vals1 = [b"mary", b"peter", b"john", b"paul"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoDupVals(db, key, vals1) == True - - vals2 = [b"dog", b"cat", b"bird"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoDupVals(db, key, vals2) == True - - vals = [bytes(val) for val in dber.getOnIoDupValLastAllPreIter(db, pre)] - lastvals = [vals0[-1], vals1[-1], vals2[-1]] - assert vals == lastvals - - # Test getIoValsAnyPreIter(self, db, pre) + # Test TopIoDupItemIter(self, db, pre) pre = b'BBPYGGwTmuupUhPx5_yZ-Wk1x4ejWzwEHHzq7K0gzhcc' vals0 = [b"gamma", b"beta"] sn = 1 # not start at zero @@ -646,7 +606,9 @@ def test_lmdber(): # dber.getTopIoDupItemIter() assert vals == allvals - # Setup Tests for getIoItemsNext and getIoItemsNextIter + + + # Setup Tests for TopIoDupItemsIter edb = dber.env.open_db(key=b'escrow.', dupsort=True) aKey = snKey(pre=b'A', sn=1) aVals = [b"z", b"m", b"x"] @@ -744,86 +706,281 @@ def test_lmdber(): assert not items - ## Test getIoItemsNextIter(self, db, key=b"") - ## get dups at first key in database - ## aVals - #items = [item for item in dber.getIoDupItemsNextIter(edb)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == aKey - #vals = [val for key, val in items] - #assert vals == aVals - - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey, skip=False)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == aKey - #vals = [val for key, val in items] - #assert vals == aVals - - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=aKey)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == bKey - #vals = [val for key, val in items] - #assert vals == bVals - - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=b'', skip=False)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == aKey - #vals = [val for key, val in items] - #assert vals == aVals - #for key, val in items: - #assert dber.delIoDupVal(edb, ikey, val) == True - - ## bVals - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == bKey - #vals = [val for key, val in items] - #assert vals == bVals - #for key, val in items: - #assert dber.delIoDupVal(edb, ikey, val) == True - - ## cVals - #items = [item for item in dber.getIoDupItemsNextIter(edb, key=ikey)] - #assert items # not empty - #ikey = items[0][0] - #assert ikey == cKey - #vals = [val for key, val in items] - #assert vals == cVals - #for key, val in items: - #assert dber.delIoDupVal(edb, ikey, val) == True + # test OnIoDup methods + ldb = dber.env.open_db(key=b'log.', dupsort=True) + # first pre + sn = 0 + key = snKey(preA, sn) + valsA0 = [b"echo", b"bravo"] + itemsA0 = [ + (preA, sn, valsA0[0]), + (preA, sn, valsA0[1]) + ] + assert dber.addIoDupVal(ldb, key, valsA0[0]) == True + assert dber.addIoDupVal(ldb, key, valsA0[1]) == True + + sn += 1 + key = snKey(preA, sn) + valsA1 = [b"sue", b"bob", b"val", b"zoe"] + itemsA1 = [ + (preA, sn, valsA1[0]), + (preA, sn, valsA1[1]), + (preA, sn, valsA1[2]), + (preA, sn, valsA1[3]), + ] + assert dber.putIoDupVals(ldb, key, valsA1) == True + + sn += 1 + key = snKey(preA, sn) + valsA2 = [b"fish", b"bat", b"snail"] + itemsA2 = [ + (preA, sn, valsA2[0]), + (preA, sn, valsA2[1]), + (preA, sn, valsA2[2]), + ] + assert dber.putIoDupVals(ldb, key, valsA2) == True + + # second pre + sn = 0 + key = snKey(preB, sn) + valsB0 = [b"gamma", b"beta"] + itemsB0 = [ + (preB, sn, valsB0[0]), + (preB, sn, valsB0[1]) + ] + assert dber.addIoDupVal(ldb, key, valsB0[0]) == True + assert dber.addIoDupVal(ldb, key, valsB0[1]) == True + + sn += 1 + key = snKey(preB, sn) + valsB1 = [b"mary", b"peter", b"john", b"paul"] + itemsB1 = [ + (preB, sn, valsB1[0]), + (preB, sn, valsB1[1]), + (preB, sn, valsB1[2]), + (preB, sn, valsB1[3]), + ] + assert dber.putIoDupVals(ldb, key, valsB1) == True + + + sn += 1 + key = snKey(preB, sn) + valsB2 = [b"dog", b"cat", b"bird"] + itemsB2 = [ + (preB, sn, valsB2[0]), + (preB, sn, valsB2[1]), + (preB, sn, valsB2[2]), + ] + assert dber.putIoDupVals(ldb, key, valsB2) == True + + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preA)] + lastitems = [itemsA0[-1], itemsA1[-1], itemsA2[-1]] + assert items == lastitems + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preA, on=1)] + lastitems = [itemsA1[-1], itemsA2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preA)] + lastvals = [valsA0[-1], valsA1[-1], valsA2[-1]] + assert vals == lastvals + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preA, on=1)] + lastvals = [valsA1[-1], valsA2[-1]] + assert vals == lastvals + + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preB)] + lastitems = [itemsB0[-1], itemsB1[-1], itemsB2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preB)] + lastvals = [valsB0[-1], valsB1[-1], valsB2[-1]] + assert vals == lastvals + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb)] + lastitems = [itemsA0[-1], itemsA1[-1], itemsA2[-1], itemsB0[-1], itemsB1[-1], itemsB2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb)] + lastvals = [valsA0[-1], valsA1[-1], valsA2[-1], valsB0[-1], valsB1[-1], valsB2[-1]] + assert vals == lastvals + + # test back iter + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preB, on=3)] + assert items ==[(preB, 2, b'bird'), + (preB, 2, b'cat'), + (preB, 2, b'dog'), + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma')] + + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preB, on=3)] + assert vals ==[ + b'bird', + b'cat', + b'dog', + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preB, on=1)] + assert items ==[ + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma') + ] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preB, on=1)] + assert vals ==[ + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preA, on=5)] + assert items ==[(preA, 2, b'snail'), + (preA, 2, b'bat'), + (preA, 2, b'fish'), + (preA, 1, b'zoe'), + (preA, 1, b'val'), + (preA, 1, b'bob'), + (preA, 1, b'sue'), + (preA, 0, b'bravo'), + (preA, 0, b'echo')] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preA, on=5)] + assert vals ==[ + b'snail', + b'bat', + b'fish', + b'zoe', + b'val', + b'bob', + b'sue', + b'bravo', + b'echo' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preA, on=0)] + assert items ==[ + (preA, 0, b'bravo'), + (preA, 0, b'echo')] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preA, on=0)] + assert vals ==[ + b'bravo', + b'echo' + ] + + # all items from last to first + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb)] + assert items ==[ + (preB, 2, b'bird'), + (preB, 2, b'cat'), + (preB, 2, b'dog'), + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma'), + (preA, 2, b'snail'), + (preA, 2, b'bat'), + (preA, 2, b'fish'), + (preA, 1, b'zoe'), + (preA, 1, b'val'), + (preA, 1, b'bob'), + (preA, 1, b'sue'), + (preA, 0, b'bravo'), + (preA, 0, b'echo'), + ] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb)] + assert vals ==[ + b'bird', + b'cat', + b'dog', + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma', + b'snail', + b'bat', + b'fish', + b'zoe', + b'val', + b'bob', + b'sue', + b'bravo', + b'echo' + ] # test OnIoDup methods key = b'Z' - assert 0 == dber.appendOnIoDupVal(edb, key, val=b'k') - assert 1 == dber.appendOnIoDupVal(edb, key, val=b'l') - assert 2 == dber.appendOnIoDupVal(edb, key, val=b'm') - assert 3 == dber.appendOnIoDupVal(edb, key, val=b'n') + assert 0 == dber.appendOnIoDupVal(ldb, key, val=b'k') + assert 1 == dber.appendOnIoDupVal(ldb, key, val=b'l') + assert 2 == dber.appendOnIoDupVal(ldb, key, val=b'm') + assert 3 == dber.appendOnIoDupVal(ldb, key, val=b'n') - assert dber.cntOnVals(edb, key) == 4 + assert dber.cntOnVals(ldb, key) == 4 + + vals = [ bytes(val) for val in dber.getOnIoDupValIter(ldb, key=key)] + assert vals == [b'k', b'l', b'm', b'n'] - items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(edb, key=key)] + vals = [ bytes(val) for val in dber.getOnIoDupValIter(ldb, key=key, on=2)] + assert vals == [ b'm', b'n'] + + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] assert items == [(b'Z', 0, b'k'), (b'Z', 1, b'l'), (b'Z', 2, b'm'), (b'Z', 3, b'n')] - items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(edb, key=key, on=2)] + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key, on=2)] assert items == [ (b'Z', 2, b'm'), (b'Z', 3, b'n')] + # test back iter - vals = [ bytes(val) for val in dber.getOnIoDupValIter(edb, key=key)] - assert vals == [b'k', b'l', b'm', b'n'] + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=3)] + assert items ==[(b'Z', 3, b'n'), + (b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=4)] + assert items ==[(b'Z', 3, b'n'), + (b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=2)] + assert items == [(b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] - vals = [ bytes(val) for val in dber.getOnIoDupValIter(edb, key=key, on=2)] - assert vals == [ b'm', b'n'] # test IoSetVals insertion order set of vals methods. diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index 7a508113..f70440ff 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -688,6 +688,18 @@ def test_on_iodup_suber(): (('a', '00000000000000000000000000000002'), 'Red apple'), (('a', '00000000000000000000000000000003'), 'White snow')] + + # test getOnIter + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == ['Red apple', + 'White snow'] + # test getOnItemIter items = [item for item in onsuber.getOnItemIter(keys='a')] assert items == [(('a',), 0, 'Blue dog'), @@ -719,6 +731,37 @@ def test_on_iodup_suber(): (('b', '00000000000000000000000000000001'), 'Green tree'), (('bc', '00000000000000000000000000000000'), 'Red apple')] + # test getOnIter + vals = [val for val in onsuber.getOnIter(keys='b')] + assert vals == ['Blue dog', 'Green tree'] + + vals = [val for val in onsuber.getOnIter(keys=('b', ))] + assert vals == ['Blue dog', 'Green tree'] + + vals = [val for val in onsuber.getOnIter(keys=('b', ""))] + assert vals == [] + + vals = [val for val in onsuber.getOnIter(keys='')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + + # test getOnItemIter items = [item for item in onsuber.getOnItemIter(keys='b')] assert items == [(('b',), 0, 'Blue dog'), From 56c47188761dae9769a8244dae6486fc783a073a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 4 Sep 2024 17:59:26 -0600 Subject: [PATCH 51/78] Added OnIoDupSuber support for BackIter methods --- src/keri/db/subing.py | 35 +++++++++++ tests/db/test_subing.py | 125 ++++++++++++++++++++++++++++++---------- 2 files changed, 129 insertions(+), 31 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 48ec914e..efec02a3 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -2154,3 +2154,38 @@ def getOnLastItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0) key=self._tokey(keys), on=on, sep=self.sep.encode())): yield (self._tokeys(keys), on, self._des(val)) + + def getOnBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + val (Iterator[bytes]): deserialized val of of each + onkey in reverse order + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValBackIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[(top keys, on, val)]): triples of (onkeys, on int, + deserialized val) in reverse order + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupItemBackIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index f70440ff..c9db9749 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -731,42 +731,18 @@ def test_on_iodup_suber(): (('b', '00000000000000000000000000000001'), 'Green tree'), (('bc', '00000000000000000000000000000000'), 'Red apple')] - # test getOnIter + + # test getOnItemIter getOnIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + vals = [val for val in onsuber.getOnIter(keys='b')] assert vals == ['Blue dog', 'Green tree'] vals = [val for val in onsuber.getOnIter(keys=('b', ))] assert vals == ['Blue dog', 'Green tree'] - vals = [val for val in onsuber.getOnIter(keys=('b', ""))] - assert vals == [] - - vals = [val for val in onsuber.getOnIter(keys='')] - assert vals == ['Blue dog', - 'Green tree', - 'Red apple', - 'White snow', - 'White snow', - 'Blue dog', - 'Green tree', - 'Red apple'] - - vals = [val for val in onsuber.getOnIter()] - assert vals == ['Blue dog', - 'Green tree', - 'Red apple', - 'White snow', - 'White snow', - 'Blue dog', - 'Green tree', - 'Red apple'] - - - # test getOnItemIter - items = [item for item in onsuber.getOnItemIter(keys='b')] - assert items == [(('b',), 0, 'Blue dog'), - (('b',), 1, 'Green tree')] - items = [item for item in onsuber.getOnItemIter(keys=('b', ))] assert items == [(('b',), 0, 'Blue dog'), (('b',), 1, 'Green tree')] @@ -774,6 +750,9 @@ def test_on_iodup_suber(): items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] assert items == [] + vals = [val for val in onsuber.getOnIter(keys=('b', ""))] + assert vals == [] + items = [item for item in onsuber.getOnItemIter(keys='')] assert items == [(('a',), 0, 'Blue dog'), (('a',), 1, 'Green tree'), @@ -784,6 +763,16 @@ def test_on_iodup_suber(): (('b',), 1, 'Green tree'), (('bc',), 0, 'Red apple')] + vals = [val for val in onsuber.getOnIter(keys='')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + items = [item for item in onsuber.getOnItemIter()] assert items == [(('a',), 0, 'Blue dog'), (('a',), 1, 'Green tree'), @@ -794,6 +783,16 @@ def test_on_iodup_suber(): (('b',), 1, 'Green tree'), (('bc',), 0, 'Red apple')] + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + # test with duplicates assert onsuber.add(keys=dbing.onKey("a", 0), val=z) assert onsuber.add(keys=dbing.onKey("a", 1), val=y) @@ -814,7 +813,7 @@ def test_on_iodup_suber(): (('a', '00000000000000000000000000000003'), 'White snow'), (('a', '00000000000000000000000000000003'), 'Blue dog')] - # test getOnItemIter + # test getOnItemIter getOnIter getOnItemBackIter getOnBackIter items = [item for item in onsuber.getOnItemIter(keys='a')] assert items == [(('a',), 0, 'Blue dog'), (('a',), 0, 'White snow'), @@ -825,17 +824,71 @@ def test_on_iodup_suber(): (('a',), 3, 'White snow'), (('a',), 3, 'Blue dog')] + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'White snow', + 'Green tree', + 'Red apple', + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog', + ] + + + items = [item for item in onsuber.getOnItemBackIter(keys='a', on=4)] + assert items == [(('a',), 3, 'Blue dog'), + (('a',), 3, 'White snow'), + (('a',), 2, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 1, 'Red apple'), + (('a',), 1, 'Green tree'), + (('a',), 0, 'White snow'), + (('a',), 0, 'Blue dog')] + + + vals = [val for val in onsuber.getOnBackIter(keys='a', on=4)] + assert vals == ['Blue dog', + 'White snow', + 'Green tree', + 'Red apple', + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog'] + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] assert items ==[(('a',), 2, 'Red apple'), (('a',), 2, 'Green tree'), (('a',), 3, 'White snow'), (('a',), 3, 'Blue dog')] + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == [ + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog', + ] + + items = [item for item in onsuber.getOnItemBackIter(keys='a', on=1)] + assert items ==[(('a',), 1, 'Red apple'), + (('a',), 1, 'Green tree'), + (('a',), 0, 'White snow'), + (('a',), 0, 'Blue dog')] + + + vals = [val for val in onsuber.getOnBackIter(keys='a', on=1)] + assert vals == ['Red apple', + 'Green tree', + 'White snow', + 'Blue dog'] # test append with duplicates assert 4 == onsuber.appendOn(keys=("a",), val=x) assert onsuber.cntOn(keys=("a",)) == 9 + # test remove assert onsuber.remOn(keys='a', on=1) assert not onsuber.remOn(keys='a', on=1) assert onsuber.remOn(keys='a', on=3) @@ -850,6 +903,16 @@ def test_on_iodup_suber(): (('a',), 2, 'Green tree'), (('a',), 4, 'Green tree')] + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == [ + 'Blue dog', + 'White snow', + 'Red apple', + 'Green tree', + 'Green tree', + ] + + assert not os.path.exists(db.path) assert not db.opened From d552f795688d8a310e9bb30747aa7305cd7e5637 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 10:22:12 -0600 Subject: [PATCH 52/78] refactored SchemerSuber to be ducktyped subclass of SerderSuberBase. Added verify parameter to Schemer.__init__ to be compatible with Suber classes that expensive verification when deserializing like Serder. Added missing unit tests. --- src/keri/core/scheming.py | 26 +++++--- src/keri/core/serdering.py | 7 ++- src/keri/db/subing.py | 121 ++++++------------------------------ tests/core/test_scheming.py | 1 + tests/db/test_subing.py | 99 ++++++++++++++++++++++++++++- 5 files changed, 140 insertions(+), 114 deletions(-) diff --git a/src/keri/core/scheming.py b/src/keri/core/scheming.py index 919d8e03..203ec950 100644 --- a/src/keri/core/scheming.py +++ b/src/keri/core/scheming.py @@ -266,7 +266,8 @@ class Schemer: """ - def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.Blake3_256): + def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), + code=MtrDex.Blake3_256, verify=True): """ Initialize instance of Schemer Deserialize if raw provided @@ -274,14 +275,19 @@ def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.B When serializing if kind provided then use kind instead of field in sed Parameters: - raw (bytes): of serialized schema - sed (dict): dict or None - if None its deserialized from raw - typ (JSONSchema): type of schema - kind (serialization): kind string value or None (see namedtuple coring.Serials) - supported kinds are 'json', 'cbor', 'msgpack', 'binary' - if kind (None): then its extracted from ked or raw - code (MtrDex): default digest code + raw (bytes): of serialized schema + sed (dict): dict or None + if None its deserialized from raw + typ (JSONSchema): type of schema + kind (serialization): kind string value or None (see namedtuple coring.Serials) + supported kinds are 'json', 'cbor', 'msgpack', 'binary' + if kind (None): then its extracted from ked or raw + code (MtrDex): default digest code + verify (bool): True means verify said(s) of given raw or sad. + Raises ValidationError if verification fails + False means don't verify. Useful to avoid unnecessary + reverification when deserializing from database + as opposed to over the wire reception. """ @@ -295,7 +301,7 @@ def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.B else: raise ValueError("Improper initialization need raw or sed.") - if not self._verify_schema(): + if verify and not self._verify_schema(): raise ValidationError("invalid kind {} for schema {}" "".format(self.kind, self.sed)) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index abb74a9b..f28ca035 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -561,8 +561,11 @@ def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, Either provided by parser from stream genus version or desired when generating Serder instance to stream verify (bool): True means verify said(s) of given raw or sad. - Raises ValidationError if verification fails - Ignore when raw not provided or when raw and saidify is True + False means don't verify. Useful to avoid unnecessary + reverification when deserializing from database + as opposed to over the wire reception. + Raises ValidationError if verification fails + Ignore when raw empty or when raw and saidify is True makify (bool): True means compute fields for sad including size and saids. proto (str | None): desired protocol type str value of Protocols diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index efec02a3..1c68056e 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1470,116 +1470,35 @@ def __init__(self, *pa, **kwa): -class SchemerSuber(Suber): +class SchemerSuber(SerderSuberBase, Suber): """ - Sub class of Suber where data is serialized Schemer instance + Sub class of SerderSuberBase and Suber where data is serialized Schemer instance + Schemer ser/des is ducktype of Serder using .raw Automatically serializes and deserializes using Schemer methods - - ToDo XXXX make this a subclass of SerderSuber since from a ser des interface - Schemer is duck type of Serder, Then can get rid of the redundant put, add, - pin, get etc definitions - """ - def __init__(self, *pa, **kwa): + def __init__(self, *pa, + klas: Type[ scheming.Schemer] = scheming.Schemer, + **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key - """ - super(SchemerSuber, self).__init__(*pa, **kwa) - - def put(self, keys: Union[str, Iterable], val: scheming.Schemer): - """ - Puts val at key made from keys. Does not overwrite - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Schemer): instance - - Returns: - result (bool): True If successful, False otherwise, such as key - already in database. - """ - return (self.db.putVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) - - def pin(self, keys: Union[str, Iterable], val: scheming.Schemer): - """ - Pins (sets) val at key made from keys. Overwrites. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Schemer): instance - - Returns: - result (bool): True If successful. False otherwise. - """ - return (self.db.setVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) - - def get(self, keys: Union[str, Iterable]): - """ - Gets Serder at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - Schemer: - None: if no entry at keys - - Usage: - Use walrus operator to catch and raise missing entry - if (srder := mydb.get(keys)) is None: - raise ExceptionHere - use srdr here - - """ - val = self.db.getVal(db=self.sdb, key=self._tokey(keys)) - return scheming.Schemer(raw=bytes(val)) if val is not None else None - - def rem(self, keys: Union[str, Iterable]): - """ - Removes entry at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - """ - return self.db.delVal(db=self.sdb, key=self._tokey(keys)) - - def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", - *, topive=False): - """ - Returns: - iterator (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb - - Parameters: - keys (str | bytes | memoryview | Iterable): tuple of bytes or - strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. - topive (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep - character. - False means treat as full branch in key space. Resultant key - does not end in .sep character. - When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[scheming.Schemer]): Class reference to ducktyped subclass + of Serder + Overridden Parameters: + klas (Type[scheming.Schemer]): Class reference to ducktyped subclass + of Serder intercepts passed in klas and forces it to Schemer """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, - top=self._tokey(keys, topive=topive)): - yield self._tokeys(iokey), scheming.Schemer(raw=bytes(val)) + super(SchemerSuber, self).__init__(*pa, klas=scheming.Schemer, **kwa) class DupSuber(SuberBase): diff --git a/tests/core/test_scheming.py b/tests/core/test_scheming.py index a79bac99..09c4df8e 100644 --- a/tests/core/test_scheming.py +++ b/tests/core/test_scheming.py @@ -172,6 +172,7 @@ def test_json_schema_dict(): sce = Schemer(sed=sed, code=MtrDex.Blake3_256) said = 'ENQKl3r1Z6HiLXOD-050aVvKziCWJtXWg3vY2FWUGSxG' assert sce.said == said + payload = b'{"a": {"b": 123, "c": "2018-11-13T20:20:39+00:00"}}' assert sce.verify(payload) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index c9db9749..dc236ebc 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -12,7 +12,7 @@ from keri import help from keri import core -from keri.core import coring, eventing, serdering, indexing +from keri.core import coring, eventing, serdering, indexing, scheming from keri.db import dbing, subing from keri.app import keeping @@ -1468,6 +1468,102 @@ def test_serder_ioset_suber(): assert not db.opened +def test_schemer_suber(): + """ + Test SchemerSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + scmber = subing.SchemerSuber(db=db, subkey='bags.') + assert isinstance(scmber, subing.SchemerSuber) + assert not scmber.sdb.flags()["dupsort"] + assert scmber.klas == scheming.Schemer + + pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + + raw0 = (b'{"$id":"EMRvS7lGxc1eDleXBkvSHkFs8vUrslRcla6UXOJdcczw","$schema":"http://json' + b'-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"str' + b'ing"},"b":{"type":"number"},"c":{"type":"string","format":"date-time"}}}') + + scmr0 = scheming.Schemer(raw=raw0) + + keys = (pre, scmr0.said) + scmber.put(keys=keys, val=scmr0) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + assert scmber.rem(keys) + actual = scmber.get(keys=keys) + assert actual is None + + scmber.put(keys=keys, val=scmr0) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + raw1 = (b'{"$id":"ENQKl3r1Z6HiLXOD-050aVvKziCWJtXWg3vY2FWUGSxG","$schema":"http://json' + b'-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"obj' + b'ect","properties":{"b":{"type":"number"},"c":{"type":"string","format":"date' + b'-time"}}}}}') + scmr1 = scheming.Schemer(raw=raw1) + + result = scmber.put(keys=keys, val=scmr1) + assert not result + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + result = scmber.pin(keys=keys, val=scmr1) + assert result + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr1.said + + # test with keys as string not tuple + keys = "{}.{}".format(pre, scmr1.said) + + scmber.put(keys=keys, val=scmr1) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr1.said + + assert scmber.rem(keys) + actual = scmber.get(keys=keys) + assert actual is None + + # test missing entry at keys + badkey = "badkey" + actual = scmber.get(badkey) + assert actual is None + + # test iteritems + scmber = subing.SchemerSuber(db=db, subkey='pugs.') + assert isinstance(scmber, subing.SchemerSuber) + scmber.put(keys=("a","1"), val=scmr0) + scmber.put(keys=("a","2"), val=scmr1) + + items = [(keys, srdr.said) for keys, srdr in scmber.getItemIter()] + assert items == [(('a', '1'), scmr0.said), + (('a', '2'), scmr1.said)] + + assert scmber.put(keys=("b","1"), val=scmr0) + assert scmber.put(keys=("b","2"), val=scmr1) + assert scmber.put(keys=("bc","1"), val=scmr0) + + topkeys = ("b", "") # append empty str to force trailing .sep + items = [(keys, srdr.said) for keys, srdr in scmber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), scmr0.said), + (('b', '2'), scmr1.said)] + + assert not os.path.exists(db.path) + assert not db.opened + + def test_cesr_suber(): """ Test CesrSuber LMDBer sub database class @@ -2533,5 +2629,6 @@ def test_crypt_signer_suber(): test_cesr_dup_suber() test_serder_suber() test_serder_ioset_suber() + test_schemer_suber() test_signer_suber() test_crypt_signer_suber() From 6a8ec7f594a4411515134104ef8ada89aebea827 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 10:45:20 -0600 Subject: [PATCH 53/78] Added test for subclass of Schemer to SchemerSuber init --- src/keri/db/subing.py | 4 +++- tests/db/test_subing.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 1c68056e..1b447d32 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -1498,7 +1498,9 @@ def __init__(self, *pa, klas (Type[scheming.Schemer]): Class reference to ducktyped subclass of Serder intercepts passed in klas and forces it to Schemer """ - super(SchemerSuber, self).__init__(*pa, klas=scheming.Schemer, **kwa) + if not issubclass(klas, scheming.Schemer): + raise TypeError(f"Invalid {klas=}, not subclass of {scheming.Schemer}.") + super(SchemerSuber, self).__init__(*pa, klas=klas, **kwa) class DupSuber(SuberBase): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index dc236ebc..c7f62fdf 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -1351,6 +1351,7 @@ def test_serder_suber(): assert items == [(('b', '1'), srdr0.said), (('b', '2'), srdr1.said)] + assert not os.path.exists(db.path) assert not db.opened @@ -1560,6 +1561,10 @@ def test_schemer_suber(): assert items == [(('b', '1'), scmr0.said), (('b', '2'), scmr1.said)] + with pytest.raises(TypeError): + scmber = subing.SchemerSuber(db=db, subkey='bags.', klas=coring.Matter) + + assert not os.path.exists(db.path) assert not db.opened From 49c9393f4bf0c8759ed91b99202e5b5bd43c752f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 10:52:59 -0600 Subject: [PATCH 54/78] some clean up --- tests/db/test_subing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index c7f62fdf..f11e229c 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -1351,7 +1351,6 @@ def test_serder_suber(): assert items == [(('b', '1'), srdr0.said), (('b', '2'), srdr1.said)] - assert not os.path.exists(db.path) assert not db.opened From 4e602efa1aa3ee526c516ef4868241bf8e68aad7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 13:55:22 -0600 Subject: [PATCH 55/78] added B64Suber subclass with unit test B64SuberBase can be mixed with IoSetSuber or OnSuber or combinations to create subclasses where the values are separated strings of Base64 charcters when the values are just groups of qb64 identifiers that don't need to be serialized and reserialized from Matter subclasses as would be the case with CatCesrSuber --- src/keri/db/subing.py | 175 +++++++++++++++++++++++++++++++++++++--- tests/db/test_subing.py | 92 ++++++++++++++++++++- 2 files changed, 251 insertions(+), 16 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 1b447d32..75b94abe 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -73,7 +73,7 @@ from collections.abc import Iterable, Iterator from .. import help -from ..help.helping import nonStringIterable +from ..help.helping import nonStringIterable, Reb64 from .. import core from ..core import coring, scheming, serdering from . import dbing @@ -133,8 +133,8 @@ def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], by partial keys by appending separator to end of partial key Returns: - key (bytes): each element of keys is joined by .sep. If top then last - char of key is also .sep + key (bytes): each element of keys is joined by .sep. If topive then + last char of key is .sep Parameters: keys (str | bytes | memoryview | Iterable[str | bytes]): db key or @@ -145,7 +145,7 @@ def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], False means treat as full branch in key space. Resultant key does not end in .sep character. When last item in keys is empty str then will treat as - partial ending in sep regardless of top value + partial ending in sep regardless of topive value """ if hasattr(keys, "encode"): # str @@ -190,11 +190,11 @@ def _ser(self, val: str | bytes | memoryview): return (val.encode("utf-8") if hasattr(val, "encode") else val) - def _des(self, val: str | bytes | memoryview): + def _des(self, val: bytes | memoryview): """ Deserialize val to str Parameters: - val (str | bytes | memoryview): decodable as str + val (bytes | memoryview): decodable as str """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes @@ -536,7 +536,8 @@ def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): class OnSuber(OnSuberBase, Suber): """ - Subclass of Suber that adds methods for keys with ordinal numbered suffixes. + Subclass of OnSuberBase andSuber that adds methods for keys with ordinal + numbered suffixes. Each key consistes of pre joined with .sep to ordinal suffix Assumes dupsort==False @@ -559,6 +560,154 @@ def __init__(self, *pa, **kwa): super(OnSuber, self).__init__(*pa, **kwa) +class B64SuberBase(SuberBase): + """ + Base Class whose values are Iterables of Base64 str or bytes that are stored + in db as .sep joined Base64 bytes. Separator character must not be valid + Base64 character so the split will work unambiguously. + + Automatically joins and splits along separator to Iterable (tuple) of Base64 + + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + Must not be Base64 character. + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + + """ + super(B64SuberBase, self).__init__(*pa, **kwa) + if Reb64.match(self.sep.encode()): + raise ValueError("Invalid sep={self.sep}, must not be Base64 char.") + + + def _toval(self, vals: str|bytes|memoryview|Iterable[str|bytes|memoryview]): + """ + Converts vals to val bytes with proper separators and returns val bytes. + If vals is already str or bytes or memoryview then returns val bytes. + Else If vals is iterable (non-str) of strs or bytes or memoryview then + joins with .sep and converts to val bytes and returns. + + Returns: + val (bytes): each element of vals is joined by .sep. + + Parameters: + vals (str | bytes | memoryview | Iterable[str | bytes]): db val or + Iterable of (str | bytes | memoryview) to form val. + Note, join of bytes sep works with memoryview. + + """ + if hasattr(vals, "encode"): # str + val = vals.encode("utf-8") + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + if isinstance(vals, memoryview): # memoryview of bytes + val = bytes(vals) # return bytes + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + elif hasattr(vals, "decode"): # bytes + val = vals + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + return (self.sep.join(val.decode() if hasattr(val, "decode") else val + for val in vals).encode("utf-8")) + + + def _tovals(self, val: bytes | memoryview): + """ + Converts val bytes to vals tuple of strs by decoding and then splitting + at separator .sep. + + Returns: + vals (tuple[str]): makes tuple by splitting val at .sep + + Parameters: + val (bytes | memoryview): db Base64 val. + + """ + if isinstance(val, memoryview): # memoryview of bytes + val = bytes(val) + if hasattr(val, "decode"): # bytes + val = val.decode("utf-8") # convert to str + return tuple(val.split(self.sep)) + + + def _ser(self, val: Union[Iterable, str, bytes]): + """ + Serialize val to bytes to store in db + When val is Iterable then joins each elements with .sep returns val bytes + + Returns: + val (bytes): .sep join of each Base64 bytes in val + + Parameters: + val (Union[Iterable, bytes]): of Base64 bytes + + """ + if not nonStringIterable(val): # not iterable + val = (val, ) # make iterable + return (self._toval(val)) + + + def _des(self, val: memoryview | bytes): + """ + Converts val bytes to vals tuple of subclass instances by deserializing + .qb64b concatenation in order of each instance in .klas + + Returns: + vals (tuple): subclass instances + + Parameters: + val (Union[bytes, memoryview]): of concatenation of .qb64b + + """ + return self._tovals(val) + + +class B64Suber(B64SuberBase, Suber): + """ + Subclass of B64SuberBase and Suber that serializes and deserializes values + as .sep joined strings of Base64 components. + + .sep must not be Base64 character. + + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + Must not be Base64 character. + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(B64Suber, self).__init__(*pa, **kwa) @@ -601,11 +750,11 @@ def _ser(self, val: coring.Matter): return val.qb64b - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: memoryview | bytes): """ Deserialize val to str Parameters: - val (Union[str, memoryview, bytes]): convertable to coring.matter + val (memoryview | bytes): convertable to coring.matter """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes @@ -710,13 +859,13 @@ def __init__(self, *pa, klas: Iterable = None, **kwa): def _ser(self, val: Union[Iterable, coring.Matter]): """ Serialize val to bytes to store in db - Concatenates .qb64b of each instance in objs and returns val bytes + Concatenates .qb64b of each instance in val and returns val bytes Returns: - val (bytes): concatenation of .qb64b of each object instance in vals + cat (bytes): concatenation of .qb64b of each object instance in vals Parameters: - subs (Union[Iterable, coring.Matter]): of subclass instances. + val (Union[Iterable, coring.Matter]): of subclass instances. """ if not nonStringIterable(val): # not iterable @@ -724,7 +873,7 @@ def _ser(self, val: Union[Iterable, coring.Matter]): return (b''.join(obj.qb64b for obj in val)) - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: memoryview | bytes | bytearray): """ Converts val bytes to vals tuple of subclass instances by deserializing .qb64b concatenation in order of each instance in .klas diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index f11e229c..e84d5f5b 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -356,6 +356,91 @@ def test_on_suber(): assert not db.opened +def test_B64_suber(): + """ + Test B64Suber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + # Test Single klas + buber = subing.B64Suber(db=db, subkey='bags.') # default klas is [Matter] + assert isinstance(buber, subing.B64Suber) + assert not buber.sdb.flags()["dupsort"] + + vals0 = ("alpha", "beta") + vals1 = ("gamma", ) + + keys0 = ("cat", "dog") + assert buber.put(keys=keys0, val=vals0) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + + assert buber.rem(keys0) + assert not buber.get(keys=keys0) + + assert buber.put(keys=keys0, val=vals0) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + + assert not buber.put(keys=keys0, val=vals1) + + assert buber.pin(keys=keys0, val=vals1) + actuals = buber.get(keys=keys0) + assert actuals == vals1 + + assert buber.rem(keys0) + assert not buber.get(keys=keys0) + + # test with vals as non Iterable or put but Iterable on get + assert buber.put(keys=keys0, val="gamma") + actuals = buber.get(keys=keys0) + assert actuals == vals1 + + assert buber.rem(keys0) + assert buber.put(keys=keys0, val="alpha.beta") + actuals = buber.get(keys=keys0) + assert actuals == vals0 + assert buber.rem(keys0) + + # test with keys as string not tuple + keys1 = "{}.{}".format("bird", "fish") + + assert buber.put(keys=keys1, val=vals1) + actuals = buber.get(keys=keys1) + assert actuals == vals1 + + assert buber.rem(keys1) + assert not buber.get(keys=keys1) + + # test missing entry at keys + badkey = "badkey" + assert not buber.get(badkey) + + # test iteritems + assert buber.put(keys0, vals0) + assert buber.put(keys1, vals1) + + items = [ items for items in buber.getItemIter()] + assert items == [(('bird', 'fish'), ('gamma',)), (('cat', 'dog'), ('alpha', 'beta'))] + + buber.put(keys=("b","1"), val=vals0) + buber.put(keys=("b","2"), val=vals1) + buber.put(keys=("c","1"), val=vals0) + buber.put(keys=("c","2"), val=vals1) + + topkeys = ("b","") # last element empty to force trailing separator + items = [items for items in buber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), ('alpha', 'beta')), + (('b', '2'), ('gamma',))] + + assert not os.path.exists(db.path) + assert not db.opened + """Done Test""" + def test_dup_suber(): """ @@ -1818,9 +1903,9 @@ def test_cesr_on_suber(): -def test_cat_suber(): +def test_cat_cesr_suber(): """ - Test CatSuber LMDBer sub database class + Test CatCesrSuber LMDBer sub database class """ with dbing.openLMDB() as db: @@ -2621,11 +2706,12 @@ def test_crypt_signer_suber(): if __name__ == "__main__": test_suber() test_on_suber() + test_B64_suber() test_dup_suber() test_iodup_suber() test_on_iodup_suber() test_ioset_suber() - test_cat_suber() + test_cat_cesr_suber() test_cesr_suber() test_cesr_on_suber() test_cesr_ioset_suber() From d78054d976788b0433766a4cda46c029ece712f0 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 5 Sep 2024 17:05:10 -0600 Subject: [PATCH 56/78] some refactoring and preliminary support for partial delegation escrow --- src/keri/core/eventing.py | 504 ++++++++++++++++++++++++++------------ src/keri/db/basing.py | 16 +- src/keri/db/subing.py | 2 + tests/db/test_basing.py | 3 +- 4 files changed, 354 insertions(+), 171 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index d07caf2b..f8f1fd5c 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -3111,39 +3111,61 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): "event = %s\n", serder.ked) - def escrowPACouple(self, serder, seqner, saider, local=True): + def escrowPWEvent(self, serder, wigers, sigers=None, + seqner=None, saider=None, local=True): """ - Update associated logs for escrow of partially authenticated issued event. - Assuming signatures are provided elsewhere. Partial authentication results - from either a partially signed event or a fully signed delegated event - but whose delegation is not yet verified. - - Escrow allows escrow processor to retrieve serder from key and source - couple from val in order to to re-verify authentication status. Sigs - are escrowed elsewhere. + Update associated logs for escrow of partially witnessed event Parameters: - serder is SerderKERI instance of delegated or issued event + serder is SerderKERI instance of event + wigers is list of Siger instance of indexed witness sigs + sigers is optional list of Siger instances of indexed controller sigs seqner is Seqner instance of sn of seal source event of delegator/issuer - saider is Saider instance of said of delegator/issuer + saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). Event validation logic is a function of local or remote + """ - local = True if local else False # ignored since not escrowing serder here + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) - #couple = seqner.qb64b + saider.qb64b - #self.db.putUde(dgkey, couple) # idempotent - self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent - logger.debug("Kever state: Escrowed source couple for partially signed " - "or delegated event = %s\n", serder.ked) + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if sigers: + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if seqner and saider: + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + self.db.putEvt(dgkey, serder.raw) + # update event source + if (esr := self.db.esrs.get(keys=dgkey)): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) - def escrowPWEvent(self, serder, wigers, sigers=None, + logger.debug("Kever state: Escrowed partially witnessed " + "event = %s\n", serder.ked) + return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) + + + def escrowPDEvent(self, serder, sigers=None, wigers=None, seqner=None, saider=None, local=True): """ - Update associated logs for escrow of partially witnessed event + Update associated logs for escrow of partially delegated or otherwise + authorized issued event. + Assumes controller signatures, sigs, and witness signatures, wigs are + provided elsewhere. Partial authentication occurs once an event is + fully signed and witnessed but the authorizing (delegating) source + seal in the authorizer's (delegator's) key has not yet been verified. + + Escrow allows escrow processor to retrieve serder from key and source + couple from val in order to to re-verify authentication status. Parameters: serder is SerderKERI instance of event @@ -3160,17 +3182,20 @@ def escrowPWEvent(self, serder, wigers, sigers=None, local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent - if wigers: - self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) - if sigers: + + if sigers: # idempotent self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) - if seqner and saider: - #couple = seqner.qb64b + saider.qb64b - #self.db.putUde(dgkey, couple) + if wigers: # idempotent + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: # idempotent self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + logger.debug(f"Kever state: Escrowed source couple sn={seqner.sn}, " + f"said={saider.said} for partially delegated/authorized" + f" event said={serder.said}.") - self.db.putEvt(dgkey, serder.raw) - # update event source + self.db.putEvt(dgkey, serder.raw) # idempotent + + # update event source local or remote if (esr := self.db.esrs.get(keys=dgkey)): # preexisting esr if local and not esr.local: # local overwrites prexisting remote esr.local = local @@ -3180,9 +3205,38 @@ def escrowPWEvent(self, serder, wigers, sigers=None, esr = basing.EventSourceRecord(local=local) self.db.esrs.put(keys=dgkey, val=esr) - logger.debug("Kever state: Escrowed partially witnessed " - "event = %s\n", serder.ked) - return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) + logger.debug(f"Kever state: Escrowed partially delegated event=\n" + f"{serder.ked}\n.") + return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) + + + def escrowPACouple(self, serder, seqner, saider, local=True): + """ + Update associated logs for escrow of partially authenticated issued event. + Assuming signatures are provided elsewhere. Partial authentication results + from either a partially signed event or a fully signed delegated event + but whose delegation is not yet verified. + + Escrow allows escrow processor to retrieve serder from key and source + couple from val in order to to re-verify authentication status. Sigs + are escrowed elsewhere. + + Parameters: + serder is SerderKERI instance of delegated or issued event + seqner is Seqner instance of sn of seal source event of delegator/issuer + saider is Saider instance of said of delegator/issuer + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + """ + local = True if local else False # ignored since not escrowing serder here + dgkey = dgKey(serder.preb, serder.saidb) + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + logger.debug("Kever state: Escrowed source couple for partially signed " + "or delegated event = %s\n", serder.ked) + + def state(self): @@ -5347,149 +5401,150 @@ def processEscrowPartialSigs(self): If successful then remove from escrow table """ - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.db.getPseItemIter(key=key): - eserder = None - try: - pre, sn = splitSnKey(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) - if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error - # no local sourde so raise ValidationError which unescrows below - raise ValidationError("Missing escrowed event source " - "at dig = {}.".format(bytes(edig))) + #key = ekey = b'' # both start same. when not same means escrows found + #while True: # break when done + for ekey, edig in self.db.getPseItemIter(key=b''): + eserder = None + try: + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) - # check date if expired then remove escrow. - dtb = self.db.getDts(dgkey) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event datetime" - " at dig = %s", bytes(edig)) + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) - raise ValidationError("Missing escrowed event datetime " - "at dig = {}.".format(bytes(edig))) + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): - # escrow stale so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Stale event escrow " - " at dig = %s", bytes(edig)) + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) - raise ValidationError("Stale event escrow " - "at dig = {}.".format(bytes(edig))) + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) - # get the escrowed event using edig - eraw = self.db.getEvt(dgkey) - if eraw is None: - # no event so so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event at." - "dig = %s", bytes(edig)) + # get the escrowed event using edig + eraw = self.db.getEvt(dgkey) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(edig))) + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - # get sigs and attach - sigs = self.db.getSigs(dgkey) - if not sigs: # otherwise its a list of sigs - # no sigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event sigs at." - "dig = %s", bytes(edig)) + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + # get sigs and attach + sigs = self.db.getSigs(dgkey) + if not sigs: # otherwise its a list of sigs + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) - raise ValidationError("Missing escrowed evt sigs at " - "dig = {}.".format(bytes(edig))) - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - if not wigs: # empty list wigs witness sigs not wits - # wigs maybe empty if not wits or if wits while waiting - # for first witness signature - # which may not arrive until some time after event is fully signed - # so just log for debugging but do not unescrow by raising - # ValidationError - logger.debug("Kevery unescrow wigs: No event wigs yet at." - "dig = %s", bytes(edig)) - - # seal source (delegator issuer if any) - delseqner = delsaider = None - if (couple := self.db.udes.get(keys=dgkey)): - delseqner, delsaider = couple - #couple = self.db.getUde(dgkey) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + if not wigs: # empty list wigs witness sigs not wits + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=dgkey)): + delseqner, delsaider = couple + #couple = self.db.getUde(dgkey) + #if couple is not None: + #delseqner, delsaider = deSourceCouple(couple) + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre + else: + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + #couple = delseqner.qb64b + delsaider.qb64b + #self.db.putUde(dgkey, couple) + # idempotent + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) - # If process does NOT validate sigs or delegation seal (when delegated), - # but there is still one valid signature then process will - # attempt to re-escrow and then raise MissingSignatureError - # or MissingDelegationSealError (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pse escrow is called by - # Kever.self.escrowPSEvent Which calls - # self.db.addPse(snKey(pre, sn), serder.digb) - # which in turn will not enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pse db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, local=esr.local) - except (MissingSignatureError, MissingDelegationError) as ex: - # still waiting on missing sigs or missing seal to validate - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrow failed: %s", ex.args[0]) + # If process does NOT validate sigs or delegation seal (when delegated), + # but there is still one valid signature then process will + # attempt to re-escrow and then raise MissingSignatureError + # or MissingDelegationSealError (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pse escrow is called by + # Kever.self.escrowPSEvent Which calls + # self.db.addPse(snKey(pre, sn), serder.digb) + # which in turn will not enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pse db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any + except (MissingSignatureError, MissingDelegationError) as ex: + # still waiting on missing sigs or missing seal to validate + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) - if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - self.cues.push(dict(kin="psUnescrow", serder=eserder)) + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed: %s", ex.args[0]) - else: - logger.error("Kevery unescrowed: %s", ex.args[0]) + if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + self.cues.push(dict(kin="psUnescrow", serder=eserder)) - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) - if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - self.cues.push(dict(kin="psUnescrow", serder=eserder)) + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.delUde(dgkey) # remove escrow if any + self.db.udes.rem(keys=dgkey) # remove escrow if any - logger.info("Kevery unescrow succeeded in valid event: " - "event=%s", eserder.said) - logger.debug(f"event=\n{eserder.pretty()}\n") + if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + self.cues.push(dict(kin="psUnescrow", serder=eserder)) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") + + #if ekey == key: # still same so no escrows found on last while iteration + #break + #key = ekey # setup next while iteration, with key after ekey def processEscrowPartialWigs(self): """ @@ -5527,10 +5582,7 @@ def processEscrowPartialWigs(self): Process event as if it came in over the wire If successful then remove from escrow table """ - - #key = ekey = b'' # both start same. when not same means escrows found - #while True: # break when done - for ekey, edig in self.db.getPweItemIter(key=b''): #self.db.getPweItemsNextIter(key=key) + for ekey, edig in self.db.getPweItemIter(key=b''): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) @@ -5645,6 +5697,7 @@ def processEscrowPartialWigs(self): except (MissingWitnessSignatureError, MissingDelegationError) as ex: # still waiting on missing witness sigs or delegation + # processEvent idempotently reescrowed if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) @@ -5669,9 +5722,6 @@ def processEscrowPartialWigs(self): "event=%s", eserder.said) logger.debug(f"event=\n{eserder.pretty()}\n") - #if ekey == key: # still same so no escrows found on last while iteration - #break - #key = ekey # setup next while iteration, with key after ekey def processEscrowPartialDels(self): """ @@ -5700,7 +5750,135 @@ def processEscrowPartialDels(self): If successful then remove from escrow table """ - pass + for ekey, edig in self.db.pdes.getItemIter(key=b''): + try: + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local sourde so raise ValidationError which unescrows below + raise ValidationError("Missing escrowed event source " + "at dig = {}.".format(bytes(edig))) + + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event datetime" + " at dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed event datetime " + "at dig = {}.".format(bytes(edig))) + + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Stale event escrow " + " at dig = %s", bytes(edig)) + + raise ValidationError("Stale event escrow " + "at dig = {}.".format(bytes(edig))) + + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt at dig = {}." + "".format(bytes(edig))) + + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + + # get sigs + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event sigs at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt sigs at " + "dig = {}.".format(bytes(edig))) + + # get witness signatures (wigs not wits) assumes wont be in this + # escrow if wigs not needed because no wits + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + if not wigs: # empty list + # no wigs so raise ValidationError which unescrows below + logger.info("Kevery unescrow error: Missing event wigs at." + "dig = %s", bytes(edig)) + + raise ValidationError("Missing escrowed evt wigs at " + "dig = {}.".format(bytes(edig))) + + # setup parameters to process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple # provided + elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find + if eserder.pre in self.kevers: + delpre = self.kevers[eserder.pre].delpre + else: + delpre = eserder.ked["di"] + seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if srdr is not None: # found seal in srdr + delseqner = coring.Seqner(sn=srdr.sn) + delsaider = coring.Saider(qb64=srdr.said) + self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, local=esr.local) + + # If process does NOT validate delegation then process will attempt + # to re-escrow and then raise MissingDelegationError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pses escrow is called by + # Kever.self.escrowPDEvent Which calls + # self.db.pdes.add(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrows in db.pdes. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except MissingDelegationError as ex: + # still waiting on missing delegation source seal + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + # removes one event escrow at key val + self.db.pdes.rem(keys=(pre, sn), val=edig) # event idx escrow + self.db.udes.rem(keys=dgkey) # remove source seal escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + # removes one event escrow at key val + self.db.pdes.rem(keys=(pre, sn), val=edig) # event idx escrow + self.db.udes.rem(keys=dgkey) # remove source seal escrow if any + logger.info("Kevery unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug(f"event=\n{eserder.pretty()}\n") def processEscrowUnverWitness(self): diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 69557fed..21980558 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -764,22 +764,23 @@ class Baser(dbing.LMDBer): More than one value per DB key is allowed .pses is named sub DB of partially signed key event escrows - that map sequence numbers to serialized event digests. + that each map pre + sequence number to serialized event digest. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .pwes is named sub DB of partially witnessed key event escrowes - that map sequence numbers to serialized event digests. + that each map pre + sequence number to serialized event digest. + these are for escrows of events with verified signatures but not + yet verified witness reciepts. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed .pdes is named sub DB of partially delegated key event escrows - that map sequence numbers to serialized event digests. This is + that each map pre + sequence number to serialized event digest. This is used in conjunction with .udes which escrows the associated seal source couple. snKey @@ -798,6 +799,8 @@ class Baser(dbing.LMDBer): KEL. DB is keyed by identifier prefix plus digest of key event Only one value per DB key is allowed + Once escrow is accepted then delegation approval source seal couples + go into .aess database of authorizing event source seal couples .uwes is named sub DB of unverified event indexed escrowed couples from witness signers. Witnesses are from witness list of latest establishment @@ -1012,11 +1015,10 @@ def reopen(self, **kwa): self.vrcs = self.env.open_db(key=b'vrcs.', dupsort=True) self.vres = self.env.open_db(key=b'vres.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) - #self.pdes = self.env.open_db(key=b'pdes.') - self.pdes = subing.IoSetSuber(db=self, subkey='pdes.') + self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) + self.pdes = subing.IoDupSuber(db=self, subkey='pdes.') self.udes = subing.CatCesrSuber(db=self, subkey='udes.', klas=(coring.Seqner, coring.Saider)) - self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) self.dels = self.env.open_db(key=b'dels.', dupsort=True) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 75b94abe..dcba2e6a 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -44,6 +44,8 @@ increasing numeric The term 'set' of values means that no value may appear more than once in the set. +Sets support idempotent adds and puts to db. This means one can add or put the same +(key, val) pair multiple times and not change the db. DupSuber provides set of lexicographic ordered values at each key. Each value has a limited size (key + value <= 511 byes). The set is performant. Good for indices. diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index f8971f7b..b4de4832 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -784,7 +784,8 @@ def test_baser(): assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") # test .pdes SerderIoSetSuber methods - assert isinstance(db.pdes, subing.IoSetSuber) + assert isinstance(db.pdes, subing.IoDupSuber) + # test .udes CatCesrSuber sub db methods assert isinstance(db.udes, subing.CatCesrSuber) From 7ea9290c9106658e19f94168dd92f0c351d5302e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 08:35:05 -0600 Subject: [PATCH 57/78] fixed bug --- src/keri/db/subing.py | 10 ++++++++-- tests/db/test_subing.py | 8 +++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index dcba2e6a..214cf5ed 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -627,8 +627,14 @@ def _toval(self, vals: str|bytes|memoryview|Iterable[str|bytes|memoryview]): if not (Reb64.match(val)): raise ValueError(f"Non Base64 {val=}.") return val - return (self.sep.join(val.decode() if hasattr(val, "decode") else val - for val in vals).encode("utf-8")) + vals = tuple(v.encode() if hasattr(v, "encode") else v for v in vals) # make bytes + for val in vals: + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return (self.sep.encode().join(vals)) + + #return (self.sep.join(val.decode() if hasattr(val, "decode") else val + #for val in vals).encode("utf-8")) def _tovals(self, val: bytes | memoryview): diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index e84d5f5b..6b97d3c5 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -401,7 +401,13 @@ def test_B64_suber(): assert actuals == vals1 assert buber.rem(keys0) - assert buber.put(keys=keys0, val="alpha.beta") + + # test val as already joined string + with pytest.raises(ValueError): + assert buber.put(keys=keys0, val="alpha.beta") + + # test bytes for val + assert buber.put(keys=keys0, val=(b"alpha", b"beta")) actuals = buber.get(keys=keys0) assert actuals == vals0 assert buber.rem(keys0) From d246047d804afde1961f35d24411eb9b4493f4a1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 10:10:43 -0600 Subject: [PATCH 58/78] updated Kever.escrows superseded EscrowPACouple so not needed anymore. --- src/keri/core/eventing.py | 55 +++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index f8f1fd5c..a44d9e8a 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2253,9 +2253,10 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # escrow if not fully signed vs signing threshold if not tholder.satisfy(indices): # at least one but not enough - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - if delseqner and delsaider: - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + #if delseqner and delsaider: + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" f" on sigs for {[siger.qb64 for siger in sigers]}" f" for evt = {serder.ked}.") @@ -2266,9 +2267,10 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # prior next threshold in .ntholder and digers in .ndigers ondices = self.exposeds(sigers) if not self.ntholder.satisfy(indices=ondices): - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - if delseqner and delsaider: # save in case not attached later - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider,local=local) + #if delseqner and delsaider: # save in case not attached later + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying prior nsith=" f"{self.ntholder.sith} with exposed " f"sigs= {[siger.qb64 for siger in sigers]}" @@ -2628,9 +2630,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # The processPSEvent should also cue a trigger to get KEL # of delegator if still missing when processing escrow later. self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, - local=local) - if delseqner and delsaider: # save in case not attached later - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + seqner=delseqner, saider=delsaider, local=local) + #if delseqner and delsaider: # save in case not attached later + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"Missing KEL of delegator " f"{delpre} of evt = {serder.ked}.") @@ -2660,7 +2662,8 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, f"event = {serder.ked}.") else: # not local delegator so escrow PSEvent - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) # since delseqner or delsaider is None there is no PACouple to escrow here #if delseqner and delsaider: # save in case not attached later #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) @@ -2687,9 +2690,10 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False #sn = validateSN(sn=serder.snh, inceptive=inceptive) sn = Number(num=serder.sn).validate(inceptive=inceptive).sn - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) - if delseqner and delsaider: # save in case not attached later - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + #if delseqner and delsaider: # save in case not attached later + #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError("No delegating event from {} at {} for " "evt = {}.".format(delpre, delsaider.qb64, @@ -3073,7 +3077,8 @@ def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): logger.debug("Kever state: escrowed delegable event=\n%s\n", json.dumps(serder.ked, indent=1)) - def escrowPSEvent(self, serder, sigers, wigers=None, local=True): + def escrowPSEvent(self, serder, *, sigers=None, wigers=None, + seqner=None, saider=None, local=True): """ Update associated logs for escrow of partially signed event or fully signed delegated event but not yet verified delegation. @@ -3082,6 +3087,8 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): serder is SerderKERI instance of event sigers is list of Siger instances of indexed controller sigs wigers is optional list of Siger instance of indexed witness sigs + seqner is Seqner instance of sn of seal source event of delegator/issuer + saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -3090,9 +3097,12 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent - self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if sigers: + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.putEvt(dgkey, serder.raw) # update event source @@ -3111,15 +3121,15 @@ def escrowPSEvent(self, serder, sigers, wigers=None, local=True): "event = %s\n", serder.ked) - def escrowPWEvent(self, serder, wigers, sigers=None, + def escrowPWEvent(self, serder, *, sigers=None, wigers=None, seqner=None, saider=None, local=True): """ Update associated logs for escrow of partially witnessed event Parameters: serder is SerderKERI instance of event - wigers is list of Siger instance of indexed witness sigs sigers is optional list of Siger instances of indexed controller sigs + wigers is list of Siger instance of indexed witness sigs seqner is Seqner instance of sn of seal source event of delegator/issuer saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic @@ -3131,10 +3141,11 @@ def escrowPWEvent(self, serder, wigers, sigers=None, local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent - if wigers: - self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if sigers: self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent @@ -3154,7 +3165,7 @@ def escrowPWEvent(self, serder, wigers, sigers=None, return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) - def escrowPDEvent(self, serder, sigers=None, wigers=None, + def escrowPDEvent(self, serder, *, sigers=None, wigers=None, seqner=None, saider=None, local=True): """ Update associated logs for escrow of partially delegated or otherwise @@ -3169,8 +3180,8 @@ def escrowPDEvent(self, serder, sigers=None, wigers=None, Parameters: serder is SerderKERI instance of event - wigers is list of Siger instance of indexed witness sigs sigers is optional list of Siger instances of indexed controller sigs + wigers is list of Siger instance of indexed witness sigs seqner is Seqner instance of sn of seal source event of delegator/issuer saider is Diger instance of digest of delegator/issuer local (bool): event source for validation logic @@ -3209,7 +3220,7 @@ def escrowPDEvent(self, serder, sigers=None, wigers=None, f"{serder.ked}\n.") return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) - + # not used anymore deprecate? def escrowPACouple(self, serder, seqner, saider, local=True): """ Update associated logs for escrow of partially authenticated issued event. From 37380e1f81057844979e885ebfaf950ba48aa889 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 12:59:06 -0600 Subject: [PATCH 59/78] enabled ProcessPDEscrow --- src/keri/core/eventing.py | 233 ++++++++++++++++++++++---------------- 1 file changed, 133 insertions(+), 100 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index a44d9e8a..7a92731d 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1561,16 +1561,17 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits, - local=local, - delseqner=delseqner, - delsaider=delsaider) + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel( + serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + local=local, + delseqner=delseqner, + delsaider=delsaider) self.delpre = delpre # may be None self.delegated = True if self.delpre else False @@ -1921,7 +1922,8 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # returned sigers and wigers are verified signatures # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel( + serder=serder, sigers=sigers, verfers=serder.verfers, tholder=tholder, @@ -1988,7 +1990,8 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, wigers, delpre = self.valSigsWigsDel(serder=serder, + sigers, wigers, delpre, _, _ = self.valSigsWigsDel( + serder=serder, sigers=sigers, verfers=self.verfers, tholder=self.tholder, @@ -2169,8 +2172,9 @@ def deriveBacks(self, serder): def valSigsWigsDel(self, serder, sigers, verfers, tholder, - wigers, toader, wits, local=True, - delseqner=None, delsaider=None): + wigers, toader, wits, *, + delseqner=None, delsaider=None, eager=False, + local=True): """ Returns triple (sigers, wigers, delegator) where: sigers is unique validated signature verified members of inputed sigers @@ -2195,14 +2199,19 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, toader (Number): instance of backer witness threshold wits (list): of qb64 non-transferable prefixes of witnesses used to derive werfers for wigers - local (bool): event source for validation logic - True means event source is local (protected). - False means event source is remote (unprotected). - Event validation logic is a function of local or remote delseqner (Seqner | None): instance of delegating event sequence number. If this event is not delegated then seqner is ignored delsaider (Saider | None): instance of of delegating event said. If this event is not delegated then saider is ignored + eager (bool): True means try harder to find delegation seals by + walking KEL of delegator. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing source seal couples + if any, either attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ if len(verfers) < tholder.size: @@ -2255,8 +2264,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, if not tholder.satisfy(indices): # at least one but not enough self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - #if delseqner and delsaider: - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" f" on sigs for {[siger.qb64 for siger in sigers]}" f" for evt = {serder.ked}.") @@ -2269,8 +2276,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, if not self.ntholder.satisfy(indices=ondices): self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider,local=local) - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingSignatureError(f"Failure satisfying prior nsith=" f"{self.ntholder.sith} with exposed " f"sigs= {[siger.qb64 for siger in sigers]}" @@ -2297,41 +2302,50 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f" delegated by {delpre} of" f"event = {serder.ked}.") - # short circuit witness validation when either locallyOwned or locallyWitnessed - # otherwise must validate fully witnessed - if not (self.locallyOwned() or self.locallyMembered() or self.locallyWitnessed(wits=wits)): - if wits: # is witnessed - if toader.num < 1 or toader.num > len(wits): # out of bounds toad - raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - else: # not witnessed - if toader.num != 0: # invalid toad - raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - - if len(windices) < toader.num: # not fully witnessed yet - if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, - seqner=delseqner, saider=delsaider, - local=local): - # cue to query for witness receipts - self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) - raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " - f"on witness sigs=" - f"{[siger.qb64 for siger in wigers]} " - f"for event={serder.ked}.") - - if delpre: - #if not (delseqner and delsaider): - #seal = dict(i=serder.ked["i"], s=serder.snh, d=serder.said) - #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - #if srdr is not None: - #delseqner = coring.Seqner(sn=srdr.sn) - #delsaider = coring.Saider(qb64=srdr.said) - - self.validateDelegation(serder, sigers=sigers, - wigers=wigers, wits=wits, - local=local, delpre=delpre, - ) # delseqner=delseqner, delsaider=delsaider - - return (sigers, wigers, delpre) # delseqner, delsaider + # this point the sigers have been verified and the wigers have been verified + # even if locallyOwned or locallyMembered or locallyWitnessed. + # But the toad has not yet been verified. + + if not wits: + if toader.num != 0: # bad toad non-zero and not witnessed bad event + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + else: # wits so may need to validate toad + # The toad is only verified against the wigers as fully witnessed if + # not (locallyOwned or locallyMembered or locallyWitnessed) + if (not (self.locallyOwned() or self.locallyMembered() or + self.locallyWitnessed(wits=wits))): + if wits: # is witnessed + if toader.num < 1 or toader.num > len(wits): # out of bounds toad + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + else: # not witnessed + if toader.num != 0: # invalid toad + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + if len(windices) < toader.num: # not fully witnessed yet + if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, + seqner=delseqner, saider=delsaider, + local=local): + # cue to query for witness receipts + self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) + raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " + f"on witness sigs=" + f"{[siger.qb64 for siger in wigers]} " + f"for event={serder.ked}.") + + # returns (None, None) when delegation validation does not apply. + # Raises ValidationError if validation applies but does not validate. + delseqner, delsaider = self.validateDelegation(serder, + sigers=sigers, + wigers=wigers, + wits=wits, + delpre=delpre, + delseqner=delseqner, + delsaider=delsaider, + eager=eager, + local=local) + + return (sigers, wigers, delpre, delseqner, delsaider) def exposeds(self, sigers): @@ -2381,8 +2395,8 @@ def exposeds(self, sigers): return odxs - def validateDelegation(self, serder, sigers, wigers, wits, local=True, - delpre=None, delseqner=None, delsaider=None): + def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, + delseqner=None, delsaider=None, eager=False, local=True): """ Returns delegator's qb64 identifier prefix if validation successful. Assumes that local vs remote source checks have been applied before @@ -2410,15 +2424,24 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, delegated event. Assumes wigers is list of unique verified sigs wits (list[str]): of qb64 non-transferable prefixes of witnesses used to derive werfers for wigers - local (bool): event source for validation logic - True means event source is local (protected). - False means event source is remote (unprotected). - Event validation logic is a function of local or remote - delpre (str | None): qb64 prefix of delegator if any + delpre (str): qb64 prefix of delegator delseqner (Seqner | None): instance of delegating event sequence number. If this event is not delegated then ignored delsaider (Saider | None): instance of of delegating event digest. If this event is not delegated ignored + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + eager (bool): True means try harder to find delegation seals by + walking KEL of delegator. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing source seal couples + if any, either attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote Returns: None @@ -2612,7 +2635,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, """ if not delpre: # not delegable so no delegation validation needed - return + return (None, None) # non-delegated so delseqner delsaider must be None # if we are the delegatee, accept the event without requiring the # delegator validation via an anchored delegation seal or by requiring @@ -2621,8 +2644,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # seal to be anchored in delegator's KEL. # Witness accepts without waiting for delegation seal to be anchored in # delegator's KEL. Witness cue in Kevery will then generate receipt - if self.locallyOwned() or self.locallyWitnessed(wits=wits): - return + if (self.locallyOwned() or self.locallyMembered() or + self.locallyWitnessed(wits=wits)): + return (None, None) # not validated so delseqner delsaider must be None if self.kevers is None or delpre not in self.kevers: # missing delegator # ToDo XXXX cue a trigger to get the KEL of the delegator. This may @@ -2736,6 +2760,10 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, f"from {delpre} in {dserder.seals} for " f"evt = {serder.ked}.") + # valid anchoring seal of delegator delpre + delseqner = Seqner(snh=dserder.snh) + delsaider = Saider(qb64=dserder.said) + # Found anchor so can confirm delegation successful unless its one of # the superseding conditions. Valid if not superseding drt of drt. # Assumes database is reverified each bootup or otherwise when @@ -2749,7 +2777,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, (serder.sner.num == self.sner.num and # superseding event self.ilk == Ilks.ixn and # superseded is ixn and serder.ilk == Ilks.drt)): # recovery rotation superseding ixn - return # indicates delegation valid + return (None, None) # not validated so delseqner delsaider must be None # indicates delegation valid # get to here means drt rotation superseding another drt rotation # Kever.logEvent saves authorizer (delegator) seal source couple in @@ -2760,7 +2788,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, serfn = serder # new potentially superseding delegated event i.e. serf new bossn = dserder # new delegating event of superseding delegated event i.e. boss new serfo = self.serder # original delegated event i.e. serf original - bosso = self.fetchDelegatingEvent(delpre, serfo) + bosso = self.fetchDelegatingEvent(delpre, serfo, eager=eager) while (True): # superseding delegated rotation of rotation recovery rules # Only get to here if same sn for drt existing and drt superseding @@ -2768,7 +2796,8 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, if (bossn.sn > bosso.sn or # later supersedes (bossn.Ilk == Ilks.drt and bosso.Ilk == Ilks.ixn) ): # drt supersedes ixn - return # delegator # valid superseding delegation + # valid superseding delegation up chain so tail link valid + return (delseqner, delsaider) # tail event's delegation source if bossn.said == bosso.said: # same delegating event nseals = [SealEvent(**seal) for seal in bossn.seals @@ -2780,7 +2809,8 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, if nindex > oindex: # later seal supersedes # assumes index can't be None - return # delegator # valid superseding delegation + # valid superseding delegation up chain so tail link valid + return (delseqner, delsaider) # tail event's delegation source else: # ToDo: XXXX may want to cue up business logic for delegator @@ -2790,10 +2820,12 @@ def validateDelegation(self, serder, sigers, wigers, wits, local=True, # tie condition same sn and drt so need to climb delegation chain serfn = bossn - bossn = self.fetchDelegatingEvent(delpre, serfn) + bossn = self.fetchDelegatingEvent(delpre, serfn, eager=eager) serfo = bosso - bosso = self.fetchDelegatingEvent(delpre, serfo) + bosso = self.fetchDelegatingEvent(delpre, serfo, eager=eager) # repeat + # should never get to here + def fetchDelegatingEvent(self, delpre, serder, *, @@ -2899,7 +2931,7 @@ def fetchDelegatingEvent(self, delpre, serder, *, # database broken this should never happen so do not supersede raise ValidationError(f"Missing delegation source seal for {serder.ked}") - return dserder # get actually found delseqner, delsaider from dserder + return dserder # extract delseqner, delsaider from actually found dserder def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, @@ -3221,31 +3253,31 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) # not used anymore deprecate? - def escrowPACouple(self, serder, seqner, saider, local=True): - """ - Update associated logs for escrow of partially authenticated issued event. - Assuming signatures are provided elsewhere. Partial authentication results - from either a partially signed event or a fully signed delegated event - but whose delegation is not yet verified. - - Escrow allows escrow processor to retrieve serder from key and source - couple from val in order to to re-verify authentication status. Sigs - are escrowed elsewhere. - - Parameters: - serder is SerderKERI instance of delegated or issued event - seqner is Seqner instance of sn of seal source event of delegator/issuer - saider is Saider instance of said of delegator/issuer - local (bool): event source for validation logic - True means event source is local (protected). - False means event source is remote (unprotected). - Event validation logic is a function of local or remote - """ - local = True if local else False # ignored since not escrowing serder here - dgkey = dgKey(serder.preb, serder.saidb) - self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent - logger.debug("Kever state: Escrowed source couple for partially signed " - "or delegated event = %s\n", serder.ked) + #def escrowPACouple(self, serder, seqner, saider, local=True): + #""" + #Update associated logs for escrow of partially authenticated issued event. + #Assuming signatures are provided elsewhere. Partial authentication results + #from either a partially signed event or a fully signed delegated event + #but whose delegation is not yet verified. + + #Escrow allows escrow processor to retrieve serder from key and source + #couple from val in order to to re-verify authentication status. Sigs + #are escrowed elsewhere. + + #Parameters: + #serder is SerderKERI instance of delegated or issued event + #seqner is Seqner instance of sn of seal source event of delegator/issuer + #saider is Saider instance of said of delegator/issuer + #local (bool): event source for validation logic + #True means event source is local (protected). + #False means event source is remote (unprotected). + #Event validation logic is a function of local or remote + #""" + #local = True if local else False # ignored since not escrowing serder here + #dgkey = dgKey(serder.preb, serder.saidb) + #self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + #logger.debug("Kever state: Escrowed source couple for partially signed " + #"or delegated event = %s\n", serder.ked) @@ -5224,6 +5256,7 @@ def processEscrows(self): self.processEscrowUnverWitness() self.processEscrowUnverNonTrans() self.processEscrowUnverTrans() + self.processEscrowPartialDels() self.processEscrowPartialWigs() self.processEscrowPartialSigs() self.processEscrowDuplicitous() @@ -5761,7 +5794,7 @@ def processEscrowPartialDels(self): If successful then remove from escrow table """ - for ekey, edig in self.db.pdes.getItemIter(key=b''): + for ekey, edig in self.db.pdes.getItemIter(keys=b''): try: pre, sn = splitSnKey(ekey) # get pre and sn from escrow item dgkey = dgKey(pre, bytes(edig)) From a4c8df781c7dd08ab6508cc5b9df5eaa9ad8ec02 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 13:24:52 -0600 Subject: [PATCH 60/78] more careful refactoring --- src/keri/core/eventing.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 7a92731d..ed6f39e7 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2651,23 +2651,19 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, if self.kevers is None or delpre not in self.kevers: # missing delegator # ToDo XXXX cue a trigger to get the KEL of the delegator. This may # require OOBIing with the delegator. - # The processPSEvent should also cue a trigger to get KEL + # The processPDEvent should also cue a trigger to get KEL # of delegator if still missing when processing escrow later. - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"Missing KEL of delegator " f"{delpre} of evt = {serder.ked}.") - - dkever = self.kevers[delpre] + dkever = self.kevers[delpre] # get delegator's KEL if dkever.doNotDelegate: # drop event if delegation not allowed raise ValidationError(f"Delegator = {delpre} for evt = {serder.ked}," f" does not allow delegation.") - # Delegator accepts here without waiting for delegation seal to be anchored in # delegator's KEL. But has already waited for fully receipted above. # Once fully receipted, cue in Kevery will then trigger cue to approve @@ -2685,18 +2681,15 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, f" delegation by {delpre} of" f"event = {serder.ked}.") - else: # not local delegator so escrow PSEvent - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + else: # otherwise escrow PDEvent since no source seal + # If eager should attempt to walk KEL to find it in delegator's KEL + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - # since delseqner or delsaider is None there is no PACouple to escrow here - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError(f"No delegation seal for delegator " f"{delpre} of evt = {serder.ked}.") - ssn = Number(num=delseqner.sn).validate(inceptive=False).sn # ToDo XXXX need to replace Seqners with Numbers - + ssn = Number(num=delseqner.sn).validate(inceptive=False).sn # get the dig of the delegating event. Using getKeLast ensures delegating # event has not already been superceded key = snKey(pre=delpre, sn=ssn) # database key @@ -2712,12 +2705,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # escrow event here inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False - #sn = validateSN(sn=serder.snh, inceptive=inceptive) sn = Number(num=serder.sn).validate(inceptive=inceptive).sn - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) raise MissingDelegationError("No delegating event from {} at {} for " "evt = {}.".format(delpre, delsaider.qb64, @@ -2739,7 +2729,6 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." "".format(delpre, ddig, serder.ked)) - found = False # find event seal of delegated event in delegating data # purported delegating event from source seal couple # XXXX ToDo need to change logic here to support native CESR seals not just dicts @@ -5862,6 +5851,8 @@ def processEscrowPartialDels(self): wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] # seal source (delegator issuer if any) + # If delegator KEL not available should also cue a trigger to + # get it if still missing when processing escrow. delseqner = delsaider = None if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): delseqner, delsaider = couple # provided From b31f74ed43b3fb4da47d64dd38fb135bf76bebbe Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 14:34:32 -0600 Subject: [PATCH 61/78] refactored misfit in valDelSigsWigs --- src/keri/core/eventing.py | 291 ++++++++++++++++++++++++++++++++------ 1 file changed, 245 insertions(+), 46 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index ed6f39e7..24cdef8e 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1663,11 +1663,13 @@ def locallyOwned(self, pre: str | None = None): Returns: (bool): True if pre is local hab but not group hab + When pre="" empty then returns False Parameters: pre (str|None): qb64 identifier prefix if any. Default None None means use self.prefixer.qb64 + """ pre = pre if pre is not None else self.prefixer.qb64 return pre in self.prefixes and pre not in self.groups @@ -2244,16 +2246,15 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, raise ValidationError("No verified signatures for evt = {}." "".format(serder.ked)) + # at least one valid controller signature # Misfit check events that must be locally sourced (protected) get # escrowed in order to repair the protection when appropriate - if (not local and - (self.locallyOwned() or - self.locallyWitnessed(wits=wits))): + if (not local and self.locallyOwned()): self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally owned" - f" or locally witnessed event" - f" = {serder.ked}, {wits}, {self.prefixes}") + raise MisfitEventSourceError(f"Nonlocal source for locally owned " + f"event = {serder.ked}, {wits}, " + f"{self.prefixes}") werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers # get unique verified wigers and windices lists from wigers list @@ -2282,25 +2283,12 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f" for new est evt={serder.ked}.") - # get delegator if any - if serder.ilk == Ilks.dip: - delpre = serder.delpre # delegator from dip event - if not delpre: # empty or None - raise ValidationError(f"Empty or missing delegator for delegated" - f" inception event = {serder.ked}.") - elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation - delpre = self.delpre - else: # not delegable event icp, rot, ixn - delpre = None - - # delpre maybe None so ensure not None to pass into .locallyOwned which - # defaults to self.prefixer.qb64 when None - if not local and self.locallyOwned(delpre if delpre is not None else ''): + if (not local and self.locallyWitnessed(wits=wits)): self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally" - f" delegated by {delpre} of" - f"event = {serder.ked}.") + seqner=delseqner, saider=delsaider, local=local) + raise MisfitEventSourceError(f"Nonlocal source for locally " + f"witnessed eventf = {serder.ked}," + f"{wits}, {self.prefixes}") # this point the sigers have been verified and the wigers have been verified # even if locallyOwned or locallyMembered or locallyWitnessed. @@ -2333,6 +2321,27 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"{[siger.qb64 for siger in wigers]} " f"for event={serder.ked}.") + # get delegator if any + if serder.ilk == Ilks.dip: + delpre = serder.delpre # delegator from dip event + if not delpre: # empty or None + raise ValidationError(f"Empty or missing delegator for delegated" + f" inception event = {serder.ked}.") + elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation + delpre = self.delpre + else: # not delegable event icp, rot, ixn + delpre = None + + # delpre maybe None so ensure not None to pass into .locallyOwned which + # defaults to self.prefixer.qb64 when None. When delpre empty "" then + # returns locallyOwned(delpre) returns False. + if not local and self.locallyOwned(delpre if delpre is not None else ''): + self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MisfitEventSourceError(f"Nonlocal source for locally" + f" delegated by {delpre} of" + f"event = {serder.ked}.") + # returns (None, None) when delegation validation does not apply. # Raises ValidationError if validation applies but does not validate. delseqner, delsaider = self.validateDelegation(serder, @@ -2348,6 +2357,184 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, return (sigers, wigers, delpre, delseqner, delsaider) + #def valSigsWigsDel(self, serder, sigers, verfers, tholder, + #wigers, toader, wits, *, + #delseqner=None, delsaider=None, eager=False, + #local=True): + #""" + #Returns triple (sigers, wigers, delegator) where: + #sigers is unique validated signature verified members of inputed sigers + #wigers is unique validated signature verified members of inputed wigers + #delegator is qb64 delegator prefix if delegated else None + + #Validates sigers signatures by validating indexes, verifying signatures, and + #validating threshold sith. + #Validate witness receipts by validating indexes, verifying + #witness signatures and validating toad. + #Witness validation is a function of wits .prefixes and .local + + #Parameters: + #serder (SerderKERI): instance of event + #sigers (list): of Siger instances of indexed controllers signatures. + #Index is offset into verfers list from which public key may be derived. + #verfers (list): of Verfer instances of keys from latest est event + #tholder (Tholder): instance of sith threshold + #wigers (list): of Siger instances of indexed witness signatures. + #Index is offset into wits list of associated witness nontrans pre + #from which public key may be derived. + #toader (Number): instance of backer witness threshold + #wits (list): of qb64 non-transferable prefixes of witnesses used to + #derive werfers for wigers + #delseqner (Seqner | None): instance of delegating event sequence number. + #If this event is not delegated then seqner is ignored + #delsaider (Saider | None): instance of of delegating event said. + #If this event is not delegated then saider is ignored + #eager (bool): True means try harder to find delegation seals by + #walking KEL of delegator. Enables only being eager + #in escrow processing not initial parsing. + #False means only use pre-existing source seal couples + #if any, either attached or in database. + #local (bool): event source for validation logic + #True means event source is local (protected). + #False means event source is remote (unprotected). + #Event validation logic is a function of local or remote + + #""" + #if len(verfers) < tholder.size: + #raise ValidationError("Invalid sith = {} for keys = {} for evt = {}." + #"".format(tholder.sith, + #[verfer.qb64 for verfer in verfers], + #serder.ked)) + + ## Filters sigers to remove any signatures from locally membered groups + ## when not local (remote) event source. So that attacker can't source + ## compromised signature remotely to satisfy threshold. + + #if not local and self.locallyMembered(): # is this Kever's pre a local group + #if indices := self.locallyContributedIndices(verfers): + #for siger in list(sigers): # copy so clean del on original elements + #if siger.index in indices: + #sigers.remove(siger) + #if self.cues: + #self.cues.push(dict(kin="remoteMemberedSig", + #serder=serder, + #index=siger.index)) + + + ## get unique verified sigers and indices lists from sigers list + #sigers, indices = verifySigs(raw=serder.raw, sigers=sigers, verfers=verfers) + ## sigers now have .verfer assigned + + ## check if minimally signed in order to continue processing + #if not indices: # must have a least one verified sig + #raise ValidationError("No verified signatures for evt = {}." + #"".format(serder.ked)) + + ## at least one valid controller signature + ## Misfit check events that must be locally sourced (protected) get + ## escrowed in order to repair the protection when appropriate + #if (not local and + #(self.locallyOwned() or + #self.locallyWitnessed(wits=wits))): + #self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, + #seqner=delseqner, saider=delsaider, local=local) + #raise MisfitEventSourceError(f"Nonlocal source for locally owned" + #f" or locally witnessed event" + #f" = {serder.ked}, {wits}, {self.prefixes}") + + #werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers + ## get unique verified wigers and windices lists from wigers list + #wigers, windices = verifySigs(raw=serder.raw, sigers=wigers, verfers=werfers) + ## each wiger now has added to it a werfer of its wit in its .verfer property + + ## escrow if not fully signed vs signing threshold + #if not tholder.satisfy(indices): # at least one but not enough + #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + #seqner=delseqner, saider=delsaider, local=local) + #raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" + #f" on sigs for {[siger.qb64 for siger in sigers]}" + #f" for evt = {serder.ked}.") + + + ## escrow if not fully signed vs prior next rotation threshold + #if serder.ilk in (Ilks.rot, Ilks.drt): # rotation so check prior next threshold + ## prior next threshold in .ntholder and digers in .ndigers + #ondices = self.exposeds(sigers) + #if not self.ntholder.satisfy(indices=ondices): + #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + #seqner=delseqner, saider=delsaider,local=local) + #raise MissingSignatureError(f"Failure satisfying prior nsith=" + #f"{self.ntholder.sith} with exposed " + #f"sigs= {[siger.qb64 for siger in sigers]}" + #f" for new est evt={serder.ked}.") + + + ## get delegator if any + #if serder.ilk == Ilks.dip: + #delpre = serder.delpre # delegator from dip event + #if not delpre: # empty or None + #raise ValidationError(f"Empty or missing delegator for delegated" + #f" inception event = {serder.ked}.") + #elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation + #delpre = self.delpre + #else: # not delegable event icp, rot, ixn + #delpre = None + + ## delpre maybe None so ensure not None to pass into .locallyOwned which + ## defaults to self.prefixer.qb64 when None + #if not local and self.locallyOwned(delpre if delpre is not None else ''): + #self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, + #seqner=delseqner, saider=delsaider, local=local) + #raise MisfitEventSourceError(f"Nonlocal source for locally" + #f" delegated by {delpre} of" + #f"event = {serder.ked}.") + + ## this point the sigers have been verified and the wigers have been verified + ## even if locallyOwned or locallyMembered or locallyWitnessed. + ## But the toad has not yet been verified. + + #if not wits: + #if toader.num != 0: # bad toad non-zero and not witnessed bad event + #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + #else: # wits so may need to validate toad + ## The toad is only verified against the wigers as fully witnessed if + ## not (locallyOwned or locallyMembered or locallyWitnessed) + #if (not (self.locallyOwned() or self.locallyMembered() or + #self.locallyWitnessed(wits=wits))): + #if wits: # is witnessed + #if toader.num < 1 or toader.num > len(wits): # out of bounds toad + #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + #else: # not witnessed + #if toader.num != 0: # invalid toad + #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + #if len(windices) < toader.num: # not fully witnessed yet + #if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, + #seqner=delseqner, saider=delsaider, + #local=local): + ## cue to query for witness receipts + #self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) + #raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " + #f"on witness sigs=" + #f"{[siger.qb64 for siger in wigers]} " + #f"for event={serder.ked}.") + + ## returns (None, None) when delegation validation does not apply. + ## Raises ValidationError if validation applies but does not validate. + #delseqner, delsaider = self.validateDelegation(serder, + #sigers=sigers, + #wigers=wigers, + #wits=wits, + #delpre=delpre, + #delseqner=delseqner, + #delsaider=delsaider, + #eager=eager, + #local=local) + + #return (sigers, wigers, delpre, delseqner, delsaider) + + def exposeds(self, sigers): """Returns list of ondices (indices) suitable for Tholder.satisfy from self.ndigers (prior next key digests ) as exposed by event sigers. @@ -2667,28 +2854,34 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # Delegator accepts here without waiting for delegation seal to be anchored in # delegator's KEL. But has already waited for fully receipted above. # Once fully receipted, cue in Kevery will then trigger cue to approve - # delegation - - if delseqner is None or delsaider is None: # missing delegation seal ref - if self.locallyOwned(delpre): # local delegator so escrow delegable. - # Won't get to here if not local and locallyOwned(delpre) because - # valSigsWigsDel will send nonlocal sourced delegable event to - # misfit escrow first. Mistfit escrow must first - # promote to local and reprocess event before we get to here + # delegation. Delegator approves delegation by attaching valid source + # seal and reprocessing event which shows up here as delseqner, delsaider. + # Won't get to here if not local and locallyOwned(delpre) because + # valSigsWigsDel will send nonlocal sourced delegable event to + # misfit escrow first. Mistfit escrow must first promote to local and + # reprocess event before we get to here. Assumes that attached source + # seal in this case can't be malicious since sourced locally. + + if self.locallyOwned(delpre): # local delegator + if delseqner is None or delsaider is None: # missing delegation seal + # so escrow delegable. So local delegator can approve OOB. self.escrowDelegableEvent(serder=serder, sigers=sigers, wigers=wigers, local=local) raise MissingDelegableApprovalError(f"Missing approval for " f" delegation by {delpre} of" f"event = {serder.ked}.") - else: # otherwise escrow PDEvent since no source seal - # If eager should attempt to walk KEL to find it in delegator's KEL - self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegation seal for delegator " - f"{delpre} of evt = {serder.ked}.") + if delseqner is None or delsaider is None: # missing delegation seal ref + # If eager should walk KEL here to attempt to find in delegator's KEL + # if not found then escrow to be found later delegating event may + # not yet have been created + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegation seal for delegator " + f"{delpre} of evt = {serder.ked}.") # ToDo XXXX need to replace Seqners with Numbers + # Get delegating event from delseqner and delpre ssn = Number(num=delseqner.sn).validate(inceptive=False).sn # get the dig of the delegating event. Using getKeLast ensures delegating # event has not already been superceded @@ -2702,10 +2895,12 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, self.cues.push(dict(kin="query", q=dict(pre=delpre, sn=delseqner.snh, dig=delsaider.qb64))) - # escrow event here inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False sn = Number(num=serder.sn).validate(inceptive=inceptive).sn + # if we were to allow for malicous local delegator source seal then we + # must check for locallyOwned(delpre) first and escrowDelegable. + # otherwise escrowPDEvent self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError("No delegating event from {} at {} for " @@ -2723,7 +2918,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event - # compare digests to make sure they match here. + # compare saids to ensure match of delegating event and source seal # should never fail unless database broken if not dserder.compare(said=delsaider.qb64): # drop event raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." @@ -2742,19 +2937,22 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, found = True break - if not found: # drop event but may want to try harder here by walking KEL - # source seal couple is bad could be malicious so need to nullify as - # also walk KEL to find if any. + if not found: # drop event but may want to try harder here. + # if try harder assume source seal was malicious so nullify it and + # attempt to repair by searching for valid one in KEL + # may not find it in KEL because not yet delegated. + # Assumes bad and unrepairable so raise ValidationError raise ValidationError(f"Missing delegation seal in designated event" f"from {delpre} in {dserder.seals} for " f"evt = {serder.ked}.") - # valid anchoring seal of delegator delpre + # Found valid anchoring seal of delegator delpre delseqner = Seqner(snh=dserder.snh) delsaider = Saider(qb64=dserder.said) - # Found anchor so can confirm delegation successful unless its one of - # the superseding conditions. Valid if not superseding drt of drt. + # Since found valid anchoring seal so can confirm delegation successful + # unless its one of the superseding conditions. + # Valid if not superseding drt of drt. # Assumes database is reverified each bootup or otherwise when # chain-of-custody of disc has been broken. # Rule for non-supeding delegated rotation of rotation. @@ -3098,6 +3296,7 @@ def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): logger.debug("Kever state: escrowed delegable event=\n%s\n", json.dumps(serder.ked, indent=1)) + def escrowPSEvent(self, serder, *, sigers=None, wigers=None, seqner=None, saider=None, local=True): """ From 6dae8ef2c93a3dde6a10bc64e0fbb05c81959031 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 15:24:30 -0600 Subject: [PATCH 62/78] fixed misfit escrow checks in valSigsWigsDel to occur upfront before event can go into any other escrow. --- src/keri/core/eventing.py | 344 ++++++++------------------------------ 1 file changed, 71 insertions(+), 273 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 24cdef8e..c5b54d4f 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1675,6 +1675,27 @@ def locallyOwned(self, pre: str | None = None): return pre in self.prefixes and pre not in self.groups + def locallyDelegated(self, pre: str): + """Returns True if pre is in .prefixes and not in .groups + False otherwise. + Indicates that provided identifier prefix is controlled by a local + controller from .prefixes but is not a group with local member. + i.e pre is a locally owned (controlled) AID (identifier prefix) + Because delpre may be None, changes the default to "" instead of + self.prefixer.pre because self.prefixer.pre is delegate not delegator + of self. Unaccepted dip events do not have self.delpre set yet. + + Returns: + (bool): True if pre is local hab but not group hab + When pre="" empty or None then returns False + + Parameters: + pre (str): qb64 identifier prefix if any. + """ + pre = pre if pre is not None else "" + return self.locallyOwned(pre=pre) + + def locallyWitnessed(self, *, wits: list[str]=None, serder: (str)=None): """Returns True if a local controller is a witness of this Kever's KEL of wits in serder of if None then current wits for this Kever. @@ -2224,8 +2245,7 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # Filters sigers to remove any signatures from locally membered groups # when not local (remote) event source. So that attacker can't source - # compromised signature remotely to satisfy threshold. - + # remotely compromised but locally membered signatures to satisfy threshold. if not local and self.locallyMembered(): # is this Kever's pre a local group if indices := self.locallyContributedIndices(verfers): for siger in list(sigers): # copy so clean del on original elements @@ -2246,15 +2266,34 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, raise ValidationError("No verified signatures for evt = {}." "".format(serder.ked)) - # at least one valid controller signature - # Misfit check events that must be locally sourced (protected) get - # escrowed in order to repair the protection when appropriate - if (not local and self.locallyOwned()): + # at least one valid controller signature so we can now escrow event + + # Misfit check events that must be locally sourced (protected). + # Misfit escrow process may repair (upgrade) the protection when appropriate + # Put all misfit checks up front so never goes into any escrow but + # misfit if fails any misfit checks. + # get delegator's delpre if any for misfit check + if serder.ilk == Ilks.dip: # dip so delpre in event "di" field + delpre = serder.delpre # delegator from dip event + if not delpre: # empty or None + raise ValidationError(f"Empty or missing delegator for delegated" + f" inception event = {serder.ked}.") + elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation + delpre = self.delpre # delpre in kever state + else: # not delegable event icp, rot, ixn + delpre = None + + # Misfit escrow checks + if (not local and (self.locallyOwned() or + self.locallyWitnessed(wits=wits) or + self.locallyDelegated(pre=delpre))): self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally owned " - f"event = {serder.ked}, {wits}, " - f"{self.prefixes}") + raise MisfitEventSourceError(f"Nonlocal source for locally owned or" + f"locally witnessed or locally delegated" + f"event={serder.ked}, local aids=" + f"{self.prefixes}, {wits=}, " + f"delgator={delpre}.") werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers # get unique verified wigers and windices lists from wigers list @@ -2282,14 +2321,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"sigs= {[siger.qb64 for siger in sigers]}" f" for new est evt={serder.ked}.") - - if (not local and self.locallyWitnessed(wits=wits)): - self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally " - f"witnessed eventf = {serder.ked}," - f"{wits}, {self.prefixes}") - # this point the sigers have been verified and the wigers have been verified # even if locallyOwned or locallyMembered or locallyWitnessed. # But the toad has not yet been verified. @@ -2321,29 +2352,31 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, f"{[siger.qb64 for siger in wigers]} " f"for event={serder.ked}.") - # get delegator if any - if serder.ilk == Ilks.dip: - delpre = serder.delpre # delegator from dip event - if not delpre: # empty or None - raise ValidationError(f"Empty or missing delegator for delegated" - f" inception event = {serder.ked}.") - elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation - delpre = self.delpre - else: # not delegable event icp, rot, ixn - delpre = None - # delpre maybe None so ensure not None to pass into .locallyOwned which - # defaults to self.prefixer.qb64 when None. When delpre empty "" then - # returns locallyOwned(delpre) returns False. - if not local and self.locallyOwned(delpre if delpre is not None else ''): - self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MisfitEventSourceError(f"Nonlocal source for locally" - f" delegated by {delpre} of" - f"event = {serder.ked}.") + # Delegator approves delegation by attaching valid source + # seal and reprocessing event which shows up here as delseqner, delsaider. + # Won't get to here if not local and locallyDelegated(delpre) misfit + # checks above will send nonlocal sourced delegable event to + # misfit escrow first. Mistfit escrow must first promote to local and + # reprocess event before we get to here. Assumes that attached source + # seal in this case can't be malicious since sourced locally. + # Doesn't get to here until fully signed and witnessed. + + if self.locallyDelegated(delpre): # local delegator + if delseqner is None or delsaider is None: # missing delegation seal + # so escrow delegable. So local delegator can approve OOB. + # and create delegator event with valid event seal of this + # delegated event and then reprocess event with attached source + # seal to delegating event, i.e. delseqner, delsaider. + self.escrowDelegableEvent(serder=serder, sigers=sigers, + wigers=wigers, local=local) + raise MissingDelegableApprovalError(f"Missing approval for " + f" delegation by {delpre} of" + f"event = {serder.ked}.") - # returns (None, None) when delegation validation does not apply. - # Raises ValidationError if validation applies but does not validate. + # validateDelegation returns (None, None) when delegation validation + # does not apply. Raises ValidationError if validation applies but + # does not validate. delseqner, delsaider = self.validateDelegation(serder, sigers=sigers, wigers=wigers, @@ -2357,182 +2390,6 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, return (sigers, wigers, delpre, delseqner, delsaider) - #def valSigsWigsDel(self, serder, sigers, verfers, tholder, - #wigers, toader, wits, *, - #delseqner=None, delsaider=None, eager=False, - #local=True): - #""" - #Returns triple (sigers, wigers, delegator) where: - #sigers is unique validated signature verified members of inputed sigers - #wigers is unique validated signature verified members of inputed wigers - #delegator is qb64 delegator prefix if delegated else None - - #Validates sigers signatures by validating indexes, verifying signatures, and - #validating threshold sith. - #Validate witness receipts by validating indexes, verifying - #witness signatures and validating toad. - #Witness validation is a function of wits .prefixes and .local - - #Parameters: - #serder (SerderKERI): instance of event - #sigers (list): of Siger instances of indexed controllers signatures. - #Index is offset into verfers list from which public key may be derived. - #verfers (list): of Verfer instances of keys from latest est event - #tholder (Tholder): instance of sith threshold - #wigers (list): of Siger instances of indexed witness signatures. - #Index is offset into wits list of associated witness nontrans pre - #from which public key may be derived. - #toader (Number): instance of backer witness threshold - #wits (list): of qb64 non-transferable prefixes of witnesses used to - #derive werfers for wigers - #delseqner (Seqner | None): instance of delegating event sequence number. - #If this event is not delegated then seqner is ignored - #delsaider (Saider | None): instance of of delegating event said. - #If this event is not delegated then saider is ignored - #eager (bool): True means try harder to find delegation seals by - #walking KEL of delegator. Enables only being eager - #in escrow processing not initial parsing. - #False means only use pre-existing source seal couples - #if any, either attached or in database. - #local (bool): event source for validation logic - #True means event source is local (protected). - #False means event source is remote (unprotected). - #Event validation logic is a function of local or remote - - #""" - #if len(verfers) < tholder.size: - #raise ValidationError("Invalid sith = {} for keys = {} for evt = {}." - #"".format(tholder.sith, - #[verfer.qb64 for verfer in verfers], - #serder.ked)) - - ## Filters sigers to remove any signatures from locally membered groups - ## when not local (remote) event source. So that attacker can't source - ## compromised signature remotely to satisfy threshold. - - #if not local and self.locallyMembered(): # is this Kever's pre a local group - #if indices := self.locallyContributedIndices(verfers): - #for siger in list(sigers): # copy so clean del on original elements - #if siger.index in indices: - #sigers.remove(siger) - #if self.cues: - #self.cues.push(dict(kin="remoteMemberedSig", - #serder=serder, - #index=siger.index)) - - - ## get unique verified sigers and indices lists from sigers list - #sigers, indices = verifySigs(raw=serder.raw, sigers=sigers, verfers=verfers) - ## sigers now have .verfer assigned - - ## check if minimally signed in order to continue processing - #if not indices: # must have a least one verified sig - #raise ValidationError("No verified signatures for evt = {}." - #"".format(serder.ked)) - - ## at least one valid controller signature - ## Misfit check events that must be locally sourced (protected) get - ## escrowed in order to repair the protection when appropriate - #if (not local and - #(self.locallyOwned() or - #self.locallyWitnessed(wits=wits))): - #self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - #seqner=delseqner, saider=delsaider, local=local) - #raise MisfitEventSourceError(f"Nonlocal source for locally owned" - #f" or locally witnessed event" - #f" = {serder.ked}, {wits}, {self.prefixes}") - - #werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers - ## get unique verified wigers and windices lists from wigers list - #wigers, windices = verifySigs(raw=serder.raw, sigers=wigers, verfers=werfers) - ## each wiger now has added to it a werfer of its wit in its .verfer property - - ## escrow if not fully signed vs signing threshold - #if not tholder.satisfy(indices): # at least one but not enough - #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, - #seqner=delseqner, saider=delsaider, local=local) - #raise MissingSignatureError(f"Failure satisfying sith = {tholder.sith}" - #f" on sigs for {[siger.qb64 for siger in sigers]}" - #f" for evt = {serder.ked}.") - - - ## escrow if not fully signed vs prior next rotation threshold - #if serder.ilk in (Ilks.rot, Ilks.drt): # rotation so check prior next threshold - ## prior next threshold in .ntholder and digers in .ndigers - #ondices = self.exposeds(sigers) - #if not self.ntholder.satisfy(indices=ondices): - #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, - #seqner=delseqner, saider=delsaider,local=local) - #raise MissingSignatureError(f"Failure satisfying prior nsith=" - #f"{self.ntholder.sith} with exposed " - #f"sigs= {[siger.qb64 for siger in sigers]}" - #f" for new est evt={serder.ked}.") - - - ## get delegator if any - #if serder.ilk == Ilks.dip: - #delpre = serder.delpre # delegator from dip event - #if not delpre: # empty or None - #raise ValidationError(f"Empty or missing delegator for delegated" - #f" inception event = {serder.ked}.") - #elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation - #delpre = self.delpre - #else: # not delegable event icp, rot, ixn - #delpre = None - - ## delpre maybe None so ensure not None to pass into .locallyOwned which - ## defaults to self.prefixer.qb64 when None - #if not local and self.locallyOwned(delpre if delpre is not None else ''): - #self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, - #seqner=delseqner, saider=delsaider, local=local) - #raise MisfitEventSourceError(f"Nonlocal source for locally" - #f" delegated by {delpre} of" - #f"event = {serder.ked}.") - - ## this point the sigers have been verified and the wigers have been verified - ## even if locallyOwned or locallyMembered or locallyWitnessed. - ## But the toad has not yet been verified. - - #if not wits: - #if toader.num != 0: # bad toad non-zero and not witnessed bad event - #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - - #else: # wits so may need to validate toad - ## The toad is only verified against the wigers as fully witnessed if - ## not (locallyOwned or locallyMembered or locallyWitnessed) - #if (not (self.locallyOwned() or self.locallyMembered() or - #self.locallyWitnessed(wits=wits))): - #if wits: # is witnessed - #if toader.num < 1 or toader.num > len(wits): # out of bounds toad - #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - #else: # not witnessed - #if toader.num != 0: # invalid toad - #raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - - #if len(windices) < toader.num: # not fully witnessed yet - #if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, - #seqner=delseqner, saider=delsaider, - #local=local): - ## cue to query for witness receipts - #self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) - #raise MissingWitnessSignatureError(f"Failure satisfying toad={toader.num} " - #f"on witness sigs=" - #f"{[siger.qb64 for siger in wigers]} " - #f"for event={serder.ked}.") - - ## returns (None, None) when delegation validation does not apply. - ## Raises ValidationError if validation applies but does not validate. - #delseqner, delsaider = self.validateDelegation(serder, - #sigers=sigers, - #wigers=wigers, - #wits=wits, - #delpre=delpre, - #delseqner=delseqner, - #delsaider=delsaider, - #eager=eager, - #local=local) - - #return (sigers, wigers, delpre, delseqner, delsaider) def exposeds(self, sigers): @@ -2851,25 +2708,6 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, raise ValidationError(f"Delegator = {delpre} for evt = {serder.ked}," f" does not allow delegation.") - # Delegator accepts here without waiting for delegation seal to be anchored in - # delegator's KEL. But has already waited for fully receipted above. - # Once fully receipted, cue in Kevery will then trigger cue to approve - # delegation. Delegator approves delegation by attaching valid source - # seal and reprocessing event which shows up here as delseqner, delsaider. - # Won't get to here if not local and locallyOwned(delpre) because - # valSigsWigsDel will send nonlocal sourced delegable event to - # misfit escrow first. Mistfit escrow must first promote to local and - # reprocess event before we get to here. Assumes that attached source - # seal in this case can't be malicious since sourced locally. - - if self.locallyOwned(delpre): # local delegator - if delseqner is None or delsaider is None: # missing delegation seal - # so escrow delegable. So local delegator can approve OOB. - self.escrowDelegableEvent(serder=serder, sigers=sigers, - wigers=wigers, local=local) - raise MissingDelegableApprovalError(f"Missing approval for " - f" delegation by {delpre} of" - f"event = {serder.ked}.") if delseqner is None or delsaider is None: # missing delegation seal ref # If eager should walk KEL here to attempt to find in delegator's KEL @@ -3865,28 +3703,6 @@ def processEvent(self, serder, sigers, *, wigers=None, # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if self.local and kever.locallyOwned(): - # ToDo XXXX process this cue of query to send event to delegator - # to trigger generation of anchor in delegating event - # note for remote validators there is query cue in - # validateDelegation to query for anchoring event seal - pass - - # Moved logic to kever.validateDelegation trigger for delegation escrow - #if (self.local and - #kever.locallyOwned(kever.delpre if kever.delpre is not None else '')): # delpre may be None - ## ToDo XXXX need to cue task here to approve delegation by generating - ## an anchoring SealEvent of serder in delegators KEL - ## may include MFA and or business logic for the delegator i.e. is local - ## event that designates this controller as delegator triggers - ## this cue to approave delegation - #self.cues.push(dict(kin="approveDelegation", - #delegator=kever.delpre, - #serder=serder)) - - - - else: # not inception so can't verify sigs etc, add to out-of-order escrow self.escrowOOEvent(serder=serder, sigers=sigers, @@ -3972,24 +3788,6 @@ def processEvent(self, serder, sigers, *, wigers=None, # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if self.local and kever.locallyOwned(): - # ToDo XXXX process this cue of query to send event to delegator - # to trigger generation of anchor in delegating event - # note for remote validators there is query cue in - # validateDelegation to query for anchoring event seal - pass - - # Moved logic to kever.validateDelegation trigger for delegation escrow - #if (self.local and - #kever.locallyOwned(kever.delpre if kever.delpre is not None else '')): # delpre may be None - ## ToDo XXXX need to cue task here to approve delegation by generating - ## an anchoring SealEvent of serder in delegators KEL - ## may include MFA and or business logic for the delegator i.e. is local - ## event that designates this controller as delegator triggers - ## this cue to approave delegation - #self.cues.push(dict(kin="approveDelegation", - #delegator=kever.delpre, - #serder=serder)) else: # maybe duplicitous # check if duplicate of existing valid accepted event From 2445710fbb095d886f5a636562b89bae5dc372ee Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 15:42:48 -0600 Subject: [PATCH 63/78] added eager parameter to processEvent and connected it through to validateDelegation --- src/keri/core/eventing.py | 60 ++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index c5b54d4f..2e7bb024 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1494,7 +1494,7 @@ class Kever: def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, db=None, estOnly=None, delseqner=None, delsaider=None, firner=None, - dater=None, cues=None, local=True, check=False): + dater=None, cues=None, eager=False, local=True, check=False): """ Create incepting kever and state from inception serder Verify incepting serder against sigers raises ValidationError if not @@ -1522,6 +1522,11 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, When dater provided then use dater for first seen datetime cues (Deck | None): reference to Kevery.cues Deck when provided i.e. notices of events or requests to respond to + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -1569,9 +1574,10 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, wigers=wigers, toader=self.toader, wits=self.wits, - local=local, delseqner=delseqner, - delsaider=delsaider) + delsaider=delsaider, + eager=eager, + local=local) self.delpre = delpre # may be None self.delegated = True if self.delpre else False @@ -1884,7 +1890,7 @@ def config(self, serder, estOnly=None, doNotDelegate=None): def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None, local=True, check=False): + firner=None, dater=None, eager=False, local=True, check=False): """ Not an inception event. Verify event serder and indexed signatures in sigers and update state @@ -1908,6 +1914,11 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, dater (Dater | None): Dater instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -1953,9 +1964,10 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, wigers=wigers, toader=toader, wits=wits, - local=local, delseqner=delseqner, - delsaider=delsaider) + delsaider=delsaider, + eager=eager, + local=local) @@ -2021,6 +2033,7 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, wigers=wigers, toader=self.toader, wits=self.wits, + eager=eager, local=local) # .validateSigsDelWigs above ensures thresholds met otherwise raises exception @@ -2226,11 +2239,11 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, If this event is not delegated then seqner is ignored delsaider (Saider | None): instance of of delegating event said. If this event is not delegated then saider is ignored - eager (bool): True means try harder to find delegation seals by - walking KEL of delegator. Enables only being eager + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager in escrow processing not initial parsing. - False means only use pre-existing source seal couples - if any, either attached or in database. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -2477,11 +2490,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, True means event source is local (protected). False means event source is remote (unprotected). Event validation logic is a function of local or remote - eager (bool): True means try harder to find delegation seals by - walking KEL of delegator. Enables only being eager + eager (bool): True means try harder to validate event by + walking KELs. Enables only being eager in escrow processing not initial parsing. - False means only use pre-existing source seal couples - if any, either attached or in database. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool): event source for validation logic True means event source is local (protected). False means event source is remote (unprotected). @@ -3619,7 +3632,7 @@ def fetchWitnessState(self, pre, sn): def processEvent(self, serder, sigers, *, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None, local=None): + firner=None, dater=None, eager=False, local=None): """ Process one event serder with attached indexd signatures sigers @@ -3639,6 +3652,11 @@ def processEvent(self, serder, sigers, *, wigers=None, dater (Dater|None): instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. local (bool|None): True means local (protected) event source. False means remote (unprotected). None means use default .local . @@ -3677,6 +3695,7 @@ def processEvent(self, serder, sigers, *, wigers=None, firner=firner if self.cloned else None, dater=dater if self.cloned else None, cues=self.cues, + eager=eager, local=local, check=self.check) self.kevers[pre] = kever # not exception so add to kevers @@ -3763,7 +3782,7 @@ def processEvent(self, serder, sigers, *, wigers=None, delseqner=delseqner, delsaider=delsaider, firner=firner if self.cloned else None, dater=dater if self.cloned else None, - local=local, check=self.check) + eager=eager, local=local, check=self.check) # At this point the non-inceptive event (rot, drt, or ixn) # given by serder together with its attachments has been @@ -5520,7 +5539,8 @@ def processEscrowPartialSigs(self): sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) # If process does NOT validate sigs or delegation seal (when delegated), # but there is still one valid signature then process will @@ -5706,7 +5726,8 @@ def processEscrowPartialWigs(self): self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) # If process does NOT validate wigs then process will attempt # to re-escrow and then raise MissingWitnessSignatureError @@ -5866,7 +5887,8 @@ def processEscrowPartialDels(self): self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider, local=esr.local) + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) # If process does NOT validate delegation then process will attempt # to re-escrow and then raise MissingDelegationError From 1709aa1372f0de7352b0d5ab30da2a146342e606 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 16:59:51 -0600 Subject: [PATCH 64/78] refactored FetchDelegatingEvent to respect eager parameter and to also repair .aess when proper to do so. --- src/keri/core/eventing.py | 137 ++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 63 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 2e7bb024..3c03478c 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2754,26 +2754,25 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # otherwise escrowPDEvent self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError("No delegating event from {} at {} for " - "evt = {}.".format(delpre, - delsaider.qb64, - serder.ked)) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") # get the delegating event from dig ddig = bytes(raw) key = dgKey(pre=delpre, dig=ddig) # database key raw = self.db.getEvt(key) # get actual last event if raw is None: # drop event should never happen unless database is broken - raise ValidationError("Missing delegation from {} at event dig = {} for evt = {}." - "".format(delpre, ddig, serder.ked)) + raise ValidationError(f"Missing delegation from {delpre} at event " + f"dig = {ddig} for evt = {serder.ked}.") dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event # compare saids to ensure match of delegating event and source seal # should never fail unless database broken if not dserder.compare(said=delsaider.qb64): # drop event - raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." - "".format(delpre, ddig, serder.ked)) + raise ValidationError(f"Invalid delegation from {delpre} at event" + f" dig={ddig} for evt={serder.ked}.") found = False # find event seal of delegated event in delegating data # purported delegating event from source seal couple @@ -2825,8 +2824,13 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # set up recursive search for superseding delegations serfn = serder # new potentially superseding delegated event i.e. serf new bossn = dserder # new delegating event of superseding delegated event i.e. boss new - serfo = self.serder # original delegated event i.e. serf original - bosso = self.fetchDelegatingEvent(delpre, serfo, eager=eager) + serfo = self.serder # original accepted delegated event i.e. serf original + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") while (True): # superseding delegated rotation of rotation recovery rules # Only get to here if same sn for drt existing and drt superseding @@ -2858,52 +2862,62 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # tie condition same sn and drt so need to climb delegation chain serfn = bossn - bossn = self.fetchDelegatingEvent(delpre, serfn, eager=eager) + if not (bossn := self.fetchDelegatingEvent(delpre, serfn, eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") serfo = bosso - bosso = self.fetchDelegatingEvent(delpre, serfo, eager=eager) + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") # repeat # should never get to here - def fetchDelegatingEvent(self, delpre, serder, *, - delseqner=None, delsaider=None, eager=False): + def fetchDelegatingEvent(self, delpre, serder, *, eager=False): """Returns delegating event of delegator given by its aid delpre of delegated event given by serder otherwise raises ValidationError. - Assumes serder is already delegated event + Assumes delegation event must have been accepted at some point even if + it has subsequently been superseded. Therefore only looks in .aess for + delegating event source seal references. So do not use for fetching + delegating eventsthat have yet to be accepted. + Checks that found delegation events have both been accepted. + + When delegated event in serder has been accepted then will repair its + .aess entry if needed. Returns: - dserder (SerderKERI): key event with delegating seal + dserder (SerderKERI): delegating key event with delegating seal of + delegated event serder. If can't fetch then + returns None when eager==False or raises + ValidationError if can't fetch but eager==True Parameters: delpre (str): qb64 of identifier prefix of delegator serder (SerderKERI): delegated serder - delseqner (Seqner | None): instance of delegating event sequence number. - delsaider (Saider | None): instance of of delegating event digest. eager (bool): True means do more expensive KEL walk instead of escrow - False means do not do expensive KEL walk now + False means do not do expensive KEL walk now. Assumes db.aes source seal couple of delegating event cannot be written unless that event has been accepted (first seen) into delegator's KEL + even if it has later been superseded. + Confirms this by checking the .fons database of the delegator. - ToDo XXXX - Use the db.fons database to lookup delegating events to ensure that - lookup only uses delegating events that were indeed accepted (first seen) - at some point even though they may now be superseded. - - Looking up in the db.evts from dig in db.aes could be malicious escrows - of delegating events?? But a malicious escrow of delegating event would - only write source seal couple to db.udes not db.aes - - When escrowing .escrowPACouple, the delegating seal source couple goes - in db.udes indexed by delegating event pre,dig + A malicious escrow of delegating event could only write source seal + couple to escrow in db.udes not in accepted db.aes Delegating event may have been superceded but delegated event validator does not know it yet because db.aes keeps original delegating event source seal from before is was superseded. - Therefore lookup of delegating event needs to find delegating event via + Therefore lookup of delegating event needs to check delegating event via db.fons and not last key event in .kels which would only return the superseding event. @@ -2912,49 +2926,31 @@ def fetchDelegatingEvent(self, delpre, serder, *, KEL. This method does not distinguish between superseding and superseded so can't assume that the delegating event is the last event at its sn, i.e. the superseding event. - So this method must use db.fons to lookup to ensure that delegating + So this method must use db.fons to ensure that delegating event was accepted (first seen) even if it has subsequently been superseded. - This assumes that the delegating source seal in the AES db could - not have been written into the delegate's db.aes unless the delegating - event had been validated and accepted (first seen) into the - delegator's kel even though the delegating event may have later - been superseded at the time of this fetch. - - - Fetch delegating event has two cases: - 1. Delegated event has yet to be accepted (not seen) so can't repair - aes db because aes db must not be written until delegated event has - been accepted (first seen) bu .logEvent. - So must indicate the newly found dserder generates new source couple. - Returned dserder has the info to generate source couple but - does not indicate that a new one is needed. - 2. Delegated event has been accepted so can repair aes source couple entry - when necessary. + When the .aess entry is missing and eager is True and a valid delegation + is found and if delegate has been accepted then repairs missing .aess + entry. Otherwise does not repair but simply returns found delegation. - Need to indicate both if delegated event has been accepted and when - aes has been repaired. - seen=True (delegated event has been accepted first seen) - repair=True found new couple so repair externally if seen == False - or else repair just happened. - - When eager == False then may return None which means to escrow event and - try later with eager = True such as in escrow processor. + Found delegation may not be superseding so do not repair .aess unless + delegate was already accepted. """ + dserder = None # when not found and not eager so caller should reescrow dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate + # extra careful double check that delegate has been accepted by checkin + # fner = first seen Number instance index if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate seqner, saider = deSourceCouple(couple) deldig = saider.qb64 # dig of delegating event # extra careful double check that .aes is valid by getting # fner = first seen Number instance index if not self.db.fons.get(keys=(delpre, deldig)): # None - raise ValidationError(f"Invalid delagation authorizing source " + raise ValidationError(f"Invalid delegation authorizing source " f"seal couple for {serder.ked}") - - - dgkey = dgKey(pre=delpre, dig=deldig) # event at its said - if not (raw := self.db.getEvt(dgkey)): + ddgkey = dgKey(pre=delpre, dig=deldig) # database key of delegation + if not (raw := self.db.getEvt(ddgkey)): # database broken this should never happen so do not supersede # ToDo XXXX should repair by deleting the erroneous aes entry and # returning found one @@ -2963,12 +2959,27 @@ def fetchDelegatingEvent(self, delpre, serder, *, # original delegating event i.e. boss original dserder = serdering.SerderKERI(raw=bytes(raw)) - else: #try to find seal the hard way by walking the delegator's KEL + elif eager: #missing but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict if not (dserder := self.db.findAnchoringSealEvent(pre=delpre, seal=seal)): - # database broken this should never happen so do not supersede + # database broken this should never happen so do not validate raise ValidationError(f"Missing delegation source seal for {serder.ked}") + # extra careful double check that .aes is valid by getting + # fner = first seen Number instance index of delegation + if not self.db.fons.get(keys=(dserder.pre, dserder.dig)): # None + raise ValidationError(f"Invalid delegation authorizing source " + f"seal couple for {serder.ked}") + # Only repair when delegated has been accepted + if self.db.fons.get(keys=(preb, serder.saidb)): + # Repair .aess of delegated event by writing found source + # seal couple of delegation. This is safe becaause we confirmed + # delegation event was accepted in delegator's kel. + couple = dserder.sner.huge.encode() + dserder.saidb + self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + + #else not found so return None to escrow and get fixed later + return dserder # extract delseqner, delsaider from actually found dserder From 04619260276554cbf791c175dd4b61ed25c6265c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 17:33:31 -0600 Subject: [PATCH 65/78] added eager lookup to validateDelegation when missing source couple --- src/keri/core/eventing.py | 15 ++++++++++++--- src/keri/db/basing.py | 14 +++++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3c03478c..8e66a78f 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2721,16 +2721,25 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, raise ValidationError(f"Delegator = {delpre} for evt = {serder.ked}," f" does not allow delegation.") - + dserder = None # no delegation event yet if delseqner is None or delsaider is None: # missing delegation seal ref - # If eager should walk KEL here to attempt to find in delegator's KEL + # If eager could walk KEL here to attempt to find in delegator's KEL # if not found then escrow to be found later delegating event may - # not yet have been created + # not yet have been created. + if eager: + seal = dict(i=serder.pre, s=serder.snh, d=serder.said) + dserder = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + if dserder is not None: # found seal in dserder + delseqner = coring.Seqner(sn=dserder.sn) + delsaider = coring.Saider(qb64=dserder.said) + + if delseqner is None or delsaider is None: # missing delegation seal ref self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegation seal for delegator " f"{delpre} of evt = {serder.ked}.") + # if delseqner and delsaider and not dserder # ToDo XXXX need to replace Seqners with Numbers # Get delegating event from delseqner and delpre ssn = Number(num=delseqner.sn).validate(inceptive=False).sn diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 21980558..5c123578 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1694,13 +1694,13 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): return None - def findAnchoringSeal(self, pre, seal, sn=0): - """ - Search through a KEL for the event that contains an anchored - Seal with same Seal type as provided seal but in dict form. - Searchs from sn forward (default = 0). Only searches last event at any - sn therefore does not search any disputed or superseded events. - Returns the Serder of the first event with the anchored Seal seal, + def findAnchoringSealLast(self, pre, seal, sn=0): + """Only searches last event at any sn therefore does not search + any disputed or superseded events. + Search through last event at each sn in KEL for the event that contains + an anchored Seal with same Seal type as provided seal but in dict form. + Searchs from sn forward (default = 0). + Returns the Serder of the first found event with the anchored Seal seal, None if not found Parameters: From f04b1d9ff6c72c2e786f2e7f115ebec2f23c3b50 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 18:03:24 -0600 Subject: [PATCH 66/78] refactor logic when can't find delegation when given source seal to nullify and walk kel --- src/keri/core/eventing.py | 159 +++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 79 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 8e66a78f..9556f526 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2723,91 +2723,92 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, dserder = None # no delegation event yet if delseqner is None or delsaider is None: # missing delegation seal ref - # If eager could walk KEL here to attempt to find in delegator's KEL - # if not found then escrow to be found later delegating event may - # not yet have been created. - if eager: + if eager: # walk kel here to find seal = dict(i=serder.pre, s=serder.snh, d=serder.said) dserder = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) if dserder is not None: # found seal in dserder delseqner = coring.Seqner(sn=dserder.sn) delsaider = coring.Saider(qb64=dserder.said) - if delseqner is None or delsaider is None: # missing delegation seal ref - self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegation seal for delegator " - f"{delpre} of evt = {serder.ked}.") - - # if delseqner and delsaider and not dserder - # ToDo XXXX need to replace Seqners with Numbers - # Get delegating event from delseqner and delpre - ssn = Number(num=delseqner.sn).validate(inceptive=False).sn - # get the dig of the delegating event. Using getKeLast ensures delegating - # event has not already been superceded - key = snKey(pre=delpre, sn=ssn) # database key - raw = self.db.getKeLast(key) # get dig of delegating event as index last - - if raw is None: # no index to delegating event at key pre, sn - # Have to wait until delegating event at sn shows up in kel - # ToDo XXXX process this cue of query to fetch delegating event from - # delegator - self.cues.push(dict(kin="query", q=dict(pre=delpre, - sn=delseqner.snh, - dig=delsaider.qb64))) - # escrow event here - inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False - sn = Number(num=serder.sn).validate(inceptive=inceptive).sn - # if we were to allow for malicous local delegator source seal then we - # must check for locallyOwned(delpre) first and escrowDelegable. - # otherwise escrowPDEvent - self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) - raise MissingDelegationError(f"No delegating event from {delpre}" - f" at {delsaider.qb64} for " - f"evt = {serder.ked}.") - - # get the delegating event from dig - ddig = bytes(raw) - key = dgKey(pre=delpre, dig=ddig) # database key - raw = self.db.getEvt(key) # get actual last event - if raw is None: # drop event should never happen unless database is broken - raise ValidationError(f"Missing delegation from {delpre} at event " - f"dig = {ddig} for evt = {serder.ked}.") - - dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event - - # compare saids to ensure match of delegating event and source seal - # should never fail unless database broken - if not dserder.compare(said=delsaider.qb64): # drop event - raise ValidationError(f"Invalid delegation from {delpre} at event" - f" dig={ddig} for evt={serder.ked}.") - - found = False # find event seal of delegated event in delegating data - # purported delegating event from source seal couple - # XXXX ToDo need to change logic here to support native CESR seals not just dicts - # for JSON, CBOR, MGPK - for dseal in dserder.seals: # find delegating seal anchor - if tuple(dseal) == SealEvent._fields: - seal = SealEvent(**dseal) - if (seal.i == serder.pre and - seal.s == serder.sner.numh and - serder.compare(said=seal.d)): - found = True - break - - if not found: # drop event but may want to try harder here. - # if try harder assume source seal was malicious so nullify it and - # attempt to repair by searching for valid one in KEL - # may not find it in KEL because not yet delegated. - # Assumes bad and unrepairable so raise ValidationError - raise ValidationError(f"Missing delegation seal in designated event" - f"from {delpre} in {dserder.seals} for " - f"evt = {serder.ked}.") - - # Found valid anchoring seal of delegator delpre - delseqner = Seqner(snh=dserder.snh) - delsaider = Saider(qb64=dserder.said) + if not dserder: # just escrow and try later + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegation seal for delegator " + f"{delpre} of evt = {serder.ked}.") + + if delseqner and delsaider and not dserder: # given couple not found + # ToDo XXXX need to replace Seqners with Numbers + # Get delegating event from delseqner and delpre + ssn = Number(num=delseqner.sn).validate(inceptive=False).sn + # get the dig of the delegating event. Using getKeLast ensures delegating + # event has not already been superceded + key = snKey(pre=delpre, sn=ssn) # database key + raw = self.db.getKeLast(key) # get dig of delegating event as index last + + if raw is None: # no index to delegating event at key pre, sn + # Have to wait until delegating event at sn shows up in kel + # ToDo XXXX process this cue of query to fetch delegating event from + # delegator + self.cues.push(dict(kin="query", q=dict(pre=delpre, + sn=delseqner.snh, + dig=delsaider.qb64))) + # escrow event here + inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False + sn = Number(num=serder.sn).validate(inceptive=inceptive).sn + # if we were to allow for malicous local delegator source seal then we + # must check for locallyOwned(delpre) first and escrowDelegable. + # otherwise escrowPDEvent + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegating event from {delpre}" + f" at {delsaider.qb64} for " + f"evt = {serder.ked}.") + + # get the delegating event from dig + ddig = bytes(raw) + key = dgKey(pre=delpre, dig=ddig) # database key + raw = self.db.getEvt(key) # get actual last event + if raw is None: # drop event should never happen unless database is broken + raise ValidationError(f"Missing delegation from {delpre} at event " + f"dig = {ddig} for evt = {serder.ked}.") + + dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event + + # compare saids to ensure match of delegating event and source seal + # should never fail unless database broken + if not dserder.compare(said=delsaider.qb64): # drop event + raise ValidationError(f"Invalid delegation from {delpre} at event" + f" dig={ddig} for evt={serder.ked}.") + + found = False # find event seal of delegated event in delegating data + # purported delegating event from source seal couple + # XXXX ToDo need to change logic here to support native CESR seals not just dicts + # for JSON, CBOR, MGPK + for dseal in dserder.seals: # find delegating seal anchor + if tuple(dseal) == SealEvent._fields: + seal = SealEvent(**dseal) + if (seal.i == serder.pre and + seal.s == serder.sner.numh and + serder.compare(said=seal.d)): + found = True + break + + if not found: # nullify and escrow to try harder later + # worst case assume source seal was malicious so nullify it and + # attempt to repair by escrowing and eager search later + delseqner = delsaider = None # nullify + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MissingDelegationError(f"No delegation seal for delegator " + f"{delpre} of evt = {serder.ked}.") + # Assumes bad and unrepairable so raise ValidationError + #raise ValidationError(f"Missing delegation seal in designated event" + #f"from {delpre} in {dserder.seals} for " + #f"evt = {serder.ked}.") + + # Found valid anchoring seal of delegator delpre + delseqner = Seqner(snh=dserder.snh) + delsaider = Saider(qb64=dserder.said) # Since found valid anchoring seal so can confirm delegation successful # unless its one of the superseding conditions. From 05912a1f14f0929f639f2034fc89b134e066b19f Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 18:20:37 -0600 Subject: [PATCH 67/78] removed estra walks from partial wig and sig escrows now only partial del escrow --- src/keri/core/eventing.py | 76 +++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 9556f526..3db80531 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5538,23 +5538,19 @@ def processEscrowPartialSigs(self): delseqner = delsaider = None if (couple := self.db.udes.get(keys=dgkey)): delseqner, delsaider = couple - #couple = self.db.getUde(dgkey) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + ## idempotent + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] @@ -5578,17 +5574,16 @@ def processEscrowPartialSigs(self): # Non re-escrow ValidationError means some other issue so unescrow. # No error at all means processed successfully so also unescrow. - except (MissingSignatureError, MissingDelegationError) as ex: + except MissingSignatureError as ex: # MissingDelegationError) # still waiting on missing sigs or missing seal to validate # processEvent idempotently reescrowed if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow + # error other than waiting on sigs so remove from escrow self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any + #self.db.udes.rem(keys=dgkey) # leave here since could PartialDelegationEscrow if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): self.cues.push(dict(kin="psUnescrow", serder=eserder)) @@ -5603,7 +5598,6 @@ def processEscrowPartialSigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any self.db.udes.rem(keys=dgkey) # remove escrow if any if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): @@ -5728,23 +5722,19 @@ def processEscrowPartialWigs(self): delseqner = delsaider = None if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): delseqner, delsaider = couple - #couple = self.db.getUde(dgKey(pre, bytes(edig))) - #if couple is not None: - #delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - #couple = delseqner.qb64b + delsaider.qb64b - #self.db.putUde(dgkey, couple) - # idempotent - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + ## idempotent + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=delseqner, delsaider=delsaider, @@ -5767,17 +5757,16 @@ def processEscrowPartialWigs(self): # validation will be successful as event would not be in # partially witnessed escrow unless they had already validated - except (MissingWitnessSignatureError, MissingDelegationError) as ex: + except MissingWitnessSignatureError as ex: # MissingDelegationError # still waiting on missing witness sigs or delegation # processEvent idempotently reescrowed if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow + # error other than waiting on wigs so remove from escrow self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any - self.db.udes.rem(keys=dgkey) # remove escrow if any + #self.db.udes.rem(keys=dgkey) # leave here since could PartialDelegationEscrow if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) else: @@ -5788,7 +5777,6 @@ def processEscrowPartialWigs(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - #self.db.delUde(dgkey) # remove escrow if any self.db.udes.rem(keys=dgkey) # remove escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) From c9def3ffa54c860fe7c492fc6d0f2af4cf28623c Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 6 Sep 2024 18:30:35 -0600 Subject: [PATCH 68/78] finished refactor of delegation escrow. --- src/keri/core/eventing.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3db80531..f60bb589 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5883,17 +5883,18 @@ def processEscrowPartialDels(self): delseqner = delsaider = None if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): delseqner, delsaider = couple # provided - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delpre - else: - delpre = eserder.ked["di"] - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: # found seal in srdr - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: # found seal in srdr + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=delseqner, delsaider=delsaider, From 697068be21c76d028b94e71806aeed75096d65de Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 7 Sep 2024 10:28:39 -0600 Subject: [PATCH 69/78] fixup delegation source seal escrow to be non-idempotent so can nullify erroneous seals and/or repair missing seals --- src/keri/core/eventing.py | 106 ++++++++++++++++---------------------- src/keri/db/basing.py | 43 ++++++++++++++-- 2 files changed, 83 insertions(+), 66 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index f60bb589..df2c50d3 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2705,7 +2705,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, self.locallyWitnessed(wits=wits)): return (None, None) # not validated so delseqner delsaider must be None - if self.kevers is None or delpre not in self.kevers: # missing delegator + if self.kevers is None or delpre not in self.kevers: # missing delegator KEL # ToDo XXXX cue a trigger to get the KEL of the delegator. This may # require OOBIing with the delegator. # The processPDEvent should also cue a trigger to get KEL @@ -2725,10 +2725,11 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, if delseqner is None or delsaider is None: # missing delegation seal ref if eager: # walk kel here to find seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - dserder = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + dserder = self.db.FetchSealingEventLastByEventSeal(pre=delpre, + seal=seal) if dserder is not None: # found seal in dserder - delseqner = coring.Seqner(sn=dserder.sn) - delsaider = coring.Saider(qb64=dserder.said) + delseqner = coring.Seqner(sn=dserder.sn) # replace with found + delsaider = coring.Saider(qb64=dserder.said) # replace with found if not dserder: # just escrow and try later self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, @@ -2743,9 +2744,9 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # get the dig of the delegating event. Using getKeLast ensures delegating # event has not already been superceded key = snKey(pre=delpre, sn=ssn) # database key - raw = self.db.getKeLast(key) # get dig of delegating event as index last - - if raw is None: # no index to delegating event at key pre, sn + # get dig of last delegating event purported at sn + raw = self.db.getKeLast(key) # last means not disputed or superseded + if raw is None: # no delegatint event yet. no index at key pre, sn # Have to wait until delegating event at sn shows up in kel # ToDo XXXX process this cue of query to fetch delegating event from # delegator @@ -2764,7 +2765,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, f" at {delsaider.qb64} for " f"evt = {serder.ked}.") - # get the delegating event from dig + # get the latest delegating event candidate from dig given by pre,sn index ddig = bytes(raw) key = dgKey(pre=delpre, dig=ddig) # database key raw = self.db.getEvt(key) # get actual last event @@ -2774,22 +2775,17 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event - # compare saids to ensure match of delegating event and source seal - # should never fail unless database broken - if not dserder.compare(said=delsaider.qb64): # drop event - raise ValidationError(f"Invalid delegation from {delpre} at event" - f" dig={ddig} for evt={serder.ked}.") - found = False # find event seal of delegated event in delegating data - # purported delegating event from source seal couple - # XXXX ToDo need to change logic here to support native CESR seals not just dicts - # for JSON, CBOR, MGPK + # search purported delegating event from source seal couple + for dseal in dserder.seals: # find delegating seal anchor + # XXXX ToDo need to change logic here to support native CESR seals not just dicts + # for JSON, CBOR, MGPK if tuple(dseal) == SealEvent._fields: - seal = SealEvent(**dseal) - if (seal.i == serder.pre and - seal.s == serder.sner.numh and - serder.compare(said=seal.d)): + dseal = SealEvent(**dseal) + if (dseal.i == serder.pre and + dseal.s == serder.sner.numh and + serder.compare(said=dseal.d)): found = True break @@ -2801,14 +2797,18 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegation seal for delegator " f"{delpre} of evt = {serder.ked}.") - # Assumes bad and unrepairable so raise ValidationError - #raise ValidationError(f"Missing delegation seal in designated event" - #f"from {delpre} in {dserder.seals} for " - #f"evt = {serder.ked}.") - # Found valid anchoring seal of delegator delpre - delseqner = Seqner(snh=dserder.snh) - delsaider = Saider(qb64=dserder.said) + ## Found valid anchoring seal of delegator delpre + ## compare saids to ensure match of delegating event and source seal + ## repair source seal to match last event at sn in source seal. + ## Original delegating event in seal may have been disputed or + ## superseded, but if latest matches then we want to repair + #if not dserder.compare(said=delsaider.qb64): # drop event + #raise ValidationError(f"Invalid delegation from {delpre} at event" + #f" dig={ddig} for evt={serder.ked}.") + + delseqner = Seqner(snh=dserder.snh) # replace with found + delsaider = Saider(qb64=dserder.said) # replace with found # Since found valid anchoring seal so can confirm delegation successful # unless its one of the superseding conditions. @@ -3262,11 +3262,15 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, """ Update associated logs for escrow of partially delegated or otherwise authorized issued event. - Assumes controller signatures, sigs, and witness signatures, wigs are + Assumes sigs (controller signatures) and wigs (witness signatures) are provided elsewhere. Partial authentication occurs once an event is fully signed and witnessed but the authorizing (delegating) source seal in the authorizer's (delegator's) key has not yet been verified. + The escrow of the authorizing event ref via source seal is not idempotent + so that malicious or erroneous source seals may be nullified and/or + replaced by found source seals. + Escrow allows escrow processor to retrieve serder from key and source couple from val in order to to re-verify authentication status. @@ -3290,11 +3294,16 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) if wigers: # idempotent self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) - if seqner and saider: # idempotent - self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent - logger.debug(f"Kever state: Escrowed source couple sn={seqner.sn}, " - f"said={saider.said} for partially delegated/authorized" - f" event said={serder.said}.") + if seqner and saider: # non-idempotent pin to repair replace + self.db.udes.pin(keys=dgkey, val=(seqner, saider)) # non-idempotent + logger.debug(f"Kever state: Replaced escrow source couple sn=" + f"{seqner.sn}, said={saider.said} for partially " + f"delegated/authorized event said={serder.said}.") + else: + self.db.udes.rem(keys=dgkey) # nullify non-idempotent + logger.debug(f"Kever state: Nullified escrow source couple for " + f"partially delegated/authorized event said=" + f"{serder.said}.") self.db.putEvt(dgkey, serder.raw) # idempotent @@ -3312,35 +3321,6 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, f"{serder.ked}\n.") return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) - # not used anymore deprecate? - #def escrowPACouple(self, serder, seqner, saider, local=True): - #""" - #Update associated logs for escrow of partially authenticated issued event. - #Assuming signatures are provided elsewhere. Partial authentication results - #from either a partially signed event or a fully signed delegated event - #but whose delegation is not yet verified. - - #Escrow allows escrow processor to retrieve serder from key and source - #couple from val in order to to re-verify authentication status. Sigs - #are escrowed elsewhere. - - #Parameters: - #serder is SerderKERI instance of delegated or issued event - #seqner is Seqner instance of sn of seal source event of delegator/issuer - #saider is Saider instance of said of delegator/issuer - #local (bool): event source for validation logic - #True means event source is local (protected). - #False means event source is remote (unprotected). - #Event validation logic is a function of local or remote - #""" - #local = True if local else False # ignored since not escrowing serder here - #dgkey = dgKey(serder.preb, serder.saidb) - #self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent - #logger.debug("Kever state: Escrowed source couple for partially signed " - #"or delegated event = %s\n", serder.ked) - - - def state(self): """ diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 5c123578..d5b6994d 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1686,7 +1686,7 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): for evt in self.getEvtPreIter(pre=pre, sn=sn): # includes disputed & superseded srdr = serdering.SerderKERI(raw=evt.tobytes()) - for eseal in srdr.seals or []: + for eseal in srdr.seals or []: # or [] for seals 'a' field missing if tuple(eseal) == eventing.SealEvent._fields: eseal = eventing.SealEvent(**eseal) # convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): @@ -1694,7 +1694,44 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): return None - def findAnchoringSealLast(self, pre, seal, sn=0): + def FetchSealingEventLastByEventSeal(self, pre, seal, sn=0): + """ + Search through a KEL for the last event at any sn but that contains a + specific anchored event seal of namedtuple SealEvent type that matches + the provided seal in dict form and is also fully witnessed. + Searchs from provided sn forward (default = 0). + Searches only last events in KEL of pre so does not include disputed + and/or superseded events. + + Returns: + srdr (Serder): instance of the first event with the matching + anchoring SealEvent seal, + None if not found + + Parameters: + pre (bytes|str): identifier of the KEL to search + seal (dict): dict form of Seal of any type SealEvent to find in anchored + seals list of each event + sn (int): beginning sn to search + + """ + if tuple(seal) != eventing.SealEvent._fields: # wrong type of seal + return None + + seal = eventing.SealEvent(**seal) #convert to namedtuple + + for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # no disputed or superseded + srdr = serdering.SerderKERI(raw=evt.tobytes()) + for eseal in srdr.seals or []: # or [] for seals 'a' field missing + if tuple(eseal) == eventing.SealEvent._fields: + eseal = eventing.SealEvent(**eseal) # convert to namedtuple + if seal == eseal and self.fullyWitnessed(srdr): + return srdr + return None + + + + def FetchSealingEventLastBySeal(self, pre, seal, sn=0): """Only searches last event at any sn therefore does not search any disputed or superseded events. Search through last event at each sn in KEL for the event that contains @@ -1715,7 +1752,7 @@ def findAnchoringSealLast(self, pre, seal, sn=0): for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # only last evt at sn srdr = serdering.SerderKERI(raw=evt.tobytes()) - for eseal in srdr.seals or []: + for eseal in srdr.seals or []: # or [] for seals 'a' field missing if tuple(eseal) == Seal._fields: # same type of seal eseal = Seal(**eseal) #convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): From ca21e191f40c2a74552f2a608de4dbf93d5fa2fc Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 7 Sep 2024 13:01:10 -0600 Subject: [PATCH 70/78] more refinement of logic for delegation --- src/keri/core/eventing.py | 163 +++++++++++++++++++++++++------------- src/keri/db/basing.py | 9 ++- 2 files changed, 114 insertions(+), 58 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index df2c50d3..3cbff9e3 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2638,17 +2638,27 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, at the same sn under either of the following conditions: B1. The superseding rotation's delegating event is later than - the superseded rotation's delegating event in the delegator's KEL, i.e. the - sn of the superseding event's delegation is higher than the superseded event's - delegation. - - B2. The sn of the superseding rotation's delegating event is the same as - the sn of the superseded rotation's delegating event in the delegator's KEL - and the superseding rotation's delegating event is a rotation and the - superseded rotation's delegating event is an interaction, - i.e. the superseding rotation's delegating event is itself a superseding - rotation of the superseded rotations delegating interaction event in the - delgator's KEL + the superseded rotation's delegating event in the delegator's KEL, + i.e. the sn of the superseding event's delegation is higher than + the superseded event's delegation. + + B2. The sn of the superseding rotation's delegating event is the + same as the sn of the superseded rotation's delegating event in the + delegator's KEL both have the same delegating event and the index + of the delegating seal of the superceding delegation seal is later + (higher) than the index of the delegating seal of the supderseded + rotation. In other words the delegating event is the same event but + the superseding delegation seal appears later in the seal list of the + delegating event than the superseded delegation seal. + + B3. The sn of the superseding rotation's delegating event is the + same as the sn of the superseded rotation's delegating event in the + delegator's KEL and the superseding rotation's delegating event is + a rotation and the superseded rotation's delegating event is an + interaction, i.e. the delegating events are not the same event and + the superseding rotation's delegating event is itself a superseding + rotation of the superseded rotations delegating interaction event + in the delgator's KEL. C. IF Neither A nor B is satisfied, then recursively apply rules A. and B. to the delegating events of those delegating events and so on until @@ -2663,22 +2673,41 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, A. or B. must be satisfied, or else the superseding rotation must be discarded. - Note: The latest seen deleagated rotation constraint means that any earlier - delegated rotations can NOT be superseded. This greatly simplifies the + Note: The latest seen delegated rotation constraint means that any earlier + delegated rotations CAN NOT be superseded. This greatly simplifies the validation logic and avoids a potential infinite regress of forks in the - delegated identifier's KEL. + delegated identifier's KEL while allowing the delegate to + detect that a compromised delegation has occurred and give an + opportunity for the delegator to refuse to approve a subsequent + delegated rotation without additional verification with the delegate + that the subsequent delegated rotation was not compromised. In order to capture control of a delegated identifier the attacker must issue a delegated rotation that rotates to keys under the control of the attacker that must be approved by the delegator. A recovery rotation must - therefore superseded the compromised rotation. If the attacker is able - to issue and get approved by the delegator a second rotation + therefore supersede the compromised rotation. If the attacker is able + to issue and get approved by the delegator a subsequent rotation that follows but does not supersede the compromising rotation then recovery is no longer possible because the delegatee would no longer control the privete keys needed to verifiably sign a recovery rotation. + One way that detectability may be assured is when the delegator imposes + a minimum time between approvals of a delegated rotation that is + sufficient for the delgate to detect a compromised rotation recovery. + Attempts to rotate sooner than the minimum time since the immediately + prior rotation are refused until further verification has occurred. - Repair of approval soruce seal couple in 'aes' database on recursive + A delegated rotation that occurs after the minimum time since the + immediately prior delegated rotation might be automatically approved + to minimize latency. While a subsequent delegated rotation that occurs + within the minimum time would not be approed to maximize safety. + The minimum time window is designed to give the delegate enough time + to detect a comprimised or duplicitious superseding rotation and + prevent the additional verification from proceding. + + ToDo: + + Repair the approval source seal couple in the 'aess' database on recursive climb the kel tree. Once an event has been accepted into its kel. Later adding a source seal couple to 'aes' should then be OK from a security perspective since its only making discovery less expensive. @@ -2725,7 +2754,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, if delseqner is None or delsaider is None: # missing delegation seal ref if eager: # walk kel here to find seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - dserder = self.db.FetchSealingEventLastByEventSeal(pre=delpre, + dserder = self.db.fetchLastSealingEventByEventSeal(pre=delpre, seal=seal) if dserder is not None: # found seal in dserder delseqner = coring.Seqner(sn=dserder.sn) # replace with found @@ -2819,12 +2848,12 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # Returning delegator indicates success and eventually results in acceptance # via Kever.logEvent which also writes the delgating event source couple to # db.aess so we can find it later - if ((serder.ilk == Ilks.dip) or # delegated inception - (serder.sner.num == self.sner.num + 1) or # inorder event - (serder.sner.num == self.sner.num and # superseding event - self.ilk == Ilks.ixn and # superseded is ixn and - serder.ilk == Ilks.drt)): # recovery rotation superseding ixn - return (None, None) # not validated so delseqner delsaider must be None # indicates delegation valid + if ((serder.ilk == Ilks.dip) or # superseding is delegated inception or + (serder.sner.num == self.sner.num + 1) or # superseding is inorder later or + (serder.sner.num == self.sner.num and # superseding event at same sn and + self.ilk == Ilks.ixn and # superseded is interaction and + serder.ilk == Ilks.drt)): # superseding is rotation + return (delseqner, delsaider) # indicates delegation valid # get to here means drt rotation superseding another drt rotation # Kever.logEvent saves authorizer (delegator) seal source couple in @@ -2832,39 +2861,46 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # recusively look up delegating events # set up recursive search for superseding delegations - serfn = serder # new potentially superseding delegated event i.e. serf new - bossn = dserder # new delegating event of superseding delegated event i.e. boss new + # get original potentially superseded delegation serfo = self.serder # original accepted delegated event i.e. serf original - if not (bosso := self.fetchDelegatingEvent(delpre, serfo, eager=eager)): + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, original=True, + eager=eager)): self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, - seqner=delseqner, saider=delsaider, local=local) + seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegating event from {delpre}" f" at {delsaider.qb64} for " f"evt = {serder.ked}.") + # already have new potential superseding delegation + serfn = serder # new potentially superseding delegated event i.e. serf new + bossn = dserder # new delegating event of superseding delegated event i.e. boss new while (True): # superseding delegated rotation of rotation recovery rules - # Only get to here if same sn for drt existing and drt superseding - - if (bossn.sn > bosso.sn or # later supersedes - (bossn.Ilk == Ilks.drt and - bosso.Ilk == Ilks.ixn) ): # drt supersedes ixn + # Only get to here if same sn for delegated event and both + # existing and superseding delegated events are rotations + if (bossn.sn > bosso.sn or # superseding delgation is later or + (bossn.Ilk == Ilks.drt and # superseding delegation is rotation and + bosso.Ilk == Ilks.ixn) ): # superseded delegation is interaction # valid superseding delegation up chain so tail link valid return (delseqner, delsaider) # tail event's delegation source if bossn.said == bosso.said: # same delegating event nseals = [SealEvent(**seal) for seal in bossn.seals if tuple(seal) == SealEvent._fields] - nindex = nseals.index(SealEvent(i=serfn.pre, s=serfn.snh, d=serfn.said)) + nindex = nseals.index(SealEvent(i=serfn.pre, + s=serfn.snh, + d=serfn.said)) oseals = [SealEvent(**seal) for seal in bosso.seals if tuple(seal) == SealEvent._fields] - oindex = oseals.index(SealEvent(i=serfo.pre, s=serfo.snh, d=serfo.said)) + oindex = oseals.index(SealEvent(i=serfo.pre, + s=serfo.snh, + d=serfo.said)) - if nindex > oindex: # later seal supersedes + if nindex > oindex: # superseding delegation seal is later # assumes index can't be None # valid superseding delegation up chain so tail link valid return (delseqner, delsaider) # tail event's delegation source - else: + else: # not superseded # ToDo: XXXX may want to cue up business logic for delegator # if self.mine(delegator): # failed attempt at recovery raise ValidationError(f"Invalid delegation recovery rotation" @@ -2872,14 +2908,18 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, # tie condition same sn and drt so need to climb delegation chain serfn = bossn - if not (bossn := self.fetchDelegatingEvent(delpre, serfn, eager=eager)): + if not (bossn := self.fetchDelegatingEvent(delpre, serfn, + original=False, + eager=eager)): self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegating event from {delpre}" f" at {delsaider.qb64} for " f"evt = {serder.ked}.") serfo = bosso - if not (bosso := self.fetchDelegatingEvent(delpre, serfo, eager=eager)): + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, + original=True, + eager=eager)): self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, seqner=delseqner, saider=delsaider, local=local) raise MissingDelegationError(f"No delegating event from {delpre}" @@ -2890,7 +2930,7 @@ def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, - def fetchDelegatingEvent(self, delpre, serder, *, eager=False): + def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): """Returns delegating event of delegator given by its aid delpre of delegated event given by serder otherwise raises ValidationError. @@ -2912,6 +2952,12 @@ def fetchDelegatingEvent(self, delpre, serder, *, eager=False): Parameters: delpre (str): qb64 of identifier prefix of delegator serder (SerderKERI): delegated serder + original (bool): True means delegated event is the original candidate + to be superseded. This means kel walk search should + include superseded or disputed events. + False means the delegated event is new candidate to + supersede. This means kel walk search should not + include superseded or disputed events. eager (bool): True means do more expensive KEL walk instead of escrow False means do not do expensive KEL walk now. @@ -2949,39 +2995,46 @@ def fetchDelegatingEvent(self, delpre, serder, *, eager=False): """ dserder = None # when not found and not eager so caller should reescrow dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate - # extra careful double check that delegate has been accepted by checkin - # fner = first seen Number instance index + if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate seqner, saider = deSourceCouple(couple) deldig = saider.qb64 # dig of delegating event # extra careful double check that .aes is valid by getting # fner = first seen Number instance index - if not self.db.fons.get(keys=(delpre, deldig)): # None + if not self.db.fons.get(keys=(delpre, deldig)): # Not first seen raise ValidationError(f"Invalid delegation authorizing source " f"seal couple for {serder.ked}") ddgkey = dgKey(pre=delpre, dig=deldig) # database key of delegation if not (raw := self.db.getEvt(ddgkey)): # database broken this should never happen so do not supersede - # ToDo XXXX should repair by deleting the erroneous aes entry and - # returning found one + # ToDo XXXX repair by deleting the erroneous aes and + # returning None so gets escrowed and subsequent check will + # search below and repair raise ValidationError(f"Missing delegation event for {serder.ked}") - # original delegating event i.e. boss original dserder = serdering.SerderKERI(raw=bytes(raw)) - elif eager: #missing but try to find seal by walking delegator's KEL + elif eager: #missing aes but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict - if not (dserder := self.db.findAnchoringSealEvent(pre=delpre, seal=seal)): - # database broken this should never happen so do not validate - raise ValidationError(f"Missing delegation source seal for {serder.ked}") - - # extra careful double check that .aes is valid by getting - # fner = first seen Number instance index of delegation + if original: # search all events in delegator's kel not just last + if not (dserder:=self.db.fetchAllSealingEventByEventSeal(pre=delpre, + seal=seal)): + # database broken this should never happen so do not validate + raise ValidationError(f"Missing delegation source seal for {serder.ked}") + else: # only search last events in delegator's kel + if not (dserder:=self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal)): + # database broken this should never happen so do not validate + raise ValidationError(f"Missing delegation source seal for {serder.ked}") + + # extra careful double check that found event is/was accepted event + # fner = first seen Number instance index of delegation from .fons if not self.db.fons.get(keys=(dserder.pre, dserder.dig)): # None raise ValidationError(f"Invalid delegation authorizing source " f"seal couple for {serder.ked}") - # Only repair when delegated has been accepted - if self.db.fons.get(keys=(preb, serder.saidb)): + + # Only repair when delegated has been accepted i.e has .fons entry + if self.db.fons.get(keys=(serder.pre, serder.said)): # Repair .aess of delegated event by writing found source # seal couple of delegation. This is safe becaause we confirmed # delegation event was accepted in delegator's kel. diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index d5b6994d..e754b670 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1662,7 +1662,7 @@ def cloneDelegation(self, kever): for dmsg in self.clonePreIter(pre=kever.delpre, fn=0): yield dmsg - def findAnchoringSealEvent(self, pre, seal, sn=0): + def fetchAllSealingEventByEventSeal(self, pre, seal, sn=0): """ Search through a KEL for the event that contains a specific anchored SealEvent type of provided seal but in dict form and is also fully @@ -1693,8 +1693,11 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): return srdr return None + # use alias here until can change everywhere for backwards compatibility + findAnchoringSealEvent = fetchAllSealingEventByEventSeal # alias - def FetchSealingEventLastByEventSeal(self, pre, seal, sn=0): + + def fetchLastSealingEventByEventSeal(self, pre, seal, sn=0): """ Search through a KEL for the last event at any sn but that contains a specific anchored event seal of namedtuple SealEvent type that matches @@ -1731,7 +1734,7 @@ def FetchSealingEventLastByEventSeal(self, pre, seal, sn=0): - def FetchSealingEventLastBySeal(self, pre, seal, sn=0): + def fetchLastSealingEventBySeal(self, pre, seal, sn=0): """Only searches last event at any sn therefore does not search any disputed or superseded events. Search through last event at each sn in KEL for the event that contains From 9cfccdf00732fe9fa60c040ec79c3e7192e2c764 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 7 Sep 2024 13:46:58 -0600 Subject: [PATCH 71/78] finally finished delegation logic --- src/keri/core/eventing.py | 45 +++++++++++++++++++++------------------ src/keri/db/basing.py | 1 - 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 3cbff9e3..d733ee7e 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2993,7 +2993,6 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): Found delegation may not be superseding so do not repair .aess unless delegate was already accepted. """ - dserder = None # when not found and not eager so caller should reescrow dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate @@ -3001,18 +3000,20 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): deldig = saider.qb64 # dig of delegating event # extra careful double check that .aes is valid by getting # fner = first seen Number instance index - if not self.db.fons.get(keys=(delpre, deldig)): # Not first seen - raise ValidationError(f"Invalid delegation authorizing source " - f"seal couple for {serder.ked}") + if not self.db.fons.get(keys=(delpre, deldig)): # Not first seen yet? + if original: # should not happen aes database broken + # repair by deleting aes and returning None so it escrows + # and then next time around find below with repair it + self.db.delAes(dgkey) # delete aes so next time repairs it + # superseding may not have happened yet so let it escrow + return None ddgkey = dgKey(pre=delpre, dig=deldig) # database key of delegation - if not (raw := self.db.getEvt(ddgkey)): - # database broken this should never happen so do not supersede - # ToDo XXXX repair by deleting the erroneous aes and - # returning None so gets escrowed and subsequent check will - # search below and repair + if not (raw := self.db.getEvt(ddgkey)): # in fons but no event + # database broken this should never happen raise ValidationError(f"Missing delegation event for {serder.ked}") # original delegating event i.e. boss original dserder = serdering.SerderKERI(raw=bytes(raw)) + return dserder elif eager: #missing aes but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict @@ -3020,20 +3021,20 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): if not (dserder:=self.db.fetchAllSealingEventByEventSeal(pre=delpre, seal=seal)): # database broken this should never happen so do not validate + # since original must have been validated so it must have + # all its delegation chain. raise ValidationError(f"Missing delegation source seal for {serder.ked}") else: # only search last events in delegator's kel if not (dserder:=self.db.fetchLastSealingEventByEventSeal(pre=delpre, seal=seal)): - # database broken this should never happen so do not validate - raise ValidationError(f"Missing delegation source seal for {serder.ked}") - - # extra careful double check that found event is/was accepted event - # fner = first seen Number instance index of delegation from .fons - if not self.db.fons.get(keys=(dserder.pre, dserder.dig)): # None - raise ValidationError(f"Invalid delegation authorizing source " - f"seal couple for {serder.ked}") - - # Only repair when delegated has been accepted i.e has .fons entry + # superseding delegation may not have happened yet so escrow + # ToDo XXXX need to cue up to get latest events in + # delegator's kel. + #raise ValidationError(f"Missing delegation source seal for {serder.ked}") + return None + + # Only repair .aess when found delegation is for delegated event that + # has been accepted delegated event i.e has .fons entry if self.db.fons.get(keys=(serder.pre, serder.said)): # Repair .aess of delegated event by writing found source # seal couple of delegation. This is safe becaause we confirmed @@ -3041,9 +3042,11 @@ def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): couple = dserder.sner.huge.encode() + dserder.saidb self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal - #else not found so return None to escrow and get fixed later + return dserder + + else: # not found so return None to escrow and get fixed later + return None - return dserder # extract delseqner, delsaider from actually found dserder def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index e754b670..0f7e3279 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1908,7 +1908,6 @@ def getEvtPreIter(self, pre, sn=0): for dig in self.getKelIter(pre, sn=sn): try: - dgkey = dbing.dgKey(pre, dig) # get message if not (raw := self.getEvt(key=dgkey)): raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) From e62f8a332b8c4a28bd1705c64f4e93222f9c4032 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 7 Sep 2024 13:55:59 -0600 Subject: [PATCH 72/78] refactored find to fetch more descriptive so use the right one --- src/keri/app/cli/commands/ipex/grant.py | 4 ++-- src/keri/app/delegating.py | 2 +- src/keri/app/grouping.py | 3 ++- src/keri/app/querying.py | 2 +- src/keri/core/eventing.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/keri/app/cli/commands/ipex/grant.py b/src/keri/app/cli/commands/ipex/grant.py index d6e309d7..cc766664 100644 --- a/src/keri/app/cli/commands/ipex/grant.py +++ b/src/keri/app/cli/commands/ipex/grant.py @@ -111,8 +111,8 @@ def grantDo(self, tymth, tock=0.0): iserder = serdering.SerderKERI(raw=bytes(iss)) seqner = coring.Seqner(sn=iserder.sn) - serder = self.hby.db.findAnchoringSealEvent(creder.sad['i'], - seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) + serder = self.hby.db.fetchAllSealingEventByEventSeal(creder.sad['i'], + seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) anc = self.hby.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said) exn, atc = protocoling.ipexGrantExn(hab=self.hab, recp=recp, message=self.message, acdc=acdc, iss=iss, anc=anc, diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index 6b90c53e..98b46808 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -138,7 +138,7 @@ def processUnanchoredEscrow(self): dkever = self.hby.kevers[kever.delpre] seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - if dserder := self.hby.db.findAnchoringSealEvent(dkever.prefixer.qb64, seal=seal): + if dserder := self.hby.db.fetchAllSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal): seqner = coring.Seqner(sn=dserder.sn) couple = seqner.qb64b + dserder.saidb dgkey = dbing.dgKey(kever.prefixer.qb64b, kever.serder.saidb) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 41f8249c..096798f7 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -171,7 +171,8 @@ def processDelegateEscrow(self): self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) else: # Not witnesser, we need to look for the anchor and then wait for receipts - if serder := self.hby.db.findAnchoringSealEvent(kever.delpre, seal=anchor): + if serder := self.hby.db.fetchAllSealingEventByEventSeal(kever.delpre, + seal=anchor): aseq = coring.Seqner(sn=serder.sn) couple = aseq.qb64b + serder.saidb dgkey = dbing.dgKey(pre, saider.qb64b) diff --git a/src/keri/app/querying.py b/src/keri/app/querying.py index 11541640..d5bfdbca 100644 --- a/src/keri/app/querying.py +++ b/src/keri/app/querying.py @@ -137,7 +137,7 @@ def recur(self, tyme, deeds=None): return False kever = self.hab.kevers[self.pre] - if self.hby.db.findAnchoringSealEvent(self.pre, seal=self.anchor): + if self.hby.db.fetchAllSealingEventByEventSeal(self.pre, seal=self.anchor): self.remove([self.witq]) return True diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index d733ee7e..9a921581 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -4888,7 +4888,7 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): kever = self.kevers[pre] if anchor: - if not self.db.findAnchoringSealEvent(pre=pre, seal=anchor): + if not self.db.fetchAllSealingEventByEventSeal(pre=pre, seal=anchor): self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) raise QueryNotFoundError("Query not found error={}.".format(ked)) From 85e82c4a874608f41a4eea8043ca1ab090b0ed94 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 8 Sep 2024 13:22:56 -0600 Subject: [PATCH 73/78] attempting to fix unit tests failing with new delegation logic --- tests/core/test_escrow.py | 125 ++++++++++++++++++++---------------- tests/core/test_eventing.py | 36 ++++++++++- 2 files changed, 102 insertions(+), 59 deletions(-) diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index 3b96e727..9ad0f014 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -466,67 +466,77 @@ def test_missing_delegator_escrow(): couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) assert couple == seqner.qb64b + bobSrdr.saidb + # Did not set up Habery so need to setup Del's prefixes so Del is locally + # owned + delDB.prefixes.add(delPre) + assert delPre in delDB.prefixes + # apply Del's inception msg to Del's Kevery # Dels event will fail but will add to its escrow psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) # delKvy.process(ims=bytearray(msg)) # process remote copy of msg - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - #escrow entry for event - escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - - # verify Kevery process partials escrow is idempotent to previously escrowed events - # assuming not stale but nothing else has changed - delKvy.processEscrowPartialSigs() - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - #escrow entry for event - escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - # apply Bob's inception to Dels' Kvy - psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(bobIcpMsg)) # process remote copy of msg - assert bobPre in delKvy.kevers # message accepted - delKvy.processEscrowPartialSigs() # process escrow - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - #assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - #escrow entry for event - escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - # apply Bob's delegating interaction to Dels' Kvy - psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(bobIxnMsg)) # process remote copy of msg - delKvy.processEscrowPartialSigs() # process escrows - assert delPre in delKvy.kevers # event removed from escrow + assert delPre in delKvy.kevers delK = delKvy.kevers[delPre] - assert delK.delegated - assert delK.serder.said == delSrdr.said - couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - assert couple == seqner.qb64b + bobSrdr.saidb - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 0 - #escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - #assert escrow is None # delegated inception delegation couple - #escrow entry for event delegated inception delegation couple - escrow = self.db.udes.get(keys=(delPre, delSrdr.said)) - assert escrow is None + + #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #escrows = delKvy.db.getPwes(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #escrows = delKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + #assert len(escrows) == 1 + #assert escrows[0] == delSrdr.saidb # escrow entry for event + ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + ##escrow entry for event + #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb + + + ## verify Kevery process partials escrow is idempotent to previously escrowed events + ## assuming not stale but nothing else has changed + #delKvy.processEscrowPartialSigs() + #assert delPre not in delKvy.kevers + #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #assert len(escrows) == 1 + #assert escrows[0] == delSrdr.saidb # escrow entry for event + ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + ##escrow entry for event + #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb + + ## apply Bob's inception to Dels' Kvy + #psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) + ## delKvy.process(ims=bytearray(bobIcpMsg)) # process remote copy of msg + #assert bobPre in delKvy.kevers # message accepted + #delKvy.processEscrowPartialSigs() # process escrow + #assert delPre not in delKvy.kevers + #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #assert len(escrows) == 1 + #assert escrows[0] == delSrdr.saidb # escrow entry for event + ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event + ##escrow entry for event + #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) + #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb + + ## apply Bob's delegating interaction to Dels' Kvy + #psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) + ## delKvy.process(ims=bytearray(bobIxnMsg)) # process remote copy of msg + #delKvy.processEscrowPartialSigs() # process escrows + #assert delPre in delKvy.kevers # event removed from escrow + #delK = delKvy.kevers[delPre] + #assert delK.delegated + #assert delK.serder.said == delSrdr.said + #couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + #assert couple == seqner.qb64b + bobSrdr.saidb + + #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + #assert len(escrows) == 0 + ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) + ##assert escrow is None # delegated inception delegation couple + ##escrow entry for event delegated inception delegation couple + #escrow = self.db.udes.get(keys=(delPre, delSrdr.said)) + #assert escrow is None # Setup Del rotation event verfers, digers = delMgr.rotate(pre=delPre, temp=True) @@ -586,8 +596,9 @@ def test_missing_delegator_escrow(): # delKvy.process(ims=bytearray(msg)) # process remote copy of msg assert delK.delegated assert delK.serder.said == delSrdr.said - couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - assert couple == seqner.qb64b + bobSrdr.saidb + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + #couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + #assert couple == seqner.qb64b + bobSrdr.saidb # apply Del's delegated Rotation event message to bob's Kevery psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index a4474d54..2b799a98 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -4907,6 +4907,40 @@ def test_load_event(mockHelpingNowUTC): # Endorse Tee's inception event with Wan's Hab just so we have non-trans receipts evt = eventing.loadEvent(wanHab.db, teeHab.pre, teeHab.pre) + #assert evt == {'ked': {'a': [], + #'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], + #'bt': '1', + #'c': [], + #'d': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', + #'di': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'i': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', + #'k': ['DLDlVl1H2Q138A5tftVRpyy834ejsY33BB71kXLRNP2h'], + #'kt': '1', + #'n': ['EBTtZqMkJOO4nf3cCt6SdezwkoCKtx2fGUKHeFApj_Yx'], + #'nt': '1', + #'s': '0', + #'t': 'dip', + #'v': 'KERI10JSON00018d_'}, + #'receipts': {'nontransferable': [{'prefix': 'BEXrSXVksXpnfno_Di6RBX2Lsr9VWRAihjLhowfjNOQQ', + #'signature': '0BCQOeNT3mwAHxh6mYU9K_B2VmbtjJh7_8115k4JrBPR3c4' + #'3jUSO197H2J73vWMi61qzOovNkSWQbnRx3NFnrk8I'}], + #'transferable': [{'prefix': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'said': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'sequence': '0AAAAAAAAAAAAAAAAAAAAAAA', + #'signature': 'AADGbcmUNw_SX7OVNX-PQYl41UZx_pgJXHOoMWrcfmCDGgkc1-' + #'MqXJjMD9S9moJ-lpPL9-AiXgITemMZL_QYGzIA'}]}, + #'signatures': [{'index': 0, + #'signature': 'AAC1-NTntZ0xkgHwooNcKxe9G4XC-rgkSryVz0B_QrZR2kkv4IKi7DMkfMBd4Eck-' + #'2NAi0DMuZeXnlvch6ZP0coO'}], + #'source_seal': {'said': 'EF7pHYN6XABC9znRdzprt5frW-MMry9rfrCI-_t5Y8VD', + #'sequence': 1}, + #'stored': True, + #'timestamp': '2021-01-01T00:00:00.000000+00:00', + #'witness_signatures': [{'index': 0, + #'signature': 'AABPMW3J1iZyMC-elPOkdIhddhZB_BJYHTdYv5SxcrOfJL_5igDVB6zKD' + #'AQiTj_cNa7oP-l6xSRRxwlHDwqgSwcB'}], + #'witnesses': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP']} + # no source seal in load assert evt == {'ked': {'a': [], 'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], 'bt': '1', @@ -4932,8 +4966,6 @@ def test_load_event(mockHelpingNowUTC): 'signatures': [{'index': 0, 'signature': 'AAC1-NTntZ0xkgHwooNcKxe9G4XC-rgkSryVz0B_QrZR2kkv4IKi7DMkfMBd4Eck-' '2NAi0DMuZeXnlvch6ZP0coO'}], - 'source_seal': {'said': 'EF7pHYN6XABC9znRdzprt5frW-MMry9rfrCI-_t5Y8VD', - 'sequence': 1}, 'stored': True, 'timestamp': '2021-01-01T00:00:00.000000+00:00', 'witness_signatures': [{'index': 0, From 1447885d69c3ef73e3cc8e837248a601d655c381 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 8 Sep 2024 13:27:48 -0600 Subject: [PATCH 74/78] added clarifying comment. --- src/keri/app/habbing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index ba295922..d9869970 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -2284,6 +2284,9 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode= # may want db method that updates .habs. and .prefixes together habord = basing.HabitatRecord(hid=self.pre, name=self.name, domain=self.ns) + # must add self.pre to self.prefixes before calling processEvent so that + # Kever.locallyOwned or Kever.locallyDelegated or Kever.locallyWitnessed + # evaluates correctly when processing own inception event. if not hidden: self.save(habord) self.prefixes.add(self.pre) From 4d7bde29c6b38a549ce1d730ddbe4540492baaf2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 13:02:44 -0600 Subject: [PATCH 75/78] fixed bugs in testing partial delegated escrow --- src/keri/core/eventing.py | 49 +++++--- src/keri/db/basing.py | 2 +- tests/core/test_escrow.py | 235 +++++++++++++++++++++----------------- tests/db/test_basing.py | 2 +- 4 files changed, 165 insertions(+), 123 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 9a921581..a02fc66e 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1683,7 +1683,10 @@ def locallyOwned(self, pre: str | None = None): def locallyDelegated(self, pre: str): """Returns True if pre is in .prefixes and not in .groups - False otherwise. + False otherwise. Use when pre is a delegator for some event and + want to confirm that pre is also locallyOwned thereby making the + associated event locallyDelegated. + Indicates that provided identifier prefix is controlled by a local controller from .prefixes but is not a group with local member. i.e pre is a locally owned (controlled) AID (identifier prefix) @@ -2156,6 +2159,7 @@ def rotate(self, serder): return tholder, toader, wits, cuts, adds + def deriveBacks(self, serder): """Derives and return tuple of (wits, cuts, adds) for backers given current set and any changes provided by serder. @@ -3353,7 +3357,7 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, if seqner and saider: # non-idempotent pin to repair replace self.db.udes.pin(keys=dgkey, val=(seqner, saider)) # non-idempotent logger.debug(f"Kever state: Replaced escrow source couple sn=" - f"{seqner.sn}, said={saider.said} for partially " + f"{seqner.sn}, said={saider.qb64} for partially " f"delegated/authorized event said={serder.said}.") else: self.db.udes.rem(keys=dgkey) # nullify non-idempotent @@ -3375,7 +3379,7 @@ def escrowPDEvent(self, serder, *, sigers=None, wigers=None, logger.debug(f"Kever state: Escrowed partially delegated event=\n" f"{serder.ked}\n.") - return self.db.pdes.add(keys =(serder.preb, serder.sn), val=serder.saidb) + return self.db.pdes.add(keys=snKey(serder.preb, serder.sn), val=serder.saidb) def state(self): @@ -5472,6 +5476,7 @@ def processEscrowOutOfOrders(self): break key = ekey # setup next while iteration, with key after ekey + def processEscrowPartialSigs(self): """ Process events escrowed by Kever that were only partially fulfilled, @@ -5846,10 +5851,10 @@ def processEscrowPartialDels(self): If successful then remove from escrow table """ - for ekey, edig in self.db.pdes.getItemIter(keys=b''): + for (epre,), esn, edig in self.db.pdes.getOnItemIter(keys=b''): try: - pre, sn = splitSnKey(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) + #pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(epre, edig) if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error # no local sourde so raise ValidationError which unescrows below raise ValidationError("Missing escrowed event source " @@ -5877,7 +5882,7 @@ def processEscrowPartialDels(self): "at dig = {}.".format(bytes(edig))) # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + eraw = self.db.getEvt(dgkey) if eraw is None: # no event so so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event at." @@ -5889,7 +5894,7 @@ def processEscrowPartialDels(self): eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event # get sigs - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + sigs = self.db.getSigs(dgkey) # list of sigs if not sigs: # empty list # no sigs so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Missing event sigs at." @@ -5900,14 +5905,20 @@ def processEscrowPartialDels(self): # get witness signatures (wigs not wits) assumes wont be in this # escrow if wigs not needed because no wits - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs - if not wigs: # empty list - # no wigs so raise ValidationError which unescrows below - logger.info("Kevery unescrow error: Missing event wigs at." - "dig = %s", bytes(edig)) - - raise ValidationError("Missing escrowed evt wigs at " - "dig = {}.".format(bytes(edig))) + wigs = self.db.getWigs(dgkey) # list of wigs if any + # may want to checks wits and wigs here. We are assuming that + # never get to this escrow if wits and not wigs + #if wits and not wigs: # non empty wits but empty wigs + ## wigs maybe empty if not wits or if wits while waiting + ## for first witness signature + ## which may not arrive until some time after event is fully signed + ## so just log for debugging but do not unescrow by raising + ## ValidationError + #logger.info("Kevery unescrow error: Missing event wigs at." + #"dig = %s", bytes(edig)) + + #raise ValidationError("Missing escrowed evt wigs at " + #"dig = {}.".format(bytes(edig))) # setup parameters to process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] @@ -5917,7 +5928,7 @@ def processEscrowPartialDels(self): # If delegator KEL not available should also cue a trigger to # get it if still missing when processing escrow. delseqner = delsaider = None - if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + if (couple := self.db.udes.get(keys=(epre, edig))): delseqner, delsaider = couple # provided #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find @@ -5962,7 +5973,7 @@ def processEscrowPartialDels(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow # removes one event escrow at key val - self.db.pdes.rem(keys=(pre, sn), val=edig) # event idx escrow + self.db.pdes.remOn(keys=snKey(epre, esn), val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) @@ -5974,7 +5985,7 @@ def processEscrowPartialDels(self): # duplicitous so we process remaining escrows in spite of found # valid event escrow. # removes one event escrow at key val - self.db.pdes.rem(keys=(pre, sn), val=edig) # event idx escrow + self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any logger.info("Kevery unescrow succeeded in valid event: " "event=%s", eserder.said) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index 0f7e3279..f5450dfe 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -1016,7 +1016,7 @@ def reopen(self, **kwa): self.vres = self.env.open_db(key=b'vres.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) - self.pdes = subing.IoDupSuber(db=self, subkey='pdes.') + self.pdes = subing.OnIoDupSuber(db=self, subkey='pdes.') self.udes = subing.CatCesrSuber(db=self, subkey='udes.', klas=(coring.Seqner, coring.Saider)) self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index 9ad0f014..7808bdc1 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -352,39 +352,77 @@ def test_partial_signed_escrow(): def test_missing_delegator_escrow(): """ Test missing delegator escrow + + bod is the delegator + del is the delegate + wat is the watcher """ - # bob is the delegator del is bob's delegate bobSalt = core.Salter(raw=b'0123456789abcdef').qb64 delSalt = core.Salter(raw=b'abcdef0123456789').qb64 + watSalt = core.Salter(raw=b'wxyzabcdefghijkl').qb64 psr = parsing.Parser() - with basing.openDB(name="bob") as bobDB, \ - keeping.openKS(name="bob") as bobKS, \ - basing.openDB(name="del") as delDB, \ - keeping.openKS(name="del") as delKS: + with (basing.openDB(name="bob") as bobDB, + keeping.openKS(name="bob") as bobKS, + basing.openDB(name="del") as delDB, + keeping.openKS(name="del") as delKS, + basing.openDB(name="wat") as watDB, \ + keeping.openKS(name="wat") as watKS ): # Init key pair managers bobMgr = keeping.Manager(ks=bobKS, salt=bobSalt) delMgr = keeping.Manager(ks=delKS, salt=delSalt) + watMgr = keeping.Manager(ks=watKS, salt=watSalt) # Init Keverys bobKvy = eventing.Kevery(db=bobDB) delKvy = eventing.Kevery(db=delDB) + watKvy = eventing.Kevery(db=watDB) + + # Setup Wat with own inception event + verfers, digers = watMgr.incept(stem='wat', temp=True) # algo default salty and rooted + + watSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + watPre = watSrdr.pre + watMgr.move(old=verfers[0].qb64, new=watPre) # move key pair label to prefix + # Setup wat's prefixes so wat's KEL will be Kever.locallyOwned() + watDB.prefixes.add(watPre) + assert watPre in watDB.prefixes + # setup wat's on kel + sigers = watMgr.sign(ser=watSrdr.raw, verfers=verfers) + msg = bytearray(watSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + watIcpMsg = msg # save for later + + # apply msg to wats's Kevery + psr.parse(ims=bytearray(watIcpMsg), kvy=watKvy, local=True) + watK = watKvy.kevers[watPre] + assert watK.prefixer.qb64 == watPre + assert watK.serder.said == watSrdr.said - # Setup Bob by creating inception event + # Setup Bob with own inception event verfers, digers = bobMgr.incept(stem='bob', temp=True) # algo default salty and rooted bobSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], ndigs=[diger.qb64 for diger in digers], code=coring.MtrDex.Blake3_256) - bobPre = bobSrdr.ked["i"] - + bobPre = bobSrdr.pre bobMgr.move(old=verfers[0].qb64, new=bobPre) # move key pair label to prefix + # Setup Bob's prefixes so bob's KEL will be Kever.locallyOwned() and + # Del's KEL will be Kever.locallyDelegated() + bobDB.prefixes.add(bobPre) + assert bobPre in bobDB.prefixes sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) - msg = bytearray(bobSrdr.raw) counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) @@ -395,28 +433,32 @@ def test_missing_delegator_escrow(): bobIcpMsg = msg # save for later # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(bobIcpMsg), kvy=bobKvy, local=True) bobK = bobKvy.kevers[bobPre] assert bobK.prefixer.qb64 == bobPre assert bobK.serder.said == bobSrdr.said + assert bobK.sn == 0 # apply msg to del's Kevery so he knows about the AID - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) + psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) assert bobK.prefixer.qb64 in delKvy.kevers + delBobK = bobKvy.kevers[bobPre] # bobs kever in dels kevery + assert delBobK.sn == 0 - # Setup Del's inception event assuming that Bob's next event will be an ixn delegating event + # Setup Del's inception event assuming that Bob's next event will be + # an ixn delegating event verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted - delSrdr = eventing.delcept(keys=[verfer.qb64 for verfer in verfers], delpre=bobPre, ndigs=[diger.qb64 for diger in digers]) - delPre = delSrdr.ked["i"] - + delPre = delSrdr.pre delMgr.move(old=verfers[0].qb64, new=delPre) # move key pair label to prefix + # Setup Del's prefixes so Del's KEL will be Kever.locallyOwned() + delDB.prefixes.add(delPre) + assert delPre in delDB.prefixes - # Now create delegating event + # Now create delegating event for Bob seal = eventing.SealEvent(i=delPre, s=delSrdr.ked["s"], d=delSrdr.said) @@ -433,14 +475,14 @@ def test_missing_delegator_escrow(): msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - bobIxnMsg = msg + bobIxnMsg1 = msg # delegating event with attachments # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(bobIxnMsg1), kvy=bobKvy, local=True) assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 1 - # now create msg with Del's delegated inception event + # now create Del's delegated inception event msg sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) @@ -455,9 +497,12 @@ def test_missing_delegator_escrow(): seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) msg.extend(bobSrdr.saidb) + delIcpMsg = msg # apply Del's delegated inception event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) + # because the attachment includes valid source seal then the Delegables + # escrow is bypassed and is validated and shows up in AES + psr.parse(ims=bytearray(delIcpMsg), kvy=bobKvy, local=True) # bobKvy.process(ims=bytearray(msg)) # process local copy of msg assert delPre in bobKvy.kevers # successfully validated bobDelK = bobKvy.kevers[delPre] # delK in bobs kevery @@ -466,77 +511,51 @@ def test_missing_delegator_escrow(): couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) assert couple == seqner.qb64b + bobSrdr.saidb - # Did not set up Habery so need to setup Del's prefixes so Del is locally - # owned - delDB.prefixes.add(delPre) - assert delPre in delDB.prefixes - # apply Del's inception msg to Del's Kevery - # Dels event will fail but will add to its escrow - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + # Because locallyOwned by delegate event does not validate delegation + # and ignores the attached source seal + psr.parse(ims=bytearray(delIcpMsg), kvy=delKvy, local=True) assert delPre in delKvy.kevers delK = delKvy.kevers[delPre] + # no AES entry for del's own delegated event when locallyOwned + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + + # apply Del's delegated inception event message to wats's Kevery as remote + # because the attachment includes valid source seal but wat does not + # yet have Bob's delegating event entry. The event goes into partial + # delegated event escrow + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + # Now apply Bob's incept to wat's kvy and process escrow + psr.parse(ims=bytearray(bobIcpMsg), kvy=watKvy, local=False) + assert bobPre in watKvy.kevers + watBobK = watKvy.kevers[bobPre] + assert watBobK.sn == 0 + watKvy.processEscrows() + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + # Now apply Bob's ixn to wat's kvy and process escrow + psr.parse(ims=bytearray(bobIxnMsg1), kvy=watKvy, local=False) + watKvy.processEscrows() + escrows = watKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) + assert len(escrows) == 0 + assert watBobK.sn == 1 + + assert delPre in watKvy.kevers # successfully validated + watDelK = watKvy.kevers[delPre] # delK in wats kevery + assert watDelK.delegated + assert watDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = watKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb - #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #escrows = delKvy.db.getPwes(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #escrows = delKvy.db.pdes.get(dbing.snKey(delPre, delSrdr.sn)) - #assert len(escrows) == 1 - #assert escrows[0] == delSrdr.saidb # escrow entry for event - ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - ##escrow entry for event - #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - - ## verify Kevery process partials escrow is idempotent to previously escrowed events - ## assuming not stale but nothing else has changed - #delKvy.processEscrowPartialSigs() - #assert delPre not in delKvy.kevers - #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #assert len(escrows) == 1 - #assert escrows[0] == delSrdr.saidb # escrow entry for event - ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - ##escrow entry for event - #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - ## apply Bob's inception to Dels' Kvy - #psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) - ## delKvy.process(ims=bytearray(bobIcpMsg)) # process remote copy of msg - #assert bobPre in delKvy.kevers # message accepted - #delKvy.processEscrowPartialSigs() # process escrow - #assert delPre not in delKvy.kevers - #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #assert len(escrows) == 1 - #assert escrows[0] == delSrdr.saidb # escrow entry for event - ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - ##assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - ##escrow entry for event - #escrowSeqner, escrowSaider = self.db.udes.get(keys=(delPre, delSrdr.said)) - #assert escrowSeqner.qb64b + escrowSaider.qb64b == seqner.qb64b + bobSrdr.saidb - - ## apply Bob's delegating interaction to Dels' Kvy - #psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy, local=True) - ## delKvy.process(ims=bytearray(bobIxnMsg)) # process remote copy of msg - #delKvy.processEscrowPartialSigs() # process escrows - #assert delPre in delKvy.kevers # event removed from escrow - #delK = delKvy.kevers[delPre] - #assert delK.delegated - #assert delK.serder.said == delSrdr.said - #couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - #assert couple == seqner.qb64b + bobSrdr.saidb - - #escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - #assert len(escrows) == 0 - ##escrow = delKvy.db.getUde(dbing.dgKey(delPre, delSrdr.said)) - ##assert escrow is None # delegated inception delegation couple - ##escrow entry for event delegated inception delegation couple - #escrow = self.db.udes.get(keys=(delPre, delSrdr.said)) - #assert escrow is None # Setup Del rotation event verfers, digers = delMgr.rotate(pre=delPre, temp=True) @@ -565,19 +584,25 @@ def test_missing_delegator_escrow(): for siger in sigers: msg.extend(siger.qb64b) - # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + bobIxnMsg2 = msg + + # apply bobs IXN msg to bob's Kevery + psr.parse(ims=bytearray(bobIxnMsg2), kvy=bobKvy, local=True) assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 2 # apply msg to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg - assert delKvy.kevers[bobPre].serder.said == bobSrdr.said + psr.parse(ims=bytearray(bobIxnMsg2), kvy=delKvy, local=True) + assert delBobK.serder.said == bobSrdr.said + assert delBobK.sn == 2 + + # apply msg to wat's Kevery + psr.parse(ims=bytearray(bobIxnMsg2), kvy=watKvy, local=True) + assert watBobK.serder.said == bobSrdr.said + assert watBobK.sn == 2 # now create msg from Del's delegated rotation event sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) - msg = bytearray(delSrdr.raw) counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) @@ -591,23 +616,28 @@ def test_missing_delegator_escrow(): msg.extend(seqner.qb64b) msg.extend(bobSrdr.saidb) + delRotMsg = msg + # apply Del's delegated Rotation event message to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy, local=True) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + psr.parse(ims=bytearray(delRotMsg), kvy=delKvy, local=True) assert delK.delegated assert delK.serder.said == delSrdr.said assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - #couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - #assert couple == seqner.qb64b + bobSrdr.saidb # apply Del's delegated Rotation event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy, local=True) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(delRotMsg), kvy=bobKvy, local=True) assert bobDelK.delegated assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) assert couple == seqner.qb64b + bobSrdr.saidb + # apply Del's delegated Rotation event message to wats's Kevery + psr.parse(ims=bytearray(delRotMsg), kvy=watKvy, local=True) + assert watDelK.delegated + assert watDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = watKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + assert not os.path.exists(delKS.path) @@ -1449,5 +1479,6 @@ def test_unverified_trans_receipt_escrow(): if __name__ == "__main__": - test_unverified_receipt_escrow() + #test_unverified_receipt_escrow() + test_missing_delegator_escrow() diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index b4de4832..659403c2 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -784,7 +784,7 @@ def test_baser(): assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") # test .pdes SerderIoSetSuber methods - assert isinstance(db.pdes, subing.IoDupSuber) + assert isinstance(db.pdes, subing.OnIoDupSuber) # test .udes CatCesrSuber sub db methods From 1a03114d9b08971297e1194206eb79ee2b684e94 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 14:01:04 -0600 Subject: [PATCH 76/78] fixed bug typo --- src/keri/core/eventing.py | 2 +- src/keri/db/dbing.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index a02fc66e..368680ff 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -5973,7 +5973,7 @@ def processEscrowPartialDels(self): except Exception as ex: # log diagnostics errors etc # error other than waiting on sigs or seal so remove from escrow # removes one event escrow at key val - self.db.pdes.remOn(keys=snKey(epre, esn), val=edig) # event idx escrow + self.db.pdes.rem(keys=snKey(epre, esn), val=edig) # event idx escrow self.db.udes.rem(keys=dgkey) # remove source seal escrow if any if logger.isEnabledFor(logging.DEBUG): logger.exception("Kevery unescrowed: %s", ex.args[0]) diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 57a3f491..5787c5ac 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -1869,12 +1869,6 @@ def getOnIoDupLastItemIter(self, db, key=b'', on=0, *, sep=b'.'): return # no values end of db raises StopIteration - - - # getOnIoDupItemBackIter symmetric with getOnIoDupItemIter - # getOnIoDupValBackIter symmetric with getOnIoDupValIter - - def getOnIoDupValBackIter(self, db, key=b'', on=0, *, sep=b'.'): """Returns iterator going backwards of values, of insertion ordered item at each key over all ordinal numbered keys From f810077e404ada22f2e34a00ff565dba6a78bfa3 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 15:16:40 -0600 Subject: [PATCH 77/78] added local check --- src/keri/core/eventing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 368680ff..b02d5ca2 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2380,7 +2380,7 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # Doesn't get to here until fully signed and witnessed. if self.locallyDelegated(delpre): # local delegator - if delseqner is None or delsaider is None: # missing delegation seal + if delseqner is None or delsaider is None or not local: # missing delegation seal # so escrow delegable. So local delegator can approve OOB. # and create delegator event with valid event seal of this # delegated event and then reprocess event with attached source From fe7ce5d6638cff8b9e2302080df3c1ca0cd4472a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 9 Sep 2024 15:20:43 -0600 Subject: [PATCH 78/78] removed local test since redundant already caught in misfit check --- src/keri/core/eventing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index b02d5ca2..d15cb65b 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -2380,7 +2380,8 @@ def valSigsWigsDel(self, serder, sigers, verfers, tholder, # Doesn't get to here until fully signed and witnessed. if self.locallyDelegated(delpre): # local delegator - if delseqner is None or delsaider is None or not local: # missing delegation seal + # must be local if locallyDelegated or caught above as misfit + if delseqner is None or delsaider is None: # missing delegation seal # so escrow delegable. So local delegator can approve OOB. # and create delegator event with valid event seal of this # delegated event and then reprocess event with attached source