From d483b0037f7fc5c7febe8430768302dcdb0c961e Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 21 Mar 2024 11:23:27 +0100 Subject: [PATCH 1/6] stores: add logging; worker: add debug logging for pruning; cmd: add name to sql logger; contractor; break revision errors up into multiple log lines --- api/worker.go | 5 ++++- autopilot/contractor.go | 6 ++++-- cmd/renterd/main.go | 2 +- stores/metadata.go | 6 ++++++ worker/rhpv2.go | 34 +++++++++++++++++++++++++++++----- worker/worker.go | 4 ++++ 6 files changed, 48 insertions(+), 9 deletions(-) diff --git a/api/worker.go b/api/worker.go index 39f075718..87f736da3 100644 --- a/api/worker.go +++ b/api/worker.go @@ -50,7 +50,10 @@ type ( // ContractsResponse is the response type for the /rhp/contracts endpoint. ContractsResponse struct { Contracts []Contract `json:"contracts"` - Error string `json:"error,omitempty"` + Errors map[types.PublicKey]string + + // deprecated + Error string `json:"error,omitempty"` } MemoryResponse struct { diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 9bfc8055e..4f4df19bd 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -220,8 +220,10 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) ( if err != nil { return false, err } - if resp.Error != "" { - c.logger.Error(resp.Error) + if resp.Errors != nil { + for pk, err := range resp.Errors { + c.logger.With("hostKey", pk).With("error", err).Warn("failed to fetch revision") + } } contracts := resp.Contracts c.logger.Infof("fetched %d contracts from the worker, took %v", len(resp.Contracts), time.Since(start)) diff --git a/cmd/renterd/main.go b/cmd/renterd/main.go index 03c27629f..bdb1dc423 100644 --- a/cmd/renterd/main.go +++ b/cmd/renterd/main.go @@ -492,7 +492,7 @@ func main() { dbLogCfg = cfg.Database.Log } busCfg.DBLogger = zapgorm2.Logger{ - ZapLogger: logger, + ZapLogger: logger.Named("SQL"), LogLevel: level, SlowThreshold: dbLogCfg.SlowThreshold, SkipCallerLookup: false, diff --git a/stores/metadata.go b/stores/metadata.go index e44cb3a63..3332a588e 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -2226,6 +2226,12 @@ func (s *SQLStore) createSlices(tx *gorm.DB, objID, multiPartID *uint, contractS DBSectorID: sectorID, DBContractID: contracts[fcid].ID, }) + } else { + s.logger.Warn("missing contract for shard", + "contract", fcid, + "root", shard.Root, + "latest_host", shard.LatestHost, + ) } } } diff --git a/worker/rhpv2.go b/worker/rhpv2.go index 49af73eb7..e73e9fdfa 100644 --- a/worker/rhpv2.go +++ b/worker/rhpv2.go @@ -284,6 +284,14 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types err = w.withContractLock(ctx, fcid, lockingPriorityPruning, func() error { return w.withTransportV2(ctx, hostKey, hostIP, func(t *rhpv2.Transport) error { return w.withRevisionV2(defaultLockTimeout, t, hostKey, fcid, lastKnownRevisionNumber, func(t *rhpv2.Transport, rev rhpv2.ContractRevision, settings rhpv2.HostSettings) (err error) { + logger := w.logger. + With("hostKey", hostKey). + With("hostVersion", settings.Version). + With("fcid", fcid). + With("revisionNumber", rev.Revision.RevisionNumber). + With("lastKnownRevisionNumber", lastKnownRevisionNumber). + Named("pruneContract") + // perform gouging checks gc, err := GougingCheckerFromContext(ctx, false) if err != nil { @@ -316,6 +324,7 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types delete(keep, root) // prevent duplicates continue } + logger.With("index", i).With("root", root).Debug("collected root for pruning") indices = append(indices, uint64(i)) } if len(indices) == 0 { @@ -339,7 +348,14 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types } func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevision, settings rhpv2.HostSettings, indices []uint64) (deleted uint64, err error) { - w.logger.Infow(fmt.Sprintf("deleting %d contract roots (%v)", len(indices), humanReadableSize(len(indices)*rhpv2.SectorSize)), "hk", rev.HostKey(), "fcid", rev.ID()) + logger := w.logger. + With("hostKey", t.HostKey()). + With("hostVersion", settings.Version). + With("fcid", rev.Revision.ParentID). + With("revisionNumber", rev.Revision.RevisionNumber). + Named("deleteContractRoots") + + logger.Infow(fmt.Sprintf("deleting %d contract roots (%v)", len(indices), humanReadableSize(len(indices)*rhpv2.SectorSize)), "hk", rev.HostKey(), "fcid", rev.ID()) // return early if len(indices) == 0 { @@ -380,9 +396,9 @@ func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevi if err = func() error { var cost types.Currency start := time.Now() - w.logger.Infow(fmt.Sprintf("starting batch %d/%d of size %d", i+1, len(batches), len(batch))) + logger.Infow(fmt.Sprintf("starting batch %d/%d of size %d", i+1, len(batches), len(batch))) defer func() { - w.logger.Infow(fmt.Sprintf("processing batch %d/%d of size %d took %v", i+1, len(batches), len(batch), time.Since(start)), "cost", cost) + logger.Infow(fmt.Sprintf("processing batch %d/%d of size %d took %v", i+1, len(batches), len(batch), time.Since(start)), "cost", cost) }() numSectors := rev.NumSectors() @@ -462,7 +478,7 @@ func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevi return err } else if err := t.ReadResponse(&merkleResp, minMessageSize+responseSize); err != nil { err := fmt.Errorf("couldn't read Merkle proof response, err: %v", err) - w.logger.Infow(fmt.Sprintf("processing batch %d/%d failed, err %v", i+1, len(batches), err)) + logger.Infow(fmt.Sprintf("processing batch %d/%d failed, err %v", i+1, len(batches), err)) return err } @@ -472,7 +488,7 @@ func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevi oldRoot, newRoot := types.Hash256(rev.Revision.FileMerkleRoot), merkleResp.NewMerkleRoot if rev.Revision.Filesize > 0 && !rhpv2.VerifyDiffProof(actions, numSectors, proofHashes, leafHashes, oldRoot, newRoot, nil) { err := fmt.Errorf("couldn't verify delete proof, host %v, version %v; %w", rev.HostKey(), settings.Version, ErrInvalidMerkleProof) - w.logger.Infow(fmt.Sprintf("processing batch %d/%d failed, err %v", i+1, len(batches), err)) + logger.Infow(fmt.Sprintf("processing batch %d/%d failed, err %v", i+1, len(batches), err)) t.WriteResponseErr(err) return err } @@ -506,6 +522,14 @@ func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevi // record spending w.contractSpendingRecorder.Record(rev.Revision, api.ContractSpending{Deletions: cost}) + + for _, action := range actions { + if action.Type == rhpv2.RPCWriteActionSwap { + logger.With("index", action.B).Debug("successfully swapped sector") + } else if action.Type == rhpv2.RPCWriteActionTrim { + logger.With("n", action.A).Debug("successfully trimmed sectors") + } + } return nil }(); err != nil { return diff --git a/worker/worker.go b/worker/worker.go index 89fe37a14..8514c086d 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -1298,6 +1298,10 @@ func (w *worker) rhpContractsHandlerGET(jc jape.Context) { resp := api.ContractsResponse{Contracts: contracts} if errs != nil { resp.Error = errs.Error() + resp.Errors = make(map[types.PublicKey]string) + for pk, err := range errs { + resp.Errors[pk] = err.Error() + } } jc.Encode(resp) } From 2f243637bb2d5dff53439994b8eec0eb40157ff8 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 21 Mar 2024 15:13:09 +0100 Subject: [PATCH 2/6] worker: add id to logger --- autopilot/accounts.go | 4 +++- worker/rhpv2.go | 20 +++++++++++--------- worker/upload.go | 3 +++ worker/uploader.go | 5 +++++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/autopilot/accounts.go b/autopilot/accounts.go index 690c2b35d..ec1e80558 100644 --- a/autopilot/accounts.go +++ b/autopilot/accounts.go @@ -147,7 +147,9 @@ func (a *accounts) refillWorkerAccounts(ctx context.Context, w Worker) { // register the alert if error is errMaxDriftExceeded a.ap.RegisterAlert(ctx, newAccountRefillAlert(accountID, contract, *rerr)) } - a.l.Errorw(rerr.err.Error(), rerr.keysAndValues...) + if _, inSet := inContractSet[contract.ID]; inSet { + a.l.Errorw(rerr.err.Error(), rerr.keysAndValues...) + } } else { // dismiss alerts on success a.ap.DismissAlert(ctx, alertIDForAccount(alertAccountRefillID, accountID)) diff --git a/worker/rhpv2.go b/worker/rhpv2.go index e73e9fdfa..1e3df8d0b 100644 --- a/worker/rhpv2.go +++ b/worker/rhpv2.go @@ -15,6 +15,8 @@ import ( "go.sia.tech/renterd/api" "go.sia.tech/siad/build" "go.sia.tech/siad/crypto" + "go.uber.org/zap" + "lukechampine.com/frand" ) const ( @@ -285,6 +287,7 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types return w.withTransportV2(ctx, hostKey, hostIP, func(t *rhpv2.Transport) error { return w.withRevisionV2(defaultLockTimeout, t, hostKey, fcid, lastKnownRevisionNumber, func(t *rhpv2.Transport, rev rhpv2.ContractRevision, settings rhpv2.HostSettings) (err error) { logger := w.logger. + With("id", frand.Entropy128()). With("hostKey", hostKey). With("hostVersion", settings.Version). With("fcid", fcid). @@ -312,6 +315,12 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types if err != nil { return err } + for _, root := range pending { + logger.With("root", root).Debug("pending root") + } + for _, root := range want { + logger.With("root", root).Debug("wanted root") + } keep := make(map[types.Hash256]struct{}) for _, root := range append(want, pending...) { keep[root] = struct{}{} @@ -332,7 +341,7 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types } // delete the roots from the contract - deleted, err = w.deleteContractRoots(t, &rev, settings, indices) + deleted, err = w.deleteContractRoots(t, &rev, settings, logger, indices) if deleted < uint64(len(indices)) { remaining = uint64(len(indices)) - deleted } @@ -347,14 +356,7 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types return } -func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevision, settings rhpv2.HostSettings, indices []uint64) (deleted uint64, err error) { - logger := w.logger. - With("hostKey", t.HostKey()). - With("hostVersion", settings.Version). - With("fcid", rev.Revision.ParentID). - With("revisionNumber", rev.Revision.RevisionNumber). - Named("deleteContractRoots") - +func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevision, settings rhpv2.HostSettings, logger *zap.SugaredLogger, indices []uint64) (deleted uint64, err error) { logger.Infow(fmt.Sprintf("deleting %d contract roots (%v)", len(indices), humanReadableSize(len(indices)*rhpv2.SectorSize)), "hk", rev.HostKey(), "fcid", rev.ID()) // return early diff --git a/worker/upload.go b/worker/upload.go index d146b920e..8f13d2132 100644 --- a/worker/upload.go +++ b/worker/upload.go @@ -616,9 +616,11 @@ func (mgr *uploadManager) UploadShards(ctx context.Context, s *object.Slab, shar } // track the upload in the bus + logger := mgr.logger.With("uploadID", hex.EncodeToString(upload.id[:])) if err := mgr.os.TrackUpload(ctx, upload.id); err != nil { return fmt.Errorf("failed to track upload '%v', err: %w", upload.id, err) } + logger.Debug("tracking upload") // defer a function that finishes the upload defer func() { @@ -626,6 +628,7 @@ func (mgr *uploadManager) UploadShards(ctx context.Context, s *object.Slab, shar if err := mgr.os.FinishUpload(ctx, upload.id); err != nil { mgr.logger.Errorf("failed to mark upload %v as finished: %v", upload.id, err) } + logger.Debug("finished upload") cancel() }() diff --git a/worker/uploader.go b/worker/uploader.go index 403accbc8..6a4e0232e 100644 --- a/worker/uploader.go +++ b/worker/uploader.go @@ -2,6 +2,7 @@ package worker import ( "context" + "encoding/hex" "errors" "fmt" "math" @@ -230,9 +231,13 @@ func (u *uploader) execute(req *sectorUploadReq) (time.Duration, error) { } // update the bus + logger := u.logger.With("uploadID", hex.EncodeToString(req.uploadID[:])). + With("root", req.sector.root). + With("fcid", fcid) if err := u.os.AddUploadingSector(ctx, req.uploadID, fcid, req.sector.root); err != nil { return 0, fmt.Errorf("failed to add uploading sector to contract %v, err: %v", fcid, err) } + logger.Debug("added uploading sector") // upload the sector start := time.Now() From 247d0371efec09dc960426a82a8c4de57f573ec3 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 22 Mar 2024 10:00:47 +0100 Subject: [PATCH 3/6] worker: hex id when logging --- worker/rhpv2.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/worker/rhpv2.go b/worker/rhpv2.go index 1e3df8d0b..749e7e547 100644 --- a/worker/rhpv2.go +++ b/worker/rhpv2.go @@ -2,6 +2,7 @@ package worker import ( "context" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -286,8 +287,9 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types err = w.withContractLock(ctx, fcid, lockingPriorityPruning, func() error { return w.withTransportV2(ctx, hostKey, hostIP, func(t *rhpv2.Transport) error { return w.withRevisionV2(defaultLockTimeout, t, hostKey, fcid, lastKnownRevisionNumber, func(t *rhpv2.Transport, rev rhpv2.ContractRevision, settings rhpv2.HostSettings) (err error) { + id := frand.Entropy128() logger := w.logger. - With("id", frand.Entropy128()). + With("id", hex.EncodeToString(id[:])). With("hostKey", hostKey). With("hostVersion", settings.Version). With("fcid", fcid). From e66f3896fd326d4bb939d70d8571e3aae582d4bd Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 27 Mar 2024 11:29:21 +0100 Subject: [PATCH 4/6] main: log avx2 --- cmd/renterd/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/renterd/main.go b/cmd/renterd/main.go index bdb1dc423..3dad91c50 100644 --- a/cmd/renterd/main.go +++ b/cmd/renterd/main.go @@ -32,6 +32,7 @@ import ( "go.sia.tech/renterd/worker" "go.sia.tech/web/renterd" "go.uber.org/zap" + "golang.org/x/sys/cpu" "golang.org/x/term" "gopkg.in/yaml.v3" "gorm.io/gorm/logger" @@ -485,6 +486,9 @@ func main() { defer closeFn(context.Background()) logger.Info("renterd", zap.String("version", build.Version()), zap.String("network", build.NetworkName()), zap.String("commit", build.Commit()), zap.Time("buildDate", build.BuildTime())) + if runtime.GOARCH == "amd64" && !cpu.X86.HasAVX2 { + logger.Warn("renterd is running on a system without AVX2 support, performance may be degraded") + } // configure database logger dbLogCfg := cfg.Log.Database From 9fc5127239739534abc96067e24a37f3c7a35fe3 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 27 Mar 2024 12:49:13 +0100 Subject: [PATCH 5/6] worker: remove some debug logging again --- worker/rhpv2.go | 38 ++++++++++---------------------------- worker/upload.go | 3 --- worker/uploader.go | 5 ----- 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/worker/rhpv2.go b/worker/rhpv2.go index 749e7e547..7207b96fa 100644 --- a/worker/rhpv2.go +++ b/worker/rhpv2.go @@ -16,7 +16,6 @@ import ( "go.sia.tech/renterd/api" "go.sia.tech/siad/build" "go.sia.tech/siad/crypto" - "go.uber.org/zap" "lukechampine.com/frand" ) @@ -287,16 +286,6 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types err = w.withContractLock(ctx, fcid, lockingPriorityPruning, func() error { return w.withTransportV2(ctx, hostKey, hostIP, func(t *rhpv2.Transport) error { return w.withRevisionV2(defaultLockTimeout, t, hostKey, fcid, lastKnownRevisionNumber, func(t *rhpv2.Transport, rev rhpv2.ContractRevision, settings rhpv2.HostSettings) (err error) { - id := frand.Entropy128() - logger := w.logger. - With("id", hex.EncodeToString(id[:])). - With("hostKey", hostKey). - With("hostVersion", settings.Version). - With("fcid", fcid). - With("revisionNumber", rev.Revision.RevisionNumber). - With("lastKnownRevisionNumber", lastKnownRevisionNumber). - Named("pruneContract") - // perform gouging checks gc, err := GougingCheckerFromContext(ctx, false) if err != nil { @@ -317,12 +306,6 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types if err != nil { return err } - for _, root := range pending { - logger.With("root", root).Debug("pending root") - } - for _, root := range want { - logger.With("root", root).Debug("wanted root") - } keep := make(map[types.Hash256]struct{}) for _, root := range append(want, pending...) { keep[root] = struct{}{} @@ -335,7 +318,6 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types delete(keep, root) // prevent duplicates continue } - logger.With("index", i).With("root", root).Debug("collected root for pruning") indices = append(indices, uint64(i)) } if len(indices) == 0 { @@ -343,7 +325,7 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types } // delete the roots from the contract - deleted, err = w.deleteContractRoots(t, &rev, settings, logger, indices) + deleted, err = w.deleteContractRoots(t, &rev, settings, indices) if deleted < uint64(len(indices)) { remaining = uint64(len(indices)) - deleted } @@ -358,7 +340,15 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types return } -func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevision, settings rhpv2.HostSettings, logger *zap.SugaredLogger, indices []uint64) (deleted uint64, err error) { +func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevision, settings rhpv2.HostSettings, indices []uint64) (deleted uint64, err error) { + id := frand.Entropy128() + logger := w.logger. + With("id", hex.EncodeToString(id[:])). + With("hostKey", rev.HostKey()). + With("hostVersion", settings.Version). + With("fcid", rev.ID()). + With("revisionNumber", rev.Revision.RevisionNumber). + Named("deleteContractRoots") logger.Infow(fmt.Sprintf("deleting %d contract roots (%v)", len(indices), humanReadableSize(len(indices)*rhpv2.SectorSize)), "hk", rev.HostKey(), "fcid", rev.ID()) // return early @@ -526,14 +516,6 @@ func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevi // record spending w.contractSpendingRecorder.Record(rev.Revision, api.ContractSpending{Deletions: cost}) - - for _, action := range actions { - if action.Type == rhpv2.RPCWriteActionSwap { - logger.With("index", action.B).Debug("successfully swapped sector") - } else if action.Type == rhpv2.RPCWriteActionTrim { - logger.With("n", action.A).Debug("successfully trimmed sectors") - } - } return nil }(); err != nil { return diff --git a/worker/upload.go b/worker/upload.go index 414dcc1c2..bc419d703 100644 --- a/worker/upload.go +++ b/worker/upload.go @@ -616,11 +616,9 @@ func (mgr *uploadManager) UploadShards(ctx context.Context, s object.Slab, shard } // track the upload in the bus - logger := mgr.logger.With("uploadID", hex.EncodeToString(upload.id[:])) if err := mgr.os.TrackUpload(ctx, upload.id); err != nil { return fmt.Errorf("failed to track upload '%v', err: %w", upload.id, err) } - logger.Debug("tracking upload") // defer a function that finishes the upload defer func() { @@ -628,7 +626,6 @@ func (mgr *uploadManager) UploadShards(ctx context.Context, s object.Slab, shard if err := mgr.os.FinishUpload(ctx, upload.id); err != nil { mgr.logger.Errorf("failed to mark upload %v as finished: %v", upload.id, err) } - logger.Debug("finished upload") cancel() }() diff --git a/worker/uploader.go b/worker/uploader.go index 6a4e0232e..403accbc8 100644 --- a/worker/uploader.go +++ b/worker/uploader.go @@ -2,7 +2,6 @@ package worker import ( "context" - "encoding/hex" "errors" "fmt" "math" @@ -231,13 +230,9 @@ func (u *uploader) execute(req *sectorUploadReq) (time.Duration, error) { } // update the bus - logger := u.logger.With("uploadID", hex.EncodeToString(req.uploadID[:])). - With("root", req.sector.root). - With("fcid", fcid) if err := u.os.AddUploadingSector(ctx, req.uploadID, fcid, req.sector.root); err != nil { return 0, fmt.Errorf("failed to add uploading sector to contract %v, err: %v", fcid, err) } - logger.Debug("added uploading sector") // upload the sector start := time.Now() From 2cf588e4b4ba0b18d4bfe48ba613e7f7786fa485 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 28 Mar 2024 11:52:55 +0100 Subject: [PATCH 6/6] autopilot: address review comments --- api/worker.go | 4 ++-- autopilot/accounts.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/worker.go b/api/worker.go index 87f736da3..6d0c0e9d2 100644 --- a/api/worker.go +++ b/api/worker.go @@ -49,8 +49,8 @@ type ( // ContractsResponse is the response type for the /rhp/contracts endpoint. ContractsResponse struct { - Contracts []Contract `json:"contracts"` - Errors map[types.PublicKey]string + Contracts []Contract `json:"contracts"` + Errors map[types.PublicKey]string `json:"errors,omitempty"` // deprecated Error string `json:"error,omitempty"` diff --git a/autopilot/accounts.go b/autopilot/accounts.go index ec1e80558..2332a325f 100644 --- a/autopilot/accounts.go +++ b/autopilot/accounts.go @@ -149,6 +149,8 @@ func (a *accounts) refillWorkerAccounts(ctx context.Context, w Worker) { } if _, inSet := inContractSet[contract.ID]; inSet { a.l.Errorw(rerr.err.Error(), rerr.keysAndValues...) + } else { + a.l.Debugw(rerr.err.Error(), rerr.keysAndValues...) } } else { // dismiss alerts on success