From 5e0c066a9522b3212c2ded16df649860de7bbf6a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 10:07:35 +0100 Subject: [PATCH 01/37] e2e: add TestV1ToV2Transition --- internal/test/e2e/cluster_test.go | 78 +++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index dd8c7b358..7a1aabf2f 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -35,6 +35,7 @@ import ( "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.sia.tech/renterd/stores/sql/sqlite" + "go.uber.org/zap" "lukechampine.com/frand" ) @@ -2845,3 +2846,80 @@ func TestContractFundsReturnWhenHostOffline(t *testing.T) { t.Errorf("expected balance to be %v, got %v, diff %v", expectedBalance, wallet.Confirmed, expectedBalance.Sub(wallet.Confirmed)) } } + +func TestV1ToV2Transition(t *testing.T) { + // create a chain manager with a custom network that starts before the v2 + // allow height + network, genesis := testNetwork() + network.HardforkV2.AllowHeight = 100 + network.HardforkV2.RequireHeight = 200 // 100 blocks after the allow height + store, state, err := chain.NewDBStore(chain.NewMemDB(), network, genesis) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(store, state) + + // custom autopilot config + apCfg := test.AutopilotConfig + apCfg.Contracts.Amount = 2 + apCfg.Contracts.Period = 1000 // make sure contracts are not scheduled for renew before reaching the allowheight + apCfg.Contracts.RenewWindow = 50 + + // create a test cluster + nHosts := 3 + l, _ := zap.NewDevelopment() + cluster := newTestCluster(t, testClusterOptions{ + autopilotConfig: &apCfg, + hosts: 0, // add hosts manually later + cm: cm, + uploadPacking: false, // disable to make sure we don't accidentally serve data from disk + logger: l, + }) + defer cluster.Shutdown() + tt := cluster.tt + + // add hosts and wait for contracts to form + cluster.AddHosts(nHosts) + + // make sure we are still before the v2 allow height + if cm.Tip().Height >= network.HardforkV2.AllowHeight { + t.Fatal("should be before the v2 allow height") + } + + // we should have 2 v1 contracts + var contracts []api.ContractMetadata + tt.Retry(100, 100*time.Millisecond, func() error { + contracts, err = cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeAll}) + tt.OK(err) + if len(contracts) != nHosts-1 { + return fmt.Errorf("expected %v contracts, got %v", nHosts-1, len(contracts)) + } + return nil + }) + contractHosts := make(map[types.PublicKey]struct{}) + for _, c := range contracts { + if c.V2 { + t.Fatal("should not have formed v2 contracts") + } + contractHosts[c.HostKey] = struct{}{} + } + + // sanity check number of hosts just to be safe + if len(contractHosts) != nHosts-1 { + t.Fatalf("expected %v unique hosts, got %v", nHosts-1, len(contractHosts)) + } + + // mine until we reach the v2 allowheight + cluster.MineBlocks(network.HardforkV2.AllowHeight - cm.Tip().Height) + + // slowly mine a few more blocks to allow renter to react + for i := 0; i < 5; i++ { + cluster.MineBlocks(1) + time.Sleep(100 * time.Millisecond) + } + + // check the contracts again + contracts, err = cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeAll}) + tt.OK(err) + fmt.Println("contracts", len(contracts)) +} From c789ff8c082bbfca9be3c482fe76f6bca77c1dd9 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 15:09:44 +0100 Subject: [PATCH 02/37] add applyV1ContractUpdate and revertV1ContractUpdate --- internal/bus/chainsubscriber.go | 181 +++++++++++++++++++++++++----- internal/test/e2e/cluster_test.go | 2 +- 2 files changed, 152 insertions(+), 31 deletions(-) diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index 5df3edfcb..9ef5f934e 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -216,12 +216,19 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply } // v1 contracts - cus := make(map[types.FileContractID]contractUpdate) + var err error cau.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { - cus[types.FileContractID(fce.ID)] = v1ContractUpdate(fce, rev, resolved, valid) + if err != nil { + return + } + err = s.applyV1ContractUpdate(tx, cau.State.Index, fce, rev, resolved, valid) }) + if err != nil { + return fmt.Errorf("failed to apply contract update: %w", err) + } // v2 contracts + cus := make(map[types.FileContractID]contractUpdate) var revisedContracts []types.V2FileContractElement cau.ForEachV2FileContractElement(func(fce types.V2FileContractElement, _ bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { if rev == nil { @@ -234,7 +241,7 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply // updates - this updates the 'known' contracts too so we do this first for _, cu := range cus { - if err := s.updateContract(tx, cau.State.Index, cu.fcid, cu.prev, cu.curr, cu.resolved, cu.valid); err != nil { + if err := s.updateV2Contract(tx, cau.State.Index, cu.fcid, cu.prev, cu.curr, cu.resolved, cu.valid); err != nil { return fmt.Errorf("failed to apply contract updates: %w", err) } } @@ -275,13 +282,19 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve // NOTE: host updates are not reverted // v1 contracts - var cus []contractUpdate + var err error cru.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { - cus = append(cus, v1ContractUpdate(fce, rev, resolved, valid)) + if err != nil { + return + } + err = s.revertV1ContractUpdate(tx, fce, rev, resolved, valid) }) + if err != nil { + return fmt.Errorf("failed to revert v1 contract update: %w", err) + } // v2 contracts - cus = cus[:0] + var cus []contractUpdate var revertedContracts []types.V2FileContractElement cru.ForEachV2FileContractElement(func(fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { if created { @@ -294,7 +307,7 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve // updates - this updates the 'known' contracts too so we do this first for _, cu := range cus { - if err := s.updateContract(tx, cru.State.Index, cu.fcid, cu.prev, cu.curr, cu.resolved, cu.valid); err != nil { + if err := s.updateV2Contract(tx, cru.State.Index, cu.fcid, cu.prev, cu.curr, cu.resolved, cu.valid); err != nil { return fmt.Errorf("failed to revert v2 contract update: %w", err) } } @@ -444,7 +457,7 @@ func (s *chainSubscriber) broadcastExpiredFileContractResolutions(tx sql.ChainUp } } -func (s *chainSubscriber) updateContract(tx sql.ChainUpdateTx, index types.ChainIndex, fcid types.FileContractID, prev, curr *revision, resolved, valid bool) error { +func (s *chainSubscriber) updateV2Contract(tx sql.ChainUpdateTx, index types.ChainIndex, fcid types.FileContractID, prev, curr *revision, resolved, valid bool) error { // sanity check at least one is not nil if prev == nil && curr == nil { return errors.New("both prev and curr revisions are nil") // developer error @@ -559,6 +572,136 @@ func (s *chainSubscriber) updateContract(tx sql.ChainUpdateTx, index types.Chain return nil } +func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index types.ChainIndex, fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) error { + fcid := fce.ID + if rev != nil { + fcid = rev.ID + } + + // ignore unknown contracts + if !s.isKnownContract(fcid) { + return nil + } + + // fetch contract state + state, err := tx.ContractState(fcid) + if err != nil && utils.IsErr(err, api.ErrContractNotFound) { + s.updateKnownContracts(fcid, false) // ignore unknown contracts + return nil + } else if err != nil { + return fmt.Errorf("failed to get contract state: %w", err) + } + + // update revision number and file size + revisionNumber := fce.FileContract.RevisionNumber + fileSize := fce.FileContract.Filesize + if rev != nil { + revisionNumber = rev.FileContract.RevisionNumber + fileSize = rev.FileContract.Filesize + } + if err := tx.UpdateContractRevision(fcid, index.Height, revisionNumber, fileSize); err != nil { + return fmt.Errorf("failed to update contract %v: %w", fcid, err) + } + + // consider a contract resolved if it has a max revision number and zero + // file size + if rev != nil && rev.FileContract.RevisionNumber == math.MaxUint64 && rev.FileContract.Filesize == 0 { + resolved = true + valid = true + } + + // update state from 'pending' -> 'active' + if state == api.ContractStatePending || state == api.ContractStateUnknown { + if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + s.logger.Infow("contract state changed: pending -> active", + "fcid", fcid, + "reason", "contract confirmed") + return nil + } + + // storage proof: 'active' -> 'complete/failed' + if resolved { + if err := tx.UpdateContractProofHeight(fcid, index.Height); err != nil { + return fmt.Errorf("failed to update contract proof height: %w", err) + } + if valid { + if err := tx.UpdateContractState(fcid, api.ContractStateComplete); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + s.logger.Infow("contract state changed: active -> complete", + "fcid", fcid, + "reason", "storage proof valid") + return nil + } else { + if err := tx.UpdateContractState(fcid, api.ContractStateFailed); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + s.logger.Infow("contract state changed: active -> failed", + "fcid", fcid, + "reason", "storage proof missed") + return nil + } + } + return nil +} + +func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) error { + fcid := fce.ID + if rev != nil { + fcid = rev.ID + } + + // ignore unknown contracts + if !s.isKnownContract(fcid) { + return nil + } + + // fetch contract state to see if contract is known + _, err := tx.ContractState(fcid) + if err != nil && utils.IsErr(err, api.ErrContractNotFound) { + s.updateKnownContracts(fcid, false) // ignore unknown contracts + return nil + } else if err != nil { + return fmt.Errorf("failed to get contract state: %w", err) + } + + // consider a contract resolved if it has a max revision number and zero + // file size + if rev != nil && rev.FileContract.RevisionNumber == math.MaxUint64 && rev.FileContract.Filesize == 0 { + resolved = true + valid = true + } + + // reverted storage proof: 'complete/failed' -> 'active' + if resolved { + if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + if valid { + s.logger.Infow("contract state changed: complete -> active", + "fcid", fcid, + "reason", "storage proof reverted") + } else { + s.logger.Infow("contract state changed: failed -> active", + "fcid", fcid, + "reason", "storage proof reverted") + } + return nil + } + + // update state from 'active' -> 'pending' + if rev == nil && fce.FileContract.RevisionNumber == 0 { + if err := tx.UpdateContractState(fcid, api.ContractStatePending); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + return nil + } + + return nil +} + func (s *chainSubscriber) isClosed() bool { select { case <-s.shutdownCtx.Done(): @@ -584,28 +727,6 @@ func (s *chainSubscriber) updateKnownContracts(fcid types.FileContractID, known s.knownContracts[fcid] = known } -func v1ContractUpdate(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) contractUpdate { - curr := &revision{ - revisionNumber: fce.FileContract.RevisionNumber, - fileSize: fce.FileContract.Filesize, - } - if rev != nil { - curr.revisionNumber = rev.FileContract.RevisionNumber - curr.fileSize = rev.FileContract.Filesize - } - if curr.revisionNumber == math.MaxUint64 && curr.fileSize == 0 { - resolved = true - valid = true - } - return contractUpdate{ - fcid: types.FileContractID(fce.ID), - prev: nil, - curr: curr, - resolved: resolved, - valid: valid, - } -} - func v2ContractUpdate(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) contractUpdate { curr := &revision{ revisionNumber: fce.V2FileContract.RevisionNumber, diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 88cfa5cdd..3ac61ffde 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2830,7 +2830,7 @@ func TestContractFundsReturnWhenHostOffline(t *testing.T) { tt.OK(hosts[0].Close()) // mine until the contract is expired - cluster.mineBlocks(types.VoidAddress, contract.WindowEnd-cs.BlockHeight+10) + cluster.mineBlocks(types.VoidAddress, contract.WindowEnd-cs.BlockHeight) expectedBalance := wallet.Confirmed.Add(contract.InitialRenterFunds).Sub(fee.Mul64(ibus.ContractResolutionTxnWeight)) cluster.tt.Retry(10, time.Second, func() error { From b5255983704d1348c8eef07e1aa6d700a2a9ea5b Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 15:51:39 +0100 Subject: [PATCH 03/37] add applyV2ContractUpdate and revertV2ContractUpdate --- internal/bus/chainsubscriber.go | 332 +++++++++++++++----------------- stores/sql/chain.go | 12 ++ stores/sql/database.go | 1 + stores/sql/mysql/chain.go | 4 + stores/sql/sqlite/chain.go | 4 + 5 files changed, 171 insertions(+), 182 deletions(-) diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index 9ef5f934e..48aa491c7 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -102,14 +102,6 @@ type ( revisionNumber uint64 fileSize uint64 } - - contractUpdate struct { - fcid types.FileContractID - prev *revision - curr *revision - resolved bool - valid bool - } ) // NewChainSubscriber creates a new chain subscriber that will sync with the @@ -224,26 +216,19 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply err = s.applyV1ContractUpdate(tx, cau.State.Index, fce, rev, resolved, valid) }) if err != nil { - return fmt.Errorf("failed to apply contract update: %w", err) + return fmt.Errorf("failed to apply v1 contract update: %w", err) } // v2 contracts - cus := make(map[types.FileContractID]contractUpdate) var revisedContracts []types.V2FileContractElement cau.ForEachV2FileContractElement(func(fce types.V2FileContractElement, _ bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { - if rev == nil { - revisedContracts = append(revisedContracts, fce) - } else { - revisedContracts = append(revisedContracts, *rev) // revised + if err != nil { + return } - cus[types.FileContractID(fce.ID)] = v2ContractUpdate(fce, rev, res) + err = s.applyV2ContractUpdate(tx, cau.State.Index, fce, rev, res) }) - - // updates - this updates the 'known' contracts too so we do this first - for _, cu := range cus { - if err := s.updateV2Contract(tx, cau.State.Index, cu.fcid, cu.prev, cu.curr, cu.resolved, cu.valid); err != nil { - return fmt.Errorf("failed to apply contract updates: %w", err) - } + if err != nil { + return fmt.Errorf("failed to apply v2 contract update: %w", err) } // new contracts - only consider the ones we are interested in @@ -294,22 +279,15 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve } // v2 contracts - var cus []contractUpdate var revertedContracts []types.V2FileContractElement cru.ForEachV2FileContractElement(func(fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { - if created { - revertedContracts = append(revertedContracts, fce) - } else if rev != nil { - revertedContracts = append(revertedContracts, *rev) + if err != nil { + return } - cus = append(cus, v2ContractUpdate(fce, rev, res)) + err = s.revertV2ContractUpdate(tx, fce, rev, res) }) - - // updates - this updates the 'known' contracts too so we do this first - for _, cu := range cus { - if err := s.updateV2Contract(tx, cru.State.Index, cu.fcid, cu.prev, cu.curr, cu.resolved, cu.valid); err != nil { - return fmt.Errorf("failed to revert v2 contract update: %w", err) - } + if err != nil { + return fmt.Errorf("failed to revert v2 contract update: %w", err) } // reverted contracts - only consider the ones that we are interested in @@ -457,10 +435,10 @@ func (s *chainSubscriber) broadcastExpiredFileContractResolutions(tx sql.ChainUp } } -func (s *chainSubscriber) updateV2Contract(tx sql.ChainUpdateTx, index types.ChainIndex, fcid types.FileContractID, prev, curr *revision, resolved, valid bool) error { - // sanity check at least one is not nil - if prev == nil && curr == nil { - return errors.New("both prev and curr revisions are nil") // developer error +func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index types.ChainIndex, fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) error { + fcid := fce.ID + if rev != nil { + fcid = rev.ID } // ignore unknown contracts @@ -475,77 +453,35 @@ func (s *chainSubscriber) updateV2Contract(tx sql.ChainUpdateTx, index types.Cha return nil } else if err != nil { return fmt.Errorf("failed to get contract state: %w", err) - } else { - s.updateKnownContracts(fcid, true) // update known contracts } - // define a helper function to update the contract state - updateState := func(update api.ContractState) (err error) { - if state != update { - err = tx.UpdateContractState(fcid, update) - if err == nil { - state = update - } - } - return + // update revision number and file size + revisionNumber := fce.FileContract.RevisionNumber + fileSize := fce.FileContract.Filesize + if rev != nil { + revisionNumber = rev.FileContract.RevisionNumber + fileSize = rev.FileContract.Filesize } - - // handle reverts - if prev != nil { - // update state from 'active' -> 'pending' - if curr == nil { - if err := updateState(api.ContractStatePending); err != nil { - return fmt.Errorf("failed to update contract state: %w", err) - } - } - - // reverted renewal: 'complete' -> 'active' - if curr != nil { - if err := tx.UpdateContractRevision(fcid, index.Height, prev.revisionNumber, prev.fileSize); err != nil { - return fmt.Errorf("failed to revert contract: %w", err) - } - if state == api.ContractStateComplete { - if err := updateState(api.ContractStateActive); err != nil { - return fmt.Errorf("failed to update contract state: %w", err) - } - s.logger.Infow("contract state changed: complete -> active", - "fcid", fcid, - "reason", "final revision reverted") - } - } - - // reverted storage proof: 'complete/failed' -> 'active' - if resolved { - if err := updateState(api.ContractStateActive); err != nil { - return fmt.Errorf("failed to update contract state: %w", err) - } - if valid { - s.logger.Infow("contract state changed: complete -> active", - "fcid", fcid, - "reason", "storage proof reverted") - } else { - s.logger.Infow("contract state changed: failed -> active", - "fcid", fcid, - "reason", "storage proof reverted") - } - } - - return nil + if err := tx.UpdateContractRevision(fcid, index.Height, revisionNumber, fileSize); err != nil { + return fmt.Errorf("failed to update contract %v: %w", fcid, err) } - // handle apply - if err := tx.UpdateContractRevision(fcid, index.Height, curr.revisionNumber, curr.fileSize); err != nil { - return fmt.Errorf("failed to update contract %v: %w", fcid, err) + // consider a contract resolved if it has a max revision number and zero + // file size + if rev != nil && rev.FileContract.RevisionNumber == math.MaxUint64 && rev.FileContract.Filesize == 0 { + resolved = true + valid = true } // update state from 'pending' -> 'active' if state == api.ContractStatePending || state == api.ContractStateUnknown { - if err := updateState(api.ContractStateActive); err != nil { + if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } s.logger.Infow("contract state changed: pending -> active", "fcid", fcid, "reason", "contract confirmed") + return nil } // storage proof: 'active' -> 'complete/failed' @@ -554,25 +490,85 @@ func (s *chainSubscriber) updateV2Contract(tx sql.ChainUpdateTx, index types.Cha return fmt.Errorf("failed to update contract proof height: %w", err) } if valid { - if err := updateState(api.ContractStateComplete); err != nil { + if err := tx.UpdateContractState(fcid, api.ContractStateComplete); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } s.logger.Infow("contract state changed: active -> complete", "fcid", fcid, "reason", "storage proof valid") + return nil } else { - if err := updateState(api.ContractStateFailed); err != nil { + if err := tx.UpdateContractState(fcid, api.ContractStateFailed); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } s.logger.Infow("contract state changed: active -> failed", "fcid", fcid, "reason", "storage proof missed") + return nil } } return nil } -func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index types.ChainIndex, fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) error { +func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) error { + fcid := fce.ID + if rev != nil { + fcid = rev.ID + } + + // ignore unknown contracts + if !s.isKnownContract(fcid) { + return nil + } + + // fetch contract state to see if contract is known + _, err := tx.ContractState(fcid) + if err != nil && utils.IsErr(err, api.ErrContractNotFound) { + s.updateKnownContracts(fcid, false) // ignore unknown contracts + return nil + } else if err != nil { + return fmt.Errorf("failed to get contract state: %w", err) + } + + // consider a contract resolved if it has a max revision number and zero + // file size + if rev != nil && rev.FileContract.RevisionNumber == math.MaxUint64 && rev.FileContract.Filesize == 0 { + resolved = true + valid = true + } + + // reverted storage proof: 'complete/failed' -> 'active' + if resolved { + if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + if valid { + s.logger.Infow("contract state changed: complete -> active", + "fcid", fcid, + "reason", "storage proof reverted") + } else { + s.logger.Infow("contract state changed: failed -> active", + "fcid", fcid, + "reason", "storage proof reverted") + } + return nil + } + + // update state from 'active' -> 'pending' + if rev == nil && fce.FileContract.RevisionNumber == 0 { + if err := tx.UpdateContractState(fcid, api.ContractStatePending); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + s.logger.Infow("contract state changed: active -> pending", + "fcid", fcid, + "reason", "contract was reverted") + return nil + } + + return nil +} + +func (s *chainSubscriber) applyV2ContractUpdate(tx sql.ChainUpdateTx, index types.ChainIndex, fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) error { fcid := fce.ID if rev != nil { fcid = rev.ID @@ -593,23 +589,16 @@ func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index type } // update revision number and file size - revisionNumber := fce.FileContract.RevisionNumber - fileSize := fce.FileContract.Filesize + revisionNumber := fce.V2FileContract.RevisionNumber + fileSize := fce.V2FileContract.Filesize if rev != nil { - revisionNumber = rev.FileContract.RevisionNumber - fileSize = rev.FileContract.Filesize + revisionNumber = rev.V2FileContract.RevisionNumber + fileSize = rev.V2FileContract.Filesize } if err := tx.UpdateContractRevision(fcid, index.Height, revisionNumber, fileSize); err != nil { return fmt.Errorf("failed to update contract %v: %w", fcid, err) } - // consider a contract resolved if it has a max revision number and zero - // file size - if rev != nil && rev.FileContract.RevisionNumber == math.MaxUint64 && rev.FileContract.Filesize == 0 { - resolved = true - valid = true - } - // update state from 'pending' -> 'active' if state == api.ContractStatePending || state == api.ContractStateUnknown { if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { @@ -621,33 +610,51 @@ func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index type return nil } - // storage proof: 'active' -> 'complete/failed' - if resolved { + // resolution: 'active' -> 'complete/failed' + if res != nil { + var newState api.ContractState + var reason string + switch res.(type) { + case *types.V2FileContractRenewal: + newState = api.ContractStateComplete + reason = "renewal" + + // link the renewed contract to the new one, this should not be + // necessary if the renewal was successfully but there is a slim + // chance that it's not when the renewal was interrupted + if err := tx.RecordContractRenewal(fcid, fcid.V2RenewalID()); err != nil { + return fmt.Errorf("failed to record contract renewal: %w", err) + } + + case *types.V2StorageProof: + newState = api.ContractStateComplete + reason = "storage proof" + case *types.V2FileContractExpiration: + newState = api.ContractStateFailed + reason = "expiration" + default: + panic("unknown resolution type") // developer error + } + + // record height of encountering the resolution if err := tx.UpdateContractProofHeight(fcid, index.Height); err != nil { return fmt.Errorf("failed to update contract proof height: %w", err) } - if valid { - if err := tx.UpdateContractState(fcid, api.ContractStateComplete); err != nil { - return fmt.Errorf("failed to update contract state: %w", err) - } - s.logger.Infow("contract state changed: active -> complete", - "fcid", fcid, - "reason", "storage proof valid") - return nil - } else { - if err := tx.UpdateContractState(fcid, api.ContractStateFailed); err != nil { - return fmt.Errorf("failed to update contract state: %w", err) - } - s.logger.Infow("contract state changed: active -> failed", - "fcid", fcid, - "reason", "storage proof missed") - return nil + + // record new state + if err := tx.UpdateContractState(fcid, api.ContractStateComplete); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) } + + s.logger.Infow(fmt.Sprintf("contract state changed: active -> %s", newState), + "fcid", fcid, + "reason", reason) + return nil } return nil } -func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) error { +func (s *chainSubscriber) revertV2ContractUpdate(tx sql.ChainUpdateTx, fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) error { fcid := fce.ID if rev != nil { fcid = rev.ID @@ -659,7 +666,7 @@ func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types } // fetch contract state to see if contract is known - _, err := tx.ContractState(fcid) + state, err := tx.ContractState(fcid) if err != nil && utils.IsErr(err, api.ErrContractNotFound) { s.updateKnownContracts(fcid, false) // ignore unknown contracts return nil @@ -667,38 +674,34 @@ func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types return fmt.Errorf("failed to get contract state: %w", err) } - // consider a contract resolved if it has a max revision number and zero - // file size - if rev != nil && rev.FileContract.RevisionNumber == math.MaxUint64 && rev.FileContract.Filesize == 0 { - resolved = true - valid = true - } + // resolution: 'complete/failed' -> 'active' + if res != nil { + // reset proof height + if err := tx.UpdateContractProofHeight(fcid, 0); err != nil { + return fmt.Errorf("failed to update contract proof height: %w", err) + } - // reverted storage proof: 'complete/failed' -> 'active' - if resolved { + // record new state if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } - if valid { - s.logger.Infow("contract state changed: complete -> active", - "fcid", fcid, - "reason", "storage proof reverted") - } else { - s.logger.Infow("contract state changed: failed -> active", - "fcid", fcid, - "reason", "storage proof reverted") - } + + s.logger.Infow(fmt.Sprintf("contract state changed: %s -> active", state), + "fcid", fcid, + "reason", "resolution was reverted") return nil } // update state from 'active' -> 'pending' - if rev == nil && fce.FileContract.RevisionNumber == 0 { + if rev == nil && fce.V2FileContract.RevisionNumber == 0 { if err := tx.UpdateContractState(fcid, api.ContractStatePending); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } + s.logger.Infow("contract state changed: active -> pending", + "fcid", fcid, + "reason", "contract was reverted") return nil } - return nil } @@ -726,38 +729,3 @@ func (s *chainSubscriber) updateKnownContracts(fcid types.FileContractID, known defer s.mu.Unlock() s.knownContracts[fcid] = known } - -func v2ContractUpdate(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) contractUpdate { - curr := &revision{ - revisionNumber: fce.V2FileContract.RevisionNumber, - fileSize: fce.V2FileContract.Filesize, - } - if rev != nil { - curr.revisionNumber = rev.V2FileContract.RevisionNumber - curr.fileSize = rev.V2FileContract.Filesize - } - - var resolved, valid bool - if res != nil { - resolved = true - switch res.(type) { - case *types.V2FileContractRenewal: - // hack to make sure v2 contracts also appear with a max revision - // number after being renewed - curr.revisionNumber = math.MaxUint64 - valid = true - case *types.V2StorageProof: - valid = true - case *types.V2FileContractExpiration: - valid = false - } - } - - return contractUpdate{ - fcid: types.FileContractID(fce.ID), - prev: nil, - curr: curr, - resolved: resolved, - valid: valid, - } -} diff --git a/stores/sql/chain.go b/stores/sql/chain.go index db38a7642..9b3808f6b 100644 --- a/stores/sql/chain.go +++ b/stores/sql/chain.go @@ -204,6 +204,18 @@ func FileContractElement(ctx context.Context, tx sql.Tx, fcid types.FileContract }, nil } +func RecordContractRenewal(ctx context.Context, tx sql.Tx, old, new types.FileContractID) error { + _, err := tx.Exec(ctx, "UPDATE contracts SET contracts.renewed_to = ? WHERE contracts.fcid = ?", FileContractID(new), FileContractID(old)) + if err != nil { + return fmt.Errorf("failed to update renewed_to of old contract: %w", err) + } + _, err = tx.Exec(ctx, "UPDATE contracts SET contracts.renewed_from = ? WHERE contracts.fcid = ?", FileContractID(old), FileContractID(new)) + if err != nil { + return fmt.Errorf("failed to update renewed_to of new contract: %w", err) + } + return nil +} + func PruneFileContractElements(ctx context.Context, tx sql.Tx, threshold uint64) error { _, err := tx.Exec(ctx, ` DELETE FROM contract_elements diff --git a/stores/sql/database.go b/stores/sql/database.go index 35bf0618d..1490e7957 100644 --- a/stores/sql/database.go +++ b/stores/sql/database.go @@ -22,6 +22,7 @@ type ( ExpiredFileContractElements(bh uint64) ([]types.V2FileContractElement, error) FileContractElement(fcid types.FileContractID) (types.V2FileContractElement, error) PruneFileContractElements(threshold uint64) error + RecordContractRenewal(old, new types.FileContractID) error UpdateFileContractElements([]types.V2FileContractElement) error UpdateChainIndex(index types.ChainIndex) error UpdateFileContractElementProofs(updater wallet.ProofUpdater) error diff --git a/stores/sql/mysql/chain.go b/stores/sql/mysql/chain.go index bed8ae9b2..1c1702dd1 100644 --- a/stores/sql/mysql/chain.go +++ b/stores/sql/mysql/chain.go @@ -214,6 +214,10 @@ func (c chainUpdateTx) PruneFileContractElements(threshold uint64) error { return ssql.PruneFileContractElements(c.ctx, c.tx, threshold) } +func (c chainUpdateTx) RecordContractRenewal(old, new types.FileContractID) error { + return ssql.RecordContractRenewal(c.ctx, c.tx, old, new) +} + func (c chainUpdateTx) UpdateFileContractElements(fces []types.V2FileContractElement) error { contractIDStmt, err := c.tx.Prepare(c.ctx, "SELECT c.id FROM contracts c WHERE c.fcid = ?") if err != nil { diff --git a/stores/sql/sqlite/chain.go b/stores/sql/sqlite/chain.go index e806b8a27..ee50ad664 100644 --- a/stores/sql/sqlite/chain.go +++ b/stores/sql/sqlite/chain.go @@ -215,6 +215,10 @@ func (c chainUpdateTx) PruneFileContractElements(threshold uint64) error { return ssql.PruneFileContractElements(c.ctx, c.tx, threshold) } +func (c chainUpdateTx) RecordContractRenewal(old, new types.FileContractID) error { + return ssql.RecordContractRenewal(c.ctx, c.tx, old, new) +} + func (c chainUpdateTx) UpdateFileContractElements(fces []types.V2FileContractElement) error { contractIDStmt, err := c.tx.Prepare(c.ctx, "SELECT c.id FROM contracts c WHERE c.fcid = ?") if err != nil { From f218019fdf700802040524d5eaeb5303573cab3b Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 16:12:31 +0100 Subject: [PATCH 04/37] move known contracts into txn --- internal/bus/chainsubscriber.go | 117 ++++++++++++-------------------- stores/sql/chain.go | 16 ++++- stores/sql/database.go | 1 + stores/sql/mysql/chain.go | 28 ++++++-- stores/sql/sqlite/chain.go | 28 ++++++-- 5 files changed, 105 insertions(+), 85 deletions(-) diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index 48aa491c7..1e18685e8 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -13,7 +13,6 @@ import ( rhp4 "go.sia.tech/coreutils/rhp/v4" "go.sia.tech/coreutils/wallet" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" @@ -91,9 +90,8 @@ type ( syncSig chan struct{} wg sync.WaitGroup - mu sync.Mutex - knownContracts map[types.FileContractID]bool - unsubscribeFn func() + mu sync.Mutex + unsubscribeFn func() } ) @@ -123,8 +121,6 @@ func NewChainSubscriber(whm WebhookManager, cm ChainManager, cs ChainStore, s Sy shutdownCtx: ctx, shutdownCtxCancel: cancel, syncSig: make(chan struct{}, 1), - - knownContracts: make(map[types.FileContractID]bool), } // start the subscriber @@ -212,6 +208,11 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply cau.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { if err != nil { return + } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { + err = lookupErr + return + } else if !known { + return // only consider known contracts } err = s.applyV1ContractUpdate(tx, cau.State.Index, fce, rev, resolved, valid) }) @@ -224,6 +225,16 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply cau.ForEachV2FileContractElement(func(fce types.V2FileContractElement, _ bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { if err != nil { return + } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { + err = lookupErr + return + } else if !known { + return // only consider known contracts + } + if rev == nil { + revisedContracts = append(revisedContracts, fce) + } else { + revisedContracts = append(revisedContracts, *rev) } err = s.applyV2ContractUpdate(tx, cau.State.Index, fce, rev, res) }) @@ -231,18 +242,12 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply return fmt.Errorf("failed to apply v2 contract update: %w", err) } - // new contracts - only consider the ones we are interested in - filtered := revisedContracts[:0] - for _, fce := range revisedContracts { - if s.isKnownContract(fce.ID) { - filtered = append(filtered, fce) - } - } - if err := tx.UpdateFileContractElements(filtered); err != nil { + // update revised contracts + if err := tx.UpdateFileContractElements(revisedContracts); err != nil { return fmt.Errorf("failed to insert v2 file contract elements: %w", err) } - // contract proofs + // update contract proofs if err := tx.UpdateFileContractElementProofs(cau); err != nil { return fmt.Errorf("failed to update file contract element proofs: %w", err) } @@ -271,6 +276,11 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve cru.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { if err != nil { return + } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { + err = lookupErr + return + } else if !known { + return // only consider known contracts } err = s.revertV1ContractUpdate(tx, fce, rev, resolved, valid) }) @@ -283,6 +293,16 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve cru.ForEachV2FileContractElement(func(fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { if err != nil { return + } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { + err = lookupErr + return + } else if !known { + return // only consider known contracts + } + if rev == nil { + revertedContracts = append(revertedContracts, fce) + } else { + revertedContracts = append(revertedContracts, *rev) } err = s.revertV2ContractUpdate(tx, fce, rev, res) }) @@ -290,18 +310,12 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve return fmt.Errorf("failed to revert v2 contract update: %w", err) } - // reverted contracts - only consider the ones that we are interested in - filtered := revertedContracts[:0] - for _, fce := range revertedContracts { - if s.isKnownContract(fce.ID) { - filtered = append(filtered, fce) - } - } - if err := tx.UpdateFileContractElements(filtered); err != nil { + // update reverted contracts + if err := tx.UpdateFileContractElements(revertedContracts); err != nil { return fmt.Errorf("failed to remove v2 file contract elements: %w", err) } - // contract proofs + // update contract proofs if err := tx.UpdateFileContractElementProofs(cru); err != nil { return fmt.Errorf("failed to update file contract element proofs: %w", err) } @@ -441,17 +455,9 @@ func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index type fcid = rev.ID } - // ignore unknown contracts - if !s.isKnownContract(fcid) { - return nil - } - // fetch contract state state, err := tx.ContractState(fcid) - if err != nil && utils.IsErr(err, api.ErrContractNotFound) { - s.updateKnownContracts(fcid, false) // ignore unknown contracts - return nil - } else if err != nil { + if err != nil { return fmt.Errorf("failed to get contract state: %w", err) } @@ -516,17 +522,9 @@ func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types fcid = rev.ID } - // ignore unknown contracts - if !s.isKnownContract(fcid) { - return nil - } - // fetch contract state to see if contract is known _, err := tx.ContractState(fcid) - if err != nil && utils.IsErr(err, api.ErrContractNotFound) { - s.updateKnownContracts(fcid, false) // ignore unknown contracts - return nil - } else if err != nil { + if err != nil { return fmt.Errorf("failed to get contract state: %w", err) } @@ -574,17 +572,9 @@ func (s *chainSubscriber) applyV2ContractUpdate(tx sql.ChainUpdateTx, index type fcid = rev.ID } - // ignore unknown contracts - if !s.isKnownContract(fcid) { - return nil - } - // fetch contract state state, err := tx.ContractState(fcid) - if err != nil && utils.IsErr(err, api.ErrContractNotFound) { - s.updateKnownContracts(fcid, false) // ignore unknown contracts - return nil - } else if err != nil { + if err != nil { return fmt.Errorf("failed to get contract state: %w", err) } @@ -661,16 +651,15 @@ func (s *chainSubscriber) revertV2ContractUpdate(tx sql.ChainUpdateTx, fce types } // ignore unknown contracts - if !s.isKnownContract(fcid) { + if known, err := tx.IsKnownContract(fcid); err != nil { + return err + } else if !known { return nil } // fetch contract state to see if contract is known state, err := tx.ContractState(fcid) - if err != nil && utils.IsErr(err, api.ErrContractNotFound) { - s.updateKnownContracts(fcid, false) // ignore unknown contracts - return nil - } else if err != nil { + if err != nil { return fmt.Errorf("failed to get contract state: %w", err) } @@ -713,19 +702,3 @@ func (s *chainSubscriber) isClosed() bool { } return false } - -func (s *chainSubscriber) isKnownContract(fcid types.FileContractID) bool { - s.mu.Lock() - defer s.mu.Unlock() - known, ok := s.knownContracts[fcid] - if !ok { - return true // assume known - } - return known -} - -func (s *chainSubscriber) updateKnownContracts(fcid types.FileContractID, known bool) { - s.mu.Lock() - defer s.mu.Unlock() - s.knownContracts[fcid] = known -} diff --git a/stores/sql/chain.go b/stores/sql/chain.go index 9b3808f6b..990c2c4cf 100644 --- a/stores/sql/chain.go +++ b/stores/sql/chain.go @@ -204,12 +204,22 @@ func FileContractElement(ctx context.Context, tx sql.Tx, fcid types.FileContract }, nil } -func RecordContractRenewal(ctx context.Context, tx sql.Tx, old, new types.FileContractID) error { - _, err := tx.Exec(ctx, "UPDATE contracts SET contracts.renewed_to = ? WHERE contracts.fcid = ?", FileContractID(new), FileContractID(old)) +func IsKnownContract(ctx context.Context, tx sql.Tx, fcid types.FileContractID) (known bool, _ error) { + err := tx.QueryRow(ctx, "SELECT 1 FROM contracts WHERE fcid = ?", FileContractID(fcid)).Scan(&known) + if errors.Is(err, dsql.ErrNoRows) { + return false, nil + } else if err != nil { + return false, err + } + return known, nil +} + +func RecordContractRenewal(ctx context.Context, tx sql.Tx, oldFCID, newFCID types.FileContractID) error { + _, err := tx.Exec(ctx, "UPDATE contracts SET contracts.renewed_to = ? WHERE contracts.fcid = ?", FileContractID(newFCID), FileContractID(oldFCID)) if err != nil { return fmt.Errorf("failed to update renewed_to of old contract: %w", err) } - _, err = tx.Exec(ctx, "UPDATE contracts SET contracts.renewed_from = ? WHERE contracts.fcid = ?", FileContractID(old), FileContractID(new)) + _, err = tx.Exec(ctx, "UPDATE contracts SET contracts.renewed_from = ? WHERE contracts.fcid = ?", FileContractID(oldFCID), FileContractID(newFCID)) if err != nil { return fmt.Errorf("failed to update renewed_to of new contract: %w", err) } diff --git a/stores/sql/database.go b/stores/sql/database.go index 1490e7957..1c5d5935a 100644 --- a/stores/sql/database.go +++ b/stores/sql/database.go @@ -21,6 +21,7 @@ type ( ContractState(fcid types.FileContractID) (api.ContractState, error) ExpiredFileContractElements(bh uint64) ([]types.V2FileContractElement, error) FileContractElement(fcid types.FileContractID) (types.V2FileContractElement, error) + IsKnownContract(fcid types.FileContractID) (bool, error) PruneFileContractElements(threshold uint64) error RecordContractRenewal(old, new types.FileContractID) error UpdateFileContractElements([]types.V2FileContractElement) error diff --git a/stores/sql/mysql/chain.go b/stores/sql/mysql/chain.go index 1c1702dd1..1d2683075 100644 --- a/stores/sql/mysql/chain.go +++ b/stores/sql/mysql/chain.go @@ -24,9 +24,10 @@ var ( ) type chainUpdateTx struct { - ctx context.Context - tx isql.Tx - l *zap.SugaredLogger + ctx context.Context + tx isql.Tx + l *zap.SugaredLogger + known map[types.FileContractID]bool // map to prevent rare duplicate selects } func (c chainUpdateTx) WalletApplyIndex(index types.ChainIndex, created, spent []types.SiacoinElement, events []wallet.Event, timestamp time.Time) error { @@ -210,12 +211,29 @@ func (c chainUpdateTx) FileContractElement(fcid types.FileContractID) (types.V2F return ssql.FileContractElement(c.ctx, c.tx, fcid) } +func (c chainUpdateTx) IsKnownContract(fcid types.FileContractID) (bool, error) { + if c.known == nil { + c.known = make(map[types.FileContractID]bool) + } + + if relevant, ok := c.known[fcid]; ok { + return relevant, nil + } + + known, err := ssql.IsKnownContract(c.ctx, c.tx, fcid) + if err != nil { + return false, err + } + c.known[fcid] = known + return known, nil +} + func (c chainUpdateTx) PruneFileContractElements(threshold uint64) error { return ssql.PruneFileContractElements(c.ctx, c.tx, threshold) } -func (c chainUpdateTx) RecordContractRenewal(old, new types.FileContractID) error { - return ssql.RecordContractRenewal(c.ctx, c.tx, old, new) +func (c chainUpdateTx) RecordContractRenewal(oldFCID, newFCID types.FileContractID) error { + return ssql.RecordContractRenewal(c.ctx, c.tx, oldFCID, newFCID) } func (c chainUpdateTx) UpdateFileContractElements(fces []types.V2FileContractElement) error { diff --git a/stores/sql/sqlite/chain.go b/stores/sql/sqlite/chain.go index ee50ad664..f3abf5353 100644 --- a/stores/sql/sqlite/chain.go +++ b/stores/sql/sqlite/chain.go @@ -25,9 +25,10 @@ var ( ) type chainUpdateTx struct { - ctx context.Context - tx isql.Tx - l *zap.SugaredLogger + ctx context.Context + tx isql.Tx + l *zap.SugaredLogger + known map[types.FileContractID]bool // map to prevent rare duplicate selects } func (c chainUpdateTx) WalletApplyIndex(index types.ChainIndex, created, spent []types.SiacoinElement, events []wallet.Event, timestamp time.Time) error { @@ -211,12 +212,29 @@ func (c chainUpdateTx) FileContractElement(fcid types.FileContractID) (types.V2F return ssql.FileContractElement(c.ctx, c.tx, fcid) } +func (c chainUpdateTx) IsKnownContract(fcid types.FileContractID) (bool, error) { + if c.known == nil { + c.known = make(map[types.FileContractID]bool) + } + + if relevant, ok := c.known[fcid]; ok { + return relevant, nil + } + + known, err := ssql.IsKnownContract(c.ctx, c.tx, fcid) + if err != nil { + return false, err + } + c.known[fcid] = known + return known, nil +} + func (c chainUpdateTx) PruneFileContractElements(threshold uint64) error { return ssql.PruneFileContractElements(c.ctx, c.tx, threshold) } -func (c chainUpdateTx) RecordContractRenewal(old, new types.FileContractID) error { - return ssql.RecordContractRenewal(c.ctx, c.tx, old, new) +func (c chainUpdateTx) RecordContractRenewal(oldFCID, newFCID types.FileContractID) error { + return ssql.RecordContractRenewal(c.ctx, c.tx, oldFCID, newFCID) } func (c chainUpdateTx) UpdateFileContractElements(fces []types.V2FileContractElement) error { From 1108bc2bf1d843f4f543c2eff81c1e5f55479199 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 16:36:23 +0100 Subject: [PATCH 05/37] fix lint --- internal/bus/chainsubscriber.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index 1e18685e8..f9530d86b 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -90,18 +90,10 @@ type ( syncSig chan struct{} wg sync.WaitGroup - mu sync.Mutex unsubscribeFn func() } ) -type ( - revision struct { - revisionNumber uint64 - fileSize uint64 - } -) - // NewChainSubscriber creates a new chain subscriber that will sync with the // given chain manager and chain store. The returned subscriber is already // running and can be stopped by calling Shutdown. From f7700ceaa87a5083f1e319b082ea2b3273ca12e7 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 12 Dec 2024 17:24:52 +0100 Subject: [PATCH 06/37] don't broadcast transactions after formation and renewal --- bus/bus.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/bus/bus.go b/bus/bus.go index dbb3ebaa0..56c846c18 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -687,9 +687,6 @@ func (b *Bus) formContract(ctx context.Context, hostSettings rhpv2.HostSettings, return api.ContractMetadata{}, fmt.Errorf("couldn't add transaction set to the pool: %w", err) } - // broadcast the transaction set - go b.s.BroadcastTransactionSet(txnSet) - return api.ContractMetadata{ ID: contract.ID(), HostKey: contract.HostKey(), @@ -727,9 +724,6 @@ func (b *Bus) formContractV2(ctx context.Context, hk types.PublicKey, hostIP str return api.ContractMetadata{}, fmt.Errorf("failed to add v2 transaction set to the pool: %w", err) } - // broadcast the transaction set - go b.s.BroadcastV2TransactionSet(res.FormationSet.Basis, res.FormationSet.Transactions) - contract := res.Contract return api.ContractMetadata{ ID: contract.ID, @@ -805,14 +799,11 @@ func (b *Bus) renewContractV1(ctx context.Context, cs consensus.State, gp api.Go // renew contract gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState) prepareRenew := b.prepareRenew(cs, rev, hs.Address, b.w.Address(), renterFunds, minNewCollateral, endHeight, expectedNewStorage) - newRevision, txnSet, contractPrice, fundAmount, err := b.rhp3Client.Renew(ctx, gc, rev, renterKey, c.HostKey, hs.SiamuxAddr(), prepareRenew, b.w.SignTransaction) + newRevision, _, contractPrice, fundAmount, err := b.rhp3Client.Renew(ctx, gc, rev, renterKey, c.HostKey, hs.SiamuxAddr(), prepareRenew, b.w.SignTransaction) if err != nil { return api.ContractMetadata{}, err } - // broadcast the transaction set - b.s.BroadcastTransactionSet(txnSet) - return api.ContractMetadata{ ID: newRevision.ID(), HostKey: newRevision.HostKey(), @@ -863,7 +854,6 @@ func (b *Bus) renewContractV2(ctx context.Context, cs consensus.State, h api.Hos } var contract cRhp4.ContractRevision - var txnSet cRhp4.TransactionSet if c.EndHeight() == endHeight { // when refreshing, the 'collateral' is added on top of the existing // collateral so we account for that by subtracting the rolled over @@ -880,7 +870,6 @@ func (b *Bus) renewContractV2(ctx context.Context, cs consensus.State, h api.Hos Collateral: collateral, }) contract = res.Contract - txnSet = res.RenewalSet } else { var res cRhp4.RPCRenewContractResult res, err = b.rhp4Client.RenewContract(ctx, h.PublicKey, h.V2SiamuxAddr(), b.cm, signer, cs, settings.Prices, rev, rhpv4.RPCRenewContractParams{ @@ -890,15 +879,11 @@ func (b *Bus) renewContractV2(ctx context.Context, cs consensus.State, h api.Hos ProofHeight: endHeight, }) contract = res.Contract - txnSet = res.RenewalSet } if err != nil { return api.ContractMetadata{}, err } - // broadcast the transaction set - b.s.BroadcastV2TransactionSet(txnSet.Basis, txnSet.Transactions) - return api.ContractMetadata{ ID: contract.ID, HostKey: h.PublicKey, From fd48afcebc396315e04eb259ad736d8e87f23294 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 11:21:49 +0100 Subject: [PATCH 07/37] make use of 'created' flag --- internal/bus/chainsubscriber.go | 136 ++++++++++++++++---------------- 1 file changed, 66 insertions(+), 70 deletions(-) diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index f9530d86b..2d2fd015d 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -197,7 +197,7 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply // v1 contracts var err error - cau.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { + cau.ForEachFileContractElement(func(fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, valid bool) { if err != nil { return } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { @@ -206,7 +206,7 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply } else if !known { return // only consider known contracts } - err = s.applyV1ContractUpdate(tx, cau.State.Index, fce, rev, resolved, valid) + err = s.applyV1ContractUpdate(tx, cau.State.Index, fce, created, rev, resolved, valid) }) if err != nil { return fmt.Errorf("failed to apply v1 contract update: %w", err) @@ -214,7 +214,7 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply // v2 contracts var revisedContracts []types.V2FileContractElement - cau.ForEachV2FileContractElement(func(fce types.V2FileContractElement, _ bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { + cau.ForEachV2FileContractElement(func(fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { if err != nil { return } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { @@ -228,7 +228,7 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply } else { revisedContracts = append(revisedContracts, *rev) } - err = s.applyV2ContractUpdate(tx, cau.State.Index, fce, rev, res) + err = s.applyV2ContractUpdate(tx, cau.State.Index, fce, created, rev, res) }) if err != nil { return fmt.Errorf("failed to apply v2 contract update: %w", err) @@ -265,7 +265,7 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve // v1 contracts var err error - cru.ForEachFileContractElement(func(fce types.FileContractElement, _ bool, rev *types.FileContractElement, resolved, valid bool) { + cru.ForEachFileContractElement(func(fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, valid bool) { if err != nil { return } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { @@ -274,7 +274,7 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve } else if !known { return // only consider known contracts } - err = s.revertV1ContractUpdate(tx, fce, rev, resolved, valid) + err = s.revertV1ContractUpdate(tx, fce, created, rev, resolved, valid) }) if err != nil { return fmt.Errorf("failed to revert v1 contract update: %w", err) @@ -296,7 +296,7 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve } else { revertedContracts = append(revertedContracts, *rev) } - err = s.revertV2ContractUpdate(tx, fce, rev, res) + err = s.revertV2ContractUpdate(tx, fce, created, rev, res) }) if err != nil { return fmt.Errorf("failed to revert v2 contract update: %w", err) @@ -441,7 +441,7 @@ func (s *chainSubscriber) broadcastExpiredFileContractResolutions(tx sql.ChainUp } } -func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index types.ChainIndex, fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) error { +func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index types.ChainIndex, fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, valid bool) error { fcid := fce.ID if rev != nil { fcid = rev.ID @@ -471,18 +471,7 @@ func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index type valid = true } - // update state from 'pending' -> 'active' - if state == api.ContractStatePending || state == api.ContractStateUnknown { - if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { - return fmt.Errorf("failed to update contract state: %w", err) - } - s.logger.Infow("contract state changed: pending -> active", - "fcid", fcid, - "reason", "contract confirmed") - return nil - } - - // storage proof: 'active' -> 'complete/failed' + // contract was resolved via proof or renewal -> 'complete/failed' if resolved { if err := tx.UpdateContractProofHeight(fcid, index.Height); err != nil { return fmt.Errorf("failed to update contract proof height: %w", err) @@ -491,31 +480,42 @@ func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index type if err := tx.UpdateContractState(fcid, api.ContractStateComplete); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } - s.logger.Infow("contract state changed: active -> complete", + s.logger.Infow(fmt.Sprintf("contract state changed: %s -> failed", state), "fcid", fcid, "reason", "storage proof valid") - return nil } else { if err := tx.UpdateContractState(fcid, api.ContractStateFailed); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } - s.logger.Infow("contract state changed: active -> failed", + s.logger.Infow(fmt.Sprintf("contract state changed: %s -> failed", state), "fcid", fcid, "reason", "storage proof missed") - return nil } + return nil + } + + // contract was created -> 'active' + if created { + if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + s.logger.Infow(fmt.Sprintf("contract state changed: %s -> active", state), + "fcid", fcid, + "reason", "contract confirmed") + return nil } + return nil } -func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) error { +func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, valid bool) error { fcid := fce.ID if rev != nil { fcid = rev.ID } // fetch contract state to see if contract is known - _, err := tx.ContractState(fcid) + state, err := tx.ContractState(fcid) if err != nil { return fmt.Errorf("failed to get contract state: %w", err) } @@ -527,38 +527,32 @@ func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types valid = true } - // reverted storage proof: 'complete/failed' -> 'active' - if resolved { - if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { + // contract was reverted -> 'pending' + if created { + if err := tx.UpdateContractState(fcid, api.ContractStatePending); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } - if valid { - s.logger.Infow("contract state changed: complete -> active", - "fcid", fcid, - "reason", "storage proof reverted") - } else { - s.logger.Infow("contract state changed: failed -> active", - "fcid", fcid, - "reason", "storage proof reverted") - } + s.logger.Infow(fmt.Sprintf("contract state changed: %s -> active", state), + "fcid", fcid, + "reason", "contract was reverted") return nil } - // update state from 'active' -> 'pending' - if rev == nil && fce.FileContract.RevisionNumber == 0 { - if err := tx.UpdateContractState(fcid, api.ContractStatePending); err != nil { + // reverted storage proof -> 'active' + if resolved { + if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } - s.logger.Infow("contract state changed: active -> pending", + s.logger.Infow(fmt.Sprintf("contract state changed: %s -> active", state), "fcid", fcid, - "reason", "contract was reverted") + "reason", "storage proof reverted") return nil } return nil } -func (s *chainSubscriber) applyV2ContractUpdate(tx sql.ChainUpdateTx, index types.ChainIndex, fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) error { +func (s *chainSubscriber) applyV2ContractUpdate(tx sql.ChainUpdateTx, index types.ChainIndex, fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) error { fcid := fce.ID if rev != nil { fcid = rev.ID @@ -581,18 +575,7 @@ func (s *chainSubscriber) applyV2ContractUpdate(tx sql.ChainUpdateTx, index type return fmt.Errorf("failed to update contract %v: %w", fcid, err) } - // update state from 'pending' -> 'active' - if state == api.ContractStatePending || state == api.ContractStateUnknown { - if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { - return fmt.Errorf("failed to update contract state: %w", err) - } - s.logger.Infow("contract state changed: pending -> active", - "fcid", fcid, - "reason", "contract confirmed") - return nil - } - - // resolution: 'active' -> 'complete/failed' + // resolution -> 'complete/failed' if res != nil { var newState api.ContractState var reason string @@ -624,19 +607,31 @@ func (s *chainSubscriber) applyV2ContractUpdate(tx sql.ChainUpdateTx, index type } // record new state - if err := tx.UpdateContractState(fcid, api.ContractStateComplete); err != nil { + if err := tx.UpdateContractState(fcid, newState); err != nil { return fmt.Errorf("failed to update contract state: %w", err) } - s.logger.Infow(fmt.Sprintf("contract state changed: active -> %s", newState), + s.logger.Infow(fmt.Sprintf("contract state changed: %s -> %s", state, newState), "fcid", fcid, "reason", reason) return nil } + + // contract was created -> 'active' + if created { + if err := tx.UpdateContractState(fcid, api.ContractStateActive); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + s.logger.Infow(fmt.Sprintf("contract state changed: %s -> active", state), + "fcid", fcid, + "reason", "contract confirmed") + return nil + } + return nil } -func (s *chainSubscriber) revertV2ContractUpdate(tx sql.ChainUpdateTx, fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) error { +func (s *chainSubscriber) revertV2ContractUpdate(tx sql.ChainUpdateTx, fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) error { fcid := fce.ID if rev != nil { fcid = rev.ID @@ -655,7 +650,18 @@ func (s *chainSubscriber) revertV2ContractUpdate(tx sql.ChainUpdateTx, fce types return fmt.Errorf("failed to get contract state: %w", err) } - // resolution: 'complete/failed' -> 'active' + // contract was reverted -> 'pending' + if created { + if err := tx.UpdateContractState(fcid, api.ContractStatePending); err != nil { + return fmt.Errorf("failed to update contract state: %w", err) + } + s.logger.Infow(fmt.Sprintf("contract state changed: %s -> active", state), + "fcid", fcid, + "reason", "contract was reverted") + return nil + } + + // reverted resolution -> 'active' if res != nil { // reset proof height if err := tx.UpdateContractProofHeight(fcid, 0); err != nil { @@ -673,16 +679,6 @@ func (s *chainSubscriber) revertV2ContractUpdate(tx sql.ChainUpdateTx, fce types return nil } - // update state from 'active' -> 'pending' - if rev == nil && fce.V2FileContract.RevisionNumber == 0 { - if err := tx.UpdateContractState(fcid, api.ContractStatePending); err != nil { - return fmt.Errorf("failed to update contract state: %w", err) - } - s.logger.Infow("contract state changed: active -> pending", - "fcid", fcid, - "reason", "contract was reverted") - return nil - } return nil } From e671991d2c8261d1fc0cfc0ad0ef8ca30fb10f99 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 11:28:04 +0100 Subject: [PATCH 08/37] fix lint --- internal/bus/chainsubscriber.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index 2d2fd015d..9445e2f19 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -265,7 +265,7 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve // v1 contracts var err error - cru.ForEachFileContractElement(func(fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, valid bool) { + cru.ForEachFileContractElement(func(fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, _ bool) { if err != nil { return } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { @@ -274,7 +274,7 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve } else if !known { return // only consider known contracts } - err = s.revertV1ContractUpdate(tx, fce, created, rev, resolved, valid) + err = s.revertV1ContractUpdate(tx, fce, created, rev, resolved) }) if err != nil { return fmt.Errorf("failed to revert v1 contract update: %w", err) @@ -508,7 +508,7 @@ func (s *chainSubscriber) applyV1ContractUpdate(tx sql.ChainUpdateTx, index type return nil } -func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, valid bool) error { +func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved bool) error { fcid := fce.ID if rev != nil { fcid = rev.ID @@ -524,7 +524,6 @@ func (s *chainSubscriber) revertV1ContractUpdate(tx sql.ChainUpdateTx, fce types // file size if rev != nil && rev.FileContract.RevisionNumber == math.MaxUint64 && rev.FileContract.Filesize == 0 { resolved = true - valid = true } // contract was reverted -> 'pending' From c20ce9228ac734c4f6285201593b5b82dbe30735 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 14:11:38 +0100 Subject: [PATCH 09/37] extend test to check if number of contracts is right and if contracts are on right hosts --- autopilot/contractor/contractor.go | 16 ++++++---- autopilot/contractor/hostfilter.go | 7 ++++- bus/bus.go | 7 +++++ bus/routes.go | 7 +++++ go.mod | 2 +- go.sum | 4 +-- internal/test/e2e/cluster.go | 2 +- internal/test/e2e/cluster_test.go | 48 +++++++++++++++++++++++++----- 8 files changed, 76 insertions(+), 17 deletions(-) diff --git a/autopilot/contractor/contractor.go b/autopilot/contractor/contractor.go index e8c1dbd26..023f5d103 100644 --- a/autopilot/contractor/contractor.go +++ b/autopilot/contractor/contractor.go @@ -463,7 +463,9 @@ func activeContracts(ctx context.Context, bus Bus, logger *zap.SugaredLogger) ([ // fetch active contracts logger.Info("fetching active contracts") start := time.Now() - metadatas, err := bus.Contracts(ctx, api.ContractsOpts{FilterMode: api.ContractFilterModeActive}) + metadatas, err := bus.Contracts(ctx, api.ContractsOpts{ + FilterMode: api.ContractFilterModeActive, + }) if err != nil { return nil, err } @@ -902,7 +904,9 @@ func performContractFormations(ctx *mCtx, bus Bus, cr contractReviser, hf hostFi wanted := int(ctx.WantedContracts()) // fetch all active contracts - contracts, err := bus.Contracts(ctx, api.ContractsOpts{}) + contracts, err := bus.Contracts(ctx, api.ContractsOpts{ + FilterMode: api.ContractFilterModeActive, + }) if err != nil { return 0, fmt.Errorf("failed to fetch contracts: %w", err) } @@ -1052,7 +1056,9 @@ func performHostChecks(ctx *mCtx, bus Bus, logger *zap.SugaredLogger) error { func performPostMaintenanceTasks(ctx *mCtx, bus Bus, alerter alerts.Alerter, cc contractChecker, rb revisionBroadcaster, logger *zap.SugaredLogger) error { // fetch some contract and host info - allContracts, err := bus.Contracts(ctx, api.ContractsOpts{}) + allContracts, err := bus.Contracts(ctx, api.ContractsOpts{ + FilterMode: api.ContractFilterModeActive, + }) if err != nil { return fmt.Errorf("failed to fetch all contracts: %w", err) } @@ -1117,7 +1123,7 @@ func performV2ContractMigration(ctx *mCtx, bus Bus, cr contractReviser, logger * } contracts, err := bus.Contracts(ctx, api.ContractsOpts{ - FilterMode: api.ContractFilterModeAll, // TODO: change to usable + FilterMode: api.ContractFilterModeActive, }) if err != nil { logger.With(zap.Error(err)).Error("failed to fetch contracts for migration") @@ -1149,7 +1155,7 @@ func performV2ContractMigration(ctx *mCtx, bus Bus, cr contractReviser, logger * } // form a new contract with the same host - contract, _, err := cr.formContract(ctx, bus, host, InitialContractFunding, logger) + _, _, err = cr.formContract(ctx, bus, host, InitialContractFunding, logger) if err != nil { logger.Errorf("failed to form a v2 contract with the host") continue diff --git a/autopilot/contractor/hostfilter.go b/autopilot/contractor/hostfilter.go index b9dd6064d..d2d349479 100644 --- a/autopilot/contractor/hostfilter.go +++ b/autopilot/contractor/hostfilter.go @@ -8,6 +8,7 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" "go.sia.tech/renterd/internal/gouging" + rhp4 "go.sia.tech/renterd/internal/rhp/v4" ) const ( @@ -165,7 +166,11 @@ func checkHost(gc gouging.Checker, sh scoredHost, minScore float64, period uint6 // calculate remaining host info fields if !h.IsAnnounced() { ub.NotAnnounced = true - } else if !h.Scanned { + } else if !h.Scanned || + // NOTE: a v2 host might have been scanned before the v2 height so strictly + // speaking it is scanned but since it hasn't been scanned since, the + // settings aren't set so we treat it as not scanned + (h.IsV2() && h.V2Settings == (rhp4.HostSettings{})) { ub.NotCompletingScan = true } else { // online check diff --git a/bus/bus.go b/bus/bus.go index dbb3ebaa0..82024d260 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -796,6 +796,13 @@ func (b *Bus) renewContractV1(ctx context.Context, cs consensus.State, gp api.Go // derive the renter key renterKey := b.masterKey.DeriveContractKey(c.HostKey) + // cap v1 renewals to the v2 require height since the host won't allow us to + // form contracts beyond that + v2ReqHeight := b.cm.TipState().Network.HardforkV2.RequireHeight + if endHeight >= v2ReqHeight { + endHeight = v2ReqHeight - 1 + } + // fetch the revision rev, err := b.rhp3Client.Revision(ctx, c.ID, c.HostKey, hs.SiamuxAddr()) if err != nil { diff --git a/bus/routes.go b/bus/routes.go index 2fe14b41a..fcb6deed7 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -2320,6 +2320,13 @@ func (b *Bus) contractsFormHandler(jc jape.Context) { return } + // cap v1 formations to the v2 require height since the host won't allow + // us to form contracts beyond that + v2ReqHeight := b.cm.TipState().Network.HardforkV2.RequireHeight + if rfr.EndHeight >= v2ReqHeight { + rfr.EndHeight = v2ReqHeight - 1 + } + // check gouging breakdown := gc.CheckSettings(settings) if breakdown.Gouging() { diff --git a/go.mod b/go.mod index a2820e7f2..f24081a06 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb go.sia.tech/coreutils v0.7.1-0.20241211045514-6881993d8806 go.sia.tech/gofakes3 v0.0.5 - go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db + go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed go.sia.tech/jape v0.12.1 go.sia.tech/mux v1.3.0 go.sia.tech/web/renterd v0.69.0 diff --git a/go.sum b/go.sum index c5157f74d..361877a3d 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,8 @@ go.sia.tech/coreutils v0.7.1-0.20241211045514-6881993d8806 h1:zmLtpmFQPKMukYMiQB go.sia.tech/coreutils v0.7.1-0.20241211045514-6881993d8806/go.mod h1:6z3oHrQqcLoFEAT/l6XnvOivEGXgIfWBKcq0OqsouWA= go.sia.tech/gofakes3 v0.0.5 h1:vFhVBUFbKE9ZplvLE2w4TQxFMQyF8qvgxV4TaTph+Vw= go.sia.tech/gofakes3 v0.0.5/go.mod h1:LXEzwGw+OHysWLmagleCttX93cJZlT9rBu/icOZjQ54= -go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db h1:ey3ezMYHPzY+FZ4yL8xsAWnCJWI2J9z4rtpmRa8dj0A= -go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db/go.mod h1:6wTgoXKmsLQT22lUcHI4/dUcb3mhXFR+9zYWIki8Qho= +go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed h1:C42AxWwwoP13EhZsdWwR17Rc9S7gXI4JnRN0AyZRxc8= +go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed/go.mod h1:6wTgoXKmsLQT22lUcHI4/dUcb3mhXFR+9zYWIki8Qho= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index eca33da30..bee8ce48e 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -630,7 +630,7 @@ func announceHosts(hosts []*Host) error { for _, host := range hosts { settings := defaultHostSettings settings.NetAddress = host.rhp4Listener.Addr().(*net.TCPAddr).IP.String() - if err := host.settings.UpdateSettings(settings); err != nil { + if err := host.UpdateSettings(settings); err != nil { return err } if err := host.settings.Announce(); err != nil { diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index b902bffa9..d06034082 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -35,7 +35,6 @@ import ( "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.sia.tech/renterd/stores/sql/sqlite" - "go.uber.org/zap" "lukechampine.com/frand" ) @@ -2859,6 +2858,7 @@ func TestContractFundsReturnWhenHostOffline(t *testing.T) { }) } +// TODO: upload and download func TestV1ToV2Transition(t *testing.T) { // create a chain manager with a custom network that starts before the v2 // allow height @@ -2874,18 +2874,16 @@ func TestV1ToV2Transition(t *testing.T) { // custom autopilot config apCfg := test.AutopilotConfig apCfg.Contracts.Amount = 2 - apCfg.Contracts.Period = 1000 // make sure contracts are not scheduled for renew before reaching the allowheight + apCfg.Contracts.Period = 1000 // make sure we handle trying to form contracts with a proof height after the v2 require height apCfg.Contracts.RenewWindow = 50 // create a test cluster nHosts := 3 - l, _ := zap.NewDevelopment() cluster := newTestCluster(t, testClusterOptions{ autopilotConfig: &apCfg, hosts: 0, // add hosts manually later cm: cm, uploadPacking: false, // disable to make sure we don't accidentally serve data from disk - logger: l, }) defer cluster.Shutdown() tt := cluster.tt @@ -2912,6 +2910,8 @@ func TestV1ToV2Transition(t *testing.T) { for _, c := range contracts { if c.V2 { t.Fatal("should not have formed v2 contracts") + } else if c.EndHeight() != network.HardforkV2.RequireHeight-1 { + t.Fatalf("expected proof height to be %v, got %v", network.HardforkV2.RequireHeight-1, c.EndHeight()) } contractHosts[c.HostKey] = struct{}{} } @@ -2929,9 +2929,43 @@ func TestV1ToV2Transition(t *testing.T) { cluster.MineBlocks(1) time.Sleep(100 * time.Millisecond) } + time.Sleep(time.Second) - // check the contracts again - contracts, err = cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeAll}) + // check that we have 1 archived contract for every contract we had before + archivedContracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeArchived}) tt.OK(err) - fmt.Println("contracts", len(contracts)) + if len(archivedContracts) != nHosts-1 { + t.Fatalf("expected %v archived contracts, got %v", 2*(nHosts-1), len(archivedContracts)) + } + + // they should be on nHosts-1 unique hosts + usedHosts := make(map[types.PublicKey]struct{}) + for _, c := range archivedContracts { + if c.ArchivalReason != "migrated to v2" { + t.Fatalf("expected archival reason to be 'migrated to v2', got %v", c.ArchivalReason) + } else if c.V2 { + t.Fatalf("expected contract to be v1, got v2") + } + usedHosts[c.HostKey] = struct{}{} + } + if len(usedHosts) != nHosts-1 { + t.Fatalf("expected %v unique hosts, got %v", nHosts-1, len(usedHosts)) + } + + // we should have the same number of active contracts + activeContracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeActive}) + tt.OK(err) + if len(activeContracts) != nHosts-1 { + t.Fatalf("expected %v active contracts, got %v", nHosts-1, len(activeContracts)) + } + + // they should be on the same hosts as before + for _, c := range activeContracts { + if _, ok := usedHosts[c.HostKey]; !ok { + t.Fatal("host not found in used hosts") + } else if !c.V2 { + t.Fatal("expected contract to be v2, got v1", c.ID, c.ArchivalReason) + } + delete(usedHosts, c.HostKey) + } } From 5f622a01538edb849150ea692bfdb651fbd86c3d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 14:32:16 +0100 Subject: [PATCH 10/37] upload an object --- internal/test/e2e/cluster_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index d06034082..332232abc 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2858,7 +2858,6 @@ func TestContractFundsReturnWhenHostOffline(t *testing.T) { }) } -// TODO: upload and download func TestV1ToV2Transition(t *testing.T) { // create a chain manager with a custom network that starts before the v2 // allow height @@ -2921,6 +2920,13 @@ func TestV1ToV2Transition(t *testing.T) { t.Fatalf("expected %v unique hosts, got %v", nHosts-1, len(contractHosts)) } + // upload some data + data := frand.Bytes(100) + tt.OKAll(cluster.Worker.UploadObject(context.Background(), bytes.NewReader(data), testBucket, "foo", api.UploadObjectOptions{ + MinShards: 1, + TotalShards: nHosts - 1, + })) + // mine until we reach the v2 allowheight cluster.MineBlocks(network.HardforkV2.AllowHeight - cm.Tip().Height) @@ -2968,4 +2974,6 @@ func TestV1ToV2Transition(t *testing.T) { } delete(usedHosts, c.HostKey) } + + // TODO: check health, contracts and download } From aae5a280310539ec4aa635240cb2c1e60433dc73 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 15:31:13 +0100 Subject: [PATCH 11/37] don't delete host sectors when contract is deleted --- stores/metadata_test.go | 13 +++++++++ stores/sql/main.go | 60 +++++++++++++++++++++++------------------ 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/stores/metadata_test.go b/stores/metadata_test.go index d03bf3203..1e572c10a 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -3634,6 +3634,19 @@ func TestUpdateSlabSanityChecks(t *testing.T) { if err := ss.UpdateSlab(context.Background(), slab.EncryptionKey, sectors); !errors.Is(err, api.ErrUnknownSector) { t.Fatal(err) } + + // delete one of the contracts - this should cause the host to still be in + // the slab but the associated slice should be empty + if err := ss.ArchiveContract(context.Background(), contracts[0].ID, "test"); err != nil { + t.Fatal(err) + } + slab.Shards[0].Contracts[hks[0]] = []types.FileContractID{} + rSlab, err = ss.Slab(context.Background(), slab.EncryptionKey) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(slab, rSlab) { + t.Fatal("unexpected slab", cmp.Diff(slab, rSlab, cmp.AllowUnexported(object.EncryptionKey{}))) + } } func TestSlabHealthInvalidation(t *testing.T) { diff --git a/stores/sql/main.go b/stores/sql/main.go index b6b020fbe..d8d8dbbeb 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -193,22 +193,19 @@ func ArchiveContract(ctx context.Context, tx sql.Tx, fcid types.FileContractID, } // delete host sectors that are no longer associated with any contract - _, err = tx.Exec(ctx, `DELETE FROM host_sectors -WHERE NOT EXISTS ( - SELECT 1 - FROM contract_sectors - WHERE contract_sectors.db_sector_id = host_sectors.db_sector_id -) -OR NOT EXISTS ( - SELECT 1 - FROM contracts - INNER JOIN hosts ON contracts.host_id = hosts.id - WHERE contracts.archival_reason IS NULL - AND hosts.id = host_sectors.db_host_id -)`) - if err != nil { - return fmt.Errorf("failed to delete host_sectors: %w", err) - } + // TODO: instead of deleting, set the timestamp of all host sectors + // belonging to a host that we don't have a contract with to 'now'. + // _, err = tx.Exec(ctx, `DELETE FROM host_sectors + // WHERE NOT EXISTS ( + // SELECT 1 + // FROM contracts + // INNER JOIN hosts ON contracts.host_id = hosts.id + // WHERE contracts.archival_reason IS NULL + // AND hosts.id = host_sectors.db_host_id + // )`) + // if err != nil { + // return fmt.Errorf("failed to delete host_sectors: %w", err) + // } return nil } @@ -1831,11 +1828,11 @@ func Slab(ctx context.Context, tx sql.Tx, key object.EncryptionKey) (object.Slab // fetch contracts for each sector stmt, err := tx.Prepare(ctx, ` - SELECT h.public_key, c.fcid - FROM contract_sectors cs - INNER JOIN contracts c ON c.id = cs.db_contract_id - INNER JOIN hosts h ON h.public_key = c.host_key - WHERE cs.db_sector_id = ? + SELECT h.public_key, COALESCE(c.fcid, ?) + FROM host_sectors hs + INNER JOIN hosts h ON h.id = hs.db_host_id + LEFT JOIN contracts c ON c.host_key = h.public_key AND c.archival_reason IS NULL + WHERE hs.db_sector_id = ? ORDER BY c.id `) if err != nil { @@ -1844,7 +1841,7 @@ func Slab(ctx context.Context, tx sql.Tx, key object.EncryptionKey) (object.Slab defer stmt.Close() for i, sectorID := range sectorIDs { - rows, err := stmt.Query(ctx, sectorID) + rows, err := stmt.Query(ctx, FileContractID{}, sectorID) if err != nil { return object.Slab{}, fmt.Errorf("failed to fetch contracts: %w", err) } @@ -1858,7 +1855,12 @@ func Slab(ctx context.Context, tx sql.Tx, key object.EncryptionKey) (object.Slab if err := rows.Scan((*PublicKey)(&pk), (*FileContractID)(&fcid)); err != nil { return fmt.Errorf("failed to scan contract: %w", err) } - slab.Shards[i].Contracts[pk] = append(slab.Shards[i].Contracts[pk], fcid) + if _, exists := slab.Shards[i].Contracts[pk]; !exists { + slab.Shards[i].Contracts[pk] = []types.FileContractID{} + } + if fcid != (types.FileContractID{}) { + slab.Shards[i].Contracts[pk] = append(slab.Shards[i].Contracts[pk], fcid) + } } return nil }(); err != nil { @@ -2531,8 +2533,9 @@ func Object(ctx context.Context, tx Tx, bucket, key string) (api.Object, error) FROM slices sli INNER JOIN slabs sla ON sli.db_slab_id = sla.id LEFT JOIN sectors sec ON sec.db_slab_id = sla.id - LEFT JOIN contract_sectors csec ON csec.db_sector_id = sec.id - LEFT JOIN contracts c ON c.id = csec.db_contract_id + LEFT JOIN host_sectors hs ON hs.db_sector_id = sec.id + LEFT JOIN hosts h ON h.id = hs.db_host_id + LEFT JOIN contracts c ON c.host_key = h.public_key AND c.archival_reason IS NULL WHERE sli.db_object_id = ? ORDER BY sli.object_index ASC, sec.slab_index ASC `, Hash256{}, FileContractID{}, PublicKey{}, objID) @@ -2595,7 +2598,12 @@ func Object(ctx context.Context, tx Tx, bucket, key string) (api.Object, error) if current.Shards[len(current.Shards)-1].Contracts == nil { current.Shards[len(current.Shards)-1].Contracts = make(map[types.PublicKey][]types.FileContractID) } - current.Shards[len(current.Shards)-1].Contracts[hk] = append(current.Shards[len(current.Shards)-1].Contracts[hk], fcid) + if _, exists := current.Shards[len(current.Shards)-1].Contracts[hk]; !exists { + current.Shards[len(current.Shards)-1].Contracts[hk] = []types.FileContractID{} + } + if fcid != (types.FileContractID{}) { + current.Shards[len(current.Shards)-1].Contracts[hk] = append(current.Shards[len(current.Shards)-1].Contracts[hk], fcid) + } } } From 4edd543f5096d6b698115606f37a90e177d28026 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 15:50:37 +0100 Subject: [PATCH 12/37] expire worker cache faster during testing --- cmd/renterd/config.go | 1 + config/config.go | 1 + internal/test/e2e/cluster.go | 1 + internal/worker/cache.go | 18 +++++++++--------- worker/worker.go | 5 ++++- worker/worker_test.go | 1 + 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/cmd/renterd/config.go b/cmd/renterd/config.go index ee9252096..07c8b4951 100644 --- a/cmd/renterd/config.go +++ b/cmd/renterd/config.go @@ -97,6 +97,7 @@ func defaultConfig() config.Config { ID: "", AccountsRefillInterval: defaultAccountRefillInterval, BusFlushInterval: 5 * time.Second, + CacheExpiry: 5 * time.Minute, DownloadMaxOverdrive: 5, DownloadOverdriveTimeout: 3 * time.Second, diff --git a/config/config.go b/config/config.go index 085503b49..a022776c4 100644 --- a/config/config.go +++ b/config/config.go @@ -130,6 +130,7 @@ type ( UploadMaxMemory uint64 `yaml:"uploadMaxMemory,omitempty"` UploadMaxOverdrive uint64 `yaml:"uploadMaxOverdrive,omitempty"` AllowUnauthenticatedDownloads bool `yaml:"allowUnauthenticatedDownloads,omitempty"` + CacheExpiry time.Duration `yaml:"cacheExpiry,omitempty"` } // Autopilot contains the configuration for an autopilot. diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index bee8ce48e..5b3c9ba3d 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -994,6 +994,7 @@ func testDBCfg() dbConfig { func testWorkerCfg() config.Worker { return config.Worker{ AccountsRefillInterval: 10 * time.Millisecond, + CacheExpiry: 100 * time.Millisecond, ID: "worker", BusFlushInterval: testBusFlushInterval, DownloadOverdriveTimeout: 500 * time.Millisecond, diff --git a/internal/worker/cache.go b/internal/worker/cache.go index 3fc69184a..f0fedded8 100644 --- a/internal/worker/cache.go +++ b/internal/worker/cache.go @@ -12,14 +12,13 @@ import ( ) const ( - cacheEntryExpiry = 5 * time.Minute - cacheKeyUsableHosts = "usablehosts" ) type memoryCache struct { - items map[string]*cacheEntry - mu sync.RWMutex + cacheEntryExpiry time.Duration + items map[string]*cacheEntry + mu sync.RWMutex } type cacheEntry struct { @@ -27,9 +26,10 @@ type cacheEntry struct { expiry time.Time } -func newMemoryCache() *memoryCache { +func newMemoryCache(expiry time.Duration) *memoryCache { return &memoryCache{ - items: make(map[string]*cacheEntry), + cacheEntryExpiry: expiry, + items: make(map[string]*cacheEntry), } } @@ -59,7 +59,7 @@ func (c *memoryCache) Set(key string, value interface{}) { defer c.mu.Unlock() c.items[key] = &cacheEntry{ value: value, - expiry: time.Now().Add(cacheEntryExpiry), + expiry: time.Now().Add(c.cacheEntryExpiry), } } @@ -79,12 +79,12 @@ type cache struct { logger *zap.SugaredLogger } -func NewCache(b Bus, logger *zap.Logger) WorkerCache { +func NewCache(b Bus, expiry time.Duration, logger *zap.Logger) WorkerCache { logger = logger.Named("workercache") return &cache{ b: b, - cache: newMemoryCache(), + cache: newMemoryCache(expiry), logger: logger.Sugar(), } } diff --git a/worker/worker.go b/worker/worker.go index f73ed59cf..1e4c63c8c 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -669,6 +669,9 @@ func New(cfg config.Worker, masterKey [32]byte, b Bus, l *zap.Logger) (*Worker, if cfg.UploadMaxMemory == 0 { return nil, errors.New("uploadMaxMemory cannot be 0") } + if cfg.CacheExpiry == 0 { + return nil, errors.New("cache expiry cannot be 0") + } a := alerts.WithOrigin(b, fmt.Sprintf("worker.%s", cfg.ID)) shutdownCtx, shutdownCancel := context.WithCancel(context.Background()) @@ -676,7 +679,7 @@ func New(cfg config.Worker, masterKey [32]byte, b Bus, l *zap.Logger) (*Worker, dialer := rhp.NewFallbackDialer(b, net.Dialer{}, l) w := &Worker{ alerts: a, - cache: iworker.NewCache(b, l), + cache: iworker.NewCache(b, cfg.CacheExpiry, l), dialer: dialer, id: cfg.ID, bus: b, diff --git a/worker/worker_test.go b/worker/worker_test.go index 0c731766b..c32e51e1e 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -156,6 +156,7 @@ func (w *testWorker) UsableHosts() []api.HostInfo { func newTestWorkerCfg() config.Worker { return config.Worker{ AccountsRefillInterval: time.Second, + CacheExpiry: 100 * time.Millisecond, ID: "test", BusFlushInterval: time.Second, DownloadOverdriveTimeout: time.Second, From 8ba3f652dfc8c4b907aaf24eac4338be0e2488fa Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 16:10:49 +0100 Subject: [PATCH 13/37] fix TestV1ToV2Transition health --- internal/test/e2e/cluster_test.go | 13 +++++++- stores/sql/main.go | 54 +++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 332232abc..2c3799c9a 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2973,7 +2973,18 @@ func TestV1ToV2Transition(t *testing.T) { t.Fatal("expected contract to be v2, got v1", c.ID, c.ArchivalReason) } delete(usedHosts, c.HostKey) + fmt.Println("new contract", c.ID) } - // TODO: check health, contracts and download + // check health is 1 + tt.Retry(100, 100*time.Millisecond, func() error { + object, err := cluster.Bus.Object(context.Background(), testBucket, "foo", api.GetObjectOptions{}) + tt.OK(err) + if object.Health != 1 { + return fmt.Errorf("expected health to be 1, got %v", object.Health) + } + return nil + }) + + // TODO: check contracts and download } diff --git a/stores/sql/main.go b/stores/sql/main.go index d8d8dbbeb..9de379b74 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -1828,11 +1828,11 @@ func Slab(ctx context.Context, tx sql.Tx, key object.EncryptionKey) (object.Slab // fetch contracts for each sector stmt, err := tx.Prepare(ctx, ` - SELECT h.public_key, COALESCE(c.fcid, ?) - FROM host_sectors hs - INNER JOIN hosts h ON h.id = hs.db_host_id - LEFT JOIN contracts c ON c.host_key = h.public_key AND c.archival_reason IS NULL - WHERE hs.db_sector_id = ? + SELECT h.public_key, c.fcid + FROM contract_sectors cs + INNER JOIN contracts c ON c.id = cs.db_contract_id + INNER JOIN hosts h ON h.public_key = c.host_key + WHERE cs.db_sector_id = ? ORDER BY c.id `) if err != nil { @@ -1840,8 +1840,21 @@ func Slab(ctx context.Context, tx sql.Tx, key object.EncryptionKey) (object.Slab } defer stmt.Close() + // fetch hosts for each sector + hostStmt, err := tx.Prepare(ctx, ` + SELECT h.public_key + FROM host_sectors hs + INNER JOIN hosts h ON h.id = hs.db_host_id + WHERE hs.db_sector_id = ? + `) + if err != nil { + return object.Slab{}, fmt.Errorf("failed to prepare statement to fetch hosts: %w", err) + } + defer hostStmt.Close() + for i, sectorID := range sectorIDs { - rows, err := stmt.Query(ctx, FileContractID{}, sectorID) + // contracts + rows, err := stmt.Query(ctx, sectorID) if err != nil { return object.Slab{}, fmt.Errorf("failed to fetch contracts: %w", err) } @@ -1866,6 +1879,26 @@ func Slab(ctx context.Context, tx sql.Tx, key object.EncryptionKey) (object.Slab }(); err != nil { return object.Slab{}, err } + + // hosts + rows, err = hostStmt.Query(ctx, sectorID) + if err != nil { + return object.Slab{}, fmt.Errorf("failed to fetch hosts: %w", err) + } + if err := func() error { + defer rows.Close() + + for rows.Next() { + var pk types.PublicKey + if err := rows.Scan((*PublicKey)(&pk)); err != nil { + return fmt.Errorf("failed to scan host: %w", err) + } + slab.Shards[i].Contracts[pk] = []types.FileContractID{} + } + return nil + }(); err != nil { + return object.Slab{}, err + } } return slab, nil } @@ -2529,13 +2562,14 @@ func Object(ctx context.Context, tx Tx, bucket, key string) (api.Object, error) // fetch slab slices rows, err = tx.Query(ctx, ` - SELECT sla.db_buffered_slab_id IS NOT NULL, sli.object_index, sli.offset, sli.length, sla.health, sla.key, sla.min_shards, COALESCE(sec.slab_index, 0), COALESCE(sec.root, ?), COALESCE(c.fcid, ?), COALESCE(c.host_key, ?) + SELECT sla.db_buffered_slab_id IS NOT NULL, sli.object_index, sli.offset, sli.length, sla.health, sla.key, sla.min_shards, COALESCE(sec.slab_index, 0), COALESCE(sec.root, ?), COALESCE(c.fcid, ?), COALESCE(h.public_key, ?) FROM slices sli INNER JOIN slabs sla ON sli.db_slab_id = sla.id - LEFT JOIN sectors sec ON sec.db_slab_id = sla.id - LEFT JOIN host_sectors hs ON hs.db_sector_id = sec.id + INNER JOIN sectors sec ON sec.db_slab_id = sla.id + LEFT JOIN contract_sectors csec ON csec.db_sector_id = sec.id + LEFT JOIN contracts c ON c.id = csec.db_contract_id + LEFT JOIN host_sectors hs ON hs.db_sector_id = csec.db_sector_id LEFT JOIN hosts h ON h.id = hs.db_host_id - LEFT JOIN contracts c ON c.host_key = h.public_key AND c.archival_reason IS NULL WHERE sli.db_object_id = ? ORDER BY sli.object_index ASC, sec.slab_index ASC `, Hash256{}, FileContractID{}, PublicKey{}, objID) From 9b23b411208ca23dccc3fdc4b12bd8e8da68045c Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 16:16:15 +0100 Subject: [PATCH 14/37] check contracts and download data --- internal/test/e2e/cluster_test.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 2c3799c9a..fde4b2625 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -21,6 +21,7 @@ import ( "github.com/google/go-cmp/cmp" rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" + rhpv4 "go.sia.tech/core/rhp/v4" "go.sia.tech/core/types" "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" @@ -2973,7 +2974,6 @@ func TestV1ToV2Transition(t *testing.T) { t.Fatal("expected contract to be v2, got v1", c.ID, c.ArchivalReason) } delete(usedHosts, c.HostKey) - fmt.Println("new contract", c.ID) } // check health is 1 @@ -2986,5 +2986,19 @@ func TestV1ToV2Transition(t *testing.T) { return nil }) - // TODO: check contracts and download + // check that the contracts now contain the data + activeContracts, err = cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeActive}) + tt.OK(err) + for _, c := range activeContracts { + if c.Size != rhpv4.SectorSize { + t.Fatalf("expected sector size to be %v, got %v", rhpv4.SectorSize, c.Size) + } + } + + // download file to make sure it's still there + buf := new(bytes.Buffer) + tt.OK(cluster.Worker.DownloadObject(context.Background(), buf, testBucket, "foo", api.DownloadObjectOptions{})) + if !bytes.Equal(data, buf.Bytes()) { + t.Fatal("data mismatch") + } } From 73ce2b218fa1b2cff33b97f9cfa51976d05c3cef Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 16:31:43 +0100 Subject: [PATCH 15/37] fix TestSQLMetadataStore --- stores/metadata_test.go | 2 +- stores/sql/main.go | 37 ++++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/stores/metadata_test.go b/stores/metadata_test.go index 1e572c10a..c23964821 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -2614,7 +2614,7 @@ func TestPartialSlab(t *testing.T) { t.Fatal(err) } if !reflect.DeepEqual(obj, *fetched.Object) { - t.Fatal("mismatch", cmp.Diff(obj, fetched.Object, cmp.AllowUnexported(object.EncryptionKey{}))) + t.Fatal("mismatch", cmp.Diff(obj, *fetched.Object, cmp.AllowUnexported(object.EncryptionKey{}))) } // Add the second slab. diff --git a/stores/sql/main.go b/stores/sql/main.go index 9de379b74..2e780a034 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -1853,47 +1853,50 @@ func Slab(ctx context.Context, tx sql.Tx, key object.EncryptionKey) (object.Slab defer hostStmt.Close() for i, sectorID := range sectorIDs { - // contracts - rows, err := stmt.Query(ctx, sectorID) + slab.Shards[i].Contracts = make(map[types.PublicKey][]types.FileContractID) + + // hosts + rows, err = hostStmt.Query(ctx, sectorID) if err != nil { - return object.Slab{}, fmt.Errorf("failed to fetch contracts: %w", err) + return object.Slab{}, fmt.Errorf("failed to fetch hosts: %w", err) } if err := func() error { defer rows.Close() - slab.Shards[i].Contracts = make(map[types.PublicKey][]types.FileContractID) for rows.Next() { var pk types.PublicKey - var fcid types.FileContractID - if err := rows.Scan((*PublicKey)(&pk), (*FileContractID)(&fcid)); err != nil { - return fmt.Errorf("failed to scan contract: %w", err) + if err := rows.Scan((*PublicKey)(&pk)); err != nil { + return fmt.Errorf("failed to scan host: %w", err) } if _, exists := slab.Shards[i].Contracts[pk]; !exists { slab.Shards[i].Contracts[pk] = []types.FileContractID{} } - if fcid != (types.FileContractID{}) { - slab.Shards[i].Contracts[pk] = append(slab.Shards[i].Contracts[pk], fcid) - } } return nil }(); err != nil { return object.Slab{}, err } - // hosts - rows, err = hostStmt.Query(ctx, sectorID) + // contracts + rows, err := stmt.Query(ctx, sectorID) if err != nil { - return object.Slab{}, fmt.Errorf("failed to fetch hosts: %w", err) + return object.Slab{}, fmt.Errorf("failed to fetch contracts: %w", err) } if err := func() error { defer rows.Close() for rows.Next() { var pk types.PublicKey - if err := rows.Scan((*PublicKey)(&pk)); err != nil { - return fmt.Errorf("failed to scan host: %w", err) + var fcid types.FileContractID + if err := rows.Scan((*PublicKey)(&pk), (*FileContractID)(&fcid)); err != nil { + return fmt.Errorf("failed to scan contract: %w", err) + } + if _, exists := slab.Shards[i].Contracts[pk]; !exists { + slab.Shards[i].Contracts[pk] = []types.FileContractID{} + } + if fcid != (types.FileContractID{}) { + slab.Shards[i].Contracts[pk] = append(slab.Shards[i].Contracts[pk], fcid) } - slab.Shards[i].Contracts[pk] = []types.FileContractID{} } return nil }(); err != nil { @@ -2565,7 +2568,7 @@ func Object(ctx context.Context, tx Tx, bucket, key string) (api.Object, error) SELECT sla.db_buffered_slab_id IS NOT NULL, sli.object_index, sli.offset, sli.length, sla.health, sla.key, sla.min_shards, COALESCE(sec.slab_index, 0), COALESCE(sec.root, ?), COALESCE(c.fcid, ?), COALESCE(h.public_key, ?) FROM slices sli INNER JOIN slabs sla ON sli.db_slab_id = sla.id - INNER JOIN sectors sec ON sec.db_slab_id = sla.id + LEFT JOIN sectors sec ON sec.db_slab_id = sla.id LEFT JOIN contract_sectors csec ON csec.db_sector_id = sec.id LEFT JOIN contracts c ON c.id = csec.db_contract_id LEFT JOIN host_sectors hs ON hs.db_sector_id = csec.db_sector_id From 5a83fc63e9b2c3b6dabaccb204d0df2f930b6b9d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 13 Dec 2024 16:44:15 +0100 Subject: [PATCH 16/37] skip TestHostSectors --- stores/metadata_test.go | 2 ++ stores/sql/main.go | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/stores/metadata_test.go b/stores/metadata_test.go index c23964821..ff4cf54b1 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -4702,6 +4702,8 @@ func TestPutContract(t *testing.T) { } func TestHostSectors(t *testing.T) { + t.Skip("enable once host sector deletion is correctly implemented") + ss := newTestSQLStore(t, defaultTestSQLStoreConfig) defer ss.Close() diff --git a/stores/sql/main.go b/stores/sql/main.go index 2e780a034..1dac02699 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -192,17 +192,17 @@ func ArchiveContract(ctx context.Context, tx sql.Tx, fcid types.FileContractID, return fmt.Errorf("failed to delete contract_sectors: %w", err) } - // delete host sectors that are no longer associated with any contract - // TODO: instead of deleting, set the timestamp of all host sectors - // belonging to a host that we don't have a contract with to 'now'. - // _, err = tx.Exec(ctx, `DELETE FROM host_sectors - // WHERE NOT EXISTS ( - // SELECT 1 - // FROM contracts - // INNER JOIN hosts ON contracts.host_id = hosts.id - // WHERE contracts.archival_reason IS NULL - // AND hosts.id = host_sectors.db_host_id - // )`) + // TODO: update all updated_at timestamps of host_sectors that are no longer used and only delete the ones + // that haven't been updated in 72 hours + // _, err = tx.Exec(ctx, `UPDATE host_sectors + // SET updated_at = ? + // WHERE NOT EXISTS ( + // SELECT 1 + // FROM contracts + // INNER JOIN hosts ON contracts.host_id = hosts.id + // WHERE contracts.archival_reason IS NULL + // AND hosts.id = host_sectors.db_host_id + // )`, time.Now()) // if err != nil { // return fmt.Errorf("failed to delete host_sectors: %w", err) // } From 8dc0af3053f50095cf6d9dae8f01717a69586df0 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 07:18:57 +0100 Subject: [PATCH 17/37] prune host_sectors when no more active contracts are available --- stores/metadata_test.go | 2 -- stores/sql/main.go | 27 +++++++++++++-------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/stores/metadata_test.go b/stores/metadata_test.go index ff4cf54b1..c23964821 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -4702,8 +4702,6 @@ func TestPutContract(t *testing.T) { } func TestHostSectors(t *testing.T) { - t.Skip("enable once host sector deletion is correctly implemented") - ss := newTestSQLStore(t, defaultTestSQLStoreConfig) defer ss.Close() diff --git a/stores/sql/main.go b/stores/sql/main.go index 1dac02699..b5748a8cf 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -192,20 +192,19 @@ func ArchiveContract(ctx context.Context, tx sql.Tx, fcid types.FileContractID, return fmt.Errorf("failed to delete contract_sectors: %w", err) } - // TODO: update all updated_at timestamps of host_sectors that are no longer used and only delete the ones - // that haven't been updated in 72 hours - // _, err = tx.Exec(ctx, `UPDATE host_sectors - // SET updated_at = ? - // WHERE NOT EXISTS ( - // SELECT 1 - // FROM contracts - // INNER JOIN hosts ON contracts.host_id = hosts.id - // WHERE contracts.archival_reason IS NULL - // AND hosts.id = host_sectors.db_host_id - // )`, time.Now()) - // if err != nil { - // return fmt.Errorf("failed to delete host_sectors: %w", err) - // } + // delete all host_sectors for every host that we don't have an active + // contract with anymore + _, err = tx.Exec(ctx, `DELETE FROM host_sectors + WHERE NOT EXISTS ( + SELECT 1 + FROM contracts + INNER JOIN hosts ON contracts.host_id = hosts.id + WHERE contracts.archival_reason IS NULL + AND hosts.id = host_sectors.db_host_id + )`, time.Now()) + if err != nil { + return fmt.Errorf("failed to delete host_sectors: %w", err) + } return nil } From 37df84d9be80f4bafedf100b1a769861e5f0f9ae Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 07:22:22 +0100 Subject: [PATCH 18/37] sql: extend DeleteHostSector --- stores/sql/main.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/stores/sql/main.go b/stores/sql/main.go index b5748a8cf..3fe31b322 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -502,14 +502,16 @@ func DeleteBucket(ctx context.Context, tx sql.Tx, bucket string) error { } func DeleteHostSector(ctx context.Context, tx sql.Tx, hk types.PublicKey, root types.Hash256) (int, error) { + // fetch sector id + var sectorID int64 + if err := tx.QueryRow(ctx, "SELECT s.id FROM sectors s WHERE root = ?", PublicKey(hk)).Scan(§orID); err != nil { + return 0, fmt.Errorf("failed to fetch sector id: %w", err) + } + // remove potential links between the host's contracts and the sector res, err := tx.Exec(ctx, ` DELETE FROM contract_sectors - WHERE db_sector_id = ( - SELECT s.id - FROM sectors s - WHERE root = ? - ) AND db_contract_id IN ( + WHERE db_sector_id = ? AND db_contract_id IN ( SELECT c.id FROM contracts c WHERE c.host_key = ? @@ -548,6 +550,13 @@ func DeleteHostSector(ctx context.Context, tx sql.Tx, hk types.PublicKey, root t if err != nil { return 0, fmt.Errorf("failed to update lost sectors: %w", err) } + + // remove sector from host_sectors + _, err = tx.Exec(ctx, "DELETE FROM host_sectors WHERE db_sector_id = ?", sectorID) + if err != nil { + return 0, fmt.Errorf("failed to delete host sector: %w", err) + } + return int(deletedSectors), nil } From 38b526c38ff1b4b69999979bc9fe4a1b65712285 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 07:48:51 +0100 Subject: [PATCH 19/37] fix TestDeleteHostSector --- stores/sql/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/stores/sql/main.go b/stores/sql/main.go index 3fe31b322..20f46ac8f 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -504,7 +504,10 @@ func DeleteBucket(ctx context.Context, tx sql.Tx, bucket string) error { func DeleteHostSector(ctx context.Context, tx sql.Tx, hk types.PublicKey, root types.Hash256) (int, error) { // fetch sector id var sectorID int64 - if err := tx.QueryRow(ctx, "SELECT s.id FROM sectors s WHERE root = ?", PublicKey(hk)).Scan(§orID); err != nil { + if err := tx.QueryRow(ctx, "SELECT s.id FROM sectors s WHERE s.root = ?", Hash256(root)). + Scan(§orID); errors.Is(err, dsql.ErrNoRows) { + return 0, nil + } else if err != nil { return 0, fmt.Errorf("failed to fetch sector id: %w", err) } @@ -516,7 +519,7 @@ func DeleteHostSector(ctx context.Context, tx sql.Tx, hk types.PublicKey, root t FROM contracts c WHERE c.host_key = ? ) - `, Hash256(root), PublicKey(hk)) + `, sectorID, PublicKey(hk)) if err != nil { return 0, fmt.Errorf("failed to delete contract sectors: %w", err) } From 6490ce4d48af37ed7805d093c976282d452b6863 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 10:26:55 +0100 Subject: [PATCH 20/37] fix TestGouging --- internal/test/e2e/gouging_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/test/e2e/gouging_test.go b/internal/test/e2e/gouging_test.go index 440879972..dc52a39f7 100644 --- a/internal/test/e2e/gouging_test.go +++ b/internal/test/e2e/gouging_test.go @@ -93,14 +93,14 @@ func TestGouging(t *testing.T) { if err := h.UpdateSettings(settings); err != nil { t.Fatal(err) } - } - // make sure the price table expires so the worker is forced to fetch it - // again, this is necessary for the host to be considered price gouging - time.Sleep(defaultHostSettings.PriceTableValidity) + // scan the host + tt.OKAll(cluster.Bus.ScanHost(context.Background(), h.PublicKey(), time.Second)) + } + time.Sleep(testWorkerCfg().CacheExpiry) // wait for cache to refresh - // download the data - should still work - tt.OKAll(w.DownloadObject(context.Background(), io.Discard, testBucket, path, api.DownloadObjectOptions{})) + // download the data - won't work since the hosts are not usable anymore + tt.FailAll(w.DownloadObject(context.Background(), io.Discard, testBucket, path, api.DownloadObjectOptions{})) // try optimising gouging settings resp, err := cluster.Autopilot.EvaluateConfig(context.Background(), test.AutopilotConfig, gs, test.RedundancySettings) From 6f84d0195c9f9d1da1434704d0dce13ee9443546 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 12:02:22 +0100 Subject: [PATCH 21/37] fix TestMigrations for v1 --- stores/sql/main.go | 81 ++++++++-------------------------------------- 1 file changed, 13 insertions(+), 68 deletions(-) diff --git a/stores/sql/main.go b/stores/sql/main.go index 20f46ac8f..41e983091 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -2576,88 +2576,33 @@ func Object(ctx context.Context, tx Tx, bucket, key string) (api.Object, error) // fetch slab slices rows, err = tx.Query(ctx, ` - SELECT sla.db_buffered_slab_id IS NOT NULL, sli.object_index, sli.offset, sli.length, sla.health, sla.key, sla.min_shards, COALESCE(sec.slab_index, 0), COALESCE(sec.root, ?), COALESCE(c.fcid, ?), COALESCE(h.public_key, ?) + SELECT sla.id, sla.health, sla.key, sla.min_shards FROM slices sli INNER JOIN slabs sla ON sli.db_slab_id = sla.id - LEFT JOIN sectors sec ON sec.db_slab_id = sla.id - LEFT JOIN contract_sectors csec ON csec.db_sector_id = sec.id - LEFT JOIN contracts c ON c.id = csec.db_contract_id - LEFT JOIN host_sectors hs ON hs.db_sector_id = csec.db_sector_id - LEFT JOIN hosts h ON h.id = hs.db_host_id WHERE sli.db_object_id = ? - ORDER BY sli.object_index ASC, sec.slab_index ASC - `, Hash256{}, FileContractID{}, PublicKey{}, objID) + ORDER BY sli.object_index ASC + `, objID) if err != nil { return api.Object{}, fmt.Errorf("failed to fetch slabs: %w", err) } defer rows.Close() - slabSlices := object.SlabSlices{} - var current *object.SlabSlice - var currObjIdx, currSlaIdx int64 + var slabSlices []object.SlabSlice for rows.Next() { - var bufferedSlab bool - var objectIndex int64 - var slabIndex int64 + var id int64 var ss object.SlabSlice - var sector object.Sector - var fcid types.FileContractID - var hk types.PublicKey - if err := rows.Scan(&bufferedSlab, // whether the slab is buffered - &objectIndex, &ss.Offset, &ss.Length, // slice info - &ss.Health, (*EncryptionKey)(&ss.EncryptionKey), &ss.MinShards, // slab info - &slabIndex, (*Hash256)(§or.Root), // sector info - (*PublicKey)(&fcid), // contract info - (*PublicKey)(&hk), // host info - ); err != nil { + if err := rows.Scan(&id, &ss.Health, (*EncryptionKey)(&ss.EncryptionKey), &ss.MinShards); err != nil { return api.Object{}, fmt.Errorf("failed to scan slab slice: %w", err) } - - // sanity check object for corruption - isFirst := current == nil && objectIndex == 1 && slabIndex == 1 - isBuffered := bufferedSlab && objectIndex == currObjIdx+1 && slabIndex == 0 - isNewSlab := isFirst || isBuffered || (current != nil && objectIndex == currObjIdx+1 && slabIndex == 1) - isNewShard := isNewSlab || (objectIndex == currObjIdx && slabIndex == currSlaIdx+1) - isNewContract := isNewShard || (objectIndex == currObjIdx && slabIndex == currSlaIdx) - if !isFirst && !isBuffered && !isNewSlab && !isNewShard && !isNewContract { - return api.Object{}, fmt.Errorf("%w: object index %d, slab index %d, current object index %d, current slab index %d", api.ErrObjectCorrupted, objectIndex, slabIndex, currObjIdx, currSlaIdx) - } - - // update indices - currObjIdx = objectIndex - currSlaIdx = slabIndex - - if isNewSlab { - if current != nil { - slabSlices = append(slabSlices, *current) - } - current = &ss - } - - // if the slab is buffered there are no sectors/contracts to add - if bufferedSlab { - continue - } - - if isNewShard { - current.Shards = append(current.Shards, sector) - } - if isNewContract { - if current.Shards[len(current.Shards)-1].Contracts == nil { - current.Shards[len(current.Shards)-1].Contracts = make(map[types.PublicKey][]types.FileContractID) - } - if _, exists := current.Shards[len(current.Shards)-1].Contracts[hk]; !exists { - current.Shards[len(current.Shards)-1].Contracts[hk] = []types.FileContractID{} - } - if fcid != (types.FileContractID{}) { - current.Shards[len(current.Shards)-1].Contracts[hk] = append(current.Shards[len(current.Shards)-1].Contracts[hk], fcid) - } - } + slabSlices = append(slabSlices, ss) } - // add last slab slice - if current != nil { - slabSlices = append(slabSlices, *current) + // fill in the shards + for i := range slabSlices { + slabSlices[i].Slab, err = Slab(ctx, tx, slabSlices[i].EncryptionKey) + if err != nil { + return api.Object{}, fmt.Errorf("failed to fetch slab: %w", err) + } } return api.Object{ From 8ce19a4bc4b8fe17aa3b281f6b9339b9dc707b29 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 12:37:38 +0100 Subject: [PATCH 22/37] fix TestMigration for v2 --- stores/sql/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stores/sql/main.go b/stores/sql/main.go index 41e983091..a45a04dff 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -1635,7 +1635,7 @@ func RecordHostScans(ctx context.Context, tx sql.Tx, scans []api.HostScan) error uptime = CASE WHEN ? AND last_scan > 0 AND last_scan < ? THEN uptime + ? - last_scan ELSE uptime END, last_scan = ?, settings = CASE WHEN ? THEN ? ELSE settings END, - v2_settings = CASE WHEN ? THEN ? ELSE settings END, + v2_settings = CASE WHEN ? THEN ? ELSE v2_settings END, price_table = CASE WHEN ? THEN ? ELSE price_table END, price_table_expiry = CASE WHEN ? THEN ? ELSE price_table_expiry END, successful_interactions = CASE WHEN ? THEN successful_interactions + 1 ELSE successful_interactions END, From 38e8f301abd67069b96ff4924a5bb48ac85e08a8 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 12:54:40 +0100 Subject: [PATCH 23/37] fix TestObjectBasic --- stores/metadata_test.go | 16 ++-------------- stores/sql/main.go | 6 +++--- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/stores/metadata_test.go b/stores/metadata_test.go index c23964821..6a6d7039e 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -245,19 +245,7 @@ func TestObjectBasic(t *testing.T) { t.Fatal(err) } if !reflect.DeepEqual(*got.Object, want) { - t.Fatal("object mismatch", got.Object, want) - } - - // update the sector to have a non-consecutive slab index - _, err = ss.DB().Exec(context.Background(), "UPDATE sectors SET slab_index = 100 WHERE slab_index = 1") - if err != nil { - t.Fatalf("failed to update sector: %v", err) - } - - // fetch the object again and assert we receive an indication it was corrupted - _, err = ss.Object(context.Background(), testBucket, "/"+t.Name()) - if !errors.Is(err, api.ErrObjectCorrupted) { - t.Fatal("unexpected err", err) + t.Fatal("object mismatch", cmp.Diff(*got.Object, want, cmp.AllowUnexported(object.EncryptionKey{}))) } // create an object without slabs @@ -272,7 +260,7 @@ func TestObjectBasic(t *testing.T) { t.Fatal(err) } if !reflect.DeepEqual(*got2.Object, want2) { - t.Fatal("object mismatch", cmp.Diff(got2.Object, want2)) + t.Fatal("object mismatch", cmp.Diff(*got2.Object, want2, cmp.AllowUnexported(object.EncryptionKey{}))) } } diff --git a/stores/sql/main.go b/stores/sql/main.go index a45a04dff..f1debfe4e 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -2576,7 +2576,7 @@ func Object(ctx context.Context, tx Tx, bucket, key string) (api.Object, error) // fetch slab slices rows, err = tx.Query(ctx, ` - SELECT sla.id, sla.health, sla.key, sla.min_shards + SELECT sla.id, sla.health, sla.key, sla.min_shards, sli.offset, sli.length FROM slices sli INNER JOIN slabs sla ON sli.db_slab_id = sla.id WHERE sli.db_object_id = ? @@ -2587,11 +2587,11 @@ func Object(ctx context.Context, tx Tx, bucket, key string) (api.Object, error) } defer rows.Close() - var slabSlices []object.SlabSlice + slabSlices := object.SlabSlices{} for rows.Next() { var id int64 var ss object.SlabSlice - if err := rows.Scan(&id, &ss.Health, (*EncryptionKey)(&ss.EncryptionKey), &ss.MinShards); err != nil { + if err := rows.Scan(&id, &ss.Health, (*EncryptionKey)(&ss.EncryptionKey), &ss.MinShards, &ss.Offset, &ss.Length); err != nil { return api.Object{}, fmt.Errorf("failed to scan slab slice: %w", err) } slabSlices = append(slabSlices, ss) From 9f89ad7bbdba1b1476d56a296c7399f5a074d3d6 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 13:16:20 +0100 Subject: [PATCH 24/37] add TestSlabSectorOnHostButNotInContract --- stores/metadata_test.go | 50 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/stores/metadata_test.go b/stores/metadata_test.go index 6a6d7039e..90a386667 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -3622,18 +3622,62 @@ func TestUpdateSlabSanityChecks(t *testing.T) { if err := ss.UpdateSlab(context.Background(), slab.EncryptionKey, sectors); !errors.Is(err, api.ErrUnknownSector) { t.Fatal(err) } +} + +func TestSlabSectorOnHostButNotInContract(t *testing.T) { + ss := newTestSQLStore(t, defaultTestSQLStoreConfig) + defer ss.Close() + + // create host and 2 contracts with it + hk := types.PublicKey{1} + err := ss.addTestHost(hk) + if err != nil { + t.Fatal(err) + } + _, contracts, err := ss.addTestContracts([]types.PublicKey{hk, hk}) + if err != nil { + t.Fatal(err) + } + + // prepare a slab - it has one shard that is pinned to contract 0. + shard := newTestShard(hk, contracts[0].ID, types.Hash256{1}) + slab := object.Slab{ + EncryptionKey: object.GenerateEncryptionKey(object.EncryptionKeyTypeSalted), + Shards: []object.Sector{shard}, + Health: 1, + } + + // set slab. + _, err = ss.addTestObject("/"+t.Name(), object.Object{ + Key: object.GenerateEncryptionKey(object.EncryptionKeyTypeSalted), + Slabs: []object.SlabSlice{{Slab: slab}}, + }) + if err != nil { + t.Fatal(err) + } + + rSlab, err := ss.Slab(context.Background(), slab.EncryptionKey) + if err != nil { + t.Fatal(err) + } else if len(rSlab.Shards) != 1 { + t.Fatal("should have 1 shard", len(rSlab.Shards)) + } else if fcids, exists := rSlab.Shards[0].Contracts[hk]; !exists || len(fcids) != 1 { + t.Fatalf("unexpected contracts %v, exists %v", fcids, exists) + } // delete one of the contracts - this should cause the host to still be in // the slab but the associated slice should be empty if err := ss.ArchiveContract(context.Background(), contracts[0].ID, "test"); err != nil { t.Fatal(err) } - slab.Shards[0].Contracts[hks[0]] = []types.FileContractID{} + rSlab, err = ss.Slab(context.Background(), slab.EncryptionKey) if err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(slab, rSlab) { - t.Fatal("unexpected slab", cmp.Diff(slab, rSlab, cmp.AllowUnexported(object.EncryptionKey{}))) + } else if len(rSlab.Shards) != 1 { + t.Fatal("should have 1 shard", len(rSlab.Shards)) + } else if fcids, exists := rSlab.Shards[0].Contracts[hk]; !exists || len(fcids) != 0 { + t.Fatalf("unexpected contracts %v, exists %v", fcids, exists) } } From 7c12fa99c4e072a379f2f4e3e295280c5636508d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 13:25:56 +0100 Subject: [PATCH 25/37] fix ndf in TestV1ToV2Transition --- internal/test/e2e/cluster_test.go | 29 +++++++++++++++++------------ stores/sql/main.go | 2 +- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index fde4b2625..3ff1ff463 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2950,8 +2950,6 @@ func TestV1ToV2Transition(t *testing.T) { for _, c := range archivedContracts { if c.ArchivalReason != "migrated to v2" { t.Fatalf("expected archival reason to be 'migrated to v2', got %v", c.ArchivalReason) - } else if c.V2 { - t.Fatalf("expected contract to be v1, got v2") } usedHosts[c.HostKey] = struct{}{} } @@ -2976,24 +2974,31 @@ func TestV1ToV2Transition(t *testing.T) { delete(usedHosts, c.HostKey) } - // check health is 1 tt.Retry(100, 100*time.Millisecond, func() error { + // check health is 1 object, err := cluster.Bus.Object(context.Background(), testBucket, "foo", api.GetObjectOptions{}) tt.OK(err) if object.Health != 1 { return fmt.Errorf("expected health to be 1, got %v", object.Health) } - return nil - }) - // check that the contracts now contain the data - activeContracts, err = cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeActive}) - tt.OK(err) - for _, c := range activeContracts { - if c.Size != rhpv4.SectorSize { - t.Fatalf("expected sector size to be %v, got %v", rhpv4.SectorSize, c.Size) + // check that the contracts now contain the data + activeContracts, err = cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeActive}) + tt.OK(err) + for _, c := range activeContracts { + // check revision + rev, err := cluster.Bus.ContractRevision(context.Background(), c.ID) + tt.OK(err) + if rev.Size != rhpv4.SectorSize { + return fmt.Errorf("expected sector size to be %v, got %v", rhpv4.SectorSize, rev.Size) + } + // check local metadata + if c.Size != rhpv4.SectorSize { + return fmt.Errorf("expected sector size to be %v, got %v", rhpv4.SectorSize, c.Size) + } } - } + return nil + }) // download file to make sure it's still there buf := new(bytes.Buffer) diff --git a/stores/sql/main.go b/stores/sql/main.go index f1debfe4e..636827822 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -201,7 +201,7 @@ func ArchiveContract(ctx context.Context, tx sql.Tx, fcid types.FileContractID, INNER JOIN hosts ON contracts.host_id = hosts.id WHERE contracts.archival_reason IS NULL AND hosts.id = host_sectors.db_host_id - )`, time.Now()) + )`) if err != nil { return fmt.Errorf("failed to delete host_sectors: %w", err) } From 68a4e3a788ccb151c4ef034f6cfdb0d8253cb717 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 14:55:17 +0100 Subject: [PATCH 26/37] address comments --- internal/bus/chainsubscriber.go | 8 ++++---- stores/sql/chain.go | 9 ++++----- stores/sql/mysql/chain.go | 4 ---- stores/sql/mysql/main.go | 7 ++++--- stores/sql/sqlite/chain.go | 4 ---- stores/sql/sqlite/main.go | 7 ++++--- 6 files changed, 16 insertions(+), 23 deletions(-) diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index 9445e2f19..23da73cfa 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -200,7 +200,7 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply cau.ForEachFileContractElement(func(fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, valid bool) { if err != nil { return - } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { + } else if known, lookupErr := tx.IsKnownContract(fce.ID); lookupErr != nil { err = lookupErr return } else if !known { @@ -217,7 +217,7 @@ func (s *chainSubscriber) applyChainUpdate(tx sql.ChainUpdateTx, cau chain.Apply cau.ForEachV2FileContractElement(func(fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { if err != nil { return - } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { + } else if known, lookupErr := tx.IsKnownContract(fce.ID); lookupErr != nil { err = lookupErr return } else if !known { @@ -268,7 +268,7 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve cru.ForEachFileContractElement(func(fce types.FileContractElement, created bool, rev *types.FileContractElement, resolved, _ bool) { if err != nil { return - } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { + } else if known, lookupErr := tx.IsKnownContract(fce.ID); lookupErr != nil { err = lookupErr return } else if !known { @@ -285,7 +285,7 @@ func (s *chainSubscriber) revertChainUpdate(tx sql.ChainUpdateTx, cru chain.Reve cru.ForEachV2FileContractElement(func(fce types.V2FileContractElement, created bool, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { if err != nil { return - } else if known, lookupErr := tx.IsKnownContract(fce.ID); err != nil { + } else if known, lookupErr := tx.IsKnownContract(fce.ID); lookupErr != nil { err = lookupErr return } else if !known { diff --git a/stores/sql/chain.go b/stores/sql/chain.go index 990c2c4cf..c872adfd0 100644 --- a/stores/sql/chain.go +++ b/stores/sql/chain.go @@ -205,10 +205,9 @@ func FileContractElement(ctx context.Context, tx sql.Tx, fcid types.FileContract } func IsKnownContract(ctx context.Context, tx sql.Tx, fcid types.FileContractID) (known bool, _ error) { - err := tx.QueryRow(ctx, "SELECT 1 FROM contracts WHERE fcid = ?", FileContractID(fcid)).Scan(&known) - if errors.Is(err, dsql.ErrNoRows) { - return false, nil - } else if err != nil { + err := tx.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM contracts WHERE fcid = ?)", FileContractID(fcid)). + Scan(&known) + if err != nil { return false, err } return known, nil @@ -221,7 +220,7 @@ func RecordContractRenewal(ctx context.Context, tx sql.Tx, oldFCID, newFCID type } _, err = tx.Exec(ctx, "UPDATE contracts SET contracts.renewed_from = ? WHERE contracts.fcid = ?", FileContractID(oldFCID), FileContractID(newFCID)) if err != nil { - return fmt.Errorf("failed to update renewed_to of new contract: %w", err) + return fmt.Errorf("failed to update renewed_from of new contract: %w", err) } return nil } diff --git a/stores/sql/mysql/chain.go b/stores/sql/mysql/chain.go index 1d2683075..8d854fb27 100644 --- a/stores/sql/mysql/chain.go +++ b/stores/sql/mysql/chain.go @@ -212,10 +212,6 @@ func (c chainUpdateTx) FileContractElement(fcid types.FileContractID) (types.V2F } func (c chainUpdateTx) IsKnownContract(fcid types.FileContractID) (bool, error) { - if c.known == nil { - c.known = make(map[types.FileContractID]bool) - } - if relevant, ok := c.known[fcid]; ok { return relevant, nil } diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index 8d856fe8c..7dba272ec 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -654,9 +654,10 @@ func (tx *MainDatabaseTx) Peers(ctx context.Context) ([]syncer.PeerInfo, error) func (tx *MainDatabaseTx) ProcessChainUpdate(ctx context.Context, fn func(ssql.ChainUpdateTx) error) error { return fn(&chainUpdateTx{ - ctx: ctx, - tx: tx, - l: tx.log.Named("ProcessChainUpdate"), + ctx: ctx, + known: make(map[types.FileContractID]bool), + tx: tx, + l: tx.log.Named("ProcessChainUpdate"), }) } diff --git a/stores/sql/sqlite/chain.go b/stores/sql/sqlite/chain.go index f3abf5353..cf642e8fe 100644 --- a/stores/sql/sqlite/chain.go +++ b/stores/sql/sqlite/chain.go @@ -213,10 +213,6 @@ func (c chainUpdateTx) FileContractElement(fcid types.FileContractID) (types.V2F } func (c chainUpdateTx) IsKnownContract(fcid types.FileContractID) (bool, error) { - if c.known == nil { - c.known = make(map[types.FileContractID]bool) - } - if relevant, ok := c.known[fcid]; ok { return relevant, nil } diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 51fcd5592..0e488c6ed 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -650,9 +650,10 @@ func (tx *MainDatabaseTx) Peers(ctx context.Context) ([]syncer.PeerInfo, error) func (tx *MainDatabaseTx) ProcessChainUpdate(ctx context.Context, fn func(ssql.ChainUpdateTx) error) (err error) { return fn(&chainUpdateTx{ - ctx: ctx, - tx: tx, - l: tx.log.Named("ProcessChainUpdate"), + ctx: ctx, + known: make(map[types.FileContractID]bool), + tx: tx, + l: tx.log.Named("ProcessChainUpdate"), }) } From 987d57c05dae2b2627218a833183b3a4d1ca7455 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 16 Dec 2024 15:09:16 +0100 Subject: [PATCH 27/37] fix panic in RecordContractRenewal --- stores/sql/chain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stores/sql/chain.go b/stores/sql/chain.go index c872adfd0..35a91b9d0 100644 --- a/stores/sql/chain.go +++ b/stores/sql/chain.go @@ -214,11 +214,11 @@ func IsKnownContract(ctx context.Context, tx sql.Tx, fcid types.FileContractID) } func RecordContractRenewal(ctx context.Context, tx sql.Tx, oldFCID, newFCID types.FileContractID) error { - _, err := tx.Exec(ctx, "UPDATE contracts SET contracts.renewed_to = ? WHERE contracts.fcid = ?", FileContractID(newFCID), FileContractID(oldFCID)) + _, err := tx.Exec(ctx, "UPDATE contracts SET renewed_to = ? WHERE fcid = ?", FileContractID(newFCID), FileContractID(oldFCID)) if err != nil { return fmt.Errorf("failed to update renewed_to of old contract: %w", err) } - _, err = tx.Exec(ctx, "UPDATE contracts SET contracts.renewed_from = ? WHERE contracts.fcid = ?", FileContractID(oldFCID), FileContractID(newFCID)) + _, err = tx.Exec(ctx, "UPDATE contracts SET renewed_from = ? WHERE fcid = ?", FileContractID(oldFCID), FileContractID(newFCID)) if err != nil { return fmt.Errorf("failed to update renewed_from of new contract: %w", err) } From 8d127ff7befac26857fe6cf666921db1fa6bf0dc Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 11:36:55 +0100 Subject: [PATCH 28/37] address comments --- autopilot/contractor/hostfilter.go | 7 +------ go.mod | 2 +- go.sum | 4 ++-- stores/sql/main.go | 13 ++++++++++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/autopilot/contractor/hostfilter.go b/autopilot/contractor/hostfilter.go index d2d349479..b9dd6064d 100644 --- a/autopilot/contractor/hostfilter.go +++ b/autopilot/contractor/hostfilter.go @@ -8,7 +8,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" "go.sia.tech/renterd/internal/gouging" - rhp4 "go.sia.tech/renterd/internal/rhp/v4" ) const ( @@ -166,11 +165,7 @@ func checkHost(gc gouging.Checker, sh scoredHost, minScore float64, period uint6 // calculate remaining host info fields if !h.IsAnnounced() { ub.NotAnnounced = true - } else if !h.Scanned || - // NOTE: a v2 host might have been scanned before the v2 height so strictly - // speaking it is scanned but since it hasn't been scanned since, the - // settings aren't set so we treat it as not scanned - (h.IsV2() && h.V2Settings == (rhp4.HostSettings{})) { + } else if !h.Scanned { ub.NotCompletingScan = true } else { // online check diff --git a/go.mod b/go.mod index d9e8802ef..a9afd1705 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/montanaflynn/stats v0.7.1 github.com/shopspring/decimal v1.4.0 go.sia.tech/core v0.8.0 - go.sia.tech/coreutils v0.8.0 + go.sia.tech/coreutils v0.8.1-0.20241217101542-5d6fc37cbb94 go.sia.tech/gofakes3 v0.0.5 go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed go.sia.tech/jape v0.12.1 diff --git a/go.sum b/go.sum index d35879f1a..5bac16294 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.sia.tech/core v0.8.0 h1:J6vZQlVhpj4bTVeuC2GKkfkGEs8jf0j651Kl1wwOxjg= go.sia.tech/core v0.8.0/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= -go.sia.tech/coreutils v0.8.0 h1:1dcl0vxY+MBgAdJ7PdewAr8RkZJn4/6wAKEZfi4iYn0= -go.sia.tech/coreutils v0.8.0/go.mod h1:ml5MefDMWCvPKNeRVIGHmyF5tv27C9h1PiI/iOiTGLg= +go.sia.tech/coreutils v0.8.1-0.20241217101542-5d6fc37cbb94 h1:1fbD59wfyA1+5LmLYNh+ukNpkbtEmQgcXYlRUZTdr+M= +go.sia.tech/coreutils v0.8.1-0.20241217101542-5d6fc37cbb94/go.mod h1:ml5MefDMWCvPKNeRVIGHmyF5tv27C9h1PiI/iOiTGLg= go.sia.tech/gofakes3 v0.0.5 h1:vFhVBUFbKE9ZplvLE2w4TQxFMQyF8qvgxV4TaTph+Vw= go.sia.tech/gofakes3 v0.0.5/go.mod h1:LXEzwGw+OHysWLmagleCttX93cJZlT9rBu/icOZjQ54= go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed h1:C42AxWwwoP13EhZsdWwR17Rc9S7gXI4JnRN0AyZRxc8= diff --git a/stores/sql/main.go b/stores/sql/main.go index 636827822..c012b7ede 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -898,6 +898,14 @@ LEFT JOIN host_checks hc ON hc.db_host_id = h.id // fill in v2 addresses err = fillInV2Addresses(ctx, tx, hostIDs, func(i int, addrs []string) { hosts[i].V2SiamuxAddresses = addrs + + // NOTE: a v2 host might have been scanned before the v2 height so strictly + // speaking it is scanned but since it hasn't been scanned since, the + // settings aren't set so we treat it as not scanned + if hosts[i].IsV2() && hosts[i].V2Settings == (rhp.HostSettings{}) { + hosts[i].Scanned = false + } + i++ }) if err != nil { @@ -2576,7 +2584,7 @@ func Object(ctx context.Context, tx Tx, bucket, key string) (api.Object, error) // fetch slab slices rows, err = tx.Query(ctx, ` - SELECT sla.id, sla.health, sla.key, sla.min_shards, sli.offset, sli.length + SELECT sla.health, sla.key, sla.min_shards, sli.offset, sli.length FROM slices sli INNER JOIN slabs sla ON sli.db_slab_id = sla.id WHERE sli.db_object_id = ? @@ -2589,9 +2597,8 @@ func Object(ctx context.Context, tx Tx, bucket, key string) (api.Object, error) slabSlices := object.SlabSlices{} for rows.Next() { - var id int64 var ss object.SlabSlice - if err := rows.Scan(&id, &ss.Health, (*EncryptionKey)(&ss.EncryptionKey), &ss.MinShards, &ss.Offset, &ss.Length); err != nil { + if err := rows.Scan(&ss.Health, (*EncryptionKey)(&ss.EncryptionKey), &ss.MinShards, &ss.Offset, &ss.Length); err != nil { return api.Object{}, fmt.Errorf("failed to scan slab slice: %w", err) } slabSlices = append(slabSlices, ss) From a73862b452d553f5970f05a7413b283f370f01b3 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 13:39:27 +0100 Subject: [PATCH 29/37] fix TestV1ToV2Transition --- internal/test/e2e/cluster_test.go | 35 ++++++++++++++++++++++++------- internal/upload/uploadmanager.go | 30 ++++++++++++++++++-------- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 0b9480fdf..992d21065 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2990,20 +2990,39 @@ func TestV1ToV2Transition(t *testing.T) { rev, err := cluster.Bus.ContractRevision(context.Background(), c.ID) tt.OK(err) if rev.Size != rhpv4.SectorSize { - return fmt.Errorf("expected sector size to be %v, got %v", rhpv4.SectorSize, rev.Size) + return fmt.Errorf("expected revision size to be %v, got %v", rhpv4.SectorSize, rev.Size) } // check local metadata if c.Size != rhpv4.SectorSize { - return fmt.Errorf("expected sector size to be %v, got %v", rhpv4.SectorSize, c.Size) + return fmt.Errorf("expected contract size to be %v, got %v", rhpv4.SectorSize, c.Size) + } + // one of the shards should be on this contract + var found bool + for _, shard := range slab.Shards { + for _, fcid := range shard.Contracts[c.HostKey] { + found = found || fcid == c.ID + } + } + if !found { + t.Fatal("expected contract to shard data") } } return nil }) - // download file to make sure it's still there - buf := new(bytes.Buffer) - tt.OK(cluster.Worker.DownloadObject(context.Background(), buf, testBucket, "foo", api.DownloadObjectOptions{})) - if !bytes.Equal(data, buf.Bytes()) { - t.Fatal("data mismatch") - } + // download file to make sure it's still available + // NOTE: 1st try fails since the accounts appear not to be funded since the + // test host has separate account managers for rhp3 and rhp4 + tt.FailAll(cluster.Worker.DownloadObject(context.Background(), bytes.NewBuffer(nil), testBucket, "foo", api.DownloadObjectOptions{})) + + // subsequent tries succeed + tt.Retry(100, 100*time.Millisecond, func() error { + buf := new(bytes.Buffer) + if err := cluster.Worker.DownloadObject(context.Background(), buf, testBucket, "foo", api.DownloadObjectOptions{}); err != nil { + return err + } else if !bytes.Equal(data, buf.Bytes()) { + t.Fatal("data mismatch") + } + return nil + }) } diff --git a/internal/upload/uploadmanager.go b/internal/upload/uploadmanager.go index 130be696e..357c9956b 100644 --- a/internal/upload/uploadmanager.go +++ b/internal/upload/uploadmanager.go @@ -471,9 +471,6 @@ func (mgr *Manager) UploadShards(ctx context.Context, s object.Slab, shardIndice // upload the shards uploaded, uploadSpeed, overdrivePct, err := upload.uploadShards(ctx, shards, mgr.candidates(upload.allowed), mem, mgr.maxOverdrive, mgr.overdriveTimeout) - if err != nil { - return err - } // build sectors var sectors []api.UploadedSector @@ -483,13 +480,22 @@ func (mgr *Manager) UploadShards(ctx context.Context, s object.Slab, shardIndice Root: sector.root, }) } + if len(sectors) > 0 { + if err := mgr.os.UpdateSlab(ctx, s.EncryptionKey, sectors); err != nil { + return fmt.Errorf("couldn't update slab: %w", err) + } + } + + // check error + if err != nil { + return err + } // track stats mgr.statsOverdrivePct.Track(overdrivePct) mgr.statsSlabUploadSpeedBytesPerMS.Track(float64(uploadSpeed)) - // update the slab - return mgr.os.UpdateSlab(ctx, s.EncryptionKey, sectors) + return nil } func (mgr *Manager) candidates(allowed map[types.PublicKey]struct{}) (candidates []*uploader.Uploader) { @@ -672,6 +678,9 @@ func (u *upload) uploadSlab(ctx context.Context, rs api.RedundancySettings, data return uploadSpeed, overdrivePct } +// uploadShards uploads the shards to the provided candidates. It returns an +// error if it fails to upload all shards but len(sectors) will be > 0 if some +// shards were uploaded successfully. func (u *upload) uploadShards(ctx context.Context, shards [][]byte, candidates []*uploader.Uploader, mem memory.Memory, maxOverdrive uint64, overdriveTimeout time.Duration) (sectors []uploadedSector, uploadSpeed int64, overdrivePct float64, err error) { // ensure inflight uploads get cancelled ctx, cancel := context.WithCancel(ctx) @@ -770,6 +779,13 @@ loop: } } + // collect the sectors + for _, sector := range slab.sectors { + if sector.isUploaded() { + sectors = append(sectors, sector.uploaded) + } + } + // calculate the upload speed bytes := slab.numUploaded * rhpv2.SectorSize ms := time.Since(start).Milliseconds() @@ -791,10 +807,6 @@ loop: return } - // collect the sectors - for _, sector := range slab.sectors { - sectors = append(sectors, sector.uploaded) - } return } From d3acee32719a48472a3fdc382594be847488b51b Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 13:42:31 +0100 Subject: [PATCH 30/37] fix lint --- internal/test/e2e/cluster_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 992d21065..8b9fb5dd0 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2981,6 +2981,7 @@ func TestV1ToV2Transition(t *testing.T) { if object.Health != 1 { return fmt.Errorf("expected health to be 1, got %v", object.Health) } + slab := object.Slabs[0] // check that the contracts now contain the data activeContracts, err = cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeActive}) From 300333fcafa9ad9d6d7447b73a5667a7a5ae68b8 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 15:26:47 +0100 Subject: [PATCH 31/37] fix TestDownloadAllHosts NDF --- internal/test/e2e/cluster_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 8b9fb5dd0..3eb37c02f 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2601,7 +2601,7 @@ func TestDownloadAllHosts(t *testing.T) { // block the new host but unblock the old one for _, host := range cluster.hosts { if host.PublicKey() == newHost { - toBlock := []string{host.settings.Settings().NetAddress, host.RHPv4Addr()} + toBlock := []string{host.RHPv2Addr(), host.RHPv4Addr()} tt.OK(b.UpdateHostBlocklist(context.Background(), toBlock, randomHost, false)) } } @@ -2614,6 +2614,7 @@ func TestDownloadAllHosts(t *testing.T) { tt.OK(host.UpdateSettings(settings)) } } + time.Sleep(testWorkerCfg().CacheExpiry) // expire cache // download the object dst = new(bytes.Buffer) From 7ea2cb42ea8109d4ee935ba171d7902c4f228d69 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 15:48:44 +0100 Subject: [PATCH 32/37] check contract usability --- stores/sql/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stores/sql/main.go b/stores/sql/main.go index c012b7ede..c8f2f1d8c 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -2237,10 +2237,10 @@ EXISTS ( h.settings, h.v2_settings FROM hosts h - INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL + INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL AND c.usability = ? INNER JOIN host_checks hc on hc.db_host_id = h.id WHERE %s - GROUP by h.id`, strings.Join(whereExprs, " AND "))) + GROUP by h.id`, strings.Join(whereExprs, " AND ")), contractUsabilityGood) if err != nil { return nil, fmt.Errorf("failed to fetch hosts: %w", err) } From a8b44e4adf5a5e621c9d19610a9301c8ab82bfd1 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 16:07:57 +0100 Subject: [PATCH 33/37] release inputs when failing to broadcast contract element --- internal/bus/chainsubscriber.go | 10 +++++++++- internal/test/e2e/cluster_test.go | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index 23da73cfa..7a1e6ab23 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math" + "strings" "sync" "time" @@ -71,6 +72,7 @@ type ( Wallet interface { FundV2Transaction(txn *types.V2Transaction, amount types.Currency, useUnconfirmed bool) (types.ChainIndex, []int, error) + ReleaseInputs(txns []types.Transaction, v2txns []types.V2Transaction) SignV2Inputs(txn *types.V2Transaction, toSign []int) UpdateChainState(tx wallet.UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error } @@ -433,8 +435,14 @@ func (s *chainSubscriber) broadcastExpiredFileContractResolutions(tx sql.ChainUp // verify txn and broadcast it _, err = s.cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn}) - if err != nil { + if err != nil && + (strings.Contains(err.Error(), "has already been resolved") || + strings.Contains(err.Error(), "not present in the accumulator")) { + s.wallet.ReleaseInputs(nil, []types.V2Transaction{txn}) + continue + } else if err != nil { s.logger.Errorf("failed to broadcast contract expiration txn: %v", err) + s.wallet.ReleaseInputs(nil, []types.V2Transaction{txn}) continue } s.s.BroadcastV2TransactionSet(basis, []types.V2Transaction{txn}) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 3eb37c02f..8ed1e61f0 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2833,6 +2833,7 @@ func TestContractFundsReturnWhenHostOffline(t *testing.T) { // mine until the contract is expired cluster.mineBlocks(types.VoidAddress, contract.WindowEnd-cs.BlockHeight) + cluster.sync() expectedBalance := wallet.Confirmed.Add(contract.InitialRenterFunds).Sub(fee.Mul64(ibus.ContractResolutionTxnWeight)) cluster.tt.Retry(10, time.Second, func() error { From 74fa72fbffd4f39876e30808345111ec54f5b637 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 16:20:23 +0100 Subject: [PATCH 34/37] fix error string --- internal/test/e2e/cluster_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 8ed1e61f0..6398c0e12 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2944,7 +2944,7 @@ func TestV1ToV2Transition(t *testing.T) { archivedContracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeArchived}) tt.OK(err) if len(archivedContracts) != nHosts-1 { - t.Fatalf("expected %v archived contracts, got %v", 2*(nHosts-1), len(archivedContracts)) + t.Fatalf("expected %v archived contracts, got %v", nHosts-1, len(archivedContracts)) } // they should be on nHosts-1 unique hosts From fd0e06be17a8de35fdb6e54f5025bad65a3cafc2 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 16:24:20 +0100 Subject: [PATCH 35/37] replace sleep with retry --- internal/test/e2e/cluster_test.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 6398c0e12..2a1713c00 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2938,14 +2938,17 @@ func TestV1ToV2Transition(t *testing.T) { cluster.MineBlocks(1) time.Sleep(100 * time.Millisecond) } - time.Sleep(time.Second) // check that we have 1 archived contract for every contract we had before - archivedContracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeArchived}) - tt.OK(err) - if len(archivedContracts) != nHosts-1 { - t.Fatalf("expected %v archived contracts, got %v", nHosts-1, len(archivedContracts)) - } + var archivedContracts []api.ContractMetadata + tt.Retry(100, 100*time.Millisecond, func() error { + archivedContracts, err = cluster.Bus.Contracts(context.Background(), api.ContractsOpts{FilterMode: api.ContractFilterModeArchived}) + tt.OK(err) + if len(archivedContracts) != nHosts-1 { + return fmt.Errorf("expected %v archived contracts, got %v", nHosts-1, len(archivedContracts)) + } + return nil + }) // they should be on nHosts-1 unique hosts usedHosts := make(map[types.PublicKey]struct{}) From 54b1ef81ba1f5299d9997460228fda67e643b47f Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 17 Dec 2024 16:49:50 +0100 Subject: [PATCH 36/37] use account token helper --- go.mod | 4 ++-- go.sum | 8 ++++---- internal/accounts/accounts.go | 8 ++------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index a9afd1705..f528783ef 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.24 github.com/montanaflynn/stats v0.7.1 github.com/shopspring/decimal v1.4.0 - go.sia.tech/core v0.8.0 - go.sia.tech/coreutils v0.8.1-0.20241217101542-5d6fc37cbb94 + go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b + go.sia.tech/coreutils v0.8.1-0.20241217153531-b5e84c03d17f go.sia.tech/gofakes3 v0.0.5 go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed go.sia.tech/jape v0.12.1 diff --git a/go.sum b/go.sum index 5bac16294..46f6e4435 100644 --- a/go.sum +++ b/go.sum @@ -55,10 +55,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.8.0 h1:J6vZQlVhpj4bTVeuC2GKkfkGEs8jf0j651Kl1wwOxjg= -go.sia.tech/core v0.8.0/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= -go.sia.tech/coreutils v0.8.1-0.20241217101542-5d6fc37cbb94 h1:1fbD59wfyA1+5LmLYNh+ukNpkbtEmQgcXYlRUZTdr+M= -go.sia.tech/coreutils v0.8.1-0.20241217101542-5d6fc37cbb94/go.mod h1:ml5MefDMWCvPKNeRVIGHmyF5tv27C9h1PiI/iOiTGLg= +go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b h1:VRkb6OOX1KawLQwuqOEHLcjha8gxVX0tAyu2Dyoq8Ek= +go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= +go.sia.tech/coreutils v0.8.1-0.20241217153531-b5e84c03d17f h1:TafvnqJgx/+0zX/QMSOOkf5HfMqaoe/73eO515fUucI= +go.sia.tech/coreutils v0.8.1-0.20241217153531-b5e84c03d17f/go.mod h1:xhIbFjjkzmCF8Dt73ZvquaBQCT2Dje7AKYBRAesn93w= go.sia.tech/gofakes3 v0.0.5 h1:vFhVBUFbKE9ZplvLE2w4TQxFMQyF8qvgxV4TaTph+Vw= go.sia.tech/gofakes3 v0.0.5/go.mod h1:LXEzwGw+OHysWLmagleCttX93cJZlT9rBu/icOZjQ54= go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed h1:C42AxWwwoP13EhZsdWwR17Rc9S7gXI4JnRN0AyZRxc8= diff --git a/internal/accounts/accounts.go b/internal/accounts/accounts.go index 2a67b7002..72d35ee9d 100644 --- a/internal/accounts/accounts.go +++ b/internal/accounts/accounts.go @@ -448,12 +448,8 @@ func (a *Manager) refillAccount(ctx context.Context, contract api.ContractMetada } func (a *Account) Token() rhpv4.AccountToken { - t := rhpv4.AccountToken{ - Account: rhpv4.Account(a.key.PublicKey()), - ValidUntil: time.Now().Add(5 * time.Minute), - } - t.Signature = a.key.SignHash(t.SigHash()) - return t + account := rhpv4.Account(a.key.PublicKey()) + return account.Token(a.key, a.acc.HostKey) } // WithSync syncs an accounts balance with the bus. To do so, the account is From 4e8ab849738fa4b1ef2e3810688a59008a1f8598 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 18 Dec 2024 11:07:31 +0100 Subject: [PATCH 37/37] address comments --- internal/bus/chainsubscriber.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index 7a1e6ab23..0224ab0f3 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -439,9 +439,10 @@ func (s *chainSubscriber) broadcastExpiredFileContractResolutions(tx sql.ChainUp (strings.Contains(err.Error(), "has already been resolved") || strings.Contains(err.Error(), "not present in the accumulator")) { s.wallet.ReleaseInputs(nil, []types.V2Transaction{txn}) + s.logger.With(zap.Error(err)).Debug("failed to broadcast contract expiration txn") continue } else if err != nil { - s.logger.Errorf("failed to broadcast contract expiration txn: %v", err) + s.logger.With(zap.Error(err)).Error("failed to broadcast contract expiration txn") s.wallet.ReleaseInputs(nil, []types.V2Transaction{txn}) continue }