e2e: Add integration test for network transition from v1 to v2

autopilot/contractor/contractor.go

@@ -463,7 +463,9 @@ func activeContracts(ctx context.Context, bus Bus, logger *zap.SugaredLogger) ([
 	// fetch active contracts
 	logger.Info("fetching active contracts")
 	start := time.Now()
-	metadatas, err := bus.Contracts(ctx, api.ContractsOpts{FilterMode: api.ContractFilterModeActive})
+	metadatas, err := bus.Contracts(ctx, api.ContractsOpts{
+		FilterMode: api.ContractFilterModeActive,
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -907,7 +909,9 @@ func performContractFormations(ctx *mCtx, bus Bus, cr contractReviser, hf hostFi
 	wanted := int(ctx.WantedContracts())
 
 	// fetch all active contracts
-	contracts, err := bus.Contracts(ctx, api.ContractsOpts{})
+	contracts, err := bus.Contracts(ctx, api.ContractsOpts{
+		FilterMode: api.ContractFilterModeActive,
+	})
 	if err != nil {
 		return 0, fmt.Errorf("failed to fetch contracts: %w", err)
 	}
@@ -1057,7 +1061,9 @@ func performHostChecks(ctx *mCtx, bus Bus, logger *zap.SugaredLogger) error {
 
 func performPostMaintenanceTasks(ctx *mCtx, bus Bus, alerter alerts.Alerter, cc contractChecker, rb revisionBroadcaster, logger *zap.SugaredLogger) error {
 	// fetch some contract and host info
-	allContracts, err := bus.Contracts(ctx, api.ContractsOpts{})
+	allContracts, err := bus.Contracts(ctx, api.ContractsOpts{
+		FilterMode: api.ContractFilterModeActive,
+	})
 	if err != nil {
 		return fmt.Errorf("failed to fetch all contracts: %w", err)
 	}
@@ -1122,7 +1128,7 @@ func performV2ContractMigration(ctx *mCtx, bus Bus, cr contractReviser, logger *
 	}
 
 	contracts, err := bus.Contracts(ctx, api.ContractsOpts{
-		FilterMode: api.ContractFilterModeAll, // TODO: change to usable
+		FilterMode: api.ContractFilterModeActive,
 	})
 	if err != nil {
 		logger.With(zap.Error(err)).Error("failed to fetch contracts for migration")
@@ -1154,7 +1160,7 @@ func performV2ContractMigration(ctx *mCtx, bus Bus, cr contractReviser, logger *
 		}
 
 		// form a new contract with the same host
-		contract, _, err := cr.formContract(ctx, bus, host, InitialContractFunding, logger)
+		_, _, err = cr.formContract(ctx, bus, host, InitialContractFunding, logger)
 		if err != nil {
 			logger.Errorf("failed to form a v2 contract with the host")
 			continue

autopilot/contractor/hostfilter.go

@@ -8,6 +8,7 @@ import (
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
 	"go.sia.tech/renterd/internal/gouging"
+	rhp4 "go.sia.tech/renterd/internal/rhp/v4"
 )
 
 const (
@@ -165,7 +166,11 @@ func checkHost(gc gouging.Checker, sh scoredHost, minScore float64, period uint6
 	// calculate remaining host info fields
 	if !h.IsAnnounced() {
 		ub.NotAnnounced = true
-	} else if !h.Scanned {
+	} else if !h.Scanned ||
+		// NOTE: a v2 host might have been scanned before the v2 height so strictly
+		// speaking it is scanned but since it hasn't been scanned since, the
+		// settings aren't set so we treat it as not scanned
+		(h.IsV2() && h.V2Settings == (rhp4.HostSettings{})) {
 		ub.NotCompletingScan = true
 	} else {
 		// online check

bus/bus.go

@@ -687,9 +687,6 @@ func (b *Bus) formContract(ctx context.Context, hostSettings rhpv2.HostSettings,
 		return api.ContractMetadata{}, fmt.Errorf("couldn't add transaction set to the pool: %w", err)
 	}
 
-	// broadcast the transaction set
-	go b.s.BroadcastTransactionSet(txnSet)
-
 	return api.ContractMetadata{
 		ID:                 contract.ID(),
 		HostKey:            contract.HostKey(),
@@ -727,9 +724,6 @@ func (b *Bus) formContractV2(ctx context.Context, hk types.PublicKey, hostIP str
 		return api.ContractMetadata{}, fmt.Errorf("failed to add v2 transaction set to the pool: %w", err)
 	}
 
-	// broadcast the transaction set
-	go b.s.BroadcastV2TransactionSet(res.FormationSet.Basis, res.FormationSet.Transactions)
-
 	contract := res.Contract
 	return api.ContractMetadata{
 		ID:                 contract.ID,
@@ -796,6 +790,13 @@ func (b *Bus) renewContractV1(ctx context.Context, cs consensus.State, gp api.Go
 	// derive the renter key
 	renterKey := b.masterKey.DeriveContractKey(c.HostKey)
 
+	// cap v1 renewals to the v2 require height since the host won't allow us to
+	// form contracts beyond that
+	v2ReqHeight := b.cm.TipState().Network.HardforkV2.RequireHeight
+	if endHeight >= v2ReqHeight {
+		endHeight = v2ReqHeight - 1
+	}
+
 	// fetch the revision
 	rev, err := b.rhp3Client.Revision(ctx, c.ID, c.HostKey, hs.SiamuxAddr())
 	if err != nil {
@@ -805,14 +806,11 @@ func (b *Bus) renewContractV1(ctx context.Context, cs consensus.State, gp api.Go
 	// renew contract
 	gc := gouging.NewChecker(gp.GougingSettings, gp.ConsensusState)
 	prepareRenew := b.prepareRenew(cs, rev, hs.Address, b.w.Address(), renterFunds, minNewCollateral, endHeight, expectedNewStorage)
-	newRevision, txnSet, contractPrice, fundAmount, err := b.rhp3Client.Renew(ctx, gc, rev, renterKey, c.HostKey, hs.SiamuxAddr(), prepareRenew, b.w.SignTransaction)
+	newRevision, _, contractPrice, fundAmount, err := b.rhp3Client.Renew(ctx, gc, rev, renterKey, c.HostKey, hs.SiamuxAddr(), prepareRenew, b.w.SignTransaction)
 	if err != nil {
 		return api.ContractMetadata{}, err
 	}
 
-	// broadcast the transaction set
-	b.s.BroadcastTransactionSet(txnSet)
-
 	return api.ContractMetadata{
 		ID:                 newRevision.ID(),
 		HostKey:            newRevision.HostKey(),
@@ -863,7 +861,6 @@ func (b *Bus) renewContractV2(ctx context.Context, cs consensus.State, h api.Hos
 	}
 
 	var contract cRhp4.ContractRevision
-	var txnSet cRhp4.TransactionSet
 	if c.EndHeight() == endHeight {
 		// when refreshing, the 'collateral' is added on top of the existing
 		// collateral so we account for that by subtracting the rolled over
@@ -880,7 +877,6 @@ func (b *Bus) renewContractV2(ctx context.Context, cs consensus.State, h api.Hos
 			Collateral: collateral,
 		})
 		contract = res.Contract
-		txnSet = res.RenewalSet
 	} else {
 		var res cRhp4.RPCRenewContractResult
 		res, err = b.rhp4Client.RenewContract(ctx, h.PublicKey, h.V2SiamuxAddr(), b.cm, signer, cs, settings.Prices, rev, rhpv4.RPCRenewContractParams{
@@ -890,15 +886,11 @@ func (b *Bus) renewContractV2(ctx context.Context, cs consensus.State, h api.Hos
 			ProofHeight: endHeight,
 		})
 		contract = res.Contract
-		txnSet = res.RenewalSet
 	}
 	if err != nil {
 		return api.ContractMetadata{}, err
 	}
 
-	// broadcast the transaction set
-	b.s.BroadcastV2TransactionSet(txnSet.Basis, txnSet.Transactions)
-
 	return api.ContractMetadata{
 		ID:                 contract.ID,
 		HostKey:            h.PublicKey,

bus/routes.go

@@ -2320,6 +2320,13 @@ func (b *Bus) contractsFormHandler(jc jape.Context) {
 			return
 		}
 
+		// cap v1 formations to the v2 require height since the host won't allow
+		// us to form contracts beyond that
+		v2ReqHeight := b.cm.TipState().Network.HardforkV2.RequireHeight
+		if rfr.EndHeight >= v2ReqHeight {
+			rfr.EndHeight = v2ReqHeight - 1
+		}
+
 		// check gouging
 		breakdown := gc.CheckSettings(settings)
 		if breakdown.Gouging() {

cmd/renterd/config.go

@@ -95,6 +95,7 @@ func defaultConfig() config.Config {
 			ID:                     "",
 			AccountsRefillInterval: defaultAccountRefillInterval,
 			BusFlushInterval:       5 * time.Second,
+			CacheExpiry:            5 * time.Minute,
 
 			DownloadMaxOverdrive:     5,
 			DownloadOverdriveTimeout: 3 * time.Second,

config/config.go

@@ -124,6 +124,7 @@ type (
 		UploadMaxMemory               uint64        `yaml:"uploadMaxMemory,omitempty"`
 		UploadMaxOverdrive            uint64        `yaml:"uploadMaxOverdrive,omitempty"`
 		AllowUnauthenticatedDownloads bool          `yaml:"allowUnauthenticatedDownloads,omitempty"`
+		CacheExpiry                   time.Duration `yaml:"cacheExpiry,omitempty"`
 	}
 
 	// Autopilot contains the configuration for an autopilot.

go.mod

@@ -17,7 +17,7 @@ require (
 	go.sia.tech/core v0.8.0
 	go.sia.tech/coreutils v0.8.0
 	go.sia.tech/gofakes3 v0.0.5
-	go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db
+	go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed
 	go.sia.tech/jape v0.12.1
 	go.sia.tech/mux v1.3.0
 	go.sia.tech/web/renterd v0.69.0

go.sum

@@ -61,8 +61,8 @@ go.sia.tech/coreutils v0.8.0 h1:1dcl0vxY+MBgAdJ7PdewAr8RkZJn4/6wAKEZfi4iYn0=
 go.sia.tech/coreutils v0.8.0/go.mod h1:ml5MefDMWCvPKNeRVIGHmyF5tv27C9h1PiI/iOiTGLg=
 go.sia.tech/gofakes3 v0.0.5 h1:vFhVBUFbKE9ZplvLE2w4TQxFMQyF8qvgxV4TaTph+Vw=
 go.sia.tech/gofakes3 v0.0.5/go.mod h1:LXEzwGw+OHysWLmagleCttX93cJZlT9rBu/icOZjQ54=
-go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db h1:ey3ezMYHPzY+FZ4yL8xsAWnCJWI2J9z4rtpmRa8dj0A=
-go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db/go.mod h1:6wTgoXKmsLQT22lUcHI4/dUcb3mhXFR+9zYWIki8Qho=
+go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed h1:C42AxWwwoP13EhZsdWwR17Rc9S7gXI4JnRN0AyZRxc8=
+go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed/go.mod h1:6wTgoXKmsLQT22lUcHI4/dUcb3mhXFR+9zYWIki8Qho=
 go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU=
 go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4=
 go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c=