Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

e2e: Add integration test for network transition from v1 to v2 #1740

Merged
merged 46 commits into from
Dec 18, 2024
Merged
Changes from 44 commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
5e0c066
e2e: add TestV1ToV2Transition
ChrisSchinnerl Dec 12, 2024
c789ff8
add applyV1ContractUpdate and revertV1ContractUpdate
ChrisSchinnerl Dec 12, 2024
b525598
add applyV2ContractUpdate and revertV2ContractUpdate
ChrisSchinnerl Dec 12, 2024
f218019
move known contracts into txn
ChrisSchinnerl Dec 12, 2024
1108bc2
fix lint
ChrisSchinnerl Dec 12, 2024
f7700ce
don't broadcast transactions after formation and renewal
ChrisSchinnerl Dec 12, 2024
fd48afc
make use of 'created' flag
ChrisSchinnerl Dec 13, 2024
e671991
fix lint
ChrisSchinnerl Dec 13, 2024
29b3b7a
Merge branch 'dev' into chris/transition-test
ChrisSchinnerl Dec 13, 2024
c20ce92
extend test to check if number of contracts is right and if contracts…
ChrisSchinnerl Dec 13, 2024
5f622a0
upload an object
ChrisSchinnerl Dec 13, 2024
aae5a28
don't delete host sectors when contract is deleted
ChrisSchinnerl Dec 13, 2024
4edd543
expire worker cache faster during testing
ChrisSchinnerl Dec 13, 2024
8ba3f65
fix TestV1ToV2Transition health
ChrisSchinnerl Dec 13, 2024
9b23b41
check contracts and download data
ChrisSchinnerl Dec 13, 2024
73ce2b2
fix TestSQLMetadataStore
ChrisSchinnerl Dec 13, 2024
5a83fc6
skip TestHostSectors
ChrisSchinnerl Dec 13, 2024
8dc0af3
prune host_sectors when no more active contracts are available
ChrisSchinnerl Dec 16, 2024
37df84d
sql: extend DeleteHostSector
ChrisSchinnerl Dec 16, 2024
38b526c
fix TestDeleteHostSector
ChrisSchinnerl Dec 16, 2024
47faaff
Merge branch 'dev' into chris/refactor-contract-updates
ChrisSchinnerl Dec 16, 2024
6490ce4
fix TestGouging
ChrisSchinnerl Dec 16, 2024
6f84d01
fix TestMigrations for v1
ChrisSchinnerl Dec 16, 2024
8ce19a4
fix TestMigration for v2
ChrisSchinnerl Dec 16, 2024
b03c139
Merge branch 'dev' into chris/transition-test
ChrisSchinnerl Dec 16, 2024
38e8f30
fix TestObjectBasic
ChrisSchinnerl Dec 16, 2024
9f89ad7
add TestSlabSectorOnHostButNotInContract
ChrisSchinnerl Dec 16, 2024
7c12fa9
fix ndf in TestV1ToV2Transition
ChrisSchinnerl Dec 16, 2024
68a4e3a
address comments
ChrisSchinnerl Dec 16, 2024
6008585
Merge branch 'dev' into chris/refactor-contract-updates
ChrisSchinnerl Dec 16, 2024
987d57c
fix panic in RecordContractRenewal
ChrisSchinnerl Dec 16, 2024
8d8fe9c
Merge branch 'chris/refactor-contract-updates' into chris/transition-…
ChrisSchinnerl Dec 16, 2024
43ad7d8
Merge branch 'dev' into chris/refactor-contract-updates
ChrisSchinnerl Dec 16, 2024
1618443
Merge branch 'chris/refactor-contract-updates' into chris/transition-…
ChrisSchinnerl Dec 16, 2024
8d127ff
address comments
ChrisSchinnerl Dec 17, 2024
a73862b
fix TestV1ToV2Transition
ChrisSchinnerl Dec 17, 2024
fe8f392
Merge branch 'dev' into chris/transition-test
ChrisSchinnerl Dec 17, 2024
d3acee3
fix lint
ChrisSchinnerl Dec 17, 2024
300333f
fix TestDownloadAllHosts NDF
ChrisSchinnerl Dec 17, 2024
7ea2cb4
check contract usability
ChrisSchinnerl Dec 17, 2024
a8b44e4
release inputs when failing to broadcast contract element
ChrisSchinnerl Dec 17, 2024
74fa72f
fix error string
ChrisSchinnerl Dec 17, 2024
fd0e06b
replace sleep with retry
ChrisSchinnerl Dec 17, 2024
54b1ef8
use account token helper
ChrisSchinnerl Dec 17, 2024
3967d1c
Merge branch 'dev' into chris/transition-test
ChrisSchinnerl Dec 18, 2024
4e8ab84
address comments
ChrisSchinnerl Dec 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions autopilot/contractor/contractor.go
Original file line number Diff line number Diff line change
@@ -463,7 +463,9 @@ func activeContracts(ctx context.Context, bus Bus, logger *zap.SugaredLogger) ([
// fetch active contracts
logger.Info("fetching active contracts")
start := time.Now()
metadatas, err := bus.Contracts(ctx, api.ContractsOpts{FilterMode: api.ContractFilterModeActive})
metadatas, err := bus.Contracts(ctx, api.ContractsOpts{
FilterMode: api.ContractFilterModeActive,
})
if err != nil {
return nil, err
}
@@ -907,7 +909,9 @@ func performContractFormations(ctx *mCtx, bus Bus, cr contractReviser, hf hostFi
wanted := int(ctx.WantedContracts())

// fetch all active contracts
contracts, err := bus.Contracts(ctx, api.ContractsOpts{})
contracts, err := bus.Contracts(ctx, api.ContractsOpts{
FilterMode: api.ContractFilterModeActive,
})
if err != nil {
return 0, fmt.Errorf("failed to fetch contracts: %w", err)
}
@@ -1057,7 +1061,9 @@ func performHostChecks(ctx *mCtx, bus Bus, logger *zap.SugaredLogger) error {

func performPostMaintenanceTasks(ctx *mCtx, bus Bus, alerter alerts.Alerter, cc contractChecker, rb revisionBroadcaster, logger *zap.SugaredLogger) error {
// fetch some contract and host info
allContracts, err := bus.Contracts(ctx, api.ContractsOpts{})
allContracts, err := bus.Contracts(ctx, api.ContractsOpts{
FilterMode: api.ContractFilterModeActive,
})
if err != nil {
return fmt.Errorf("failed to fetch all contracts: %w", err)
}
@@ -1122,7 +1128,7 @@ func performV2ContractMigration(ctx *mCtx, bus Bus, cr contractReviser, logger *
}

contracts, err := bus.Contracts(ctx, api.ContractsOpts{
FilterMode: api.ContractFilterModeAll, // TODO: change to usable
FilterMode: api.ContractFilterModeActive,
ChrisSchinnerl marked this conversation as resolved.
Show resolved Hide resolved
})
if err != nil {
logger.With(zap.Error(err)).Error("failed to fetch contracts for migration")
@@ -1154,7 +1160,7 @@ func performV2ContractMigration(ctx *mCtx, bus Bus, cr contractReviser, logger *
}

// form a new contract with the same host
contract, _, err := cr.formContract(ctx, bus, host, InitialContractFunding, logger)
_, _, err = cr.formContract(ctx, bus, host, InitialContractFunding, logger)
if err != nil {
logger.Errorf("failed to form a v2 contract with the host")
continue
7 changes: 7 additions & 0 deletions bus/bus.go
Original file line number Diff line number Diff line change
@@ -790,6 +790,13 @@ func (b *Bus) renewContractV1(ctx context.Context, cs consensus.State, gp api.Go
// derive the renter key
renterKey := b.masterKey.DeriveContractKey(c.HostKey)

// cap v1 renewals to the v2 require height since the host won't allow us to
// form contracts beyond that
v2ReqHeight := b.cm.TipState().Network.HardforkV2.RequireHeight
if endHeight >= v2ReqHeight {
endHeight = v2ReqHeight - 1
}

// fetch the revision
rev, err := b.rhp3Client.Revision(ctx, c.ID, c.HostKey, hs.SiamuxAddr())
if err != nil {
7 changes: 7 additions & 0 deletions bus/routes.go
Original file line number Diff line number Diff line change
@@ -2320,6 +2320,13 @@ func (b *Bus) contractsFormHandler(jc jape.Context) {
return
}

// cap v1 formations to the v2 require height since the host won't allow
ChrisSchinnerl marked this conversation as resolved.
Show resolved Hide resolved
// us to form contracts beyond that
v2ReqHeight := b.cm.TipState().Network.HardforkV2.RequireHeight
if rfr.EndHeight >= v2ReqHeight {
rfr.EndHeight = v2ReqHeight - 1
}

// check gouging
breakdown := gc.CheckSettings(settings)
if breakdown.Gouging() {
1 change: 1 addition & 0 deletions cmd/renterd/config.go
Original file line number Diff line number Diff line change
@@ -95,6 +95,7 @@ func defaultConfig() config.Config {
ID: "",
AccountsRefillInterval: defaultAccountRefillInterval,
BusFlushInterval: 5 * time.Second,
CacheExpiry: 5 * time.Minute,

DownloadMaxOverdrive: 5,
DownloadOverdriveTimeout: 3 * time.Second,
1 change: 1 addition & 0 deletions config/config.go
Original file line number Diff line number Diff line change
@@ -124,6 +124,7 @@ type (
UploadMaxMemory uint64 `yaml:"uploadMaxMemory,omitempty"`
UploadMaxOverdrive uint64 `yaml:"uploadMaxOverdrive,omitempty"`
AllowUnauthenticatedDownloads bool `yaml:"allowUnauthenticatedDownloads,omitempty"`
CacheExpiry time.Duration `yaml:"cacheExpiry,omitempty"`
}

// Autopilot contains the configuration for an autopilot.
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
@@ -14,10 +14,10 @@ require (
github.com/mattn/go-sqlite3 v1.14.24
github.com/montanaflynn/stats v0.7.1
github.com/shopspring/decimal v1.4.0
go.sia.tech/core v0.8.0
go.sia.tech/coreutils v0.8.0
go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b
go.sia.tech/coreutils v0.8.1-0.20241217153531-b5e84c03d17f
go.sia.tech/gofakes3 v0.0.5
go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db
go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed
go.sia.tech/jape v0.12.1
go.sia.tech/mux v1.3.0
go.sia.tech/web/renterd v0.69.0
12 changes: 6 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
@@ -55,14 +55,14 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
go.sia.tech/core v0.8.0 h1:J6vZQlVhpj4bTVeuC2GKkfkGEs8jf0j651Kl1wwOxjg=
go.sia.tech/core v0.8.0/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA=
go.sia.tech/coreutils v0.8.0 h1:1dcl0vxY+MBgAdJ7PdewAr8RkZJn4/6wAKEZfi4iYn0=
go.sia.tech/coreutils v0.8.0/go.mod h1:ml5MefDMWCvPKNeRVIGHmyF5tv27C9h1PiI/iOiTGLg=
go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b h1:VRkb6OOX1KawLQwuqOEHLcjha8gxVX0tAyu2Dyoq8Ek=
go.sia.tech/core v0.8.1-0.20241217152409-7950a7ca324b/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA=
go.sia.tech/coreutils v0.8.1-0.20241217153531-b5e84c03d17f h1:TafvnqJgx/+0zX/QMSOOkf5HfMqaoe/73eO515fUucI=
go.sia.tech/coreutils v0.8.1-0.20241217153531-b5e84c03d17f/go.mod h1:xhIbFjjkzmCF8Dt73ZvquaBQCT2Dje7AKYBRAesn93w=
go.sia.tech/gofakes3 v0.0.5 h1:vFhVBUFbKE9ZplvLE2w4TQxFMQyF8qvgxV4TaTph+Vw=
go.sia.tech/gofakes3 v0.0.5/go.mod h1:LXEzwGw+OHysWLmagleCttX93cJZlT9rBu/icOZjQ54=
go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db h1:ey3ezMYHPzY+FZ4yL8xsAWnCJWI2J9z4rtpmRa8dj0A=
go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db/go.mod h1:6wTgoXKmsLQT22lUcHI4/dUcb3mhXFR+9zYWIki8Qho=
go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed h1:C42AxWwwoP13EhZsdWwR17Rc9S7gXI4JnRN0AyZRxc8=
go.sia.tech/hostd v1.1.3-0.20241212215223-9e3440475bed/go.mod h1:6wTgoXKmsLQT22lUcHI4/dUcb3mhXFR+9zYWIki8Qho=
go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU=
go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4=
go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c=
8 changes: 2 additions & 6 deletions internal/accounts/accounts.go
Original file line number Diff line number Diff line change
@@ -448,12 +448,8 @@ func (a *Manager) refillAccount(ctx context.Context, contract api.ContractMetada
}

func (a *Account) Token() rhpv4.AccountToken {
t := rhpv4.AccountToken{
Account: rhpv4.Account(a.key.PublicKey()),
ValidUntil: time.Now().Add(5 * time.Minute),
}
t.Signature = a.key.SignHash(t.SigHash())
return t
account := rhpv4.Account(a.key.PublicKey())
return account.Token(a.key, a.acc.HostKey)
}

// WithSync syncs an accounts balance with the bus. To do so, the account is
10 changes: 9 additions & 1 deletion internal/bus/chainsubscriber.go
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"math"
"strings"
"sync"
"time"

@@ -71,6 +72,7 @@ type (

Wallet interface {
FundV2Transaction(txn *types.V2Transaction, amount types.Currency, useUnconfirmed bool) (types.ChainIndex, []int, error)
ReleaseInputs(txns []types.Transaction, v2txns []types.V2Transaction)
SignV2Inputs(txn *types.V2Transaction, toSign []int)
UpdateChainState(tx wallet.UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error
}
@@ -433,8 +435,14 @@ func (s *chainSubscriber) broadcastExpiredFileContractResolutions(tx sql.ChainUp

// verify txn and broadcast it
_, err = s.cm.AddV2PoolTransactions(basis, []types.V2Transaction{txn})
if err != nil {
if err != nil &&
(strings.Contains(err.Error(), "has already been resolved") ||
strings.Contains(err.Error(), "not present in the accumulator")) {
ChrisSchinnerl marked this conversation as resolved.
Show resolved Hide resolved
s.wallet.ReleaseInputs(nil, []types.V2Transaction{txn})
ChrisSchinnerl marked this conversation as resolved.
Show resolved Hide resolved
continue
} else if err != nil {
s.logger.Errorf("failed to broadcast contract expiration txn: %v", err)
s.wallet.ReleaseInputs(nil, []types.V2Transaction{txn})
continue
}
s.s.BroadcastV2TransactionSet(basis, []types.V2Transaction{txn})
3 changes: 2 additions & 1 deletion internal/test/e2e/cluster.go
Original file line number Diff line number Diff line change
@@ -636,7 +636,7 @@ func announceHosts(hosts []*Host) error {
for _, host := range hosts {
settings := defaultHostSettings
settings.NetAddress = host.rhp4Listener.Addr().(*net.TCPAddr).IP.String()
if err := host.settings.UpdateSettings(settings); err != nil {
if err := host.UpdateSettings(settings); err != nil {
return err
}
if err := host.settings.Announce(); err != nil {
@@ -1000,6 +1000,7 @@ func testDBCfg() dbConfig {
func testWorkerCfg() config.Worker {
return config.Worker{
AccountsRefillInterval: 10 * time.Millisecond,
CacheExpiry: 100 * time.Millisecond,
ID: "worker",
BusFlushInterval: testBusFlushInterval,
DownloadOverdriveTimeout: 500 * time.Millisecond,
Loading