From 7b773912a6035b59b9b116a49a9dd1f9acf90135 Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 13 Dec 2024 09:42:42 -0800 Subject: [PATCH 1/5] chore: update core and coreutils --- cmd/hostd/run.go | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- host/contracts/lock.go | 12 ++++++++++-- internal/testutil/testutil.go | 12 ++++++------ 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/cmd/hostd/run.go b/cmd/hostd/run.go index 6b1ddb00..41c445dc 100644 --- a/cmd/hostd/run.go +++ b/cmd/hostd/run.go @@ -357,7 +357,7 @@ func runRootCmd(ctx context.Context, cfg config.Config, walletKey types.PrivateK go rhp3.Serve() defer rhp3.Close() - rhp4 := rhp4.NewServer(hostKey, cm, s, contractManager, wm, sm, vm, rhp4.WithPriceTableValidity(30*time.Minute), rhp4.WithContractProofWindowBuffer(72)) + rhp4 := rhp4.NewServer(hostKey, cm, s, contractManager, wm, sm, vm, rhp4.WithPriceTableValidity(30*time.Minute)) var stopListenerFuncs []func() error defer func() { diff --git a/go.mod b/go.mod index d44d7aad..ca7f494a 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/mattn/go-sqlite3 v1.14.24 github.com/shopspring/decimal v1.4.0 - go.sia.tech/core v0.7.1 - go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31 + go.sia.tech/core v0.8.0 + go.sia.tech/coreutils v0.8.0 go.sia.tech/jape v0.12.1 go.sia.tech/mux v1.3.0 go.sia.tech/web/hostd v0.52.0 diff --git a/go.sum b/go.sum index 43da0c93..66a854c6 100644 --- a/go.sum +++ b/go.sum @@ -40,10 +40,10 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= -go.sia.tech/core v0.7.1 h1:PrKh19Ql5vJbQbB5YGtTHQ8W3fRF8hhYnR4kPOIOIME= -go.sia.tech/core v0.7.1/go.mod h1:gB8iXFJFSV8XIHRaL00CL6Be+hyykB+SYnvRPHCCc/E= -go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31 h1:Qskaf8d6oDKG5emNvGHZsd9iZRqz2GeouVNKY5paXlE= -go.sia.tech/coreutils v0.7.1-0.20241203172514-7bf95dd18f31/go.mod h1:d6jrawloc02MCXi/EVc8FIN5h3C6XDiMs4fuFMcU0PU= +go.sia.tech/core v0.8.0 h1:J6vZQlVhpj4bTVeuC2GKkfkGEs8jf0j651Kl1wwOxjg= +go.sia.tech/core v0.8.0/go.mod h1:Wj1qzvpMM2rqEQjwWJEbCBbe9VWX/mSJUu2Y2ABl1QA= +go.sia.tech/coreutils v0.8.0 h1:1dcl0vxY+MBgAdJ7PdewAr8RkZJn4/6wAKEZfi4iYn0= +go.sia.tech/coreutils v0.8.0/go.mod h1:ml5MefDMWCvPKNeRVIGHmyF5tv27C9h1PiI/iOiTGLg= go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU= go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4= go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c= diff --git a/host/contracts/lock.go b/host/contracts/lock.go index 86f9850b..e8819138 100644 --- a/host/contracts/lock.go +++ b/host/contracts/lock.go @@ -129,9 +129,17 @@ func (cm *Manager) LockV2Contract(id types.FileContractID) (rev rhp4.RevisionSta return rhp4.RevisionState{}, nil, fmt.Errorf("failed to get contract: %w", err) } + renewed := contract.RenewedTo != (types.FileContractID{}) + var maxRevisionHeight uint64 + if contract.ProofHeight > cm.revisionSubmissionBuffer { + maxRevisionHeight = contract.ProofHeight - cm.revisionSubmissionBuffer + } + revisable := !renewed && cm.chain.Tip().Height < maxRevisionHeight return rhp4.RevisionState{ - Revision: contract.V2FileContract, - Roots: cm.getSectorRoots(id), + Revision: contract.V2FileContract, + Renewed: contract.RenewedTo != (types.FileContractID{}), + Revisable: revisable, + Roots: cm.getSectorRoots(id), }, func() { cm.locks.Unlock(id) }, nil diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 79ce43b1..ff0e25b7 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -62,7 +62,7 @@ func V2Network() (*consensus.Network, types.Block) { } // WaitForSync is a helper to wait for the chain and indexer to sync -func WaitForSync(t *testing.T, cm *chain.Manager, idx *index.Manager) { +func WaitForSync(t testing.TB, cm *chain.Manager, idx *index.Manager) { t.Helper() for { @@ -74,7 +74,7 @@ func WaitForSync(t *testing.T, cm *chain.Manager, idx *index.Manager) { } // MineBlocks is a helper to mine blocks and broadcast the headers -func MineBlocks(t *testing.T, cn *ConsensusNode, addr types.Address, n int) { +func MineBlocks(t testing.TB, cn *ConsensusNode, addr types.Address, n int) { t.Helper() for i := 0; i < n; i++ { @@ -95,7 +95,7 @@ func MineBlocks(t *testing.T, cn *ConsensusNode, addr types.Address, n int) { // MineAndSync is a helper to mine blocks and wait for the index to catch up // between each block -func MineAndSync(t *testing.T, hn *HostNode, addr types.Address, n int) { +func MineAndSync(t testing.TB, hn *HostNode, addr types.Address, n int) { t.Helper() for i := 0; i < n; i++ { @@ -106,7 +106,7 @@ func MineAndSync(t *testing.T, hn *HostNode, addr types.Address, n int) { // NewConsensusNode initializes all of the consensus components and returns them. // The function will clean up all resources when the test is done. -func NewConsensusNode(t *testing.T, network *consensus.Network, genesis types.Block, log *zap.Logger) *ConsensusNode { +func NewConsensusNode(t testing.TB, network *consensus.Network, genesis types.Block, log *zap.Logger) *ConsensusNode { t.Helper() dir := t.TempDir() @@ -156,7 +156,7 @@ func NewConsensusNode(t *testing.T, network *consensus.Network, genesis types.Bl // NewHostNode initializes all of the hostd components and returns them. The function // will clean up all resources when the test is done. -func NewHostNode(t *testing.T, pk types.PrivateKey, network *consensus.Network, genesis types.Block, log *zap.Logger) *HostNode { +func NewHostNode(t testing.TB, pk types.PrivateKey, network *consensus.Network, genesis types.Block, log *zap.Logger) *HostNode { t.Helper() cn := NewConsensusNode(t, network, genesis, log) @@ -181,7 +181,7 @@ func NewHostNode(t *testing.T, pk types.PrivateKey, network *consensus.Network, initialSettings := settings.DefaultSettings initialSettings.AcceptingContracts = true - initialSettings.NetAddress = "127.0.0.1:9981" + initialSettings.NetAddress = "127.0.0.1" initialSettings.WindowSize = 10 sm, err := settings.NewConfigManager(pk, cn.Store, cn.Chain, cn.Syncer, vm, wm, settings.WithAnnounceInterval(10), settings.WithValidateNetAddress(false), settings.WithInitialSettings(initialSettings)) if err != nil { From 60f3972ec3bb85851206c34b81b30dc69563a79f Mon Sep 17 00:00:00 2001 From: Nate Date: Fri, 13 Dec 2024 09:42:54 -0800 Subject: [PATCH 2/5] refactor: add RHP4 integration tests --- internal/integration/rhp/v4/rhp4_test.go | 1383 ++++++++++++++++++++++ 1 file changed, 1383 insertions(+) create mode 100644 internal/integration/rhp/v4/rhp4_test.go diff --git a/internal/integration/rhp/v4/rhp4_test.go b/internal/integration/rhp/v4/rhp4_test.go new file mode 100644 index 00000000..a489e81e --- /dev/null +++ b/internal/integration/rhp/v4/rhp4_test.go @@ -0,0 +1,1383 @@ +package rhp_test + +import ( + "bytes" + "context" + "net" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + proto4 "go.sia.tech/core/rhp/v4" + "go.sia.tech/core/types" + rhp4 "go.sia.tech/coreutils/rhp/v4" + "go.sia.tech/coreutils/wallet" + "go.sia.tech/hostd/internal/testutil" + "go.sia.tech/hostd/rhp" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" + "lukechampine.com/frand" +) + +type fundAndSign struct { + w *wallet.SingleAddressWallet + pk types.PrivateKey +} + +func (fs *fundAndSign) FundV2Transaction(txn *types.V2Transaction, amount types.Currency) (types.ChainIndex, []int, error) { + return fs.w.FundV2Transaction(txn, amount, true) +} +func (fs *fundAndSign) ReleaseInputs(txns []types.V2Transaction) { + fs.w.ReleaseInputs(nil, txns) +} + +func (fs *fundAndSign) SignV2Inputs(txn *types.V2Transaction, toSign []int) { + fs.w.SignV2Inputs(txn, toSign) +} +func (fs *fundAndSign) SignHash(h types.Hash256) types.Signature { + return fs.pk.SignHash(h) +} +func (fs *fundAndSign) PublicKey() types.PublicKey { + return fs.pk.PublicKey() +} +func (fs *fundAndSign) Address() types.Address { + return fs.w.Address() +} + +func testRenterHostPair(tb testing.TB, hostKey types.PrivateKey, hn *testutil.HostNode, log *zap.Logger) rhp4.TransportClient { + rs := rhp4.NewServer(hostKey, hn.Chain, hn.Syncer, hn.Contracts, hn.Wallet, hn.Settings, hn.Volumes, rhp4.WithPriceTableValidity(2*time.Minute)) + + l, err := net.Listen("tcp", ":0") + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { l.Close() }) + go rhp.ServeRHP4SiaMux(l, rs, log.Named("siamux")) + + transport, err := rhp4.DialSiaMux(context.Background(), l.Addr().String(), hostKey.PublicKey()) + if err != nil { + tb.Fatal(err) + } + tb.Cleanup(func() { transport.Close() }) + + return transport +} + +func TestSettings(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey := types.GeneratePrivateKey() + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + testutil.MineAndSync(t, hn, hn.Wallet.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } else if settings.Prices.ValidUntil.Before(time.Now()) { + t.Fatal("settings expired") + } + + // verify the signature + sigHash := settings.Prices.SigHash() + if !hostKey.PublicKey().VerifyHash(sigHash, settings.Prices.Signature) { + t.Fatal("signature verification failed") + } + + // adjust the calculated fields to match the expected values + expected := hn.Settings.RHP4Settings() + expected.ProtocolVersion = settings.ProtocolVersion + expected.Prices.Signature = settings.Prices.Signature + expected.Prices.ValidUntil = settings.Prices.ValidUntil + expected.Prices.TipHeight = settings.Prices.TipHeight + + if !reflect.DeepEqual(settings, expected) { + t.Error("retrieved", settings) + t.Error("expected", expected) + t.Fatal("settings mismatch") + } +} + +func TestFormContract(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + result, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + + // verify the transaction set is valid + if known, err := hn.Chain.AddV2PoolTransactions(result.FormationSet.Basis, result.FormationSet.Transactions); err != nil { + t.Fatal(err) + } else if !known { + t.Fatal("expected transaction set to be known") + } + + sigHash := cm.TipState().ContractSigHash(result.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, result.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, result.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } +} + +func TestFormContractBasis(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + result, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + + // verify the transaction set is valid + if known, err := cm.AddV2PoolTransactions(result.FormationSet.Basis, result.FormationSet.Transactions); err != nil { + t.Fatal(err) + } else if !known { + t.Fatal("expected transaction set to be known") + } +} + +func TestRPCRefresh(t *testing.T) { + log := zaptest.NewLogger(t) + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + hn := testutil.NewHostNode(t, hostKey, n, genesis, log) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(t.TempDir(), "test.dat"), 64, results); err != nil { + t.Fatal(err) + } else if err := <-results; err != nil { + t.Fatal(err) + } + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, log.Named("renterhost")) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + fundAndSign := &fundAndSign{w, renterKey} + + formContractUploadSector := func(t *testing.T, renterAllowance, hostCollateral, accountBalance types.Currency) rhp4.ContractRevision { + t.Helper() + + result, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + revision := result.Contract + + // verify the transaction set is valid + if known, err := cm.AddV2PoolTransactions(result.FormationSet.Basis, result.FormationSet.Transactions); err != nil { + t.Fatal(err) + } else if !known { + t.Fatal("expected transaction set to be known") + } + + // mine a few blocks to confirm the contract + testutil.MineAndSync(t, hn, types.VoidAddress, 10) + + // fund an account to transfer funds to the host + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountBalance}, + }) + if err != nil { + t.Fatal(err) + } + revision.Revision = fundResult.Revision + + // upload data + at := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(5 * time.Minute), + } + at.Signature = renterKey.SignHash(at.SigHash()) + wRes, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, at, bytes.NewReader(bytes.Repeat([]byte{1}, proto4.LeafSize)), proto4.LeafSize) + if err != nil { + t.Fatal(err) + } + aRes, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, []types.Hash256{wRes.Root}) + if err != nil { + t.Fatal(err) + } + revision.Revision = aRes.Revision + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if rs.Renewed { + t.Fatal("expected contract to not be renewed") + } else if !rs.Revisable { + t.Fatal("expected contract to be revisable") + } + return revision + } + + t.Run("no allowance or collateral", func(t *testing.T) { + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + + // refresh the contract + _, err = rhp4.RPCRefreshContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRefreshContractParams{ + ContractID: revision.ID, + Allowance: types.ZeroCurrency, + Collateral: types.ZeroCurrency, + }) + if err == nil { + t.Fatal(err) + } else if !strings.Contains(err.Error(), "allowance must be greater than zero") { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("valid refresh", func(t *testing.T) { + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + // refresh the contract + refreshResult, err := rhp4.RPCRefreshContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRefreshContractParams{ + ContractID: revision.ID, + Allowance: types.Siacoins(10), + Collateral: types.Siacoins(20), + }) + if err != nil { + t.Fatal(err) + } + + // verify the transaction set is valid + if known, err := cm.AddV2PoolTransactions(refreshResult.RenewalSet.Basis, refreshResult.RenewalSet.Transactions); err != nil { + t.Fatal(err) + } else if !known { + t.Fatal("expected transaction set to be known") + } + + sigHash := cm.TipState().ContractSigHash(refreshResult.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, refreshResult.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, refreshResult.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if !rs.Renewed { + t.Fatal("expected contract to be renewed") + } else if rs.Revisable { + t.Fatal("expected contract to not be revisable") + } + }) +} + +func TestRPCRenew(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(t.TempDir(), "test.dat"), 64, results); err != nil { + t.Fatal(err) + } else if err := <-results; err != nil { + t.Fatal(err) + } + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + fundAndSign := &fundAndSign{w, renterKey} + + formContractUploadSector := func(t *testing.T, renterAllowance, hostCollateral, accountBalance types.Currency) rhp4.ContractRevision { + t.Helper() + + result, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + revision := result.Contract + sigHash := cm.TipState().ContractSigHash(revision.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, revision.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, revision.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } + + // verify the transaction set is valid + if known, err := cm.AddV2PoolTransactions(result.FormationSet.Basis, result.FormationSet.Transactions); err != nil { + t.Fatal(err) + } else if !known { + t.Fatal("expected transaction set to be known") + } + + // mine a few blocks to confirm the contract + testutil.MineAndSync(t, hn, types.VoidAddress, 10) + + // fund an account to transfer funds to the host + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountBalance}, + }) + if err != nil { + t.Fatal(err) + } + revision.Revision = fundResult.Revision + + // upload data + at := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(5 * time.Minute), + } + at.Signature = renterKey.SignHash(at.SigHash()) + wRes, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, at, bytes.NewReader(bytes.Repeat([]byte{1}, proto4.LeafSize)), proto4.LeafSize) + if err != nil { + t.Fatal(err) + } + aRes, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, []types.Hash256{wRes.Root}) + if err != nil { + t.Fatal(err) + } + revision.Revision = aRes.Revision + return revision + } + + t.Run("same duration", func(t *testing.T) { + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + + // renew the contract + _, err = rhp4.RPCRenewContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRenewContractParams{ + ContractID: revision.ID, + Allowance: types.Siacoins(150), + Collateral: types.Siacoins(300), + ProofHeight: revision.Revision.ProofHeight, + }) + if err == nil { + t.Fatal(err) + } else if !strings.Contains(err.Error(), "renewal proof height must be greater than existing proof height") { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("partial rollover", func(t *testing.T) { + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + + // renew the contract + renewResult, err := rhp4.RPCRenewContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRenewContractParams{ + ContractID: revision.ID, + Allowance: types.Siacoins(150), + Collateral: types.Siacoins(300), + ProofHeight: revision.Revision.ProofHeight + 10, + }) + if err != nil { + t.Fatal(err) + } + + // verify the transaction set is valid + if known, err := cm.AddV2PoolTransactions(renewResult.RenewalSet.Basis, renewResult.RenewalSet.Transactions); err != nil { + t.Fatal(err) + } else if !known { + t.Fatal("expected transaction set to be known") + } + + sigHash := cm.TipState().ContractSigHash(renewResult.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if !rs.Renewed { + t.Fatal("expected contract to be renewed") + } else if rs.Revisable { + t.Fatal("expected contract to not be revisable") + } + }) + + t.Run("full rollover", func(t *testing.T) { + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + + // renew the contract + renewResult, err := rhp4.RPCRenewContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRenewContractParams{ + ContractID: revision.ID, + Allowance: types.Siacoins(50), + Collateral: types.Siacoins(100), + ProofHeight: revision.Revision.ProofHeight + 10, + }) + if err != nil { + t.Fatal(err) + } + + // verify the transaction set is valid + if known, err := cm.AddV2PoolTransactions(renewResult.RenewalSet.Basis, renewResult.RenewalSet.Transactions); err != nil { + t.Fatal(err) + } else if !known { + t.Fatal("expected transaction set to be known") + } + + sigHash := cm.TipState().ContractSigHash(renewResult.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if !rs.Renewed { + t.Fatal("expected contract to be renewed") + } else if rs.Revisable { + t.Fatal("expected contract to not be revisable") + } + }) + + t.Run("no rollover", func(t *testing.T) { + revision := formContractUploadSector(t, types.Siacoins(100), types.Siacoins(200), types.Siacoins(25)) + + // renew the contract + renewResult, err := rhp4.RPCRenewContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, revision.Revision, proto4.RPCRenewContractParams{ + ContractID: revision.ID, + Allowance: types.Siacoins(150), + Collateral: types.Siacoins(300), + ProofHeight: revision.Revision.ProofHeight + 10, + }) + if err != nil { + t.Fatal(err) + } + + // verify the transaction set is valid + if known, err := cm.AddV2PoolTransactions(renewResult.RenewalSet.Basis, renewResult.RenewalSet.Transactions); err != nil { + t.Fatal(err) + } else if !known { + t.Fatal("expected transaction set to be known") + } + + sigHash := cm.TipState().ContractSigHash(renewResult.Contract.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.RenterSignature) { + t.Fatal("renter signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(sigHash, renewResult.Contract.Revision.HostSignature) { + t.Fatal("host signature verification failed") + } + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } else if !rs.Renewed { + t.Fatal("expected contract to be renewed") + } else if rs.Revisable { + t.Fatal("expected contract to not be revisable") + } + }) +} + +func TestAccounts(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + account := proto4.Account(renterKey.PublicKey()) + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + revision := formResult.Contract + + cs := cm.TipState() + + balance, err := rhp4.RPCAccountBalance(context.Background(), transport, account) + if err != nil { + t.Fatal(err) + } else if !balance.IsZero() { + t.Fatal("expected zero balance") + } + + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + t.Fatal(err) + } + revised := fundResult.Revision + + renterOutputValue := revision.Revision.RenterOutput.Value.Sub(accountFundAmount) + hostOutputValue := revision.Revision.HostOutput.Value.Add(accountFundAmount) + + // verify the contract was modified correctly + switch { + case fundResult.Balances[0].Account != account: + t.Fatalf("expected %v, got %v", account, fundResult.Balances[0].Account) + case fundResult.Balances[0].Balance != accountFundAmount: + t.Fatalf("expected %v, got %v", accountFundAmount, fundResult.Balances[0].Balance) + case !fundResult.Usage.RenterCost().Equals(accountFundAmount): + t.Fatalf("expected %v, got %v", accountFundAmount, fundResult.Usage.RenterCost()) + case !revised.HostOutput.Value.Equals(hostOutputValue): + t.Fatalf("expected %v, got %v", hostOutputValue, revised.HostOutput.Value) + case !revised.RenterOutput.Value.Equals(renterOutputValue): + t.Fatalf("expected %v, got %v", renterOutputValue, revised.RenterOutput.Value) + case !revised.MissedHostValue.Equals(revision.Revision.MissedHostValue): + t.Fatalf("expected %v, got %v", revision.Revision.MissedHostValue, revised.MissedHostValue) + case revised.RevisionNumber != revision.Revision.RevisionNumber+1: + t.Fatalf("expected %v, got %v", revision.Revision.RevisionNumber+1, revised.RevisionNumber) + } + + revisionSigHash := cs.ContractSigHash(revised) + if !renterKey.PublicKey().VerifyHash(revisionSigHash, revised.RenterSignature) { + t.Fatal("revision signature verification failed") + } else if !hostKey.PublicKey().VerifyHash(revisionSigHash, revised.HostSignature) { + t.Fatal("revision signature verification failed") + } + + // verify the account balance + balance, err = rhp4.RPCAccountBalance(context.Background(), transport, account) + if err != nil { + t.Fatal(err) + } else if !balance.Equals(accountFundAmount) { + t.Fatalf("expected %v, got %v", accountFundAmount, balance) + } +} + +func TestReadWriteSector(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(t.TempDir(), "test.dat"), 10, results); err != nil { + t.Fatal(err) + } else if err := <-results; err != nil { + t.Fatal(err) + } + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + revision := formResult.Contract + + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + t.Fatal(err) + } + revision.Revision = fundResult.Revision + + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + tokenSigHash := token.SigHash() + token.Signature = renterKey.SignHash(tokenSigHash) + + data := frand.Bytes(1024) + + // store the sector + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) + if err != nil { + t.Fatal(err) + } + + // verify the sector root + var sector [proto4.SectorSize]byte + copy(sector[:], data) + if writeResult.Root != proto4.SectorRoot(§or) { + t.Fatal("root mismatch") + } + + // read the sector back + buf := bytes.NewBuffer(nil) + _, err = rhp4.RPCReadSector(context.Background(), transport, settings.Prices, token, buf, writeResult.Root, 0, 64) + if err != nil { + t.Fatal(err) + } else if !bytes.Equal(buf.Bytes(), data[:64]) { + t.Fatal("data mismatch") + } +} + +func TestAppendSectors(t *testing.T) { + const sectors = 10 + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(t.TempDir(), "test.dat"), sectors, results); err != nil { + t.Fatal(err) + } else if err := <-results; err != nil { + t.Fatal(err) + } + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + revision := formResult.Contract + + assertLastRevision := func(t *testing.T) { + t.Helper() + + rs, err := rhp4.RPCLatestRevision(context.Background(), transport, revision.ID) + if err != nil { + t.Fatal(err) + } + lastRev := rs.Contract + if !reflect.DeepEqual(lastRev, revision.Revision) { + t.Log(lastRev) + t.Log(revision.Revision) + t.Fatalf("expected last revision to match") + } + + sigHash := cm.TipState().ContractSigHash(revision.Revision) + if !renterKey.PublicKey().VerifyHash(sigHash, lastRev.RenterSignature) { + t.Fatal("renter signature invalid") + } else if !hostKey.PublicKey().VerifyHash(sigHash, lastRev.HostSignature) { + t.Fatal("host signature invalid") + } + } + assertLastRevision(t) + + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + t.Fatal(err) + } + revision.Revision = fundResult.Revision + assertLastRevision(t) + + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + tokenSigHash := token.SigHash() + token.Signature = renterKey.SignHash(tokenSigHash) + + // store random sectors + roots := make([]types.Hash256, 0, 10) + for i := 0; i < 10; i++ { + var sector [proto4.SectorSize]byte + frand.Read(sector[:]) + root := proto4.SectorRoot(§or) + + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sector[:]), proto4.SectorSize) + if err != nil { + t.Fatal(err) + } else if writeResult.Root != root { + t.Fatal("root mismatch") + } + roots = append(roots, root) + } + + // corrupt a random root + excludedIndex := frand.Intn(len(roots)) + roots[excludedIndex] = frand.Entropy256() + + // append the sectors to the contract + appendResult, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, roots) + if err != nil { + t.Fatal(err) + } else if len(appendResult.Sectors) != len(roots)-1 { + t.Fatalf("expected %v, got %v", len(roots)-1, len(appendResult.Sectors)) + } + roots = append(roots[:excludedIndex], roots[excludedIndex+1:]...) + if appendResult.Revision.FileMerkleRoot != proto4.MetaRoot(roots) { + t.Fatal("root mismatch") + } + revision.Revision = appendResult.Revision + assertLastRevision(t) + + // read the sectors back + buf := bytes.NewBuffer(make([]byte, 0, proto4.SectorSize)) + for _, root := range roots { + buf.Reset() + + _, err = rhp4.RPCReadSector(context.Background(), transport, settings.Prices, token, buf, root, 0, proto4.SectorSize) + if err != nil { + t.Fatal(err) + } else if proto4.SectorRoot((*[proto4.SectorSize]byte)(buf.Bytes())) != root { + t.Fatal("data mismatch") + } + } +} + +func TestVerifySector(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(t.TempDir(), "test.dat"), 64, results); err != nil { + t.Fatal(err) + } else if err := <-results; err != nil { + t.Fatal(err) + } + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + revision := formResult.Contract + + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + t.Fatal(err) + } + revision.Revision = fundResult.Revision + + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + tokenSigHash := token.SigHash() + token.Signature = renterKey.SignHash(tokenSigHash) + + data := frand.Bytes(1024) + + // store the sector + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) + if err != nil { + t.Fatal(err) + } + + // verify the sector root + var sector [proto4.SectorSize]byte + copy(sector[:], data) + if writeResult.Root != proto4.SectorRoot(§or) { + t.Fatal("root mismatch") + } + + // verify the host is storing the sector + _, err = rhp4.RPCVerifySector(context.Background(), transport, settings.Prices, token, writeResult.Root) + if err != nil { + t.Fatal(err) + } +} + +func TestRPCFreeSectors(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(t.TempDir(), "test.dat"), 64, results); err != nil { + t.Fatal(err) + } else if err := <-results; err != nil { + t.Fatal(err) + } + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + revision := formResult.Contract + + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + t.Fatal(err) + } + revision.Revision = fundResult.Revision + + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + tokenSigHash := token.SigHash() + token.Signature = renterKey.SignHash(tokenSigHash) + + roots := make([]types.Hash256, 10) + for i := range roots { + // store random sectors on the host + data := frand.Bytes(1024) + + // store the sector + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) + if err != nil { + t.Fatal(err) + } + roots[i] = writeResult.Root + } + + assertRevision := func(t *testing.T, revision types.V2FileContract, roots []types.Hash256) { + t.Helper() + + expectedRoot := proto4.MetaRoot(roots) + n := len(roots) + + if revision.Filesize/proto4.SectorSize != uint64(n) { + t.Fatalf("expected %v sectors, got %v", n, revision.Filesize/proto4.SectorSize) + } else if revision.FileMerkleRoot != expectedRoot { + t.Fatalf("expected %v, got %v", expectedRoot, revision.FileMerkleRoot) + } + } + + // append all the sector roots to the contract + appendResult, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, roots) + if err != nil { + t.Fatal(err) + } + assertRevision(t, appendResult.Revision, roots) + revision.Revision = appendResult.Revision + + // randomly remove half the sectors + indices := make([]uint64, len(roots)/2) + for i, n := range frand.Perm(len(roots))[:len(roots)/2] { + indices[i] = uint64(n) + } + newRoots := append([]types.Hash256(nil), roots...) + for i, n := range indices { + newRoots[n] = newRoots[len(newRoots)-i-1] + } + newRoots = newRoots[:len(newRoots)-len(indices)] + + removeResult, err := rhp4.RPCFreeSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, indices) + if err != nil { + t.Fatal(err) + } + assertRevision(t, removeResult.Revision, newRoots) +} + +func TestRPCSectorRoots(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(t.TempDir(), "test.dat"), 64, results); err != nil { + t.Fatal(err) + } else if err := <-results; err != nil { + t.Fatal(err) + } + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + revision := formResult.Contract + + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + t.Fatal(err) + } + revision.Revision = fundResult.Revision + + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + tokenSigHash := token.SigHash() + token.Signature = renterKey.SignHash(tokenSigHash) + + roots := make([]types.Hash256, 0, 50) + + checkRoots := func(t *testing.T, expected []types.Hash256) { + t.Helper() + + rootsResult, err := rhp4.RPCSectorRoots(context.Background(), transport, cs, settings.Prices, renterKey, revision, 0, uint64(len(expected))) + if err != nil { + t.Fatal(err) + } else if len(roots) != len(expected) { + t.Fatalf("expected %v roots, got %v", len(expected), len(roots)) + } + for i := range rootsResult.Roots { + if roots[i] != expected[i] { + t.Fatalf("expected %v, got %v", expected[i], roots[i]) + } + } + revision.Revision = rootsResult.Revision + } + + for i := 0; i < cap(roots); i++ { + // store random sectors on the host + data := frand.Bytes(1024) + + // store the sector + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) + if err != nil { + t.Fatal(err) + } + roots = append(roots, writeResult.Root) + + appendResult, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, []types.Hash256{writeResult.Root}) + if err != nil { + t.Fatal(err) + } + revision.Revision = appendResult.Revision + checkRoots(t, roots) + } +} + +func BenchmarkWrite(b *testing.B) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(b, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(b.TempDir(), "test.dat"), uint64(b.N), results); err != nil { + b.Fatal(err) + } else if err := <-results; err != nil { + b.Fatal(err) + } + + testutil.MineAndSync(b, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(b, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + b.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + b.Fatal(err) + } + revision := formResult.Contract + + // fund an account + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + b.Fatal(err) + } + revision.Revision = fundResult.Revision + + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + token.Signature = renterKey.SignHash(token.SigHash()) + + var sectors [][proto4.SectorSize]byte + for i := 0; i < b.N; i++ { + var sector [proto4.SectorSize]byte + frand.Read(sector[:256]) + sectors = append(sectors, sector) + } + + b.ResetTimer() + b.ReportAllocs() + b.SetBytes(proto4.SectorSize) + + for i := 0; i < b.N; i++ { + // store the sector + _, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sectors[i][:]), proto4.SectorSize) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkRead(b *testing.B) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(b, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(b.TempDir(), "test.dat"), uint64(b.N), results); err != nil { + b.Fatal(err) + } else if err := <-results; err != nil { + b.Fatal(err) + } + + testutil.MineAndSync(b, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(b, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + b.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + b.Fatal(err) + } + revision := formResult.Contract + + // fund an account + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + b.Fatal(err) + } + revision.Revision = fundResult.Revision + + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + token.Signature = renterKey.SignHash(token.SigHash()) + + var sectors [][proto4.SectorSize]byte + roots := make([]types.Hash256, 0, b.N) + for i := 0; i < b.N; i++ { + var sector [proto4.SectorSize]byte + frand.Read(sector[:256]) + sectors = append(sectors, sector) + + // store the sector + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sectors[i][:]), proto4.SectorSize) + if err != nil { + b.Fatal(err) + } + roots = append(roots, writeResult.Root) + } + + b.ResetTimer() + b.ReportAllocs() + b.SetBytes(proto4.SectorSize) + + buf := bytes.NewBuffer(make([]byte, 0, proto4.SectorSize)) + for i := 0; i < b.N; i++ { + buf.Reset() + // store the sector + _, err = rhp4.RPCReadSector(context.Background(), transport, settings.Prices, token, buf, roots[i], 0, proto4.SectorSize) + if err != nil { + b.Fatal(err) + } else if !bytes.Equal(buf.Bytes(), sectors[i][:]) { + b.Fatal("data mismatch") + } + } +} + +func BenchmarkContractUpload(b *testing.B) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(b, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(b.TempDir(), "test.dat"), uint64(b.N), results); err != nil { + b.Fatal(err) + } else if err := <-results; err != nil { + b.Fatal(err) + } + + testutil.MineAndSync(b, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(b, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + b.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + b.Fatal(err) + } + revision := formResult.Contract + + // fund an account + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + b.Fatal(err) + } + revision.Revision = fundResult.Revision + + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + token.Signature = renterKey.SignHash(token.SigHash()) + + var sectors [][proto4.SectorSize]byte + roots := make([]types.Hash256, 0, b.N) + for i := 0; i < b.N; i++ { + var sector [proto4.SectorSize]byte + frand.Read(sector[:256]) + sectors = append(sectors, sector) + roots = append(roots, proto4.SectorRoot(§or)) + } + + b.ResetTimer() + b.ReportAllocs() + b.SetBytes(proto4.SectorSize) + + var wg sync.WaitGroup + for i := 0; i < b.N; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(sectors[i][:]), proto4.SectorSize) + if err != nil { + b.Error(err) + } else if writeResult.Root != roots[i] { + b.Errorf("expected %v, got %v", roots[i], writeResult.Root) + } + }(i) + } + + wg.Wait() + + appendResult, err := rhp4.RPCAppendSectors(context.Background(), transport, cs, settings.Prices, renterKey, revision, roots) + if err != nil { + b.Fatal(err) + } else if appendResult.Revision.Filesize != uint64(b.N)*proto4.SectorSize { + b.Fatalf("expected %v sectors, got %v", b.N, appendResult.Revision.Filesize/proto4.SectorSize) + } +} From b79ebe075eaf681142699b35c1b0f29e5fb3c682 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 16 Dec 2024 06:18:06 -0800 Subject: [PATCH 3/5] Update host/contracts/lock.go Co-authored-by: Christopher Schinnerl --- host/contracts/lock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/host/contracts/lock.go b/host/contracts/lock.go index e8819138..270c6c1b 100644 --- a/host/contracts/lock.go +++ b/host/contracts/lock.go @@ -137,7 +137,7 @@ func (cm *Manager) LockV2Contract(id types.FileContractID) (rev rhp4.RevisionSta revisable := !renewed && cm.chain.Tip().Height < maxRevisionHeight return rhp4.RevisionState{ Revision: contract.V2FileContract, - Renewed: contract.RenewedTo != (types.FileContractID{}), + Renewed: renewed, Revisable: revisable, Roots: cm.getSectorRoots(id), }, func() { From 1e80100fc596372d59324f8a76e5c35619888e8d Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 16 Dec 2024 14:50:15 -0800 Subject: [PATCH 4/5] rhp4: remove TestContractBasis --- internal/integration/rhp/v4/rhp4_test.go | 120 ++++++++++++++++------- 1 file changed, 82 insertions(+), 38 deletions(-) diff --git a/internal/integration/rhp/v4/rhp4_test.go b/internal/integration/rhp/v4/rhp4_test.go index a489e81e..83dbc2bf 100644 --- a/internal/integration/rhp/v4/rhp4_test.go +++ b/internal/integration/rhp/v4/rhp4_test.go @@ -147,44 +147,6 @@ func TestFormContract(t *testing.T) { } } -func TestFormContractBasis(t *testing.T) { - n, genesis := testutil.V2Network() - hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() - - hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) - cm := hn.Chain - w := hn.Wallet - - testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) - - transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) - - settings, err := rhp4.RPCSettings(context.Background(), transport) - if err != nil { - t.Fatal(err) - } - - fundAndSign := &fundAndSign{w, renterKey} - renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) - result, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ - RenterPublicKey: renterKey.PublicKey(), - RenterAddress: w.Address(), - Allowance: renterAllowance, - Collateral: hostCollateral, - ProofHeight: cm.Tip().Height + 50, - }) - if err != nil { - t.Fatal(err) - } - - // verify the transaction set is valid - if known, err := cm.AddV2PoolTransactions(result.FormationSet.Basis, result.FormationSet.Transactions); err != nil { - t.Fatal(err) - } else if !known { - t.Fatal("expected transaction set to be known") - } -} - func TestRPCRefresh(t *testing.T) { log := zaptest.NewLogger(t) n, genesis := testutil.V2Network() @@ -1125,6 +1087,88 @@ func TestRPCSectorRoots(t *testing.T) { } } +func TestPrune(t *testing.T) { + n, genesis := testutil.V2Network() + hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() + + hn := testutil.NewHostNode(t, hostKey, n, genesis, zap.NewNop()) + cm := hn.Chain + w := hn.Wallet + + results := make(chan error, 1) + if _, err := hn.Volumes.AddVolume(context.Background(), filepath.Join(t.TempDir(), "test.dat"), 10, results); err != nil { + t.Fatal(err) + } else if err := <-results; err != nil { + t.Fatal(err) + } + + testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20)) + + transport := testRenterHostPair(t, hostKey, hn, zap.NewNop()) + + settings, err := rhp4.RPCSettings(context.Background(), transport) + if err != nil { + t.Fatal(err) + } + + fundAndSign := &fundAndSign{w, renterKey} + renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200) + formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{ + RenterPublicKey: renterKey.PublicKey(), + RenterAddress: w.Address(), + Allowance: renterAllowance, + Collateral: hostCollateral, + ProofHeight: cm.Tip().Height + 50, + }) + if err != nil { + t.Fatal(err) + } + revision := formResult.Contract + + cs := cm.TipState() + account := proto4.Account(renterKey.PublicKey()) + + accountFundAmount := types.Siacoins(25) + fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, []proto4.AccountDeposit{ + {Account: account, Amount: accountFundAmount}, + }) + if err != nil { + t.Fatal(err) + } + revision.Revision = fundResult.Revision + + token := proto4.AccountToken{ + Account: account, + ValidUntil: time.Now().Add(time.Hour), + } + tokenSigHash := token.SigHash() + token.Signature = renterKey.SignHash(tokenSigHash) + + data := frand.Bytes(1024) + + // store the sector + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) + if err != nil { + t.Fatal(err) + } + + // verify the sector root + var sector [proto4.SectorSize]byte + copy(sector[:], data) + if writeResult.Root != proto4.SectorRoot(§or) { + t.Fatal("root mismatch") + } + + // read the sector back + buf := bytes.NewBuffer(nil) + _, err = rhp4.RPCReadSector(context.Background(), transport, settings.Prices, token, buf, writeResult.Root, 0, 64) + if err != nil { + t.Fatal(err) + } else if !bytes.Equal(buf.Bytes(), data[:64]) { + t.Fatal("data mismatch") + } +} + func BenchmarkWrite(b *testing.B) { n, genesis := testutil.V2Network() hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey() From 2b1e1183ae6c28ca94f73d850e1f8f263189fb87 Mon Sep 17 00:00:00 2001 From: Nate Date: Mon, 16 Dec 2024 18:49:33 -0800 Subject: [PATCH 5/5] rhp4: add test prune --- internal/integration/rhp/v4/rhp4_test.go | 68 ++++++++++++++++++------ internal/testutil/testutil.go | 2 +- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/internal/integration/rhp/v4/rhp4_test.go b/internal/integration/rhp/v4/rhp4_test.go index 83dbc2bf..7ad5f5fe 100644 --- a/internal/integration/rhp/v4/rhp4_test.go +++ b/internal/integration/rhp/v4/rhp4_test.go @@ -6,6 +6,7 @@ import ( "net" "path/filepath" "reflect" + "slices" "strings" "sync" "testing" @@ -1118,7 +1119,7 @@ func TestPrune(t *testing.T) { RenterAddress: w.Address(), Allowance: renterAllowance, Collateral: hostCollateral, - ProofHeight: cm.Tip().Height + 50, + ProofHeight: cm.Tip().Height + proto4.TempSectorDuration + 10, }) if err != nil { t.Fatal(err) @@ -1144,29 +1145,64 @@ func TestPrune(t *testing.T) { tokenSigHash := token.SigHash() token.Signature = renterKey.SignHash(tokenSigHash) - data := frand.Bytes(1024) + tempExpirationHeight := cm.Tip().Height + proto4.TempSectorDuration + roots := make([]types.Hash256, 10) + for i := 0; i < len(roots); i++ { + data := frand.Bytes(1024) - // store the sector - writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) - if err != nil { - t.Fatal(err) + // store the sector + writeResult, err := rhp4.RPCWriteSector(context.Background(), transport, settings.Prices, token, bytes.NewReader(data), uint64(len(data))) + if err != nil { + t.Fatal(err) + } + roots[i] = writeResult.Root } - // verify the sector root - var sector [proto4.SectorSize]byte - copy(sector[:], data) - if writeResult.Root != proto4.SectorRoot(§or) { - t.Fatal("root mismatch") + assertSectors := func(t *testing.T, available, deleted []types.Hash256) { + t.Helper() + + buf := bytes.NewBuffer(nil) + for _, root := range available { + buf.Reset() + + _, err := rhp4.RPCReadSector(context.Background(), transport, settings.Prices, token, buf, root, 0, proto4.SectorSize) + if err != nil { + t.Fatal(err) + } else if r2 := proto4.SectorRoot((*[4194304]byte)(buf.Bytes())); root != r2 { + t.Fatalf("expected root %q, got %q", root, r2) + } + } + + for _, root := range deleted { + buf.Reset() + + _, err := rhp4.RPCReadSector(context.Background(), transport, settings.Prices, token, buf, root, 0, proto4.SectorSize) + if err == nil || !strings.Contains(err.Error(), "sector not found") { + t.Fatalf("expected err %q, got %q", proto4.ErrSectorNotFound, err) + } + } } - // read the sector back - buf := bytes.NewBuffer(nil) - _, err = rhp4.RPCReadSector(context.Background(), transport, settings.Prices, token, buf, writeResult.Root, 0, 64) + tempSectors, contractSectors := roots[:len(roots)/2], roots[len(roots)/2:] + + appendResult, err := rhp4.RPCAppendSectors(context.Background(), transport, cm.TipState(), settings.Prices, renterKey, revision, contractSectors) if err != nil { t.Fatal(err) - } else if !bytes.Equal(buf.Bytes(), data[:64]) { - t.Fatal("data mismatch") + } else if !slices.Equal(appendResult.Sectors, contractSectors) { + t.Fatal("expect contract sectors") } + + assertSectors(t, roots, nil) + + // mine until the temp sector expire + testutil.MineAndSync(t, hn, types.VoidAddress, int(tempExpirationHeight-cm.Tip().Height)+1) + time.Sleep(time.Second) // wait for the sectors to be pruned + assertSectors(t, contractSectors, tempSectors) + + // mine until the contract sector expires + testutil.MineAndSync(t, hn, types.VoidAddress, int(revision.Revision.ExpirationHeight-cm.Tip().Height)+1) + time.Sleep(time.Second) + assertSectors(t, nil, roots) } func BenchmarkWrite(b *testing.B) { diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index ff0e25b7..fac6ddde 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -167,7 +167,7 @@ func NewHostNode(t testing.TB, pk types.PrivateKey, network *consensus.Network, } t.Cleanup(func() { wm.Close() }) - vm, err := storage.NewVolumeManager(cn.Store, storage.WithLogger(log.Named("storage"))) + vm, err := storage.NewVolumeManager(cn.Store, storage.WithLogger(log.Named("storage")), storage.WithPruneInterval(500*time.Millisecond)) if err != nil { t.Fatal("failed to create volume manager:", err) }