From a12f028ed44cea762d5ef8df6d77bfa546ce7180 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 13 Dec 2023 16:30:27 +0100 Subject: [PATCH 01/14] go.mod: update deps --- go.mod | 16 ++++++++-------- go.sum | 34 +++++++++++++++++----------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 1c1cc0a88..62990c342 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/go-gormigrate/gormigrate/v2 v2.1.1 github.com/google/go-cmp v0.6.0 github.com/gotd/contrib v0.19.0 - github.com/klauspost/reedsolomon v1.11.8 + github.com/klauspost/reedsolomon v1.12.0 github.com/minio/minio-go/v7 v7.0.65 github.com/montanaflynn/stats v0.7.1 gitlab.com/NebulousLabs/encoding v0.0.0-20200604091946-456c3dc907fe @@ -36,7 +36,7 @@ require ( require ( github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect - github.com/aws/aws-sdk-go v1.48.13 // indirect + github.com/aws/aws-sdk-go v1.49.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cloudflare/cloudflare-go v0.75.0 // indirect github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf // indirect @@ -48,7 +48,7 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/uuid v1.4.0 // indirect + github.com/google/uuid v1.5.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -86,16 +86,16 @@ require ( gitlab.com/NebulousLabs/threadgroup v0.0.0-20200608151952-38921fbef213 // indirect go.opentelemetry.io/otel/metric v1.21.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect - go.sia.tech/web v0.0.0-20231205212549-23f4fb47d5f6 // indirect + go.sia.tech/web v0.0.0-20231213145933-3f175a86abff // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect - google.golang.org/grpc v1.59.0 // indirect + golang.org/x/tools v0.16.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect + google.golang.org/grpc v1.60.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect nhooyr.io/websocket v1.8.10 // indirect diff --git a/go.sum b/go.sum index 306b968db..d7d313ed2 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.48.13 h1:6N4GTme6MpxfCisWf5pql8k3TBORiKTmbeutZCDXlG8= -github.com/aws/aws-sdk-go v1.48.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.49.1 h1:Dsamcd8d/nNb3A+bZ0ucfGl0vGZsW5wlRW0vhoYGoeQ= +github.com/aws/aws-sdk-go v1.49.1/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -76,8 +76,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -126,8 +126,8 @@ github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/reedsolomon v1.9.3/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= -github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= -github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= +github.com/klauspost/reedsolomon v1.12.0 h1:I5FEp3xSwVCcEh3F5A7dofEfhXdF/bWhQWPH+XwBFno= +github.com/klauspost/reedsolomon v1.12.0/go.mod h1:EPLZJeh4l27pUGC3aXOjheaoh1I9yut7xTURiW3LQ9Y= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -274,8 +274,8 @@ go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca h1:aZMg2AKevn7jKx+wlusWQfwSM5pNU9aGtRZme29q3O4= go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca/go.mod h1:h/1afFwpxzff6/gG5i1XdAgPK7dEY6FaibhK7N5F86Y= -go.sia.tech/web v0.0.0-20231205212549-23f4fb47d5f6 h1:JmEYO5jlKx8c4lNJ9oh3k0I55JewQKM6cNvg1ZF8GUE= -go.sia.tech/web v0.0.0-20231205212549-23f4fb47d5f6/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= +go.sia.tech/web v0.0.0-20231213145933-3f175a86abff h1:/nE7nhewDRxzEdtSKT4SkiUwtjPSiy7Xz7CHEW3MaGQ= +go.sia.tech/web v0.0.0-20231213145933-3f175a86abff/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= go.sia.tech/web/renterd v0.36.0 h1:78FXkALF4ZiJw6IrZY1IZ9G0Z+Bp9hZbR8sXxk28Jxg= go.sia.tech/web/renterd v0.36.0/go.mod h1:FgXrdmAnu591a3h96RB/15pMZ74xO9457g902uE06BM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -383,22 +383,22 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg= -google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 h1:ZcOkrmX74HbKFYnpPY8Qsw93fC29TbJXspYKaBkSXDQ= -google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4/go.mod h1:k2dtGpRrbsSyKcNPKKI5sstZkrNCZwpU/ns96JoHbGg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 h1:1hfbdAfFbkmpg41000wDVqr7jUpK/Yo+LPnIxxGzmkg= +google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 h1:s1w3X6gQxwrLEpxnLd/qXTVLgQE2yXwaOaoa6IlY/+o= +google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0/go.mod h1:CAny0tYF+0/9rmDB9fahA9YLzX3+AEVl1qXbv5hhj6c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k= +google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= From 36a839783ade21237ea33196fd9ee031f017e31d Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 14 Dec 2023 09:35:37 +0100 Subject: [PATCH 02/14] bus: improve error message --- bus/bus.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bus/bus.go b/bus/bus.go index abde20e79..9689f307c 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -2066,7 +2066,11 @@ func (b *bus) metricsHandlerGET(jc jape.Context) { if jc.DecodeForm("n", &n) != nil { return } else if n == 0 { - jc.Error(errors.New("parameter 'n' is required"), http.StatusBadRequest) + if jc.Request.FormValue("n") == "" { + jc.Error(errors.New("parameter 'n' is required"), http.StatusBadRequest) + } else { + jc.Error(errors.New("'n' has to be greater than zero"), http.StatusBadRequest) + } return } From b2ae0817d56a72690ca69fa40fb9871a1699e386 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 13 Dec 2023 18:46:09 +0100 Subject: [PATCH 03/14] worker: add mock host --- worker/host.go | 82 ++++++++++++++++++++++++++++--- worker/host_test.go | 117 ++++++++++++++++++++++++++++++++++++++++++++ worker/rhpv3.go | 63 ------------------------ 3 files changed, 192 insertions(+), 70 deletions(-) create mode 100644 worker/host_test.go diff --git a/worker/host.go b/worker/host.go index ccf9f5645..b5e4b7429 100644 --- a/worker/host.go +++ b/worker/host.go @@ -19,12 +19,15 @@ import ( type ( Host interface { DownloadSector(ctx context.Context, w io.Writer, root types.Hash256, offset, length uint32, overpay bool) error + UploadSector(ctx context.Context, sector *[rhpv2.SectorSize]byte, rev types.FileContractRevision) (types.Hash256, error) + FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt hostdb.HostPriceTable, err error) FetchRevision(ctx context.Context, fetchTimeout time.Duration, blockHeight uint64) (types.FileContractRevision, error) + FundAccount(ctx context.Context, balance types.Currency, rev *types.FileContractRevision) error - RenewContract(ctx context.Context, rrr api.RHPRenewRequest) (_ rhpv2.ContractRevision, _ []types.Transaction, _ types.Currency, err error) SyncAccount(ctx context.Context, rev *types.FileContractRevision) error - UploadSector(ctx context.Context, sector *[rhpv2.SectorSize]byte, rev types.FileContractRevision) (types.Hash256, error) + + RenewContract(ctx context.Context, rrr api.RHPRenewRequest) (_ rhpv2.ContractRevision, _ []types.Transaction, _ types.Currency, err error) } HostManager interface { @@ -49,7 +52,10 @@ type ( } ) -var _ Host = (*host)(nil) +var ( + _ Host = (*host)(nil) + _ HostManager = (*worker)(nil) +) func (w *worker) Host(hk types.PublicKey, fcid types.FileContractID, siamuxAddr string) Host { return &host{ @@ -107,7 +113,6 @@ func (h *host) DownloadSector(ctx context.Context, w io.Writer, root types.Hash2 }) } -// UploadSector uploads a sector to the host. func (h *host) UploadSector(ctx context.Context, sector *[rhpv2.SectorSize]byte, rev types.FileContractRevision) (root types.Hash256, err error) { // fetch price table pt, err := h.priceTable(ctx, nil) @@ -145,9 +150,6 @@ func (h *host) UploadSector(ctx context.Context, sector *[rhpv2.SectorSize]byte, return root, nil } -// Renew renews a contract with a host. To avoid an edge case where the contract -// is drained and can therefore not be used to pay for the revision, we simply -// don't pay for it. func (h *host) RenewContract(ctx context.Context, rrr api.RHPRenewRequest) (_ rhpv2.ContractRevision, _ []types.Transaction, _ types.Currency, err error) { // Try to get a valid pricetable. ptCtx, cancel := context.WithTimeout(ctx, 10*time.Second) @@ -165,6 +167,9 @@ func (h *host) RenewContract(ctx context.Context, rrr api.RHPRenewRequest) (_ rh var txnSet []types.Transaction var renewErr error err = h.transportPool.withTransportV3(ctx, h.hk, h.siamuxAddr, func(ctx context.Context, t *transportV3) (err error) { + // NOTE: to avoid an edge case where the contract is drained and can + // therefore not be used to pay for the revision, we simply don't pay + // for it. _, err = RPCLatestRevision(ctx, t, h.fcid, func(revision *types.FileContractRevision) (rhpv3.HostPriceTable, rhpv3.PaymentMethod, error) { // Renew contract. contractPrice = pt.ContractPrice @@ -207,3 +212,66 @@ func (h *host) FetchPriceTable(ctx context.Context, rev *types.FileContractRevis } return fetchPT(h.preparePriceTableAccountPayment(cs.BlockHeight)) } + +func (h *host) FundAccount(ctx context.Context, balance types.Currency, rev *types.FileContractRevision) error { + // fetch pricetable + pt, err := h.priceTable(ctx, rev) + if err != nil { + return err + } + + // calculate the amount to deposit + curr, err := h.acc.Balance(ctx) + if err != nil { + return err + } + if curr.Cmp(balance) >= 0 { + return nil + } + amount := balance.Sub(curr) + + // cap the amount by the amount of money left in the contract + renterFunds := rev.ValidRenterPayout() + possibleFundCost := pt.FundAccountCost.Add(pt.UpdatePriceTableCost) + if renterFunds.Cmp(possibleFundCost) <= 0 { + return fmt.Errorf("insufficient funds to fund account: %v <= %v", renterFunds, possibleFundCost) + } else if maxAmount := renterFunds.Sub(possibleFundCost); maxAmount.Cmp(amount) < 0 { + amount = maxAmount + } + + return h.acc.WithDeposit(ctx, func() (types.Currency, error) { + return amount, h.transportPool.withTransportV3(ctx, h.hk, h.siamuxAddr, func(ctx context.Context, t *transportV3) (err error) { + cost := amount.Add(pt.FundAccountCost) + payment, err := payByContract(rev, cost, rhpv3.Account{}, h.renterKey) // no account needed for funding + if err != nil { + return err + } + if err := RPCFundAccount(ctx, t, &payment, h.acc.id, pt.UID); err != nil { + return fmt.Errorf("failed to fund account with %v;%w", amount, err) + } + h.contractSpendingRecorder.Record(*rev, api.ContractSpending{FundAccount: cost}) + return nil + }) + }) +} + +func (h *host) SyncAccount(ctx context.Context, rev *types.FileContractRevision) error { + // fetch pricetable + pt, err := h.priceTable(ctx, rev) + if err != nil { + return err + } + + return h.acc.WithSync(ctx, func() (types.Currency, error) { + var balance types.Currency + err := h.transportPool.withTransportV3(ctx, h.hk, h.siamuxAddr, func(ctx context.Context, t *transportV3) error { + payment, err := payByContract(rev, pt.AccountBalanceCost, h.acc.id, h.renterKey) + if err != nil { + return err + } + balance, err = RPCAccountBalance(ctx, t, &payment, h.acc.id, pt.UID) + return err + }) + return balance, err + }) +} diff --git a/worker/host_test.go b/worker/host_test.go new file mode 100644 index 000000000..c73118612 --- /dev/null +++ b/worker/host_test.go @@ -0,0 +1,117 @@ +package worker + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "time" + + rhpv2 "go.sia.tech/core/rhp/v2" + "go.sia.tech/core/types" + "go.sia.tech/renterd/api" + "go.sia.tech/renterd/hostdb" + "lukechampine.com/frand" +) + +type ( + mockHost struct { + hk types.PublicKey + fcid types.FileContractID + sectors map[types.Hash256]*[rhpv2.SectorSize]byte + } +) + +var ( + _ Host = (*mockHost)(nil) +) + +func (h *mockHost) DownloadSector(ctx context.Context, w io.Writer, root types.Hash256, offset, length uint32, overpay bool) error { + sector, exist := h.sectors[root] + if !exist { + return errors.New("sector not found") + } + if offset+length > rhpv2.SectorSize { + return errors.New("sector out of bounds") + } + _, err := w.Write(sector[offset : offset+length]) + return err +} + +func (h *mockHost) UploadSector(ctx context.Context, sector *[rhpv2.SectorSize]byte, rev types.FileContractRevision) (types.Hash256, error) { + root := rhpv2.SectorRoot(sector) + h.sectors[root] = sector + return root, nil +} + +func (h *mockHost) FetchRevision(ctx context.Context, fetchTimeout time.Duration, blockHeight uint64) (rev types.FileContractRevision, _ error) { + return +} + +func (h *mockHost) FetchPriceTable(ctx context.Context, rev *types.FileContractRevision) (hpt hostdb.HostPriceTable, err error) { + return +} + +func (h *mockHost) FundAccount(ctx context.Context, balance types.Currency, rev *types.FileContractRevision) error { + return nil +} + +func (h *mockHost) RenewContract(ctx context.Context, rrr api.RHPRenewRequest) (_ rhpv2.ContractRevision, _ []types.Transaction, _ types.Currency, err error) { + return +} + +func (h *mockHost) SyncAccount(ctx context.Context, rev *types.FileContractRevision) error { + return nil +} + +func TestHost(t *testing.T) { + h := newTestHost() + s := newTestSector() + + root, err := h.UploadSector(context.Background(), s, types.FileContractRevision{}) + if err != nil { + t.Fatal(err) + } + + // download entire sector + var buf bytes.Buffer + err = h.DownloadSector(context.Background(), &buf, root, 0, rhpv2.SectorSize, false) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf.Bytes(), s[:]) { + t.Fatal("sector mismatch") + } + + // download part of the sector + buf.Reset() + err = h.DownloadSector(context.Background(), &buf, root, 64, 64, false) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf.Bytes(), s[64:128]) { + t.Fatal("sector mismatch") + } + + // try downloading out of bounds + err = h.DownloadSector(context.Background(), &buf, root, rhpv2.SectorSize, 64, false) + if err == nil || !strings.Contains(err.Error(), "out of bounds") { + t.Fatal("expected out of bounds error", err) + } +} + +func newTestHost() Host { + return &mockHost{ + hk: types.PublicKey{1}, + fcid: types.FileContractID{1}, + sectors: make(map[types.Hash256]*[rhpv2.SectorSize]byte), + } +} + +func newTestSector() *[rhpv2.SectorSize]byte { + var sector [rhpv2.SectorSize]byte + frand.Read(sector[:]) + return §or +} diff --git a/worker/rhpv3.go b/worker/rhpv3.go index 15bc96575..ff45390f9 100644 --- a/worker/rhpv3.go +++ b/worker/rhpv3.go @@ -332,69 +332,6 @@ func (h *host) fetchRevisionNoPayment(ctx context.Context, hostKey types.PublicK return rev, err } -func (h *host) FundAccount(ctx context.Context, balance types.Currency, rev *types.FileContractRevision) error { - // fetch pricetable - pt, err := h.priceTable(ctx, rev) - if err != nil { - return err - } - - // calculate the amount to deposit - curr, err := h.acc.Balance(ctx) - if err != nil { - return err - } - if curr.Cmp(balance) >= 0 { - return nil - } - amount := balance.Sub(curr) - - // cap the amount by the amount of money left in the contract - renterFunds := rev.ValidRenterPayout() - possibleFundCost := pt.FundAccountCost.Add(pt.UpdatePriceTableCost) - if renterFunds.Cmp(possibleFundCost) <= 0 { - return fmt.Errorf("insufficient funds to fund account: %v <= %v", renterFunds, possibleFundCost) - } else if maxAmount := renterFunds.Sub(possibleFundCost); maxAmount.Cmp(amount) < 0 { - amount = maxAmount - } - - return h.acc.WithDeposit(ctx, func() (types.Currency, error) { - return amount, h.transportPool.withTransportV3(ctx, h.hk, h.siamuxAddr, func(ctx context.Context, t *transportV3) (err error) { - cost := amount.Add(pt.FundAccountCost) - payment, err := payByContract(rev, cost, rhpv3.Account{}, h.renterKey) // no account needed for funding - if err != nil { - return err - } - if err := RPCFundAccount(ctx, t, &payment, h.acc.id, pt.UID); err != nil { - return fmt.Errorf("failed to fund account with %v;%w", amount, err) - } - h.contractSpendingRecorder.Record(*rev, api.ContractSpending{FundAccount: cost}) - return nil - }) - }) -} - -func (h *host) SyncAccount(ctx context.Context, rev *types.FileContractRevision) error { - // fetch pricetable - pt, err := h.priceTable(ctx, rev) - if err != nil { - return err - } - - return h.acc.WithSync(ctx, func() (types.Currency, error) { - var balance types.Currency - err := h.transportPool.withTransportV3(ctx, h.hk, h.siamuxAddr, func(ctx context.Context, t *transportV3) error { - payment, err := payByContract(rev, pt.AccountBalanceCost, h.acc.id, h.renterKey) - if err != nil { - return err - } - balance, err = RPCAccountBalance(ctx, t, &payment, h.acc.id, pt.UID) - return err - }) - return balance, err - }) -} - type ( // accounts stores the balance and other metrics of accounts that the // worker maintains with a host. From 8def6147b97fc913d3e883137dc2ee1f057faab6 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 14 Dec 2023 12:57:45 +0100 Subject: [PATCH 04/14] worker: fix refresh panic --- api/contract.go | 6 +++--- autopilot/contractor.go | 35 +++++++++++++++++++++++++++++------ autopilot/hostfilter.go | 2 +- worker/host.go | 3 +-- worker/rhpv3.go | 31 +++++++++++++++++-------------- 5 files changed, 51 insertions(+), 26 deletions(-) diff --git a/api/contract.go b/api/contract.go index ae0cf2956..29f75ea28 100644 --- a/api/contract.go +++ b/api/contract.go @@ -206,9 +206,9 @@ func (c Contract) RenterFunds() types.Currency { } // RemainingCollateral returns the remaining collateral in the contract. -func (c Contract) RemainingCollateral(s rhpv2.HostSettings) types.Currency { - if c.Revision.MissedHostPayout().Cmp(s.ContractPrice) < 0 { +func (c Contract) RemainingCollateral() types.Currency { + if c.Revision.MissedHostPayout().Cmp(c.ContractPrice) < 0 { return types.ZeroCurrency } - return c.Revision.MissedHostPayout().Sub(s.ContractPrice) + return c.Revision.MissedHostPayout().Sub(c.ContractPrice) } diff --git a/autopilot/contractor.go b/autopilot/contractor.go index eb2b66763..deac9ee42 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -1441,10 +1441,15 @@ func (c *contractor) refreshContract(ctx context.Context, w Worker, ci contractI } // calculate the renter funds - renterFunds, err := c.refreshFundingEstimate(ctx, state.cfg, ci, state.fee) - if err != nil { - c.logger.Errorw(fmt.Sprintf("could not get refresh funding estimate, err: %v", err), "hk", hk, "fcid", fcid) - return api.ContractMetadata{}, true, err + var renterFunds types.Currency + if isOutOfFunds(state.cfg, ci.settings, ci.contract) { + renterFunds, err = c.refreshFundingEstimate(ctx, state.cfg, ci, state.fee) + if err != nil { + c.logger.Errorw(fmt.Sprintf("could not get refresh funding estimate, err: %v", err), "hk", hk, "fcid", fcid) + return api.ContractMetadata{}, true, err + } + } else { + renterFunds = rev.ValidRenterPayout() // don't increase funds } // check our budget @@ -1456,11 +1461,28 @@ func (c *contractor) refreshContract(ctx context.Context, w Worker, ci contractI // calculate the new collateral expectedStorage := renterFundsToExpectedStorage(renterFunds, contract.EndHeight()-cs.BlockHeight, ci.priceTable) unallocatedCollateral := rev.MissedHostPayout().Sub(contract.ContractPrice) - minNewCollateral := minNewCollateral(unallocatedCollateral) + + // calculate the expected new collateral to determine the minNewCollateral. + // If the contract isn't below the min collateral, we don't enforce a + // minimum. + var minNewColl types.Currency + _, _, expectedNewCollateral := rhpv3.RenewalCosts(contract.Revision.FileContract, ci.priceTable, expectedStorage, contract.EndHeight()) + if isBelowCollateralThreshold(expectedNewCollateral, unallocatedCollateral) { + minNewColl = minNewCollateral(unallocatedCollateral) + } // renew the contract - resp, err := w.RHPRenew(ctx, contract.ID, contract.EndHeight(), hk, contract.SiamuxAddr, settings.Address, state.address, renterFunds, minNewCollateral, expectedStorage, settings.WindowSize) + resp, err := w.RHPRenew(ctx, contract.ID, contract.EndHeight(), hk, contract.SiamuxAddr, settings.Address, state.address, renterFunds, minNewColl, expectedStorage, settings.WindowSize) if err != nil { + if strings.Contains(err.Error(), "new collateral is too low") { + c.logger.Debugw("refresh failed: contract wouldn't have enough collateral after refresh", + "hk", hk, + "fcid", fcid, + "unallocatedCollateral", unallocatedCollateral.String(), + "minNewCollateral", minNewColl.String(), + ) + return api.ContractMetadata{}, true, err + } c.logger.Errorw("refresh failed", zap.Error(err), "hk", hk, "fcid", fcid) if strings.Contains(err.Error(), wallet.ErrInsufficientBalance.Error()) { return api.ContractMetadata{}, false, err @@ -1484,6 +1506,7 @@ func (c *contractor) refreshContract(ctx context.Context, w Worker, ci contractI "fcid", refreshedContract.ID, "renewedFrom", contract.ID, "renterFunds", renterFunds.String(), + "minNewCollateral", minNewColl.String(), "newCollateral", newCollateral.String(), ) return refreshedContract, true, nil diff --git a/autopilot/hostfilter.go b/autopilot/hostfilter.go index 22ddb9bae..a8a674e0a 100644 --- a/autopilot/hostfilter.go +++ b/autopilot/hostfilter.go @@ -313,7 +313,7 @@ func isOutOfCollateral(c api.Contract, s rhpv2.HostSettings, pt rhpv3.HostPriceT expectedStorage = s.RemainingStorage } _, _, newCollateral := rhpv3.RenewalCosts(c.Revision.FileContract, pt, expectedStorage, c.EndHeight()) - return isBelowCollateralThreshold(newCollateral, c.RemainingCollateral(s)) + return isBelowCollateralThreshold(newCollateral, c.RemainingCollateral()) } // isBelowCollateralThreshold returns true if the remainingCollateral is below a diff --git a/worker/host.go b/worker/host.go index ccf9f5645..7f6d28d72 100644 --- a/worker/host.go +++ b/worker/host.go @@ -167,8 +167,7 @@ func (h *host) RenewContract(ctx context.Context, rrr api.RHPRenewRequest) (_ rh err = h.transportPool.withTransportV3(ctx, h.hk, h.siamuxAddr, func(ctx context.Context, t *transportV3) (err error) { _, err = RPCLatestRevision(ctx, t, h.fcid, func(revision *types.FileContractRevision) (rhpv3.HostPriceTable, rhpv3.PaymentMethod, error) { // Renew contract. - contractPrice = pt.ContractPrice - rev, txnSet, renewErr = RPCRenew(ctx, rrr, h.bus, t, pt, *revision, h.renterKey, h.logger) + rev, txnSet, contractPrice, renewErr = RPCRenew(ctx, rrr, h.bus, t, pt, *revision, h.renterKey, h.logger) return rhpv3.HostPriceTable{}, nil, nil }) return err diff --git a/worker/rhpv3.go b/worker/rhpv3.go index 15bc96575..45cb6ad0d 100644 --- a/worker/rhpv3.go +++ b/worker/rhpv3.go @@ -1184,11 +1184,11 @@ func RPCAppendSector(ctx context.Context, t *transportV3, renterKey types.Privat return } -func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transportV3, pt *rhpv3.HostPriceTable, rev types.FileContractRevision, renterKey types.PrivateKey, l *zap.SugaredLogger) (_ rhpv2.ContractRevision, _ []types.Transaction, err error) { +func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transportV3, pt *rhpv3.HostPriceTable, rev types.FileContractRevision, renterKey types.PrivateKey, l *zap.SugaredLogger) (_ rhpv2.ContractRevision, _ []types.Transaction, _ types.Currency, err error) { defer wrapErr(&err, "RPCRenew") s, err := t.DialStream(ctx) if err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to dial stream: %w", err) + return rhpv2.ContractRevision{}, nil, types.ZeroCurrency, fmt.Errorf("failed to dial stream: %w", err) } defer s.Close() @@ -1198,7 +1198,7 @@ func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transpor ptUID = pt.UID } if err = s.WriteRequest(rhpv3.RPCRenewContractID, &ptUID); err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to send ptUID: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to send ptUID: %w", err) } // If we didn't have a valid pricetable, read the temporary one from the @@ -1206,28 +1206,31 @@ func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transpor if ptUID == (rhpv3.SettingsID{}) { var ptResp rhpv3.RPCUpdatePriceTableResponse if err = s.ReadResponse(&ptResp, defaultRPCResponseMaxSize); err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to read RPCUpdatePriceTableResponse: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to read RPCUpdatePriceTableResponse: %w", err) } pt = new(rhpv3.HostPriceTable) if err = json.Unmarshal(ptResp.PriceTableJSON, pt); err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to unmarshal price table: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to unmarshal price table: %w", err) } } + // Remember contract price. + contractPrice := pt.ContractPrice + // Perform gouging checks. gc, err := GougingCheckerFromContext(ctx, false) if err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to get gouging checker: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to get gouging checker: %w", err) } if breakdown := gc.Check(nil, pt); breakdown.Gouging() { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("host gouging during renew: %v", breakdown) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("host gouging during renew: %v", breakdown) } // Prepare the signed transaction that contains the final revision as well // as the new contract wprr, err := bus.WalletPrepareRenew(ctx, rev, rrr.HostAddress, rrr.RenterAddress, renterKey, rrr.RenterFunds, rrr.MinNewCollateral, *pt, rrr.EndHeight, rrr.WindowSize, rrr.ExpectedNewStorage) if err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to prepare renew: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to prepare renew: %w", err) } // Starting from here, we need to make sure to release the txn on error. @@ -1250,13 +1253,13 @@ func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transpor FinalRevisionSignature: finalRevisionSignature, } if err = s.WriteResponse(&req); err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to send RPCRenewContractRequest: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to send RPCRenewContractRequest: %w", err) } // Incorporate the host's additions. var hostAdditions rhpv3.RPCRenewContractHostAdditions if err = s.ReadResponse(&hostAdditions, defaultRPCResponseMaxSize); err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to read RPCRenewContractHostAdditions: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to read RPCRenewContractHostAdditions: %w", err) } parents = append(parents, hostAdditions.Parents...) txn.SiacoinInputs = append(txn.SiacoinInputs, hostAdditions.SiacoinInputs...) @@ -1288,7 +1291,7 @@ func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transpor Signatures: []uint64{0, 1}, } if err := bus.WalletSign(ctx, &txn, wprr.ToSign, cf); err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to sign transaction: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to sign transaction: %w", err) } // Create a new no-op revision and sign it. @@ -1312,13 +1315,13 @@ func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transpor RevisionSignature: renterNoOpRevisionSignature, } if err = s.WriteResponse(&rs); err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to send RPCRenewSignatures: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to send RPCRenewSignatures: %w", err) } // Receive the host's signatures. var hostSigs rhpv3.RPCRenewSignatures if err = s.ReadResponse(&hostSigs, defaultRPCResponseMaxSize); err != nil { - return rhpv2.ContractRevision{}, nil, fmt.Errorf("failed to read RPCRenewSignatures: %w", err) + return rhpv2.ContractRevision{}, nil, types.Currency{}, fmt.Errorf("failed to read RPCRenewSignatures: %w", err) } txn.Signatures = append(txn.Signatures, hostSigs.TransactionSignatures...) @@ -1328,7 +1331,7 @@ func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transpor return rhpv2.ContractRevision{ Revision: noOpRevision, Signatures: [2]types.TransactionSignature{renterNoOpRevisionSignature, hostSigs.RevisionSignature}, - }, txnSet, nil + }, txnSet, contractPrice, nil } // initialRevision returns the first revision of a file contract formation From 6b533dad64171e5fcbef12c5af3d90228476b563 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 14 Dec 2023 15:01:42 +0100 Subject: [PATCH 05/14] worker: update host_test.go --- worker/host_test.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/worker/host_test.go b/worker/host_test.go index c73118612..15e272ca6 100644 --- a/worker/host_test.go +++ b/worker/host_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "strings" "testing" "time" @@ -28,13 +27,17 @@ var ( _ Host = (*mockHost)(nil) ) +var ( + errSectorOutOfBounds = errors.New("sector out of bounds") +) + func (h *mockHost) DownloadSector(ctx context.Context, w io.Writer, root types.Hash256, offset, length uint32, overpay bool) error { sector, exist := h.sectors[root] if !exist { - return errors.New("sector not found") + return errSectorNotFound } if offset+length > rhpv2.SectorSize { - return errors.New("sector out of bounds") + return errSectorOutOfBounds } _, err := w.Write(sector[offset : offset+length]) return err @@ -67,12 +70,15 @@ func (h *mockHost) SyncAccount(ctx context.Context, rev *types.FileContractRevis } func TestHost(t *testing.T) { - h := newTestHost() - s := newTestSector() + h := newMockHost() + sector, root := newMockSector() - root, err := h.UploadSector(context.Background(), s, types.FileContractRevision{}) + // upload the sector + uploaded, err := h.UploadSector(context.Background(), sector, types.FileContractRevision{}) if err != nil { t.Fatal(err) + } else if uploaded != root { + t.Fatal("root mismatch") } // download entire sector @@ -80,8 +86,7 @@ func TestHost(t *testing.T) { err = h.DownloadSector(context.Background(), &buf, root, 0, rhpv2.SectorSize, false) if err != nil { t.Fatal(err) - } - if !bytes.Equal(buf.Bytes(), s[:]) { + } else if !bytes.Equal(buf.Bytes(), sector[:]) { t.Fatal("sector mismatch") } @@ -90,19 +95,18 @@ func TestHost(t *testing.T) { err = h.DownloadSector(context.Background(), &buf, root, 64, 64, false) if err != nil { t.Fatal(err) - } - if !bytes.Equal(buf.Bytes(), s[64:128]) { + } else if !bytes.Equal(buf.Bytes(), sector[64:128]) { t.Fatal("sector mismatch") } // try downloading out of bounds err = h.DownloadSector(context.Background(), &buf, root, rhpv2.SectorSize, 64, false) - if err == nil || !strings.Contains(err.Error(), "out of bounds") { + if !errors.Is(err, errSectorOutOfBounds) { t.Fatal("expected out of bounds error", err) } } -func newTestHost() Host { +func newMockHost() *mockHost { return &mockHost{ hk: types.PublicKey{1}, fcid: types.FileContractID{1}, @@ -110,8 +114,8 @@ func newTestHost() Host { } } -func newTestSector() *[rhpv2.SectorSize]byte { +func newMockSector() (*[rhpv2.SectorSize]byte, types.Hash256) { var sector [rhpv2.SectorSize]byte frand.Read(sector[:]) - return §or + return §or, rhpv2.SectorRoot(§or) } From aa521cc91d639ee4586bc64e798fa1f69d155e6b Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 14 Dec 2023 15:56:23 +0100 Subject: [PATCH 06/14] cmd: update version --- cmd/renterd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/renterd/main.go b/cmd/renterd/main.go index 9a7004999..a884a3c42 100644 --- a/cmd/renterd/main.go +++ b/cmd/renterd/main.go @@ -296,7 +296,7 @@ func main() { flag.Parse() - log.Println("renterd v0.6.0") + log.Println("renterd v0.7.1") log.Println("Network", build.NetworkName()) if flag.Arg(0) == "version" { log.Println("Commit:", githash) From b6e34ac8d0745137d5401953b1d70ff1784c26bf Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 14 Dec 2023 16:14:56 +0100 Subject: [PATCH 07/14] stores: fix migration 35 --- stores/migrations.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/stores/migrations.go b/stores/migrations.go index 86d4b9c13..a8cd8225c 100644 --- a/stores/migrations.go +++ b/stores/migrations.go @@ -1392,6 +1392,14 @@ func performMigration00034_objectHealth(txn *gorm.DB, logger *zap.SugaredLogger) func performMigration00035_bufferedSlabsDropSizeAndComplete(txn *gorm.DB, logger *zap.SugaredLogger) error { logger.Info("performing migration 00035_bufferedSlabsDropSizeAndComplete") + + // Disable foreign keys in SQLite to avoid issues with updating constraints. + if isSQLite(txn) { + if err := txn.Exec(`PRAGMA foreign_keys = 0`).Error; err != nil { + return err + } + } + if txn.Migrator().HasColumn(&dbBufferedSlab{}, "size") { if err := txn.Migrator().DropColumn(&dbBufferedSlab{}, "size"); err != nil { return err @@ -1402,6 +1410,16 @@ func performMigration00035_bufferedSlabsDropSizeAndComplete(txn *gorm.DB, logger return err } } + + // Enable foreign keys again. + if isSQLite(txn) { + if err := txn.Exec(`PRAGMA foreign_keys = 1`).Error; err != nil { + return err + } + if err := txn.Exec(`PRAGMA foreign_key_check(buffered_slabs)`).Error; err != nil { + return err + } + } logger.Info("migration 00035_bufferedSlabsDropSizeAndComplete complete") return nil } From d9507ff8aeaf46c012a1f8e25148745c2ccf182a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 14 Dec 2023 16:20:46 +0100 Subject: [PATCH 08/14] docker: use alpine instead of debian --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a4d27d01a..4b68df053 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,7 +26,7 @@ RUN --mount=type=cache,target=/root/go/pkg/mod \ CGO_ENABLED=1 go build -ldflags="-s -w" -tags="${BUILD_TAGS}" ./cmd/renterd # Build image that will be used to run renterd. -FROM debian:bookworm-slim +FROM alpine:3 LABEL maintainer="The Sia Foundation " \ org.opencontainers.image.description.vendor="The Sia Foundation" \ org.opencontainers.image.description="A renterd container - next-generation Sia renter" \ From d20bf3ec8a595d49499bc6ebcc390f76c469ca9a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 14 Dec 2023 17:00:14 +0100 Subject: [PATCH 09/14] docker: fix linking --- docker/Dockerfile | 2 +- docker/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4b68df053..55d27ac99 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,7 +23,7 @@ RUN if [ "$BUILD_RUN_GO_GENERATE" = "true" ] ; then go generate ./... ; fi # Build renterd. RUN --mount=type=cache,target=/root/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=1 go build -ldflags="-s -w" -tags="${BUILD_TAGS}" ./cmd/renterd + CGO_ENABLED=1 go build -ldflags='-s -w -linkmode external -extldflags "-static"' -tags="${BUILD_TAGS}" ./cmd/renterd # Build image that will be used to run renterd. FROM alpine:3 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 167838af5..bac2ba601 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh if [[ "$BUILD_TAGS" == *'testnet'* ]]; then exec renterd -http=':9880' -s3.address=':7070' "$@" From 28f668f04fda4896c0280e756e22a3d476f8356a Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 14 Dec 2023 17:19:14 +0100 Subject: [PATCH 10/14] stores: print version --- stores/sql.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/stores/sql.go b/stores/sql.go index 9e4cf1eee..26b1e91f5 100644 --- a/stores/sql.go +++ b/stores/sql.go @@ -161,6 +161,21 @@ func NewSQLStore(conn, connMetrics gorm.Dialector, alerts alerts.Alerter, partia } l := logger.Named("sql") + // Print SQLite version + var dbName string + var dbVersion string + if isSQLite(db) { + err = db.Raw("select sqlite_version()").Scan(&dbVersion).Error + dbName = "SQLite" + } else { + err = db.Raw("select version()").Scan(&dbVersion).Error + dbName = "MySQL" + } + if err != nil { + return nil, modules.ConsensusChangeID{}, fmt.Errorf("failed to fetch db version: %v", err) + } + l.Infof("Using %s version %s", dbName, dbVersion) + // Perform migrations. if migrate { if err := performMigrations(db, l); err != nil { From ab1f39107befa9c619c19f4bbb3ee393dc3fe962 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 14 Dec 2023 17:38:31 +0100 Subject: [PATCH 11/14] autopilot: address comments --- autopilot/contractor.go | 1 - worker/rhpv3.go | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/autopilot/contractor.go b/autopilot/contractor.go index deac9ee42..16666d16c 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -1458,7 +1458,6 @@ func (c *contractor) refreshContract(ctx context.Context, w Worker, ci contractI return api.ContractMetadata{}, false, fmt.Errorf("insufficient budget: %s < %s", budget.String(), renterFunds.String()) } - // calculate the new collateral expectedStorage := renterFundsToExpectedStorage(renterFunds, contract.EndHeight()-cs.BlockHeight, ci.priceTable) unallocatedCollateral := rev.MissedHostPayout().Sub(contract.ContractPrice) diff --git a/worker/rhpv3.go b/worker/rhpv3.go index 45cb6ad0d..06254e0f0 100644 --- a/worker/rhpv3.go +++ b/worker/rhpv3.go @@ -1214,9 +1214,6 @@ func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transpor } } - // Remember contract price. - contractPrice := pt.ContractPrice - // Perform gouging checks. gc, err := GougingCheckerFromContext(ctx, false) if err != nil { @@ -1331,7 +1328,7 @@ func RPCRenew(ctx context.Context, rrr api.RHPRenewRequest, bus Bus, t *transpor return rhpv2.ContractRevision{ Revision: noOpRevision, Signatures: [2]types.TransactionSignature{renterNoOpRevisionSignature, hostSigs.RevisionSignature}, - }, txnSet, contractPrice, nil + }, txnSet, pt.ContractPrice, nil } // initialRevision returns the first revision of a file contract formation From 9436b7750b9afe64aa42185d8759d0cff848badf Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Dec 2023 11:10:04 +0100 Subject: [PATCH 12/14] wallet: redistribute wallet in batches --- autopilot/autopilot.go | 2 +- autopilot/contractor.go | 16 ++-- bus/bus.go | 23 +++--- bus/client/wallet.go | 4 +- wallet/wallet.go | 174 +++++++++++++++++++++++----------------- wallet/wallet_test.go | 24 ++++-- 6 files changed, 143 insertions(+), 100 deletions(-) diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 43b54023d..b4ab8cc5e 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -87,7 +87,7 @@ type Bus interface { WalletDiscard(ctx context.Context, txn types.Transaction) error WalletOutputs(ctx context.Context) (resp []wallet.SiacoinElement, err error) WalletPending(ctx context.Context) (resp []types.Transaction, err error) - WalletRedistribute(ctx context.Context, outputs int, amount types.Currency) (id types.TransactionID, err error) + WalletRedistribute(ctx context.Context, outputs int, amount types.Currency) (ids []types.TransactionID, err error) } type Autopilot struct { diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 16666d16c..04b68c640 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -91,7 +91,7 @@ type ( resolver *ipResolver logger *zap.SugaredLogger - maintenanceTxnID types.TransactionID + maintenanceTxnIDs []types.TransactionID revisionBroadcastInterval time.Duration revisionLastBroadcast map[types.FileContractID]time.Time @@ -579,9 +579,11 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { return nil } for _, txn := range pending { - if c.maintenanceTxnID == txn.ID() { - l.Debugf("wallet maintenance skipped, pending transaction found with id %v", c.maintenanceTxnID) - return nil + for _, mTxnID := range c.maintenanceTxnIDs { + if mTxnID == txn.ID() { + l.Debugf("wallet maintenance skipped, pending transaction found with id %v", mTxnID) + return nil + } } } @@ -607,13 +609,13 @@ func (c *contractor) performWalletMaintenance(ctx context.Context) error { } // redistribute outputs - id, err := b.WalletRedistribute(ctx, int(outputs), amount) + ids, err := b.WalletRedistribute(ctx, int(outputs), amount) if err != nil { return fmt.Errorf("failed to redistribute wallet into %d outputs of amount %v, balance %v, err %v", outputs, amount, balance, err) } - l.Debugf("wallet maintenance succeeded, tx %v", id) - c.maintenanceTxnID = id + l.Debugf("wallet maintenance succeeded, txns %v", ids) + c.maintenanceTxnIDs = ids return nil } diff --git a/bus/bus.go b/bus/bus.go index 9689f307c..7fc7e0c8b 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -83,8 +83,8 @@ type ( Balance() (spendable, confirmed, unconfirmed types.Currency, _ error) FundTransaction(cs consensus.State, txn *types.Transaction, amount types.Currency, useUnconfirmedTxns bool) ([]types.Hash256, error) Height() uint64 - Redistribute(cs consensus.State, outputs int, amount, feePerByte types.Currency, pool []types.Transaction) (types.Transaction, []types.Hash256, error) - ReleaseInputs(txn types.Transaction) + Redistribute(cs consensus.State, outputs int, amount, feePerByte types.Currency, pool []types.Transaction) ([]types.Transaction, []types.Hash256, error) + ReleaseInputs(txn ...types.Transaction) SignTransaction(cs consensus.State, txn *types.Transaction, toSign []types.Hash256, cf types.CoveredFields) error Transactions(before, since time.Time, offset, limit int) ([]wallet.Transaction, error) UnspentOutputs() ([]wallet.SiacoinElement, error) @@ -602,22 +602,27 @@ func (b *bus) walletRedistributeHandler(jc jape.Context) { } cs := b.cm.TipState() - txn, toSign, err := b.w.Redistribute(cs, wfr.Outputs, wfr.Amount, b.tp.RecommendedFee(), b.tp.Transactions()) + txns, toSign, err := b.w.Redistribute(cs, wfr.Outputs, wfr.Amount, b.tp.RecommendedFee(), b.tp.Transactions()) if jc.Check("couldn't redistribute money in the wallet into the desired outputs", err) != nil { return } - err = b.w.SignTransaction(cs, &txn, toSign, types.CoveredFields{WholeTransaction: true}) - if jc.Check("couldn't sign the transaction", err) != nil { - return + var ids []types.TransactionID + for i := 0; i < len(txns); i++ { + err = b.w.SignTransaction(cs, &txns[i], toSign, types.CoveredFields{WholeTransaction: true}) + if jc.Check("couldn't sign the transaction", err) != nil { + b.w.ReleaseInputs(txns...) + return + } + ids = append(ids, txns[i].ID()) } - if jc.Check("couldn't broadcast the transaction", b.tp.AcceptTransactionSet([]types.Transaction{txn})) != nil { - b.w.ReleaseInputs(txn) + if jc.Check("couldn't broadcast the transaction", b.tp.AcceptTransactionSet(txns)) != nil { + b.w.ReleaseInputs(txns...) return } - jc.Encode(txn.ID()) + jc.Encode(ids) } func (b *bus) walletDiscardHandler(jc jape.Context) { diff --git a/bus/client/wallet.go b/bus/client/wallet.go index 7630654f6..0d4761e51 100644 --- a/bus/client/wallet.go +++ b/bus/client/wallet.go @@ -116,13 +116,13 @@ func (c *Client) WalletPrepareRenew(ctx context.Context, revision types.FileCont // WalletRedistribute broadcasts a transaction that redistributes the money in // the wallet in the desired number of outputs of given amount. If the // transaction was successfully broadcasted it will return the transaction ID. -func (c *Client) WalletRedistribute(ctx context.Context, outputs int, amount types.Currency) (id types.TransactionID, err error) { +func (c *Client) WalletRedistribute(ctx context.Context, outputs int, amount types.Currency) (ids []types.TransactionID, err error) { req := api.WalletRedistributeRequest{ Amount: amount, Outputs: outputs, } - err = c.c.WithContext(ctx).POST("/wallet/redistribute", req, &id) + err = c.c.WithContext(ctx).POST("/wallet/redistribute", req, &ids) return } diff --git a/wallet/wallet.go b/wallet/wallet.go index 6882d9163..76e8e60a9 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "reflect" "sort" "sync" "time" @@ -16,12 +15,17 @@ import ( "go.sia.tech/renterd/api" "go.sia.tech/siad/modules" "go.uber.org/zap" - "lukechampine.com/frand" ) -// BytesPerInput is the encoded size of a SiacoinInput and corresponding -// TransactionSignature, assuming standard UnlockConditions. -const BytesPerInput = 241 +const ( + // BytesPerInput is the encoded size of a SiacoinInput and corresponding + // TransactionSignature, assuming standard UnlockConditions. + BytesPerInput = 241 + + // redistributeBatchSize is the number of outputs to redistribute per txn to + // avoid creating a txn that is too large. + redistributeBatchSize = 10 +) // ErrInsufficientBalance is returned when there aren't enough unused outputs to // cover the requested amount. @@ -223,14 +227,22 @@ func (w *SingleAddressWallet) FundTransaction(cs consensus.State, txn *types.Tra return nil, err } - // choose outputs randomly - frand.Shuffle(len(utxos), reflect.Swapper(utxos)) + // desc sort + sort.Slice(utxos, func(i, j int) bool { + return utxos[i].Value.Cmp(utxos[j].Value) > 0 + }) // add all unconfirmed outputs to the end of the slice as a last resort if useUnconfirmedTxns { + var tpoolUtxos []SiacoinElement for _, sco := range w.tpoolUtxos { - utxos = append(utxos, sco) + tpoolUtxos = append(tpoolUtxos, sco) } + // desc sort + sort.Slice(tpoolUtxos, func(i, j int) bool { + return tpoolUtxos[i].Value.Cmp(tpoolUtxos[j].Value) > 0 + }) + utxos = append(utxos, tpoolUtxos...) } var outputSum types.Currency @@ -270,9 +282,17 @@ func (w *SingleAddressWallet) FundTransaction(cs consensus.State, txn *types.Tra // ReleaseInputs is a helper function that releases the inputs of txn for use in // other transactions. It should only be called on transactions that are invalid // or will never be broadcast. -func (w *SingleAddressWallet) ReleaseInputs(txn types.Transaction) { - for _, in := range txn.SiacoinInputs { - delete(w.lastUsed, types.Hash256(in.ParentID)) +func (w *SingleAddressWallet) ReleaseInputs(txns ...types.Transaction) { + w.mu.Lock() + defer w.mu.Unlock() + w.releaseInputs(txns...) +} + +func (w *SingleAddressWallet) releaseInputs(txns ...types.Transaction) { + for _, txn := range txns { + for _, in := range txn.SiacoinInputs { + delete(w.lastUsed, types.Hash256(in.ParentID)) + } } } @@ -303,7 +323,7 @@ func (w *SingleAddressWallet) SignTransaction(cs consensus.State, txn *types.Tra // // NOTE: we can not reuse 'FundTransaction' because it randomizes the unspent // transaction outputs it uses and we need a minimal set of inputs -func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amount, feePerByte types.Currency, pool []types.Transaction) (types.Transaction, []types.Hash256, error) { +func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amount, feePerByte types.Currency, pool []types.Transaction) ([]types.Transaction, []types.Hash256, error) { w.mu.Lock() defer w.mu.Unlock() @@ -318,7 +338,7 @@ func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amou // fetch unspent transaction outputs utxos, err := w.store.UnspentSiacoinElements(false) if err != nil { - return types.Transaction{}, nil, err + return nil, nil, err } // check whether a redistribution is necessary, adjust number of desired @@ -332,16 +352,7 @@ func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amou } } if outputs <= 0 { - return types.Transaction{}, nil, nil - } - - // prepare all outputs - var txn types.Transaction - for i := 0; i < int(outputs); i++ { - txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{ - Value: amount, - Address: w.Address(), - }) + return nil, nil, nil } // desc sort @@ -349,67 +360,84 @@ func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amou return utxos[i].Value.Cmp(utxos[j].Value) > 0 }) - // estimate the fees - outputFees := feePerByte.Mul64(uint64(len(encoding.Marshal(txn.SiacoinOutputs)))) - feePerInput := feePerByte.Mul64(BytesPerInput) + // prepare all outputs + var txns []types.Transaction + var toSign []types.Hash256 + + for outputs > 0 { + var txn types.Transaction + for i := 0; outputs > 0 && i < redistributeBatchSize; i++ { + txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{ + Value: amount, + Address: w.Address(), + }) + } - // collect outputs that cover the total amount - var inputs []SiacoinElement - want := amount.Mul64(uint64(outputs)) - var amtInUse, amtSameValue, amtNotMatured types.Currency - for _, sce := range utxos { - inUse := w.isOutputUsed(sce.ID) || inPool[sce.ID] - matured := cs.Index.Height >= sce.MaturityHeight - sameValue := sce.Value.Equals(amount) - if inUse { - amtInUse = amtInUse.Add(sce.Value) - continue - } else if sameValue { - amtSameValue = amtSameValue.Add(sce.Value) - continue - } else if !matured { - amtNotMatured = amtNotMatured.Add(sce.Value) - continue + // estimate the fees + outputFees := feePerByte.Mul64(uint64(len(encoding.Marshal(txn.SiacoinOutputs)))) + feePerInput := feePerByte.Mul64(BytesPerInput) + + // collect outputs that cover the total amount + var inputs []SiacoinElement + want := amount.Mul64(uint64(outputs)) + var amtInUse, amtSameValue, amtNotMatured types.Currency + for _, sce := range utxos { + inUse := w.isOutputUsed(sce.ID) || inPool[sce.ID] + matured := cs.Index.Height >= sce.MaturityHeight + sameValue := sce.Value.Equals(amount) + if inUse { + amtInUse = amtInUse.Add(sce.Value) + continue + } else if sameValue { + amtSameValue = amtSameValue.Add(sce.Value) + continue + } else if !matured { + amtNotMatured = amtNotMatured.Add(sce.Value) + continue + } + + inputs = append(inputs, sce) + fee := feePerInput.Mul64(uint64(len(inputs))).Add(outputFees) + if SumOutputs(inputs).Cmp(want.Add(fee)) > 0 { + break + } } - inputs = append(inputs, sce) + // not enough outputs found fee := feePerInput.Mul64(uint64(len(inputs))).Add(outputFees) - if SumOutputs(inputs).Cmp(want.Add(fee)) > 0 { - break + if sumOut := SumOutputs(inputs); sumOut.Cmp(want.Add(fee)) < 0 { + // in case of an error we need to free all inputs + w.releaseInputs(txns...) + return nil, nil, fmt.Errorf("%w: inputs %v < needed %v + txnFee %v (usable: %v, inUse: %v, sameValue: %v, notMatured: %v)", + ErrInsufficientBalance, sumOut.String(), want.String(), fee.String(), sumOut.String(), amtInUse.String(), amtSameValue.String(), amtNotMatured.String()) } - } - // not enough outputs found - fee := feePerInput.Mul64(uint64(len(inputs))).Add(outputFees) - if sumOut := SumOutputs(inputs); sumOut.Cmp(want.Add(fee)) < 0 { - return types.Transaction{}, nil, fmt.Errorf("%w: inputs %v < needed %v + txnFee %v (usable: %v, inUse: %v, sameValue: %v, notMatured: %v)", - ErrInsufficientBalance, sumOut.String(), want.String(), fee.String(), sumOut.String(), amtInUse.String(), amtSameValue.String(), amtNotMatured.String()) - } + // set the miner fee + txn.MinerFees = []types.Currency{fee} - // set the miner fee - txn.MinerFees = []types.Currency{fee} + // add the change output + change := SumOutputs(inputs).Sub(want.Add(fee)) + if !change.IsZero() { + txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{ + Value: change, + Address: w.addr, + }) + } - // add the change output - change := SumOutputs(inputs).Sub(want.Add(fee)) - if !change.IsZero() { - txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{ - Value: change, - Address: w.addr, - }) - } + // add the inputs + for _, sce := range inputs { + txn.SiacoinInputs = append(txn.SiacoinInputs, types.SiacoinInput{ + ParentID: types.SiacoinOutputID(sce.ID), + UnlockConditions: StandardUnlockConditions(w.priv.PublicKey()), + }) + toSign = append(toSign, sce.ID) + w.lastUsed[sce.ID] = time.Now() + } - // add the inputs - toSign := make([]types.Hash256, len(inputs)) - for i, sce := range inputs { - txn.SiacoinInputs = append(txn.SiacoinInputs, types.SiacoinInput{ - ParentID: types.SiacoinOutputID(sce.ID), - UnlockConditions: StandardUnlockConditions(w.priv.PublicKey()), - }) - toSign[i] = sce.ID - w.lastUsed[sce.ID] = time.Now() + txns = append(txns, txn) } - return txn, toSign, nil + return txns, toSign, nil } func (w *SingleAddressWallet) isOutputUsed(id types.Hash256) bool { diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index a6b02fcc0..922aed59f 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -91,10 +91,12 @@ func TestWalletRedistribute(t *testing.T) { // split into 3 outputs of 6SC each amount := oneSC.Mul64(6) - if txn, _, err := w.Redistribute(cs, 3, amount, types.NewCurrency64(1), nil); err != nil { + if txns, _, err := w.Redistribute(cs, 3, amount, types.NewCurrency64(1), nil); err != nil { t.Fatal(err) + } else if len(txns) != 1 { + t.Fatalf("unexpected number of txns, %v != 1", len(txns)) } else { - applyTxn(txn) + applyTxn(txns[0]) } // assert number of outputs @@ -117,10 +119,12 @@ func TestWalletRedistribute(t *testing.T) { // split into 2 outputs of 9SC amount = oneSC.Mul64(9) - if txn, _, err := w.Redistribute(cs, 2, amount, types.NewCurrency64(1), nil); err != nil { + if txns, _, err := w.Redistribute(cs, 2, amount, types.NewCurrency64(1), nil); err != nil { t.Fatal(err) + } else if len(txns) != 1 { + t.Fatalf("unexpected number of txns, %v != 1", len(txns)) } else { - applyTxn(txn) + applyTxn(txns[0]) } // assert number of outputs @@ -137,10 +141,12 @@ func TestWalletRedistribute(t *testing.T) { // split into 5 outputs of 3SC amount = oneSC.Mul64(3) - if txn, _, err := w.Redistribute(cs, 5, amount, types.NewCurrency64(1), nil); err != nil { + if txns, _, err := w.Redistribute(cs, 5, amount, types.NewCurrency64(1), nil); err != nil { t.Fatal(err) + } else if len(txns) != 1 { + t.Fatalf("unexpected number of txns, %v != 1", len(txns)) } else { - applyTxn(txn) + applyTxn(txns[0]) } // assert number of outputs that hold 3SC @@ -154,10 +160,12 @@ func TestWalletRedistribute(t *testing.T) { } // split into 6 outputs of 3SC - if txn, _, err := w.Redistribute(cs, 6, amount, types.NewCurrency64(1), nil); err != nil { + if txns, _, err := w.Redistribute(cs, 6, amount, types.NewCurrency64(1), nil); err != nil { t.Fatal(err) + } else if len(txns) != 1 { + t.Fatalf("unexpected number of txns, %v != 1", len(txns)) } else { - applyTxn(txn) + applyTxn(txns[0]) } // assert number of outputs that hold 3SC From 53dac7c65f33fae5c17589b6b8580a34fd3076e7 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Dec 2023 11:27:32 +0100 Subject: [PATCH 13/14] wallet: fix TestWalletRedistribute --- wallet/wallet.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index 76e8e60a9..aa14acb0f 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -366,7 +366,7 @@ func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amou for outputs > 0 { var txn types.Transaction - for i := 0; outputs > 0 && i < redistributeBatchSize; i++ { + for i := 0; i < outputs && i < redistributeBatchSize; i++ { txn.SiacoinOutputs = append(txn.SiacoinOutputs, types.SiacoinOutput{ Value: amount, Address: w.Address(), @@ -434,6 +434,7 @@ func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amou w.lastUsed[sce.ID] = time.Now() } + outputs -= len(txn.SiacoinOutputs) txns = append(txns, txn) } From 3fd61f64e1d6cfe070a1ee122a151974e75a2721 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 15 Dec 2023 14:14:03 +0100 Subject: [PATCH 14/14] wallet: address comments --- wallet/wallet.go | 7 ++----- wallet/wallet_test.go | 35 +++++++++++++++++++++++++---------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/wallet/wallet.go b/wallet/wallet.go index aa14acb0f..ea2859aa0 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -320,9 +320,6 @@ func (w *SingleAddressWallet) SignTransaction(cs consensus.State, txn *types.Tra // Redistribute returns a transaction that redistributes money in the wallet by // selecting a minimal set of inputs to cover the creation of the requested // outputs. It also returns a list of output IDs that need to be signed. -// -// NOTE: we can not reuse 'FundTransaction' because it randomizes the unspent -// transaction outputs it uses and we need a minimal set of inputs func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amount, feePerByte types.Currency, pool []types.Transaction) ([]types.Transaction, []types.Hash256, error) { w.mu.Lock() defer w.mu.Unlock() @@ -372,6 +369,7 @@ func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amou Address: w.Address(), }) } + outputs -= len(txn.SiacoinOutputs) // estimate the fees outputFees := feePerByte.Mul64(uint64(len(encoding.Marshal(txn.SiacoinOutputs)))) @@ -379,7 +377,7 @@ func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amou // collect outputs that cover the total amount var inputs []SiacoinElement - want := amount.Mul64(uint64(outputs)) + want := amount.Mul64(uint64(len(txn.SiacoinOutputs))) var amtInUse, amtSameValue, amtNotMatured types.Currency for _, sce := range utxos { inUse := w.isOutputUsed(sce.ID) || inPool[sce.ID] @@ -434,7 +432,6 @@ func (w *SingleAddressWallet) Redistribute(cs consensus.State, outputs int, amou w.lastUsed[sce.ID] = time.Now() } - outputs -= len(txn.SiacoinOutputs) txns = append(txns, txn) } diff --git a/wallet/wallet_test.go b/wallet/wallet_test.go index 922aed59f..0538d50af 100644 --- a/wallet/wallet_test.go +++ b/wallet/wallet_test.go @@ -1,4 +1,4 @@ -package wallet_test +package wallet import ( "context" @@ -9,7 +9,6 @@ import ( "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/wallet" "go.uber.org/zap" "lukechampine.com/frand" ) @@ -17,15 +16,15 @@ import ( // mockStore implements wallet.SingleAddressStore and allows to manipulate the // wallet's utxos type mockStore struct { - utxos []wallet.SiacoinElement + utxos []SiacoinElement } func (s *mockStore) Balance() (types.Currency, error) { return types.ZeroCurrency, nil } func (s *mockStore) Height() uint64 { return 0 } -func (s *mockStore) UnspentSiacoinElements(bool) ([]wallet.SiacoinElement, error) { +func (s *mockStore) UnspentSiacoinElements(bool) ([]SiacoinElement, error) { return s.utxos, nil } -func (s *mockStore) Transactions(before, since time.Time, offset, limit int) ([]wallet.Transaction, error) { +func (s *mockStore) Transactions(before, since time.Time, offset, limit int) ([]Transaction, error) { return nil, nil } func (s *mockStore) RecordWalletMetric(ctx context.Context, metrics ...api.WalletMetric) error { @@ -47,16 +46,16 @@ func TestWalletRedistribute(t *testing.T) { // create a wallet with one output priv := types.GeneratePrivateKey() pub := priv.PublicKey() - utxo := wallet.SiacoinElement{ + utxo := SiacoinElement{ types.SiacoinOutput{ Value: oneSC.Mul64(20), - Address: wallet.StandardAddress(pub), + Address: StandardAddress(pub), }, randomOutputID(), 0, } - s := &mockStore{utxos: []wallet.SiacoinElement{utxo}} - w := wallet.NewSingleAddressWallet(priv, s, 0, zap.NewNop().Sugar()) + s := &mockStore{utxos: []SiacoinElement{utxo}} + w := NewSingleAddressWallet(priv, s, 0, zap.NewNop().Sugar()) numOutputsWithValue := func(v types.Currency) (c uint64) { utxos, _ := w.UnspentOutputs() @@ -78,7 +77,7 @@ func TestWalletRedistribute(t *testing.T) { } } for _, output := range txn.SiacoinOutputs { - s.utxos = append(s.utxos, wallet.SiacoinElement{output, randomOutputID(), 0}) + s.utxos = append(s.utxos, SiacoinElement{output, randomOutputID(), 0}) } } @@ -172,6 +171,22 @@ func TestWalletRedistribute(t *testing.T) { if cnt := numOutputsWithValue(amount); cnt != 6 { t.Fatalf("unexpected number of 3SC outputs, %v != 6", cnt) } + + // split into 2 times the redistributeBatchSize + amount = oneSC.Div64(10) + if txns, _, err := w.Redistribute(cs, 2*redistributeBatchSize, amount, types.NewCurrency64(1), nil); err != nil { + t.Fatal(err) + } else if len(txns) != 2 { + t.Fatalf("unexpected number of txns, %v != 2", len(txns)) + } else { + applyTxn(txns[0]) + applyTxn(txns[1]) + } + + // assert number of outputs that hold 0.1SC + if cnt := numOutputsWithValue(amount); cnt != 2*redistributeBatchSize { + t.Fatalf("unexpected number of 0.1SC outputs, %v != 20", cnt) + } } func randomOutputID() (t types.Hash256) {