Merge branch 'dev' of github.com:SiaFoundation/renterd into pj/move-m…

…igrations

.changeset/add_bus_routes_to_openapi_spec.md

@@ -0,0 +1,11 @@
+---
+default: major
+---
+
+# Add bus section to openapi spec
+
+Added routes:
+- accounts
+- alerts
+- autopilot
+- buckets

.changeset/improve_migration_out_after_foreignkey_check_fails.md

@@ -0,0 +1,5 @@
+---
+default: patch
+---
+
+# Improve migration out after foreignkey check fails

bus/routes.go

@@ -45,13 +45,16 @@ func (b *Bus) accountsFundHandler(jc jape.Context) {
 		return
 	}
 
-	// contract metadata
+	// fetch contract
 	cm, err := b.store.Contract(jc.Request.Context(), req.ContractID)
-	if jc.Check("failed to fetch contract metadata", err) != nil {
+	if errors.Is(err, api.ErrContractNotFound) {
+		jc.Error(err, http.StatusNotFound)
+		return
+	} else if jc.Check("failed to fetch contract metadata", err) != nil {
 		return
 	}
 
-	// host
+	// fetch host
 	host, err := b.store.Host(jc.Request.Context(), cm.HostKey)
 	if jc.Check("failed to fetch host for contract", err) != nil {
 		return
@@ -276,21 +279,33 @@ func (b *Bus) bucketsHandlerPOST(jc jape.Context) {
 	} else if err := req.Validate(); err != nil {
 		jc.Error(err, http.StatusBadRequest)
 		return
-	} else if jc.Check("failed to create bucket", b.store.CreateBucket(jc.Request.Context(), req.Name, req.Policy)) != nil {
+	}
+
+	err := b.store.CreateBucket(jc.Request.Context(), req.Name, req.Policy)
+	if errors.Is(err, api.ErrBucketExists) {
+		jc.Error(err, http.StatusConflict)
 		return
 	}
+	jc.Check("failed to create bucket", err)
 }
 
 func (b *Bus) bucketsHandlerPolicyPUT(jc jape.Context) {
 	var req api.BucketUpdatePolicyRequest
 	if jc.Decode(&req) != nil {
 		return
-	} else if bucket := jc.PathParam("name"); bucket == "" {
+	}
+	bucket := jc.PathParam("name")
+	if bucket == "" {
 		jc.Error(errors.New("no bucket name provided"), http.StatusBadRequest)
 		return
-	} else if jc.Check("failed to create bucket", b.store.UpdateBucketPolicy(jc.Request.Context(), bucket, req.Policy)) != nil {
+	}
+
+	err := b.store.UpdateBucketPolicy(jc.Request.Context(), bucket, req.Policy)
+	if errors.Is(err, api.ErrBucketNotFound) {
+		jc.Error(err, http.StatusNotFound)
 		return
 	}
+	jc.Check("failed to create bucket", err)
 }
 
 func (b *Bus) bucketHandlerDELETE(jc jape.Context) {
@@ -300,9 +315,18 @@ func (b *Bus) bucketHandlerDELETE(jc jape.Context) {
 	} else if name == "" {
 		jc.Error(errors.New("no name provided"), http.StatusBadRequest)
 		return
-	} else if jc.Check("failed to delete bucket", b.store.DeleteBucket(jc.Request.Context(), name)) != nil {
+	}
+
+	err := b.store.DeleteBucket(jc.Request.Context(), name)
+	if errors.Is(err, api.ErrBucketNotFound) {
+		jc.Error(err, http.StatusNotFound)
+		return
+	} else if errors.Is(err, api.ErrBucketNotEmpty) {
+		jc.Error(err, http.StatusConflict)
 		return
 	}
+
+	jc.Check("failed to delete bucket", err)
 }
 
 func (b *Bus) bucketHandlerGET(jc jape.Context) {
@@ -1864,9 +1888,7 @@ func (b *Bus) accountsHandlerPOST(jc jape.Context) {
 			return
 		}
 	}
-	if b.store.SaveAccounts(jc.Request.Context(), req.Accounts) != nil {
-		return
-	}
+	jc.Check("failed to save accounts", b.store.SaveAccounts(jc.Request.Context(), req.Accounts))
 }
 
 func (b *Bus) hostsCheckHandlerPUT(jc jape.Context) {

go.mod

@@ -17,7 +17,7 @@ require (
 	go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb
 	go.sia.tech/coreutils v0.7.1-0.20241211045514-6881993d8806
 	go.sia.tech/gofakes3 v0.0.5
-	go.sia.tech/hostd v1.1.3-0.20241203052717-10e79b2b8e85
+	go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db
 	go.sia.tech/jape v0.12.1
 	go.sia.tech/mux v1.3.0
 	go.sia.tech/web/renterd v0.69.0
@@ -32,7 +32,7 @@ require (
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/cloudflare/cloudflare-go v0.109.0 // indirect
+	github.com/cloudflare/cloudflare-go v0.111.0 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect

go.sum

@@ -2,8 +2,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
 github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
-github.com/cloudflare/cloudflare-go v0.109.0 h1:Wjp+RfJD1lidIFUlrTBqUQnCBrUnmVsLxgzWYiURueg=
-github.com/cloudflare/cloudflare-go v0.109.0/go.mod h1:m492eNahT/9MsN7Ppnoge8AaI7QhVFtEgVm3I9HJFeU=
+github.com/cloudflare/cloudflare-go v0.111.0 h1:bFgl5OyR7iaV9DkTaoI2jU8X4rXDzEaFDaPfMTp+Ewo=
+github.com/cloudflare/cloudflare-go v0.111.0/go.mod h1:w5c4Vm00JjZM+W0mPi6QOC+eWLncGQPURtgDck3z5xU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -51,8 +51,8 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dc
 github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
 github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
 go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
 go.sia.tech/core v0.7.2-0.20241210224920-0534a5928ddb h1:JHX+qWKS9sAXmEroICAu2jPQkr3CYUF7iWd/zlATsBM=
@@ -61,8 +61,8 @@ go.sia.tech/coreutils v0.7.1-0.20241211045514-6881993d8806 h1:zmLtpmFQPKMukYMiQB
 go.sia.tech/coreutils v0.7.1-0.20241211045514-6881993d8806/go.mod h1:6z3oHrQqcLoFEAT/l6XnvOivEGXgIfWBKcq0OqsouWA=
 go.sia.tech/gofakes3 v0.0.5 h1:vFhVBUFbKE9ZplvLE2w4TQxFMQyF8qvgxV4TaTph+Vw=
 go.sia.tech/gofakes3 v0.0.5/go.mod h1:LXEzwGw+OHysWLmagleCttX93cJZlT9rBu/icOZjQ54=
-go.sia.tech/hostd v1.1.3-0.20241203052717-10e79b2b8e85 h1:iWitZVJDsazpQHjvmSUHMP6u4p5gySvogkAIQNaaVBU=
-go.sia.tech/hostd v1.1.3-0.20241203052717-10e79b2b8e85/go.mod h1:7REKFrGbO6/Nv49K7p1G8ZwYhZWgIgWyD7iscEMRsEY=
+go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db h1:ey3ezMYHPzY+FZ4yL8xsAWnCJWI2J9z4rtpmRa8dj0A=
+go.sia.tech/hostd v1.1.3-0.20241212081824-0f6d95b852db/go.mod h1:6wTgoXKmsLQT22lUcHI4/dUcb3mhXFR+9zYWIki8Qho=
 go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU=
 go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4=
 go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c=

internal/gouging/gouging.go

@@ -210,7 +210,7 @@ func checkPriceGougingHS(gs api.GougingSettings, hs *rhpv2.HostSettings) error {
 
 	// check EA expiry
 	if hs.EphemeralAccountExpiry < time.Duration(gs.MinAccountExpiry) {
-		return fmt.Errorf("'EphemeralAccountExpiry' is less than the allowed minimum value, %v < %v", hs.EphemeralAccountExpiry, gs.MinAccountExpiry)
+		return fmt.Errorf("'EphemeralAccountExpiry' is less than the allowed minimum value, %v < %v", hs.EphemeralAccountExpiry, time.Duration(gs.MinAccountExpiry))
 	}
 
 	return nil
@@ -280,7 +280,7 @@ func checkPriceGougingPT(gs api.GougingSettings, cs api.ConsensusState, pt *rhpv
 
 	// check Validity
 	if pt.Validity < time.Duration(gs.MinPriceTableValidity) {
-		return fmt.Errorf("'Validity' is less than the allowed minimum value, %v < %v", pt.Validity, gs.MinPriceTableValidity)
+		return fmt.Errorf("'Validity' is less than the allowed minimum value, %v < %v", pt.Validity, time.Duration(gs.MinPriceTableValidity))
 	}
 
 	return nil

internal/rhp/v4/rhp.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"io"
 	"net"
-	"strings"
 	"time"
 
 	"go.sia.tech/core/consensus"
@@ -140,11 +139,6 @@ func (c *Client) AccountBalance(ctx context.Context, hk types.PublicKey, hostIP
 	err := c.tpool.withTransport(ctx, hk, hostIP, func(c rhp.TransportClient) (err error) {
 		balance, err = rhp.RPCAccountBalance(ctx, c, account)
 		if err != nil {
-			// TODO: remove this hack once the host is fixed
-			if strings.Contains(err.Error(), "internal error") {
-				err = nil
-				balance = types.ZeroCurrency
-			}
 			return err
 		}
 		return err

internal/sql/migrations.go

@@ -537,6 +537,12 @@ var (
 					return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00033_remove_contract_sets", log)
 				},
 			},
+			{
+				ID: "00034_v2",
+				Migrate: func(tx Tx) error {
+					return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00034_v2", log)
+				},
+			},
 		}
 	}
 	MetricsMigrations = func(ctx context.Context, migrationsFs embed.FS, log *zap.SugaredLogger) []Migration {

internal/test/e2e/cluster.go

@@ -629,7 +629,7 @@ func addStorageFolderToHost(ctx context.Context, hosts []*Host) error {
 func announceHosts(hosts []*Host) error {
 	for _, host := range hosts {
 		settings := defaultHostSettings
-		settings.NetAddress = host.RHPv2Addr()
+		settings.NetAddress = host.rhp4Listener.Addr().(*net.TCPAddr).IP.String()
 		if err := host.settings.UpdateSettings(settings); err != nil {
 			return err
 		}

internal/test/e2e/cluster_test.go

@@ -2564,7 +2564,7 @@ func TestDownloadAllHosts(t *testing.T) {
 	var randomHost []string
 	for _, host := range cluster.hosts {
 		if _, used := usedHosts[host.PublicKey()]; used {
-			randomHost = []string{host.settings.Settings().NetAddress, host.RHPv4Addr()}
+			randomHost = []string{host.RHPv2Addr(), host.RHPv4Addr()}
 			break
 		}
 	}

internal/test/e2e/host.go

@@ -196,12 +196,12 @@ func (h *Host) UpdateSettings(settings settings.Settings) error {
 
 // RHPv2Settings returns the host's current RHPv2 settings
 func (h *Host) RHPv2Settings() (crhpv2.HostSettings, error) {
-	return h.rhpv2.Settings()
+	return h.settings.RHP2Settings()
 }
 
 // RHPv3PriceTable returns the host's current RHPv3 price table
 func (h *Host) RHPv3PriceTable() (crhpv3.HostPriceTable, error) {
-	return h.rhpv3.PriceTable()
+	return h.settings.RHP3PriceTable()
 }
 
 // WalletAddress returns the host's wallet address
@@ -283,9 +283,11 @@ func NewHost(privKey types.PrivateKey, cm *chain.Manager, dir string, network *c
 		return nil, fmt.Errorf("failed to create rhp3 listener: %w", err)
 	}
 
-	settings, err := settings.NewConfigManager(privKey, db, cm, s, wallet, storage,
+	settings, err := settings.NewConfigManager(privKey, db, cm, s, storage, wallet,
 		settings.WithValidateNetAddress(false),
-		settings.WithRHP4AnnounceAddresses([]chain.NetAddress{{Protocol: rhp4.ProtocolTCPSiaMux, Address: rhp4Listener.Addr().String()}}),
+		settings.WithRHP2Port(uint16(rhp2Listener.Addr().(*net.TCPAddr).Port)),
+		settings.WithRHP3Port(uint16(rhp3Listener.Addr().(*net.TCPAddr).Port)),
+		settings.WithRHP4Port(uint16(rhp4Listener.Addr().(*net.TCPAddr).Port)),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create settings manager: %w", err)
@@ -299,16 +301,10 @@ func NewHost(privKey types.PrivateKey, cm *chain.Manager, dir string, network *c
 	registry := registry.NewManager(privKey, db, zap.NewNop())
 	accounts := accounts.NewManager(db, settings)
 
-	rhpv2, err := rhpv2.NewSessionHandler(rhp2Listener, privKey, rhp3Listener.Addr().String(), cm, s, wallet, contracts, settings, storage, log.Named("rhpv2"))
-	if err != nil {
-		return nil, fmt.Errorf("failed to create rhpv2 session handler: %w", err)
-	}
+	rhpv2 := rhpv2.NewSessionHandler(rhp2Listener, privKey, cm, s, wallet, contracts, settings, storage, log.Named("rhpv2"))
 	go rhpv2.Serve()
 
-	rhpv3, err := rhpv3.NewSessionHandler(rhp3Listener, privKey, cm, s, wallet, accounts, contracts, registry, storage, settings, log.Named("rhpv2"))
-	if err != nil {
-		return nil, fmt.Errorf("failed to create rhpv3 session handler: %w", err)
-	}
+	rhpv3 := rhpv3.NewSessionHandler(rhp3Listener, privKey, cm, s, wallet, accounts, contracts, registry, storage, settings, log.Named("rhpv3"))
 	go rhpv3.Serve()
 
 	rhpv4 := rhp4.NewServer(privKey, cm, s, contracts, wallet, settings, storage, rhp4.WithPriceTableValidity(30*time.Minute), rhp4.WithContractProofWindowBuffer(1))