Merge branch 'dev' of github.com:SiaFoundation/renterd into its-happe…

…ning

.github/workflows/publish.yml

@@ -9,7 +9,7 @@ on:
       - dev
     tags: 
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-**'
 
 concurrency:
   group: ${{ github.workflow }}

.github/workflows/ui.yml

@@ -15,7 +15,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: '1.21.0'
 

README.md

@@ -214,14 +214,14 @@ setting does not have a default value, it can be updated using the settings API:
 
 In most cases the default set should match the set from your autopilot
 configuration in order for migrations to work properly. The contract set can be
-overriden by passing it as a query string parameter to the worker's upload and
+overridden by passing it as a query string parameter to the worker's upload and
 migrate endpoints.
 
 - `PUT /api/worker/objects/foo?contractset=foo`
 
 ### Redundancy
 
-The default redundancy on mainnet is 30-10, on testnet it is 6-2. The redunancy
+The default redundancy on mainnet is 30-10, on testnet it is 6-2. The redundancy
 can be updated using the settings API:
 
 - `GET /api/bus/setting/redundancy`

api/autopilot.go

@@ -91,9 +91,37 @@ type (
 		StartTime TimeRFC3339 `json:"startTime"`
 		BuildState
 	}
-)
 
-type (
+	ConfigEvaluationRequest struct {
+		AutopilotConfig    AutopilotConfig    `json:"autopilotConfig"`
+		GougingSettings    GougingSettings    `json:"gougingSettings"`
+		RedundancySettings RedundancySettings `json:"redundancySettings"`
+	}
+
+	ConfigRecommendation struct {
+		GougingSettings GougingSettings `json:"gougingSettings"`
+	}
+
+	// ConfigEvaluationResponse is the response type for /evaluate
+	ConfigEvaluationResponse struct {
+		Hosts    uint64 `json:"hosts"`
+		Usable   uint64 `json:"usable"`
+		Unusable struct {
+			Blocked uint64 `json:"blocked"`
+			Gouging struct {
+				Contract uint64 `json:"contract"`
+				Download uint64 `json:"download"`
+				Gouging  uint64 `json:"gouging"`
+				Pruning  uint64 `json:"pruning"`
+				Upload   uint64 `json:"upload"`
+			} `json:"gouging"`
+			NotAcceptingContracts uint64 `json:"notAcceptingContracts"`
+			NotScanned            uint64 `json:"notScanned"`
+			Unknown               uint64 `json:"unknown"`
+		}
+		Recommendation *ConfigRecommendation `json:"recommendation,omitempty"`
+	}
+
 	// HostHandlerResponse is the response type for the /host/:hostkey endpoint.
 	HostHandlerResponse struct {
 		Host   hostdb.Host                `json:"host"`

api/host.go

@@ -112,6 +112,6 @@ func (opts HostsForScanningOptions) Apply(values url.Values) {
 		values.Set("limit", fmt.Sprint(opts.Limit))
 	}
 	if !opts.MaxLastScan.IsZero() {
-		values.Set("maxLastScan", fmt.Sprint(TimeRFC3339(opts.MaxLastScan)))
+		values.Set("lastScan", fmt.Sprint(TimeRFC3339(opts.MaxLastScan)))
 	}
 }

api/object.go

@@ -83,9 +83,14 @@ type (
 		Object  *Object          `json:"object,omitempty"`
 	}
 
-	// GetObjectResponse is the response type for the /worker/object endpoint.
+	// GetObjectResponse is the response type for the GET /worker/object endpoint.
 	GetObjectResponse struct {
-		Content      io.ReadCloser      `json:"content"`
+		Content io.ReadCloser `json:"content"`
+		HeadObjectResponse
+	}
+
+	// HeadObjectResponse is the response type for the HEAD /worker/object endpoint.
+	HeadObjectResponse struct {
 		ContentType  string             `json:"contentType"`
 		LastModified string             `json:"lastModified"`
 		Range        *DownloadRange     `json:"range,omitempty"`
@@ -206,6 +211,10 @@ type (
 		Batch bool
 	}
 
+	HeadObjectOptions struct {
+		Range DownloadRange
+	}
+
 	DownloadObjectOptions struct {
 		GetObjectOptions
 		Range DownloadRange
@@ -301,6 +310,16 @@ func (opts DeleteObjectOptions) Apply(values url.Values) {
 	}
 }
 
+func (opts HeadObjectOptions) ApplyHeaders(h http.Header) {
+	if opts.Range != (DownloadRange{}) {
+		if opts.Range.Length == -1 {
+			h.Set("Range", fmt.Sprintf("bytes=%v-", opts.Range.Offset))
+		} else {
+			h.Set("Range", fmt.Sprintf("bytes=%v-%v", opts.Range.Offset, opts.Range.Offset+opts.Range.Length-1))
+		}
+	}
+}
+
 func (opts GetObjectOptions) Apply(values url.Values) {
 	if opts.Prefix != "" {
 		values.Set("prefix", opts.Prefix)

api/worker.go

@@ -19,6 +19,10 @@ var (
 	// ErrContractSetNotSpecified is returned by the worker API by endpoints that
 	// need a contract set to be able to upload data.
 	ErrContractSetNotSpecified = errors.New("contract set is not specified")
+
+	// ErrHostOnPrivateNetwork is returned by the worker API when a host can't
+	// be scanned since it is on a private network.
+	ErrHostOnPrivateNetwork = errors.New("host is on a private network")
 )
 
 type (

autopilot/accounts.go

@@ -136,22 +136,18 @@ func (a *accounts) refillWorkerAccounts(ctx context.Context, w Worker) {
 
 	// refill accounts in separate goroutines
 	for _, c := range contracts {
-		// add logging for contracts in the set
-		_, inSet := inContractSet[c.ID]
-
 		// launch refill if not already in progress
 		if a.markRefillInProgress(workerID, c.HostKey) {
-			go func(contract api.ContractMetadata, inSet bool) {
+			go func(contract api.ContractMetadata) {
 				rCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
 				defer cancel()
 				accountID, refilled, rerr := refillWorkerAccount(rCtx, a.a, w, workerID, contract)
 				if rerr != nil {
-					if inSet || rerr.Is(errMaxDriftExceeded) {
-						// register the alert on failure if the contract is in
-						// the set or the error is errMaxDriftExceeded
+					if rerr.Is(errMaxDriftExceeded) {
+						// register the alert if error is errMaxDriftExceeded
 						a.ap.RegisterAlert(ctx, newAccountRefillAlert(accountID, contract, *rerr))
-						a.l.Errorw(rerr.err.Error(), rerr.keysAndValues...)
 					}
+					a.l.Errorw(rerr.err.Error(), rerr.keysAndValues...)
 				} else {
 					// dismiss alerts on success
 					a.ap.DismissAlert(ctx, alertIDForAccount(alertAccountRefillID, accountID))
@@ -167,7 +163,7 @@ func (a *accounts) refillWorkerAccounts(ctx context.Context, w Worker) {
 				}
 
 				a.markRefillDone(workerID, contract.HostKey)
-			}(c, inSet)
+			}(c)
 		}
 	}
 }

autopilot/autopilot.go

@@ -21,6 +21,7 @@ import (
 	"go.sia.tech/renterd/object"
 	"go.sia.tech/renterd/wallet"
 	"go.sia.tech/renterd/webhooks"
+	"go.sia.tech/renterd/worker"
 	"go.uber.org/zap"
 )
 
@@ -166,13 +167,43 @@ func (ap *Autopilot) Handler() http.Handler {
 	return jape.Mux(map[string]jape.Handler{
 		"GET    /config":        ap.configHandlerGET,
 		"PUT    /config":        ap.configHandlerPUT,
+		"POST   /config":        ap.configHandlerPOST,
 		"POST   /hosts":         ap.hostsHandlerPOST,
 		"GET    /host/:hostKey": ap.hostHandlerGET,
 		"GET    /state":         ap.stateHandlerGET,
 		"POST   /trigger":       ap.triggerHandlerPOST,
 	})
 }
 
+func (ap *Autopilot) configHandlerPOST(jc jape.Context) {
+	ctx := jc.Request.Context()
+
+	// decode request
+	var req api.ConfigEvaluationRequest
+	if jc.Decode(&req) != nil {
+		return
+	}
+
+	// fetch necessary information
+	cfg := req.AutopilotConfig
+	gs := req.GougingSettings
+	rs := req.RedundancySettings
+	cs, err := ap.bus.ConsensusState(ctx)
+	if jc.Check("failed to get consensus state", err) != nil {
+		return
+	}
+	state := ap.State()
+
+	// fetch hosts
+	hosts, err := ap.bus.Hosts(ctx, api.GetHostsOptions{})
+	if jc.Check("failed to get hosts", err) != nil {
+		return
+	}
+
+	// evaluate the config
+	jc.Encode(evaluateConfig(cfg, cs, state.fee, state.period, rs, gs, hosts))
+}
+
 func (ap *Autopilot) Run() error {
 	ap.startStopMu.Lock()
 	if ap.isRunning() {
@@ -702,3 +733,172 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) {
 	}
 	jc.Encode(hosts)
 }
+
+func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.Host) (usables uint64) {
+	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
+	for _, host := range hosts {
+		usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0)
+		if usable {
+			usables++
+		}
+	}
+	return
+}
+
+// evaluateConfig evaluates the given configuration and if the gouging settings
+// are too strict for the number of contracts required by 'cfg', it will provide
+// a recommendation on how to loosen it.
+func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.Host) (resp api.ConfigEvaluationResponse) {
+	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
+
+	resp.Hosts = uint64(len(hosts))
+	for _, host := range hosts {
+		usable, usableBreakdown := isUsableHost(cfg, rs, gc, host, 0, 0)
+		if usable {
+			resp.Usable++
+			continue
+		}
+		if usableBreakdown.blocked > 0 {
+			resp.Unusable.Blocked++
+		}
+		if usableBreakdown.notacceptingcontracts > 0 {
+			resp.Unusable.NotAcceptingContracts++
+		}
+		if usableBreakdown.notcompletingscan > 0 {
+			resp.Unusable.NotScanned++
+		}
+		if usableBreakdown.unknown > 0 {
+			resp.Unusable.Unknown++
+		}
+		if usableBreakdown.gougingBreakdown.ContractErr != "" {
+			resp.Unusable.Gouging.Contract++
+		}
+		if usableBreakdown.gougingBreakdown.DownloadErr != "" {
+			resp.Unusable.Gouging.Download++
+		}
+		if usableBreakdown.gougingBreakdown.GougingErr != "" {
+			resp.Unusable.Gouging.Gouging++
+		}
+		if usableBreakdown.gougingBreakdown.PruneErr != "" {
+			resp.Unusable.Gouging.Pruning++
+		}
+		if usableBreakdown.gougingBreakdown.UploadErr != "" {
+			resp.Unusable.Gouging.Upload++
+		}
+	}
+
+	if resp.Usable >= cfg.Contracts.Amount {
+		return // no recommendation needed
+	}
+
+	// optimise gouging settings
+	maxGS := func() api.GougingSettings {
+		return api.GougingSettings{
+			// these are the fields we optimise one-by-one
+			MaxRPCPrice:      types.MaxCurrency,
+			MaxContractPrice: types.MaxCurrency,
+			MaxDownloadPrice: types.MaxCurrency,
+			MaxUploadPrice:   types.MaxCurrency,
+			MaxStoragePrice:  types.MaxCurrency,
+
+			// these are not optimised, so we keep the same values as the user
+			// provided
+			MinMaxCollateral:              gs.MinMaxCollateral,
+			HostBlockHeightLeeway:         gs.HostBlockHeightLeeway,
+			MinPriceTableValidity:         gs.MinPriceTableValidity,
+			MinAccountExpiry:              gs.MinAccountExpiry,
+			MinMaxEphemeralAccountBalance: gs.MinMaxEphemeralAccountBalance,
+			MigrationSurchargeMultiplier:  gs.MigrationSurchargeMultiplier,
+		}
+	}
+
+	// use the input gouging settings as the starting point and try to optimise
+	// each field independent of the other fields we want to optimise
+	optimisedGS := gs
+	success := false
+
+	// MaxRPCPrice
+	tmpGS := maxGS()
+	tmpGS.MaxRPCPrice = gs.MaxRPCPrice
+	if optimiseGougingSetting(&tmpGS, &tmpGS.MaxRPCPrice, cfg, cs, fee, currentPeriod, rs, hosts) {
+		optimisedGS.MaxRPCPrice = tmpGS.MaxRPCPrice
+		success = true
+	}
+	// MaxContractPrice
+	tmpGS = maxGS()
+	tmpGS.MaxContractPrice = gs.MaxContractPrice
+	if optimiseGougingSetting(&tmpGS, &tmpGS.MaxContractPrice, cfg, cs, fee, currentPeriod, rs, hosts) {
+		optimisedGS.MaxContractPrice = tmpGS.MaxContractPrice
+		success = true
+	}
+	// MaxDownloadPrice
+	tmpGS = maxGS()
+	tmpGS.MaxDownloadPrice = gs.MaxDownloadPrice
+	if optimiseGougingSetting(&tmpGS, &tmpGS.MaxDownloadPrice, cfg, cs, fee, currentPeriod, rs, hosts) {
+		optimisedGS.MaxDownloadPrice = tmpGS.MaxDownloadPrice
+		success = true
+	}
+	// MaxUploadPrice
+	tmpGS = maxGS()
+	tmpGS.MaxUploadPrice = gs.MaxUploadPrice
+	if optimiseGougingSetting(&tmpGS, &tmpGS.MaxUploadPrice, cfg, cs, fee, currentPeriod, rs, hosts) {
+		optimisedGS.MaxUploadPrice = tmpGS.MaxUploadPrice
+		success = true
+	}
+	// MaxStoragePrice
+	tmpGS = maxGS()
+	tmpGS.MaxStoragePrice = gs.MaxStoragePrice
+	if optimiseGougingSetting(&tmpGS, &tmpGS.MaxStoragePrice, cfg, cs, fee, currentPeriod, rs, hosts) {
+		optimisedGS.MaxStoragePrice = tmpGS.MaxStoragePrice
+		success = true
+	}
+	// If one of the optimisations was successful, we return the optimised
+	// gouging settings
+	if success {
+		resp.Recommendation = &api.ConfigRecommendation{
+			GougingSettings: optimisedGS,
+		}
+	}
+	return
+}
+
+// optimiseGougingSetting tries to optimise one field of the gouging settings to
+// try and hit the target number of contracts.
+func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []hostdb.Host) bool {
+	if cfg.Contracts.Amount == 0 {
+		return true // nothing to do
+	}
+	stepSize := []uint64{200, 150, 125, 110, 105}
+	maxSteps := 12
+
+	stepIdx := 0
+	nSteps := 0
+	prevVal := *field // to keep accurate value
+	for {
+		nUsable := countUsableHosts(cfg, cs, fee, currentPeriod, rs, *gs, hosts)
+		targetHit := nUsable >= cfg.Contracts.Amount
+
+		if targetHit && nSteps == 0 {
+			return true // target already hit without optimising
+		} else if targetHit && stepIdx == len(stepSize)-1 {
+			return true // target hit after optimising
+		} else if targetHit {
+			// move one step back and decrease step size
+			stepIdx++
+			nSteps--
+			*field = prevVal
+		} else if nSteps >= maxSteps {
+			return false // ran out of steps
+		}
+
+		// apply next step
+		prevVal = *field
+		newValue, overflow := prevVal.Mul64WithOverflow(stepSize[stepIdx])
+		if overflow {
+			return false
+		}
+		newValue = newValue.Div64(100)
+		*field = newValue
+		nSteps++
+	}
+}