Merge branch 'dev' of github.com:SiaFoundation/renterd into its-happe…

…ning

api/host.go

@@ -158,6 +158,7 @@ type (
 		Scanned          bool                 `json:"scanned"`
 		Blocked          bool                 `json:"blocked"`
 		Checks           map[string]HostCheck `json:"checks"`
+		StoredData       uint64               `json:"storedData"`
 	}
 
 	HostAddress struct {

api/multipart.go

@@ -7,6 +7,11 @@ import (
 )
 
 var (
+	// ErrInvalidMultipartEncryptionSettings is returned if the multipart upload
+	// has an invalid combination of encryption params. e.g. when encryption is
+	// enabled but not offset is set.
+	ErrInvalidMultipartEncryptionSettings = errors.New("invalid multipart encryption settings")
+
 	// ErrMultipartUploadNotFound is returned if the specified multipart upload
 	// wasn't found.
 	ErrMultipartUploadNotFound = errors.New("multipart upload not found")

api/object.go

@@ -91,12 +91,12 @@ type (
 
 	// HeadObjectResponse is the response type for the HEAD /worker/object endpoint.
 	HeadObjectResponse struct {
-		ContentType  string             `json:"contentType"`
-		Etag         string             `json:"eTag"`
-		LastModified string             `json:"lastModified"`
-		Range        *DownloadRange     `json:"range,omitempty"`
-		Size         int64              `json:"size"`
-		Metadata     ObjectUserMetadata `json:"metadata"`
+		ContentType  string
+		Etag         string
+		LastModified TimeRFC3339
+		Range        *ContentRange
+		Size         int64
+		Metadata     ObjectUserMetadata
 	}
 
 	// ObjectsDeleteRequest is the request type for the /bus/objects/list endpoint.
@@ -151,12 +151,6 @@ func ExtractObjectUserMetadataFrom(metadata map[string]string) ObjectUserMetadat
 	return oum
 }
 
-// LastModified returns the object's ModTime formatted for use in the
-// 'Last-Modified' header
-func (o ObjectMetadata) LastModified() string {
-	return o.ModTime.Std().Format(http.TimeFormat)
-}
-
 // ContentType returns the object's MimeType for use in the 'Content-Type'
 // header, if the object's mime type is empty we try and deduce it from the
 // extension in the object's name.
@@ -214,12 +208,12 @@ type (
 
 	HeadObjectOptions struct {
 		IgnoreDelim bool
-		Range       DownloadRange
+		Range       *DownloadRange
 	}
 
 	DownloadObjectOptions struct {
 		GetObjectOptions
-		Range DownloadRange
+		Range *DownloadRange
 	}
 
 	GetObjectOptions struct {
@@ -247,7 +241,6 @@ type (
 
 	// UploadObjectOptions is the options type for the worker client.
 	UploadObjectOptions struct {
-		Offset        int
 		MinShards     int
 		TotalShards   int
 		ContractSet   string
@@ -257,15 +250,15 @@ type (
 	}
 
 	UploadMultipartUploadPartOptions struct {
+		ContractSet      string
+		MinShards        int
+		TotalShards      int
 		EncryptionOffset *int
 		ContentLength    int64
 	}
 )
 
 func (opts UploadObjectOptions) ApplyValues(values url.Values) {
-	if opts.Offset != 0 {
-		values.Set("offset", fmt.Sprint(opts.Offset))
-	}
 	if opts.MinShards != 0 {
 		values.Set("minshards", fmt.Sprint(opts.MinShards))
 	}
@@ -290,14 +283,23 @@ func (opts UploadMultipartUploadPartOptions) Apply(values url.Values) {
 	if opts.EncryptionOffset != nil {
 		values.Set("offset", fmt.Sprint(*opts.EncryptionOffset))
 	}
+	if opts.MinShards != 0 {
+		values.Set("minshards", fmt.Sprint(opts.MinShards))
+	}
+	if opts.TotalShards != 0 {
+		values.Set("totalshards", fmt.Sprint(opts.TotalShards))
+	}
+	if opts.ContractSet != "" {
+		values.Set("contractset", opts.ContractSet)
+	}
 }
 
 func (opts DownloadObjectOptions) ApplyValues(values url.Values) {
 	opts.GetObjectOptions.Apply(values)
 }
 
 func (opts DownloadObjectOptions) ApplyHeaders(h http.Header) {
-	if opts.Range != (DownloadRange{}) {
+	if opts.Range != nil {
 		if opts.Range.Length == -1 {
 			h.Set("Range", fmt.Sprintf("bytes=%v-", opts.Range.Offset))
 		} else {
@@ -319,7 +321,7 @@ func (opts HeadObjectOptions) Apply(values url.Values) {
 }
 
 func (opts HeadObjectOptions) ApplyHeaders(h http.Header) {
-	if opts.Range != (DownloadRange{}) {
+	if opts.Range != nil {
 		if opts.Range.Length == -1 {
 			h.Set("Range", fmt.Sprintf("bytes=%v-", opts.Range.Offset))
 		} else {

api/setting.go

@@ -24,6 +24,10 @@ const (
 )
 
 var (
+	// ErrInvalidRedundancySettings is returned if the redundancy settings are
+	// not valid
+	ErrInvalidRedundancySettings = errors.New("invalid redundancy settings")
+
 	// ErrSettingNotFound is returned if a requested setting is not present in the
 	// database.
 	ErrSettingNotFound = errors.New("setting not found")
@@ -136,13 +140,13 @@ func (rs RedundancySettings) SlabSizeNoRedundancy() uint64 {
 // valid.
 func (rs RedundancySettings) Validate() error {
 	if rs.MinShards < 1 {
-		return errors.New("MinShards must be greater than 0")
+		return fmt.Errorf("%w: MinShards must be greater than 0", ErrInvalidRedundancySettings)
 	}
 	if rs.TotalShards < rs.MinShards {
-		return errors.New("TotalShards must be at least MinShards")
+		return fmt.Errorf("%w: TotalShards must be at least MinShards", ErrInvalidRedundancySettings)
 	}
 	if rs.TotalShards > 255 {
-		return errors.New("TotalShards must be less than 256")
+		return fmt.Errorf("%w: TotalShards must be less than 256", ErrInvalidRedundancySettings)
 	}
 	return nil
 }

api/worker.go

@@ -3,9 +3,12 @@ package api
 import (
 	"errors"
 	"fmt"
+	"math"
+	"net/http"
 	"strconv"
 	"strings"
 
+	"github.com/gotd/contrib/http_range"
 	rhpv2 "go.sia.tech/core/rhp/v2"
 	rhpv3 "go.sia.tech/core/rhp/v3"
 	"go.sia.tech/core/types"
@@ -23,6 +26,10 @@ var (
 	// ErrHostOnPrivateNetwork is returned by the worker API when a host can't
 	// be scanned since it is on a private network.
 	ErrHostOnPrivateNetwork = errors.New("host is on a private network")
+
+	// ErrMultiRangeNotSupported is returned by the worker API when a request
+	// tries to download multiple ranges at once.
+	ErrMultiRangeNotSupported = errors.New("multipart ranges are not supported")
 )
 
 type (
@@ -216,41 +223,76 @@ type (
 	}
 )
 
-type DownloadRange struct {
+// ContentRange represents a content range returned via the "Content-Range"
+// header.
+type ContentRange struct {
 	Offset int64
 	Length int64
 	Size   int64
 }
 
-func ParseDownloadRange(contentRange string) (DownloadRange, error) {
+// DownloadRange represents a requested range for a download via the "Range"
+// header.
+type DownloadRange struct {
+	Offset int64
+	Length int64
+}
+
+func (r *DownloadRange) ContentRange(size int64) *ContentRange {
+	return &ContentRange{
+		Offset: r.Offset,
+		Length: r.Length,
+		Size:   size,
+	}
+}
+
+func ParseContentRange(contentRange string) (ContentRange, error) {
 	parts := strings.Split(contentRange, " ")
 	if len(parts) != 2 || parts[0] != "bytes" {
-		return DownloadRange{}, errors.New("missing 'bytes' prefix in range header")
+		return ContentRange{}, errors.New("missing 'bytes' prefix in range header")
 	}
 	parts = strings.Split(parts[1], "/")
 	if len(parts) != 2 {
-		return DownloadRange{}, fmt.Errorf("invalid Content-Range header: %s", contentRange)
+		return ContentRange{}, fmt.Errorf("invalid Content-Range header: %s", contentRange)
 	}
 	rangeStr := parts[0]
 	rangeParts := strings.Split(rangeStr, "-")
 	if len(rangeParts) != 2 {
-		return DownloadRange{}, errors.New("invalid Content-Range header")
+		return ContentRange{}, errors.New("invalid Content-Range header")
 	}
 	start, err := strconv.ParseInt(rangeParts[0], 10, 64)
 	if err != nil {
-		return DownloadRange{}, err
+		return ContentRange{}, err
 	}
 	end, err := strconv.ParseInt(rangeParts[1], 10, 64)
 	if err != nil {
-		return DownloadRange{}, err
+		return ContentRange{}, err
 	}
 	size, err := strconv.ParseInt(parts[1], 10, 64)
 	if err != nil {
-		return DownloadRange{}, err
+		return ContentRange{}, err
 	}
-	return DownloadRange{
+	return ContentRange{
 		Offset: start,
 		Length: end - start + 1,
 		Size:   size,
 	}, nil
 }
+
+func ParseDownloadRange(req *http.Request) (DownloadRange, error) {
+	// parse the request range we pass math.MaxInt64 since a range header in a
+	// request doesn't have a size
+	ranges, err := http_range.ParseRange(req.Header.Get("Range"), math.MaxInt64)
+	if err != nil {
+		return DownloadRange{}, err
+	}
+
+	// extract requested offset and length
+	dr := DownloadRange{Offset: 0, Length: -1}
+	if len(ranges) == 1 {
+		dr.Offset, dr.Length = ranges[0].Start, ranges[0].Length
+	} else if len(ranges) > 1 {
+		return DownloadRange{}, ErrMultiRangeNotSupported
+	}
+	return dr, nil
+}

autopilot/contractor/contractor.go

@@ -282,12 +282,10 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro
 		usedHosts[contract.HostKey] = struct{}{}
 	}
 
-	// compile map of stored data per host
+	// compile map of stored data per contract
 	contractData := make(map[types.FileContractID]uint64)
-	hostData := make(map[types.PublicKey]uint64)
 	for _, c := range contracts {
 		contractData[c.ID] = c.FileSize()
-		hostData[c.HostKey] += c.FileSize()
 	}
 
 	// fetch all hosts
@@ -310,7 +308,7 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro
 	}
 
 	// fetch candidate hosts
-	candidates, unusableHosts, err := c.candidateHosts(mCtx, hosts, usedHosts, hostData, minValidScore) // avoid 0 score hosts
+	candidates, unusableHosts, err := c.candidateHosts(mCtx, hosts, usedHosts, minValidScore) // avoid 0 score hosts
 	if err != nil {
 		return false, err
 	}
@@ -324,7 +322,7 @@ func (c *Contractor) performContractMaintenance(ctx *mCtx, w Worker) (bool, erro
 	}
 
 	// run host checks
-	checks, err := c.runHostChecks(mCtx, hosts, hostData, minScore)
+	checks, err := c.runHostChecks(mCtx, hosts, minScore)
 	if err != nil {
 		return false, fmt.Errorf("failed to run host checks, err: %v", err)
 	}
@@ -742,7 +740,7 @@ LOOP:
 	return toKeep, toArchive, toStopUsing, toRefresh, toRenew
 }
 
-func (c *Contractor) runHostChecks(ctx *mCtx, hosts []api.Host, hostData map[types.PublicKey]uint64, minScore float64) (map[types.PublicKey]*api.HostCheck, error) {
+func (c *Contractor) runHostChecks(ctx *mCtx, hosts []api.Host, minScore float64) (map[types.PublicKey]*api.HostCheck, error) {
 	// fetch consensus state
 	cs, err := c.bus.ConsensusState(ctx)
 	if err != nil {
@@ -756,7 +754,7 @@ func (c *Contractor) runHostChecks(ctx *mCtx, hosts []api.Host, hostData map[typ
 	checks := make(map[types.PublicKey]*api.HostCheck)
 	for _, h := range hosts {
 		h.PriceTable.HostBlockHeight = cs.BlockHeight // ignore HostBlockHeight
-		checks[h.PublicKey] = checkHost(ctx.AutopilotConfig(), ctx.state.RS, gc, h, minScore, hostData[h.PublicKey])
+		checks[h.PublicKey] = checkHost(ctx.AutopilotConfig(), ctx.state.RS, gc, h, minScore)
 	}
 	return checks, nil
 }
@@ -984,7 +982,16 @@ func (c *Contractor) runContractRenewals(ctx *mCtx, w Worker, toRenew []contract
 		contract := toRenew[i].contract.ContractMetadata
 		renewed, proceed, err := c.renewContract(ctx, w, toRenew[i], budget)
 		if err != nil {
-			c.alerter.RegisterAlert(ctx, newContractRenewalFailedAlert(contract, !proceed, err))
+			// don't register an alert for hosts that are out of funds since the
+			// user can't do anything about it
+			if !(worker.IsErrHost(err) && utils.IsErr(err, wallet.ErrNotEnoughFunds)) {
+				c.alerter.RegisterAlert(ctx, newContractRenewalFailedAlert(contract, !proceed, err))
+			}
+			c.logger.With(zap.Error(err)).
+				With("fcid", toRenew[i].contract.ID).
+				With("hostKey", toRenew[i].contract.HostKey).
+				With("proceed", proceed).
+				Errorw("failed to renew contract")
 			if toRenew[i].usable {
 				toKeep = append(toKeep, toRenew[i].contract.ContractMetadata)
 			}
@@ -1220,7 +1227,7 @@ func (c *Contractor) calculateMinScore(candidates []scoredHost, numContracts uin
 	return minScore
 }
 
-func (c *Contractor) candidateHosts(ctx *mCtx, hosts []api.Host, usedHosts map[types.PublicKey]struct{}, storedData map[types.PublicKey]uint64, minScore float64) ([]scoredHost, unusableHostsBreakdown, error) {
+func (c *Contractor) candidateHosts(ctx *mCtx, hosts []api.Host, usedHosts map[types.PublicKey]struct{}, minScore float64) ([]scoredHost, unusableHostsBreakdown, error) {
 	start := time.Now()
 
 	// fetch consensus state
@@ -1273,7 +1280,7 @@ func (c *Contractor) candidateHosts(ctx *mCtx, hosts []api.Host, usedHosts map[t
 		// NOTE: ignore the pricetable's HostBlockHeight by setting it to our
 		// own blockheight
 		h.PriceTable.HostBlockHeight = cs.BlockHeight
-		hc := checkHost(ctx.AutopilotConfig(), ctx.state.RS, gc, h, minScore, storedData[h.PublicKey])
+		hc := checkHost(ctx.AutopilotConfig(), ctx.state.RS, gc, h, minScore)
 		if hc.Usability.IsUsable() {
 			candidates = append(candidates, scoredHost{h, hc.Score.Score()})
 			continue

autopilot/contractor/evaluate.go

@@ -9,7 +9,7 @@ import (
 func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) {
 	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
 	for _, host := range hosts {
-		hc := checkHost(cfg, rs, gc, host, minValidScore, 0)
+		hc := checkHost(cfg, rs, gc, host, minValidScore)
 		if hc.Usability.IsUsable() {
 			usables++
 		}
@@ -25,7 +25,7 @@ func EvaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu
 
 	resp.Hosts = uint64(len(hosts))
 	for _, host := range hosts {
-		hc := checkHost(cfg, rs, gc, host, 0, 0)
+		hc := checkHost(cfg, rs, gc, host, 0)
 		if hc.Usability.IsUsable() {
 			resp.Usable++
 			continue

autopilot/contractor/hostfilter.go

@@ -236,7 +236,7 @@ func isUpForRenewal(cfg api.AutopilotConfig, r types.FileContractRevision, block
 }
 
 // checkHost performs a series of checks on the host.
-func checkHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h api.Host, minScore float64, storedData uint64) *api.HostCheck {
+func checkHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.GougingChecker, h api.Host, minScore float64) *api.HostCheck {
 	if rs.Validate() != nil {
 		panic("invalid redundancy settings were supplied - developer error")
 	}
@@ -278,7 +278,7 @@ func checkHost(cfg api.AutopilotConfig, rs api.RedundancySettings, gc worker.Gou
 			// not gouging, this because the core package does not have overflow
 			// checks in its cost calculations needed to calculate the period
 			// cost
-			sb = hostScore(cfg, h, storedData, rs.Redundancy())
+			sb = hostScore(cfg, h, rs.Redundancy())
 			if sb.Score() < minScore {
 				ub.LowScore = true
 			}