Merge branch 'dev' of github.com:SiaFoundation/renterd into its-happe…

…ning

.github/workflows/test.yml

@@ -17,13 +17,6 @@ jobs:
       - name: Configure Windows
         if: matrix.os == 'windows-latest'
         run: git config --global core.autocrlf false # fixes go lint fmt error
-      - name: Configure MySQL
-        if: matrix.os == 'ubuntu-latest'
-        uses: mirromutth/[email protected]
-        with:
-          host port: 3800
-          mysql version: '8'
-          mysql root password: test
       - name: Checkout
         uses: actions/checkout@v3
       - name: Setup Go
@@ -43,6 +36,13 @@ jobs:
             autopilot
             bus bus/client
             worker worker/client
+      - name: Configure MySQL
+        if: matrix.os == 'ubuntu-latest'
+        uses: mirromutth/[email protected]
+        with:
+          host port: 3800
+          mysql version: '8'
+          mysql root password: test
       - name: Test
         uses: n8maninger/action-golang-test@v1
         with:

.golangci.yml

@@ -17,13 +17,6 @@ run:
   # list of build tags, all linters use it. Default is empty list.
   build-tags: []
 
-  # which dirs to skip: issues from them won't be reported;
-  # can use regexp here: generated.*, regexp is applied on full path;
-  # default value is empty list, but default dirs are skipped independently
-  # from this option's value (see skip-dirs-use-default).
-  skip-dirs:
-    - cover
-
   # default is true. Enables skipping of directories:
   #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
   skip-dirs-use-default: true
@@ -36,9 +29,6 @@ run:
 
 # output configuration options
 output:
-  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
-  format: colored-line-number
-
   # print lines of code with issue, default is true
   print-issued-lines: true
 
@@ -61,23 +51,28 @@ linters-settings:
     # See https://go-critic.github.io/overview#checks-overview
     # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
     # By default list of stable checks is used.
-    enabled-checks:
-      - argOrder # Diagnostic options
-      - badCond
-      - caseOrder
-      - dupArg
-      - dupBranchBody
-      - dupCase
-      - dupSubExpr
-      - nilValReturn
-      - offBy1
-      - weakCond
-      - boolExprSimplify # Style options here and below.
-      - builtinShadow
-      - emptyFallthrough
-      - hexLiteral
-      - underef
-      - equalFold
+    enabled-tags:
+      - diagnostic
+      - style
+    disabled-checks:
+      # diagnostic
+      - commentedOutCode
+      - uncheckedInlineErr
+
+      # style
+      - exitAfterDefer
+      - ifElseChain
+      - importShadow
+      - octalLiteral
+      - paramTypeCombine
+      - ptrToRefParam
+      - stringsCompare
+      - tooManyResultsChecker
+      - typeDefFirst
+      - typeUnparen
+      - unlabelStmt
+      - unnamedResult
+      - whyNoLint
   revive:
     ignore-generated-header: true
     rules:

README.md

@@ -250,7 +250,6 @@ updated using the settings API:
 	"maxUploadPrice": "3000000000000000000000000000",             // 3000 SC per 1 TiB
 	"migrationSurchargeMultiplier": 10,                           // overpay up to 10x for sectors migrations on critical slabs
 	"minAccountExpiry": 86400000000000,                           // 1 day
-	"minMaxCollateral": "10000000000000000000000000",             // at least up to 10 SC per contract
 	"minMaxEphemeralAccountBalance": "1000000000000000000000000", // 1 SC
 	"minPriceTableValidity": 300000000000                         // 5 minutes
 }

api/host.go

@@ -79,6 +79,7 @@ type (
 		Limit       int
 		Offset      int
 	}
+
 	SearchHostOptions struct {
 		AddressContains string
 		FilterMode      string
@@ -88,13 +89,6 @@ type (
 	}
 )
 
-func DefaultSearchHostOptions() SearchHostOptions {
-	return SearchHostOptions{
-		Limit:      -1,
-		FilterMode: HostFilterModeAll,
-	}
-}
-
 func (opts GetHostsOptions) Apply(values url.Values) {
 	if opts.Offset != 0 {
 		values.Set("offset", fmt.Sprint(opts.Offset))
@@ -112,6 +106,6 @@ func (opts HostsForScanningOptions) Apply(values url.Values) {
 		values.Set("limit", fmt.Sprint(opts.Limit))
 	}
 	if !opts.MaxLastScan.IsZero() {
-		values.Set("lastScan", fmt.Sprint(TimeRFC3339(opts.MaxLastScan)))
+		values.Set("lastScan", TimeRFC3339(opts.MaxLastScan).String())
 	}
 }

api/object.go

@@ -367,8 +367,8 @@ func (opts SearchObjectOptions) Apply(values url.Values) {
 	}
 }
 
-func FormatETag(ETag string) string {
-	return fmt.Sprintf("\"%s\"", ETag)
+func FormatETag(eTag string) string {
+	return fmt.Sprintf("%q", eTag)
 }
 
 func ObjectPathEscape(path string) string {

api/param.go

@@ -105,7 +105,7 @@ func (t *TimeRFC3339) UnmarshalText(b []byte) error {
 
 // MarshalJSON implements json.Marshaler.
 func (t TimeRFC3339) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf(`"%s"`, (time.Time)(t).UTC().Format(time.RFC3339Nano))), nil
+	return []byte(fmt.Sprintf("%q", (time.Time)(t).UTC().Format(time.RFC3339Nano))), nil
 }
 
 // String implements fmt.Stringer.

api/setting.go

@@ -38,10 +38,6 @@ type (
 
 	// GougingSettings contain some price settings used in price gouging.
 	GougingSettings struct {
-		// MinMaxCollateral is the minimum value for 'MaxCollateral' in the host's
-		// price settings
-		MinMaxCollateral types.Currency `json:"minMaxCollateral"`
-
 		// MaxRPCPrice is the maximum allowed base price for RPCs
 		MaxRPCPrice types.Currency `json:"maxRPCPrice"`
 
@@ -155,11 +151,11 @@ func (rs RedundancySettings) Validate() error {
 // valid.
 func (s3as S3AuthenticationSettings) Validate() error {
 	for accessKeyID, secretAccessKey := range s3as.V4Keypairs {
-		if len(accessKeyID) == 0 {
+		if accessKeyID == "" {
 			return fmt.Errorf("AccessKeyID cannot be empty")
 		} else if len(accessKeyID) < S3MinAccessKeyLen || len(accessKeyID) > S3MaxAccessKeyLen {
 			return fmt.Errorf("AccessKeyID must be between %d and %d characters long but was %d", S3MinAccessKeyLen, S3MaxAccessKeyLen, len(accessKeyID))
-		} else if len(secretAccessKey) == 0 {
+		} else if secretAccessKey == "" {
 			return fmt.Errorf("SecretAccessKey cannot be empty")
 		} else if len(secretAccessKey) != S3SecretKeyLen {
 			return fmt.Errorf("SecretAccessKey must be %d characters long but was %d", S3SecretKeyLen, len(secretAccessKey))

autopilot/alerts.go

@@ -194,22 +194,37 @@ func newCriticalMigrationSucceededAlert(slabKey object.EncryptionKey) alerts.Ale
 	}
 }
 
-func newCriticalMigrationFailedAlert(slabKey object.EncryptionKey, health float64, err error) alerts.Alert {
+func newCriticalMigrationFailedAlert(slabKey object.EncryptionKey, health float64, objectIds map[string][]string, err error) alerts.Alert {
+	data := map[string]interface{}{
+		"error":   err.Error(),
+		"health":  health,
+		"slabKey": slabKey.String(),
+		"hint":    "If migrations of low-health slabs fail, it might be necessary to increase the MigrationSurchargeMultiplier in the gouging settings to ensure it has every chance of succeeding.",
+	}
+	if objectIds != nil {
+		data["objectIDs"] = objectIds
+	}
+
 	return alerts.Alert{
-		ID:       alertIDForSlab(alertMigrationID, slabKey),
-		Severity: alerts.SeverityCritical,
-		Message:  "Critical migration failed",
-		Data: map[string]interface{}{
-			"error":   err.Error(),
-			"health":  health,
-			"slabKey": slabKey.String(),
-			"hint":    "If migrations of low-health slabs fail, it might be necessary to increase the MigrationSurchargeMultiplier in the gouging settings to ensure it has every chance of succeeding.",
-		},
+		ID:        alertIDForSlab(alertMigrationID, slabKey),
+		Severity:  alerts.SeverityCritical,
+		Message:   "Critical migration failed",
+		Data:      data,
 		Timestamp: time.Now(),
 	}
 }
 
-func newMigrationFailedAlert(slabKey object.EncryptionKey, health float64, err error) alerts.Alert {
+func newMigrationFailedAlert(slabKey object.EncryptionKey, health float64, objectIds map[string][]string, err error) alerts.Alert {
+	data := map[string]interface{}{
+		"error":   err.Error(),
+		"health":  health,
+		"slabKey": slabKey.String(),
+		"hint":    "Migration failures can be temporary, but if they persist it can eventually lead to data loss and should therefor be taken very seriously.",
+	}
+	if objectIds != nil {
+		data["objectIDs"] = objectIds
+	}
+
 	severity := alerts.SeverityError
 	if health < 0.25 {
 		severity = alerts.SeverityCritical
@@ -218,15 +233,10 @@ func newMigrationFailedAlert(slabKey object.EncryptionKey, health float64, err e
 	}
 
 	return alerts.Alert{
-		ID:       alertIDForSlab(alertMigrationID, slabKey),
-		Severity: severity,
-		Message:  "Slab migration failed",
-		Data: map[string]interface{}{
-			"error":   err.Error(),
-			"health":  health,
-			"slabKey": slabKey.String(),
-			"hint":    "Migration failures can be temporary, but if they persist it can eventually lead to data loss and should therefor be taken very seriously.",
-		},
+		ID:        alertIDForSlab(alertMigrationID, slabKey),
+		Severity:  severity,
+		Message:   "Slab migration failed",
+		Data:      data,
 		Timestamp: time.Now(),
 	}
 }

autopilot/autopilot.go

@@ -53,15 +53,17 @@ type Bus interface {
 
 	// hostdb
 	Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error)
-	Hosts(ctx context.Context, opts api.GetHostsOptions) ([]hostdb.Host, error)
 	HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error)
 	RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error)
-	SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.Host, error)
+	SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error)
 
 	// metrics
 	RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error
 	RecordContractPruneMetric(ctx context.Context, metrics ...api.ContractPruneMetric) error
 
+	// buckets
+	ListBuckets(ctx context.Context) ([]api.Bucket, error)
+
 	// objects
 	ObjectsBySlabKey(ctx context.Context, bucket string, key object.EncryptionKey) (objects []api.ObjectMetadata, err error)
 	RefreshHealth(ctx context.Context) error
@@ -195,7 +197,7 @@ func (ap *Autopilot) configHandlerPOST(jc jape.Context) {
 	state := ap.State()
 
 	// fetch hosts
-	hosts, err := ap.bus.Hosts(ctx, api.GetHostsOptions{})
+	hosts, err := ap.bus.SearchHosts(ctx, api.SearchHostOptions{Limit: -1, FilterMode: api.HostFilterModeAllowed})
 	if jc.Check("failed to get hosts", err) != nil {
 		return
 	}
@@ -732,7 +734,7 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) {
 	jc.Encode(hosts)
 }
 
-func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.Host) (usables uint64) {
+func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (usables uint64) {
 	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
 	for _, host := range hosts {
 		usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0)
@@ -746,7 +748,7 @@ func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.
 // evaluateConfig evaluates the given configuration and if the gouging settings
 // are too strict for the number of contracts required by 'cfg', it will provide
 // a recommendation on how to loosen it.
-func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.Host) (resp api.ConfigEvaluationResponse) {
+func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (resp api.ConfigEvaluationResponse) {
 	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
 
 	resp.Hosts = uint64(len(hosts))
@@ -801,7 +803,6 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu
 
 			// these are not optimised, so we keep the same values as the user
 			// provided
-			MinMaxCollateral:              gs.MinMaxCollateral,
 			HostBlockHeightLeeway:         gs.HostBlockHeightLeeway,
 			MinPriceTableValidity:         gs.MinPriceTableValidity,
 			MinAccountExpiry:              gs.MinAccountExpiry,
@@ -862,7 +863,7 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu
 
 // optimiseGougingSetting tries to optimise one field of the gouging settings to
 // try and hit the target number of contracts.
-func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []hostdb.Host) bool {
+func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []hostdb.HostInfo) bool {
 	if cfg.Contracts.Amount == 0 {
 		return true // nothing to do
 	}

autopilot/autopilot_test.go

@@ -14,31 +14,34 @@ import (
 
 func TestOptimiseGougingSetting(t *testing.T) {
 	// create 10 hosts that should all be usable
-	var hosts []hostdb.Host
+	var hosts []hostdb.HostInfo
 	for i := 0; i < 10; i++ {
-		hosts = append(hosts, hostdb.Host{
-			KnownSince: time.Unix(0, 0),
-			PriceTable: hostdb.HostPriceTable{
-				HostPriceTable: rhpv3.HostPriceTable{
-					CollateralCost: types.Siacoins(1),
-					MaxCollateral:  types.Siacoins(1000),
+		hosts = append(hosts, hostdb.HostInfo{
+			Host: hostdb.Host{
+				KnownSince: time.Unix(0, 0),
+				PriceTable: hostdb.HostPriceTable{
+					HostPriceTable: rhpv3.HostPriceTable{
+						CollateralCost: types.Siacoins(1),
+						MaxCollateral:  types.Siacoins(1000),
+					},
 				},
+				Settings: rhpv2.HostSettings{
+					AcceptingContracts: true,
+					Collateral:         types.Siacoins(1),
+					MaxCollateral:      types.Siacoins(1000),
+					Version:            "1.6.0",
+				},
+				Interactions: hostdb.Interactions{
+					Uptime:                  time.Hour * 1000,
+					LastScan:                time.Now(),
+					LastScanSuccess:         true,
+					SecondToLastScanSuccess: true,
+					TotalScans:              100,
+				},
+				LastAnnouncement: time.Unix(0, 0),
+				Scanned:          true,
 			},
-			Settings: rhpv2.HostSettings{
-				AcceptingContracts: true,
-				Collateral:         types.Siacoins(1),
-				MaxCollateral:      types.Siacoins(1000),
-				Version:            "1.6.0",
-			},
-			Interactions: hostdb.Interactions{
-				Uptime:                  time.Hour * 1000,
-				LastScan:                time.Now(),
-				LastScanSuccess:         true,
-				SecondToLastScanSuccess: true,
-				TotalScans:              100,
-			},
-			LastAnnouncement: time.Unix(0, 0),
-			Scanned:          true,
+			Blocked: false,
 		})
 	}