Merge branch 'dev' into chris/improve-logging

.github/workflows/project-add.yml

@@ -10,12 +10,6 @@ on:
 
 jobs:
   add-to-project:
-    name: Add issue to project
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/[email protected]
-        with:
-          # You can target a project in a different organization
-          # to the issue
-          project-url: https://github.com/orgs/SiaFoundation/projects/5
-          github-token: ${{ secrets.PAT_ADD_TO_PROJECT }}
+    uses: SiaFoundation/workflows/.github/workflows/project-add.yml@master
+    secrets: inherit
+

.github/workflows/publish.yml

@@ -1,13 +1,13 @@
 name: Publish
 
-# Controls when the action will run. 
+# Controls when the action will run.
 on:
   # Triggers the workflow on new SemVer tags
   push:
     branches:
       - master
       - dev
-    tags: 
+    tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+-**'
 
@@ -116,7 +116,7 @@ jobs:
         with:
           name: renterd
           path: release/
-  build-mac: 
+  build-mac:
     runs-on: macos-latest
     strategy:
       matrix:
@@ -212,7 +212,7 @@ jobs:
         with:
           name: renterd
           path: release/
-  build-windows: 
+  build-windows:
     runs-on: windows-latest
     strategy:
       matrix:
@@ -253,28 +253,62 @@ jobs:
         with:
           name: renterd
           path: release/
-  dispatch:
+
+  dispatch-homebrew: # only runs on full releases
     if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
     needs: [docker, build-linux, build-mac, build-windows]
-    strategy:
-        matrix:
-            repo: ['siafoundation/homebrew-sia', 'siafoundation/linux']
     runs-on: ubuntu-latest
     steps:
       - name: Extract Tag Name
         id: get_tag
         run: echo "tag_name=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
 
-      - name: Repository Dispatch
+      - name: Dispatch
         uses: peter-evans/repository-dispatch@v3
         with:
           token: ${{ secrets.PAT_REPOSITORY_DISPATCH }}
-          repository: ${{ matrix.repo }}
+          repository: siafoundation/homebrew-sia
           event-type: release-tagged
           client-payload: >
             {
               "description": "Renterd: The Next-Gen Sia Renter",
               "tag": "${{ env.tag_name }}",
               "project": "renterd",
               "workflow_id": "${{ github.run_id }}"
-            }
+            }
+  dispatch-linux: # run on full releases, release candidates, and master branch
+    if: startsWith(github.ref, 'refs/tags/v') || endsWith(github.ref, 'master')
+    needs: [docker, build-linux, build-mac, build-windows]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Build Dispatch Payload
+        id: get_payload
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const isRelease = context.ref.startsWith('refs/tags/v'),
+              isBeta = isRelease && context.ref.includes('-beta'),
+              tag = isRelease ? context.ref.replace('refs/tags/', '') : 'master';
+
+            let component = 'nightly';
+            if (isBeta) {
+              component = 'beta';
+            } else if (isRelease) {
+              component = 'main';
+            }
+
+            return {
+              description: "renterd: The Next-Gen Sia Renter",
+              tag: tag,
+              project: "renterd",
+              workflow_id: context.runId,
+              component: component
+            };
+
+      - name: Dispatch
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.PAT_REPOSITORY_DISPATCH }}
+          repository: siafoundation/linux
+          event-type: release-tagged
+          client-payload: ${{ steps.get_payload.outputs.result }}

api/autopilot.go

@@ -2,8 +2,6 @@ package api
 
 import (
 	"errors"
-	"fmt"
-	"strings"
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/hostdb"
@@ -136,65 +134,8 @@ type (
 		Usable           bool                 `json:"usable"`
 		UnusableReasons  []string             `json:"unusableReasons"`
 	}
-
-	HostGougingBreakdown struct {
-		ContractErr string `json:"contractErr"`
-		DownloadErr string `json:"downloadErr"`
-		GougingErr  string `json:"gougingErr"`
-		PruneErr    string `json:"pruneErr"`
-		UploadErr   string `json:"uploadErr"`
-	}
-
-	HostScoreBreakdown struct {
-		Age              float64 `json:"age"`
-		Collateral       float64 `json:"collateral"`
-		Interactions     float64 `json:"interactions"`
-		StorageRemaining float64 `json:"storageRemaining"`
-		Uptime           float64 `json:"uptime"`
-		Version          float64 `json:"version"`
-		Prices           float64 `json:"prices"`
-	}
 )
 
-func (sb HostScoreBreakdown) String() string {
-	return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices)
-}
-
-func (hgb HostGougingBreakdown) Gouging() bool {
-	for _, err := range []string{
-		hgb.ContractErr,
-		hgb.DownloadErr,
-		hgb.GougingErr,
-		hgb.PruneErr,
-		hgb.UploadErr,
-	} {
-		if err != "" {
-			return true
-		}
-	}
-	return false
-}
-
-func (hgb HostGougingBreakdown) String() string {
-	var reasons []string
-	for _, errStr := range []string{
-		hgb.ContractErr,
-		hgb.DownloadErr,
-		hgb.GougingErr,
-		hgb.PruneErr,
-		hgb.UploadErr,
-	} {
-		if errStr != "" {
-			reasons = append(reasons, errStr)
-		}
-	}
-	return strings.Join(reasons, ";")
-}
-
-func (sb HostScoreBreakdown) Score() float64 {
-	return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices
-}
-
 func (c AutopilotConfig) Validate() error {
 	if c.Hosts.MaxDowntimeHours > 99*365*24 {
 		return ErrMaxDowntimeHoursTooHigh

api/host.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"strings"
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/hostdb"
@@ -42,6 +43,8 @@ type (
 		MinRecentScanFailures uint64    `json:"minRecentScanFailures"`
 	}
 
+	// SearchHostsRequest is the request type for the /api/bus/search/hosts
+	// endpoint.
 	SearchHostsRequest struct {
 		Offset          int               `json:"offset"`
 		Limit           int               `json:"limit"`
@@ -109,3 +112,85 @@ func (opts HostsForScanningOptions) Apply(values url.Values) {
 		values.Set("lastScan", TimeRFC3339(opts.MaxLastScan).String())
 	}
 }
+
+type (
+	Host struct {
+		hostdb.Host
+		Blocked bool                 `json:"blocked"`
+		Checks  map[string]HostCheck `json:"checks"`
+	}
+
+	HostCheck struct {
+		Gouging   HostGougingBreakdown   `json:"gouging"`
+		Score     HostScoreBreakdown     `json:"score"`
+		Usability HostUsabilityBreakdown `json:"usability"`
+	}
+
+	HostGougingBreakdown struct {
+		ContractErr string `json:"contractErr"`
+		DownloadErr string `json:"downloadErr"`
+		GougingErr  string `json:"gougingErr"`
+		PruneErr    string `json:"pruneErr"`
+		UploadErr   string `json:"uploadErr"`
+	}
+
+	HostScoreBreakdown struct {
+		Age              float64 `json:"age"`
+		Collateral       float64 `json:"collateral"`
+		Interactions     float64 `json:"interactions"`
+		StorageRemaining float64 `json:"storageRemaining"`
+		Uptime           float64 `json:"uptime"`
+		Version          float64 `json:"version"`
+		Prices           float64 `json:"prices"`
+	}
+
+	HostUsabilityBreakdown struct {
+		Blocked               bool `json:"blocked"`
+		Offline               bool `json:"offline"`
+		LowScore              bool `json:"lowScore"`
+		RedundantIP           bool `json:"redundantIP"`
+		Gouging               bool `json:"gouging"`
+		NotAcceptingContracts bool `json:"notAcceptingContracts"`
+		NotAnnounced          bool `json:"notAnnounced"`
+		NotCompletingScan     bool `json:"notCompletingScan"`
+	}
+)
+
+func (sb HostScoreBreakdown) String() string {
+	return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices)
+}
+
+func (hgb HostGougingBreakdown) Gouging() bool {
+	for _, err := range []string{
+		hgb.ContractErr,
+		hgb.DownloadErr,
+		hgb.GougingErr,
+		hgb.PruneErr,
+		hgb.UploadErr,
+	} {
+		if err != "" {
+			return true
+		}
+	}
+	return false
+}
+
+func (hgb HostGougingBreakdown) String() string {
+	var reasons []string
+	for _, errStr := range []string{
+		hgb.ContractErr,
+		hgb.DownloadErr,
+		hgb.GougingErr,
+		hgb.PruneErr,
+		hgb.UploadErr,
+	} {
+		if errStr != "" {
+			reasons = append(reasons, errStr)
+		}
+	}
+	return strings.Join(reasons, ";")
+}
+
+func (sb HostScoreBreakdown) Score() float64 {
+	return sb.Age * sb.Collateral * sb.Interactions * sb.StorageRemaining * sb.Uptime * sb.Version * sb.Prices
+}

autopilot/autopilot.go

@@ -53,10 +53,10 @@ type Bus interface {
 	PrunableData(ctx context.Context) (prunableData api.ContractsPrunableDataResponse, err error)
 
 	// hostdb
-	Host(ctx context.Context, hostKey types.PublicKey) (hostdb.HostInfo, error)
+	Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error)
 	HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]hostdb.HostAddress, error)
 	RemoveOfflineHosts(ctx context.Context, minRecentScanFailures uint64, maxDowntime time.Duration) (uint64, error)
-	SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]hostdb.HostInfo, error)
+	SearchHosts(ctx context.Context, opts api.SearchHostOptions) ([]api.Host, error)
 
 	// metrics
 	RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error
@@ -737,7 +737,7 @@ func (ap *Autopilot) hostsHandlerPOST(jc jape.Context) {
 	jc.Encode(hosts)
 }
 
-func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (usables uint64) {
+func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) {
 	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
 	for _, host := range hosts {
 		usable, _ := isUsableHost(cfg, rs, gc, host, smallestValidScore, 0)
@@ -751,7 +751,7 @@ func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.
 // evaluateConfig evaluates the given configuration and if the gouging settings
 // are too strict for the number of contracts required by 'cfg', it will provide
 // a recommendation on how to loosen it.
-func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []hostdb.HostInfo) (resp api.ConfigEvaluationResponse) {
+func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (resp api.ConfigEvaluationResponse) {
 	gc := worker.NewGougingChecker(gs, cs, fee, currentPeriod, cfg.Contracts.RenewWindow)
 
 	resp.Hosts = uint64(len(hosts))
@@ -866,7 +866,7 @@ func evaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Cu
 
 // optimiseGougingSetting tries to optimise one field of the gouging settings to
 // try and hit the target number of contracts.
-func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []hostdb.HostInfo) bool {
+func optimiseGougingSetting(gs *api.GougingSettings, field *types.Currency, cfg api.AutopilotConfig, cs api.ConsensusState, fee types.Currency, currentPeriod uint64, rs api.RedundancySettings, hosts []api.Host) bool {
 	if cfg.Contracts.Amount == 0 {
 		return true // nothing to do
 	}

autopilot/autopilot_test.go

@@ -14,9 +14,10 @@ import (
 
 func TestOptimiseGougingSetting(t *testing.T) {
 	// create 10 hosts that should all be usable
-	var hosts []hostdb.HostInfo
+	var hosts []api.Host
 	for i := 0; i < 10; i++ {
-		hosts = append(hosts, hostdb.HostInfo{
+		hosts = append(hosts, api.Host{
+
 			Host: hostdb.Host{
 				KnownSince: time.Unix(0, 0),
 				PriceTable: hostdb.HostPriceTable{
@@ -42,6 +43,7 @@ func TestOptimiseGougingSetting(t *testing.T) {
 				Scanned:          true,
 			},
 			Blocked: false,
+			Checks:  nil,
 		})
 	}
 

autopilot/client.go

@@ -40,7 +40,7 @@ func (c *Client) HostInfo(hostKey types.PublicKey) (resp api.HostHandlerResponse
 }
 
 // HostInfo returns information about all hosts.
-func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode string, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) {
+func (c *Client) HostInfos(ctx context.Context, filterMode, usabilityMode, addressContains string, keyIn []types.PublicKey, offset, limit int) (resp []api.HostHandlerResponse, err error) {
 	err = c.c.POST("/hosts", api.SearchHostsRequest{
 		Offset:          offset,
 		Limit:           limit,