Merge branch 'dev' into pj/host-infos

.github/workflows/publish.yml

@@ -1,13 +1,13 @@
 name: Publish
 
-# Controls when the action will run. 
+# Controls when the action will run.
 on:
   # Triggers the workflow on new SemVer tags
   push:
     branches:
       - master
       - dev
-    tags: 
+    tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
       - 'v[0-9]+.[0-9]+.[0-9]+-**'
 
@@ -116,7 +116,7 @@ jobs:
         with:
           name: renterd
           path: release/
-  build-mac: 
+  build-mac:
     runs-on: macos-latest
     strategy:
       matrix:
@@ -212,7 +212,7 @@ jobs:
         with:
           name: renterd
           path: release/
-  build-windows: 
+  build-windows:
     runs-on: windows-latest
     strategy:
       matrix:
@@ -253,28 +253,62 @@ jobs:
         with:
           name: renterd
           path: release/
-  dispatch:
+
+  dispatch-homebrew: # only runs on full releases
     if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
     needs: [docker, build-linux, build-mac, build-windows]
-    strategy:
-        matrix:
-            repo: ['siafoundation/homebrew-sia', 'siafoundation/linux']
     runs-on: ubuntu-latest
     steps:
       - name: Extract Tag Name
         id: get_tag
         run: echo "tag_name=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
 
-      - name: Repository Dispatch
+      - name: Dispatch
         uses: peter-evans/repository-dispatch@v3
         with:
           token: ${{ secrets.PAT_REPOSITORY_DISPATCH }}
-          repository: ${{ matrix.repo }}
+          repository: siafoundation/homebrew-sia
           event-type: release-tagged
           client-payload: >
             {
               "description": "Renterd: The Next-Gen Sia Renter",
               "tag": "${{ env.tag_name }}",
               "project": "renterd",
               "workflow_id": "${{ github.run_id }}"
-            }
+            }
+  dispatch-linux: # run on full releases, release candidates, and master branch
+    if: startsWith(github.ref, 'refs/tags/v') || endsWith(github.ref, 'master')
+    needs: [docker, build-linux, build-mac, build-windows]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Build Dispatch Payload
+        id: get_payload
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const isRelease = context.ref.startsWith('refs/tags/v'),
+              isBeta = isRelease && context.ref.includes('-beta'),
+              tag = isRelease ? context.ref.replace('refs/tags/', '') : 'master';
+
+            let component = 'nightly';
+            if (isBeta) {
+              component = 'beta';
+            } else if (isRelease) {
+              component = 'main';
+            }
+
+            return {
+              description: "renterd: The Next-Gen Sia Renter",
+              tag: tag,
+              project: "renterd",
+              workflow_id: context.runId,
+              component: component
+            };
+
+      - name: Dispatch
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.PAT_REPOSITORY_DISPATCH }}
+          repository: siafoundation/linux
+          event-type: release-tagged
+          client-payload: ${{ steps.get_payload.outputs.result }}

autopilot/contractor.go

@@ -32,6 +32,10 @@ const (
 	// contract.
 	estimatedFileContractTransactionSetSize = 2048
 
+	// failedRenewalForgivenessPeriod is the amount of time we wait before
+	// punishing a contract for not being able to refresh
+	failedRefreshForgivenessPeriod = 24 * time.Hour
+
 	// leewayPctCandidateHosts is the leeway we apply when fetching candidate
 	// hosts, we fetch ~10% more than required
 	leewayPctCandidateHosts = 1.1
@@ -95,6 +99,8 @@ type (
 		revisionLastBroadcast     map[types.FileContractID]time.Time
 		revisionSubmissionBuffer  uint64
 
+		firstRefreshFailure map[types.FileContractID]time.Time
+
 		mu sync.Mutex
 
 		pruning          bool
@@ -152,6 +158,8 @@ func newContractor(ap *Autopilot, revisionSubmissionBuffer uint64, revisionBroad
 		revisionLastBroadcast:     make(map[types.FileContractID]time.Time),
 		revisionSubmissionBuffer:  revisionSubmissionBuffer,
 
+		firstRefreshFailure: make(map[types.FileContractID]time.Time),
+
 		resolver: newIPResolver(ap.shutdownCtx, resolverLookupTimeout, ap.logger.Named("resolver")),
 	}
 }
@@ -216,6 +224,9 @@ func (c *contractor) performContractMaintenance(ctx context.Context, w Worker) (
 	contracts := resp.Contracts
 	c.logger.Infof("fetched %d contracts from the worker, took %v", len(resp.Contracts), time.Since(start))
 
+	// prune contract refresh failure map
+	c.pruneContractRefreshFailures(contracts)
+
 	// run revision broadcast
 	c.runRevisionBroadcast(ctx, w, contracts, isInCurrentSet)
 
@@ -1595,6 +1606,27 @@ func (c *contractor) hostForContract(ctx context.Context, fcid types.FileContrac
 	return
 }
 
+func (c *contractor) pruneContractRefreshFailures(contracts []api.Contract) {
+	contractMap := make(map[types.FileContractID]struct{})
+	for _, contract := range contracts {
+		contractMap[contract.ID] = struct{}{}
+	}
+	for fcid := range c.firstRefreshFailure {
+		if _, ok := contractMap[fcid]; !ok {
+			delete(c.firstRefreshFailure, fcid)
+		}
+	}
+}
+
+func (c *contractor) shouldForgiveFailedRefresh(fcid types.FileContractID) bool {
+	lastFailure, exists := c.firstRefreshFailure[fcid]
+	if !exists {
+		lastFailure = time.Now()
+		c.firstRefreshFailure[fcid] = lastFailure
+	}
+	return time.Since(lastFailure) < failedRefreshForgivenessPeriod
+}
+
 func addLeeway(n uint64, pct float64) uint64 {
 	if pct < 0 {
 		panic("given leeway percent has to be positive")

autopilot/contractor_test.go

@@ -3,8 +3,12 @@ package autopilot
 import (
 	"math"
 	"testing"
+	"time"
 
+	"go.sia.tech/core/types"
+	"go.sia.tech/renterd/api"
 	"go.uber.org/zap"
+	"lukechampine.com/frand"
 )
 
 func TestCalculateMinScore(t *testing.T) {
@@ -35,3 +39,30 @@ func TestCalculateMinScore(t *testing.T) {
 		t.Fatalf("expected minScore to be math.SmallestNonzeroFLoat64 but was %v", minScore)
 	}
 }
+
+func TestShouldForgiveFailedRenewal(t *testing.T) {
+	var fcid types.FileContractID
+	frand.Read(fcid[:])
+	c := &contractor{
+		firstRefreshFailure: make(map[types.FileContractID]time.Time),
+	}
+
+	// try twice since the first time will set the failure time
+	if !c.shouldForgiveFailedRefresh(fcid) {
+		t.Fatal("should forgive")
+	} else if !c.shouldForgiveFailedRefresh(fcid) {
+		t.Fatal("should forgive")
+	}
+
+	// set failure to be a full period in the past
+	c.firstRefreshFailure[fcid] = time.Now().Add(-failedRefreshForgivenessPeriod - time.Second)
+	if c.shouldForgiveFailedRefresh(fcid) {
+		t.Fatal("should not forgive")
+	}
+
+	// prune map
+	c.pruneContractRefreshFailures([]api.Contract{})
+	if len(c.firstRefreshFailure) != 0 {
+		t.Fatal("expected no failures")
+	}
+}

autopilot/hostfilter.go

@@ -185,8 +185,8 @@ func (c *contractor) isUsableContract(cfg api.AutopilotConfig, state state, ci c
 		}
 		if isOutOfFunds(cfg, pt, contract) {
 			reasons = append(reasons, errContractOutOfFunds.Error())
-			usable = false
-			recoverable = true
+			usable = usable && c.shouldForgiveFailedRefresh(contract.ID)
+			recoverable = !usable // only needs to be recoverable if !usable
 			refresh = true
 			renew = false
 		}

bus/bus.go

@@ -1019,7 +1019,7 @@ func (b *bus) contractsPrunableDataHandlerGET(jc jape.Context) {
 		// adjust the amount of prunable data with the pending uploads, due to
 		// how we record contract spending a contract's size might already
 		// include pending sectors
-		pending := b.uploadingSectors.pending(fcid)
+		pending := b.uploadingSectors.Pending(fcid)
 		if pending > size.Prunable {
 			size.Prunable = 0
 		} else {
@@ -1066,7 +1066,7 @@ func (b *bus) contractSizeHandlerGET(jc jape.Context) {
 	// adjust the amount of prunable data with the pending uploads, due to how
 	// we record contract spending a contract's size might already include
 	// pending sectors
-	pending := b.uploadingSectors.pending(id)
+	pending := b.uploadingSectors.Pending(id)
 	if pending > size.Prunable {
 		size.Prunable = 0
 	} else {
@@ -1143,6 +1143,7 @@ func (b *bus) contractIDRenewedHandlerPOST(jc jape.Context) {
 	if jc.Check("couldn't store contract", err) == nil {
 		jc.Encode(r)
 	}
+	b.uploadingSectors.HandleRenewal(req.Contract.ID(), req.RenewedFrom)
 }
 
 func (b *bus) contractIDRootsHandlerGET(jc jape.Context) {
@@ -1155,7 +1156,7 @@ func (b *bus) contractIDRootsHandlerGET(jc jape.Context) {
 	if jc.Check("couldn't fetch contract sectors", err) == nil {
 		jc.Encode(api.ContractRootsResponse{
 			Roots:     roots,
-			Uploading: b.uploadingSectors.sectors(id),
+			Uploading: b.uploadingSectors.Sectors(id),
 		})
 	}
 }
@@ -2016,7 +2017,7 @@ func (b *bus) stateHandlerGET(jc jape.Context) {
 func (b *bus) uploadTrackHandlerPOST(jc jape.Context) {
 	var id api.UploadID
 	if jc.DecodeParam("id", &id) == nil {
-		jc.Check("failed to track upload", b.uploadingSectors.trackUpload(id))
+		jc.Check("failed to track upload", b.uploadingSectors.StartUpload(id))
 	}
 }
 
@@ -2029,13 +2030,13 @@ func (b *bus) uploadAddSectorHandlerPOST(jc jape.Context) {
 	if jc.Decode(&req) != nil {
 		return
 	}
-	jc.Check("failed to add sector", b.uploadingSectors.addUploadingSector(id, req.ContractID, req.Root))
+	jc.Check("failed to add sector", b.uploadingSectors.AddSector(id, req.ContractID, req.Root))
 }
 
 func (b *bus) uploadFinishedHandlerDELETE(jc jape.Context) {
 	var id api.UploadID
 	if jc.DecodeParam("id", &id) == nil {
-		b.uploadingSectors.finishUpload(id)
+		b.uploadingSectors.FinishUpload(id)
 	}
 }