Merge branch 'dev' into pj/sql-backups

.github/workflows/project-add.yml

@@ -0,0 +1,21 @@
+name: Add issues and PRs to Sia project
+
+on:
+  issues:
+    types:
+      - opened
+  pull_request:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          # You can target a project in a different organization
+          # to the issue
+          project-url: https://github.com/orgs/SiaFoundation/projects/5
+          github-token: ${{ secrets.PAT_ADD_TO_PROJECT }}

.github/workflows/publish.yml

@@ -9,7 +9,7 @@ on:
       - dev
     tags: 
       - 'v[0-9]+.[0-9]+.[0-9]+'
-      - 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-**'
 
 concurrency:
   group: ${{ github.workflow }}
@@ -263,7 +263,7 @@ jobs:
     steps:
       - name: Extract Tag Name
         id: get_tag
-        run: echo "::set-output name=tag_name::${GITHUB_REF#refs/tags/}"
+        run: echo "tag_name=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
 
       - name: Repository Dispatch
         uses: peter-evans/repository-dispatch@v3
@@ -274,7 +274,7 @@ jobs:
           client-payload: >
             {
               "description": "Renterd: The Next-Gen Sia Renter",
-              "tag": "${{ steps.get_tag.outputs.tag_name }}",
+              "tag": "${{ env.tag_name }}",
               "project": "renterd",
               "workflow_id": "${{ github.run_id }}"
             }

.golangci.yml

@@ -102,6 +102,8 @@ linters:
     - typecheck
     - whitespace
     - tagliatelle
+    - unused
+    - unparam
 
 issues:
   # Maximum issues count per one linter. Set to 0 to disable. Default is 50.

alerts/alerts_test.go

@@ -22,27 +22,29 @@ type testWebhookStore struct {
 	listed  int
 }
 
-func (s *testWebhookStore) DeleteWebhook(wb webhooks.Webhook) error {
+func (s *testWebhookStore) DeleteWebhook(_ context.Context, wb webhooks.Webhook) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	s.deleted++
 	return nil
 }
 
-func (s *testWebhookStore) AddWebhook(wb webhooks.Webhook) error {
+func (s *testWebhookStore) AddWebhook(_ context.Context, wb webhooks.Webhook) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	s.added++
 	return nil
 }
 
-func (s *testWebhookStore) Webhooks() ([]webhooks.Webhook, error) {
+func (s *testWebhookStore) Webhooks(_ context.Context) ([]webhooks.Webhook, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	s.listed++
 	return nil, nil
 }
 
+var _ webhooks.WebhookStore = (*testWebhookStore)(nil)
+
 func TestWebhooks(t *testing.T) {
 	store := &testWebhookStore{}
 	mgr, err := webhooks.NewManager(zap.NewNop().Sugar(), store)
@@ -75,7 +77,7 @@ func TestWebhooks(t *testing.T) {
 	if hookID := wh.String(); hookID != fmt.Sprintf("%v.%v.%v", wh.URL, wh.Module, "") {
 		t.Fatalf("wrong result for wh.String(): %v != %v", wh.String(), hookID)
 	}
-	err = mgr.Register(wh)
+	err = mgr.Register(context.Background(), wh)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -110,7 +112,7 @@ func TestWebhooks(t *testing.T) {
 	}
 
 	// unregister hook
-	if err := mgr.Delete(webhooks.Webhook{
+	if err := mgr.Delete(context.Background(), webhooks.Webhook{
 		Event:  hooks[0].Event,
 		Module: hooks[0].Module,
 		URL:    hooks[0].URL,

api/multipart.go

@@ -51,6 +51,10 @@ type (
 		MimeType    string
 		Metadata    ObjectUserMetadata
 	}
+
+	CompleteMultipartOptions struct {
+		Metadata ObjectUserMetadata
+	}
 )
 
 type (
@@ -75,10 +79,11 @@ type (
 	}
 
 	MultipartCompleteRequest struct {
-		Bucket   string `json:"bucket"`
-		Path     string `json:"path"`
-		UploadID string `json:"uploadID"`
-		Parts    []MultipartCompletedPart
+		Bucket   string                   `json:"bucket"`
+		Metadata ObjectUserMetadata       `json:"metadata"`
+		Path     string                   `json:"path"`
+		UploadID string                   `json:"uploadID"`
+		Parts    []MultipartCompletedPart `json:"parts"`
 	}
 
 	MultipartCreateRequest struct {

api/object.go

@@ -92,6 +92,7 @@ type (
 	// HeadObjectResponse is the response type for the HEAD /worker/object endpoint.
 	HeadObjectResponse struct {
 		ContentType  string             `json:"contentType"`
+		Etag         string             `json:"eTag"`
 		LastModified string             `json:"lastModified"`
 		Range        *DownloadRange     `json:"range,omitempty"`
 		Size         int64              `json:"size"`
@@ -212,7 +213,8 @@ type (
 	}
 
 	HeadObjectOptions struct {
-		Range DownloadRange
+		IgnoreDelim bool
+		Range       DownloadRange
 	}
 
 	DownloadObjectOptions struct {
@@ -310,6 +312,12 @@ func (opts DeleteObjectOptions) Apply(values url.Values) {
 	}
 }
 
+func (opts HeadObjectOptions) Apply(values url.Values) {
+	if opts.IgnoreDelim {
+		values.Set("ignoreDelim", "true")
+	}
+}
+
 func (opts HeadObjectOptions) ApplyHeaders(h http.Header) {
 	if opts.Range != (DownloadRange{}) {
 		if opts.Range.Length == -1 {

autopilot/accounts.go

@@ -136,22 +136,18 @@ func (a *accounts) refillWorkerAccounts(ctx context.Context, w Worker) {
 
 	// refill accounts in separate goroutines
 	for _, c := range contracts {
-		// add logging for contracts in the set
-		_, inSet := inContractSet[c.ID]
-
 		// launch refill if not already in progress
 		if a.markRefillInProgress(workerID, c.HostKey) {
-			go func(contract api.ContractMetadata, inSet bool) {
+			go func(contract api.ContractMetadata) {
 				rCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
 				defer cancel()
-				accountID, refilled, rerr := refillWorkerAccount(rCtx, a.a, w, workerID, contract)
+				accountID, refilled, rerr := refillWorkerAccount(rCtx, a.a, w, contract)
 				if rerr != nil {
-					if inSet || rerr.Is(errMaxDriftExceeded) {
-						// register the alert on failure if the contract is in
-						// the set or the error is errMaxDriftExceeded
+					if rerr.Is(errMaxDriftExceeded) {
+						// register the alert if error is errMaxDriftExceeded
 						a.ap.RegisterAlert(ctx, newAccountRefillAlert(accountID, contract, *rerr))
-						a.l.Errorw(rerr.err.Error(), rerr.keysAndValues...)
 					}
+					a.l.Errorw(rerr.err.Error(), rerr.keysAndValues...)
 				} else {
 					// dismiss alerts on success
 					a.ap.DismissAlert(ctx, alertIDForAccount(alertAccountRefillID, accountID))
@@ -167,7 +163,7 @@ func (a *accounts) refillWorkerAccounts(ctx context.Context, w Worker) {
 				}
 
 				a.markRefillDone(workerID, contract.HostKey)
-			}(c, inSet)
+			}(c)
 		}
 	}
 }
@@ -188,7 +184,7 @@ func (err *refillError) Is(target error) bool {
 	return errors.Is(err.err, target)
 }
 
-func refillWorkerAccount(ctx context.Context, a AccountStore, w Worker, workerID string, contract api.ContractMetadata) (accountID rhpv3.Account, refilled bool, rerr *refillError) {
+func refillWorkerAccount(ctx context.Context, a AccountStore, w Worker, contract api.ContractMetadata) (accountID rhpv3.Account, refilled bool, rerr *refillError) {
 	wrapErr := func(err error, keysAndValues ...interface{}) *refillError {
 		if err == nil {
 			return nil

autopilot/autopilot.go

@@ -18,6 +18,7 @@ import (
 	"go.sia.tech/renterd/api"
 	"go.sia.tech/renterd/build"
 	"go.sia.tech/renterd/hostdb"
+	"go.sia.tech/renterd/internal/utils"
 	"go.sia.tech/renterd/object"
 	"go.sia.tech/renterd/wallet"
 	"go.sia.tech/renterd/webhooks"
@@ -299,7 +300,7 @@ func (ap *Autopilot) Run() error {
 
 			// perform maintenance
 			setChanged, err := ap.c.performContractMaintenance(ap.shutdownCtx, w)
-			if err != nil && isErr(err, context.Canceled) {
+			if err != nil && utils.IsErr(err, context.Canceled) {
 				return
 			} else if err != nil {
 				ap.logger.Errorf("contract maintenance failed, err: %v", err)
@@ -321,11 +322,11 @@ func (ap *Autopilot) Run() error {
 			}
 
 			// migration
-			ap.m.tryPerformMigrations(ap.shutdownCtx, ap.workers)
+			ap.m.tryPerformMigrations(ap.workers)
 
 			// pruning
 			if ap.state.cfg.Contracts.Prune {
-				ap.c.tryPerformPruning(ap.shutdownCtx, ap.workers)
+				ap.c.tryPerformPruning(ap.workers)
 			} else {
 				ap.logger.Debug("pruning disabled")
 			}
@@ -405,9 +406,9 @@ func (ap *Autopilot) blockUntilConfigured(interrupt <-chan time.Time) (configure
 		cancel()
 
 		// if the config was not found, or we were unable to fetch it, keep blocking
-		if isErr(err, context.Canceled) {
+		if utils.IsErr(err, context.Canceled) {
 			return
-		} else if isErr(err, api.ErrAutopilotNotFound) {
+		} else if utils.IsErr(err, api.ErrAutopilotNotFound) {
 			once.Do(func() { ap.logger.Info("autopilot is waiting to be configured...") })
 		} else if err != nil {
 			ap.logger.Errorf("autopilot is unable to fetch its configuration from the bus, err: %v", err)
@@ -438,7 +439,7 @@ func (ap *Autopilot) blockUntilOnline() (online bool) {
 		online = len(peers) > 0
 		cancel()
 
-		if isErr(err, context.Canceled) {
+		if utils.IsErr(err, context.Canceled) {
 			return
 		} else if err != nil {
 			ap.logger.Errorf("failed to get peers, err: %v", err)
@@ -472,7 +473,7 @@ func (ap *Autopilot) blockUntilSynced(interrupt <-chan time.Time) (synced, block
 		cancel()
 
 		// if an error occurred, or if we're not synced, we continue
-		if isErr(err, context.Canceled) {
+		if utils.IsErr(err, context.Canceled) {
 			return
 		} else if err != nil {
 			ap.logger.Errorf("failed to get consensus state, err: %v", err)
@@ -631,7 +632,7 @@ func (ap *Autopilot) isStopped() bool {
 
 func (ap *Autopilot) configHandlerGET(jc jape.Context) {
 	autopilot, err := ap.bus.Autopilot(jc.Request.Context(), ap.id)
-	if err != nil && strings.Contains(err.Error(), api.ErrAutopilotNotFound.Error()) {
+	if utils.IsErr(err, api.ErrAutopilotNotFound) {
 		jc.Error(errors.New("autopilot is not configured yet"), http.StatusNotFound)
 		return
 	}
@@ -653,7 +654,7 @@ func (ap *Autopilot) configHandlerPUT(jc jape.Context) {
 	// fetch the autopilot and update its config
 	var contractSetChanged bool
 	autopilot, err := ap.bus.Autopilot(jc.Request.Context(), ap.id)
-	if err != nil && strings.Contains(err.Error(), api.ErrAutopilotNotFound.Error()) {
+	if utils.IsErr(err, api.ErrAutopilotNotFound) {
 		autopilot = api.Autopilot{ID: ap.id, Config: cfg}
 	} else {
 		if autopilot.Config.Contracts.Set != cfg.Contracts.Set {

autopilot/contract_pruning.go

@@ -9,6 +9,7 @@ import (
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/alerts"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/internal/utils"
 	"go.sia.tech/siad/build"
 )
 
@@ -65,14 +66,14 @@ func (pm pruneMetrics) String() string {
 func (pr pruneResult) toAlert() (id types.Hash256, alert *alerts.Alert) {
 	id = alertIDForContract(alertPruningID, pr.fcid)
 
-	if shouldTrigger := pr.err != nil && !((isErr(pr.err, errInvalidMerkleProof) && build.VersionCmp(pr.version, "1.6.0") < 0) ||
-		isErr(pr.err, api.ErrContractNotFound) || // contract got archived
-		isErr(pr.err, errConnectionRefused) ||
-		isErr(pr.err, errConnectionTimedOut) ||
-		isErr(pr.err, errConnectionResetByPeer) ||
-		isErr(pr.err, errInvalidHandshakeSignature) ||
-		isErr(pr.err, errNoRouteToHost) ||
-		isErr(pr.err, errNoSuchHost)); shouldTrigger {
+	if shouldTrigger := pr.err != nil && !((utils.IsErr(pr.err, errInvalidMerkleProof) && build.VersionCmp(pr.version, "1.6.0") < 0) ||
+		utils.IsErr(pr.err, api.ErrContractNotFound) || // contract got archived
+		utils.IsErr(pr.err, errConnectionRefused) ||
+		utils.IsErr(pr.err, errConnectionTimedOut) ||
+		utils.IsErr(pr.err, errConnectionResetByPeer) ||
+		utils.IsErr(pr.err, errInvalidHandshakeSignature) ||
+		utils.IsErr(pr.err, errNoRouteToHost) ||
+		utils.IsErr(pr.err, errNoSuchHost)); shouldTrigger {
 		alert = newContractPruningFailedAlert(pr.hk, pr.version, pr.fcid, pr.err)
 	}
 	return
@@ -196,7 +197,7 @@ func (c *contractor) pruneContract(w Worker, fcid types.FileContractID) pruneRes
 	pruned, remaining, err := w.RHPPruneContract(ctx, fcid, timeoutPruneContract)
 	if err != nil && pruned == 0 {
 		return pruneResult{fcid: fcid, hk: host.PublicKey, version: host.Settings.Version, err: err}
-	} else if err != nil && isErr(err, context.DeadlineExceeded) {
+	} else if err != nil && utils.IsErr(err, context.DeadlineExceeded) {
 		err = nil
 	}
 

autopilot/contract_spending.go

@@ -20,7 +20,7 @@ func (c *contractor) contractSpending(ctx context.Context, contract api.Contract
 	return total, nil
 }
 
-func (c *contractor) currentPeriodSpending(contracts []api.Contract, currentPeriod uint64) (types.Currency, error) {
+func (c *contractor) currentPeriodSpending(contracts []api.Contract, currentPeriod uint64) types.Currency {
 	totalCosts := make(map[types.FileContractID]types.Currency)
 	for _, c := range contracts {
 		totalCosts[c.ID] = c.TotalCost
@@ -41,22 +41,19 @@ func (c *contractor) currentPeriodSpending(contracts []api.Contract, currentPeri
 	for _, contract := range filtered {
 		totalAllocated = totalAllocated.Add(contract.TotalCost)
 	}
-	return totalAllocated, nil
+	return totalAllocated
 }
 
-func (c *contractor) remainingFunds(contracts []api.Contract) (types.Currency, error) {
+func (c *contractor) remainingFunds(contracts []api.Contract) types.Currency {
 	state := c.ap.State()
 
 	// find out how much we spent in the current period
-	spent, err := c.currentPeriodSpending(contracts, state.period)
-	if err != nil {
-		return types.ZeroCurrency, err
-	}
+	spent := c.currentPeriodSpending(contracts, state.period)
 
 	// figure out remaining funds
 	var remaining types.Currency
 	if state.cfg.Contracts.Allowance.Cmp(spent) > 0 {
 		remaining = state.cfg.Contracts.Allowance.Sub(spent)
 	}
-	return remaining, nil
+	return remaining
 }