Skip to content

Commit

Permalink
Migrate PruneMetrics to raw SQL and cleanup metric models (#1314)
Browse files Browse the repository at this point in the history
  • Loading branch information
@peterjan
peterjan authored Jun 21, 2024
1 parent e646259 commit a82d3d3
Show file tree
Hide file tree
Showing 7 changed files with 360 additions and 497 deletions.
147 changes: 3 additions & 144 deletions stores/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,119 +2,12 @@ package stores

import (
"context"
"errors"
"fmt"
"time"

"go.sia.tech/renterd/api"
sql "go.sia.tech/renterd/stores/sql"
)

type (
// dbContractMetric tracks information about a contract's funds. It is
// supposed to be reported by a worker every time a contract is revised.
dbContractMetric struct {
Model

Timestamp unixTimeMS `gorm:"index;NOT NULL"`

FCID fileContractID `gorm:"index;size:32;NOT NULL;column:fcid"`
Host publicKey `gorm:"index;size:32;NOT NULL"`

RemainingCollateralLo unsigned64 `gorm:"index:idx_remaining_collateral;NOT NULL"`
RemainingCollateralHi unsigned64 `gorm:"index:idx_remaining_collateral;NOT NULL"`
RemainingFundsLo unsigned64 `gorm:"index:idx_remaining_funds;NOT NULL"`
RemainingFundsHi unsigned64 `gorm:"index:idx_remaining_funds;NOT NULL"`
RevisionNumber unsigned64 `gorm:"index;NOT NULL"`

UploadSpendingLo unsigned64 `gorm:"index:idx_upload_spending;NOT NULL"`
UploadSpendingHi unsigned64 `gorm:"index:idx_upload_spending;NOT NULL"`
DownloadSpendingLo unsigned64 `gorm:"index:idx_download_spending;NOT NULL"`
DownloadSpendingHi unsigned64 `gorm:"index:idx_download_spending;NOT NULL"`
FundAccountSpendingLo unsigned64 `gorm:"index:idx_fund_account_spending;NOT NULL"`
FundAccountSpendingHi unsigned64 `gorm:"index:idx_fund_account_spending;NOT NULL"`
DeleteSpendingLo unsigned64 `gorm:"index:idx_delete_spending;NOT NULL"`
DeleteSpendingHi unsigned64 `gorm:"index:idx_delete_spending;NOT NULL"`
ListSpendingLo unsigned64 `gorm:"index:idx_list_spending;NOT NULL"`
ListSpendingHi unsigned64 `gorm:"index:idx_list_spending;NOT NULL"`
}

// dbContractPruneMetric tracks information about contract pruning. Such as
// the number of bytes pruned, how much data there is left to prune and how
// long it took, along with potential errors that occurred while trying to
// prune the contract.
dbContractPruneMetric struct {
Model

Timestamp unixTimeMS `gorm:"index;NOT NULL"`

FCID fileContractID `gorm:"index;size:32;NOT NULL;column:fcid"`
Host publicKey `gorm:"index;size:32;NOT NULL"`
HostVersion string `gorm:"index"`

Pruned unsigned64 `gorm:"index;NOT NULL"`
Remaining unsigned64 `gorm:"index;NOT NULL"`
Duration time.Duration `gorm:"index;NOT NULL"`
}

// dbContractSetMetric tracks information about a specific contract set.
// Such as the number of contracts it contains. Intended to be reported by
// the bus every time the set is updated.
dbContractSetMetric struct {
Model
Timestamp unixTimeMS `gorm:"index;NOT NULL"`

Name string `gorm:"index;NOT NULL"`
Contracts int `gorm:"index;NOT NULL"`
}

// dbContractSetChurnMetric contains information about contracts being added
// to / removed from a contract set. Expected to be reported by the entity
// updating the set. e.g. the autopilot.
dbContractSetChurnMetric struct {
Model
Timestamp unixTimeMS `gorm:"index;NOT NULL"`

Name string `gorm:"index;NOT NULL"`
FCID fileContractID `gorm:"index;size:32;NOT NULL"`
Direction string `gorm:"index;NOT NULL"` // "added" or "removed"
Reason string `gorm:"index;NOT NULL"`
}

// dbPerformanceMetric is a generic metric used to track the performance of
// an action. Such an action could be a ReadSector operation. Expected to be
// reported by workers.
dbPerformanceMetric struct {
Model
Timestamp unixTimeMS `gorm:"index;NOT NULL"`

Action string `gorm:"index;NOT NULL"`
Host publicKey `gorm:"index;size:32;NOT NULL"`
Origin string `gorm:"index;NOT NULL"`
Duration time.Duration `gorm:"index;NOT NULL"`
}

// dbWalletMetric tracks information about a specific wallet.
dbWalletMetric struct {
Model
Timestamp unixTimeMS `gorm:"index;NOT NULL"`

ConfirmedLo unsigned64 `gorm:"index:idx_confirmed;NOT NULL"`
ConfirmedHi unsigned64 `gorm:"index:idx_confirmed;NOT NULL"`
SpendableLo unsigned64 `gorm:"index:idx_spendable;NOT NULL"`
SpendableHi unsigned64 `gorm:"index:idx_spendable;NOT NULL"`
UnconfirmedLo unsigned64 `gorm:"index:idx_unconfirmed;NOT NULL"`
UnconfirmedHi unsigned64 `gorm:"index:idx_unconfirmed;NOT NULL"`
}
)

func (dbContractMetric) TableName() string { return "contracts" }
func (dbContractPruneMetric) TableName() string { return "contract_prunes" }
func (dbContractSetMetric) TableName() string { return "contract_sets" }
func (dbContractSetChurnMetric) TableName() string { return "contract_sets_churn" }
func (dbPerformanceMetric) TableName() string { return "performance" }
func (dbWalletMetric) TableName() string { return "wallets" }

func (s *SQLStore) ContractMetrics(ctx context.Context, start time.Time, n uint64, interval time.Duration, opts api.ContractMetricsQueryOpts) (metrics []api.ContractMetric, err error) {
err = s.bMetrics.Transaction(ctx, func(tx sql.MetricsDatabaseTx) (txErr error) {
metrics, txErr = tx.ContractMetrics(ctx, start, n, interval, opts)
Expand Down Expand Up @@ -200,41 +93,7 @@ func (s *SQLStore) WalletMetrics(ctx context.Context, start time.Time, n uint64,
}

func (s *SQLStore) PruneMetrics(ctx context.Context, metric string, cutoff time.Time) error {
if metric == "" {
return errors.New("metric must be set")
} else if cutoff.IsZero() {
return errors.New("cutoff time must be set")
}
var model interface{}
switch metric {
case api.MetricContractPrune:
model = &dbContractPruneMetric{}
case api.MetricContractSet:
model = &dbContractSetMetric{}
case api.MetricContractSetChurn:
model = &dbContractSetChurnMetric{}
case api.MetricContract:
model = &dbContractMetric{}
case api.MetricPerformance:
model = &dbPerformanceMetric{}
case api.MetricWallet:
model = &dbWalletMetric{}
default:
return fmt.Errorf("unknown metric '%s'", metric)
}
return s.dbMetrics.Model(model).
Where("timestamp < ?", unixTimeMS(cutoff)).
Delete(model).
Error
}

func normaliseTimestamp(start time.Time, interval time.Duration, t unixTimeMS) unixTimeMS {
startMS := start.UnixMilli()
toNormaliseMS := time.Time(t).UnixMilli()
intervalMS := interval.Milliseconds()
if startMS > toNormaliseMS {
return unixTimeMS(start)
}
normalizedMS := (toNormaliseMS-startMS)/intervalMS*intervalMS + start.UnixMilli()
return unixTimeMS(time.UnixMilli(normalizedMS))
return s.bMetrics.Transaction(ctx, func(tx sql.MetricsDatabaseTx) error {
return tx.PruneMetrics(ctx, metric, cutoff)
})
}
Loading

0 comments on commit a82d3d3

Please sign in to comment.