From 766f66d9233ebad38a451ebdd717b390345e726d Mon Sep 17 00:00:00 2001 From: PJ Date: Tue, 13 Aug 2024 10:51:32 +0200 Subject: [PATCH 01/10] all: add node.go --- autopilot/autopilot.go | 12 +- autopilot/node.go | 17 + bus/bus.go | 14 +- bus/client/client_test.go | 7 +- {internal/node => bus}/node.go | 57 +-- {internal/node => bus}/syncer.go | 6 +- cmd/renterd/commands.go | 15 +- cmd/renterd/config.go | 358 ++++++++++++++- cmd/renterd/logger.go | 6 +- cmd/renterd/main.go | 734 +------------------------------ cmd/renterd/node.go | 462 +++++++++++++++++++ config/config.go | 6 +- internal/chain/chain.go | 48 -- internal/test/e2e/cluster.go | 57 ++- internal/worker/auth.go | 20 - stores/hostdb_test.go | 2 +- worker/mocks_test.go | 51 +++ worker/node.go | 28 ++ worker/worker.go | 3 + 19 files changed, 1001 insertions(+), 902 deletions(-) create mode 100644 autopilot/node.go rename {internal/node => bus}/node.go (76%) rename {internal/node => bus}/syncer.go (92%) create mode 100644 cmd/renterd/node.go delete mode 100644 internal/chain/chain.go delete mode 100644 internal/worker/auth.go create mode 100644 worker/node.go diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 52da95099..577ceb2b6 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -209,11 +209,11 @@ func (ap *Autopilot) configHandlerPOST(jc jape.Context) { jc.Encode(res) } -func (ap *Autopilot) Run() error { +func (ap *Autopilot) Run() { ap.startStopMu.Lock() if ap.isRunning() { ap.startStopMu.Unlock() - return errors.New("already running") + return } ap.startTime = time.Now() ap.triggerChan = make(chan bool, 1) @@ -226,7 +226,7 @@ func (ap *Autopilot) Run() error { // block until the autopilot is online if online := ap.blockUntilOnline(); !online { ap.logger.Error("autopilot stopped before it was able to come online") - return nil + return } // schedule a trigger when the wallet receives its first deposit @@ -234,7 +234,7 @@ func (ap *Autopilot) Run() error { if !errors.Is(err, context.Canceled) { ap.logger.Error(err) } - return nil + return } var forceScan bool @@ -342,7 +342,7 @@ func (ap *Autopilot) Run() error { select { case <-ap.shutdownCtx.Done(): - return nil + return case forceScan = <-ap.triggerChan: ap.logger.Info("autopilot iteration triggered") ap.ticker.Reset(ap.tickerDuration) @@ -350,7 +350,7 @@ func (ap *Autopilot) Run() error { case <-tickerFired: } } - return nil + return } // Shutdown shuts down the autopilot. diff --git a/autopilot/node.go b/autopilot/node.go new file mode 100644 index 000000000..eec478b3c --- /dev/null +++ b/autopilot/node.go @@ -0,0 +1,17 @@ +package autopilot + +import ( + "context" + "net/http" + + "go.sia.tech/renterd/config" + "go.uber.org/zap" +) + +func NewNode(cfg config.Autopilot, b Bus, workers []Worker, l *zap.Logger) (http.Handler, func(), func(context.Context) error, error) { + ap, err := New(cfg.ID, b, workers, l, cfg.Heartbeat, cfg.ScannerInterval, cfg.ScannerBatchSize, cfg.ScannerNumThreads, cfg.MigrationHealthCutoff, cfg.AccountsRefillInterval, cfg.RevisionSubmissionBuffer, cfg.MigratorParallelSlabsPerWorker, cfg.RevisionBroadcastInterval) + if err != nil { + return nil, nil, nil, err + } + return ap.Handler(), ap.Run, ap.Shutdown, nil +} diff --git a/bus/bus.go b/bus/bus.go index d5f0ad383..48f7720fe 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -1,5 +1,9 @@ package bus +// TODOs: +// - add wallet metrics +// - add UPNP support + import ( "context" "encoding/json" @@ -9,11 +13,9 @@ import ( "time" "go.sia.tech/core/consensus" - "go.sia.tech/core/gateway" rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" - "go.sia.tech/coreutils/syncer" "go.sia.tech/coreutils/wallet" "go.sia.tech/jape" "go.sia.tech/renterd/alerts" @@ -203,14 +205,6 @@ type ( TriggerUpdate() } - Syncer interface { - Addr() string - BroadcastHeader(h gateway.BlockHeader) - BroadcastTransactionSet([]types.Transaction) - Connect(ctx context.Context, addr string) (*syncer.Peer, error) - Peers() []*syncer.Peer - } - Wallet interface { Address() types.Address Balance() (wallet.Balance, error) diff --git a/bus/client/client_test.go b/bus/client/client_test.go index 5ec93d7eb..8f26b992c 100644 --- a/bus/client/client_test.go +++ b/bus/client/client_test.go @@ -10,12 +10,13 @@ import ( "time" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" "go.sia.tech/jape" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/bus" "go.sia.tech/renterd/bus/client" + "go.sia.tech/renterd/config" - "go.sia.tech/renterd/internal/chain" - "go.sia.tech/renterd/internal/node" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -70,7 +71,7 @@ func newTestClient(dir string) (*client.Client, func() error, func(context.Conte // create bus network, genesis := chain.Mainnet() - b, shutdown, _, err := node.NewBus(node.BusConfig{ + b, shutdown, _, err := bus.NewNode(bus.NodeConfig{ Bus: config.Bus{ AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year Bootstrap: false, diff --git a/internal/node/node.go b/bus/node.go similarity index 76% rename from internal/node/node.go rename to bus/node.go index 94a895796..940ff8807 100644 --- a/internal/node/node.go +++ b/bus/node.go @@ -1,4 +1,4 @@ -package node +package bus import ( "context" @@ -17,8 +17,6 @@ import ( "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/wallet" "go.sia.tech/renterd/alerts" - "go.sia.tech/renterd/autopilot" - "go.sia.tech/renterd/bus" "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/stores" @@ -26,25 +24,13 @@ import ( "go.sia.tech/renterd/stores/sql/mysql" "go.sia.tech/renterd/stores/sql/sqlite" "go.sia.tech/renterd/webhooks" - "go.sia.tech/renterd/worker" - "go.sia.tech/renterd/worker/s3" "go.uber.org/zap" - "golang.org/x/crypto/blake2b" "gorm.io/gorm" "gorm.io/gorm/logger" "moul.io/zapgorm2" ) -// TODOs: -// - add wallet metrics -// - add UPNP support - -type Bus interface { - worker.Bus - s3.Bus -} - -type BusConfig struct { +type NodeConfig struct { config.Bus Database config.Database DatabaseLog config.DatabaseLog @@ -56,20 +42,7 @@ type BusConfig struct { SyncerPeerDiscoveryInterval time.Duration } -type AutopilotConfig struct { - config.Autopilot - ID string -} - -type ( - RunFn = func() error - WorkerSetupFn = func(context.Context, string, string) error - ShutdownFn = func(context.Context) error -) - -var NoopFn = func(context.Context) error { return nil } - -func NewBus(cfg BusConfig, dir string, seed types.PrivateKey, logger *zap.Logger) (http.Handler, ShutdownFn, *chain.Manager, error) { +func NewNode(cfg NodeConfig, dir string, seed types.PrivateKey, logger *zap.Logger) (http.Handler, func(context.Context) error, *chain.Manager, error) { // ensure we don't hang indefinitely ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() @@ -223,7 +196,7 @@ func NewBus(cfg BusConfig, dir string, seed types.PrivateKey, logger *zap.Logger // create bus announcementMaxAgeHours := time.Duration(cfg.AnnouncementMaxAgeHours) * time.Hour - b, err := bus.New(ctx, alertsMgr, wh, cm, s, w, sqlStore, sqlStore, sqlStore, sqlStore, sqlStore, sqlStore, sqlStore, announcementMaxAgeHours, logger) + b, err := New(ctx, alertsMgr, wh, cm, s, w, sqlStore, sqlStore, sqlStore, sqlStore, sqlStore, sqlStore, sqlStore, announcementMaxAgeHours, logger) if err != nil { return nil, nil, nil, err } @@ -240,28 +213,6 @@ func NewBus(cfg BusConfig, dir string, seed types.PrivateKey, logger *zap.Logger return b.Handler(), shutdownFn, cm, nil } -func NewWorker(cfg config.Worker, s3Opts s3.Opts, b Bus, seed types.PrivateKey, l *zap.Logger) (http.Handler, http.Handler, WorkerSetupFn, ShutdownFn, error) { - workerKey := blake2b.Sum256(append([]byte("worker"), seed...)) - w, err := worker.New(workerKey, cfg.ID, b, cfg.ContractLockTimeout, cfg.BusFlushInterval, cfg.DownloadOverdriveTimeout, cfg.UploadOverdriveTimeout, cfg.DownloadMaxOverdrive, cfg.UploadMaxOverdrive, cfg.DownloadMaxMemory, cfg.UploadMaxMemory, cfg.AllowPrivateIPs, l) - if err != nil { - return nil, nil, nil, nil, err - } - s3Handler, err := s3.New(b, w, l.Named("s3").Sugar(), s3Opts) - if err != nil { - err = errors.Join(err, w.Shutdown(context.Background())) - return nil, nil, nil, nil, fmt.Errorf("failed to create s3 handler: %w", err) - } - return w.Handler(), s3Handler, w.Setup, w.Shutdown, nil -} - -func NewAutopilot(cfg AutopilotConfig, b autopilot.Bus, workers []autopilot.Worker, l *zap.Logger) (http.Handler, RunFn, ShutdownFn, error) { - ap, err := autopilot.New(cfg.ID, b, workers, l, cfg.Heartbeat, cfg.ScannerInterval, cfg.ScannerBatchSize, cfg.ScannerNumThreads, cfg.MigrationHealthCutoff, cfg.AccountsRefillInterval, cfg.RevisionSubmissionBuffer, cfg.MigratorParallelSlabsPerWorker, cfg.RevisionBroadcastInterval) - if err != nil { - return nil, nil, nil, err - } - return ap.Handler(), ap.Run, ap.Shutdown, nil -} - func gormLogLevel(cfg config.DatabaseLog) logger.LogLevel { level := logger.Silent if cfg.Enabled { diff --git a/internal/node/syncer.go b/bus/syncer.go similarity index 92% rename from internal/node/syncer.go rename to bus/syncer.go index 0a374c78e..1e5f9f966 100644 --- a/internal/node/syncer.go +++ b/bus/syncer.go @@ -1,4 +1,4 @@ -package node +package bus import ( "context" @@ -26,7 +26,7 @@ type Syncer interface { // NewSyncer creates a syncer using the given configuration. The syncer that is // returned is already running, closing it will close the underlying listener // causing the syncer to stop. -func NewSyncer(cfg BusConfig, cm syncer.ChainManager, ps syncer.PeerStore, logger *zap.Logger) (Syncer, error) { +func NewSyncer(cfg NodeConfig, cm syncer.ChainManager, ps syncer.PeerStore, logger *zap.Logger) (Syncer, error) { // validate config if cfg.Bootstrap && cfg.Network == nil { return nil, errors.New("cannot bootstrap without a network") @@ -72,7 +72,7 @@ func NewSyncer(cfg BusConfig, cm syncer.ChainManager, ps syncer.PeerStore, logge return s, nil } -func options(cfg BusConfig, logger *zap.Logger) (opts []syncer.Option) { +func options(cfg NodeConfig, logger *zap.Logger) (opts []syncer.Option) { opts = append(opts, syncer.WithLogger(logger.Named("syncer")), syncer.WithSendBlocksTimeout(time.Minute), diff --git a/cmd/renterd/commands.go b/cmd/renterd/commands.go index 63898cfe1..819eb9de5 100644 --- a/cmd/renterd/commands.go +++ b/cmd/renterd/commands.go @@ -7,10 +7,11 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/wallet" "go.sia.tech/renterd/build" + "go.sia.tech/renterd/config" "gopkg.in/yaml.v3" ) -func cmdBuildConfig() { +func cmdBuildConfig(cfg *config.Config) { if _, err := os.Stat("renterd.yml"); err == nil { if !promptYesNo("renterd.yml already exists. Would you like to overwrite it?") { return @@ -23,10 +24,10 @@ func cmdBuildConfig() { fmt.Println("If you change your wallet seed phrase, your renter will not be able to access Siacoin associated with this wallet.") fmt.Println("Ensure that you have backed up your wallet seed phrase before continuing.") if promptYesNo("Would you like to change your wallet seed phrase?") { - setSeedPhrase() + setSeedPhrase(cfg) } } else { - setSeedPhrase() + setSeedPhrase(cfg) } fmt.Println("") @@ -34,17 +35,17 @@ func cmdBuildConfig() { fmt.Println(wrapANSI("\033[33m", "An admin password is already set.", "\033[0m")) fmt.Println("If you change your admin password, you will need to update any scripts or applications that use the admin API.") if promptYesNo("Would you like to change your admin password?") { - setAPIPassword() + setAPIPassword(cfg) } } else { - setAPIPassword() + setAPIPassword(cfg) } fmt.Println("") - setS3Config() + setS3Config(cfg) fmt.Println("") - setAdvancedConfig() + setAdvancedConfig(cfg) // write the config file configPath := "renterd.yml" diff --git a/cmd/renterd/config.go b/cmd/renterd/config.go index f4b728c7e..fb2fb507e 100644 --- a/cmd/renterd/config.go +++ b/cmd/renterd/config.go @@ -3,21 +3,365 @@ package main import ( "bufio" "encoding/hex" + "flag" "fmt" + "log" "net" "os" "runtime" "strconv" "strings" + "time" "go.sia.tech/core/types" "go.sia.tech/coreutils/wallet" + "go.sia.tech/renterd/api" "go.sia.tech/renterd/config" + "go.sia.tech/renterd/worker/s3" "golang.org/x/term" + "gopkg.in/yaml.v3" "lukechampine.com/frand" ) -var enableANSI = runtime.GOOS != "windows" +const ( + // accountRefillInterval is the amount of time between refills of ephemeral + // accounts. If we conservatively assume that a good host charges 500 SC / + // TiB, we can pay for about 2.2 GiB with 1 SC. Since we want to refill + // ahead of time at 0.5 SC, that makes 1.1 GiB. Considering a 1 Gbps uplink + // that is shared across 30 uploads, we upload at around 33 Mbps to each + // host. That means uploading 1.1 GiB to drain 0.5 SC takes around 5 + // minutes. That's why we assume 10 seconds to be more than frequent enough + // to refill an account when it's due for another refill. + defaultAccountRefillInterval = 10 * time.Second +) + +var ( + disableStdin bool + enableANSI = runtime.GOOS != "windows" +) + +func defaultConfig() config.Config { + return config.Config{ + Directory: ".", + Seed: os.Getenv("RENTERD_SEED"), + AutoOpenWebUI: true, + Network: "mainnet", + HTTP: config.HTTP{ + Address: "localhost:9980", + Password: os.Getenv("RENTERD_API_PASSWORD"), + }, + ShutdownTimeout: 5 * time.Minute, + Database: config.Database{ + MySQL: config.MySQL{ + User: "renterd", + Database: "renterd", + MetricsDatabase: "renterd_metrics", + }, + }, + Log: config.Log{ + Path: "", // deprecated. included for compatibility. + Level: "", + File: config.LogFile{ + Enabled: true, + Format: "json", + Path: os.Getenv("RENTERD_LOG_FILE"), + }, + StdOut: config.StdOut{ + Enabled: true, + Format: "human", + EnableANSI: runtime.GOOS != "windows", + }, + Database: config.DatabaseLog{ + Enabled: true, + IgnoreRecordNotFoundError: true, + SlowThreshold: 100 * time.Millisecond, + }, + }, + Bus: config.Bus{ + AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year + Bootstrap: true, + GatewayAddr: ":9981", + UsedUTXOExpiry: 24 * time.Hour, + SlabBufferCompletionThreshold: 1 << 12, + }, + Worker: config.Worker{ + Enabled: true, + + ID: "worker", + ContractLockTimeout: 30 * time.Second, + BusFlushInterval: 5 * time.Second, + + DownloadMaxOverdrive: 5, + DownloadOverdriveTimeout: 3 * time.Second, + + DownloadMaxMemory: 1 << 30, // 1 GiB + UploadMaxMemory: 1 << 30, // 1 GiB + UploadMaxOverdrive: 5, + UploadOverdriveTimeout: 3 * time.Second, + }, + Autopilot: config.Autopilot{ + Enabled: true, + + ID: api.DefaultAutopilotID, + RevisionSubmissionBuffer: 150, // 144 + 6 blocks leeway + AccountsRefillInterval: defaultAccountRefillInterval, + Heartbeat: 30 * time.Minute, + MigrationHealthCutoff: 0.75, + RevisionBroadcastInterval: 7 * 24 * time.Hour, + ScannerBatchSize: 100, + ScannerInterval: 4 * time.Hour, + ScannerNumThreads: 10, + MigratorParallelSlabsPerWorker: 1, + }, + S3: config.S3{ + Address: "localhost:8080", + Enabled: true, + DisableAuth: false, + KeypairsV4: nil, + }, + } +} + +func loadConfig() config.Config { + cfg := defaultConfig() + + parseYamlConfig(&cfg) + parseCLIFlags(&cfg) + parseEnvironmentVariables(&cfg) + + // check that the API password is set + if cfg.HTTP.Password == "" { + if disableStdin { + stdoutFatalError("API password must be set via environment variable or config file when --env flag is set") + return config.Config{} + } + } + setAPIPassword(&cfg) + + // check that the seed is set + if cfg.Seed == "" && (cfg.Worker.Enabled || cfg.Bus.RemoteAddr == "") { // only worker & bus require a seed + if disableStdin { + stdoutFatalError("Seed must be set via environment variable or config file when --env flag is set") + return config.Config{} + } + setSeedPhrase(&cfg) + } + + // default log levels + if cfg.Log.Level == "" { + cfg.Log.Level = "info" + } + if cfg.Log.Database.Level == "" { + cfg.Log.Database.Level = cfg.Log.Level + } + + return cfg +} + +// parseYamlConfig loads the config file specified by the RENTERD_CONFIG_FILE +// environment variable. If the config file does not exist, it will not be +// loaded. +func parseYamlConfig(cfg *config.Config) { + configPath := "renterd.yml" + if str := os.Getenv("RENTERD_CONFIG_FILE"); str != "" { + configPath = str + } + + // If the config file doesn't exist, don't try to load it. + if _, err := os.Stat(configPath); os.IsNotExist(err) { + return + } + + f, err := os.Open(configPath) + if err != nil { + log.Fatal("failed to open config file:", err) + } + defer f.Close() + + dec := yaml.NewDecoder(f) + dec.KnownFields(true) + + if err := dec.Decode(&cfg); err != nil { + log.Fatal("failed to decode config file:", err) + } +} + +func parseCLIFlags(cfg *config.Config) { + // deprecated - these go first so that they can be overwritten by the non-deprecated flags + flag.StringVar(&cfg.Log.Database.Level, "db.logger.logLevel", cfg.Log.Database.Level, "(deprecated) Logger level (overrides with RENTERD_DB_LOGGER_LOG_LEVEL)") + flag.BoolVar(&cfg.Database.Log.IgnoreRecordNotFoundError, "db.logger.ignoreNotFoundError", cfg.Database.Log.IgnoreRecordNotFoundError, "(deprecated) Ignores 'not found' errors in logger (overrides with RENTERD_DB_LOGGER_IGNORE_NOT_FOUND_ERROR)") + flag.DurationVar(&cfg.Database.Log.SlowThreshold, "db.logger.slowThreshold", cfg.Database.Log.SlowThreshold, "(deprecated) Threshold for slow queries in logger (overrides with RENTERD_DB_LOGGER_SLOW_THRESHOLD)") + flag.StringVar(&cfg.Log.Path, "log-path", cfg.Log.Path, "(deprecated) Path to directory for logs (overrides with RENTERD_LOG_PATH)") + + // node + flag.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "Address for serving the API") + flag.StringVar(&cfg.Directory, "dir", cfg.Directory, "Directory for storing node state") + flag.BoolVar(&disableStdin, "env", false, "disable stdin prompts for environment variables (default false)") + flag.BoolVar(&cfg.AutoOpenWebUI, "openui", cfg.AutoOpenWebUI, "automatically open the web UI on startup") + flag.StringVar(&cfg.Network, "network", cfg.Network, "Network to connect to (mainnet|zen|anagami). Defaults to 'mainnet' (overrides with RENTERD_NETWORK)") + + // logger + flag.StringVar(&cfg.Log.Level, "log.level", cfg.Log.Level, "Global logger level (debug|info|warn|error). Defaults to 'info' (overrides with RENTERD_LOG_LEVEL)") + flag.BoolVar(&cfg.Log.File.Enabled, "log.file.enabled", cfg.Log.File.Enabled, "Enables logging to disk. Defaults to 'true'. (overrides with RENTERD_LOG_FILE_ENABLED)") + flag.StringVar(&cfg.Log.File.Format, "log.file.format", cfg.Log.File.Format, "Format of log file (json|human). Defaults to 'json' (overrides with RENTERD_LOG_FILE_FORMAT)") + flag.StringVar(&cfg.Log.File.Path, "log.file.path", cfg.Log.File.Path, "Path of log file. Defaults to 'renterd.log' within the renterd directory. (overrides with RENTERD_LOG_FILE_PATH)") + flag.BoolVar(&cfg.Log.StdOut.Enabled, "log.stdout.enabled", cfg.Log.StdOut.Enabled, "Enables logging to stdout. Defaults to 'true'. (overrides with RENTERD_LOG_STDOUT_ENABLED)") + flag.StringVar(&cfg.Log.StdOut.Format, "log.stdout.format", cfg.Log.StdOut.Format, "Format of log output (json|human). Defaults to 'human' (overrides with RENTERD_LOG_STDOUT_FORMAT)") + flag.BoolVar(&cfg.Log.StdOut.EnableANSI, "log.stdout.enableANSI", cfg.Log.StdOut.EnableANSI, "Enables ANSI color codes in log output. Defaults to 'true' on non-Windows systems. (overrides with RENTERD_LOG_STDOUT_ENABLE_ANSI)") + flag.BoolVar(&cfg.Log.Database.Enabled, "log.database.enabled", cfg.Log.Database.Enabled, "Enable logging database queries. Defaults to 'true' (overrides with RENTERD_LOG_DATABASE_ENABLED)") + flag.StringVar(&cfg.Log.Database.Level, "log.database.level", cfg.Log.Database.Level, "Logger level for database queries (info|warn|error). Defaults to 'warn' (overrides with RENTERD_LOG_LEVEL and RENTERD_LOG_DATABASE_LEVEL)") + flag.BoolVar(&cfg.Log.Database.IgnoreRecordNotFoundError, "log.database.ignoreRecordNotFoundError", cfg.Log.Database.IgnoreRecordNotFoundError, "Enable ignoring 'not found' errors resulting from database queries. Defaults to 'true' (overrides with RENTERD_LOG_DATABASE_IGNORE_RECORD_NOT_FOUND_ERROR)") + flag.DurationVar(&cfg.Log.Database.SlowThreshold, "log.database.slowThreshold", cfg.Log.Database.SlowThreshold, "Threshold for slow queries in logger. Defaults to 100ms (overrides with RENTERD_LOG_DATABASE_SLOW_THRESHOLD)") + + // db + flag.StringVar(&cfg.Database.MySQL.URI, "db.uri", cfg.Database.MySQL.URI, "Database URI for the bus (overrides with RENTERD_DB_URI)") + flag.StringVar(&cfg.Database.MySQL.User, "db.user", cfg.Database.MySQL.User, "Database username for the bus (overrides with RENTERD_DB_USER)") + flag.StringVar(&cfg.Database.MySQL.Database, "db.name", cfg.Database.MySQL.Database, "Database name for the bus (overrides with RENTERD_DB_NAME)") + flag.StringVar(&cfg.Database.MySQL.MetricsDatabase, "db.metricsName", cfg.Database.MySQL.MetricsDatabase, "Database for metrics (overrides with RENTERD_DB_METRICS_NAME)") + + // bus + flag.Uint64Var(&cfg.Bus.AnnouncementMaxAgeHours, "bus.announcementMaxAgeHours", cfg.Bus.AnnouncementMaxAgeHours, "Max age for announcements") + flag.BoolVar(&cfg.Bus.Bootstrap, "bus.bootstrap", cfg.Bus.Bootstrap, "Bootstraps gateway and consensus modules") + flag.StringVar(&cfg.Bus.GatewayAddr, "bus.gatewayAddr", cfg.Bus.GatewayAddr, "Address for Sia peer connections (overrides with RENTERD_BUS_GATEWAY_ADDR)") + flag.DurationVar(&cfg.Bus.PersistInterval, "bus.persistInterval", cfg.Bus.PersistInterval, "(deprecated) Interval for persisting consensus updates") + flag.DurationVar(&cfg.Bus.UsedUTXOExpiry, "bus.usedUTXOExpiry", cfg.Bus.UsedUTXOExpiry, "Expiry for used UTXOs in transactions") + flag.Int64Var(&cfg.Bus.SlabBufferCompletionThreshold, "bus.slabBufferCompletionThreshold", cfg.Bus.SlabBufferCompletionThreshold, "Threshold for slab buffer upload (overrides with RENTERD_BUS_SLAB_BUFFER_COMPLETION_THRESHOLD)") + + // worker + flag.BoolVar(&cfg.Worker.AllowPrivateIPs, "worker.allowPrivateIPs", cfg.Worker.AllowPrivateIPs, "Allows hosts with private IPs") + flag.DurationVar(&cfg.Worker.BusFlushInterval, "worker.busFlushInterval", cfg.Worker.BusFlushInterval, "Interval for flushing data to bus") + flag.Uint64Var(&cfg.Worker.DownloadMaxMemory, "worker.downloadMaxMemory", cfg.Worker.DownloadMaxMemory, "Max amount of RAM the worker allocates for slabs when downloading (overrides with RENTERD_WORKER_DOWNLOAD_MAX_MEMORY)") + flag.Uint64Var(&cfg.Worker.DownloadMaxOverdrive, "worker.downloadMaxOverdrive", cfg.Worker.DownloadMaxOverdrive, "Max overdrive workers for downloads") + flag.StringVar(&cfg.Worker.ID, "worker.id", cfg.Worker.ID, "Unique ID for worker (overrides with RENTERD_WORKER_ID)") + flag.DurationVar(&cfg.Worker.DownloadOverdriveTimeout, "worker.downloadOverdriveTimeout", cfg.Worker.DownloadOverdriveTimeout, "Timeout for overdriving slab downloads") + flag.Uint64Var(&cfg.Worker.UploadMaxMemory, "worker.uploadMaxMemory", cfg.Worker.UploadMaxMemory, "Max amount of RAM the worker allocates for slabs when uploading (overrides with RENTERD_WORKER_UPLOAD_MAX_MEMORY)") + flag.Uint64Var(&cfg.Worker.UploadMaxOverdrive, "worker.uploadMaxOverdrive", cfg.Worker.UploadMaxOverdrive, "Max overdrive workers for uploads") + flag.DurationVar(&cfg.Worker.UploadOverdriveTimeout, "worker.uploadOverdriveTimeout", cfg.Worker.UploadOverdriveTimeout, "Timeout for overdriving slab uploads") + flag.BoolVar(&cfg.Worker.Enabled, "worker.enabled", cfg.Worker.Enabled, "Enables/disables worker (overrides with RENTERD_WORKER_ENABLED)") + flag.BoolVar(&cfg.Worker.AllowUnauthenticatedDownloads, "worker.unauthenticatedDownloads", cfg.Worker.AllowUnauthenticatedDownloads, "Allows unauthenticated downloads (overrides with RENTERD_WORKER_UNAUTHENTICATED_DOWNLOADS)") + flag.StringVar(&cfg.Worker.ExternalAddress, "worker.externalAddress", cfg.Worker.ExternalAddress, "Address of the worker on the network, only necessary when the bus is remote (overrides with RENTERD_WORKER_EXTERNAL_ADDR)") + + // autopilot + flag.DurationVar(&cfg.Autopilot.AccountsRefillInterval, "autopilot.accountRefillInterval", cfg.Autopilot.AccountsRefillInterval, "Interval for refilling workers' account balances") + flag.DurationVar(&cfg.Autopilot.Heartbeat, "autopilot.heartbeat", cfg.Autopilot.Heartbeat, "Interval for autopilot loop execution") + flag.Float64Var(&cfg.Autopilot.MigrationHealthCutoff, "autopilot.migrationHealthCutoff", cfg.Autopilot.MigrationHealthCutoff, "Threshold for migrating slabs based on health") + flag.DurationVar(&cfg.Autopilot.RevisionBroadcastInterval, "autopilot.revisionBroadcastInterval", cfg.Autopilot.RevisionBroadcastInterval, "Interval for broadcasting contract revisions (overrides with RENTERD_AUTOPILOT_REVISION_BROADCAST_INTERVAL)") + flag.Uint64Var(&cfg.Autopilot.ScannerBatchSize, "autopilot.scannerBatchSize", cfg.Autopilot.ScannerBatchSize, "Batch size for host scanning") + flag.DurationVar(&cfg.Autopilot.ScannerInterval, "autopilot.scannerInterval", cfg.Autopilot.ScannerInterval, "Interval for scanning hosts") + flag.Uint64Var(&cfg.Autopilot.ScannerNumThreads, "autopilot.scannerNumThreads", cfg.Autopilot.ScannerNumThreads, "Number of threads for scanning hosts") + flag.Uint64Var(&cfg.Autopilot.MigratorParallelSlabsPerWorker, "autopilot.migratorParallelSlabsPerWorker", cfg.Autopilot.MigratorParallelSlabsPerWorker, "Parallel slab migrations per worker (overrides with RENTERD_MIGRATOR_PARALLEL_SLABS_PER_WORKER)") + flag.BoolVar(&cfg.Autopilot.Enabled, "autopilot.enabled", cfg.Autopilot.Enabled, "Enables/disables autopilot (overrides with RENTERD_AUTOPILOT_ENABLED)") + flag.DurationVar(&cfg.ShutdownTimeout, "node.shutdownTimeout", cfg.ShutdownTimeout, "Timeout for node shutdown") + + // s3 + var hostBasesStr string + flag.StringVar(&cfg.S3.Address, "s3.address", cfg.S3.Address, "Address for serving S3 API (overrides with RENTERD_S3_ADDRESS)") + flag.BoolVar(&cfg.S3.DisableAuth, "s3.disableAuth", cfg.S3.DisableAuth, "Disables authentication for S3 API (overrides with RENTERD_S3_DISABLE_AUTH)") + flag.BoolVar(&cfg.S3.Enabled, "s3.enabled", cfg.S3.Enabled, "Enables/disables S3 API (requires worker.enabled to be 'true', overrides with RENTERD_S3_ENABLED)") + flag.StringVar(&hostBasesStr, "s3.hostBases", "", "Enables bucket rewriting in the router for specific hosts provided via comma-separated list (overrides with RENTERD_S3_HOST_BUCKET_BASES)") + flag.BoolVar(&cfg.S3.HostBucketEnabled, "s3.hostBucketEnabled", cfg.S3.HostBucketEnabled, "Enables bucket rewriting in the router for all hosts (overrides with RENTERD_S3_HOST_BUCKET_ENABLED)") + + // combine host bucket bases + for _, base := range strings.Split(hostBasesStr, ",") { + if trimmed := strings.TrimSpace(base); trimmed != "" { + cfg.S3.HostBucketBases = append(cfg.S3.HostBucketBases, base) + } + } + + flag.Parse() +} + +func parseEnvironmentVariables(cfg *config.Config) { + parseEnvVar("RENTERD_NETWORK", &cfg.Network) + + parseEnvVar("RENTERD_BUS_REMOTE_ADDR", &cfg.Bus.RemoteAddr) + parseEnvVar("RENTERD_BUS_API_PASSWORD", &cfg.Bus.RemotePassword) + parseEnvVar("RENTERD_BUS_GATEWAY_ADDR", &cfg.Bus.GatewayAddr) + parseEnvVar("RENTERD_BUS_SLAB_BUFFER_COMPLETION_THRESHOLD", &cfg.Bus.SlabBufferCompletionThreshold) + + parseEnvVar("RENTERD_DB_URI", &cfg.Database.MySQL.URI) + parseEnvVar("RENTERD_DB_USER", &cfg.Database.MySQL.User) + parseEnvVar("RENTERD_DB_PASSWORD", &cfg.Database.MySQL.Password) + parseEnvVar("RENTERD_DB_NAME", &cfg.Database.MySQL.Database) + parseEnvVar("RENTERD_DB_METRICS_NAME", &cfg.Database.MySQL.MetricsDatabase) + + parseEnvVar("RENTERD_DB_LOGGER_IGNORE_NOT_FOUND_ERROR", &cfg.Database.Log.IgnoreRecordNotFoundError) + parseEnvVar("RENTERD_DB_LOGGER_LOG_LEVEL", &cfg.Log.Level) + parseEnvVar("RENTERD_DB_LOGGER_SLOW_THRESHOLD", &cfg.Database.Log.SlowThreshold) + + parseEnvVar("RENTERD_WORKER_ENABLED", &cfg.Worker.Enabled) + parseEnvVar("RENTERD_WORKER_ID", &cfg.Worker.ID) + parseEnvVar("RENTERD_WORKER_UNAUTHENTICATED_DOWNLOADS", &cfg.Worker.AllowUnauthenticatedDownloads) + parseEnvVar("RENTERD_WORKER_DOWNLOAD_MAX_MEMORY", &cfg.Worker.DownloadMaxMemory) + parseEnvVar("RENTERD_WORKER_UPLOAD_MAX_MEMORY", &cfg.Worker.UploadMaxMemory) + parseEnvVar("RENTERD_WORKER_EXTERNAL_ADDR", &cfg.Worker.ExternalAddress) + + parseEnvVar("RENTERD_AUTOPILOT_ENABLED", &cfg.Autopilot.Enabled) + parseEnvVar("RENTERD_AUTOPILOT_REVISION_BROADCAST_INTERVAL", &cfg.Autopilot.RevisionBroadcastInterval) + parseEnvVar("RENTERD_MIGRATOR_PARALLEL_SLABS_PER_WORKER", &cfg.Autopilot.MigratorParallelSlabsPerWorker) + + parseEnvVar("RENTERD_S3_ADDRESS", &cfg.S3.Address) + parseEnvVar("RENTERD_S3_ENABLED", &cfg.S3.Enabled) + parseEnvVar("RENTERD_S3_DISABLE_AUTH", &cfg.S3.DisableAuth) + parseEnvVar("RENTERD_S3_HOST_BUCKET_ENABLED", &cfg.S3.HostBucketEnabled) + parseEnvVar("RENTERD_S3_HOST_BUCKET_BASES", &cfg.S3.HostBucketBases) + + parseEnvVar("RENTERD_LOG_PATH", &cfg.Log.Path) + parseEnvVar("RENTERD_LOG_LEVEL", &cfg.Log.Level) + parseEnvVar("RENTERD_LOG_FILE_ENABLED", &cfg.Log.File.Enabled) + parseEnvVar("RENTERD_LOG_FILE_FORMAT", &cfg.Log.File.Format) + parseEnvVar("RENTERD_LOG_FILE_PATH", &cfg.Log.File.Path) + parseEnvVar("RENTERD_LOG_STDOUT_ENABLED", &cfg.Log.StdOut.Enabled) + parseEnvVar("RENTERD_LOG_STDOUT_FORMAT", &cfg.Log.StdOut.Format) + parseEnvVar("RENTERD_LOG_STDOUT_ENABLE_ANSI", &cfg.Log.StdOut.EnableANSI) + parseEnvVar("RENTERD_LOG_DATABASE_ENABLED", &cfg.Log.Database.Enabled) + parseEnvVar("RENTERD_LOG_DATABASE_LEVEL", &cfg.Log.Database.Level) + parseEnvVar("RENTERD_LOG_DATABASE_IGNORE_RECORD_NOT_FOUND_ERROR", &cfg.Log.Database.IgnoreRecordNotFoundError) + parseEnvVar("RENTERD_LOG_DATABASE_SLOW_THRESHOLD", &cfg.Log.Database.SlowThreshold) + + // parse remotes, we duplicate the old behavior of all workers sharing the + // same password + var workerRemotePassStr string + var workerRemoteAddrsStr string + parseEnvVar("RENTERD_WORKER_REMOTE_ADDRS", &workerRemoteAddrsStr) + parseEnvVar("RENTERD_WORKER_API_PASSWORD", &workerRemotePassStr) + if workerRemoteAddrsStr != "" && workerRemotePassStr != "" { + cfg.Worker.Remotes = cfg.Worker.Remotes[:0] + for _, addr := range strings.Split(workerRemoteAddrsStr, ";") { + cfg.Worker.Remotes = append(cfg.Worker.Remotes, config.RemoteWorker{ + Address: addr, + Password: workerRemotePassStr, + }) + } + } + + // disable worker if remotes are set + if len(cfg.Worker.Remotes) > 0 { + cfg.Worker.Enabled = false + } + + // parse S3 auth keys + if cfg.S3.Enabled { + var keyPairsV4 string + parseEnvVar("RENTERD_S3_KEYPAIRS_V4", &keyPairsV4) + if !cfg.S3.DisableAuth && keyPairsV4 != "" { + var err error + cfg.S3.KeypairsV4, err = s3.Parsev4AuthKeys(strings.Split(keyPairsV4, ";")) + if err != nil { + log.Fatalf("failed to parse keypairs: %v", err) + } + } + } +} + +func parseEnvVar(s string, v interface{}) { + if env, ok := os.LookupEnv(s); ok { + if _, err := fmt.Sscan(env, v); err != nil { + log.Fatalf("failed to parse %s: %v", s, err) + } + fmt.Printf("Using %s environment variable\n", s) + } +} // readPasswordInput reads a password from stdin. func readPasswordInput(context string) string { @@ -147,7 +491,7 @@ func setListenAddress(context string, value *string, allowEmpty bool) { // setSeedPhrase prompts the user to enter a seed phrase if one is not already // set via environment variable or config file. -func setSeedPhrase() { +func setSeedPhrase(cfg *config.Config) { // retry until a valid seed phrase is entered for { fmt.Println("") @@ -203,7 +547,7 @@ func setSeedPhrase() { // setAPIPassword prompts the user to enter an API password if one is not // already set via environment variable or config file. -func setAPIPassword() { +func setAPIPassword(cfg *config.Config) { // return early if the password is already set if len(cfg.HTTP.Password) >= 4 { return @@ -224,7 +568,7 @@ func setAPIPassword() { } } -func setAdvancedConfig() { +func setAdvancedConfig(cfg *config.Config) { if !promptYesNo("Would you like to configure advanced settings?") { return } @@ -251,10 +595,10 @@ func setAdvancedConfig() { fmt.Println("The database is used to store the renter's metadata.") fmt.Println("The embedded SQLite database requires no additional configuration and is ideal for testing or demo purposes.") fmt.Println("For production usage, we recommend MySQL, which requires a separate MySQL server.") - setStoreConfig() + setStoreConfig(cfg) } -func setStoreConfig() { +func setStoreConfig(cfg *config.Config) { store := promptQuestion("Which data store would you like to use?", []string{"mysql", "sqlite"}) switch store { case "mysql": @@ -278,7 +622,7 @@ func setStoreConfig() { } } -func setS3Config() { +func setS3Config(cfg *config.Config) { if !promptYesNo("Would you like to configure S3 settings?") { return } diff --git a/cmd/renterd/logger.go b/cmd/renterd/logger.go index 4b21a1925..d107cc4a0 100644 --- a/cmd/renterd/logger.go +++ b/cmd/renterd/logger.go @@ -11,11 +11,11 @@ import ( "go.uber.org/zap/zapcore" ) -func NewLogger(dir string, cfg config.Log) (*zap.Logger, func(context.Context) error, error) { +func NewLogger(dir, filename string, cfg config.Log) (*zap.Logger, func(context.Context) error, error) { // path - path := filepath.Join(dir, "renterd.log") + path := filepath.Join(dir, filename) if cfg.Path != "" { - path = filepath.Join(cfg.Path, "renterd.log") + path = filepath.Join(cfg.Path, filename) } if cfg.File.Path != "" { diff --git a/cmd/renterd/main.go b/cmd/renterd/main.go index f1b1c212b..c3df11de2 100644 --- a/cmd/renterd/main.go +++ b/cmd/renterd/main.go @@ -1,55 +1,18 @@ package main import ( - "context" - "encoding/json" - "errors" "flag" - "fmt" "log" - "net" - "net/http" "os" - "os/exec" "os/signal" - "path/filepath" - "runtime" - "strings" "syscall" - "time" "go.sia.tech/core/consensus" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" - "go.sia.tech/coreutils/wallet" - "go.sia.tech/jape" - "go.sia.tech/renterd/api" - "go.sia.tech/renterd/autopilot" - "go.sia.tech/renterd/build" - "go.sia.tech/renterd/bus" - "go.sia.tech/renterd/config" - "go.sia.tech/renterd/internal/node" - "go.sia.tech/renterd/internal/utils" - iworker "go.sia.tech/renterd/internal/worker" - "go.sia.tech/renterd/worker" - "go.sia.tech/renterd/worker/s3" - "go.sia.tech/web/renterd" - "go.uber.org/zap" - "golang.org/x/sys/cpu" - "gopkg.in/yaml.v3" ) const ( - // accountRefillInterval is the amount of time between refills of ephemeral - // accounts. If we conservatively assume that a good host charges 500 SC / - // TiB, we can pay for about 2.2 GiB with 1 SC. Since we want to refill - // ahead of time at 0.5 SC, that makes 1.1 GiB. Considering a 1 Gbps uplink - // that is shared across 30 uploads, we upload at around 33 Mbps to each - // host. That means uploading 1.1 GiB to drain 0.5 SC takes around 5 - // minutes. That's why we assume 10 seconds to be more than frequent enough - // to refill an account when it's due for another refill. - defaultAccountRefillInterval = 10 * time.Second - // usageHeader is the header for the CLI usage text. usageHeader = ` Renterd is the official Sia renter daemon. It provides a REST API for forming @@ -74,296 +37,20 @@ on how to configure and use renterd. ` ) -var ( - cfg = config.Config{ - Directory: ".", - Seed: os.Getenv("RENTERD_SEED"), - AutoOpenWebUI: true, - Network: "mainnet", - HTTP: config.HTTP{ - Address: "localhost:9980", - Password: os.Getenv("RENTERD_API_PASSWORD"), - }, - ShutdownTimeout: 5 * time.Minute, - Database: config.Database{ - MySQL: config.MySQL{ - User: "renterd", - Database: "renterd", - MetricsDatabase: "renterd_metrics", - }, - }, - Log: config.Log{ - Path: "", // deprecated. included for compatibility. - Level: "", - File: config.LogFile{ - Enabled: true, - Format: "json", - Path: os.Getenv("RENTERD_LOG_FILE"), - }, - StdOut: config.StdOut{ - Enabled: true, - Format: "human", - EnableANSI: runtime.GOOS != "windows", - }, - Database: config.DatabaseLog{ - Enabled: true, - IgnoreRecordNotFoundError: true, - SlowThreshold: 100 * time.Millisecond, - }, - }, - Bus: config.Bus{ - AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year - Bootstrap: true, - GatewayAddr: ":9981", - UsedUTXOExpiry: 24 * time.Hour, - SlabBufferCompletionThreshold: 1 << 12, - }, - Worker: config.Worker{ - Enabled: true, - - ID: "worker", - ContractLockTimeout: 30 * time.Second, - BusFlushInterval: 5 * time.Second, - - DownloadMaxOverdrive: 5, - DownloadOverdriveTimeout: 3 * time.Second, - - DownloadMaxMemory: 1 << 30, // 1 GiB - UploadMaxMemory: 1 << 30, // 1 GiB - UploadMaxOverdrive: 5, - UploadOverdriveTimeout: 3 * time.Second, - }, - Autopilot: config.Autopilot{ - Enabled: true, - RevisionSubmissionBuffer: 150, // 144 + 6 blocks leeway - AccountsRefillInterval: defaultAccountRefillInterval, - Heartbeat: 30 * time.Minute, - MigrationHealthCutoff: 0.75, - RevisionBroadcastInterval: 7 * 24 * time.Hour, - ScannerBatchSize: 100, - ScannerInterval: 4 * time.Hour, - ScannerNumThreads: 10, - MigratorParallelSlabsPerWorker: 1, - }, - S3: config.S3{ - Address: "localhost:8080", - Enabled: true, - DisableAuth: false, - KeypairsV4: nil, - }, - } - disableStdin bool -) - -func mustParseWorkers(workers, password string) { - if workers == "" { - return - } - // if the CLI flag/environment variable is set, overwrite the config file - cfg.Worker.Remotes = cfg.Worker.Remotes[:0] - for _, addr := range strings.Split(workers, ";") { - // note: duplicates the old behavior of all workers sharing the same - // password - cfg.Worker.Remotes = append(cfg.Worker.Remotes, config.RemoteWorker{ - Address: addr, - Password: password, - }) - } -} - -// tryLoadConfig loads the config file specified by the RENTERD_CONFIG_FILE -// environment variable. If the config file does not exist, it will not be -// loaded. -func tryLoadConfig() { - configPath := "renterd.yml" - if str := os.Getenv("RENTERD_CONFIG_FILE"); str != "" { - configPath = str - } - - // If the config file doesn't exist, don't try to load it. - if _, err := os.Stat(configPath); os.IsNotExist(err) { - return - } - - f, err := os.Open(configPath) - if err != nil { - log.Fatal("failed to open config file:", err) - } - defer f.Close() - - dec := yaml.NewDecoder(f) - dec.KnownFields(true) - - if err := dec.Decode(&cfg); err != nil { - log.Fatal("failed to decode config file:", err) - } -} - -func parseEnvVar(s string, v interface{}) { - if env, ok := os.LookupEnv(s); ok { - if _, err := fmt.Sscan(env, v); err != nil { - log.Fatalf("failed to parse %s: %v", s, err) - } - fmt.Printf("Using %s environment variable\n", s) - } -} - -func listenTCP(logger *zap.Logger, addr string) (net.Listener, error) { - l, err := net.Listen("tcp", addr) - if utils.IsErr(err, errors.New("no such host")) && strings.Contains(addr, "localhost") { - // fall back to 127.0.0.1 if 'localhost' doesn't work - _, port, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - fallbackAddr := fmt.Sprintf("127.0.0.1:%s", port) - logger.Sugar().Warnf("failed to listen on %s, falling back to %s", addr, fallbackAddr) - return net.Listen("tcp", fallbackAddr) - } else if err != nil { - return nil, err - } - return l, nil -} - func main() { log.SetFlags(0) - // load the YAML config first. CLI flags and environment variables will - // overwrite anything set in the config file. - tryLoadConfig() - - // deprecated - these go first so that they can be overwritten by the non-deprecated flags - flag.StringVar(&cfg.Log.Database.Level, "db.logger.logLevel", cfg.Log.Database.Level, "(deprecated) Logger level (overrides with RENTERD_DB_LOGGER_LOG_LEVEL)") - flag.BoolVar(&cfg.Database.Log.IgnoreRecordNotFoundError, "db.logger.ignoreNotFoundError", cfg.Database.Log.IgnoreRecordNotFoundError, "(deprecated) Ignores 'not found' errors in logger (overrides with RENTERD_DB_LOGGER_IGNORE_NOT_FOUND_ERROR)") - flag.DurationVar(&cfg.Database.Log.SlowThreshold, "db.logger.slowThreshold", cfg.Database.Log.SlowThreshold, "(deprecated) Threshold for slow queries in logger (overrides with RENTERD_DB_LOGGER_SLOW_THRESHOLD)") - flag.StringVar(&cfg.Log.Path, "log-path", cfg.Log.Path, "(deprecated) Path to directory for logs (overrides with RENTERD_LOG_PATH)") - - // node - flag.StringVar(&cfg.HTTP.Address, "http", cfg.HTTP.Address, "Address for serving the API") - flag.StringVar(&cfg.Directory, "dir", cfg.Directory, "Directory for storing node state") - flag.BoolVar(&disableStdin, "env", false, "disable stdin prompts for environment variables (default false)") - flag.BoolVar(&cfg.AutoOpenWebUI, "openui", cfg.AutoOpenWebUI, "automatically open the web UI on startup") - flag.StringVar(&cfg.Network, "network", cfg.Network, "Network to connect to (mainnet|zen|anagami). Defaults to 'mainnet' (overrides with RENTERD_NETWORK)") - - // logger - flag.StringVar(&cfg.Log.Level, "log.level", cfg.Log.Level, "Global logger level (debug|info|warn|error). Defaults to 'info' (overrides with RENTERD_LOG_LEVEL)") - flag.BoolVar(&cfg.Log.File.Enabled, "log.file.enabled", cfg.Log.File.Enabled, "Enables logging to disk. Defaults to 'true'. (overrides with RENTERD_LOG_FILE_ENABLED)") - flag.StringVar(&cfg.Log.File.Format, "log.file.format", cfg.Log.File.Format, "Format of log file (json|human). Defaults to 'json' (overrides with RENTERD_LOG_FILE_FORMAT)") - flag.StringVar(&cfg.Log.File.Path, "log.file.path", cfg.Log.File.Path, "Path of log file. Defaults to 'renterd.log' within the renterd directory. (overrides with RENTERD_LOG_FILE_PATH)") - flag.BoolVar(&cfg.Log.StdOut.Enabled, "log.stdout.enabled", cfg.Log.StdOut.Enabled, "Enables logging to stdout. Defaults to 'true'. (overrides with RENTERD_LOG_STDOUT_ENABLED)") - flag.StringVar(&cfg.Log.StdOut.Format, "log.stdout.format", cfg.Log.StdOut.Format, "Format of log output (json|human). Defaults to 'human' (overrides with RENTERD_LOG_STDOUT_FORMAT)") - flag.BoolVar(&cfg.Log.StdOut.EnableANSI, "log.stdout.enableANSI", cfg.Log.StdOut.EnableANSI, "Enables ANSI color codes in log output. Defaults to 'true' on non-Windows systems. (overrides with RENTERD_LOG_STDOUT_ENABLE_ANSI)") - flag.BoolVar(&cfg.Log.Database.Enabled, "log.database.enabled", cfg.Log.Database.Enabled, "Enable logging database queries. Defaults to 'true' (overrides with RENTERD_LOG_DATABASE_ENABLED)") - flag.StringVar(&cfg.Log.Database.Level, "log.database.level", cfg.Log.Database.Level, "Logger level for database queries (info|warn|error). Defaults to 'warn' (overrides with RENTERD_LOG_LEVEL and RENTERD_LOG_DATABASE_LEVEL)") - flag.BoolVar(&cfg.Log.Database.IgnoreRecordNotFoundError, "log.database.ignoreRecordNotFoundError", cfg.Log.Database.IgnoreRecordNotFoundError, "Enable ignoring 'not found' errors resulting from database queries. Defaults to 'true' (overrides with RENTERD_LOG_DATABASE_IGNORE_RECORD_NOT_FOUND_ERROR)") - flag.DurationVar(&cfg.Log.Database.SlowThreshold, "log.database.slowThreshold", cfg.Log.Database.SlowThreshold, "Threshold for slow queries in logger. Defaults to 100ms (overrides with RENTERD_LOG_DATABASE_SLOW_THRESHOLD)") - - // db - flag.StringVar(&cfg.Database.MySQL.URI, "db.uri", cfg.Database.MySQL.URI, "Database URI for the bus (overrides with RENTERD_DB_URI)") - flag.StringVar(&cfg.Database.MySQL.User, "db.user", cfg.Database.MySQL.User, "Database username for the bus (overrides with RENTERD_DB_USER)") - flag.StringVar(&cfg.Database.MySQL.Database, "db.name", cfg.Database.MySQL.Database, "Database name for the bus (overrides with RENTERD_DB_NAME)") - flag.StringVar(&cfg.Database.MySQL.MetricsDatabase, "db.metricsName", cfg.Database.MySQL.MetricsDatabase, "Database for metrics (overrides with RENTERD_DB_METRICS_NAME)") - - // bus - flag.Uint64Var(&cfg.Bus.AnnouncementMaxAgeHours, "bus.announcementMaxAgeHours", cfg.Bus.AnnouncementMaxAgeHours, "Max age for announcements") - flag.BoolVar(&cfg.Bus.Bootstrap, "bus.bootstrap", cfg.Bus.Bootstrap, "Bootstraps gateway and consensus modules") - flag.StringVar(&cfg.Bus.GatewayAddr, "bus.gatewayAddr", cfg.Bus.GatewayAddr, "Address for Sia peer connections (overrides with RENTERD_BUS_GATEWAY_ADDR)") - flag.DurationVar(&cfg.Bus.PersistInterval, "bus.persistInterval", cfg.Bus.PersistInterval, "(deprecated) Interval for persisting consensus updates") - flag.DurationVar(&cfg.Bus.UsedUTXOExpiry, "bus.usedUTXOExpiry", cfg.Bus.UsedUTXOExpiry, "Expiry for used UTXOs in transactions") - flag.Int64Var(&cfg.Bus.SlabBufferCompletionThreshold, "bus.slabBufferCompletionThreshold", cfg.Bus.SlabBufferCompletionThreshold, "Threshold for slab buffer upload (overrides with RENTERD_BUS_SLAB_BUFFER_COMPLETION_THRESHOLD)") - - // worker - flag.BoolVar(&cfg.Worker.AllowPrivateIPs, "worker.allowPrivateIPs", cfg.Worker.AllowPrivateIPs, "Allows hosts with private IPs") - flag.DurationVar(&cfg.Worker.BusFlushInterval, "worker.busFlushInterval", cfg.Worker.BusFlushInterval, "Interval for flushing data to bus") - flag.Uint64Var(&cfg.Worker.DownloadMaxMemory, "worker.downloadMaxMemory", cfg.Worker.DownloadMaxMemory, "Max amount of RAM the worker allocates for slabs when downloading (overrides with RENTERD_WORKER_DOWNLOAD_MAX_MEMORY)") - flag.Uint64Var(&cfg.Worker.DownloadMaxOverdrive, "worker.downloadMaxOverdrive", cfg.Worker.DownloadMaxOverdrive, "Max overdrive workers for downloads") - flag.StringVar(&cfg.Worker.ID, "worker.id", cfg.Worker.ID, "Unique ID for worker (overrides with RENTERD_WORKER_ID)") - flag.DurationVar(&cfg.Worker.DownloadOverdriveTimeout, "worker.downloadOverdriveTimeout", cfg.Worker.DownloadOverdriveTimeout, "Timeout for overdriving slab downloads") - flag.Uint64Var(&cfg.Worker.UploadMaxMemory, "worker.uploadMaxMemory", cfg.Worker.UploadMaxMemory, "Max amount of RAM the worker allocates for slabs when uploading (overrides with RENTERD_WORKER_UPLOAD_MAX_MEMORY)") - flag.Uint64Var(&cfg.Worker.UploadMaxOverdrive, "worker.uploadMaxOverdrive", cfg.Worker.UploadMaxOverdrive, "Max overdrive workers for uploads") - flag.DurationVar(&cfg.Worker.UploadOverdriveTimeout, "worker.uploadOverdriveTimeout", cfg.Worker.UploadOverdriveTimeout, "Timeout for overdriving slab uploads") - flag.BoolVar(&cfg.Worker.Enabled, "worker.enabled", cfg.Worker.Enabled, "Enables/disables worker (overrides with RENTERD_WORKER_ENABLED)") - flag.BoolVar(&cfg.Worker.AllowUnauthenticatedDownloads, "worker.unauthenticatedDownloads", cfg.Worker.AllowUnauthenticatedDownloads, "Allows unauthenticated downloads (overrides with RENTERD_WORKER_UNAUTHENTICATED_DOWNLOADS)") - flag.StringVar(&cfg.Worker.ExternalAddress, "worker.externalAddress", cfg.Worker.ExternalAddress, "Address of the worker on the network, only necessary when the bus is remote (overrides with RENTERD_WORKER_EXTERNAL_ADDR)") - - // autopilot - flag.DurationVar(&cfg.Autopilot.AccountsRefillInterval, "autopilot.accountRefillInterval", cfg.Autopilot.AccountsRefillInterval, "Interval for refilling workers' account balances") - flag.DurationVar(&cfg.Autopilot.Heartbeat, "autopilot.heartbeat", cfg.Autopilot.Heartbeat, "Interval for autopilot loop execution") - flag.Float64Var(&cfg.Autopilot.MigrationHealthCutoff, "autopilot.migrationHealthCutoff", cfg.Autopilot.MigrationHealthCutoff, "Threshold for migrating slabs based on health") - flag.DurationVar(&cfg.Autopilot.RevisionBroadcastInterval, "autopilot.revisionBroadcastInterval", cfg.Autopilot.RevisionBroadcastInterval, "Interval for broadcasting contract revisions (overrides with RENTERD_AUTOPILOT_REVISION_BROADCAST_INTERVAL)") - flag.Uint64Var(&cfg.Autopilot.ScannerBatchSize, "autopilot.scannerBatchSize", cfg.Autopilot.ScannerBatchSize, "Batch size for host scanning") - flag.DurationVar(&cfg.Autopilot.ScannerInterval, "autopilot.scannerInterval", cfg.Autopilot.ScannerInterval, "Interval for scanning hosts") - flag.Uint64Var(&cfg.Autopilot.ScannerNumThreads, "autopilot.scannerNumThreads", cfg.Autopilot.ScannerNumThreads, "Number of threads for scanning hosts") - flag.Uint64Var(&cfg.Autopilot.MigratorParallelSlabsPerWorker, "autopilot.migratorParallelSlabsPerWorker", cfg.Autopilot.MigratorParallelSlabsPerWorker, "Parallel slab migrations per worker (overrides with RENTERD_MIGRATOR_PARALLEL_SLABS_PER_WORKER)") - flag.BoolVar(&cfg.Autopilot.Enabled, "autopilot.enabled", cfg.Autopilot.Enabled, "Enables/disables autopilot (overrides with RENTERD_AUTOPILOT_ENABLED)") - flag.DurationVar(&cfg.ShutdownTimeout, "node.shutdownTimeout", cfg.ShutdownTimeout, "Timeout for node shutdown") - - // s3 - var hostBasesStr string - flag.StringVar(&cfg.S3.Address, "s3.address", cfg.S3.Address, "Address for serving S3 API (overrides with RENTERD_S3_ADDRESS)") - flag.BoolVar(&cfg.S3.DisableAuth, "s3.disableAuth", cfg.S3.DisableAuth, "Disables authentication for S3 API (overrides with RENTERD_S3_DISABLE_AUTH)") - flag.BoolVar(&cfg.S3.Enabled, "s3.enabled", cfg.S3.Enabled, "Enables/disables S3 API (requires worker.enabled to be 'true', overrides with RENTERD_S3_ENABLED)") - flag.StringVar(&hostBasesStr, "s3.hostBases", "", "Enables bucket rewriting in the router for specific hosts provided via comma-separated list (overrides with RENTERD_S3_HOST_BUCKET_BASES)") - flag.BoolVar(&cfg.S3.HostBucketEnabled, "s3.hostBucketEnabled", cfg.S3.HostBucketEnabled, "Enables bucket rewriting in the router for all hosts (overrides with RENTERD_S3_HOST_BUCKET_ENABLED)") - - // custom usage + // set usage flag.Usage = func() { log.Print(usageHeader) flag.PrintDefaults() log.Print(usageFooter) } - flag.Parse() - - // Overwrite flags from environment if set. - parseEnvVar("RENTERD_NETWORK", &cfg.Network) - - parseEnvVar("RENTERD_BUS_REMOTE_ADDR", &cfg.Bus.RemoteAddr) - parseEnvVar("RENTERD_BUS_API_PASSWORD", &cfg.Bus.RemotePassword) - parseEnvVar("RENTERD_BUS_GATEWAY_ADDR", &cfg.Bus.GatewayAddr) - parseEnvVar("RENTERD_BUS_SLAB_BUFFER_COMPLETION_THRESHOLD", &cfg.Bus.SlabBufferCompletionThreshold) - - parseEnvVar("RENTERD_DB_URI", &cfg.Database.MySQL.URI) - parseEnvVar("RENTERD_DB_USER", &cfg.Database.MySQL.User) - parseEnvVar("RENTERD_DB_PASSWORD", &cfg.Database.MySQL.Password) - parseEnvVar("RENTERD_DB_NAME", &cfg.Database.MySQL.Database) - parseEnvVar("RENTERD_DB_METRICS_NAME", &cfg.Database.MySQL.MetricsDatabase) - - parseEnvVar("RENTERD_DB_LOGGER_IGNORE_NOT_FOUND_ERROR", &cfg.Database.Log.IgnoreRecordNotFoundError) - parseEnvVar("RENTERD_DB_LOGGER_LOG_LEVEL", &cfg.Log.Level) - parseEnvVar("RENTERD_DB_LOGGER_SLOW_THRESHOLD", &cfg.Database.Log.SlowThreshold) + // load the config + cfg := loadConfig() - parseEnvVar("RENTERD_WORKER_ENABLED", &cfg.Worker.Enabled) - parseEnvVar("RENTERD_WORKER_ID", &cfg.Worker.ID) - parseEnvVar("RENTERD_WORKER_UNAUTHENTICATED_DOWNLOADS", &cfg.Worker.AllowUnauthenticatedDownloads) - parseEnvVar("RENTERD_WORKER_DOWNLOAD_MAX_MEMORY", &cfg.Worker.DownloadMaxMemory) - parseEnvVar("RENTERD_WORKER_UPLOAD_MAX_MEMORY", &cfg.Worker.UploadMaxMemory) - parseEnvVar("RENTERD_WORKER_EXTERNAL_ADDR", &cfg.Worker.ExternalAddress) - - parseEnvVar("RENTERD_AUTOPILOT_ENABLED", &cfg.Autopilot.Enabled) - parseEnvVar("RENTERD_AUTOPILOT_REVISION_BROADCAST_INTERVAL", &cfg.Autopilot.RevisionBroadcastInterval) - parseEnvVar("RENTERD_MIGRATOR_PARALLEL_SLABS_PER_WORKER", &cfg.Autopilot.MigratorParallelSlabsPerWorker) - - parseEnvVar("RENTERD_S3_ADDRESS", &cfg.S3.Address) - parseEnvVar("RENTERD_S3_ENABLED", &cfg.S3.Enabled) - parseEnvVar("RENTERD_S3_DISABLE_AUTH", &cfg.S3.DisableAuth) - parseEnvVar("RENTERD_S3_HOST_BUCKET_ENABLED", &cfg.S3.HostBucketEnabled) - parseEnvVar("RENTERD_S3_HOST_BUCKET_BASES", &cfg.S3.HostBucketBases) - - parseEnvVar("RENTERD_LOG_PATH", &cfg.Log.Path) - parseEnvVar("RENTERD_LOG_LEVEL", &cfg.Log.Level) - parseEnvVar("RENTERD_LOG_FILE_ENABLED", &cfg.Log.File.Enabled) - parseEnvVar("RENTERD_LOG_FILE_FORMAT", &cfg.Log.File.Format) - parseEnvVar("RENTERD_LOG_FILE_PATH", &cfg.Log.File.Path) - parseEnvVar("RENTERD_LOG_STDOUT_ENABLED", &cfg.Log.StdOut.Enabled) - parseEnvVar("RENTERD_LOG_STDOUT_FORMAT", &cfg.Log.StdOut.Format) - parseEnvVar("RENTERD_LOG_STDOUT_ENABLE_ANSI", &cfg.Log.StdOut.EnableANSI) - parseEnvVar("RENTERD_LOG_DATABASE_ENABLED", &cfg.Log.Database.Enabled) - parseEnvVar("RENTERD_LOG_DATABASE_LEVEL", &cfg.Log.Database.Level) - parseEnvVar("RENTERD_LOG_DATABASE_IGNORE_RECORD_NOT_FOUND_ERROR", &cfg.Log.Database.IgnoreRecordNotFoundError) - parseEnvVar("RENTERD_LOG_DATABASE_SLOW_THRESHOLD", &cfg.Log.Database.SlowThreshold) - - // check network + // validate the network var network *consensus.Network var genesis types.Block switch cfg.Network { @@ -385,419 +72,36 @@ func main() { cmdSeed() return } else if flag.Arg(0) == "config" { - cmdBuildConfig() + cmdBuildConfig(&cfg) return } else if flag.Arg(0) != "" { flag.Usage() return } - // parse remotes - var workerRemotePassStr string - var workerRemoteAddrsStr string - parseEnvVar("RENTERD_WORKER_REMOTE_ADDRS", &workerRemoteAddrsStr) - parseEnvVar("RENTERD_WORKER_API_PASSWORD", &workerRemotePassStr) - if workerRemoteAddrsStr != "" && workerRemotePassStr != "" { - mustParseWorkers(workerRemoteAddrsStr, workerRemotePassStr) - } - - // disable worker if remotes are set - if len(cfg.Worker.Remotes) > 0 { - cfg.Worker.Enabled = false - } - - // combine host bucket bases - for _, base := range strings.Split(hostBasesStr, ",") { - if trimmed := strings.TrimSpace(base); trimmed != "" { - cfg.S3.HostBucketBases = append(cfg.S3.HostBucketBases, base) - } - } - - // check that the API password is set - if cfg.HTTP.Password == "" { - if disableStdin { - stdoutFatalError("API password must be set via environment variable or config file when --env flag is set") - return - } - } - setAPIPassword() - - // check that the seed is set - if cfg.Seed == "" && (cfg.Worker.Enabled || cfg.Bus.RemoteAddr == "") { // only worker & bus require a seed - if disableStdin { - stdoutFatalError("Seed must be set via environment variable or config file when --env flag is set") - return - } - setSeedPhrase() - } - - // generate private key from seed - var pk types.PrivateKey - if cfg.Seed != "" { - var rawSeed [32]byte - if err := wallet.SeedFromPhrase(&rawSeed, cfg.Seed); err != nil { - log.Fatal("failed to load wallet", zap.Error(err)) - } - pk = wallet.KeyFromSeed(&rawSeed, 0) - } - - // parse S3 auth keys - if cfg.S3.Enabled { - var keyPairsV4 string - parseEnvVar("RENTERD_S3_KEYPAIRS_V4", &keyPairsV4) - if !cfg.S3.DisableAuth && keyPairsV4 != "" { - var err error - cfg.S3.KeypairsV4, err = s3.Parsev4AuthKeys(strings.Split(keyPairsV4, ";")) - if err != nil { - log.Fatalf("failed to parse keypairs: %v", err) - } - } - } - - // create logger - if cfg.Log.Level == "" { - cfg.Log.Level = "info" // default to 'info' if not set - } - logger, closeFn, err := NewLogger(cfg.Directory, cfg.Log) + // create node + node, err := newNode(cfg, network, genesis) if err != nil { - log.Fatalln("failed to create logger:", err) - } - defer closeFn(context.Background()) - - logger.Info("renterd", zap.String("version", build.Version()), zap.String("network", network.Name), zap.String("commit", build.Commit()), zap.Time("buildDate", build.BuildTime())) - if runtime.GOARCH == "amd64" && !cpu.X86.HasAVX2 { - logger.Warn("renterd is running on a system without AVX2 support, performance may be degraded") - } - - if cfg.Log.Database.Level == "" { - cfg.Log.Database.Level = cfg.Log.Level + log.Fatal("failed to create node: " + err.Error()) } - busCfg := node.BusConfig{ - Bus: cfg.Bus, - Database: cfg.Database, - DatabaseLog: cfg.Log.Database, - Logger: logger, - Network: network, - Genesis: genesis, - RetryTxIntervals: []time.Duration{ - 200 * time.Millisecond, - 500 * time.Millisecond, - time.Second, - 3 * time.Second, - 10 * time.Second, - 10 * time.Second, - }, - } - - type shutdownFnEntry struct { - name string - fn func(context.Context) error - } - var shutdownFns []shutdownFnEntry - - if cfg.Bus.RemoteAddr != "" && !cfg.Worker.Enabled && !cfg.Autopilot.Enabled { - logger.Fatal("remote bus, remote worker, and no autopilot -- nothing to do!") - } - if cfg.Worker.Enabled && cfg.Bus.RemoteAddr != "" && cfg.Worker.ExternalAddress == "" { - logger.Fatal("can't enable the worker using a remote bus, without configuring the worker's external address") - } - if cfg.Autopilot.Enabled && !cfg.Worker.Enabled && len(cfg.Worker.Remotes) == 0 { - logger.Fatal("can't enable autopilot without providing either workers to connect to or creating a worker") - } - - // create listener first, so that we know the actual apiAddr if the user - // specifies port :0 - l, err := listenTCP(logger, cfg.HTTP.Address) - if err != nil { - logger.Fatal("failed to create listener: " + err.Error()) - } - - // override the address with the actual one - cfg.HTTP.Address = "http://" + l.Addr().String() - - auth := jape.BasicAuth(cfg.HTTP.Password) - mux := &utils.TreeMux{ - Sub: make(map[string]utils.TreeMux), - } - - // Create the webserver. - srv := &http.Server{Handler: mux} - shutdownFns = append(shutdownFns, shutdownFnEntry{ - name: "HTTP Server", - fn: srv.Shutdown, - }) - - if err := os.MkdirAll(cfg.Directory, 0700); err != nil { - logger.Fatal("failed to create directory: " + err.Error()) - } - - busAddr, busPassword := cfg.Bus.RemoteAddr, cfg.Bus.RemotePassword - if cfg.Bus.RemoteAddr == "" { - b, shutdownFn, _, err := node.NewBus(busCfg, cfg.Directory, pk, logger) - if err != nil { - logger.Fatal("failed to create bus, err: " + err.Error()) - } - - shutdownFns = append(shutdownFns, shutdownFnEntry{ - name: "Bus", - fn: shutdownFn, - }) - - mux.Sub["/api/bus"] = utils.TreeMux{Handler: auth(b)} - busAddr = cfg.HTTP.Address + "/api/bus" - busPassword = cfg.HTTP.Password - - // only serve the UI if a bus is created - mux.Handler = renterd.Handler() - } else { - logger.Info("connecting to remote bus at " + busAddr) - } - bc := bus.NewClient(busAddr, busPassword) - - var s3Srv *http.Server - var s3Listener net.Listener - var workers []autopilot.Worker - setupWorkerFn := node.NoopFn - if len(cfg.Worker.Remotes) == 0 { - if cfg.Worker.Enabled { - workerAddr := cfg.HTTP.Address + "/api/worker" - var shutdownFn node.ShutdownFn - w, s3Handler, setupFn, shutdownFn, err := node.NewWorker(cfg.Worker, s3.Opts{ - AuthDisabled: cfg.S3.DisableAuth, - HostBucketBases: cfg.S3.HostBucketBases, - HostBucketEnabled: cfg.S3.HostBucketEnabled, - }, bc, pk, logger) - if err != nil { - logger.Fatal("failed to create worker: " + err.Error()) - } - var workerExternAddr string - if cfg.Bus.RemoteAddr != "" { - workerExternAddr = cfg.Worker.ExternalAddress - } else { - workerExternAddr = workerAddr - } - setupWorkerFn = func(ctx context.Context) error { - return setupFn(ctx, workerExternAddr, cfg.HTTP.Password) - } - shutdownFns = append(shutdownFns, shutdownFnEntry{ - name: "Worker", - fn: shutdownFn, - }) - - mux.Sub["/api/worker"] = utils.TreeMux{Handler: iworker.Auth(cfg.HTTP.Password, cfg.Worker.AllowUnauthenticatedDownloads)(w)} - wc := worker.NewClient(workerAddr, cfg.HTTP.Password) - workers = append(workers, wc) - - if cfg.S3.Enabled { - s3Srv = &http.Server{ - Addr: cfg.S3.Address, - Handler: s3Handler, - } - s3Listener, err = listenTCP(logger, cfg.S3.Address) - if err != nil { - logger.Fatal("failed to create listener: " + err.Error()) - } - shutdownFns = append(shutdownFns, shutdownFnEntry{ - name: "S3", - fn: s3Srv.Shutdown, - }) - } - } - } else { - for _, remote := range cfg.Worker.Remotes { - workers = append(workers, worker.NewClient(remote.Address, remote.Password)) - logger.Info("connecting to remote worker at " + remote.Address) - } - } - - autopilotErr := make(chan error, 1) - autopilotDir := filepath.Join(cfg.Directory, api.DefaultAutopilotID) - if cfg.Autopilot.Enabled { - apCfg := node.AutopilotConfig{ - ID: api.DefaultAutopilotID, - Autopilot: cfg.Autopilot, - } - ap, runFn, fn, err := node.NewAutopilot(apCfg, bc, workers, logger) - if err != nil { - logger.Fatal("failed to create autopilot: " + err.Error()) - } - - // NOTE: the autopilot shutdown function needs to be called first. - shutdownFns = append(shutdownFns, shutdownFnEntry{ - name: "Autopilot", - fn: fn, - }) - - go func() { autopilotErr <- runFn() }() - mux.Sub["/api/autopilot"] = utils.TreeMux{Handler: auth(ap)} - } - - // Start server. - go srv.Serve(l) - - // Finish worker setup. - if err := setupWorkerFn(context.Background()); err != nil { - logger.Fatal("failed to setup worker: " + err.Error()) - } - - // Set initial S3 keys. - if cfg.S3.Enabled && !cfg.S3.DisableAuth { - as, err := bc.S3AuthenticationSettings(context.Background()) - if err != nil && !strings.Contains(err.Error(), api.ErrSettingNotFound.Error()) { - logger.Fatal("failed to fetch S3 authentication settings: " + err.Error()) - } else if as.V4Keypairs == nil { - as.V4Keypairs = make(map[string]string) - } - - // S3 key pair validation was broken at one point, we need to remove the - // invalid key pairs here to ensure we don't fail when we update the - // setting below. - for k, v := range as.V4Keypairs { - if err := (api.S3AuthenticationSettings{V4Keypairs: map[string]string{k: v}}).Validate(); err != nil { - logger.Sugar().Infof("removing invalid S3 keypair for AccessKeyID %s, reason: %v", k, err) - delete(as.V4Keypairs, k) - } - } - - // merge keys - for k, v := range cfg.S3.KeypairsV4 { - as.V4Keypairs[k] = v - } - // update settings - if err := bc.UpdateSetting(context.Background(), api.SettingS3Authentication, as); err != nil { - logger.Fatal("failed to update S3 authentication settings: " + err.Error()) - } - } - - logger.Info("api: Listening on " + l.Addr().String()) - - if s3Srv != nil { - go s3Srv.Serve(s3Listener) - logger.Info("s3: Listening on " + s3Listener.Addr().String()) - } - - syncerAddress, err := bc.SyncerAddress(context.Background()) + // start node + err = node.Run() if err != nil { - logger.Fatal("failed to fetch syncer address: " + err.Error()) - } - logger.Info("bus: Listening on " + syncerAddress) - - if cfg.Autopilot.Enabled { - if err := runCompatMigrateAutopilotJSONToStore(bc, "autopilot", autopilotDir); err != nil { - logger.Fatal("failed to migrate autopilot JSON: " + err.Error()) - } - } - - if cfg.AutoOpenWebUI { - time.Sleep(time.Millisecond) // give the web server a chance to start - _, port, err := net.SplitHostPort(l.Addr().String()) - if err != nil { - logger.Debug("failed to parse API address", zap.Error(err)) - } else if err := openBrowser(fmt.Sprintf("http://127.0.0.1:%s", port)); err != nil { - logger.Debug("failed to open browser", zap.Error(err)) - } + log.Fatal("failed to run node: " + err.Error()) } + // wait for interrupt signal signalCh := make(chan os.Signal, 1) signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM) - select { - case <-signalCh: - logger.Info("Shutting down...") - case err := <-autopilotErr: - logger.Fatal("Fatal autopilot error: " + err.Error()) - } - - // Give each service a fraction of the total shutdown timeout. One service - // timing out shouldn't prevent the others from attempting a shutdown. - timeout := cfg.ShutdownTimeout / time.Duration(len(shutdownFns)) - shutdown := func(fn func(ctx context.Context) error) error { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - return fn(ctx) - } + <-signalCh - // Shut down the autopilot first, then the rest of the services in reverse order and then - exitCode := 0 - for i := len(shutdownFns) - 1; i >= 0; i-- { - if err := shutdown(shutdownFns[i].fn); err != nil { - logger.Sugar().Errorf("Failed to shut down %v: %v", shutdownFns[i].name, err) - exitCode = 1 - } else { - logger.Sugar().Infof("%v shut down successfully", shutdownFns[i].name) - } - } - logger.Info("Shutdown complete") - os.Exit(exitCode) -} - -func openBrowser(url string) error { - switch runtime.GOOS { - case "linux": - return exec.Command("xdg-open", url).Start() - case "windows": - return exec.Command("rundll32", "url.dll,FileProtocolHandler", url).Start() - case "darwin": - return exec.Command("open", url).Start() - default: - return fmt.Errorf("unsupported platform %q", runtime.GOOS) - } -} - -func runCompatMigrateAutopilotJSONToStore(bc *bus.Client, id, dir string) (err error) { - // check if the file exists - path := filepath.Join(dir, "autopilot.json") - if _, err := os.Stat(path); os.IsNotExist(err) { - return nil - } - - // defer autopilot dir cleanup - defer func() { - if err == nil { - log.Println("migration: removing autopilot directory") - if err = os.RemoveAll(dir); err == nil { - log.Println("migration: done") - } - } - }() - - // read the json config - log.Println("migration: reading autopilot.json") - //nolint:tagliatelle - var cfg struct { - Config api.AutopilotConfig `json:"Config"` - } - if data, err := os.ReadFile(path); err != nil { - return err - } else if err := json.Unmarshal(data, &cfg); err != nil { - return err - } - - // make sure we don't hang - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - // check if the autopilot already exists, if so we don't need to migrate - _, err = bc.Autopilot(ctx, api.DefaultAutopilotID) - if err == nil { - log.Printf("migration: autopilot already exists in the bus, the autopilot.json won't be migrated\n old config: %+v\n", cfg.Config) - return nil - } - - // create an autopilot entry - log.Println("migration: persisting autopilot to the bus") - if err := bc.UpdateAutopilot(ctx, api.Autopilot{ - ID: id, - Config: cfg.Config, - }); err != nil { - return err - } - - // remove autopilot folder and config - log.Println("migration: cleaning up autopilot directory") - if err = os.RemoveAll(dir); err == nil { - log.Println("migration: done") + // shut down the node + err = node.Shutdown() + if err != nil { + os.Exit(1) + return } - return nil + os.Exit(0) } diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go new file mode 100644 index 000000000..14ceda8ed --- /dev/null +++ b/cmd/renterd/node.go @@ -0,0 +1,462 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "go.sia.tech/core/consensus" + "go.sia.tech/core/types" + "go.sia.tech/coreutils/wallet" + "go.sia.tech/jape" + "go.sia.tech/renterd/api" + "go.sia.tech/renterd/autopilot" + "go.sia.tech/renterd/build" + "go.sia.tech/renterd/bus" + "go.sia.tech/renterd/config" + "go.sia.tech/renterd/internal/utils" + "go.sia.tech/renterd/worker" + "go.sia.tech/renterd/worker/s3" + "go.sia.tech/web/renterd" + "go.uber.org/zap" + "golang.org/x/sys/cpu" +) + +type ( + node struct { + cfg config.Config + + apiSrv *http.Server + apiListener net.Listener + + s3Srv *http.Server + s3Listener net.Listener + + setupFns []fn + shutdownFns []fn + + bus *bus.Client + logger *zap.SugaredLogger + } + + fn struct { + name string + fn func(context.Context) error + } +) + +func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) (*node, error) { + var setupFns, shutdownFns []fn + + // validate config + if cfg.Bus.RemoteAddr != "" && !cfg.Worker.Enabled && !cfg.Autopilot.Enabled { + return nil, errors.New("remote bus, remote worker, and no autopilot -- nothing to do!") + } + if cfg.Worker.Enabled && cfg.Bus.RemoteAddr != "" && cfg.Worker.ExternalAddress == "" { + return nil, errors.New("can't enable the worker using a remote bus, without configuring the worker's external address") + } + if cfg.Autopilot.Enabled && !cfg.Worker.Enabled && len(cfg.Worker.Remotes) == 0 { + return nil, errors.New("can't enable autopilot without providing either workers to connect to or creating a worker") + } + + // initialise directory + err := os.MkdirAll(cfg.Directory, 0700) + if err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } + + // initialise logger + logger, closeFn, err := NewLogger(cfg.Directory, "renterd.log", cfg.Log) + if err != nil { + return nil, fmt.Errorf("failed to create logger: %w", err) + } + shutdownFns = append(shutdownFns, fn{ + name: "Logger", + fn: closeFn, + }) + + // print network and version + logger.Info("renterd", zap.String("version", build.Version()), zap.String("network", network.Name), zap.String("commit", build.Commit()), zap.Time("buildDate", build.BuildTime())) + if runtime.GOARCH == "amd64" && !cpu.X86.HasAVX2 { + logger.Warn("renterd is running on a system without AVX2 support, performance may be degraded") + } + + // initialise a listener and override the HTTP address, we have to do this + // first so we know the actual api address if the user specifies port :0 + l, err := listenTCP(logger, cfg.HTTP.Address) + if err != nil { + return nil, fmt.Errorf("failed to create listener: %w", err) + } + cfg.HTTP.Address = "http://" + l.Addr().String() + + // initialise a web server + mux := &utils.TreeMux{Sub: make(map[string]utils.TreeMux)} + srv := &http.Server{Handler: mux} + shutdownFns = append(shutdownFns, fn{ + name: "HTTP Server", + fn: srv.Shutdown, + }) + + // initialise auth handler + auth := jape.BasicAuth(cfg.HTTP.Password) + + // generate private key from seed + var pk types.PrivateKey + if cfg.Seed != "" { + var rawSeed [32]byte + err := wallet.SeedFromPhrase(&rawSeed, cfg.Seed) + if err != nil { + return nil, fmt.Errorf("failed to load wallet: %w", err) + } + pk = wallet.KeyFromSeed(&rawSeed, 0) + } + + // initialise bus + busAddr, busPassword := cfg.Bus.RemoteAddr, cfg.Bus.RemotePassword + if cfg.Bus.RemoteAddr == "" { + b, shutdownFn, _, err := bus.NewNode(bus.NodeConfig{ + Bus: cfg.Bus, + Database: cfg.Database, + DatabaseLog: cfg.Log.Database, + Logger: logger, + Network: network, + Genesis: genesis, + RetryTxIntervals: []time.Duration{ + 200 * time.Millisecond, + 500 * time.Millisecond, + time.Second, + 3 * time.Second, + 10 * time.Second, + 10 * time.Second, + }, + }, cfg.Directory, pk, logger) + if err != nil { + return nil, fmt.Errorf("failed to create bus: %w", err) + } + + shutdownFns = append(shutdownFns, fn{ + name: "Bus", + fn: shutdownFn, + }) + + mux.Sub["/api/bus"] = utils.TreeMux{Handler: auth(b)} + busAddr = cfg.HTTP.Address + "/api/bus" + busPassword = cfg.HTTP.Password + + // only serve the UI if a bus is created + mux.Handler = renterd.Handler() + } else { + logger.Info("connecting to remote bus at " + busAddr) + } + bc := bus.NewClient(busAddr, busPassword) + + // initialise workers + var s3Srv *http.Server + var s3Listener net.Listener + var workers []autopilot.Worker + if len(cfg.Worker.Remotes) == 0 { + if cfg.Worker.Enabled { + workerAddr := cfg.HTTP.Address + "/api/worker" + w, s3Handler, setupFn, shutdownFn, err := worker.NewNode(cfg.Worker, s3.Opts{ + AuthDisabled: cfg.S3.DisableAuth, + HostBucketBases: cfg.S3.HostBucketBases, + HostBucketEnabled: cfg.S3.HostBucketEnabled, + }, bc, pk, logger) + if err != nil { + logger.Fatal("failed to create worker: " + err.Error()) + } + var workerExternAddr string + if cfg.Bus.RemoteAddr != "" { + workerExternAddr = cfg.Worker.ExternalAddress + } else { + workerExternAddr = workerAddr + } + + setupFns = append(setupFns, fn{ + name: "Worker", + fn: func(ctx context.Context) error { + return setupFn(ctx, workerExternAddr, cfg.HTTP.Password) + }, + }) + shutdownFns = append(shutdownFns, fn{ + name: "Worker", + fn: shutdownFn, + }) + + mux.Sub["/api/worker"] = utils.TreeMux{Handler: authHandler(cfg.HTTP.Password, cfg.Worker.AllowUnauthenticatedDownloads)(w)} + wc := worker.NewClient(workerAddr, cfg.HTTP.Password) + workers = append(workers, wc) + + if cfg.S3.Enabled { + s3Srv = &http.Server{ + Addr: cfg.S3.Address, + Handler: s3Handler, + } + s3Listener, err = listenTCP(logger, cfg.S3.Address) + if err != nil { + logger.Fatal("failed to create listener: " + err.Error()) + } + shutdownFns = append(shutdownFns, fn{ + name: "S3", + fn: s3Srv.Shutdown, + }) + } + } + } else { + for _, remote := range cfg.Worker.Remotes { + workers = append(workers, worker.NewClient(remote.Address, remote.Password)) + logger.Info("connecting to remote worker at " + remote.Address) + } + } + + // initialise autopilot + if cfg.Autopilot.Enabled { + ap, runFn, shutdownFn, err := autopilot.NewNode(cfg.Autopilot, bc, workers, logger) + if err != nil { + logger.Fatal("failed to create autopilot: " + err.Error()) + } + + setupFns = append(setupFns, fn{ + name: "Autopilot", + fn: func(_ context.Context) error { runFn(); return nil }, + }) + + // NOTE: shutdown functions are executed in reverse order, it's + // important the autopilot is shut down first so we don't needlessly + // make worker and bus calls while they're shutting down + shutdownFns = append(shutdownFns, fn{ + name: "Autopilot", + fn: shutdownFn, + }) + + mux.Sub["/api/autopilot"] = utils.TreeMux{Handler: auth(ap)} + } + + return &node{ + apiSrv: srv, + apiListener: l, + + s3Srv: s3Srv, + s3Listener: s3Listener, + + setupFns: setupFns, + shutdownFns: shutdownFns, + + cfg: cfg, + + logger: logger.Sugar(), + }, nil +} + +func (n *node) Run() error { + // start server + go n.apiSrv.Serve(n.apiListener) + n.logger.Info("api: Listening on " + n.apiListener.Addr().String()) + + // execute run functions + for _, fn := range n.setupFns { + if err := fn.fn(context.Background()); err != nil { + return fmt.Errorf("failed to run %v: %w", fn.name, err) + } + } + + // set initial S3 keys + if n.cfg.S3.Enabled && !n.cfg.S3.DisableAuth { + as, err := n.bus.S3AuthenticationSettings(context.Background()) + if err != nil && !strings.Contains(err.Error(), api.ErrSettingNotFound.Error()) { + return fmt.Errorf("failed to fetch S3 authentication settings: %w", err) + } else if as.V4Keypairs == nil { + as.V4Keypairs = make(map[string]string) + } + + // S3 key pair validation was broken at one point, we need to remove the + // invalid key pairs here to ensure we don't fail when we update the + // setting below. + for k, v := range as.V4Keypairs { + if err := (api.S3AuthenticationSettings{V4Keypairs: map[string]string{k: v}}).Validate(); err != nil { + n.logger.Infof("removing invalid S3 keypair for AccessKeyID %s, reason: %v", k, err) + delete(as.V4Keypairs, k) + } + } + + // merge keys + for k, v := range n.cfg.S3.KeypairsV4 { + as.V4Keypairs[k] = v + } + // update settings + if err := n.bus.UpdateSetting(context.Background(), api.SettingS3Authentication, as); err != nil { + return fmt.Errorf("failed to update S3 authentication settings: %w", err) + } + } + + // start S3 server + if n.s3Srv != nil { + go n.s3Srv.Serve(n.s3Listener) + n.logger.Info("s3: Listening on " + n.s3Listener.Addr().String()) + } + + // fetch the syncer address + syncerAddress, err := n.bus.SyncerAddress(context.Background()) + if err != nil { + return fmt.Errorf("failed to fetch syncer address: %w", err) + } + n.logger.Info("bus: Listening on " + syncerAddress) + + // run autopilot store migration + // + // TODO: we can safely remove this already + if n.cfg.Autopilot.Enabled { + autopilotDir := filepath.Join(n.cfg.Directory, n.cfg.Autopilot.ID) + if err := runCompatMigrateAutopilotJSONToStore(n.bus, "autopilot", autopilotDir); err != nil { + return fmt.Errorf("failed to migrate autopilot JSON: %w", err) + } + } + + // open the web UI if enabled + if n.cfg.AutoOpenWebUI { + time.Sleep(time.Millisecond) // give the web server a chance to start + _, port, err := net.SplitHostPort(n.apiListener.Addr().String()) + if err != nil { + n.logger.Debug("failed to parse API address", zap.Error(err)) + } else if err := openBrowser(fmt.Sprintf("http://127.0.0.1:%s", port)); err != nil { + n.logger.Debug("failed to open browser", zap.Error(err)) + } + } + return nil +} + +func (n *node) Shutdown() error { + n.logger.Info("Shutting down...") + + // give each service a fraction of the total shutdown timeout. One service + // timing out shouldn't prevent the others from attempting a shutdown. + timeout := n.cfg.ShutdownTimeout / time.Duration(len(n.shutdownFns)) + shutdown := func(fn func(ctx context.Context) error) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return fn(ctx) + } + + // shut down the services in reverse order + var errs []error + for i := len(n.shutdownFns) - 1; i >= 0; i-- { + if err := shutdown(n.shutdownFns[i].fn); err != nil { + n.logger.Errorf("failed to shut down %v: %v", n.shutdownFns[i].name, err) + errs = append(errs, err) + } else { + n.logger.Infof("%v shut down successfully", n.shutdownFns[i].name) + } + } + + return errors.Join(errs...) +} + +func authHandler(password string, unauthenticatedDownloads bool) func(http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if unauthenticatedDownloads && req.Method == http.MethodGet && strings.HasPrefix(req.URL.Path, "/objects/") { + h.ServeHTTP(w, req) + } else { + jape.BasicAuth(password)(h).ServeHTTP(w, req) + } + }) + } +} + +func listenTCP(logger *zap.Logger, addr string) (net.Listener, error) { + l, err := net.Listen("tcp", addr) + if utils.IsErr(err, errors.New("no such host")) && strings.Contains(addr, "localhost") { + // fall back to 127.0.0.1 if 'localhost' doesn't work + _, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + fallbackAddr := fmt.Sprintf("127.0.0.1:%s", port) + logger.Sugar().Warnf("failed to listen on %s, falling back to %s", addr, fallbackAddr) + return net.Listen("tcp", fallbackAddr) + } else if err != nil { + return nil, err + } + return l, nil +} + +func openBrowser(url string) error { + switch runtime.GOOS { + case "linux": + return exec.Command("xdg-open", url).Start() + case "windows": + return exec.Command("rundll32", "url.dll,FileProtocolHandler", url).Start() + case "darwin": + return exec.Command("open", url).Start() + default: + return fmt.Errorf("unsupported platform %q", runtime.GOOS) + } +} + +func runCompatMigrateAutopilotJSONToStore(bc *bus.Client, id, dir string) (err error) { + // check if the file exists + path := filepath.Join(dir, "autopilot.json") + if _, err := os.Stat(path); os.IsNotExist(err) { + return nil + } + + // defer autopilot dir cleanup + defer func() { + if err == nil { + log.Println("migration: removing autopilot directory") + if err = os.RemoveAll(dir); err == nil { + log.Println("migration: done") + } + } + }() + + // read the json config + log.Println("migration: reading autopilot.json") + //nolint:tagliatelle + var cfg struct { + Config api.AutopilotConfig `json:"Config"` + } + if data, err := os.ReadFile(path); err != nil { + return err + } else if err := json.Unmarshal(data, &cfg); err != nil { + return err + } + + // make sure we don't hang + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // check if the autopilot already exists, if so we don't need to migrate + _, err = bc.Autopilot(ctx, api.DefaultAutopilotID) + if err == nil { + log.Printf("migration: autopilot already exists in the bus, the autopilot.json won't be migrated\n old config: %+v\n", cfg.Config) + return nil + } + + // create an autopilot entry + log.Println("migration: persisting autopilot to the bus") + if err := bc.UpdateAutopilot(ctx, api.Autopilot{ + ID: id, + Config: cfg.Config, + }); err != nil { + return err + } + + // remove autopilot folder and config + log.Println("migration: cleaning up autopilot directory") + if err = os.RemoveAll(dir); err == nil { + log.Println("migration: done") + } + + return nil +} diff --git a/config/config.go b/config/config.go index 3951ea1d8..99382240b 100644 --- a/config/config.go +++ b/config/config.go @@ -17,11 +17,12 @@ type ( Log Log `yaml:"log,omitempty"` - HTTP HTTP `yaml:"http,omitempty"` + HTTP HTTP `yaml:"http,omitempty"` + + Autopilot Autopilot `yaml:"autopilot,omitempty"` Bus Bus `yaml:"bus,omitempty"` Worker Worker `yaml:"worker,omitempty"` S3 S3 `yaml:"s3,omitempty"` - Autopilot Autopilot `yaml:"autopilot,omitempty"` Database Database `yaml:"database,omitempty"` } @@ -132,6 +133,7 @@ type ( // Autopilot contains the configuration for an autopilot. Autopilot struct { Enabled bool `yaml:"enabled,omitempty"` + ID string `yaml:"id,omitempty"` AccountsRefillInterval time.Duration `yaml:"accountsRefillInterval,omitempty"` Heartbeat time.Duration `yaml:"heartbeat,omitempty"` MigrationHealthCutoff float64 `yaml:"migrationHealthCutoff,omitempty"` diff --git a/internal/chain/chain.go b/internal/chain/chain.go deleted file mode 100644 index 140fd5b7d..000000000 --- a/internal/chain/chain.go +++ /dev/null @@ -1,48 +0,0 @@ -package chain - -import ( - "time" - - "go.sia.tech/core/consensus" - "go.sia.tech/core/types" - "go.sia.tech/coreutils/chain" - "go.sia.tech/coreutils/wallet" - "go.sia.tech/renterd/api" -) - -type ( - Manager = chain.Manager - HostAnnouncement = chain.HostAnnouncement - ApplyUpdate = chain.ApplyUpdate - RevertUpdate = chain.RevertUpdate -) - -var ForEachHostAnnouncement = chain.ForEachHostAnnouncement - -type ChainUpdateTx interface { - ContractState(fcid types.FileContractID) (api.ContractState, error) - UpdateChainIndex(index types.ChainIndex) error - UpdateContract(fcid types.FileContractID, revisionHeight, revisionNumber, size uint64) error - UpdateContractState(fcid types.FileContractID, state api.ContractState) error - UpdateContractProofHeight(fcid types.FileContractID, proofHeight uint64) error - UpdateFailedContracts(blockHeight uint64) error - UpdateHost(hk types.PublicKey, ha chain.HostAnnouncement, bh uint64, blockID types.BlockID, ts time.Time) error - - wallet.UpdateTx -} - -func TestnetZen() (*consensus.Network, types.Block) { - return chain.TestnetZen() -} - -func Mainnet() (*consensus.Network, types.Block) { - return chain.Mainnet() -} - -func NewDBStore(db chain.DB, n *consensus.Network, genesisBlock types.Block) (_ *chain.DBStore, _ consensus.State, err error) { - return chain.NewDBStore(db, n, genesisBlock) -} - -func NewManager(store chain.Store, cs consensus.State) *Manager { - return chain.NewManager(store, cs) -} diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 00046b2fe..b022a26fe 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -23,10 +24,8 @@ import ( "go.sia.tech/renterd/autopilot" "go.sia.tech/renterd/bus" "go.sia.tech/renterd/config" - "go.sia.tech/renterd/internal/node" "go.sia.tech/renterd/internal/test" "go.sia.tech/renterd/internal/utils" - iworker "go.sia.tech/renterd/internal/worker" "go.sia.tech/renterd/stores" "go.sia.tech/renterd/worker/s3" "go.sia.tech/web/renterd" @@ -161,9 +160,9 @@ type testClusterOptions struct { skipRunningAutopilot bool walletKey *types.PrivateKey - autopilotCfg *node.AutopilotConfig + autopilotCfg *config.Autopilot autopilotSettings *api.AutopilotConfig - busCfg *node.BusConfig + busCfg *bus.NodeConfig workerCfg *config.Worker } @@ -308,7 +307,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { tt.OK(err) // Create bus. - b, bShutdownFn, cm, err := node.NewBus(busCfg, busDir, wk, logger) + b, bShutdownFn, cm, err := bus.NewNode(busCfg, busDir, wk, logger) tt.OK(err) busAuth := jape.BasicAuth(busPassword) @@ -328,10 +327,10 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { busShutdownFns = append(busShutdownFns, bShutdownFn) // Create worker. - w, s3Handler, wSetupFn, wShutdownFn, err := node.NewWorker(workerCfg, s3.Opts{}, busClient, wk, logger) + w, s3Handler, wSetupFn, wShutdownFn, err := worker.NewNode(workerCfg, s3.Opts{}, busClient, wk, logger) tt.OK(err) workerServer := http.Server{ - Handler: iworker.Auth(workerPassword, false)(w), + Handler: auth(workerPassword, false)(w), } var workerShutdownFns []func(context.Context) error @@ -347,7 +346,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { s3ShutdownFns = append(s3ShutdownFns, s3Server.Shutdown) // Create autopilot. - ap, aStartFn, aStopFn, err := node.NewAutopilot(apCfg, busClient, []autopilot.Worker{workerClient}, logger) + ap, aStartFn, aStopFn, err := autopilot.NewNode(apCfg, busClient, []autopilot.Worker{workerClient}, logger) tt.OK(err) autopilotAuth := jape.BasicAuth(autopilotPassword) @@ -406,7 +405,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if !opts.skipRunningAutopilot { cluster.wg.Add(1) go func() { - _ = aStartFn() + aStartFn() cluster.wg.Done() }() } @@ -863,6 +862,18 @@ func (c *TestCluster) mineBlocks(addr types.Address, n uint64) error { return nil } +func auth(password string, unauthenticatedDownloads bool) func(http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if unauthenticatedDownloads && req.Method == http.MethodGet && strings.HasPrefix(req.URL.Path, "/objects/") { + h.ServeHTTP(w, req) + } else { + jape.BasicAuth(password)(h).ServeHTTP(w, req) + } + }) + } +} + // testNetwork returns a modified version of Zen used for testing func testNetwork() (*consensus.Network, types.Block) { // use a modified version of Zen @@ -888,10 +899,10 @@ func testNetwork() (*consensus.Network, types.Block) { return n, genesis } -func testBusCfg() node.BusConfig { +func testBusCfg() bus.NodeConfig { network, genesis := testNetwork() - return node.BusConfig{ + return bus.NodeConfig{ Bus: config.Bus{ AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year Bootstrap: false, @@ -936,18 +947,16 @@ func testWorkerCfg() config.Worker { } } -func testApCfg() node.AutopilotConfig { - return node.AutopilotConfig{ - ID: api.DefaultAutopilotID, - Autopilot: config.Autopilot{ - AccountsRefillInterval: time.Second, - Heartbeat: time.Second, - MigrationHealthCutoff: 0.99, - MigratorParallelSlabsPerWorker: 1, - RevisionSubmissionBuffer: 0, - ScannerInterval: time.Second, - ScannerBatchSize: 10, - ScannerNumThreads: 1, - }, +func testApCfg() config.Autopilot { + return config.Autopilot{ + AccountsRefillInterval: time.Second, + Heartbeat: time.Second, + ID: api.DefaultAutopilotID, + MigrationHealthCutoff: 0.99, + MigratorParallelSlabsPerWorker: 1, + RevisionSubmissionBuffer: 0, + ScannerInterval: time.Second, + ScannerBatchSize: 10, + ScannerNumThreads: 1, } } diff --git a/internal/worker/auth.go b/internal/worker/auth.go deleted file mode 100644 index 032d2536c..000000000 --- a/internal/worker/auth.go +++ /dev/null @@ -1,20 +0,0 @@ -package worker - -import ( - "net/http" - "strings" - - "go.sia.tech/jape" -) - -func Auth(password string, unauthenticatedDownloads bool) func(http.Handler) http.Handler { - return func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if unauthenticatedDownloads && req.Method == http.MethodGet && strings.HasPrefix(req.URL.Path, "/objects/") { - h.ServeHTTP(w, req) - } else { - jape.BasicAuth(password)(h).ServeHTTP(w, req) - } - }) - } -} diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index f30960011..33c57ce95 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -12,8 +12,8 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/internal/chain" sql "go.sia.tech/renterd/stores/sql" ) diff --git a/worker/mocks_test.go b/worker/mocks_test.go index 4c3929205..dd0243008 100644 --- a/worker/mocks_test.go +++ b/worker/mocks_test.go @@ -83,6 +83,7 @@ type busMock struct { *objectStoreMock *settingStoreMock *syncerMock + *s3Mock *walletMock *webhookBroadcasterMock *webhookStoreMock @@ -634,6 +635,56 @@ func (os *objectStoreMock) forEachObject(fn func(bucket, path string, o object.O } } +type s3Mock struct{} + +func (*s3Mock) CreateBucket(context.Context, string, api.CreateBucketOptions) error { + return nil +} + +func (*s3Mock) DeleteBucket(context.Context, string) error { + return nil +} + +func (*s3Mock) ListBuckets(context.Context) (buckets []api.Bucket, err error) { + return nil, nil +} + +func (*s3Mock) CopyObject(context.Context, string, string, string, string, api.CopyObjectOptions) (om api.ObjectMetadata, err error) { + return api.ObjectMetadata{}, nil +} + +func (*s3Mock) ListObjects(context.Context, string, api.ListObjectOptions) (resp api.ObjectsListResponse, err error) { + return api.ObjectsListResponse{}, nil +} + +func (*s3Mock) AbortMultipartUpload(context.Context, string, string, string) (err error) { + return nil +} + +func (*s3Mock) CompleteMultipartUpload(context.Context, string, string, string, []api.MultipartCompletedPart, api.CompleteMultipartOptions) (_ api.MultipartCompleteResponse, err error) { + return api.MultipartCompleteResponse{}, nil +} + +func (*s3Mock) CreateMultipartUpload(context.Context, string, string, api.CreateMultipartOptions) (api.MultipartCreateResponse, error) { + return api.MultipartCreateResponse{}, nil +} + +func (*s3Mock) MultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker string, maxUploads int) (resp api.MultipartListUploadsResponse, _ error) { + return api.MultipartListUploadsResponse{}, nil +} + +func (*s3Mock) MultipartUploadParts(ctx context.Context, bucket, object string, uploadID string, marker int, limit int64) (resp api.MultipartListPartsResponse, _ error) { + return api.MultipartListPartsResponse{}, nil +} + +func (*s3Mock) S3AuthenticationSettings(context.Context) (as api.S3AuthenticationSettings, err error) { + return api.S3AuthenticationSettings{}, nil +} + +func (*s3Mock) UpdateSetting(context.Context, string, interface{}) error { + return nil +} + var _ SettingStore = (*settingStoreMock)(nil) type settingStoreMock struct{} diff --git a/worker/node.go b/worker/node.go new file mode 100644 index 000000000..d9a0d9467 --- /dev/null +++ b/worker/node.go @@ -0,0 +1,28 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "net/http" + + "go.sia.tech/core/types" + "go.sia.tech/renterd/config" + "go.sia.tech/renterd/worker/s3" + "go.uber.org/zap" + "golang.org/x/crypto/blake2b" +) + +func NewNode(cfg config.Worker, s3Opts s3.Opts, b Bus, seed types.PrivateKey, l *zap.Logger) (http.Handler, http.Handler, func(context.Context, string, string) error, func(context.Context) error, error) { + workerKey := blake2b.Sum256(append([]byte("worker"), seed...)) + w, err := New(workerKey, cfg.ID, b, cfg.ContractLockTimeout, cfg.BusFlushInterval, cfg.DownloadOverdriveTimeout, cfg.UploadOverdriveTimeout, cfg.DownloadMaxOverdrive, cfg.UploadMaxOverdrive, cfg.DownloadMaxMemory, cfg.UploadMaxMemory, cfg.AllowPrivateIPs, l) + if err != nil { + return nil, nil, nil, nil, err + } + s3Handler, err := s3.New(b, w, l.Named("s3").Sugar(), s3Opts) + if err != nil { + err = errors.Join(err, w.Shutdown(context.Background())) + return nil, nil, nil, nil, fmt.Errorf("failed to create s3 handler: %w", err) + } + return w.Handler(), s3Handler, w.Setup, w.Shutdown, nil +} diff --git a/worker/worker.go b/worker/worker.go index fb4f8c49d..a4dc4945a 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -29,6 +29,7 @@ import ( "go.sia.tech/renterd/object" "go.sia.tech/renterd/webhooks" "go.sia.tech/renterd/worker/client" + "go.sia.tech/renterd/worker/s3" "go.uber.org/zap" "golang.org/x/crypto/blake2b" ) @@ -68,6 +69,8 @@ func NewClient(address, password string) *Client { type ( Bus interface { + s3.Bus + alerts.Alerter ConsensusState webhooks.Broadcaster From 17e6d30620987cb61eba4262a3e173c445ae42e9 Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 14 Aug 2024 09:24:32 +0200 Subject: [PATCH 02/10] all: add utils package, fix order of events when loading config --- cmd/renterd/config.go | 153 ++++++++++++++++++++++------------- cmd/renterd/main.go | 37 +++------ cmd/renterd/node.go | 39 +-------- internal/test/e2e/cluster.go | 2 +- internal/utils/web.go | 32 -------- utils/web.go | 83 +++++++++++++++++++ 6 files changed, 193 insertions(+), 153 deletions(-) delete mode 100644 internal/utils/web.go create mode 100644 utils/web.go diff --git a/cmd/renterd/config.go b/cmd/renterd/config.go index fb2fb507e..38231458d 100644 --- a/cmd/renterd/config.go +++ b/cmd/renterd/config.go @@ -3,6 +3,7 @@ package main import ( "bufio" "encoding/hex" + "errors" "flag" "fmt" "log" @@ -13,7 +14,9 @@ import ( "strings" "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/wallet" "go.sia.tech/renterd/api" "go.sia.tech/renterd/config" @@ -23,6 +26,8 @@ import ( "lukechampine.com/frand" ) +// TODO: handle RENTERD_S3_HOST_BUCKET_BASES correctly + const ( // accountRefillInterval is the amount of time between refills of ephemeral // accounts. If we conservatively assume that a good host charges 500 SC / @@ -38,6 +43,11 @@ const ( var ( disableStdin bool enableANSI = runtime.GOOS != "windows" + + hostBasesStr string + keyPairsV4 string + workerRemotePassStr string + workerRemoteAddrsStr string ) func defaultConfig() config.Config { @@ -122,29 +132,88 @@ func defaultConfig() config.Config { } } -func loadConfig() config.Config { - cfg := defaultConfig() - +// loadConfig creates a default config and overrides it with the contents of the +// YAML file (specified by the RENTERD_CONFIG_FILE), CLI flags, and environment +// variables, in that order. +func loadConfig() (cfg config.Config, network *consensus.Network, genesis types.Block, err error) { + cfg = defaultConfig() parseYamlConfig(&cfg) parseCLIFlags(&cfg) parseEnvironmentVariables(&cfg) + // check network + switch cfg.Network { + case "anagami": + network, genesis = chain.TestnetAnagami() + case "mainnet": + network, genesis = chain.Mainnet() + case "zen": + network, genesis = chain.TestnetZen() + default: + err = fmt.Errorf("unknown network '%s'", cfg.Network) + return + } + + return +} + +func sanitizeConfig(cfg *config.Config) error { + // parse remotes + if workerRemoteAddrsStr != "" && workerRemotePassStr != "" { + cfg.Worker.Remotes = cfg.Worker.Remotes[:0] + for _, addr := range strings.Split(workerRemoteAddrsStr, ";") { + cfg.Worker.Remotes = append(cfg.Worker.Remotes, config.RemoteWorker{ + Address: addr, + Password: workerRemotePassStr, + }) + } + } + + // disable worker if remotes are set + if len(cfg.Worker.Remotes) > 0 { + cfg.Worker.Enabled = false + } + + // combine host bucket bases + for _, base := range strings.Split(hostBasesStr, ",") { + if trimmed := strings.TrimSpace(base); trimmed != "" { + cfg.S3.HostBucketBases = append(cfg.S3.HostBucketBases, base) + } + } + // check that the API password is set if cfg.HTTP.Password == "" { if disableStdin { - stdoutFatalError("API password must be set via environment variable or config file when --env flag is set") - return config.Config{} + return errors.New("API password must be set via environment variable or config file when --env flag is set") } } - setAPIPassword(&cfg) + setAPIPassword(cfg) // check that the seed is set if cfg.Seed == "" && (cfg.Worker.Enabled || cfg.Bus.RemoteAddr == "") { // only worker & bus require a seed if disableStdin { - stdoutFatalError("Seed must be set via environment variable or config file when --env flag is set") - return config.Config{} + return errors.New("Seed must be set via environment variable or config file when --env flag is set") + } + setSeedPhrase(cfg) + } + + // validate the seed is valid + if cfg.Seed != "" { + var rawSeed [32]byte + if err := wallet.SeedFromPhrase(&rawSeed, cfg.Seed); err != nil { + return fmt.Errorf("failed to load wallet: %v", err) + } + } + + // parse S3 auth keys + if cfg.S3.Enabled { + if !cfg.S3.DisableAuth && keyPairsV4 != "" { + var err error + cfg.S3.KeypairsV4, err = s3.Parsev4AuthKeys(strings.Split(keyPairsV4, ";")) + if err != nil { + return fmt.Errorf("failed to parse keypairs: %v", err) + } } - setSeedPhrase(&cfg) } // default log levels @@ -155,12 +224,9 @@ func loadConfig() config.Config { cfg.Log.Database.Level = cfg.Log.Level } - return cfg + return nil } -// parseYamlConfig loads the config file specified by the RENTERD_CONFIG_FILE -// environment variable. If the config file does not exist, it will not be -// loaded. func parseYamlConfig(cfg *config.Config) { configPath := "renterd.yml" if str := os.Getenv("RENTERD_CONFIG_FILE"); str != "" { @@ -254,24 +320,33 @@ func parseCLIFlags(cfg *config.Config) { flag.DurationVar(&cfg.ShutdownTimeout, "node.shutdownTimeout", cfg.ShutdownTimeout, "Timeout for node shutdown") // s3 - var hostBasesStr string flag.StringVar(&cfg.S3.Address, "s3.address", cfg.S3.Address, "Address for serving S3 API (overrides with RENTERD_S3_ADDRESS)") flag.BoolVar(&cfg.S3.DisableAuth, "s3.disableAuth", cfg.S3.DisableAuth, "Disables authentication for S3 API (overrides with RENTERD_S3_DISABLE_AUTH)") flag.BoolVar(&cfg.S3.Enabled, "s3.enabled", cfg.S3.Enabled, "Enables/disables S3 API (requires worker.enabled to be 'true', overrides with RENTERD_S3_ENABLED)") flag.StringVar(&hostBasesStr, "s3.hostBases", "", "Enables bucket rewriting in the router for specific hosts provided via comma-separated list (overrides with RENTERD_S3_HOST_BUCKET_BASES)") flag.BoolVar(&cfg.S3.HostBucketEnabled, "s3.hostBucketEnabled", cfg.S3.HostBucketEnabled, "Enables bucket rewriting in the router for all hosts (overrides with RENTERD_S3_HOST_BUCKET_ENABLED)") - // combine host bucket bases - for _, base := range strings.Split(hostBasesStr, ",") { - if trimmed := strings.TrimSpace(base); trimmed != "" { - cfg.S3.HostBucketBases = append(cfg.S3.HostBucketBases, base) - } + // custom usage + flag.Usage = func() { + log.Print(usageHeader) + flag.PrintDefaults() + log.Print(usageFooter) } flag.Parse() } func parseEnvironmentVariables(cfg *config.Config) { + // define helper function to parse environment variables + parseEnvVar := func(s string, v interface{}) { + if env, ok := os.LookupEnv(s); ok { + if _, err := fmt.Sscan(env, v); err != nil { + log.Fatalf("failed to parse %s: %v", s, err) + } + fmt.Printf("Using %s environment variable\n", s) + } + } + parseEnvVar("RENTERD_NETWORK", &cfg.Network) parseEnvVar("RENTERD_BUS_REMOTE_ADDR", &cfg.Bus.RemoteAddr) @@ -319,48 +394,10 @@ func parseEnvironmentVariables(cfg *config.Config) { parseEnvVar("RENTERD_LOG_DATABASE_IGNORE_RECORD_NOT_FOUND_ERROR", &cfg.Log.Database.IgnoreRecordNotFoundError) parseEnvVar("RENTERD_LOG_DATABASE_SLOW_THRESHOLD", &cfg.Log.Database.SlowThreshold) - // parse remotes, we duplicate the old behavior of all workers sharing the - // same password - var workerRemotePassStr string - var workerRemoteAddrsStr string parseEnvVar("RENTERD_WORKER_REMOTE_ADDRS", &workerRemoteAddrsStr) parseEnvVar("RENTERD_WORKER_API_PASSWORD", &workerRemotePassStr) - if workerRemoteAddrsStr != "" && workerRemotePassStr != "" { - cfg.Worker.Remotes = cfg.Worker.Remotes[:0] - for _, addr := range strings.Split(workerRemoteAddrsStr, ";") { - cfg.Worker.Remotes = append(cfg.Worker.Remotes, config.RemoteWorker{ - Address: addr, - Password: workerRemotePassStr, - }) - } - } - - // disable worker if remotes are set - if len(cfg.Worker.Remotes) > 0 { - cfg.Worker.Enabled = false - } - - // parse S3 auth keys - if cfg.S3.Enabled { - var keyPairsV4 string - parseEnvVar("RENTERD_S3_KEYPAIRS_V4", &keyPairsV4) - if !cfg.S3.DisableAuth && keyPairsV4 != "" { - var err error - cfg.S3.KeypairsV4, err = s3.Parsev4AuthKeys(strings.Split(keyPairsV4, ";")) - if err != nil { - log.Fatalf("failed to parse keypairs: %v", err) - } - } - } -} -func parseEnvVar(s string, v interface{}) { - if env, ok := os.LookupEnv(s); ok { - if _, err := fmt.Sscan(env, v); err != nil { - log.Fatalf("failed to parse %s: %v", s, err) - } - fmt.Printf("Using %s environment variable\n", s) - } + parseEnvVar("RENTERD_S3_KEYPAIRS_V4", &keyPairsV4) } // readPasswordInput reads a password from stdin. diff --git a/cmd/renterd/main.go b/cmd/renterd/main.go index c3df11de2..a32ecceed 100644 --- a/cmd/renterd/main.go +++ b/cmd/renterd/main.go @@ -6,10 +6,6 @@ import ( "os" "os/signal" "syscall" - - "go.sia.tech/core/consensus" - "go.sia.tech/core/types" - "go.sia.tech/coreutils/chain" ) const ( @@ -40,28 +36,10 @@ on how to configure and use renterd. func main() { log.SetFlags(0) - // set usage - flag.Usage = func() { - log.Print(usageHeader) - flag.PrintDefaults() - log.Print(usageFooter) - } - // load the config - cfg := loadConfig() - - // validate the network - var network *consensus.Network - var genesis types.Block - switch cfg.Network { - case "anagami": - network, genesis = chain.TestnetAnagami() - case "mainnet": - network, genesis = chain.Mainnet() - case "zen": - network, genesis = chain.TestnetZen() - default: - log.Fatalf("unknown network '%s'", cfg.Network) + cfg, network, genesis, err := loadConfig() + if err != nil { + stdoutFatalError("failed to load config: " + err.Error()) } // NOTE: update the usage header when adding new commands @@ -79,16 +57,21 @@ func main() { return } + // sanitize the config + if err := sanitizeConfig(&cfg); err != nil { + stdoutFatalError("failed to sanitize config: " + err.Error()) + } + // create node node, err := newNode(cfg, network, genesis) if err != nil { - log.Fatal("failed to create node: " + err.Error()) + stdoutFatalError("failed to create node: " + err.Error()) } // start node err = node.Run() if err != nil { - log.Fatal("failed to run node: " + err.Error()) + stdoutFatalError("failed to run node: " + err.Error()) } // wait for interrupt signal diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index 14ceda8ed..f7e69aa04 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -9,7 +9,6 @@ import ( "net" "net/http" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -24,7 +23,7 @@ import ( "go.sia.tech/renterd/build" "go.sia.tech/renterd/bus" "go.sia.tech/renterd/config" - "go.sia.tech/renterd/internal/utils" + "go.sia.tech/renterd/utils" "go.sia.tech/renterd/worker" "go.sia.tech/renterd/worker/s3" "go.sia.tech/web/renterd" @@ -93,7 +92,7 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) // initialise a listener and override the HTTP address, we have to do this // first so we know the actual api address if the user specifies port :0 - l, err := listenTCP(logger, cfg.HTTP.Address) + l, err := utils.ListenTCP(cfg.HTTP.Address, logger) if err != nil { return nil, fmt.Errorf("failed to create listener: %w", err) } @@ -202,7 +201,7 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) Addr: cfg.S3.Address, Handler: s3Handler, } - s3Listener, err = listenTCP(logger, cfg.S3.Address) + s3Listener, err = utils.ListenTCP(cfg.S3.Address, logger) if err != nil { logger.Fatal("failed to create listener: " + err.Error()) } @@ -328,7 +327,7 @@ func (n *node) Run() error { _, port, err := net.SplitHostPort(n.apiListener.Addr().String()) if err != nil { n.logger.Debug("failed to parse API address", zap.Error(err)) - } else if err := openBrowser(fmt.Sprintf("http://127.0.0.1:%s", port)); err != nil { + } else if err := utils.OpenBrowser(fmt.Sprintf("http://127.0.0.1:%s", port)); err != nil { n.logger.Debug("failed to open browser", zap.Error(err)) } } @@ -373,36 +372,6 @@ func authHandler(password string, unauthenticatedDownloads bool) func(http.Handl } } -func listenTCP(logger *zap.Logger, addr string) (net.Listener, error) { - l, err := net.Listen("tcp", addr) - if utils.IsErr(err, errors.New("no such host")) && strings.Contains(addr, "localhost") { - // fall back to 127.0.0.1 if 'localhost' doesn't work - _, port, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - fallbackAddr := fmt.Sprintf("127.0.0.1:%s", port) - logger.Sugar().Warnf("failed to listen on %s, falling back to %s", addr, fallbackAddr) - return net.Listen("tcp", fallbackAddr) - } else if err != nil { - return nil, err - } - return l, nil -} - -func openBrowser(url string) error { - switch runtime.GOOS { - case "linux": - return exec.Command("xdg-open", url).Start() - case "windows": - return exec.Command("rundll32", "url.dll,FileProtocolHandler", url).Start() - case "darwin": - return exec.Command("open", url).Start() - default: - return fmt.Errorf("unsupported platform %q", runtime.GOOS) - } -} - func runCompatMigrateAutopilotJSONToStore(bc *bus.Client, id, dir string) (err error) { // check if the file exists path := filepath.Join(dir, "autopilot.json") diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index b022a26fe..4849ddf75 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -25,8 +25,8 @@ import ( "go.sia.tech/renterd/bus" "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/test" - "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/stores" + "go.sia.tech/renterd/utils" "go.sia.tech/renterd/worker/s3" "go.sia.tech/web/renterd" "go.uber.org/zap" diff --git a/internal/utils/web.go b/internal/utils/web.go deleted file mode 100644 index 0a490acd6..000000000 --- a/internal/utils/web.go +++ /dev/null @@ -1,32 +0,0 @@ -package utils - -import ( - "net/http" - _ "net/http/pprof" - "strings" -) - -type TreeMux struct { - Handler http.Handler - Sub map[string]TreeMux -} - -func (t TreeMux) ServeHTTP(w http.ResponseWriter, req *http.Request) { - if strings.HasPrefix(req.URL.Path, "/debug/pprof") { - http.DefaultServeMux.ServeHTTP(w, req) - return - } - - for prefix, c := range t.Sub { - if strings.HasPrefix(req.URL.Path, prefix) { - req.URL.Path = strings.TrimPrefix(req.URL.Path, prefix) - c.ServeHTTP(w, req) - return - } - } - if t.Handler != nil { - t.Handler.ServeHTTP(w, req) - return - } - http.NotFound(w, req) -} diff --git a/utils/web.go b/utils/web.go new file mode 100644 index 000000000..85d1ec644 --- /dev/null +++ b/utils/web.go @@ -0,0 +1,83 @@ +package utils + +import ( + "errors" + "fmt" + "net" + "net/http" + _ "net/http/pprof" + "os/exec" + "runtime" + "strings" + + "go.sia.tech/jape" + "go.sia.tech/renterd/internal/utils" + "go.uber.org/zap" +) + +type TreeMux struct { + Handler http.Handler + Sub map[string]TreeMux +} + +func (t TreeMux) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if strings.HasPrefix(req.URL.Path, "/debug/pprof") { + http.DefaultServeMux.ServeHTTP(w, req) + return + } + + for prefix, c := range t.Sub { + if strings.HasPrefix(req.URL.Path, prefix) { + req.URL.Path = strings.TrimPrefix(req.URL.Path, prefix) + c.ServeHTTP(w, req) + return + } + } + if t.Handler != nil { + t.Handler.ServeHTTP(w, req) + return + } + http.NotFound(w, req) +} + +func Auth(password string, unauthenticatedDownloads bool) func(http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if unauthenticatedDownloads && req.Method == http.MethodGet && strings.HasPrefix(req.URL.Path, "/objects/") { + h.ServeHTTP(w, req) + } else { + jape.BasicAuth(password)(h).ServeHTTP(w, req) + } + }) + } +} + +func ListenTCP(addr string, logger *zap.Logger) (net.Listener, error) { + l, err := net.Listen("tcp", addr) + if utils.IsErr(err, errors.New("no such host")) && strings.Contains(addr, "localhost") { + // fall back to 127.0.0.1 if 'localhost' doesn't work + _, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + fallbackAddr := fmt.Sprintf("127.0.0.1:%s", port) + logger.Sugar().Warnf("failed to listen on %s, falling back to %s", addr, fallbackAddr) + return net.Listen("tcp", fallbackAddr) + } else if err != nil { + return nil, err + } + return l, nil +} + +func OpenBrowser(url string) error { + switch runtime.GOOS { + case "linux": + return exec.Command("xdg-open", url).Start() + case "windows": + return exec.Command("rundll32", "url.dll,FileProtocolHandler", url).Start() + case "darwin": + return exec.Command("open", url).Start() + default: + return fmt.Errorf("unsupported platform %q", runtime.GOOS) + } +} From 4e616e753b98b31bd5eba4f6b04fb4093469456f Mon Sep 17 00:00:00 2001 From: PJ Date: Wed, 14 Aug 2024 09:35:41 +0200 Subject: [PATCH 03/10] util: use Auth --- cmd/renterd/node.go | 14 +------------- internal/test/e2e/cluster.go | 15 +-------------- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index f7e69aa04..f2789c0f1 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -192,7 +192,7 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) fn: shutdownFn, }) - mux.Sub["/api/worker"] = utils.TreeMux{Handler: authHandler(cfg.HTTP.Password, cfg.Worker.AllowUnauthenticatedDownloads)(w)} + mux.Sub["/api/worker"] = utils.TreeMux{Handler: utils.Auth(cfg.HTTP.Password, cfg.Worker.AllowUnauthenticatedDownloads)(w)} wc := worker.NewClient(workerAddr, cfg.HTTP.Password) workers = append(workers, wc) @@ -360,18 +360,6 @@ func (n *node) Shutdown() error { return errors.Join(errs...) } -func authHandler(password string, unauthenticatedDownloads bool) func(http.Handler) http.Handler { - return func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if unauthenticatedDownloads && req.Method == http.MethodGet && strings.HasPrefix(req.URL.Path, "/objects/") { - h.ServeHTTP(w, req) - } else { - jape.BasicAuth(password)(h).ServeHTTP(w, req) - } - }) - } -} - func runCompatMigrateAutopilotJSONToStore(bc *bus.Client, id, dir string) (err error) { // check if the file exists path := filepath.Join(dir, "autopilot.json") diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 4849ddf75..c4ef08c86 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -9,7 +9,6 @@ import ( "net/http" "os" "path/filepath" - "strings" "sync" "testing" "time" @@ -330,7 +329,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { w, s3Handler, wSetupFn, wShutdownFn, err := worker.NewNode(workerCfg, s3.Opts{}, busClient, wk, logger) tt.OK(err) workerServer := http.Server{ - Handler: auth(workerPassword, false)(w), + Handler: utils.Auth(workerPassword, false)(w), } var workerShutdownFns []func(context.Context) error @@ -862,18 +861,6 @@ func (c *TestCluster) mineBlocks(addr types.Address, n uint64) error { return nil } -func auth(password string, unauthenticatedDownloads bool) func(http.Handler) http.Handler { - return func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if unauthenticatedDownloads && req.Method == http.MethodGet && strings.HasPrefix(req.URL.Path, "/objects/") { - h.ServeHTTP(w, req) - } else { - jape.BasicAuth(password)(h).ServeHTTP(w, req) - } - }) - } -} - // testNetwork returns a modified version of Zen used for testing func testNetwork() (*consensus.Network, types.Block) { // use a modified version of Zen From 925c3bb637e6703c15224c1834c8a4570371eaf9 Mon Sep 17 00:00:00 2001 From: Peter-Jan Brone Date: Thu, 15 Aug 2024 09:02:06 +0200 Subject: [PATCH 04/10] Expose `worker` and `bus` (#1438) --- bus/bus.go | 16 +-- bus/routes.go | 242 ++++++++++++++++++++-------------------- worker/contract_lock.go | 4 +- worker/download.go | 2 +- worker/host.go | 4 +- worker/migrations.go | 2 +- worker/pricetables.go | 2 +- worker/rhpv2.go | 14 +-- worker/rhpv3.go | 4 +- worker/spending.go | 2 +- worker/upload.go | 8 +- worker/worker.go | 86 +++++++------- worker/worker_test.go | 2 +- 13 files changed, 194 insertions(+), 194 deletions(-) diff --git a/bus/bus.go b/bus/bus.go index 48f7720fe..5446f46f2 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -229,7 +229,7 @@ type ( } ) -type bus struct { +type Bus struct { startTime time.Time alerts alerts.Alerter @@ -257,9 +257,9 @@ type bus struct { } // New returns a new Bus -func New(ctx context.Context, am *alerts.Manager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, hdb HostDB, as AutopilotStore, cs ChainStore, ms MetadataStore, ss SettingStore, eas EphemeralAccountStore, mtrcs MetricsStore, announcementMaxAge time.Duration, l *zap.Logger) (*bus, error) { +func New(ctx context.Context, am *alerts.Manager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, hdb HostDB, as AutopilotStore, cs ChainStore, ms MetadataStore, ss SettingStore, eas EphemeralAccountStore, mtrcs MetricsStore, announcementMaxAge time.Duration, l *zap.Logger) (*Bus, error) { l = l.Named("bus") - b := &bus{ + b := &Bus{ s: s, cm: cm, w: w, @@ -300,7 +300,7 @@ func New(ctx context.Context, am *alerts.Manager, wm WebhooksManager, cm ChainMa } // Handler returns an HTTP handler that serves the bus API. -func (b *bus) Handler() http.Handler { +func (b *Bus) Handler() http.Handler { return jape.Mux(map[string]jape.Handler{ "GET /accounts": b.accountsHandlerGET, "POST /account/:id": b.accountHandlerGET, @@ -442,7 +442,7 @@ func (b *bus) Handler() http.Handler { } // Shutdown shuts down the bus. -func (b *bus) Shutdown(ctx context.Context) error { +func (b *Bus) Shutdown(ctx context.Context) error { return errors.Join( b.saveAccounts(ctx), b.webhooksMgr.Shutdown(ctx), @@ -452,7 +452,7 @@ func (b *bus) Shutdown(ctx context.Context) error { } // initAccounts loads the accounts into memory -func (b *bus) initAccounts(ctx context.Context) error { +func (b *Bus) initAccounts(ctx context.Context) error { accounts, err := b.eas.Accounts(ctx) if err != nil { return err @@ -470,7 +470,7 @@ func (b *bus) initAccounts(ctx context.Context) error { // initSettings loads the default settings if the setting is not already set and // ensures the settings are valid -func (b *bus) initSettings(ctx context.Context) error { +func (b *Bus) initSettings(ctx context.Context) error { // testnets have different redundancy settings defaultRedundancySettings := api.DefaultRedundancySettings if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { @@ -547,7 +547,7 @@ func (b *bus) initSettings(ctx context.Context) error { } // saveAccounts saves the accounts to the db when the bus is stopped -func (b *bus) saveAccounts(ctx context.Context) error { +func (b *Bus) saveAccounts(ctx context.Context) error { accounts := b.accounts.ToPersist() if err := b.eas.SaveAccounts(ctx, accounts); err != nil { b.logger.Errorf("failed to save %v accounts: %v", len(accounts), err) diff --git a/bus/routes.go b/bus/routes.go index 81f180402..f6e97ee06 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -32,7 +32,7 @@ import ( "go.uber.org/zap" ) -func (b *bus) fetchSetting(ctx context.Context, key string, value interface{}) error { +func (b *Bus) fetchSetting(ctx context.Context, key string, value interface{}) error { if val, err := b.ss.Setting(ctx, key); err != nil { return fmt.Errorf("could not get contract set settings: %w", err) } else if err := json.Unmarshal([]byte(val), &value); err != nil { @@ -41,7 +41,7 @@ func (b *bus) fetchSetting(ctx context.Context, key string, value interface{}) e return nil } -func (b *bus) consensusAcceptBlock(jc jape.Context) { +func (b *Bus) consensusAcceptBlock(jc jape.Context) { var block types.Block if jc.Decode(&block) != nil { return @@ -61,11 +61,11 @@ func (b *bus) consensusAcceptBlock(jc jape.Context) { } } -func (b *bus) syncerAddrHandler(jc jape.Context) { +func (b *Bus) syncerAddrHandler(jc jape.Context) { jc.Encode(b.s.Addr()) } -func (b *bus) syncerPeersHandler(jc jape.Context) { +func (b *Bus) syncerPeersHandler(jc jape.Context) { var peers []string for _, p := range b.s.Peers() { peers = append(peers, p.String()) @@ -73,7 +73,7 @@ func (b *bus) syncerPeersHandler(jc jape.Context) { jc.Encode(peers) } -func (b *bus) syncerConnectHandler(jc jape.Context) { +func (b *Bus) syncerConnectHandler(jc jape.Context) { var addr string if jc.Decode(&addr) == nil { _, err := b.s.Connect(jc.Request.Context(), addr) @@ -81,7 +81,7 @@ func (b *bus) syncerConnectHandler(jc jape.Context) { } } -func (b *bus) consensusStateHandler(jc jape.Context) { +func (b *Bus) consensusStateHandler(jc jape.Context) { cs, err := b.consensusState(jc.Request.Context()) if jc.Check("couldn't fetch consensus state", err) != nil { return @@ -89,21 +89,21 @@ func (b *bus) consensusStateHandler(jc jape.Context) { jc.Encode(cs) } -func (b *bus) consensusNetworkHandler(jc jape.Context) { +func (b *Bus) consensusNetworkHandler(jc jape.Context) { jc.Encode(api.ConsensusNetwork{ Name: b.cm.TipState().Network.Name, }) } -func (b *bus) txpoolFeeHandler(jc jape.Context) { +func (b *Bus) txpoolFeeHandler(jc jape.Context) { jc.Encode(b.cm.RecommendedFee()) } -func (b *bus) txpoolTransactionsHandler(jc jape.Context) { +func (b *Bus) txpoolTransactionsHandler(jc jape.Context) { jc.Encode(b.cm.PoolTransactions()) } -func (b *bus) txpoolBroadcastHandler(jc jape.Context) { +func (b *Bus) txpoolBroadcastHandler(jc jape.Context) { var txnSet []types.Transaction if jc.Decode(&txnSet) != nil { return @@ -117,7 +117,7 @@ func (b *bus) txpoolBroadcastHandler(jc jape.Context) { b.s.BroadcastTransactionSet(txnSet) } -func (b *bus) bucketsHandlerGET(jc jape.Context) { +func (b *Bus) bucketsHandlerGET(jc jape.Context) { resp, err := b.ms.ListBuckets(jc.Request.Context()) if jc.Check("couldn't list buckets", err) != nil { return @@ -125,7 +125,7 @@ func (b *bus) bucketsHandlerGET(jc jape.Context) { jc.Encode(resp) } -func (b *bus) bucketsHandlerPOST(jc jape.Context) { +func (b *Bus) bucketsHandlerPOST(jc jape.Context) { var bucket api.BucketCreateRequest if jc.Decode(&bucket) != nil { return @@ -137,7 +137,7 @@ func (b *bus) bucketsHandlerPOST(jc jape.Context) { } } -func (b *bus) bucketsHandlerPolicyPUT(jc jape.Context) { +func (b *Bus) bucketsHandlerPolicyPUT(jc jape.Context) { var req api.BucketUpdatePolicyRequest if jc.Decode(&req) != nil { return @@ -149,7 +149,7 @@ func (b *bus) bucketsHandlerPolicyPUT(jc jape.Context) { } } -func (b *bus) bucketHandlerDELETE(jc jape.Context) { +func (b *Bus) bucketHandlerDELETE(jc jape.Context) { var name string if jc.DecodeParam("name", &name) != nil { return @@ -161,7 +161,7 @@ func (b *bus) bucketHandlerDELETE(jc jape.Context) { } } -func (b *bus) bucketHandlerGET(jc jape.Context) { +func (b *Bus) bucketHandlerGET(jc jape.Context) { var name string if jc.DecodeParam("name", &name) != nil { return @@ -179,7 +179,7 @@ func (b *bus) bucketHandlerGET(jc jape.Context) { jc.Encode(bucket) } -func (b *bus) walletHandler(jc jape.Context) { +func (b *Bus) walletHandler(jc jape.Context) { address := b.w.Address() balance, err := b.w.Balance() if jc.Check("couldn't fetch wallet balance", err) != nil { @@ -201,7 +201,7 @@ func (b *bus) walletHandler(jc jape.Context) { }) } -func (b *bus) walletTransactionsHandler(jc jape.Context) { +func (b *Bus) walletTransactionsHandler(jc jape.Context) { offset := 0 limit := -1 if jc.DecodeForm("offset", &offset) != nil || @@ -287,7 +287,7 @@ func (b *bus) walletTransactionsHandler(jc jape.Context) { } } -func (b *bus) walletOutputsHandler(jc jape.Context) { +func (b *Bus) walletOutputsHandler(jc jape.Context) { utxos, err := b.w.SpendableOutputs() if jc.Check("couldn't load outputs", err) == nil { // convert to siacoin elements @@ -306,7 +306,7 @@ func (b *bus) walletOutputsHandler(jc jape.Context) { } } -func (b *bus) walletFundHandler(jc jape.Context) { +func (b *Bus) walletFundHandler(jc jape.Context) { var wfr api.WalletFundRequest if jc.Decode(&wfr) != nil { return @@ -331,7 +331,7 @@ func (b *bus) walletFundHandler(jc jape.Context) { }) } -func (b *bus) walletSignHandler(jc jape.Context) { +func (b *Bus) walletSignHandler(jc jape.Context) { var wsr api.WalletSignRequest if jc.Decode(&wsr) != nil { return @@ -340,7 +340,7 @@ func (b *bus) walletSignHandler(jc jape.Context) { jc.Encode(wsr.Transaction) } -func (b *bus) walletRedistributeHandler(jc jape.Context) { +func (b *Bus) walletRedistributeHandler(jc jape.Context) { var wfr api.WalletRedistributeRequest if jc.Decode(&wfr) != nil { return @@ -375,14 +375,14 @@ func (b *bus) walletRedistributeHandler(jc jape.Context) { jc.Encode(ids) } -func (b *bus) walletDiscardHandler(jc jape.Context) { +func (b *Bus) walletDiscardHandler(jc jape.Context) { var txn types.Transaction if jc.Decode(&txn) == nil { b.w.ReleaseInputs([]types.Transaction{txn}, nil) } } -func (b *bus) walletPrepareFormHandler(jc jape.Context) { +func (b *Bus) walletPrepareFormHandler(jc jape.Context) { var wpfr api.WalletPrepareFormRequest if jc.Decode(&wpfr) != nil { return @@ -413,7 +413,7 @@ func (b *bus) walletPrepareFormHandler(jc jape.Context) { jc.Encode(append(b.cm.UnconfirmedParents(txn), txn)) } -func (b *bus) walletPrepareRenewHandler(jc jape.Context) { +func (b *Bus) walletPrepareRenewHandler(jc jape.Context) { var wprr api.WalletPrepareRenewRequest if jc.Decode(&wprr) != nil { return @@ -470,7 +470,7 @@ func (b *bus) walletPrepareRenewHandler(jc jape.Context) { }) } -func (b *bus) walletPendingHandler(jc jape.Context) { +func (b *Bus) walletPendingHandler(jc jape.Context) { isRelevant := func(txn types.Transaction) bool { addr := b.w.Address() for _, sci := range txn.SiacoinInputs { @@ -496,7 +496,7 @@ func (b *bus) walletPendingHandler(jc jape.Context) { jc.Encode(relevant) } -func (b *bus) hostsHandlerGETDeprecated(jc jape.Context) { +func (b *Bus) hostsHandlerGETDeprecated(jc jape.Context) { offset := 0 limit := -1 if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil { @@ -511,7 +511,7 @@ func (b *bus) hostsHandlerGETDeprecated(jc jape.Context) { jc.Encode(hosts) } -func (b *bus) searchHostsHandlerPOST(jc jape.Context) { +func (b *Bus) searchHostsHandlerPOST(jc jape.Context) { var req api.SearchHostsRequest if jc.Decode(&req) != nil { return @@ -528,7 +528,7 @@ func (b *bus) searchHostsHandlerPOST(jc jape.Context) { jc.Encode(hosts) } -func (b *bus) hostsRemoveHandlerPOST(jc jape.Context) { +func (b *Bus) hostsRemoveHandlerPOST(jc jape.Context) { var hrr api.HostsRemoveRequest if jc.Decode(&hrr) != nil { return @@ -548,7 +548,7 @@ func (b *bus) hostsRemoveHandlerPOST(jc jape.Context) { jc.Encode(removed) } -func (b *bus) hostsScanningHandlerGET(jc jape.Context) { +func (b *Bus) hostsScanningHandlerGET(jc jape.Context) { offset := 0 limit := -1 maxLastScan := time.Now() @@ -562,7 +562,7 @@ func (b *bus) hostsScanningHandlerGET(jc jape.Context) { jc.Encode(hosts) } -func (b *bus) hostsPubkeyHandlerGET(jc jape.Context) { +func (b *Bus) hostsPubkeyHandlerGET(jc jape.Context) { var hostKey types.PublicKey if jc.DecodeParam("hostkey", &hostKey) != nil { return @@ -573,7 +573,7 @@ func (b *bus) hostsPubkeyHandlerGET(jc jape.Context) { } } -func (b *bus) hostsResetLostSectorsPOST(jc jape.Context) { +func (b *Bus) hostsResetLostSectorsPOST(jc jape.Context) { var hostKey types.PublicKey if jc.DecodeParam("hostkey", &hostKey) != nil { return @@ -584,7 +584,7 @@ func (b *bus) hostsResetLostSectorsPOST(jc jape.Context) { } } -func (b *bus) hostsScanHandlerPOST(jc jape.Context) { +func (b *Bus) hostsScanHandlerPOST(jc jape.Context) { var req api.HostsScanRequest if jc.Decode(&req) != nil { return @@ -594,7 +594,7 @@ func (b *bus) hostsScanHandlerPOST(jc jape.Context) { } } -func (b *bus) hostsPricetableHandlerPOST(jc jape.Context) { +func (b *Bus) hostsPricetableHandlerPOST(jc jape.Context) { var req api.HostsPriceTablesRequest if jc.Decode(&req) != nil { return @@ -604,7 +604,7 @@ func (b *bus) hostsPricetableHandlerPOST(jc jape.Context) { } } -func (b *bus) contractsSpendingHandlerPOST(jc jape.Context) { +func (b *Bus) contractsSpendingHandlerPOST(jc jape.Context) { var records []api.ContractSpendingRecord if jc.Decode(&records) != nil { return @@ -614,14 +614,14 @@ func (b *bus) contractsSpendingHandlerPOST(jc jape.Context) { } } -func (b *bus) hostsAllowlistHandlerGET(jc jape.Context) { +func (b *Bus) hostsAllowlistHandlerGET(jc jape.Context) { allowlist, err := b.hdb.HostAllowlist(jc.Request.Context()) if jc.Check("couldn't load allowlist", err) == nil { jc.Encode(allowlist) } } -func (b *bus) hostsAllowlistHandlerPUT(jc jape.Context) { +func (b *Bus) hostsAllowlistHandlerPUT(jc jape.Context) { ctx := jc.Request.Context() var req api.UpdateAllowlistRequest if jc.Decode(&req) == nil { @@ -634,14 +634,14 @@ func (b *bus) hostsAllowlistHandlerPUT(jc jape.Context) { } } -func (b *bus) hostsBlocklistHandlerGET(jc jape.Context) { +func (b *Bus) hostsBlocklistHandlerGET(jc jape.Context) { blocklist, err := b.hdb.HostBlocklist(jc.Request.Context()) if jc.Check("couldn't load blocklist", err) == nil { jc.Encode(blocklist) } } -func (b *bus) hostsBlocklistHandlerPUT(jc jape.Context) { +func (b *Bus) hostsBlocklistHandlerPUT(jc jape.Context) { ctx := jc.Request.Context() var req api.UpdateBlocklistRequest if jc.Decode(&req) == nil { @@ -654,7 +654,7 @@ func (b *bus) hostsBlocklistHandlerPUT(jc jape.Context) { } } -func (b *bus) contractsHandlerGET(jc jape.Context) { +func (b *Bus) contractsHandlerGET(jc jape.Context) { var cs string if jc.DecodeForm("contractset", &cs) != nil { return @@ -667,7 +667,7 @@ func (b *bus) contractsHandlerGET(jc jape.Context) { } } -func (b *bus) contractsRenewedIDHandlerGET(jc jape.Context) { +func (b *Bus) contractsRenewedIDHandlerGET(jc jape.Context) { var id types.FileContractID if jc.DecodeParam("id", &id) != nil { return @@ -679,7 +679,7 @@ func (b *bus) contractsRenewedIDHandlerGET(jc jape.Context) { } } -func (b *bus) contractsArchiveHandlerPOST(jc jape.Context) { +func (b *Bus) contractsArchiveHandlerPOST(jc jape.Context) { var toArchive api.ContractsArchiveRequest if jc.Decode(&toArchive) != nil { return @@ -700,14 +700,14 @@ func (b *bus) contractsArchiveHandlerPOST(jc jape.Context) { } } -func (b *bus) contractsSetsHandlerGET(jc jape.Context) { +func (b *Bus) contractsSetsHandlerGET(jc jape.Context) { sets, err := b.ms.ContractSets(jc.Request.Context()) if jc.Check("couldn't fetch contract sets", err) == nil { jc.Encode(sets) } } -func (b *bus) contractsSetHandlerPUT(jc jape.Context) { +func (b *Bus) contractsSetHandlerPUT(jc jape.Context) { var contractIds []types.FileContractID if set := jc.PathParam("set"); set == "" { jc.Error(errors.New("path parameter 'set' can not be empty"), http.StatusBadRequest) @@ -729,13 +729,13 @@ func (b *bus) contractsSetHandlerPUT(jc jape.Context) { } } -func (b *bus) contractsSetHandlerDELETE(jc jape.Context) { +func (b *Bus) contractsSetHandlerDELETE(jc jape.Context) { if set := jc.PathParam("set"); set != "" { jc.Check("could not remove contract set", b.ms.RemoveContractSet(jc.Request.Context(), set)) } } -func (b *bus) contractAcquireHandlerPOST(jc jape.Context) { +func (b *Bus) contractAcquireHandlerPOST(jc jape.Context) { var id types.FileContractID if jc.DecodeParam("id", &id) != nil { return @@ -754,7 +754,7 @@ func (b *bus) contractAcquireHandlerPOST(jc jape.Context) { }) } -func (b *bus) contractKeepaliveHandlerPOST(jc jape.Context) { +func (b *Bus) contractKeepaliveHandlerPOST(jc jape.Context) { var id types.FileContractID if jc.DecodeParam("id", &id) != nil { return @@ -770,7 +770,7 @@ func (b *bus) contractKeepaliveHandlerPOST(jc jape.Context) { } } -func (b *bus) contractsPrunableDataHandlerGET(jc jape.Context) { +func (b *Bus) contractsPrunableDataHandlerGET(jc jape.Context) { sizes, err := b.ms.ContractSizes(jc.Request.Context()) if jc.Check("failed to fetch contract sizes", err) != nil { return @@ -815,7 +815,7 @@ func (b *bus) contractsPrunableDataHandlerGET(jc jape.Context) { }) } -func (b *bus) contractSizeHandlerGET(jc jape.Context) { +func (b *Bus) contractSizeHandlerGET(jc jape.Context) { var id types.FileContractID if jc.DecodeParam("id", &id) != nil { return @@ -842,7 +842,7 @@ func (b *bus) contractSizeHandlerGET(jc jape.Context) { jc.Encode(size) } -func (b *bus) contractReleaseHandlerPOST(jc jape.Context) { +func (b *Bus) contractReleaseHandlerPOST(jc jape.Context) { var id types.FileContractID if jc.DecodeParam("id", &id) != nil { return @@ -856,7 +856,7 @@ func (b *bus) contractReleaseHandlerPOST(jc jape.Context) { } } -func (b *bus) contractIDHandlerGET(jc jape.Context) { +func (b *Bus) contractIDHandlerGET(jc jape.Context) { var id types.FileContractID if jc.DecodeParam("id", &id) != nil { return @@ -867,7 +867,7 @@ func (b *bus) contractIDHandlerGET(jc jape.Context) { } } -func (b *bus) contractIDHandlerPOST(jc jape.Context) { +func (b *Bus) contractIDHandlerPOST(jc jape.Context) { var id types.FileContractID var req api.ContractAddRequest if jc.DecodeParam("id", &id) != nil || jc.Decode(&req) != nil { @@ -888,7 +888,7 @@ func (b *bus) contractIDHandlerPOST(jc jape.Context) { } } -func (b *bus) contractIDRenewedHandlerPOST(jc jape.Context) { +func (b *Bus) contractIDRenewedHandlerPOST(jc jape.Context) { var id types.FileContractID var req api.ContractRenewedRequest if jc.DecodeParam("id", &id) != nil || jc.Decode(&req) != nil { @@ -923,7 +923,7 @@ func (b *bus) contractIDRenewedHandlerPOST(jc jape.Context) { jc.Encode(r) } -func (b *bus) contractIDRootsHandlerGET(jc jape.Context) { +func (b *Bus) contractIDRootsHandlerGET(jc jape.Context) { var id types.FileContractID if jc.DecodeParam("id", &id) != nil { return @@ -938,7 +938,7 @@ func (b *bus) contractIDRootsHandlerGET(jc jape.Context) { } } -func (b *bus) contractIDHandlerDELETE(jc jape.Context) { +func (b *Bus) contractIDHandlerDELETE(jc jape.Context) { var id types.FileContractID if jc.DecodeParam("id", &id) != nil { return @@ -946,11 +946,11 @@ func (b *bus) contractIDHandlerDELETE(jc jape.Context) { jc.Check("couldn't remove contract", b.ms.ArchiveContract(jc.Request.Context(), id, api.ContractArchivalReasonRemoved)) } -func (b *bus) contractsAllHandlerDELETE(jc jape.Context) { +func (b *Bus) contractsAllHandlerDELETE(jc jape.Context) { jc.Check("couldn't remove contracts", b.ms.ArchiveAllContracts(jc.Request.Context(), api.ContractArchivalReasonRemoved)) } -func (b *bus) searchObjectsHandlerGET(jc jape.Context) { +func (b *Bus) searchObjectsHandlerGET(jc jape.Context) { offset := 0 limit := -1 var key string @@ -968,7 +968,7 @@ func (b *bus) searchObjectsHandlerGET(jc jape.Context) { jc.Encode(keys) } -func (b *bus) objectsHandlerGET(jc jape.Context) { +func (b *Bus) objectsHandlerGET(jc jape.Context) { var ignoreDelim bool if jc.DecodeForm("ignoreDelim", &ignoreDelim) != nil { return @@ -1003,7 +1003,7 @@ func (b *bus) objectsHandlerGET(jc jape.Context) { jc.Encode(api.ObjectsResponse{Object: &o}) } -func (b *bus) objectEntriesHandlerGET(jc jape.Context, path string) { +func (b *Bus) objectEntriesHandlerGET(jc jape.Context, path string) { bucket := api.DefaultBucketName if jc.DecodeForm("bucket", &bucket) != nil { return @@ -1047,7 +1047,7 @@ func (b *bus) objectEntriesHandlerGET(jc jape.Context, path string) { jc.Encode(api.ObjectsResponse{Entries: entries, HasMore: hasMore}) } -func (b *bus) objectsHandlerPUT(jc jape.Context) { +func (b *Bus) objectsHandlerPUT(jc jape.Context) { var aor api.AddObjectRequest if jc.Decode(&aor) != nil { return @@ -1057,7 +1057,7 @@ func (b *bus) objectsHandlerPUT(jc jape.Context) { jc.Check("couldn't store object", b.ms.UpdateObject(jc.Request.Context(), aor.Bucket, jc.PathParam("path"), aor.ContractSet, aor.ETag, aor.MimeType, aor.Metadata, aor.Object)) } -func (b *bus) objectsCopyHandlerPOST(jc jape.Context) { +func (b *Bus) objectsCopyHandlerPOST(jc jape.Context) { var orr api.CopyObjectsRequest if jc.Decode(&orr) != nil { return @@ -1072,7 +1072,7 @@ func (b *bus) objectsCopyHandlerPOST(jc jape.Context) { jc.Encode(om) } -func (b *bus) objectsListHandlerPOST(jc jape.Context) { +func (b *Bus) objectsListHandlerPOST(jc jape.Context) { var req api.ObjectsListRequest if jc.Decode(&req) != nil { return @@ -1090,7 +1090,7 @@ func (b *bus) objectsListHandlerPOST(jc jape.Context) { jc.Encode(resp) } -func (b *bus) objectsRenameHandlerPOST(jc jape.Context) { +func (b *Bus) objectsRenameHandlerPOST(jc jape.Context) { var orr api.ObjectsRenameRequest if jc.Decode(&orr) != nil { return @@ -1120,7 +1120,7 @@ func (b *bus) objectsRenameHandlerPOST(jc jape.Context) { } } -func (b *bus) objectsHandlerDELETE(jc jape.Context) { +func (b *Bus) objectsHandlerDELETE(jc jape.Context) { var batch bool if jc.DecodeForm("batch", &batch) != nil { return @@ -1142,7 +1142,7 @@ func (b *bus) objectsHandlerDELETE(jc jape.Context) { jc.Check("couldn't delete object", err) } -func (b *bus) slabbuffersHandlerGET(jc jape.Context) { +func (b *Bus) slabbuffersHandlerGET(jc jape.Context) { buffers, err := b.ms.SlabBuffers(jc.Request.Context()) if jc.Check("couldn't get slab buffers info", err) != nil { return @@ -1150,7 +1150,7 @@ func (b *bus) slabbuffersHandlerGET(jc jape.Context) { jc.Encode(buffers) } -func (b *bus) objectsStatshandlerGET(jc jape.Context) { +func (b *Bus) objectsStatshandlerGET(jc jape.Context) { opts := api.ObjectsStatsOpts{} if jc.DecodeForm("bucket", &opts.Bucket) != nil { return @@ -1162,7 +1162,7 @@ func (b *bus) objectsStatshandlerGET(jc jape.Context) { jc.Encode(info) } -func (b *bus) packedSlabsHandlerFetchPOST(jc jape.Context) { +func (b *Bus) packedSlabsHandlerFetchPOST(jc jape.Context) { var psrg api.PackedSlabsRequestGET if jc.Decode(&psrg) != nil { return @@ -1186,7 +1186,7 @@ func (b *bus) packedSlabsHandlerFetchPOST(jc jape.Context) { jc.Encode(slabs) } -func (b *bus) packedSlabsHandlerDonePOST(jc jape.Context) { +func (b *Bus) packedSlabsHandlerDonePOST(jc jape.Context) { var psrp api.PackedSlabsRequestPOST if jc.Decode(&psrp) != nil { return @@ -1194,7 +1194,7 @@ func (b *bus) packedSlabsHandlerDonePOST(jc jape.Context) { jc.Check("failed to mark packed slab(s) as uploaded", b.ms.MarkPackedSlabsUploaded(jc.Request.Context(), psrp.Slabs)) } -func (b *bus) sectorsHostRootHandlerDELETE(jc jape.Context) { +func (b *Bus) sectorsHostRootHandlerDELETE(jc jape.Context) { var hk types.PublicKey var root types.Hash256 if jc.DecodeParam("hk", &hk) != nil { @@ -1210,7 +1210,7 @@ func (b *bus) sectorsHostRootHandlerDELETE(jc jape.Context) { } } -func (b *bus) slabObjectsHandlerGET(jc jape.Context) { +func (b *Bus) slabObjectsHandlerGET(jc jape.Context) { var key object.EncryptionKey if jc.DecodeParam("key", &key) != nil { return @@ -1226,7 +1226,7 @@ func (b *bus) slabObjectsHandlerGET(jc jape.Context) { jc.Encode(objects) } -func (b *bus) slabHandlerGET(jc jape.Context) { +func (b *Bus) slabHandlerGET(jc jape.Context) { var key object.EncryptionKey if jc.DecodeParam("key", &key) != nil { return @@ -1242,18 +1242,18 @@ func (b *bus) slabHandlerGET(jc jape.Context) { jc.Encode(slab) } -func (b *bus) slabHandlerPUT(jc jape.Context) { +func (b *Bus) slabHandlerPUT(jc jape.Context) { var usr api.UpdateSlabRequest if jc.Decode(&usr) == nil { jc.Check("couldn't update slab", b.ms.UpdateSlab(jc.Request.Context(), usr.Slab, usr.ContractSet)) } } -func (b *bus) slabsRefreshHealthHandlerPOST(jc jape.Context) { +func (b *Bus) slabsRefreshHealthHandlerPOST(jc jape.Context) { jc.Check("failed to recompute health", b.ms.RefreshHealth(jc.Request.Context())) } -func (b *bus) slabsMigrationHandlerPOST(jc jape.Context) { +func (b *Bus) slabsMigrationHandlerPOST(jc jape.Context) { var msr api.MigrationSlabsRequest if jc.Decode(&msr) == nil { if slabs, err := b.ms.UnhealthySlabs(jc.Request.Context(), msr.HealthCutoff, msr.ContractSet, msr.Limit); jc.Check("couldn't fetch slabs for migration", err) == nil { @@ -1264,7 +1264,7 @@ func (b *bus) slabsMigrationHandlerPOST(jc jape.Context) { } } -func (b *bus) slabsPartialHandlerGET(jc jape.Context) { +func (b *Bus) slabsPartialHandlerGET(jc jape.Context) { jc.Custom(nil, []byte{}) var key object.EncryptionKey @@ -1294,7 +1294,7 @@ func (b *bus) slabsPartialHandlerGET(jc jape.Context) { jc.ResponseWriter.Write(data) } -func (b *bus) slabsPartialHandlerPOST(jc jape.Context) { +func (b *Bus) slabsPartialHandlerPOST(jc jape.Context) { var minShards int if jc.DecodeForm("minShards", &minShards) != nil { return @@ -1338,13 +1338,13 @@ func (b *bus) slabsPartialHandlerPOST(jc jape.Context) { }) } -func (b *bus) settingsHandlerGET(jc jape.Context) { +func (b *Bus) settingsHandlerGET(jc jape.Context) { if settings, err := b.ss.Settings(jc.Request.Context()); jc.Check("couldn't load settings", err) == nil { jc.Encode(settings) } } -func (b *bus) settingKeyHandlerGET(jc jape.Context) { +func (b *Bus) settingKeyHandlerGET(jc jape.Context) { jc.Custom(nil, (any)(nil)) key := jc.PathParam("key") @@ -1392,7 +1392,7 @@ func (b *bus) settingKeyHandlerGET(jc jape.Context) { jc.ResponseWriter.Write(resp) } -func (b *bus) settingKeyHandlerPUT(jc jape.Context) { +func (b *Bus) settingKeyHandlerPUT(jc jape.Context) { key := jc.PathParam("key") if key == "" { jc.Error(errors.New("path parameter 'key' can not be empty"), http.StatusBadRequest) @@ -1469,7 +1469,7 @@ func (b *bus) settingKeyHandlerPUT(jc jape.Context) { } } -func (b *bus) settingKeyHandlerDELETE(jc jape.Context) { +func (b *Bus) settingKeyHandlerDELETE(jc jape.Context) { key := jc.PathParam("key") if key == "" { jc.Error(errors.New("path parameter 'key' can not be empty"), http.StatusBadRequest) @@ -1488,7 +1488,7 @@ func (b *bus) settingKeyHandlerDELETE(jc jape.Context) { } } -func (b *bus) contractIDAncestorsHandler(jc jape.Context) { +func (b *Bus) contractIDAncestorsHandler(jc jape.Context) { var fcid types.FileContractID if jc.DecodeParam("id", &fcid) != nil { return @@ -1504,7 +1504,7 @@ func (b *bus) contractIDAncestorsHandler(jc jape.Context) { jc.Encode(ancestors) } -func (b *bus) paramsHandlerUploadGET(jc jape.Context) { +func (b *Bus) paramsHandlerUploadGET(jc jape.Context) { gp, err := b.gougingParams(jc.Request.Context()) if jc.Check("could not get gouging parameters", err) != nil { return @@ -1536,7 +1536,7 @@ func (b *bus) paramsHandlerUploadGET(jc jape.Context) { }) } -func (b *bus) consensusState(ctx context.Context) (api.ConsensusState, error) { +func (b *Bus) consensusState(ctx context.Context) (api.ConsensusState, error) { index, err := b.cs.ChainIndex(ctx) if err != nil { return api.ConsensusState{}, err @@ -1555,7 +1555,7 @@ func (b *bus) consensusState(ctx context.Context) (api.ConsensusState, error) { }, nil } -func (b *bus) paramsHandlerGougingGET(jc jape.Context) { +func (b *Bus) paramsHandlerGougingGET(jc jape.Context) { gp, err := b.gougingParams(jc.Request.Context()) if jc.Check("could not get gouging parameters", err) != nil { return @@ -1563,7 +1563,7 @@ func (b *bus) paramsHandlerGougingGET(jc jape.Context) { jc.Encode(gp) } -func (b *bus) gougingParams(ctx context.Context) (api.GougingParams, error) { +func (b *Bus) gougingParams(ctx context.Context) (api.GougingParams, error) { var gs api.GougingSettings if gss, err := b.ss.Setting(ctx, api.SettingGouging); err != nil { return api.GougingParams{}, err @@ -1591,7 +1591,7 @@ func (b *bus) gougingParams(ctx context.Context) (api.GougingParams, error) { }, nil } -func (b *bus) handleGETAlertsDeprecated(jc jape.Context) { +func (b *Bus) handleGETAlertsDeprecated(jc jape.Context) { ar, err := b.alertMgr.Alerts(jc.Request.Context(), alerts.AlertsOpts{Offset: 0, Limit: -1}) if jc.Check("failed to fetch alerts", err) != nil { return @@ -1599,7 +1599,7 @@ func (b *bus) handleGETAlertsDeprecated(jc jape.Context) { jc.Encode(ar.Alerts) } -func (b *bus) handleGETAlerts(jc jape.Context) { +func (b *Bus) handleGETAlerts(jc jape.Context) { if jc.Request.FormValue("offset") == "" && jc.Request.FormValue("limit") == "" { b.handleGETAlertsDeprecated(jc) return @@ -1627,7 +1627,7 @@ func (b *bus) handleGETAlerts(jc jape.Context) { jc.Encode(ar) } -func (b *bus) handlePOSTAlertsDismiss(jc jape.Context) { +func (b *Bus) handlePOSTAlertsDismiss(jc jape.Context) { var ids []types.Hash256 if jc.Decode(&ids) != nil { return @@ -1635,7 +1635,7 @@ func (b *bus) handlePOSTAlertsDismiss(jc jape.Context) { jc.Check("failed to dismiss alerts", b.alertMgr.DismissAlerts(jc.Request.Context(), ids...)) } -func (b *bus) handlePOSTAlertsRegister(jc jape.Context) { +func (b *Bus) handlePOSTAlertsRegister(jc jape.Context) { var alert alerts.Alert if jc.Decode(&alert) != nil { return @@ -1643,11 +1643,11 @@ func (b *bus) handlePOSTAlertsRegister(jc jape.Context) { jc.Check("failed to register alert", b.alertMgr.RegisterAlert(jc.Request.Context(), alert)) } -func (b *bus) accountsHandlerGET(jc jape.Context) { +func (b *Bus) accountsHandlerGET(jc jape.Context) { jc.Encode(b.accounts.Accounts()) } -func (b *bus) accountHandlerGET(jc jape.Context) { +func (b *Bus) accountHandlerGET(jc jape.Context) { var id rhpv3.Account if jc.DecodeParam("id", &id) != nil { return @@ -1663,7 +1663,7 @@ func (b *bus) accountHandlerGET(jc jape.Context) { jc.Encode(acc) } -func (b *bus) accountsAddHandlerPOST(jc jape.Context) { +func (b *Bus) accountsAddHandlerPOST(jc jape.Context) { var id rhpv3.Account if jc.DecodeParam("id", &id) != nil { return @@ -1683,7 +1683,7 @@ func (b *bus) accountsAddHandlerPOST(jc jape.Context) { b.accounts.AddAmount(id, req.HostKey, req.Amount) } -func (b *bus) accountsResetDriftHandlerPOST(jc jape.Context) { +func (b *Bus) accountsResetDriftHandlerPOST(jc jape.Context) { var id rhpv3.Account if jc.DecodeParam("id", &id) != nil { return @@ -1698,7 +1698,7 @@ func (b *bus) accountsResetDriftHandlerPOST(jc jape.Context) { } } -func (b *bus) accountsUpdateHandlerPOST(jc jape.Context) { +func (b *Bus) accountsUpdateHandlerPOST(jc jape.Context) { var id rhpv3.Account if jc.DecodeParam("id", &id) != nil { return @@ -1718,7 +1718,7 @@ func (b *bus) accountsUpdateHandlerPOST(jc jape.Context) { b.accounts.SetBalance(id, req.HostKey, req.Amount) } -func (b *bus) accountsRequiresSyncHandlerPOST(jc jape.Context) { +func (b *Bus) accountsRequiresSyncHandlerPOST(jc jape.Context) { var id rhpv3.Account if jc.DecodeParam("id", &id) != nil { return @@ -1745,7 +1745,7 @@ func (b *bus) accountsRequiresSyncHandlerPOST(jc jape.Context) { } } -func (b *bus) accountsLockHandlerPOST(jc jape.Context) { +func (b *Bus) accountsLockHandlerPOST(jc jape.Context) { var id rhpv3.Account if jc.DecodeParam("id", &id) != nil { return @@ -1762,7 +1762,7 @@ func (b *bus) accountsLockHandlerPOST(jc jape.Context) { }) } -func (b *bus) accountsUnlockHandlerPOST(jc jape.Context) { +func (b *Bus) accountsUnlockHandlerPOST(jc jape.Context) { var id rhpv3.Account if jc.DecodeParam("id", &id) != nil { return @@ -1778,13 +1778,13 @@ func (b *bus) accountsUnlockHandlerPOST(jc jape.Context) { } } -func (b *bus) autopilotsListHandlerGET(jc jape.Context) { +func (b *Bus) autopilotsListHandlerGET(jc jape.Context) { if autopilots, err := b.as.Autopilots(jc.Request.Context()); jc.Check("failed to fetch autopilots", err) == nil { jc.Encode(autopilots) } } -func (b *bus) autopilotsHandlerGET(jc jape.Context) { +func (b *Bus) autopilotsHandlerGET(jc jape.Context) { var id string if jc.DecodeParam("id", &id) != nil { return @@ -1801,7 +1801,7 @@ func (b *bus) autopilotsHandlerGET(jc jape.Context) { jc.Encode(ap) } -func (b *bus) autopilotsHandlerPUT(jc jape.Context) { +func (b *Bus) autopilotsHandlerPUT(jc jape.Context) { var id string if jc.DecodeParam("id", &id) != nil { return @@ -1822,7 +1822,7 @@ func (b *bus) autopilotsHandlerPUT(jc jape.Context) { } } -func (b *bus) autopilotHostCheckHandlerPUT(jc jape.Context) { +func (b *Bus) autopilotHostCheckHandlerPUT(jc jape.Context) { var id string if jc.DecodeParam("id", &id) != nil { return @@ -1845,7 +1845,7 @@ func (b *bus) autopilotHostCheckHandlerPUT(jc jape.Context) { } } -func (b *bus) broadcastAction(e webhooks.Event) { +func (b *Bus) broadcastAction(e webhooks.Event) { log := b.logger.With("event", e.Event).With("module", e.Module) err := b.webhooksMgr.BroadcastAction(context.Background(), e) if err != nil { @@ -1855,7 +1855,7 @@ func (b *bus) broadcastAction(e webhooks.Event) { } } -func (b *bus) contractTaxHandlerGET(jc jape.Context) { +func (b *Bus) contractTaxHandlerGET(jc jape.Context) { var payout types.Currency if jc.DecodeParam("payout", (*api.ParamCurrency)(&payout)) != nil { return @@ -1864,7 +1864,7 @@ func (b *bus) contractTaxHandlerGET(jc jape.Context) { jc.Encode(cs.FileContractTax(types.FileContract{Payout: payout})) } -func (b *bus) stateHandlerGET(jc jape.Context) { +func (b *Bus) stateHandlerGET(jc jape.Context) { jc.Encode(api.BusStateResponse{ StartTime: api.TimeRFC3339(b.startTime), BuildState: api.BuildState{ @@ -1877,14 +1877,14 @@ func (b *bus) stateHandlerGET(jc jape.Context) { }) } -func (b *bus) uploadTrackHandlerPOST(jc jape.Context) { +func (b *Bus) uploadTrackHandlerPOST(jc jape.Context) { var id api.UploadID if jc.DecodeParam("id", &id) == nil { jc.Check("failed to track upload", b.uploadingSectors.StartUpload(id)) } } -func (b *bus) uploadAddSectorHandlerPOST(jc jape.Context) { +func (b *Bus) uploadAddSectorHandlerPOST(jc jape.Context) { var id api.UploadID if jc.DecodeParam("id", &id) != nil { return @@ -1896,14 +1896,14 @@ func (b *bus) uploadAddSectorHandlerPOST(jc jape.Context) { jc.Check("failed to add sector", b.uploadingSectors.AddSector(id, req.ContractID, req.Root)) } -func (b *bus) uploadFinishedHandlerDELETE(jc jape.Context) { +func (b *Bus) uploadFinishedHandlerDELETE(jc jape.Context) { var id api.UploadID if jc.DecodeParam("id", &id) == nil { b.uploadingSectors.FinishUpload(id) } } -func (b *bus) webhookActionHandlerPost(jc jape.Context) { +func (b *Bus) webhookActionHandlerPost(jc jape.Context) { var action webhooks.Event if jc.Check("failed to decode action", jc.Decode(&action)) != nil { return @@ -1911,7 +1911,7 @@ func (b *bus) webhookActionHandlerPost(jc jape.Context) { b.broadcastAction(action) } -func (b *bus) webhookHandlerDelete(jc jape.Context) { +func (b *Bus) webhookHandlerDelete(jc jape.Context) { var wh webhooks.Webhook if jc.Decode(&wh) != nil { return @@ -1925,7 +1925,7 @@ func (b *bus) webhookHandlerDelete(jc jape.Context) { } } -func (b *bus) webhookHandlerGet(jc jape.Context) { +func (b *Bus) webhookHandlerGet(jc jape.Context) { webhooks, queueInfos := b.webhooksMgr.Info() jc.Encode(api.WebhookResponse{ Queues: queueInfos, @@ -1933,7 +1933,7 @@ func (b *bus) webhookHandlerGet(jc jape.Context) { }) } -func (b *bus) webhookHandlerPost(jc jape.Context) { +func (b *Bus) webhookHandlerPost(jc jape.Context) { var req webhooks.Webhook if jc.Decode(&req) != nil { return @@ -1951,7 +1951,7 @@ func (b *bus) webhookHandlerPost(jc jape.Context) { } } -func (b *bus) metricsHandlerDELETE(jc jape.Context) { +func (b *Bus) metricsHandlerDELETE(jc jape.Context) { metric := jc.PathParam("key") if metric == "" { jc.Error(errors.New("parameter 'metric' is required"), http.StatusBadRequest) @@ -1972,7 +1972,7 @@ func (b *bus) metricsHandlerDELETE(jc jape.Context) { } } -func (b *bus) metricsHandlerPUT(jc jape.Context) { +func (b *Bus) metricsHandlerPUT(jc jape.Context) { jc.Custom((*interface{})(nil), nil) key := jc.PathParam("key") @@ -2001,7 +2001,7 @@ func (b *bus) metricsHandlerPUT(jc jape.Context) { } } -func (b *bus) metricsHandlerGET(jc jape.Context) { +func (b *Bus) metricsHandlerGET(jc jape.Context) { // parse mandatory query parameters var start time.Time if jc.DecodeForm("start", (*api.TimeRFC3339)(&start)) != nil { @@ -2086,7 +2086,7 @@ func (b *bus) metricsHandlerGET(jc jape.Context) { jc.Encode(metrics) } -func (b *bus) metrics(ctx context.Context, key string, start time.Time, n uint64, interval time.Duration, opts interface{}) (interface{}, error) { +func (b *Bus) metrics(ctx context.Context, key string, start time.Time, n uint64, interval time.Duration, opts interface{}) (interface{}, error) { switch key { case api.MetricContract: return b.mtrcs.ContractMetrics(ctx, start, n, interval, opts.(api.ContractMetricsQueryOpts)) @@ -2102,7 +2102,7 @@ func (b *bus) metrics(ctx context.Context, key string, start time.Time, n uint64 return nil, fmt.Errorf("unknown metric '%s'", key) } -func (b *bus) multipartHandlerCreatePOST(jc jape.Context) { +func (b *Bus) multipartHandlerCreatePOST(jc jape.Context) { var req api.MultipartCreateRequest if jc.Decode(&req) != nil { return @@ -2124,7 +2124,7 @@ func (b *bus) multipartHandlerCreatePOST(jc jape.Context) { jc.Encode(resp) } -func (b *bus) multipartHandlerAbortPOST(jc jape.Context) { +func (b *Bus) multipartHandlerAbortPOST(jc jape.Context) { var req api.MultipartAbortRequest if jc.Decode(&req) != nil { return @@ -2135,7 +2135,7 @@ func (b *bus) multipartHandlerAbortPOST(jc jape.Context) { } } -func (b *bus) multipartHandlerCompletePOST(jc jape.Context) { +func (b *Bus) multipartHandlerCompletePOST(jc jape.Context) { var req api.MultipartCompleteRequest if jc.Decode(&req) != nil { return @@ -2149,7 +2149,7 @@ func (b *bus) multipartHandlerCompletePOST(jc jape.Context) { jc.Encode(resp) } -func (b *bus) multipartHandlerUploadPartPUT(jc jape.Context) { +func (b *Bus) multipartHandlerUploadPartPUT(jc jape.Context) { var req api.MultipartAddPartRequest if jc.Decode(&req) != nil { return @@ -2175,7 +2175,7 @@ func (b *bus) multipartHandlerUploadPartPUT(jc jape.Context) { } } -func (b *bus) multipartHandlerUploadGET(jc jape.Context) { +func (b *Bus) multipartHandlerUploadGET(jc jape.Context) { resp, err := b.ms.MultipartUpload(jc.Request.Context(), jc.PathParam("id")) if jc.Check("failed to get multipart upload", err) != nil { return @@ -2183,7 +2183,7 @@ func (b *bus) multipartHandlerUploadGET(jc jape.Context) { jc.Encode(resp) } -func (b *bus) multipartHandlerListUploadsPOST(jc jape.Context) { +func (b *Bus) multipartHandlerListUploadsPOST(jc jape.Context) { var req api.MultipartListUploadsRequest if jc.Decode(&req) != nil { return @@ -2195,7 +2195,7 @@ func (b *bus) multipartHandlerListUploadsPOST(jc jape.Context) { jc.Encode(resp) } -func (b *bus) multipartHandlerListPartsPOST(jc jape.Context) { +func (b *Bus) multipartHandlerListPartsPOST(jc jape.Context) { var req api.MultipartListPartsRequest if jc.Decode(&req) != nil { return diff --git a/worker/contract_lock.go b/worker/contract_lock.go index f5115d37f..4f9d147ba 100644 --- a/worker/contract_lock.go +++ b/worker/contract_lock.go @@ -51,7 +51,7 @@ func newContractLock(ctx context.Context, fcid types.FileContractID, lockID uint return cl } -func (w *worker) acquireContractLock(ctx context.Context, fcid types.FileContractID, priority int) (_ *contractLock, err error) { +func (w *Worker) acquireContractLock(ctx context.Context, fcid types.FileContractID, priority int) (_ *contractLock, err error) { lockID, err := w.bus.AcquireContract(ctx, fcid, priority, w.contractLockingDuration) if err != nil { return nil, err @@ -59,7 +59,7 @@ func (w *worker) acquireContractLock(ctx context.Context, fcid types.FileContrac return newContractLock(w.shutdownCtx, fcid, lockID, w.contractLockingDuration, w.bus, w.logger), nil } -func (w *worker) withContractLock(ctx context.Context, fcid types.FileContractID, priority int, fn func() error) error { +func (w *Worker) withContractLock(ctx context.Context, fcid types.FileContractID, priority int, fn func() error) error { contractLock, err := w.acquireContractLock(ctx, fcid, priority) if err != nil { return err diff --git a/worker/download.go b/worker/download.go index 261088700..118120c94 100644 --- a/worker/download.go +++ b/worker/download.go @@ -126,7 +126,7 @@ type ( } ) -func (w *worker) initDownloadManager(maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.SugaredLogger) { +func (w *Worker) initDownloadManager(maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.SugaredLogger) { if w.downloadManager != nil { panic("download manager already initialized") // developer error } diff --git a/worker/host.go b/worker/host.go index ceb9b24b2..56b5876d0 100644 --- a/worker/host.go +++ b/worker/host.go @@ -59,10 +59,10 @@ type ( var ( _ Host = (*host)(nil) - _ HostManager = (*worker)(nil) + _ HostManager = (*Worker)(nil) ) -func (w *worker) Host(hk types.PublicKey, fcid types.FileContractID, siamuxAddr string) Host { +func (w *Worker) Host(hk types.PublicKey, fcid types.FileContractID, siamuxAddr string) Host { return &host{ hk: hk, acc: w.accounts.ForHost(hk), diff --git a/worker/migrations.go b/worker/migrations.go index 075642dd5..d2d1c6474 100644 --- a/worker/migrations.go +++ b/worker/migrations.go @@ -10,7 +10,7 @@ import ( "go.sia.tech/renterd/object" ) -func (w *worker) migrate(ctx context.Context, s object.Slab, contractSet string, dlContracts, ulContracts []api.ContractMetadata, bh uint64) (int, bool, error) { +func (w *Worker) migrate(ctx context.Context, s object.Slab, contractSet string, dlContracts, ulContracts []api.ContractMetadata, bh uint64) (int, bool, error) { // make a map of good hosts goodHosts := make(map[types.PublicKey]map[types.FileContractID]bool) for _, c := range ulContracts { diff --git a/worker/pricetables.go b/worker/pricetables.go index 6cc5f181a..3ffdef459 100644 --- a/worker/pricetables.go +++ b/worker/pricetables.go @@ -58,7 +58,7 @@ type ( } ) -func (w *worker) initPriceTables() { +func (w *Worker) initPriceTables() { if w.priceTables != nil { panic("priceTables already initialized") // developer error } diff --git a/worker/rhpv2.go b/worker/rhpv2.go index 048e42962..890f28fd1 100644 --- a/worker/rhpv2.go +++ b/worker/rhpv2.go @@ -245,7 +245,7 @@ func RPCFormContract(ctx context.Context, t *rhpv2.Transport, renterKey types.Pr // FetchSignedRevision fetches the latest signed revision for a contract from a host. // TODO: stop using rhpv2 and upgrade to newer protocol when possible. -func (w *worker) FetchSignedRevision(ctx context.Context, hostIP string, hostKey types.PublicKey, renterKey types.PrivateKey, contractID types.FileContractID, timeout time.Duration) (rhpv2.ContractRevision, error) { +func (w *Worker) FetchSignedRevision(ctx context.Context, hostIP string, hostKey types.PublicKey, renterKey types.PrivateKey, contractID types.FileContractID, timeout time.Duration) (rhpv2.ContractRevision, error) { var rev rhpv2.ContractRevision err := w.withTransportV2(ctx, hostKey, hostIP, func(t *rhpv2.Transport) error { req := &rhpv2.RPCLockRequest{ @@ -289,7 +289,7 @@ func (w *worker) FetchSignedRevision(ctx context.Context, hostIP string, hostKey return rev, err } -func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types.PublicKey, fcid types.FileContractID, lastKnownRevisionNumber uint64) (deleted, remaining uint64, err error) { +func (w *Worker) PruneContract(ctx context.Context, hostIP string, hostKey types.PublicKey, fcid types.FileContractID, lastKnownRevisionNumber uint64) (deleted, remaining uint64, err error) { err = w.withContractLock(ctx, fcid, lockingPriorityPruning, func() error { return w.withTransportV2(ctx, hostKey, hostIP, func(t *rhpv2.Transport) error { return w.withRevisionV2(defaultLockTimeout, t, hostKey, fcid, lastKnownRevisionNumber, func(t *rhpv2.Transport, rev rhpv2.ContractRevision, settings rhpv2.HostSettings) (err error) { @@ -347,7 +347,7 @@ func (w *worker) PruneContract(ctx context.Context, hostIP string, hostKey types return } -func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevision, settings rhpv2.HostSettings, indices []uint64) (deleted uint64, err error) { +func (w *Worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevision, settings rhpv2.HostSettings, indices []uint64) (deleted uint64, err error) { id := frand.Entropy128() logger := w.logger. With("id", hex.EncodeToString(id[:])). @@ -529,7 +529,7 @@ func (w *worker) deleteContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevi return } -func (w *worker) FetchContractRoots(ctx context.Context, hostIP string, hostKey types.PublicKey, fcid types.FileContractID, lastKnownRevisionNumber uint64) (roots []types.Hash256, err error) { +func (w *Worker) FetchContractRoots(ctx context.Context, hostIP string, hostKey types.PublicKey, fcid types.FileContractID, lastKnownRevisionNumber uint64) (roots []types.Hash256, err error) { err = w.withTransportV2(ctx, hostKey, hostIP, func(t *rhpv2.Transport) error { return w.withRevisionV2(defaultLockTimeout, t, hostKey, fcid, lastKnownRevisionNumber, func(t *rhpv2.Transport, rev rhpv2.ContractRevision, settings rhpv2.HostSettings) (err error) { gc, err := GougingCheckerFromContext(ctx, false) @@ -546,7 +546,7 @@ func (w *worker) FetchContractRoots(ctx context.Context, hostIP string, hostKey return } -func (w *worker) fetchContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevision, settings rhpv2.HostSettings) (roots []types.Hash256, _ error) { +func (w *Worker) fetchContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevision, settings rhpv2.HostSettings) (roots []types.Hash256, _ error) { // derive the renter key renterKey := w.deriveRenterKey(rev.HostKey()) @@ -634,7 +634,7 @@ func (w *worker) fetchContractRoots(t *rhpv2.Transport, rev *rhpv2.ContractRevis return } -func (w *worker) withTransportV2(ctx context.Context, hostKey types.PublicKey, hostIP string, fn func(*rhpv2.Transport) error) (err error) { +func (w *Worker) withTransportV2(ctx context.Context, hostKey types.PublicKey, hostIP string, fn func(*rhpv2.Transport) error) (err error) { conn, err := dial(ctx, hostIP) if err != nil { return err @@ -667,7 +667,7 @@ func (w *worker) withTransportV2(ctx context.Context, hostKey types.PublicKey, h return fn(t) } -func (w *worker) withRevisionV2(lockTimeout time.Duration, t *rhpv2.Transport, hk types.PublicKey, fcid types.FileContractID, lastKnownRevisionNumber uint64, fn func(t *rhpv2.Transport, rev rhpv2.ContractRevision, settings rhpv2.HostSettings) error) error { +func (w *Worker) withRevisionV2(lockTimeout time.Duration, t *rhpv2.Transport, hk types.PublicKey, fcid types.FileContractID, lastKnownRevisionNumber uint64, fn func(t *rhpv2.Transport, rev rhpv2.ContractRevision, settings rhpv2.HostSettings) error) error { renterKey := w.deriveRenterKey(hk) // execute lock RPC diff --git a/worker/rhpv3.go b/worker/rhpv3.go index f95f596d3..2c0739703 100644 --- a/worker/rhpv3.go +++ b/worker/rhpv3.go @@ -396,7 +396,7 @@ type ( } ) -func (w *worker) initAccounts(as AccountStore) { +func (w *Worker) initAccounts(as AccountStore) { if w.accounts != nil { panic("accounts already initialized") // developer error } @@ -406,7 +406,7 @@ func (w *worker) initAccounts(as AccountStore) { } } -func (w *worker) initTransportPool() { +func (w *Worker) initTransportPool() { if w.transportPoolV3 != nil { panic("transport pool already initialized") // developer error } diff --git a/worker/spending.go b/worker/spending.go index 87d2ec17d..6b9914c58 100644 --- a/worker/spending.go +++ b/worker/spending.go @@ -35,7 +35,7 @@ var ( _ ContractSpendingRecorder = (*contractSpendingRecorder)(nil) ) -func (w *worker) initContractSpendingRecorder(flushInterval time.Duration) { +func (w *Worker) initContractSpendingRecorder(flushInterval time.Duration) { if w.contractSpendingRecorder != nil { panic("ContractSpendingRecorder already initialized") // developer error } diff --git a/worker/upload.go b/worker/upload.go index 3522584a1..423613ee5 100644 --- a/worker/upload.go +++ b/worker/upload.go @@ -146,7 +146,7 @@ type ( } ) -func (w *worker) initUploadManager(maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.SugaredLogger) { +func (w *Worker) initUploadManager(maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.SugaredLogger) { if w.uploadManager != nil { panic("upload manager already initialized") // developer error } @@ -155,7 +155,7 @@ func (w *worker) initUploadManager(maxMemory, maxOverdrive uint64, overdriveTime w.uploadManager = newUploadManager(w.shutdownCtx, w, mm, w.bus, w.bus, w.bus, maxOverdrive, overdriveTimeout, w.contractLockingDuration, logger) } -func (w *worker) upload(ctx context.Context, bucket, path string, rs api.RedundancySettings, r io.Reader, contracts []api.ContractMetadata, opts ...UploadOption) (_ string, err error) { +func (w *Worker) upload(ctx context.Context, bucket, path string, rs api.RedundancySettings, r io.Reader, contracts []api.ContractMetadata, opts ...UploadOption) (_ string, err error) { // apply the options up := defaultParameters(bucket, path, rs) for _, opt := range opts { @@ -212,7 +212,7 @@ func (w *worker) upload(ctx context.Context, bucket, path string, rs api.Redunda return eTag, nil } -func (w *worker) threadedUploadPackedSlabs(rs api.RedundancySettings, contractSet string, lockPriority int) { +func (w *Worker) threadedUploadPackedSlabs(rs api.RedundancySettings, contractSet string, lockPriority int) { key := fmt.Sprintf("%d-%d_%s", rs.MinShards, rs.TotalShards, contractSet) w.uploadsMu.Lock() if _, ok := w.uploadingPackedSlabs[key]; ok { @@ -278,7 +278,7 @@ func (w *worker) threadedUploadPackedSlabs(rs api.RedundancySettings, contractSe wg.Wait() } -func (w *worker) tryUploadPackedSlab(ctx context.Context, mem Memory, ps api.PackedSlab, rs api.RedundancySettings, contractSet string, lockPriority int) error { +func (w *Worker) tryUploadPackedSlab(ctx context.Context, mem Memory, ps api.PackedSlab, rs api.RedundancySettings, contractSet string, lockPriority int) error { // fetch contracts contracts, err := w.bus.Contracts(ctx, api.ContractsOpts{ContractSet: contractSet}) if err != nil { diff --git a/worker/worker.go b/worker/worker.go index a4dc4945a..1f23b2cc3 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -171,7 +171,7 @@ type ( // deriveSubKey can be used to derive a sub-masterkey from the worker's // masterkey to use for a specific purpose. Such as deriving more keys for // ephemeral accounts. -func (w *worker) deriveSubKey(purpose string) types.PrivateKey { +func (w *Worker) deriveSubKey(purpose string) types.PrivateKey { seed := blake2b.Sum256(append(w.masterKey[:], []byte(purpose)...)) pk := types.NewPrivateKeyFromSeed(seed[:]) for i := range seed { @@ -192,7 +192,7 @@ func (w *worker) deriveSubKey(purpose string) types.PrivateKey { // // TODO: instead of deriving a renter key use a randomly generated salt so we're // not limited to one key per host -func (w *worker) deriveRenterKey(hostKey types.PublicKey) types.PrivateKey { +func (w *Worker) deriveRenterKey(hostKey types.PublicKey) types.PrivateKey { seed := blake2b.Sum256(append(w.deriveSubKey("renterkey"), hostKey[:]...)) pk := types.NewPrivateKeyFromSeed(seed[:]) for i := range seed { @@ -203,7 +203,7 @@ func (w *worker) deriveRenterKey(hostKey types.PublicKey) types.PrivateKey { // A worker talks to Sia hosts to perform contract and storage operations within // a renterd system. -type worker struct { +type Worker struct { alerts alerts.Alerter allowPrivateIPs bool @@ -233,7 +233,7 @@ type worker struct { logger *zap.SugaredLogger } -func (w *worker) isStopped() bool { +func (w *Worker) isStopped() bool { select { case <-w.shutdownCtx.Done(): return true @@ -242,7 +242,7 @@ func (w *worker) isStopped() bool { return false } -func (w *worker) withRevision(ctx context.Context, fetchTimeout time.Duration, fcid types.FileContractID, hk types.PublicKey, siamuxAddr string, lockPriority int, fn func(rev types.FileContractRevision) error) error { +func (w *Worker) withRevision(ctx context.Context, fetchTimeout time.Duration, fcid types.FileContractID, hk types.PublicKey, siamuxAddr string, lockPriority int, fn func(rev types.FileContractRevision) error) error { return w.withContractLock(ctx, fcid, lockPriority, func() error { h := w.Host(hk, fcid, siamuxAddr) rev, err := h.FetchRevision(ctx, fetchTimeout) @@ -253,7 +253,7 @@ func (w *worker) withRevision(ctx context.Context, fetchTimeout time.Duration, f }) } -func (w *worker) registerAlert(a alerts.Alert) { +func (w *Worker) registerAlert(a alerts.Alert) { ctx, cancel := context.WithTimeout(w.shutdownCtx, time.Minute) if err := w.alerts.RegisterAlert(ctx, a); err != nil { w.logger.Errorf("failed to register alert, err: %v", err) @@ -261,7 +261,7 @@ func (w *worker) registerAlert(a alerts.Alert) { cancel() } -func (w *worker) rhpScanHandler(jc jape.Context) { +func (w *Worker) rhpScanHandler(jc jape.Context) { ctx := jc.Request.Context() // decode the request @@ -295,7 +295,7 @@ func (w *worker) rhpScanHandler(jc jape.Context) { }) } -func (w *worker) fetchContracts(ctx context.Context, metadatas []api.ContractMetadata, timeout time.Duration) (contracts []api.Contract, errs HostErrorSet) { +func (w *Worker) fetchContracts(ctx context.Context, metadatas []api.ContractMetadata, timeout time.Duration) (contracts []api.Contract, errs HostErrorSet) { errs = make(HostErrorSet) // create requests channel @@ -347,7 +347,7 @@ func (w *worker) fetchContracts(ctx context.Context, metadatas []api.ContractMet return } -func (w *worker) rhpPriceTableHandler(jc jape.Context) { +func (w *Worker) rhpPriceTableHandler(jc jape.Context) { // decode the request var rptr api.RHPPriceTableRequest if jc.Decode(&rptr) != nil { @@ -390,11 +390,11 @@ func (w *worker) rhpPriceTableHandler(jc jape.Context) { jc.Encode(hpt) } -func (w *worker) discardTxnOnErr(txn types.Transaction, errContext string, err *error) { +func (w *Worker) discardTxnOnErr(txn types.Transaction, errContext string, err *error) { discardTxnOnErr(w.shutdownCtx, w.bus, w.logger, txn, errContext, err) } -func (w *worker) rhpFormHandler(jc jape.Context) { +func (w *Worker) rhpFormHandler(jc jape.Context) { ctx := jc.Request.Context() // decode the request @@ -471,7 +471,7 @@ func (w *worker) rhpFormHandler(jc jape.Context) { }) } -func (w *worker) rhpBroadcastHandler(jc jape.Context) { +func (w *Worker) rhpBroadcastHandler(jc jape.Context) { ctx := jc.Request.Context() // decode the fcid @@ -528,7 +528,7 @@ func (w *worker) rhpBroadcastHandler(jc jape.Context) { } } -func (w *worker) rhpPruneContractHandlerPOST(jc jape.Context) { +func (w *Worker) rhpPruneContractHandlerPOST(jc jape.Context) { ctx := jc.Request.Context() // decode fcid @@ -595,7 +595,7 @@ func (w *worker) rhpPruneContractHandlerPOST(jc jape.Context) { jc.Encode(res) } -func (w *worker) rhpContractRootsHandlerGET(jc jape.Context) { +func (w *Worker) rhpContractRootsHandlerGET(jc jape.Context) { ctx := jc.Request.Context() // decode fcid @@ -629,7 +629,7 @@ func (w *worker) rhpContractRootsHandlerGET(jc jape.Context) { } } -func (w *worker) rhpRenewHandler(jc jape.Context) { +func (w *Worker) rhpRenewHandler(jc jape.Context) { ctx := jc.Request.Context() // decode request @@ -679,7 +679,7 @@ func (w *worker) rhpRenewHandler(jc jape.Context) { }) } -func (w *worker) rhpFundHandler(jc jape.Context) { +func (w *Worker) rhpFundHandler(jc jape.Context) { ctx := jc.Request.Context() // decode request @@ -717,7 +717,7 @@ func (w *worker) rhpFundHandler(jc jape.Context) { })) } -func (w *worker) rhpSyncHandler(jc jape.Context) { +func (w *Worker) rhpSyncHandler(jc jape.Context) { ctx := jc.Request.Context() // decode the request @@ -740,7 +740,7 @@ func (w *worker) rhpSyncHandler(jc jape.Context) { })) } -func (w *worker) slabMigrateHandler(jc jape.Context) { +func (w *Worker) slabMigrateHandler(jc jape.Context) { ctx := jc.Request.Context() // decode the slab @@ -816,7 +816,7 @@ func (w *worker) slabMigrateHandler(jc jape.Context) { }) } -func (w *worker) downloadsStatsHandlerGET(jc jape.Context) { +func (w *Worker) downloadsStatsHandlerGET(jc jape.Context) { stats := w.downloadManager.Stats() // prepare downloaders stats @@ -846,7 +846,7 @@ func (w *worker) downloadsStatsHandlerGET(jc jape.Context) { }) } -func (w *worker) uploadsStatsHandlerGET(jc jape.Context) { +func (w *Worker) uploadsStatsHandlerGET(jc jape.Context) { stats := w.uploadManager.Stats() // prepare upload stats @@ -871,7 +871,7 @@ func (w *worker) uploadsStatsHandlerGET(jc jape.Context) { }) } -func (w *worker) objectsHandlerHEAD(jc jape.Context) { +func (w *Worker) objectsHandlerHEAD(jc jape.Context) { // parse bucket bucket := api.DefaultBucketName if jc.DecodeForm("bucket", &bucket) != nil { @@ -930,7 +930,7 @@ func (w *worker) objectsHandlerHEAD(jc jape.Context) { serveContent(jc.ResponseWriter, jc.Request, path, bytes.NewReader(nil), *hor) } -func (w *worker) objectsHandlerGET(jc jape.Context) { +func (w *Worker) objectsHandlerGET(jc jape.Context) { jc.Custom(nil, []api.ObjectMetadata{}) ctx := jc.Request.Context() @@ -1023,7 +1023,7 @@ func (w *worker) objectsHandlerGET(jc jape.Context) { serveContent(jc.ResponseWriter, jc.Request, path, gor.Content, gor.HeadObjectResponse) } -func (w *worker) objectsHandlerPUT(jc jape.Context) { +func (w *Worker) objectsHandlerPUT(jc jape.Context) { jc.Custom((*[]byte)(nil), nil) ctx := jc.Request.Context() @@ -1094,7 +1094,7 @@ func (w *worker) objectsHandlerPUT(jc jape.Context) { jc.ResponseWriter.Header().Set("ETag", api.FormatETag(resp.ETag)) } -func (w *worker) multipartUploadHandlerPUT(jc jape.Context) { +func (w *Worker) multipartUploadHandlerPUT(jc jape.Context) { jc.Custom((*[]byte)(nil), nil) ctx := jc.Request.Context() @@ -1182,7 +1182,7 @@ func (w *worker) multipartUploadHandlerPUT(jc jape.Context) { jc.ResponseWriter.Header().Set("ETag", api.FormatETag(resp.ETag)) } -func (w *worker) objectsHandlerDELETE(jc jape.Context) { +func (w *Worker) objectsHandlerDELETE(jc jape.Context) { var batch bool if jc.DecodeForm("batch", &batch) != nil { return @@ -1199,7 +1199,7 @@ func (w *worker) objectsHandlerDELETE(jc jape.Context) { jc.Check("couldn't delete object", err) } -func (w *worker) rhpContractsHandlerGET(jc jape.Context) { +func (w *Worker) rhpContractsHandlerGET(jc jape.Context) { ctx := jc.Request.Context() // fetch contracts @@ -1235,18 +1235,18 @@ func (w *worker) rhpContractsHandlerGET(jc jape.Context) { jc.Encode(resp) } -func (w *worker) idHandlerGET(jc jape.Context) { +func (w *Worker) idHandlerGET(jc jape.Context) { jc.Encode(w.id) } -func (w *worker) memoryGET(jc jape.Context) { +func (w *Worker) memoryGET(jc jape.Context) { jc.Encode(api.MemoryResponse{ Download: w.downloadManager.mm.Status(), Upload: w.uploadManager.mm.Status(), }) } -func (w *worker) accountHandlerGET(jc jape.Context) { +func (w *Worker) accountHandlerGET(jc jape.Context) { var hostKey types.PublicKey if jc.DecodeParam("hostkey", &hostKey) != nil { return @@ -1255,7 +1255,7 @@ func (w *worker) accountHandlerGET(jc jape.Context) { jc.Encode(account) } -func (w *worker) eventsHandlerPOST(jc jape.Context) { +func (w *Worker) eventsHandlerPOST(jc jape.Context) { var event webhooks.Event if jc.Decode(&event) != nil { return @@ -1266,7 +1266,7 @@ func (w *worker) eventsHandlerPOST(jc jape.Context) { } } -func (w *worker) stateHandlerGET(jc jape.Context) { +func (w *Worker) stateHandlerGET(jc jape.Context) { jc.Encode(api.WorkerStateResponse{ ID: w.id, StartTime: api.TimeRFC3339(w.startTime), @@ -1280,7 +1280,7 @@ func (w *worker) stateHandlerGET(jc jape.Context) { } // New returns an HTTP handler that serves the worker API. -func New(masterKey [32]byte, id string, b Bus, contractLockingDuration, busFlushInterval, downloadOverdriveTimeout, uploadOverdriveTimeout time.Duration, downloadMaxOverdrive, uploadMaxOverdrive, downloadMaxMemory, uploadMaxMemory uint64, allowPrivateIPs bool, l *zap.Logger) (*worker, error) { +func New(masterKey [32]byte, id string, b Bus, contractLockingDuration, busFlushInterval, downloadOverdriveTimeout, uploadOverdriveTimeout time.Duration, downloadMaxOverdrive, uploadMaxOverdrive, downloadMaxMemory, uploadMaxMemory uint64, allowPrivateIPs bool, l *zap.Logger) (*Worker, error) { if contractLockingDuration == 0 { return nil, errors.New("contract lock duration must be positive") } @@ -1303,7 +1303,7 @@ func New(masterKey [32]byte, id string, b Bus, contractLockingDuration, busFlush a := alerts.WithOrigin(b, fmt.Sprintf("worker.%s", id)) l = l.Named("worker").Named(id) shutdownCtx, shutdownCancel := context.WithCancel(context.Background()) - w := &worker{ + w := &Worker{ alerts: a, allowPrivateIPs: allowPrivateIPs, contractLockingDuration: contractLockingDuration, @@ -1331,7 +1331,7 @@ func New(masterKey [32]byte, id string, b Bus, contractLockingDuration, busFlush } // Handler returns an HTTP handler that serves the worker API. -func (w *worker) Handler() http.Handler { +func (w *Worker) Handler() http.Handler { return jape.Mux(map[string]jape.Handler{ "GET /account/:hostkey": w.accountHandlerGET, "GET /id": w.idHandlerGET, @@ -1367,7 +1367,7 @@ func (w *worker) Handler() http.Handler { } // Setup register event webhooks that enable the worker cache. -func (w *worker) Setup(ctx context.Context, apiURL, apiPassword string) error { +func (w *Worker) Setup(ctx context.Context, apiURL, apiPassword string) error { go func() { eventsURL := fmt.Sprintf("%s/events", apiURL) webhookOpts := []webhooks.HeaderOption{webhooks.WithBasicAuth("", apiPassword)} @@ -1380,7 +1380,7 @@ func (w *worker) Setup(ctx context.Context, apiURL, apiPassword string) error { } // Shutdown shuts down the worker. -func (w *worker) Shutdown(ctx context.Context) error { +func (w *Worker) Shutdown(ctx context.Context) error { // cancel shutdown context w.shutdownCtxCancel() @@ -1395,7 +1395,7 @@ func (w *worker) Shutdown(ctx context.Context) error { return w.eventSubscriber.Shutdown(ctx) } -func (w *worker) scanHost(ctx context.Context, timeout time.Duration, hostKey types.PublicKey, hostIP string) (rhpv2.HostSettings, rhpv3.HostPriceTable, time.Duration, error) { +func (w *Worker) scanHost(ctx context.Context, timeout time.Duration, hostKey types.PublicKey, hostIP string) (rhpv2.HostSettings, rhpv3.HostPriceTable, time.Duration, error) { logger := w.logger.With("host", hostKey).With("hostIP", hostIP).With("timeout", timeout) // prepare a helper to create a context for scanning @@ -1537,7 +1537,7 @@ func isErrHostUnreachable(err error) bool { utils.IsErr(err, errors.New("cannot assign requested address")) } -func (w *worker) headObject(ctx context.Context, bucket, path string, onlyMetadata bool, opts api.HeadObjectOptions) (*api.HeadObjectResponse, api.ObjectsResponse, error) { +func (w *Worker) headObject(ctx context.Context, bucket, path string, onlyMetadata bool, opts api.HeadObjectOptions) (*api.HeadObjectResponse, api.ObjectsResponse, error) { // fetch object res, err := w.bus.Object(ctx, bucket, path, api.GetObjectOptions{ IgnoreDelim: opts.IgnoreDelim, @@ -1572,7 +1572,7 @@ func (w *worker) headObject(ctx context.Context, bucket, path string, onlyMetada }, res, nil } -func (w *worker) GetObject(ctx context.Context, bucket, path string, opts api.DownloadObjectOptions) (*api.GetObjectResponse, error) { +func (w *Worker) GetObject(ctx context.Context, bucket, path string, opts api.DownloadObjectOptions) (*api.GetObjectResponse, error) { // head object hor, res, err := w.headObject(ctx, bucket, path, false, api.HeadObjectOptions{ IgnoreDelim: opts.IgnoreDelim, @@ -1638,12 +1638,12 @@ func (w *worker) GetObject(ctx context.Context, bucket, path string, opts api.Do }, nil } -func (w *worker) HeadObject(ctx context.Context, bucket, path string, opts api.HeadObjectOptions) (*api.HeadObjectResponse, error) { +func (w *Worker) HeadObject(ctx context.Context, bucket, path string, opts api.HeadObjectOptions) (*api.HeadObjectResponse, error) { res, _, err := w.headObject(ctx, bucket, path, true, opts) return res, err } -func (w *worker) UploadObject(ctx context.Context, r io.Reader, bucket, path string, opts api.UploadObjectOptions) (*api.UploadObjectResponse, error) { +func (w *Worker) UploadObject(ctx context.Context, r io.Reader, bucket, path string, opts api.UploadObjectOptions) (*api.UploadObjectResponse, error) { // prepare upload params up, err := w.prepareUploadParams(ctx, bucket, opts.ContractSet, opts.MinShards, opts.TotalShards) if err != nil { @@ -1679,7 +1679,7 @@ func (w *worker) UploadObject(ctx context.Context, r io.Reader, bucket, path str }, nil } -func (w *worker) UploadMultipartUploadPart(ctx context.Context, r io.Reader, bucket, path, uploadID string, partNumber int, opts api.UploadMultipartUploadPartOptions) (*api.UploadMultipartUploadPartResponse, error) { +func (w *Worker) UploadMultipartUploadPart(ctx context.Context, r io.Reader, bucket, path, uploadID string, partNumber int, opts api.UploadMultipartUploadPartOptions) (*api.UploadMultipartUploadPartResponse, error) { // prepare upload params up, err := w.prepareUploadParams(ctx, bucket, opts.ContractSet, opts.MinShards, opts.TotalShards) if err != nil { @@ -1734,7 +1734,7 @@ func (w *worker) UploadMultipartUploadPart(ctx context.Context, r io.Reader, buc }, nil } -func (w *worker) prepareUploadParams(ctx context.Context, bucket string, contractSet string, minShards, totalShards int) (api.UploadParams, error) { +func (w *Worker) prepareUploadParams(ctx context.Context, bucket string, contractSet string, minShards, totalShards int) (api.UploadParams, error) { // return early if the bucket does not exist _, err := w.bus.Bucket(ctx, bucket) if err != nil { diff --git a/worker/worker_test.go b/worker/worker_test.go index 706fae14e..6b03e8c3f 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -17,7 +17,7 @@ import ( type ( testWorker struct { tt test.TT - *worker + *Worker cs *contractStoreMock os *objectStoreMock From 747232c40e6cba4827b749dc6c1e41e9237ff0f5 Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 15 Aug 2024 09:24:53 +0200 Subject: [PATCH 05/10] bus: add Store interface --- bus/bus.go | 149 +++++++++++++++++++++++++++----------------------- bus/node.go | 2 +- bus/routes.go | 26 ++++----- 3 files changed, 95 insertions(+), 82 deletions(-) diff --git a/bus/bus.go b/bus/bus.go index 5446f46f2..701a125b8 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -68,11 +68,6 @@ type ( Shutdown(context.Context) error } - ChainStore interface { - ChainIndex(ctx context.Context) (types.ChainIndex, error) - ProcessChainUpdate(ctx context.Context, applyFn func(sql.ChainUpdateTx) error) error - } - // A TransactionPool can validate and relay unconfirmed transactions. TransactionPool interface { AcceptTransactionSet(txns []types.Transaction) error @@ -82,8 +77,69 @@ type ( UnconfirmedParents(txn types.Transaction) ([]types.Transaction, error) } - // A HostDB stores information about hosts. - HostDB interface { + PinManager interface { + Shutdown(context.Context) error + TriggerUpdate() + } + + Wallet interface { + Address() types.Address + Balance() (wallet.Balance, error) + Close() error + FundTransaction(txn *types.Transaction, amount types.Currency, useUnconfirmed bool) ([]types.Hash256, error) + Redistribute(outputs int, amount, feePerByte types.Currency) (txns []types.Transaction, toSign []types.Hash256, err error) + ReleaseInputs(txns []types.Transaction, v2txns []types.V2Transaction) + SignTransaction(txn *types.Transaction, toSign []types.Hash256, cf types.CoveredFields) + SpendableOutputs() ([]types.SiacoinElement, error) + Tip() (types.ChainIndex, error) + UnconfirmedEvents() ([]wallet.Event, error) + UpdateChainState(tx wallet.UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error + Events(offset, limit int) ([]wallet.Event, error) + } + + WebhooksManager interface { + webhooks.Broadcaster + Delete(context.Context, webhooks.Webhook) error + Info() ([]webhooks.Webhook, []webhooks.WebhookQueueInfo) + Register(context.Context, webhooks.Webhook) error + Shutdown(context.Context) error + } + + // Store is a collection of stores used by the bus. + Store interface { + AutopilotStore + ChainStore + EphemeralAccountStore + HostStore + MetadataStore + MetricsStore + SettingStore + } + + // An AutopilotStore stores autopilots. + AutopilotStore interface { + Autopilot(ctx context.Context, id string) (api.Autopilot, error) + Autopilots(ctx context.Context) ([]api.Autopilot, error) + UpdateAutopilot(ctx context.Context, ap api.Autopilot) error + } + + // A ChainStore stores information about the chain. + ChainStore interface { + ChainIndex(ctx context.Context) (types.ChainIndex, error) + ProcessChainUpdate(ctx context.Context, applyFn func(sql.ChainUpdateTx) error) error + } + + // EphemeralAccountStore persists information about accounts. Since accounts + // are rapidly updated and can be recovered, they are only loaded upon + // startup and persisted upon shutdown. + EphemeralAccountStore interface { + Accounts(context.Context) ([]api.Account, error) + SaveAccounts(context.Context, []api.Account) error + SetUncleanShutdown(context.Context) error + } + + // A HostStore stores information about hosts. + HostStore interface { Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) HostAllowlist(ctx context.Context) ([]types.PublicKey, error) HostBlocklist(ctx context.Context) ([]string, error) @@ -160,30 +216,7 @@ type ( UpdateSlab(ctx context.Context, s object.Slab, contractSet string) error } - // An AutopilotStore stores autopilots. - AutopilotStore interface { - Autopilot(ctx context.Context, id string) (api.Autopilot, error) - Autopilots(ctx context.Context) ([]api.Autopilot, error) - UpdateAutopilot(ctx context.Context, ap api.Autopilot) error - } - - // A SettingStore stores settings. - SettingStore interface { - DeleteSetting(ctx context.Context, key string) error - Setting(ctx context.Context, key string) (string, error) - Settings(ctx context.Context) ([]string, error) - UpdateSetting(ctx context.Context, key, value string) error - } - - // EphemeralAccountStore persists information about accounts. Since accounts - // are rapidly updated and can be recovered, they are only loaded upon - // startup and persisted upon shutdown. - EphemeralAccountStore interface { - Accounts(context.Context) ([]api.Account, error) - SaveAccounts(context.Context, []api.Account) error - SetUncleanShutdown(context.Context) error - } - + // A MetricsStore stores metrics. MetricsStore interface { ContractSetMetrics(ctx context.Context, start time.Time, n uint64, interval time.Duration, opts api.ContractSetMetricsQueryOpts) ([]api.ContractSetMetric, error) @@ -200,32 +233,12 @@ type ( WalletMetrics(ctx context.Context, start time.Time, n uint64, interval time.Duration, opts api.WalletMetricsQueryOpts) ([]api.WalletMetric, error) } - PinManager interface { - Shutdown(context.Context) error - TriggerUpdate() - } - - Wallet interface { - Address() types.Address - Balance() (wallet.Balance, error) - Close() error - FundTransaction(txn *types.Transaction, amount types.Currency, useUnconfirmed bool) ([]types.Hash256, error) - Redistribute(outputs int, amount, feePerByte types.Currency) (txns []types.Transaction, toSign []types.Hash256, err error) - ReleaseInputs(txns []types.Transaction, v2txns []types.V2Transaction) - SignTransaction(txn *types.Transaction, toSign []types.Hash256, cf types.CoveredFields) - SpendableOutputs() ([]types.SiacoinElement, error) - Tip() (types.ChainIndex, error) - UnconfirmedEvents() ([]wallet.Event, error) - UpdateChainState(tx wallet.UpdateTx, reverted []chain.RevertUpdate, applied []chain.ApplyUpdate) error - Events(offset, limit int) ([]wallet.Event, error) - } - - WebhooksManager interface { - webhooks.Broadcaster - Delete(context.Context, webhooks.Webhook) error - Info() ([]webhooks.Webhook, []webhooks.WebhookQueueInfo) - Register(context.Context, webhooks.Webhook) error - Shutdown(context.Context) error + // A SettingStore stores settings. + SettingStore interface { + DeleteSetting(ctx context.Context, key string) error + Setting(ctx context.Context, key string) (string, error) + Settings(ctx context.Context) ([]string, error) + UpdateSetting(ctx context.Context, key, value string) error } ) @@ -244,10 +257,10 @@ type Bus struct { as AutopilotStore eas EphemeralAccountStore - hdb HostDB + hs HostStore ms MetadataStore - ss SettingStore mtrcs MetricsStore + ss SettingStore accounts *accounts contractLocks *contractLocks @@ -257,18 +270,18 @@ type Bus struct { } // New returns a new Bus -func New(ctx context.Context, am *alerts.Manager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, hdb HostDB, as AutopilotStore, cs ChainStore, ms MetadataStore, ss SettingStore, eas EphemeralAccountStore, mtrcs MetricsStore, announcementMaxAge time.Duration, l *zap.Logger) (*Bus, error) { +func New(ctx context.Context, am *alerts.Manager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, store Store, announcementMaxAge time.Duration, l *zap.Logger) (*Bus, error) { l = l.Named("bus") b := &Bus{ s: s, cm: cm, w: w, - hdb: hdb, - as: as, - ms: ms, - mtrcs: mtrcs, - ss: ss, - eas: eas, + hs: store, + as: store, + ms: store, + mtrcs: store, + ss: store, + eas: store, contractLocks: newContractLocks(), uploadingSectors: newUploadingSectorsCache(), @@ -291,10 +304,10 @@ func New(ctx context.Context, am *alerts.Manager, wm WebhooksManager, cm ChainMa } // create pin manager - b.pinMgr = ibus.NewPinManager(b.alerts, wm, as, ss, defaultPinUpdateInterval, defaultPinRateWindow, l) + b.pinMgr = ibus.NewPinManager(b.alerts, wm, store, store, defaultPinUpdateInterval, defaultPinRateWindow, l) // create chain subscriber - b.cs = ibus.NewChainSubscriber(wm, cm, cs, w, announcementMaxAge, l) + b.cs = ibus.NewChainSubscriber(wm, cm, store, w, announcementMaxAge, l) return b, nil } diff --git a/bus/node.go b/bus/node.go index 94e01f3c4..2fbaf9c96 100644 --- a/bus/node.go +++ b/bus/node.go @@ -194,7 +194,7 @@ func NewNode(cfg NodeConfig, dir string, seed types.PrivateKey, logger *zap.Logg // create bus announcementMaxAgeHours := time.Duration(cfg.AnnouncementMaxAgeHours) * time.Hour - b, err := New(ctx, alertsMgr, wh, cm, s, w, sqlStore, sqlStore, sqlStore, sqlStore, sqlStore, sqlStore, sqlStore, announcementMaxAgeHours, logger) + b, err := New(ctx, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, logger) if err != nil { return nil, nil, nil, err } diff --git a/bus/routes.go b/bus/routes.go index f6e97ee06..20266d6b8 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -504,7 +504,7 @@ func (b *Bus) hostsHandlerGETDeprecated(jc jape.Context) { } // fetch hosts - hosts, err := b.hdb.SearchHosts(jc.Request.Context(), "", api.HostFilterModeAllowed, api.UsabilityFilterModeAll, "", nil, offset, limit) + hosts, err := b.hs.SearchHosts(jc.Request.Context(), "", api.HostFilterModeAllowed, api.UsabilityFilterModeAll, "", nil, offset, limit) if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", offset, offset+limit), err) != nil { return } @@ -521,7 +521,7 @@ func (b *Bus) searchHostsHandlerPOST(jc jape.Context) { // - properly default search params (currently no defaults are set) // - properly validate and return 400 (currently validation is done in autopilot and the store) - hosts, err := b.hdb.SearchHosts(jc.Request.Context(), req.AutopilotID, req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) + hosts, err := b.hs.SearchHosts(jc.Request.Context(), req.AutopilotID, req.FilterMode, req.UsabilityMode, req.AddressContains, req.KeyIn, req.Offset, req.Limit) if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil { return } @@ -541,7 +541,7 @@ func (b *Bus) hostsRemoveHandlerPOST(jc jape.Context) { jc.Error(errors.New("minRecentScanFailures must be non-zero"), http.StatusBadRequest) return } - removed, err := b.hdb.RemoveOfflineHosts(jc.Request.Context(), hrr.MinRecentScanFailures, time.Duration(hrr.MaxDowntimeHours)) + removed, err := b.hs.RemoveOfflineHosts(jc.Request.Context(), hrr.MinRecentScanFailures, time.Duration(hrr.MaxDowntimeHours)) if jc.Check("couldn't remove offline hosts", err) != nil { return } @@ -555,7 +555,7 @@ func (b *Bus) hostsScanningHandlerGET(jc jape.Context) { if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("lastScan", (*api.TimeRFC3339)(&maxLastScan)) != nil { return } - hosts, err := b.hdb.HostsForScanning(jc.Request.Context(), maxLastScan, offset, limit) + hosts, err := b.hs.HostsForScanning(jc.Request.Context(), maxLastScan, offset, limit) if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", offset, offset+limit), err) != nil { return } @@ -567,7 +567,7 @@ func (b *Bus) hostsPubkeyHandlerGET(jc jape.Context) { if jc.DecodeParam("hostkey", &hostKey) != nil { return } - host, err := b.hdb.Host(jc.Request.Context(), hostKey) + host, err := b.hs.Host(jc.Request.Context(), hostKey) if jc.Check("couldn't load host", err) == nil { jc.Encode(host) } @@ -578,7 +578,7 @@ func (b *Bus) hostsResetLostSectorsPOST(jc jape.Context) { if jc.DecodeParam("hostkey", &hostKey) != nil { return } - err := b.hdb.ResetLostSectors(jc.Request.Context(), hostKey) + err := b.hs.ResetLostSectors(jc.Request.Context(), hostKey) if jc.Check("couldn't reset lost sectors", err) != nil { return } @@ -589,7 +589,7 @@ func (b *Bus) hostsScanHandlerPOST(jc jape.Context) { if jc.Decode(&req) != nil { return } - if jc.Check("failed to record scans", b.hdb.RecordHostScans(jc.Request.Context(), req.Scans)) != nil { + if jc.Check("failed to record scans", b.hs.RecordHostScans(jc.Request.Context(), req.Scans)) != nil { return } } @@ -599,7 +599,7 @@ func (b *Bus) hostsPricetableHandlerPOST(jc jape.Context) { if jc.Decode(&req) != nil { return } - if jc.Check("failed to record interactions", b.hdb.RecordPriceTables(jc.Request.Context(), req.PriceTableUpdates)) != nil { + if jc.Check("failed to record interactions", b.hs.RecordPriceTables(jc.Request.Context(), req.PriceTableUpdates)) != nil { return } } @@ -615,7 +615,7 @@ func (b *Bus) contractsSpendingHandlerPOST(jc jape.Context) { } func (b *Bus) hostsAllowlistHandlerGET(jc jape.Context) { - allowlist, err := b.hdb.HostAllowlist(jc.Request.Context()) + allowlist, err := b.hs.HostAllowlist(jc.Request.Context()) if jc.Check("couldn't load allowlist", err) == nil { jc.Encode(allowlist) } @@ -628,14 +628,14 @@ func (b *Bus) hostsAllowlistHandlerPUT(jc jape.Context) { if len(req.Add)+len(req.Remove) > 0 && req.Clear { jc.Error(errors.New("cannot add or remove entries while clearing the allowlist"), http.StatusBadRequest) return - } else if jc.Check("couldn't update allowlist entries", b.hdb.UpdateHostAllowlistEntries(ctx, req.Add, req.Remove, req.Clear)) != nil { + } else if jc.Check("couldn't update allowlist entries", b.hs.UpdateHostAllowlistEntries(ctx, req.Add, req.Remove, req.Clear)) != nil { return } } } func (b *Bus) hostsBlocklistHandlerGET(jc jape.Context) { - blocklist, err := b.hdb.HostBlocklist(jc.Request.Context()) + blocklist, err := b.hs.HostBlocklist(jc.Request.Context()) if jc.Check("couldn't load blocklist", err) == nil { jc.Encode(blocklist) } @@ -648,7 +648,7 @@ func (b *Bus) hostsBlocklistHandlerPUT(jc jape.Context) { if len(req.Add)+len(req.Remove) > 0 && req.Clear { jc.Error(errors.New("cannot add or remove entries while clearing the blocklist"), http.StatusBadRequest) return - } else if jc.Check("couldn't update blocklist entries", b.hdb.UpdateHostBlocklistEntries(ctx, req.Add, req.Remove, req.Clear)) != nil { + } else if jc.Check("couldn't update blocklist entries", b.hs.UpdateHostBlocklistEntries(ctx, req.Add, req.Remove, req.Clear)) != nil { return } } @@ -1836,7 +1836,7 @@ func (b *Bus) autopilotHostCheckHandlerPUT(jc jape.Context) { return } - err := b.hdb.UpdateHostCheck(jc.Request.Context(), id, hk, hc) + err := b.hs.UpdateHostCheck(jc.Request.Context(), id, hk, hc) if errors.Is(err, api.ErrAutopilotNotFound) { jc.Error(err, http.StatusNotFound) return From 46dc438ea4737596136afee04c9be607bc63f62d Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 15 Aug 2024 09:28:19 +0200 Subject: [PATCH 06/10] bus: update pin manager store --- bus/bus.go | 2 +- internal/bus/pinmanager.go | 25 ++++++++++--------------- internal/bus/pinmanager_test.go | 2 +- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/bus/bus.go b/bus/bus.go index 701a125b8..56ccfbc41 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -304,7 +304,7 @@ func New(ctx context.Context, am *alerts.Manager, wm WebhooksManager, cm ChainMa } // create pin manager - b.pinMgr = ibus.NewPinManager(b.alerts, wm, store, store, defaultPinUpdateInterval, defaultPinRateWindow, l) + b.pinMgr = ibus.NewPinManager(b.alerts, wm, store, defaultPinUpdateInterval, defaultPinRateWindow, l) // create chain subscriber b.cs = ibus.NewChainSubscriber(wm, cm, store, w, announcementMaxAge, l) diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index ad256b7e1..f9833c887 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -18,13 +18,10 @@ import ( ) type ( - AutopilotStore interface { + Store interface { Autopilot(ctx context.Context, id string) (api.Autopilot, error) - UpdateAutopilot(ctx context.Context, ap api.Autopilot) error - } - - SettingStore interface { Setting(ctx context.Context, key string) (string, error) + UpdateAutopilot(ctx context.Context, ap api.Autopilot) error UpdateSetting(ctx context.Context, key, value string) error } ) @@ -32,8 +29,7 @@ type ( type ( pinManager struct { a alerts.Alerter - as AutopilotStore - ss SettingStore + s Store broadcaster webhooks.Broadcaster updateInterval time.Duration @@ -54,11 +50,10 @@ type ( // NewPinManager returns a new PinManager, responsible for pinning prices to a // fixed value in an underlying currency. The returned pin manager is already // running and can be stopped by calling Shutdown. -func NewPinManager(alerts alerts.Alerter, broadcaster webhooks.Broadcaster, as AutopilotStore, ss SettingStore, updateInterval, rateWindow time.Duration, l *zap.Logger) *pinManager { +func NewPinManager(alerts alerts.Alerter, broadcaster webhooks.Broadcaster, s Store, updateInterval, rateWindow time.Duration, l *zap.Logger) *pinManager { pm := &pinManager{ a: alerts, - as: as, - ss: ss, + s: s, broadcaster: broadcaster, logger: l.Named("pricemanager").Sugar(), @@ -110,7 +105,7 @@ func (pm *pinManager) averageRate() decimal.Decimal { func (pm *pinManager) pinnedSettings(ctx context.Context) (api.PricePinSettings, error) { var ps api.PricePinSettings - if pss, err := pm.ss.Setting(ctx, api.SettingPricePinning); err != nil { + if pss, err := pm.s.Setting(ctx, api.SettingPricePinning); err != nil { return api.PricePinSettings{}, err } else if err := json.Unmarshal([]byte(pss), &ps); err != nil { pm.logger.Panicf("failed to unmarshal pinned settings '%s': %v", pss, err) @@ -185,7 +180,7 @@ func (pm *pinManager) run() { func (pm *pinManager) updateAutopilotSettings(ctx context.Context, autopilotID string, pins api.AutopilotPins, rate decimal.Decimal) error { var updated bool - ap, err := pm.as.Autopilot(ctx, autopilotID) + ap, err := pm.s.Autopilot(ctx, autopilotID) if err != nil { return err } @@ -222,7 +217,7 @@ func (pm *pinManager) updateAutopilotSettings(ctx context.Context, autopilotID s } // update autopilto - return pm.as.UpdateAutopilot(ctx, ap) + return pm.s.UpdateAutopilot(ctx, ap) } func (pm *pinManager) updateExchangeRates(currency string, rate float64) error { @@ -249,7 +244,7 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin // fetch gouging settings var gs api.GougingSettings - if gss, err := pm.ss.Setting(ctx, api.SettingGouging); err != nil { + if gss, err := pm.s.Setting(ctx, api.SettingGouging); err != nil { return err } else if err := json.Unmarshal([]byte(gss), &gs); err != nil { pm.logger.Panicf("failed to unmarshal gouging settings '%s': %v", gss, err) @@ -319,7 +314,7 @@ func (pm *pinManager) updateGougingSettings(ctx context.Context, pins api.Gougin // update settings bytes, _ := json.Marshal(gs) - err = pm.ss.UpdateSetting(ctx, api.SettingGouging, string(bytes)) + err = pm.s.UpdateSetting(ctx, api.SettingGouging, string(bytes)) // broadcast event if err == nil { diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index 45fb83fad..d57ef742c 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -195,7 +195,7 @@ func TestPinManager(t *testing.T) { defer forex.Close() // create a pinmanager - pm := NewPinManager(a, eb, ms, ms, testUpdateInterval, time.Minute, zap.NewNop()) + pm := NewPinManager(a, eb, ms, testUpdateInterval, time.Minute, zap.NewNop()) defer func() { if err := pm.Shutdown(context.Background()); err != nil { t.Fatal(err) From 5088287daf781fb7b24cc9adc0602023b5691adc Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 15 Aug 2024 10:03:51 +0200 Subject: [PATCH 07/10] bus: add AlertManager interface --- bus/bus.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bus/bus.go b/bus/bus.go index 56ccfbc41..8b2d32b6d 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -49,6 +49,11 @@ func NewClient(addr, password string) *Client { } type ( + AlertManager interface { + alerts.Alerter + RegisterWebhookBroadcaster(b webhooks.Broadcaster) + } + ChainManager interface { AddBlocks(blocks []types.Block) error AddPoolTransactions(txns []types.Transaction) (bool, error) @@ -246,7 +251,7 @@ type Bus struct { startTime time.Time alerts alerts.Alerter - alertMgr *alerts.Manager + alertMgr AlertManager pinMgr PinManager webhooksMgr WebhooksManager @@ -270,7 +275,7 @@ type Bus struct { } // New returns a new Bus -func New(ctx context.Context, am *alerts.Manager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, store Store, announcementMaxAge time.Duration, l *zap.Logger) (*Bus, error) { +func New(ctx context.Context, am AlertManager, wm WebhooksManager, cm ChainManager, s Syncer, w Wallet, store Store, announcementMaxAge time.Duration, l *zap.Logger) (*Bus, error) { l = l.Named("bus") b := &Bus{ s: s, From 296ed876e783306f2e6c2ed3f72c330c1b72f48c Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 15 Aug 2024 13:38:08 +0200 Subject: [PATCH 08/10] all: refactor node setup --- autopilot/autopilot.go | 19 +- autopilot/node.go | 17 -- bus/bus.go | 10 + bus/client/client_test.go | 112 ---------- bus/node.go | 212 ------------------ bus/syncer.go | 102 --------- cmd/renterd/node.go | 364 +++++++++++++++++++++++-------- internal/test/e2e/cluster.go | 271 ++++++++++++++++++----- internal/test/e2e/host.go | 2 +- {utils => internal/utils}/web.go | 3 +- worker/download.go | 13 +- worker/memory.go | 14 +- worker/node.go | 28 --- worker/s3/s3.go | 9 +- worker/upload.go | 12 +- worker/worker.go | 32 +-- worker/worker_test.go | 14 +- 17 files changed, 572 insertions(+), 662 deletions(-) delete mode 100644 autopilot/node.go delete mode 100644 bus/client/client_test.go delete mode 100644 bus/node.go delete mode 100644 bus/syncer.go rename {utils => internal/utils}/web.go (93%) delete mode 100644 worker/node.go diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 577ceb2b6..89f43ee68 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -19,6 +19,7 @@ import ( "go.sia.tech/renterd/autopilot/contractor" "go.sia.tech/renterd/autopilot/scanner" "go.sia.tech/renterd/build" + "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.sia.tech/renterd/webhooks" @@ -123,31 +124,31 @@ type Autopilot struct { } // New initializes an Autopilot. -func New(id string, bus Bus, workers []Worker, logger *zap.Logger, heartbeat time.Duration, scannerScanInterval time.Duration, scannerBatchSize, scannerNumThreads uint64, migrationHealthCutoff float64, accountsRefillInterval time.Duration, revisionSubmissionBuffer, migratorParallelSlabsPerWorker uint64, revisionBroadcastInterval time.Duration) (_ *Autopilot, err error) { +func New(cfg config.Autopilot, bus Bus, workers []Worker, logger *zap.Logger) (_ *Autopilot, err error) { shutdownCtx, shutdownCtxCancel := context.WithCancel(context.Background()) ap := &Autopilot{ - alerts: alerts.WithOrigin(bus, fmt.Sprintf("autopilot.%s", id)), - id: id, + alerts: alerts.WithOrigin(bus, fmt.Sprintf("autopilot.%s", cfg.ID)), + id: cfg.ID, bus: bus, - logger: logger.Sugar().Named("autopilot").Named(id), + logger: logger.Sugar().Named("autopilot").Named(cfg.ID), workers: newWorkerPool(workers), shutdownCtx: shutdownCtx, shutdownCtxCancel: shutdownCtxCancel, - tickerDuration: heartbeat, + tickerDuration: cfg.Heartbeat, pruningAlertIDs: make(map[types.FileContractID]types.Hash256), } - ap.s, err = scanner.New(ap.bus, scannerBatchSize, scannerNumThreads, scannerScanInterval, ap.logger) + ap.s, err = scanner.New(ap.bus, cfg.ScannerBatchSize, cfg.ScannerNumThreads, cfg.ScannerInterval, ap.logger) if err != nil { return } - ap.c = contractor.New(bus, bus, ap.logger, revisionSubmissionBuffer, revisionBroadcastInterval) - ap.m = newMigrator(ap, migrationHealthCutoff, migratorParallelSlabsPerWorker) - ap.a = newAccounts(ap, ap.bus, ap.bus, ap.workers, ap.logger, accountsRefillInterval, revisionSubmissionBuffer) + ap.c = contractor.New(bus, bus, ap.logger, cfg.RevisionSubmissionBuffer, cfg.RevisionBroadcastInterval) + ap.m = newMigrator(ap, cfg.MigrationHealthCutoff, cfg.MigratorParallelSlabsPerWorker) + ap.a = newAccounts(ap, ap.bus, ap.bus, ap.workers, ap.logger, cfg.AccountsRefillInterval, cfg.RevisionSubmissionBuffer) return ap, nil } diff --git a/autopilot/node.go b/autopilot/node.go deleted file mode 100644 index eec478b3c..000000000 --- a/autopilot/node.go +++ /dev/null @@ -1,17 +0,0 @@ -package autopilot - -import ( - "context" - "net/http" - - "go.sia.tech/renterd/config" - "go.uber.org/zap" -) - -func NewNode(cfg config.Autopilot, b Bus, workers []Worker, l *zap.Logger) (http.Handler, func(), func(context.Context) error, error) { - ap, err := New(cfg.ID, b, workers, l, cfg.Heartbeat, cfg.ScannerInterval, cfg.ScannerBatchSize, cfg.ScannerNumThreads, cfg.MigrationHealthCutoff, cfg.AccountsRefillInterval, cfg.RevisionSubmissionBuffer, cfg.MigratorParallelSlabsPerWorker, cfg.RevisionBroadcastInterval) - if err != nil { - return nil, nil, nil, err - } - return ap.Handler(), ap.Run, ap.Shutdown, nil -} diff --git a/bus/bus.go b/bus/bus.go index 8b2d32b6d..26e3fb143 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -13,9 +13,11 @@ import ( "time" "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" "go.sia.tech/coreutils/wallet" "go.sia.tech/jape" "go.sia.tech/renterd/alerts" @@ -87,6 +89,14 @@ type ( TriggerUpdate() } + Syncer interface { + Addr() string + BroadcastHeader(h gateway.BlockHeader) + BroadcastTransactionSet([]types.Transaction) + Connect(ctx context.Context, addr string) (*syncer.Peer, error) + Peers() []*syncer.Peer + } + Wallet interface { Address() types.Address Balance() (wallet.Balance, error) diff --git a/bus/client/client_test.go b/bus/client/client_test.go deleted file mode 100644 index 8f26b992c..000000000 --- a/bus/client/client_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package client_test - -import ( - "context" - "net" - "net/http" - "path/filepath" - "strings" - "testing" - "time" - - "go.sia.tech/core/types" - "go.sia.tech/coreutils/chain" - "go.sia.tech/jape" - "go.sia.tech/renterd/api" - "go.sia.tech/renterd/bus" - "go.sia.tech/renterd/bus/client" - - "go.sia.tech/renterd/config" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -func TestClient(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - c, serveFn, shutdownFn, err := newTestClient(t.TempDir()) - if err != nil { - t.Fatal(err) - } - defer func() { - if err := shutdownFn(ctx); err != nil { - t.Error(err) - } - }() - go serveFn() - - // assert setting 'foo' is not found - if err := c.Setting(ctx, "foo", nil); err == nil || !strings.Contains(err.Error(), api.ErrSettingNotFound.Error()) { - t.Fatal("unexpected err", err) - } - - // update setting 'foo' - if err := c.UpdateSetting(ctx, "foo", "bar"); err != nil { - t.Fatal(err) - } - - // fetch setting 'foo' and assert it matches - var value string - if err := c.Setting(ctx, "foo", &value); err != nil { - t.Fatal("unexpected err", err) - } else if value != "bar" { - t.Fatal("unexpected result", value) - } - - // fetch redundancy settings and assert they're configured to the default values - if rs, err := c.RedundancySettings(ctx); err != nil { - t.Fatal(err) - } else if rs.MinShards != api.DefaultRedundancySettings.MinShards || rs.TotalShards != api.DefaultRedundancySettings.TotalShards { - t.Fatal("unexpected redundancy settings", rs) - } -} - -func newTestClient(dir string) (*client.Client, func() error, func(context.Context) error, error) { - // create listener - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - return nil, nil, nil, err - } - - // create bus - network, genesis := chain.Mainnet() - b, shutdown, _, err := bus.NewNode(bus.NodeConfig{ - Bus: config.Bus{ - AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year - Bootstrap: false, - GatewayAddr: "127.0.0.1:0", - UsedUTXOExpiry: time.Minute, - SlabBufferCompletionThreshold: 0, - }, - Network: network, - Genesis: genesis, - DatabaseLog: config.DatabaseLog{ - SlowThreshold: 100 * time.Millisecond, - }, - Logger: zap.NewNop(), - }, filepath.Join(dir, "bus"), types.GeneratePrivateKey(), zap.New(zapcore.NewNopCore())) - if err != nil { - return nil, nil, nil, err - } - - // create client - client := client.New("http://"+l.Addr().String(), "test") - - // create server - server := http.Server{Handler: jape.BasicAuth("test")(b)} - - serveFn := func() error { - err := server.Serve(l) - if err != nil && !strings.Contains(err.Error(), "Server closed") { - return err - } - return nil - } - - shutdownFn := func(ctx context.Context) error { - server.Shutdown(ctx) - return shutdown(ctx) - } - return client, serveFn, shutdownFn, nil -} diff --git a/bus/node.go b/bus/node.go deleted file mode 100644 index 2fbaf9c96..000000000 --- a/bus/node.go +++ /dev/null @@ -1,212 +0,0 @@ -package bus - -import ( - "context" - "errors" - "fmt" - "net/http" - "os" - "path/filepath" - "time" - - "go.sia.tech/core/consensus" - "go.sia.tech/core/types" - "go.sia.tech/coreutils" - "go.sia.tech/coreutils/chain" - "go.sia.tech/coreutils/wallet" - "go.sia.tech/renterd/alerts" - "go.sia.tech/renterd/config" - "go.sia.tech/renterd/internal/utils" - "go.sia.tech/renterd/stores" - "go.sia.tech/renterd/stores/sql" - "go.sia.tech/renterd/stores/sql/mysql" - "go.sia.tech/renterd/stores/sql/sqlite" - "go.sia.tech/renterd/webhooks" - "go.uber.org/zap" -) - -type NodeConfig struct { - config.Bus - Database config.Database - DatabaseLog config.DatabaseLog - Genesis types.Block - Logger *zap.Logger - Network *consensus.Network - RetryTxIntervals []time.Duration - SyncerSyncInterval time.Duration - SyncerPeerDiscoveryInterval time.Duration -} - -func NewNode(cfg NodeConfig, dir string, seed types.PrivateKey, logger *zap.Logger) (http.Handler, func(context.Context) error, *chain.Manager, error) { - // ensure we don't hang indefinitely - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - // create database connections - var dbMain sql.Database - var dbMetrics sql.MetricsDatabase - if cfg.Database.MySQL.URI != "" { - // create MySQL connections - connMain, err := mysql.Open( - cfg.Database.MySQL.User, - cfg.Database.MySQL.Password, - cfg.Database.MySQL.URI, - cfg.Database.MySQL.Database, - ) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to open MySQL main database: %w", err) - } - connMetrics, err := mysql.Open( - cfg.Database.MySQL.User, - cfg.Database.MySQL.Password, - cfg.Database.MySQL.URI, - cfg.Database.MySQL.MetricsDatabase, - ) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to open MySQL metrics database: %w", err) - } - dbMain, err = mysql.NewMainDatabase(connMain, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create MySQL main database: %w", err) - } - dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger.Named("metrics").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create MySQL metrics database: %w", err) - } - } else { - // create database directory - dbDir := filepath.Join(dir, "db") - if err := os.MkdirAll(dbDir, 0700); err != nil { - return nil, nil, nil, err - } - - // create SQLite connections - db, err := sqlite.Open(filepath.Join(dbDir, "db.sqlite")) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to open SQLite main database: %w", err) - } - dbMain, err = sqlite.NewMainDatabase(db, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create SQLite main database: %w", err) - } - - dbm, err := sqlite.Open(filepath.Join(dbDir, "metrics.sqlite")) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to open SQLite metrics database: %w", err) - } - dbMetrics, err = sqlite.NewMetricsDatabase(dbm, logger.Named("metrics").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create SQLite metrics database: %w", err) - } - } - - alertsMgr := alerts.NewManager() - sqlStoreDir := filepath.Join(dir, "partial_slabs") - sqlStore, err := stores.NewSQLStore(stores.Config{ - Alerts: alerts.WithOrigin(alertsMgr, "bus"), - DB: dbMain, - DBMetrics: dbMetrics, - PartialSlabDir: sqlStoreDir, - Migrate: true, - SlabBufferCompletionThreshold: cfg.SlabBufferCompletionThreshold, - Logger: logger.Sugar(), - RetryTransactionIntervals: cfg.RetryTxIntervals, - WalletAddress: types.StandardUnlockHash(seed.PublicKey()), - LongQueryDuration: cfg.DatabaseLog.SlowThreshold, - LongTxDuration: cfg.DatabaseLog.SlowThreshold, - }) - if err != nil { - return nil, nil, nil, err - } - - // create webhooks manager - wh, err := webhooks.NewManager(sqlStore, logger) - if err != nil { - return nil, nil, nil, err - } - - // hookup webhooks <-> alerts - alertsMgr.RegisterWebhookBroadcaster(wh) - - // create consensus directory - consensusDir := filepath.Join(dir, "consensus") - if err := os.MkdirAll(consensusDir, 0700); err != nil { - return nil, nil, nil, err - } - - // migrate consensus database - oldConsensus, err := os.Stat(filepath.Join(consensusDir, "consensus.db")) - if err != nil && !os.IsNotExist(err) { - return nil, nil, nil, err - } else if err == nil { - logger.Warn("found old consensus.db, indicating a migration is necessary") - - // reset chain state - logger.Warn("Resetting chain state...") - if err := sqlStore.ResetChainState(ctx); err != nil { - return nil, nil, nil, err - } - logger.Warn("Chain state was successfully reset.") - - // remove consensus.db and consensus.log file - logger.Warn("Removing consensus database...") - _ = os.RemoveAll(filepath.Join(consensusDir, "consensus.log")) // ignore error - if err := os.Remove(filepath.Join(consensusDir, "consensus.db")); err != nil { - return nil, nil, nil, err - } - logger.Warn(fmt.Sprintf("Old 'consensus.db' was successfully removed, reclaimed %v of disk space.", utils.HumanReadableSize(int(oldConsensus.Size())))) - logger.Warn("ATTENTION: consensus will now resync from scratch, this process may take several hours to complete") - } - - // reset chain state if blockchain.db does not exist to make sure deleting - // it forces a resync - chainPath := filepath.Join(consensusDir, "blockchain.db") - if _, err := os.Stat(chainPath); os.IsNotExist(err) { - if err := sqlStore.ResetChainState(context.Background()); err != nil { - return nil, nil, nil, err - } - } - - // create chain database - bdb, err := coreutils.OpenBoltChainDB(chainPath) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to open chain database: %w", err) - } - - // create chain manager - store, state, err := chain.NewDBStore(bdb, cfg.Network, cfg.Genesis) - if err != nil { - return nil, nil, nil, err - } - cm := chain.NewManager(store, state) - - // create wallet - w, err := wallet.NewSingleAddressWallet(seed, cm, sqlStore, wallet.WithReservationDuration(cfg.UsedUTXOExpiry)) - if err != nil { - return nil, nil, nil, err - } - - // create syncer - s, err := NewSyncer(cfg, cm, sqlStore, logger) - if err != nil { - return nil, nil, nil, err - } - - // create bus - announcementMaxAgeHours := time.Duration(cfg.AnnouncementMaxAgeHours) * time.Hour - b, err := New(ctx, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, logger) - if err != nil { - return nil, nil, nil, err - } - - shutdownFn := func(ctx context.Context) error { - return errors.Join( - s.Close(), - w.Close(), - b.Shutdown(ctx), - sqlStore.Close(), - bdb.Close(), - ) - } - return b.Handler(), shutdownFn, cm, nil -} diff --git a/bus/syncer.go b/bus/syncer.go deleted file mode 100644 index c1f941cce..000000000 --- a/bus/syncer.go +++ /dev/null @@ -1,102 +0,0 @@ -package bus - -import ( - "context" - "errors" - "fmt" - "io" - "net" - "time" - - "go.sia.tech/core/gateway" - "go.sia.tech/core/types" - "go.sia.tech/coreutils/syncer" - "go.uber.org/zap" -) - -type Syncer interface { - io.Closer - Addr() string - BroadcastHeader(h gateway.BlockHeader) - BroadcastTransactionSet([]types.Transaction) - Connect(ctx context.Context, addr string) (*syncer.Peer, error) - Peers() []*syncer.Peer -} - -// NewSyncer creates a syncer using the given configuration. The syncer that is -// returned is already running, closing it will close the underlying listener -// causing the syncer to stop. -func NewSyncer(cfg NodeConfig, cm syncer.ChainManager, ps syncer.PeerStore, logger *zap.Logger) (Syncer, error) { - // validate config - if cfg.Bootstrap && cfg.Network == nil { - return nil, errors.New("cannot bootstrap without a network") - } - - // bootstrap the syncer - if cfg.Bootstrap { - peers, err := peers(cfg.Network.Name) - if err != nil { - return nil, err - } - for _, addr := range peers { - if err := ps.AddPeer(addr); err != nil { - return nil, fmt.Errorf("%w: failed to add bootstrap peer '%s'", err, addr) - } - } - } - - // create syncer - l, err := net.Listen("tcp", cfg.GatewayAddr) - if err != nil { - return nil, err - } - syncerAddr := l.Addr().String() - - // peers will reject us if our hostname is empty or unspecified, so use loopback - host, port, _ := net.SplitHostPort(syncerAddr) - if ip := net.ParseIP(host); ip == nil || ip.IsUnspecified() { - syncerAddr = net.JoinHostPort("127.0.0.1", port) - } - - // create header - header := gateway.Header{ - GenesisID: cfg.Genesis.ID(), - UniqueID: gateway.GenerateUniqueID(), - NetAddress: syncerAddr, - } - - // start the syncer - s := syncer.New(l, cm, ps, header, options(cfg, logger)...) - go s.Run(context.Background()) - - return s, nil -} - -func options(cfg NodeConfig, logger *zap.Logger) (opts []syncer.Option) { - opts = append(opts, - syncer.WithLogger(logger.Named("syncer")), - syncer.WithSendBlocksTimeout(time.Minute), - ) - - if cfg.SyncerPeerDiscoveryInterval > 0 { - opts = append(opts, syncer.WithPeerDiscoveryInterval(cfg.SyncerPeerDiscoveryInterval)) - } - if cfg.SyncerSyncInterval > 0 { - opts = append(opts, syncer.WithSyncInterval(cfg.SyncerSyncInterval)) - } - - return -} - -func peers(network string) ([]string, error) { - switch network { - case "mainnet": - return syncer.MainnetBootstrapPeers, nil - case "zen": - return syncer.ZenBootstrapPeers, nil - case "anagami": - return syncer.AnagamiBootstrapPeers, nil - default: - return nil, fmt.Errorf("no available bootstrap peers for unknown network '%s'", network) - } -} diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index f2789c0f1..be73a61c5 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -2,10 +2,8 @@ package main import ( "context" - "encoding/json" "errors" "fmt" - "log" "net" "net/http" "os" @@ -15,19 +13,30 @@ import ( "time" "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" "go.sia.tech/core/types" + "go.sia.tech/coreutils" + "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" "go.sia.tech/coreutils/wallet" "go.sia.tech/jape" + "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" "go.sia.tech/renterd/autopilot" "go.sia.tech/renterd/build" "go.sia.tech/renterd/bus" "go.sia.tech/renterd/config" - "go.sia.tech/renterd/utils" + "go.sia.tech/renterd/internal/utils" + "go.sia.tech/renterd/stores" + "go.sia.tech/renterd/stores/sql" + "go.sia.tech/renterd/stores/sql/mysql" + "go.sia.tech/renterd/stores/sql/sqlite" + "go.sia.tech/renterd/webhooks" "go.sia.tech/renterd/worker" "go.sia.tech/renterd/worker/s3" "go.sia.tech/web/renterd" "go.uber.org/zap" + "golang.org/x/crypto/blake2b" "golang.org/x/sys/cpu" ) @@ -123,24 +132,14 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) // initialise bus busAddr, busPassword := cfg.Bus.RemoteAddr, cfg.Bus.RemotePassword if cfg.Bus.RemoteAddr == "" { - b, shutdownFn, _, err := bus.NewNode(bus.NodeConfig{ - Bus: cfg.Bus, - Database: cfg.Database, - DatabaseLog: cfg.Log.Database, - Logger: logger, - Network: network, - Genesis: genesis, - RetryTxIntervals: []time.Duration{ - 200 * time.Millisecond, - 500 * time.Millisecond, - time.Second, - 3 * time.Second, - 10 * time.Second, - 10 * time.Second, - }, - }, cfg.Directory, pk, logger) + // ensure we don't hang indefinitely + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // create bus + b, shutdownFn, err := newBus(ctx, cfg, pk, network, genesis, logger) if err != nil { - return nil, fmt.Errorf("failed to create bus: %w", err) + return nil, err } shutdownFns = append(shutdownFns, fn{ @@ -148,7 +147,7 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) fn: shutdownFn, }) - mux.Sub["/api/bus"] = utils.TreeMux{Handler: auth(b)} + mux.Sub["/api/bus"] = utils.TreeMux{Handler: auth(b.Handler())} busAddr = cfg.HTTP.Address + "/api/bus" busPassword = cfg.HTTP.Password @@ -166,14 +165,6 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) if len(cfg.Worker.Remotes) == 0 { if cfg.Worker.Enabled { workerAddr := cfg.HTTP.Address + "/api/worker" - w, s3Handler, setupFn, shutdownFn, err := worker.NewNode(cfg.Worker, s3.Opts{ - AuthDisabled: cfg.S3.DisableAuth, - HostBucketBases: cfg.S3.HostBucketBases, - HostBucketEnabled: cfg.S3.HostBucketEnabled, - }, bc, pk, logger) - if err != nil { - logger.Fatal("failed to create worker: " + err.Error()) - } var workerExternAddr string if cfg.Bus.RemoteAddr != "" { workerExternAddr = cfg.Worker.ExternalAddress @@ -181,22 +172,37 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) workerExternAddr = workerAddr } + workerKey := blake2b.Sum256(append([]byte("worker"), pk...)) + w, err := worker.New(cfg.Worker, workerKey, bc, logger.Named("worker")) + if err != nil { + logger.Fatal("failed to create worker: " + err.Error()) + } setupFns = append(setupFns, fn{ name: "Worker", fn: func(ctx context.Context) error { - return setupFn(ctx, workerExternAddr, cfg.HTTP.Password) + return w.Setup(ctx, workerExternAddr, cfg.HTTP.Password) }, }) shutdownFns = append(shutdownFns, fn{ name: "Worker", - fn: shutdownFn, + fn: w.Shutdown, }) - mux.Sub["/api/worker"] = utils.TreeMux{Handler: utils.Auth(cfg.HTTP.Password, cfg.Worker.AllowUnauthenticatedDownloads)(w)} + mux.Sub["/api/worker"] = utils.TreeMux{Handler: utils.Auth(cfg.HTTP.Password, cfg.Worker.AllowUnauthenticatedDownloads)(w.Handler())} wc := worker.NewClient(workerAddr, cfg.HTTP.Password) workers = append(workers, wc) if cfg.S3.Enabled { + s3Handler, err := s3.New(bc, w, logger.Named("s3"), s3.Opts{ + AuthDisabled: cfg.S3.DisableAuth, + HostBucketBases: cfg.S3.HostBucketBases, + HostBucketEnabled: cfg.S3.HostBucketEnabled, + }) + if err != nil { + err = errors.Join(err, w.Shutdown(context.Background())) + logger.Fatal("failed to create s3 handler: " + err.Error()) + } + s3Srv = &http.Server{ Addr: cfg.S3.Address, Handler: s3Handler, @@ -220,25 +226,20 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) // initialise autopilot if cfg.Autopilot.Enabled { - ap, runFn, shutdownFn, err := autopilot.NewNode(cfg.Autopilot, bc, workers, logger) + ap, err := autopilot.New(cfg.Autopilot, bc, workers, logger) if err != nil { logger.Fatal("failed to create autopilot: " + err.Error()) } - setupFns = append(setupFns, fn{ name: "Autopilot", - fn: func(_ context.Context) error { runFn(); return nil }, + fn: func(_ context.Context) error { ap.Run(); return nil }, }) - - // NOTE: shutdown functions are executed in reverse order, it's - // important the autopilot is shut down first so we don't needlessly - // make worker and bus calls while they're shutting down shutdownFns = append(shutdownFns, fn{ name: "Autopilot", - fn: shutdownFn, + fn: ap.Shutdown, }) - mux.Sub["/api/autopilot"] = utils.TreeMux{Handler: auth(ap)} + mux.Sub["/api/autopilot"] = utils.TreeMux{Handler: auth(ap.Handler())} } return &node{ @@ -257,6 +258,142 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) }, nil } +func newBus(ctx context.Context, cfg config.Config, pk types.PrivateKey, network *consensus.Network, genesis types.Block, logger *zap.Logger) (*bus.Bus, func(ctx context.Context) error, error) { + // create store + alertsMgr := alerts.NewManager() + storeCfg, err := buildStoreConfig(alertsMgr, cfg, pk, logger) + if err != nil { + return nil, nil, err + } + sqlStore, err := stores.NewSQLStore(storeCfg) + if err != nil { + return nil, nil, err + } + + // create webhooks manager + wh, err := webhooks.NewManager(sqlStore, logger) + if err != nil { + return nil, nil, err + } + + // hookup webhooks <-> alerts + alertsMgr.RegisterWebhookBroadcaster(wh) + + // create consensus directory + consensusDir := filepath.Join(cfg.Directory, "consensus") + if err := os.MkdirAll(consensusDir, 0700); err != nil { + return nil, nil, err + } + + // migrate consensus database if necessary + migrateConsensusDatabase(ctx, sqlStore, consensusDir, logger) + + // reset chain state if blockchain.db does not exist to make sure deleting + // it forces a resync + chainPath := filepath.Join(consensusDir, "blockchain.db") + if _, err := os.Stat(chainPath); os.IsNotExist(err) { + if err := sqlStore.ResetChainState(context.Background()); err != nil { + return nil, nil, err + } + } + + // create chain database + bdb, err := coreutils.OpenBoltChainDB(chainPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to open chain database: %w", err) + } + + // create chain manager + store, state, err := chain.NewDBStore(bdb, network, genesis) + if err != nil { + return nil, nil, err + } + cm := chain.NewManager(store, state) + + // create wallet + w, err := wallet.NewSingleAddressWallet(pk, cm, sqlStore, wallet.WithReservationDuration(cfg.Bus.UsedUTXOExpiry)) + if err != nil { + return nil, nil, err + } + + // bootstrap the syncer + if cfg.Bus.Bootstrap { + var peers []string + switch network.Name { + case "mainnet": + peers = syncer.MainnetBootstrapPeers + case "zen": + peers = syncer.ZenBootstrapPeers + case "anagami": + peers = syncer.AnagamiBootstrapPeers + default: + return nil, nil, fmt.Errorf("no available bootstrap peers for unknown network '%s'", network.Name) + } + for _, addr := range peers { + if err := sqlStore.AddPeer(addr); err != nil { + return nil, nil, fmt.Errorf("%w: failed to add bootstrap peer '%s'", err, addr) + } + } + } + + // create syncer, peers will reject us if our hostname is empty or + // unspecified, so use loopback + l, err := net.Listen("tcp", cfg.Bus.GatewayAddr) + if err != nil { + return nil, nil, err + } + syncerAddr := l.Addr().String() + host, port, _ := net.SplitHostPort(syncerAddr) + if ip := net.ParseIP(host); ip == nil || ip.IsUnspecified() { + syncerAddr = net.JoinHostPort("127.0.0.1", port) + } + + // create header + header := gateway.Header{ + GenesisID: genesis.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerAddr, + } + + // create the syncer + s := syncer.New(l, cm, sqlStore, header, syncer.WithLogger(logger.Named("syncer")), syncer.WithSendBlocksTimeout(time.Minute)) + + // start syncer + errChan := make(chan error, 1) + go func() { + errChan <- s.Run(context.Background()) + close(errChan) + }() + + // create a helper function to wait for syncer to wind down on shutdown + syncerShutdown := func(ctx context.Context) error { + select { + case err := <-errChan: + return err + case <-ctx.Done(): + return context.Cause(ctx) + } + } + + // create bus + announcementMaxAgeHours := time.Duration(cfg.Bus.AnnouncementMaxAgeHours) * time.Hour + b, err := bus.New(ctx, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, logger) + if err != nil { + return nil, nil, fmt.Errorf("failed to create bus: %w", err) + } + + return b, func(ctx context.Context) error { + return errors.Join( + s.Close(), + w.Close(), + b.Shutdown(ctx), + sqlStore.Close(), + bdb.Close(), + syncerShutdown(ctx), + ) + }, nil +} + func (n *node) Run() error { // start server go n.apiSrv.Serve(n.apiListener) @@ -311,16 +448,6 @@ func (n *node) Run() error { } n.logger.Info("bus: Listening on " + syncerAddress) - // run autopilot store migration - // - // TODO: we can safely remove this already - if n.cfg.Autopilot.Enabled { - autopilotDir := filepath.Join(n.cfg.Directory, n.cfg.Autopilot.ID) - if err := runCompatMigrateAutopilotJSONToStore(n.bus, "autopilot", autopilotDir); err != nil { - return fmt.Errorf("failed to migrate autopilot JSON: %w", err) - } - } - // open the web UI if enabled if n.cfg.AutoOpenWebUI { time.Sleep(time.Millisecond) // give the web server a chance to start @@ -360,60 +487,113 @@ func (n *node) Shutdown() error { return errors.Join(errs...) } -func runCompatMigrateAutopilotJSONToStore(bc *bus.Client, id, dir string) (err error) { - // check if the file exists - path := filepath.Join(dir, "autopilot.json") - if _, err := os.Stat(path); os.IsNotExist(err) { - return nil - } +// TODO: needs a better spot +func buildStoreConfig(am alerts.Alerter, cfg config.Config, pk types.PrivateKey, logger *zap.Logger) (stores.Config, error) { + // create database connections + var dbMain sql.Database + var dbMetrics sql.MetricsDatabase + if cfg.Database.MySQL.URI != "" { + // create MySQL connections + connMain, err := mysql.Open( + cfg.Database.MySQL.User, + cfg.Database.MySQL.Password, + cfg.Database.MySQL.URI, + cfg.Database.MySQL.Database, + ) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to open MySQL main database: %w", err) + } + connMetrics, err := mysql.Open( + cfg.Database.MySQL.User, + cfg.Database.MySQL.Password, + cfg.Database.MySQL.URI, + cfg.Database.MySQL.MetricsDatabase, + ) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to open MySQL metrics database: %w", err) + } + dbMain, err = mysql.NewMainDatabase(connMain, logger.Named("main").Sugar(), cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to create MySQL main database: %w", err) + } + dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger.Named("metrics").Sugar(), cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to create MySQL metrics database: %w", err) + } + } else { + // create database directory + dbDir := filepath.Join(cfg.Directory, "db") + if err := os.MkdirAll(dbDir, 0700); err != nil { + return stores.Config{}, err + } - // defer autopilot dir cleanup - defer func() { - if err == nil { - log.Println("migration: removing autopilot directory") - if err = os.RemoveAll(dir); err == nil { - log.Println("migration: done") - } + // create SQLite connections + db, err := sqlite.Open(filepath.Join(dbDir, "db.sqlite")) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to open SQLite main database: %w", err) + } + dbMain, err = sqlite.NewMainDatabase(db, logger.Named("main").Sugar(), cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to create SQLite main database: %w", err) } - }() - // read the json config - log.Println("migration: reading autopilot.json") - //nolint:tagliatelle - var cfg struct { - Config api.AutopilotConfig `json:"Config"` - } - if data, err := os.ReadFile(path); err != nil { - return err - } else if err := json.Unmarshal(data, &cfg); err != nil { - return err + dbm, err := sqlite.Open(filepath.Join(dbDir, "metrics.sqlite")) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to open SQLite metrics database: %w", err) + } + dbMetrics, err = sqlite.NewMetricsDatabase(dbm, logger.Named("metrics").Sugar(), cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to create SQLite metrics database: %w", err) + } } - // make sure we don't hang - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() + return stores.Config{ + Alerts: alerts.WithOrigin(am, "bus"), + DB: dbMain, + DBMetrics: dbMetrics, + PartialSlabDir: filepath.Join(cfg.Directory, "partial_slabs"), + Migrate: true, + SlabBufferCompletionThreshold: cfg.Bus.SlabBufferCompletionThreshold, + Logger: logger.Sugar(), + RetryTransactionIntervals: []time.Duration{ + 200 * time.Millisecond, + 500 * time.Millisecond, + time.Second, + 3 * time.Second, + 10 * time.Second, + 10 * time.Second, + }, + WalletAddress: types.StandardUnlockHash(pk.PublicKey()), + LongQueryDuration: cfg.Log.Database.SlowThreshold, + LongTxDuration: cfg.Log.Database.SlowThreshold, + }, nil +} - // check if the autopilot already exists, if so we don't need to migrate - _, err = bc.Autopilot(ctx, api.DefaultAutopilotID) - if err == nil { - log.Printf("migration: autopilot already exists in the bus, the autopilot.json won't be migrated\n old config: %+v\n", cfg.Config) +func migrateConsensusDatabase(ctx context.Context, store *stores.SQLStore, consensusDir string, logger *zap.Logger) error { + oldConsensus, err := os.Stat(filepath.Join(consensusDir, "consensus.db")) + if os.IsNotExist(err) { return nil + } else if err != nil { + return err } - // create an autopilot entry - log.Println("migration: persisting autopilot to the bus") - if err := bc.UpdateAutopilot(ctx, api.Autopilot{ - ID: id, - Config: cfg.Config, - }); err != nil { + logger.Warn("found old consensus.db, indicating a migration is necessary") + + // reset chain state + logger.Warn("Resetting chain state...") + if err := store.ResetChainState(ctx); err != nil { return err } + logger.Warn("Chain state was successfully reset.") - // remove autopilot folder and config - log.Println("migration: cleaning up autopilot directory") - if err = os.RemoveAll(dir); err == nil { - log.Println("migration: done") + // remove consensus.db and consensus.log file + logger.Warn("Removing consensus database...") + _ = os.RemoveAll(filepath.Join(consensusDir, "consensus.log")) // ignore error + if err := os.Remove(filepath.Join(consensusDir, "consensus.db")); err != nil { + return err } + logger.Warn(fmt.Sprintf("Old 'consensus.db' was successfully removed, reclaimed %v of disk space.", utils.HumanReadableSize(int(oldConsensus.Size())))) + logger.Warn("ATTENTION: consensus will now resync from scratch, this process may take several hours to complete") return nil } diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 5d582749c..5a1c91c55 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -15,21 +15,30 @@ import ( "github.com/minio/minio-go/v7" "go.sia.tech/core/consensus" + "go.sia.tech/core/gateway" "go.sia.tech/core/types" "go.sia.tech/coreutils" "go.sia.tech/coreutils/chain" + "go.sia.tech/coreutils/syncer" + "go.sia.tech/coreutils/wallet" "go.sia.tech/jape" + "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" "go.sia.tech/renterd/autopilot" "go.sia.tech/renterd/bus" "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/test" + "go.sia.tech/renterd/internal/utils" + "go.sia.tech/renterd/stores" + "go.sia.tech/renterd/stores/sql" "go.sia.tech/renterd/stores/sql/mysql" - "go.sia.tech/renterd/utils" + "go.sia.tech/renterd/stores/sql/sqlite" + "go.sia.tech/renterd/webhooks" "go.sia.tech/renterd/worker/s3" "go.sia.tech/web/renterd" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "golang.org/x/crypto/blake2b" "lukechampine.com/frand" "go.sia.tech/renterd/worker" @@ -72,6 +81,12 @@ type TestCluster struct { wg sync.WaitGroup } +type dbConfig struct { + Database config.Database + DatabaseLog config.DatabaseLog + RetryTxIntervals []time.Duration +} + func (tc *TestCluster) ShutdownAutopilot(ctx context.Context) { tc.tt.Helper() for _, fn := range tc.autopilotShutdownFns { @@ -160,7 +175,7 @@ type testClusterOptions struct { autopilotCfg *config.Autopilot autopilotSettings *api.AutopilotConfig - busCfg *bus.NodeConfig + busCfg *config.Bus workerCfg *config.Worker } @@ -210,7 +225,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if opts.walletKey != nil { wk = *opts.walletKey } - busCfg, workerCfg, apCfg := testBusCfg(), testWorkerCfg(), testApCfg() + busCfg, workerCfg, apCfg, dbCfg := testBusCfg(), testWorkerCfg(), testApCfg(), testDBCfg() if opts.busCfg != nil { busCfg = *opts.busCfg } @@ -236,34 +251,28 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if opts.autopilotSettings != nil { apSettings = *opts.autopilotSettings } - if busCfg.Logger == nil { - busCfg.Logger = logger - } if opts.dbName != "" { - busCfg.Database.MySQL.Database = opts.dbName + dbCfg.Database.MySQL.Database = opts.dbName } // Check if we are testing against an external database. If so, we create a // database with a random name first. if mysqlCfg := config.MySQLConfigFromEnv(); mysqlCfg.URI != "" { // generate a random database name if none are set - if busCfg.Database.MySQL.Database == "" { - busCfg.Database.MySQL.Database = "db" + hex.EncodeToString(frand.Bytes(16)) + if dbCfg.Database.MySQL.Database == "" { + dbCfg.Database.MySQL.Database = "db" + hex.EncodeToString(frand.Bytes(16)) } - if busCfg.Database.MySQL.MetricsDatabase == "" { - busCfg.Database.MySQL.MetricsDatabase = "db" + hex.EncodeToString(frand.Bytes(16)) + if dbCfg.Database.MySQL.MetricsDatabase == "" { + dbCfg.Database.MySQL.MetricsDatabase = "db" + hex.EncodeToString(frand.Bytes(16)) } tmpDB, err := mysql.Open(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, "") tt.OK(err) - tt.OKAll(tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", busCfg.Database.MySQL.Database))) - tt.OKAll(tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", busCfg.Database.MySQL.MetricsDatabase))) + tt.OKAll(tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", dbCfg.Database.MySQL.Database))) + tt.OKAll(tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", dbCfg.Database.MySQL.MetricsDatabase))) tt.OK(tmpDB.Close()) } - // Prepare individual dirs. - busDir := filepath.Join(dir, "bus") - // Generate API passwords. busPassword := randomPassword() workerPassword := randomPassword() @@ -303,7 +312,8 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { tt.OK(err) // Create bus. - b, bShutdownFn, cm, err := bus.NewNode(busCfg, busDir, wk, logger) + busDir := filepath.Join(dir, "bus") + b, bShutdownFn, cm, err := newTestBus(ctx, busDir, busCfg, dbCfg, wk, logger) tt.OK(err) busAuth := jape.BasicAuth(busPassword) @@ -312,7 +322,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { Handler: renterd.Handler(), // ui Sub: map[string]utils.TreeMux{ "/bus": { - Handler: busAuth(b), + Handler: busAuth(b.Handler()), }, }, }, @@ -323,44 +333,44 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { busShutdownFns = append(busShutdownFns, bShutdownFn) // Create worker. - w, s3Handler, wSetupFn, wShutdownFn, err := worker.NewNode(workerCfg, s3.Opts{}, busClient, wk, logger) + workerKey := blake2b.Sum256(append([]byte("worker"), wk...)) + w, err := worker.New(workerCfg, workerKey, busClient, logger.Named("worker")) tt.OK(err) - workerServer := http.Server{ - Handler: utils.Auth(workerPassword, false)(w), - } + workerServer := http.Server{Handler: utils.Auth(workerPassword, false)(w.Handler())} var workerShutdownFns []func(context.Context) error workerShutdownFns = append(workerShutdownFns, workerServer.Shutdown) - workerShutdownFns = append(workerShutdownFns, wShutdownFn) + workerShutdownFns = append(workerShutdownFns, w.Shutdown) // Create S3 API. - s3Server := http.Server{ - Handler: s3Handler, - } + s3Handler, err := s3.New(busClient, w, logger.Named("s3"), s3.Opts{}) + tt.OK(err) + s3Server := http.Server{Handler: s3Handler} var s3ShutdownFns []func(context.Context) error s3ShutdownFns = append(s3ShutdownFns, s3Server.Shutdown) // Create autopilot. - ap, aStartFn, aStopFn, err := autopilot.NewNode(apCfg, busClient, []autopilot.Worker{workerClient}, logger) + ap, err := autopilot.New(apCfg, busClient, []autopilot.Worker{workerClient}, logger) tt.OK(err) autopilotAuth := jape.BasicAuth(autopilotPassword) autopilotServer := http.Server{ - Handler: autopilotAuth(ap), + Handler: autopilotAuth(ap.Handler()), } var autopilotShutdownFns []func(context.Context) error autopilotShutdownFns = append(autopilotShutdownFns, autopilotServer.Shutdown) - autopilotShutdownFns = append(autopilotShutdownFns, aStopFn) + autopilotShutdownFns = append(autopilotShutdownFns, ap.Shutdown) + network, genesis := testNetwork() cluster := &TestCluster{ apID: apCfg.ID, dir: dir, - dbName: busCfg.Database.MySQL.Database, + dbName: dbCfg.Database.MySQL.Database, logger: logger, - network: busCfg.Network, - genesisBlock: busCfg.Genesis, + network: network, + genesisBlock: genesis, cm: cm, tt: tt, wk: wk, @@ -401,13 +411,13 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if !opts.skipRunningAutopilot { cluster.wg.Add(1) go func() { - aStartFn() + ap.Run() cluster.wg.Done() }() } // Finish worker setup. - if err := wSetupFn(ctx, workerAddr, workerPassword); err != nil { + if err := w.Setup(ctx, workerAddr, workerPassword); err != nil { tt.Fatalf("failed to setup worker, err: %v", err) } @@ -438,7 +448,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { // Fund the bus. if funding { - cluster.MineBlocks(busCfg.Network.HardforkFoundation.Height + blocksPerDay) // mine until the first block reward matures + cluster.MineBlocks(network.HardforkFoundation.Height + blocksPerDay) // mine until the first block reward matures tt.Retry(100, 100*time.Millisecond, func() error { if cs, err := busClient.ConsensusState(ctx); err != nil { return err @@ -474,6 +484,97 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { return cluster } +func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, pk types.PrivateKey, logger *zap.Logger) (*bus.Bus, func(ctx context.Context) error, *chain.Manager, error) { + // create store + alertsMgr := alerts.NewManager() + storeCfg, err := buildStoreConfig(alertsMgr, dir, cfg.SlabBufferCompletionThreshold, cfgDb, pk, logger) + if err != nil { + return nil, nil, nil, err + } + + sqlStore, err := stores.NewSQLStore(storeCfg) + if err != nil { + return nil, nil, nil, err + } + + // create webhooks manager + wh, err := webhooks.NewManager(sqlStore, logger) + if err != nil { + return nil, nil, nil, err + } + + // hookup webhooks <-> alerts + alertsMgr.RegisterWebhookBroadcaster(wh) + + // create consensus directory + consensusDir := filepath.Join(dir, "consensus") + if err := os.MkdirAll(consensusDir, 0700); err != nil { + return nil, nil, nil, err + } + + // create chain database + chainPath := filepath.Join(consensusDir, "blockchain.db") + bdb, err := coreutils.OpenBoltChainDB(chainPath) + if err != nil { + return nil, nil, nil, err + } + + // create chain manager + network, genesis := testNetwork() + store, state, err := chain.NewDBStore(bdb, network, genesis) + if err != nil { + return nil, nil, nil, err + } + cm := chain.NewManager(store, state) + + // create wallet + w, err := wallet.NewSingleAddressWallet(pk, cm, sqlStore, wallet.WithReservationDuration(cfg.UsedUTXOExpiry)) + if err != nil { + return nil, nil, nil, err + } + + // create syncer, peers will reject us if our hostname is empty or + // unspecified, so use loopback + l, err := net.Listen("tcp", cfg.GatewayAddr) + if err != nil { + return nil, nil, nil, err + } + syncerAddr := l.Addr().String() + host, port, _ := net.SplitHostPort(syncerAddr) + if ip := net.ParseIP(host); ip == nil || ip.IsUnspecified() { + syncerAddr = net.JoinHostPort("127.0.0.1", port) + } + + // create header + header := gateway.Header{ + GenesisID: genesis.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: syncerAddr, + } + + // create the syncer + s := syncer.New(l, cm, sqlStore, header, syncer.WithLogger(logger.Named("syncer")), syncer.WithSendBlocksTimeout(time.Minute)) + go s.Run(context.Background()) + + // create bus + announcementMaxAgeHours := time.Duration(cfg.AnnouncementMaxAgeHours) * time.Hour + b, err := bus.New(ctx, alertsMgr, wh, cm, s, w, sqlStore, announcementMaxAgeHours, logger) + if err != nil { + return nil, nil, nil, err + } + + shutdownFn := func(ctx context.Context) error { + return errors.Join( + s.Close(), + w.Close(), + b.Shutdown(ctx), + sqlStore.Close(), + bdb.Close(), + ) + } + return b, shutdownFn, cm, nil +} + // addStorageFolderToHosts adds a single storage folder to each host. func addStorageFolderToHost(ctx context.Context, hosts []*Host) error { for _, host := range hosts { @@ -883,17 +984,18 @@ func testNetwork() (*consensus.Network, types.Block) { return n, genesis } -func testBusCfg() bus.NodeConfig { - network, genesis := testNetwork() +func testBusCfg() config.Bus { + return config.Bus{ + AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year + Bootstrap: false, + GatewayAddr: "127.0.0.1:0", + UsedUTXOExpiry: time.Minute, + SlabBufferCompletionThreshold: 0, + } +} - return bus.NodeConfig{ - Bus: config.Bus{ - AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year - Bootstrap: false, - GatewayAddr: "127.0.0.1:0", - UsedUTXOExpiry: time.Minute, - SlabBufferCompletionThreshold: 0, - }, +func testDBCfg() dbConfig { + return dbConfig{ Database: config.Database{ MySQL: config.MySQLConfigFromEnv(), }, @@ -902,10 +1004,6 @@ func testBusCfg() bus.NodeConfig { IgnoreRecordNotFoundError: true, SlowThreshold: 100 * time.Millisecond, }, - Network: network, - Genesis: genesis, - SyncerSyncInterval: 100 * time.Millisecond, - SyncerPeerDiscoveryInterval: 100 * time.Millisecond, RetryTxIntervals: []time.Duration{ 50 * time.Millisecond, 100 * time.Millisecond, @@ -944,3 +1042,78 @@ func testApCfg() config.Autopilot { ScannerNumThreads: 1, } } + +func buildStoreConfig(am alerts.Alerter, dir string, slabBufferCompletionThreshold int64, cfg dbConfig, pk types.PrivateKey, logger *zap.Logger) (stores.Config, error) { + // create database connections + var dbMain sql.Database + var dbMetrics sql.MetricsDatabase + if cfg.Database.MySQL.URI != "" { + // create MySQL connections + connMain, err := mysql.Open( + cfg.Database.MySQL.User, + cfg.Database.MySQL.Password, + cfg.Database.MySQL.URI, + cfg.Database.MySQL.Database, + ) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to open MySQL main database: %w", err) + } + connMetrics, err := mysql.Open( + cfg.Database.MySQL.User, + cfg.Database.MySQL.Password, + cfg.Database.MySQL.URI, + cfg.Database.MySQL.MetricsDatabase, + ) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to open MySQL metrics database: %w", err) + } + dbMain, err = mysql.NewMainDatabase(connMain, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to create MySQL main database: %w", err) + } + dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger.Named("metrics").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to create MySQL metrics database: %w", err) + } + } else { + // create database directory + dbDir := filepath.Join(dir, "db") + if err := os.MkdirAll(dbDir, 0700); err != nil { + return stores.Config{}, err + } + + // create SQLite connections + db, err := sqlite.Open(filepath.Join(dbDir, "db.sqlite")) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to open SQLite main database: %w", err) + } + dbMain, err = sqlite.NewMainDatabase(db, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to create SQLite main database: %w", err) + } + + dbm, err := sqlite.Open(filepath.Join(dbDir, "metrics.sqlite")) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to open SQLite metrics database: %w", err) + } + dbMetrics, err = sqlite.NewMetricsDatabase(dbm, logger.Named("metrics").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + if err != nil { + return stores.Config{}, fmt.Errorf("failed to create SQLite metrics database: %w", err) + } + } + + return stores.Config{ + Alerts: alerts.WithOrigin(am, "bus"), + DB: dbMain, + DBMetrics: dbMetrics, + PartialSlabDir: filepath.Join(dir, "partial_slabs"), + Migrate: true, + SlabBufferCompletionThreshold: slabBufferCompletionThreshold, + Logger: logger.Sugar(), + WalletAddress: types.StandardUnlockHash(pk.PublicKey()), + + RetryTransactionIntervals: cfg.RetryTxIntervals, + LongQueryDuration: cfg.DatabaseLog.SlowThreshold, + LongTxDuration: cfg.DatabaseLog.SlowThreshold, + }, nil +} diff --git a/internal/test/e2e/host.go b/internal/test/e2e/host.go index 61d63d4bd..c6dd44381 100644 --- a/internal/test/e2e/host.go +++ b/internal/test/e2e/host.go @@ -240,7 +240,7 @@ func NewHost(privKey types.PrivateKey, dir string, network *consensus.Network, g GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), NetAddress: l.Addr().String(), - }, syncer.WithPeerDiscoveryInterval(testBusCfg().SyncerPeerDiscoveryInterval), syncer.WithSyncInterval(testBusCfg().SyncerSyncInterval)) + }, syncer.WithPeerDiscoveryInterval(100*time.Millisecond), syncer.WithSyncInterval(100*time.Millisecond)) syncErrChan := make(chan error, 1) go func() { syncErrChan <- s.Run(context.Background()) }() diff --git a/utils/web.go b/internal/utils/web.go similarity index 93% rename from utils/web.go rename to internal/utils/web.go index 85d1ec644..6f0caa571 100644 --- a/utils/web.go +++ b/internal/utils/web.go @@ -11,7 +11,6 @@ import ( "strings" "go.sia.tech/jape" - "go.sia.tech/renterd/internal/utils" "go.uber.org/zap" ) @@ -54,7 +53,7 @@ func Auth(password string, unauthenticatedDownloads bool) func(http.Handler) htt func ListenTCP(addr string, logger *zap.Logger) (net.Listener, error) { l, err := net.Listen("tcp", addr) - if utils.IsErr(err, errors.New("no such host")) && strings.Contains(addr, "localhost") { + if IsErr(err, errors.New("no such host")) && strings.Contains(addr, "localhost") { // fall back to 127.0.0.1 if 'localhost' doesn't work _, port, err := net.SplitHostPort(addr) if err != nil { diff --git a/worker/download.go b/worker/download.go index 118120c94..98d1d833d 100644 --- a/worker/download.go +++ b/worker/download.go @@ -126,21 +126,20 @@ type ( } ) -func (w *Worker) initDownloadManager(maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.SugaredLogger) { +func (w *Worker) initDownloadManager(maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.Logger) { if w.downloadManager != nil { panic("download manager already initialized") // developer error } - - mm := newMemoryManager(logger.Named("memorymanager"), maxMemory) - w.downloadManager = newDownloadManager(w.shutdownCtx, w, mm, w.bus, maxOverdrive, overdriveTimeout, logger) + w.downloadManager = newDownloadManager(w.shutdownCtx, w, w.bus, maxMemory, maxOverdrive, overdriveTimeout, logger) } -func newDownloadManager(ctx context.Context, hm HostManager, mm MemoryManager, os ObjectStore, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.SugaredLogger) *downloadManager { +func newDownloadManager(ctx context.Context, hm HostManager, os ObjectStore, maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.Logger) *downloadManager { + logger = logger.Named("downloadmanager") return &downloadManager{ hm: hm, - mm: mm, + mm: newMemoryManager(maxMemory, logger), os: os, - logger: logger, + logger: logger.Sugar(), maxOverdrive: maxOverdrive, overdriveTimeout: overdriveTimeout, diff --git a/worker/memory.go b/worker/memory.go index 1dbd680ec..a5306b2b4 100644 --- a/worker/memory.go +++ b/worker/memory.go @@ -41,9 +41,17 @@ type ( var _ MemoryManager = (*memoryManager)(nil) -func newMemoryManager(logger *zap.SugaredLogger, maxMemory uint64) MemoryManager { +func newMemoryManager(maxMemory uint64, logger *zap.Logger) MemoryManager { + logger = logger.Named("memorymanager") + return newMemoryManagerCustom(maxMemory, logger) +} + +// newMemoryManagerCustom is an internal constructor that doesn't name the +// logger being passed in, this avoids that we chain the logger name for every +// limit memory manager being created. +func newMemoryManagerCustom(maxMemory uint64, logger *zap.Logger) MemoryManager { mm := &memoryManager{ - logger: logger, + logger: logger.Sugar(), totalAvailable: maxMemory, } mm.available = mm.totalAvailable @@ -57,7 +65,7 @@ func (mm *memoryManager) Limit(amt uint64) (MemoryManager, error) { } return &limitMemoryManager{ parent: mm, - child: newMemoryManager(mm.logger, amt), + child: newMemoryManagerCustom(amt, mm.logger.Desugar()), }, nil } diff --git a/worker/node.go b/worker/node.go deleted file mode 100644 index d9a0d9467..000000000 --- a/worker/node.go +++ /dev/null @@ -1,28 +0,0 @@ -package worker - -import ( - "context" - "errors" - "fmt" - "net/http" - - "go.sia.tech/core/types" - "go.sia.tech/renterd/config" - "go.sia.tech/renterd/worker/s3" - "go.uber.org/zap" - "golang.org/x/crypto/blake2b" -) - -func NewNode(cfg config.Worker, s3Opts s3.Opts, b Bus, seed types.PrivateKey, l *zap.Logger) (http.Handler, http.Handler, func(context.Context, string, string) error, func(context.Context) error, error) { - workerKey := blake2b.Sum256(append([]byte("worker"), seed...)) - w, err := New(workerKey, cfg.ID, b, cfg.ContractLockTimeout, cfg.BusFlushInterval, cfg.DownloadOverdriveTimeout, cfg.UploadOverdriveTimeout, cfg.DownloadMaxOverdrive, cfg.UploadMaxOverdrive, cfg.DownloadMaxMemory, cfg.UploadMaxMemory, cfg.AllowPrivateIPs, l) - if err != nil { - return nil, nil, nil, nil, err - } - s3Handler, err := s3.New(b, w, l.Named("s3").Sugar(), s3Opts) - if err != nil { - err = errors.Join(err, w.Shutdown(context.Background())) - return nil, nil, nil, nil, fmt.Errorf("failed to create s3 handler: %w", err) - } - return w.Handler(), s3Handler, w.Setup, w.Shutdown, nil -} diff --git a/worker/s3/s3.go b/worker/s3/s3.go index cc6021652..e144e31ad 100644 --- a/worker/s3/s3.go +++ b/worker/s3/s3.go @@ -69,12 +69,11 @@ func (l *gofakes3Logger) Print(level gofakes3.LogLevel, v ...interface{}) { } } -func New(b Bus, w Worker, logger *zap.SugaredLogger, opts Opts) (http.Handler, error) { - namedLogger := logger.Named("s3") +func New(b Bus, w Worker, logger *zap.Logger, opts Opts) (http.Handler, error) { s3Backend := &s3{ b: b, w: w, - logger: namedLogger, + logger: logger.Sugar(), } backend := gofakes3.Backend(s3Backend) if !opts.AuthDisabled { @@ -84,9 +83,7 @@ func New(b Bus, w Worker, logger *zap.SugaredLogger, opts Opts) (http.Handler, e backend, gofakes3.WithHostBucket(opts.HostBucketEnabled), gofakes3.WithHostBucketBase(opts.HostBucketBases...), - gofakes3.WithLogger(&gofakes3Logger{ - l: namedLogger, - }), + gofakes3.WithLogger(&gofakes3Logger{l: logger.Sugar()}), gofakes3.WithRequestID(rand.Uint64()), gofakes3.WithoutVersioning(), ) diff --git a/worker/upload.go b/worker/upload.go index 423613ee5..6ced77f9a 100644 --- a/worker/upload.go +++ b/worker/upload.go @@ -146,13 +146,12 @@ type ( } ) -func (w *Worker) initUploadManager(maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.SugaredLogger) { +func (w *Worker) initUploadManager(maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, logger *zap.Logger) { if w.uploadManager != nil { panic("upload manager already initialized") // developer error } - mm := newMemoryManager(logger.Named("memorymanager"), maxMemory) - w.uploadManager = newUploadManager(w.shutdownCtx, w, mm, w.bus, w.bus, w.bus, maxOverdrive, overdriveTimeout, w.contractLockingDuration, logger) + w.uploadManager = newUploadManager(w.shutdownCtx, w, w.bus, w.bus, w.bus, maxMemory, maxOverdrive, overdriveTimeout, w.contractLockingDuration, logger) } func (w *Worker) upload(ctx context.Context, bucket, path string, rs api.RedundancySettings, r io.Reader, contracts []api.ContractMetadata, opts ...UploadOption) (_ string, err error) { @@ -303,14 +302,15 @@ func (w *Worker) tryUploadPackedSlab(ctx context.Context, mem Memory, ps api.Pac return nil } -func newUploadManager(ctx context.Context, hm HostManager, mm MemoryManager, os ObjectStore, cl ContractLocker, cs ContractStore, maxOverdrive uint64, overdriveTimeout time.Duration, contractLockDuration time.Duration, logger *zap.SugaredLogger) *uploadManager { +func newUploadManager(ctx context.Context, hm HostManager, os ObjectStore, cl ContractLocker, cs ContractStore, maxMemory, maxOverdrive uint64, overdriveTimeout time.Duration, contractLockDuration time.Duration, logger *zap.Logger) *uploadManager { + logger = logger.Named("uploadmanager") return &uploadManager{ hm: hm, - mm: mm, + mm: newMemoryManager(maxMemory, logger), os: os, cl: cl, cs: cs, - logger: logger, + logger: logger.Sugar(), contractLockDuration: contractLockDuration, diff --git a/worker/worker.go b/worker/worker.go index 1f23b2cc3..1456f808a 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -24,6 +24,7 @@ import ( "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" "go.sia.tech/renterd/build" + "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/utils" iworker "go.sia.tech/renterd/internal/worker" "go.sia.tech/renterd/object" @@ -1280,36 +1281,37 @@ func (w *Worker) stateHandlerGET(jc jape.Context) { } // New returns an HTTP handler that serves the worker API. -func New(masterKey [32]byte, id string, b Bus, contractLockingDuration, busFlushInterval, downloadOverdriveTimeout, uploadOverdriveTimeout time.Duration, downloadMaxOverdrive, uploadMaxOverdrive, downloadMaxMemory, uploadMaxMemory uint64, allowPrivateIPs bool, l *zap.Logger) (*Worker, error) { - if contractLockingDuration == 0 { +func New(cfg config.Worker, masterKey [32]byte, b Bus, l *zap.Logger) (*Worker, error) { + l = l.Named("worker").Named(cfg.ID) + + if cfg.ContractLockTimeout == 0 { return nil, errors.New("contract lock duration must be positive") } - if busFlushInterval == 0 { + if cfg.BusFlushInterval == 0 { return nil, errors.New("bus flush interval must be positive") } - if downloadOverdriveTimeout == 0 { + if cfg.DownloadOverdriveTimeout == 0 { return nil, errors.New("download overdrive timeout must be positive") } - if uploadOverdriveTimeout == 0 { + if cfg.UploadOverdriveTimeout == 0 { return nil, errors.New("upload overdrive timeout must be positive") } - if downloadMaxMemory == 0 { + if cfg.DownloadMaxMemory == 0 { return nil, errors.New("downloadMaxMemory cannot be 0") } - if uploadMaxMemory == 0 { + if cfg.UploadMaxMemory == 0 { return nil, errors.New("uploadMaxMemory cannot be 0") } - a := alerts.WithOrigin(b, fmt.Sprintf("worker.%s", id)) - l = l.Named("worker").Named(id) + a := alerts.WithOrigin(b, fmt.Sprintf("worker.%s", cfg.ID)) shutdownCtx, shutdownCancel := context.WithCancel(context.Background()) w := &Worker{ alerts: a, - allowPrivateIPs: allowPrivateIPs, - contractLockingDuration: contractLockingDuration, + allowPrivateIPs: cfg.AllowPrivateIPs, + contractLockingDuration: cfg.ContractLockTimeout, cache: iworker.NewCache(b, l), eventSubscriber: iworker.NewEventSubscriber(a, b, l, 10*time.Second), - id: id, + id: cfg.ID, bus: b, masterKey: masterKey, logger: l.Sugar(), @@ -1323,10 +1325,10 @@ func New(masterKey [32]byte, id string, b Bus, contractLockingDuration, busFlush w.initPriceTables() w.initTransportPool() - w.initDownloadManager(downloadMaxMemory, downloadMaxOverdrive, downloadOverdriveTimeout, l.Named("downloadmanager").Sugar()) - w.initUploadManager(uploadMaxMemory, uploadMaxOverdrive, uploadOverdriveTimeout, l.Named("uploadmanager").Sugar()) + w.initDownloadManager(cfg.DownloadMaxMemory, cfg.DownloadMaxOverdrive, cfg.DownloadOverdriveTimeout, l) + w.initUploadManager(cfg.UploadMaxMemory, cfg.UploadMaxOverdrive, cfg.UploadOverdriveTimeout, l) - w.initContractSpendingRecorder(busFlushInterval) + w.initContractSpendingRecorder(cfg.BusFlushInterval) return w, nil } diff --git a/worker/worker_test.go b/worker/worker_test.go index 6b03e8c3f..9b10f3074 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -8,6 +8,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/core/types" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/test" "go.uber.org/zap" "golang.org/x/crypto/blake2b" @@ -41,8 +42,19 @@ func newTestWorker(t test.TestingCommon) *testWorker { dlmm := newMemoryManagerMock() ulmm := newMemoryManagerMock() + // create cfg + cfg := config.Worker{ + ID: "test", + ContractLockTimeout: time.Second, + BusFlushInterval: time.Second, + DownloadOverdriveTimeout: time.Second, + UploadOverdriveTimeout: time.Second, + DownloadMaxMemory: 1 << 12, // 4 KiB + UploadMaxMemory: 1 << 12, // 4 KiB + } + // create worker - w, err := New(blake2b.Sum256([]byte("testwork")), "test", b, time.Second, time.Second, time.Second, time.Second, 0, 0, 1, 1, false, zap.NewNop()) + w, err := New(cfg, blake2b.Sum256([]byte("testwork")), b, zap.NewNop()) if err != nil { t.Fatal(err) } From 81111c0036509420aff270a1b716f22f8d6d1c0c Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 15 Aug 2024 13:57:55 +0200 Subject: [PATCH 09/10] all: cleanup PR --- autopilot/autopilot.go | 5 +++-- autopilot/scanner/scanner.go | 5 +++-- autopilot/scanner/scanner_test.go | 2 +- cmd/renterd/node.go | 14 +++++++------- internal/bus/chainsubscriber.go | 3 ++- internal/test/e2e/cluster.go | 14 +++++++------- internal/worker/cache.go | 3 ++- stores/slabbuffer.go | 7 ++++--- stores/slabbuffer_test.go | 4 ++-- stores/sql.go | 8 ++++---- stores/sql/mysql/main.go | 7 ++++--- stores/sql/mysql/metrics.go | 7 ++++--- stores/sql/sqlite/main.go | 7 ++++--- stores/sql/sqlite/metrics.go | 7 ++++--- stores/sql_test.go | 16 ++++++++-------- worker/memory.go | 9 ++++----- worker/s3/s3.go | 1 + worker/worker_test.go | 25 +++++++++++++------------ 18 files changed, 77 insertions(+), 67 deletions(-) diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 89f43ee68..58fb0a9ec 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -125,12 +125,13 @@ type Autopilot struct { // New initializes an Autopilot. func New(cfg config.Autopilot, bus Bus, workers []Worker, logger *zap.Logger) (_ *Autopilot, err error) { + logger = logger.Named("autopilot").Named(cfg.ID) shutdownCtx, shutdownCtxCancel := context.WithCancel(context.Background()) ap := &Autopilot{ alerts: alerts.WithOrigin(bus, fmt.Sprintf("autopilot.%s", cfg.ID)), id: cfg.ID, bus: bus, - logger: logger.Sugar().Named("autopilot").Named(cfg.ID), + logger: logger.Sugar(), workers: newWorkerPool(workers), shutdownCtx: shutdownCtx, @@ -141,7 +142,7 @@ func New(cfg config.Autopilot, bus Bus, workers []Worker, logger *zap.Logger) (_ pruningAlertIDs: make(map[types.FileContractID]types.Hash256), } - ap.s, err = scanner.New(ap.bus, cfg.ScannerBatchSize, cfg.ScannerNumThreads, cfg.ScannerInterval, ap.logger) + ap.s, err = scanner.New(ap.bus, cfg.ScannerBatchSize, cfg.ScannerNumThreads, cfg.ScannerInterval, logger) if err != nil { return } diff --git a/autopilot/scanner/scanner.go b/autopilot/scanner/scanner.go index b5acda651..6c34274ad 100644 --- a/autopilot/scanner/scanner.go +++ b/autopilot/scanner/scanner.go @@ -65,7 +65,8 @@ type ( } ) -func New(hs HostStore, scanBatchSize, scanThreads uint64, scanMinInterval time.Duration, logger *zap.SugaredLogger) (Scanner, error) { +func New(hs HostStore, scanBatchSize, scanThreads uint64, scanMinInterval time.Duration, logger *zap.Logger) (Scanner, error) { + logger = logger.Named("scanner") if scanBatchSize == 0 { return nil, errors.New("scanner batch size has to be greater than zero") } @@ -80,7 +81,7 @@ func New(hs HostStore, scanBatchSize, scanThreads uint64, scanMinInterval time.D scanInterval: scanMinInterval, statsHostPingMS: utils.NewDataPoints(0), - logger: logger.Named("scanner"), + logger: logger.Sugar(), interruptChan: make(chan struct{}), shutdownChan: make(chan struct{}), diff --git a/autopilot/scanner/scanner_test.go b/autopilot/scanner/scanner_test.go index 537d878c0..ee847395b 100644 --- a/autopilot/scanner/scanner_test.go +++ b/autopilot/scanner/scanner_test.go @@ -88,7 +88,7 @@ func TestScanner(t *testing.T) { hs := &mockHostStore{hosts: test.NewHosts(100)} // create test scanner - s, err := New(hs, testBatchSize, testNumThreads, time.Minute, zap.NewNop().Sugar()) + s, err := New(hs, testBatchSize, testNumThreads, time.Minute, zap.NewNop()) if err != nil { t.Fatal(err) } diff --git a/cmd/renterd/node.go b/cmd/renterd/node.go index be73a61c5..b4c330fc4 100644 --- a/cmd/renterd/node.go +++ b/cmd/renterd/node.go @@ -173,7 +173,7 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) } workerKey := blake2b.Sum256(append([]byte("worker"), pk...)) - w, err := worker.New(cfg.Worker, workerKey, bc, logger.Named("worker")) + w, err := worker.New(cfg.Worker, workerKey, bc, logger) if err != nil { logger.Fatal("failed to create worker: " + err.Error()) } @@ -193,7 +193,7 @@ func newNode(cfg config.Config, network *consensus.Network, genesis types.Block) workers = append(workers, wc) if cfg.S3.Enabled { - s3Handler, err := s3.New(bc, w, logger.Named("s3"), s3.Opts{ + s3Handler, err := s3.New(bc, w, logger, s3.Opts{ AuthDisabled: cfg.S3.DisableAuth, HostBucketBases: cfg.S3.HostBucketBases, HostBucketEnabled: cfg.S3.HostBucketEnabled, @@ -512,11 +512,11 @@ func buildStoreConfig(am alerts.Alerter, cfg config.Config, pk types.PrivateKey, if err != nil { return stores.Config{}, fmt.Errorf("failed to open MySQL metrics database: %w", err) } - dbMain, err = mysql.NewMainDatabase(connMain, logger.Named("main").Sugar(), cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + dbMain, err = mysql.NewMainDatabase(connMain, logger, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) if err != nil { return stores.Config{}, fmt.Errorf("failed to create MySQL main database: %w", err) } - dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger.Named("metrics").Sugar(), cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) if err != nil { return stores.Config{}, fmt.Errorf("failed to create MySQL metrics database: %w", err) } @@ -532,7 +532,7 @@ func buildStoreConfig(am alerts.Alerter, cfg config.Config, pk types.PrivateKey, if err != nil { return stores.Config{}, fmt.Errorf("failed to open SQLite main database: %w", err) } - dbMain, err = sqlite.NewMainDatabase(db, logger.Named("main").Sugar(), cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + dbMain, err = sqlite.NewMainDatabase(db, logger, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) if err != nil { return stores.Config{}, fmt.Errorf("failed to create SQLite main database: %w", err) } @@ -541,7 +541,7 @@ func buildStoreConfig(am alerts.Alerter, cfg config.Config, pk types.PrivateKey, if err != nil { return stores.Config{}, fmt.Errorf("failed to open SQLite metrics database: %w", err) } - dbMetrics, err = sqlite.NewMetricsDatabase(dbm, logger.Named("metrics").Sugar(), cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) + dbMetrics, err = sqlite.NewMetricsDatabase(dbm, logger, cfg.Log.Database.SlowThreshold, cfg.Log.Database.SlowThreshold) if err != nil { return stores.Config{}, fmt.Errorf("failed to create SQLite metrics database: %w", err) } @@ -554,7 +554,7 @@ func buildStoreConfig(am alerts.Alerter, cfg config.Config, pk types.PrivateKey, PartialSlabDir: filepath.Join(cfg.Directory, "partial_slabs"), Migrate: true, SlabBufferCompletionThreshold: cfg.Bus.SlabBufferCompletionThreshold, - Logger: logger.Sugar(), + Logger: logger, RetryTransactionIntervals: []time.Duration{ 200 * time.Millisecond, 500 * time.Millisecond, diff --git a/internal/bus/chainsubscriber.go b/internal/bus/chainsubscriber.go index 94cc55293..e1200c24b 100644 --- a/internal/bus/chainsubscriber.go +++ b/internal/bus/chainsubscriber.go @@ -94,12 +94,13 @@ type ( // given chain manager and chain store. The returned subscriber is already // running and can be stopped by calling Shutdown. func NewChainSubscriber(whm WebhookManager, cm ChainManager, cs ChainStore, w Wallet, announcementMaxAge time.Duration, logger *zap.Logger) *chainSubscriber { + logger = logger.Named("chainsubscriber") ctx, cancel := context.WithCancelCause(context.Background()) subscriber := &chainSubscriber{ cm: cm, cs: cs, wm: whm, - logger: logger.Sugar().Named("chainsubscriber"), + logger: logger.Sugar(), announcementMaxAge: announcementMaxAge, wallet: w, diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 250c4d3ec..f22c8dff0 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -334,7 +334,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { // Create worker. workerKey := blake2b.Sum256(append([]byte("worker"), wk...)) - w, err := worker.New(workerCfg, workerKey, busClient, logger.Named("worker")) + w, err := worker.New(workerCfg, workerKey, busClient, logger) tt.OK(err) workerServer := http.Server{Handler: utils.Auth(workerPassword, false)(w.Handler())} @@ -343,7 +343,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { workerShutdownFns = append(workerShutdownFns, w.Shutdown) // Create S3 API. - s3Handler, err := s3.New(busClient, w, logger.Named("s3"), s3.Opts{}) + s3Handler, err := s3.New(busClient, w, logger, s3.Opts{}) tt.OK(err) s3Server := http.Server{Handler: s3Handler} @@ -1060,11 +1060,11 @@ func buildStoreConfig(am alerts.Alerter, dir string, slabBufferCompletionThresho if err != nil { return stores.Config{}, fmt.Errorf("failed to open MySQL metrics database: %w", err) } - dbMain, err = mysql.NewMainDatabase(connMain, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + dbMain, err = mysql.NewMainDatabase(connMain, logger, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) if err != nil { return stores.Config{}, fmt.Errorf("failed to create MySQL main database: %w", err) } - dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger.Named("metrics").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) if err != nil { return stores.Config{}, fmt.Errorf("failed to create MySQL metrics database: %w", err) } @@ -1080,7 +1080,7 @@ func buildStoreConfig(am alerts.Alerter, dir string, slabBufferCompletionThresho if err != nil { return stores.Config{}, fmt.Errorf("failed to open SQLite main database: %w", err) } - dbMain, err = sqlite.NewMainDatabase(db, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + dbMain, err = sqlite.NewMainDatabase(db, logger, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) if err != nil { return stores.Config{}, fmt.Errorf("failed to create SQLite main database: %w", err) } @@ -1089,7 +1089,7 @@ func buildStoreConfig(am alerts.Alerter, dir string, slabBufferCompletionThresho if err != nil { return stores.Config{}, fmt.Errorf("failed to open SQLite metrics database: %w", err) } - dbMetrics, err = sqlite.NewMetricsDatabase(dbm, logger.Named("metrics").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) + dbMetrics, err = sqlite.NewMetricsDatabase(dbm, logger, cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold) if err != nil { return stores.Config{}, fmt.Errorf("failed to create SQLite metrics database: %w", err) } @@ -1102,7 +1102,7 @@ func buildStoreConfig(am alerts.Alerter, dir string, slabBufferCompletionThresho PartialSlabDir: filepath.Join(dir, "partial_slabs"), Migrate: true, SlabBufferCompletionThreshold: slabBufferCompletionThreshold, - Logger: logger.Sugar(), + Logger: logger, WalletAddress: types.StandardUnlockHash(pk.PublicKey()), RetryTransactionIntervals: cfg.RetryTxIntervals, diff --git a/internal/worker/cache.go b/internal/worker/cache.go index dc8ae6357..5081710c2 100644 --- a/internal/worker/cache.go +++ b/internal/worker/cache.go @@ -96,11 +96,12 @@ type cache struct { } func NewCache(b Bus, logger *zap.Logger) WorkerCache { + logger = logger.Named("workercache") return &cache{ b: b, cache: newMemoryCache(), - logger: logger.Sugar().Named("workercache"), + logger: logger.Sugar(), } } diff --git a/stores/slabbuffer.go b/stores/slabbuffer.go index 6337a53a4..5e8a542b8 100644 --- a/stores/slabbuffer.go +++ b/stores/slabbuffer.go @@ -53,7 +53,8 @@ type SlabBufferManager struct { buffersByKey map[string]*SlabBuffer } -func newSlabBufferManager(ctx context.Context, a alerts.Alerter, db sql.Database, logger *zap.SugaredLogger, slabBufferCompletionThreshold int64, partialSlabDir string) (*SlabBufferManager, error) { +func newSlabBufferManager(ctx context.Context, a alerts.Alerter, db sql.Database, logger *zap.Logger, slabBufferCompletionThreshold int64, partialSlabDir string) (*SlabBufferManager, error) { + logger = logger.Named("slabbuffers") if slabBufferCompletionThreshold < 0 || slabBufferCompletionThreshold > 1<<22 { return nil, fmt.Errorf("invalid slabBufferCompletionThreshold %v", slabBufferCompletionThreshold) } @@ -69,7 +70,7 @@ func newSlabBufferManager(ctx context.Context, a alerts.Alerter, db sql.Database bufferedSlabCompletionThreshold: slabBufferCompletionThreshold, db: db, dir: partialSlabDir, - logger: logger, + logger: logger.Sugar(), completeBuffers: make(map[bufferGroupID][]*SlabBuffer), incompleteBuffers: make(map[bufferGroupID][]*SlabBuffer), @@ -98,7 +99,7 @@ func newSlabBufferManager(ctx context.Context, a alerts.Alerter, db sql.Database }, Timestamp: time.Now(), }) - logger.Errorf("failed to open buffer file %v for slab %v: %v", buffer.Filename, buffer.Key, err) + logger.Sugar().Errorf("failed to open buffer file %v for slab %v: %v", buffer.Filename, buffer.Key, err) continue } diff --git a/stores/slabbuffer_test.go b/stores/slabbuffer_test.go index d9c0db48c..0a0c03192 100644 --- a/stores/slabbuffer_test.go +++ b/stores/slabbuffer_test.go @@ -21,7 +21,7 @@ func TestRecordAppendToCompletedBuffer(t *testing.T) { defer ss.Close() completionThreshold := int64(1000) - mgr, err := newSlabBufferManager(context.Background(), ss.alerts, ss.db, ss.logger, completionThreshold, t.TempDir()) + mgr, err := newSlabBufferManager(context.Background(), ss.alerts, ss.db, ss.logger.Desugar(), completionThreshold, t.TempDir()) if err != nil { t.Fatal(err) } @@ -71,7 +71,7 @@ func TestMarkBufferCompleteTwice(t *testing.T) { ss := newTestSQLStore(t, defaultTestSQLStoreConfig) defer ss.Close() - mgr, err := newSlabBufferManager(context.Background(), ss.alerts, ss.db, ss.logger, 0, t.TempDir()) + mgr, err := newSlabBufferManager(context.Background(), ss.alerts, ss.db, ss.logger.Desugar(), 0, t.TempDir()) if err != nil { t.Fatal(err) } diff --git a/stores/sql.go b/stores/sql.go index e6658f341..50533768d 100644 --- a/stores/sql.go +++ b/stores/sql.go @@ -25,7 +25,7 @@ type ( AnnouncementMaxAge time.Duration WalletAddress types.Address SlabBufferCompletionThreshold int64 - Logger *zap.SugaredLogger + Logger *zap.Logger RetryTransactionIntervals []time.Duration LongQueryDuration time.Duration LongTxDuration time.Duration @@ -77,7 +77,7 @@ func NewSQLStore(cfg Config) (*SQLStore, error) { if err != nil { return nil, fmt.Errorf("failed to fetch db version: %v", err) } - l.Infof("Using %s version %s", dbName, dbVersion) + l.Sugar().Infof("Using %s version %s", dbName, dbVersion) // Perform migrations. if cfg.Migrate { @@ -93,7 +93,7 @@ func NewSQLStore(cfg Config) (*SQLStore, error) { alerts: cfg.Alerts, db: dbMain, dbMetrics: dbMetrics, - logger: l, + logger: l.Sugar(), settings: make(map[string]string), walletAddress: cfg.WalletAddress, @@ -106,7 +106,7 @@ func NewSQLStore(cfg Config) (*SQLStore, error) { shutdownCtxCancel: shutdownCtxCancel, } - ss.slabBufferMgr, err = newSlabBufferManager(shutdownCtx, cfg.Alerts, dbMain, l.Named("slabbuffers"), cfg.SlabBufferCompletionThreshold, cfg.PartialSlabDir) + ss.slabBufferMgr, err = newSlabBufferManager(shutdownCtx, cfg.Alerts, dbMain, l, cfg.SlabBufferCompletionThreshold, cfg.PartialSlabDir) if err != nil { return nil, err } diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index 500cf091a..08ff0010e 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -39,11 +39,12 @@ type ( ) // NewMainDatabase creates a new MySQL backend. -func NewMainDatabase(db *dsql.DB, log *zap.SugaredLogger, lqd, ltd time.Duration) (*MainDatabase, error) { - store, err := sql.NewDB(db, log.Desugar(), deadlockMsgs, lqd, ltd) +func NewMainDatabase(db *dsql.DB, log *zap.Logger, lqd, ltd time.Duration) (*MainDatabase, error) { + log = log.Named("main") + store, err := sql.NewDB(db, log, deadlockMsgs, lqd, ltd) return &MainDatabase{ db: store, - log: log, + log: log.Sugar(), }, err } diff --git a/stores/sql/mysql/metrics.go b/stores/sql/mysql/metrics.go index dd51f228e..e7ef23813 100644 --- a/stores/sql/mysql/metrics.go +++ b/stores/sql/mysql/metrics.go @@ -30,11 +30,12 @@ type ( var _ ssql.MetricsDatabaseTx = (*MetricsDatabaseTx)(nil) // NewMetricsDatabase creates a new MySQL backend. -func NewMetricsDatabase(db *dsql.DB, log *zap.SugaredLogger, lqd, ltd time.Duration) (*MetricsDatabase, error) { - store, err := sql.NewDB(db, log.Desugar(), deadlockMsgs, lqd, ltd) +func NewMetricsDatabase(db *dsql.DB, log *zap.Logger, lqd, ltd time.Duration) (*MetricsDatabase, error) { + log = log.Named("metrics") + store, err := sql.NewDB(db, log, deadlockMsgs, lqd, ltd) return &MetricsDatabase{ db: store, - log: log, + log: log.Sugar(), }, err } diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index c81b93e9e..b72ec5e8c 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -38,11 +38,12 @@ type ( ) // NewMainDatabase creates a new SQLite backend. -func NewMainDatabase(db *dsql.DB, log *zap.SugaredLogger, lqd, ltd time.Duration) (*MainDatabase, error) { - store, err := sql.NewDB(db, log.Desugar(), deadlockMsgs, lqd, ltd) +func NewMainDatabase(db *dsql.DB, log *zap.Logger, lqd, ltd time.Duration) (*MainDatabase, error) { + log = log.Named("main") + store, err := sql.NewDB(db, log, deadlockMsgs, lqd, ltd) return &MainDatabase{ db: store, - log: log, + log: log.Sugar(), }, err } diff --git a/stores/sql/sqlite/metrics.go b/stores/sql/sqlite/metrics.go index c52417fee..df912d7c7 100644 --- a/stores/sql/sqlite/metrics.go +++ b/stores/sql/sqlite/metrics.go @@ -29,11 +29,12 @@ type ( var _ ssql.MetricsDatabaseTx = (*MetricsDatabaseTx)(nil) // NewSQLiteDatabase creates a new SQLite backend. -func NewMetricsDatabase(db *dsql.DB, log *zap.SugaredLogger, lqd, ltd time.Duration) (*MetricsDatabase, error) { - store, err := sql.NewDB(db, log.Desugar(), deadlockMsgs, lqd, ltd) +func NewMetricsDatabase(db *dsql.DB, log *zap.Logger, lqd, ltd time.Duration) (*MetricsDatabase, error) { + log = log.Named("metrics") + store, err := sql.NewDB(db, log, deadlockMsgs, lqd, ltd) return &MetricsDatabase{ db: store, - log: log, + log: log.Sugar(), }, err } diff --git a/stores/sql_test.go b/stores/sql_test.go index ba20bfc53..0846254cb 100644 --- a/stores/sql_test.go +++ b/stores/sql_test.go @@ -90,17 +90,17 @@ func (cfg *testSQLStoreConfig) dbConnections() (sql.Database, sql.MetricsDatabas // create MySQL conns connMain, err := mysql.Open(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, mysqlCfg.Database) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("failed to open MySQL main database: %w", err) } connMetrics, err := mysql.Open(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, mysqlCfg.MetricsDatabase) if err != nil { return nil, nil, fmt.Errorf("failed to open MySQL metrics database: %w", err) } - dbMain, err = mysql.NewMainDatabase(connMain, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond) + dbMain, err = mysql.NewMainDatabase(connMain, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) if err != nil { return nil, nil, fmt.Errorf("failed to create MySQL main database: %w", err) } - dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond) + dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) if err != nil { return nil, nil, fmt.Errorf("failed to create MySQL metrics database: %w", err) } @@ -114,11 +114,11 @@ func (cfg *testSQLStoreConfig) dbConnections() (sql.Database, sql.MetricsDatabas if err != nil { return nil, nil, fmt.Errorf("failed to open SQLite metrics database: %w", err) } - dbMain, err = sqlite.NewMainDatabase(connMain, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond) + dbMain, err = sqlite.NewMainDatabase(connMain, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) if err != nil { return nil, nil, fmt.Errorf("failed to create SQLite main database: %w", err) } - dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond) + dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) if err != nil { return nil, nil, fmt.Errorf("failed to create SQLite metrics database: %w", err) } @@ -132,11 +132,11 @@ func (cfg *testSQLStoreConfig) dbConnections() (sql.Database, sql.MetricsDatabas if err != nil { return nil, nil, fmt.Errorf("failed to open ephemeral SQLite metrics database: %w", err) } - dbMain, err = sqlite.NewMainDatabase(connMain, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond) + dbMain, err = sqlite.NewMainDatabase(connMain, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) if err != nil { return nil, nil, fmt.Errorf("failed to create ephemeral SQLite main database: %w", err) } - dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond) + dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, zap.NewNop(), 100*time.Millisecond, 100*time.Millisecond) if err != nil { return nil, nil, fmt.Errorf("failed to create ephemeral SQLite metrics database: %w", err) } @@ -175,7 +175,7 @@ func newTestSQLStore(t *testing.T, cfg testSQLStoreConfig) *testSQLStore { PartialSlabDir: cfg.dir, Migrate: !cfg.skipMigrate, SlabBufferCompletionThreshold: 0, - Logger: zap.NewNop().Sugar(), + Logger: zap.NewNop(), LongQueryDuration: 100 * time.Millisecond, LongTxDuration: 100 * time.Millisecond, RetryTransactionIntervals: []time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond}, diff --git a/worker/memory.go b/worker/memory.go index a5306b2b4..d6f459bc8 100644 --- a/worker/memory.go +++ b/worker/memory.go @@ -42,16 +42,15 @@ type ( var _ MemoryManager = (*memoryManager)(nil) func newMemoryManager(maxMemory uint64, logger *zap.Logger) MemoryManager { - logger = logger.Named("memorymanager") - return newMemoryManagerCustom(maxMemory, logger) + return newMemoryManagerCustom(maxMemory, logger.Named("memorymanager").Sugar()) } // newMemoryManagerCustom is an internal constructor that doesn't name the // logger being passed in, this avoids that we chain the logger name for every // limit memory manager being created. -func newMemoryManagerCustom(maxMemory uint64, logger *zap.Logger) MemoryManager { +func newMemoryManagerCustom(maxMemory uint64, logger *zap.SugaredLogger) MemoryManager { mm := &memoryManager{ - logger: logger.Sugar(), + logger: logger, totalAvailable: maxMemory, } mm.available = mm.totalAvailable @@ -65,7 +64,7 @@ func (mm *memoryManager) Limit(amt uint64) (MemoryManager, error) { } return &limitMemoryManager{ parent: mm, - child: newMemoryManagerCustom(amt, mm.logger.Desugar()), + child: newMemoryManagerCustom(amt, mm.logger), }, nil } diff --git a/worker/s3/s3.go b/worker/s3/s3.go index e144e31ad..d5cbb71a3 100644 --- a/worker/s3/s3.go +++ b/worker/s3/s3.go @@ -70,6 +70,7 @@ func (l *gofakes3Logger) Print(level gofakes3.LogLevel, v ...interface{}) { } func New(b Bus, w Worker, logger *zap.Logger, opts Opts) (http.Handler, error) { + logger = logger.Named("s3") s3Backend := &s3{ b: b, w: w, diff --git a/worker/worker_test.go b/worker/worker_test.go index 9b10f3074..f0822f03f 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -42,19 +42,8 @@ func newTestWorker(t test.TestingCommon) *testWorker { dlmm := newMemoryManagerMock() ulmm := newMemoryManagerMock() - // create cfg - cfg := config.Worker{ - ID: "test", - ContractLockTimeout: time.Second, - BusFlushInterval: time.Second, - DownloadOverdriveTimeout: time.Second, - UploadOverdriveTimeout: time.Second, - DownloadMaxMemory: 1 << 12, // 4 KiB - UploadMaxMemory: 1 << 12, // 4 KiB - } - // create worker - w, err := New(cfg, blake2b.Sum256([]byte("testwork")), b, zap.NewNop()) + w, err := New(newTestWorkerCfg(), blake2b.Sum256([]byte("testwork")), b, zap.NewNop()) if err != nil { t.Fatal(err) } @@ -141,6 +130,18 @@ func (w *testWorker) RenewContract(hk types.PublicKey) *contractMock { return renewal } +func newTestWorkerCfg() config.Worker { + return config.Worker{ + ID: "test", + ContractLockTimeout: time.Second, + BusFlushInterval: time.Second, + DownloadOverdriveTimeout: time.Second, + UploadOverdriveTimeout: time.Second, + DownloadMaxMemory: 1 << 12, // 4 KiB + UploadMaxMemory: 1 << 12, // 4 KiB + } +} + func newTestSector() (*[rhpv2.SectorSize]byte, types.Hash256) { var sector [rhpv2.SectorSize]byte frand.Read(sector[:]) From e3c9f373a88cf55c37e395ff217f40d6718eb6ac Mon Sep 17 00:00:00 2001 From: PJ Date: Thu, 15 Aug 2024 14:43:07 +0200 Subject: [PATCH 10/10] testing: shut down syncer --- internal/test/e2e/cluster.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index f22c8dff0..2ac6e5a76 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -554,7 +554,23 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, // create the syncer s := syncer.New(l, cm, sqlStore, header, syncer.WithLogger(logger.Named("syncer")), syncer.WithSendBlocksTimeout(time.Minute)) - go s.Run(context.Background()) + + // start syncer + errChan := make(chan error, 1) + go func() { + errChan <- s.Run(context.Background()) + close(errChan) + }() + + // create a helper function to wait for syncer to wind down on shutdown + syncerShutdown := func(ctx context.Context) error { + select { + case err := <-errChan: + return err + case <-ctx.Done(): + return context.Cause(ctx) + } + } // create bus announcementMaxAgeHours := time.Duration(cfg.AnnouncementMaxAgeHours) * time.Hour @@ -570,6 +586,7 @@ func newTestBus(ctx context.Context, dir string, cfg config.Bus, cfgDb dbConfig, b.Shutdown(ctx), sqlStore.Close(), bdb.Close(), + syncerShutdown(ctx), ) } return b, shutdownFn, cm, nil