From de0682ba6e98204c571d93424de6d45226106aa9 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 8 Aug 2024 14:08:25 +0200 Subject: [PATCH 01/12] cmd: add network cli flag --- api/bus.go | 1 + api/setting.go | 48 ++++++++++++++++++ api/state.go | 1 - autopilot/autopilot.go | 1 - build/env_default.go | 58 ---------------------- build/env_testnet.go | 62 ------------------------ build/network.go | 33 ------------- bus/bus.go | 28 ++++++----- bus/client/client_test.go | 6 +-- cmd/renterd/commands.go | 4 +- cmd/renterd/main.go | 86 +++++++++++++++++++++++++-------- config/config.go | 1 + internal/bus/pinmanager_test.go | 7 ++- internal/chain/chain.go | 4 ++ internal/test/e2e/cluster.go | 3 +- worker/upload.go | 4 +- worker/upload_params.go | 11 +---- worker/upload_test.go | 7 ++- worker/worker.go | 7 +-- 19 files changed, 155 insertions(+), 217 deletions(-) delete mode 100644 build/env_default.go delete mode 100644 build/env_testnet.go delete mode 100644 build/network.go diff --git a/api/bus.go b/api/bus.go index 8652a5f49..453af61ca 100644 --- a/api/bus.go +++ b/api/bus.go @@ -48,6 +48,7 @@ type ( // BusStateResponse is the response type for the /bus/state endpoint. BusStateResponse struct { StartTime TimeRFC3339 `json:"startTime"` + Network string `json:"network"` BuildState } ) diff --git a/api/setting.go b/api/setting.go index 19c73ac78..175794432 100644 --- a/api/setting.go +++ b/api/setting.go @@ -32,6 +32,54 @@ var ( // ErrSettingNotFound is returned if a requested setting is not present in the // database. ErrSettingNotFound = errors.New("setting not found") + + // DefaultGougingSettings define the default gouging settings the bus is + // configured with on startup. These values can be adjusted using the + // settings API. + // + DefaultGougingSettings = GougingSettings{ + MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC + MaxContractPrice: types.Siacoins(15), // 15 SC per contract + MaxDownloadPrice: types.Siacoins(3000), // 3000 SC per 1 TiB + MaxUploadPrice: types.Siacoins(3000), // 3000 SC per 1 TiB + MaxStoragePrice: types.Siacoins(3000).Div64(1 << 40).Div64(144 * 30), // 3000 SC per TiB per month + HostBlockHeightLeeway: 6, // 6 blocks + MinPriceTableValidity: 5 * time.Minute, // 5 minutes + MinAccountExpiry: 24 * time.Hour, // 1 day + MinMaxEphemeralAccountBalance: types.Siacoins(1), // 1 SC + MigrationSurchargeMultiplier: 10, // 10x + } + + // DefaultPricePinSettings define the default price pin settings the bus is + // configured with on startup. These values can be adjusted using the + // settings API. + DefaultPricePinSettings = PricePinSettings{ + Enabled: false, + } + + // DefaultUploadPackingSettings define the default upload packing settings + // the bus is configured with on startup. + DefaultUploadPackingSettings = UploadPackingSettings{ + Enabled: true, + SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB + } + + // DefaultRedundancySettings define the default redundancy settings the bus + // is configured with on startup. These values can be adjusted using the + // settings API. + // + // NOTE: default redundancy settings for testnet are different from mainnet. + DefaultRedundancySettings = RedundancySettings{ + MinShards: 10, + TotalShards: 30, + } + + // Same as DefaultRedundancySettings but for running on testnet networks due + // to their reduced number of hosts. + DefaultRedundancySettingsTestnet = RedundancySettings{ + MinShards: 2, + TotalShards: 6, + } ) type ( diff --git a/api/state.go b/api/state.go index 63faa95ea..ff292e177 100644 --- a/api/state.go +++ b/api/state.go @@ -3,7 +3,6 @@ package api type ( // BuildState contains static information about the build. BuildState struct { - Network string `json:"network"` Version string `json:"version"` Commit string `json:"commit"` OS string `json:"os"` diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index d16e0e8aa..52da95099 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -825,7 +825,6 @@ func (ap *Autopilot) stateHandlerGET(jc jape.Context) { StartTime: api.TimeRFC3339(ap.StartTime()), BuildState: api.BuildState{ - Network: build.NetworkName(), Version: build.Version(), Commit: build.Commit(), OS: runtime.GOOS, diff --git a/build/env_default.go b/build/env_default.go deleted file mode 100644 index 3730fd5b2..000000000 --- a/build/env_default.go +++ /dev/null @@ -1,58 +0,0 @@ -//go:build !testnet - -package build - -import ( - "time" - - "go.sia.tech/core/types" - "go.sia.tech/renterd/api" -) - -const ( - network = "mainnet" - - DefaultAPIAddress = "localhost:9980" - DefaultGatewayAddress = ":9981" - DefaultS3Address = "localhost:8080" -) - -var ( - // DefaultGougingSettings define the default gouging settings the bus is - // configured with on startup. These values can be adjusted using the - // settings API. - DefaultGougingSettings = api.GougingSettings{ - MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC - MaxContractPrice: types.Siacoins(1), // 1 SC per contract - MaxDownloadPrice: types.Siacoins(3000), // 3000 SC per 1 TiB - MaxUploadPrice: types.Siacoins(3000), // 3000 SC per 1 TiB - MaxStoragePrice: types.Siacoins(3000).Div64(1 << 40).Div64(144 * 30), // 3000 SC per TiB per month - HostBlockHeightLeeway: 6, // 6 blocks - MinPriceTableValidity: 5 * time.Minute, // 5 minutes - MinAccountExpiry: 24 * time.Hour, // 1 day - MinMaxEphemeralAccountBalance: types.Siacoins(1), // 1 SC - MigrationSurchargeMultiplier: 10, // 10x - } - - // DefaultPricePinSettings define the default price pin settings the bus is - // configured with on startup. These values can be adjusted using the - // settings API. - DefaultPricePinSettings = api.PricePinSettings{ - Enabled: false, - } - - // DefaultUploadPackingSettings define the default upload packing settings - // the bus is configured with on startup. - DefaultUploadPackingSettings = api.UploadPackingSettings{ - Enabled: true, - SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB - } - - // DefaultRedundancySettings define the default redundancy settings the bus - // is configured with on startup. These values can be adjusted using the - // settings API. - DefaultRedundancySettings = api.RedundancySettings{ - MinShards: 10, - TotalShards: 30, - } -) diff --git a/build/env_testnet.go b/build/env_testnet.go deleted file mode 100644 index 5ccf6f24f..000000000 --- a/build/env_testnet.go +++ /dev/null @@ -1,62 +0,0 @@ -//go:build testnet - -package build - -import ( - "time" - - "go.sia.tech/core/types" - "go.sia.tech/renterd/api" -) - -const ( - network = "zen" - - DefaultAPIAddress = "localhost:9880" - DefaultGatewayAddress = ":9881" - DefaultS3Address = "localhost:7070" -) - -var ( - // DefaultGougingSettings define the default gouging settings the bus is - // configured with on startup. These values can be adjusted using the - // settings API. - // - // NOTE: default gouging settings for testnet are identical to mainnet. - DefaultGougingSettings = api.GougingSettings{ - MaxRPCPrice: types.Siacoins(1).Div64(1000), // 1mS per RPC - MaxContractPrice: types.Siacoins(15), // 15 SC per contract - MaxDownloadPrice: types.Siacoins(3000), // 3000 SC per 1 TiB - MaxUploadPrice: types.Siacoins(3000), // 3000 SC per 1 TiB - MaxStoragePrice: types.Siacoins(3000).Div64(1 << 40).Div64(144 * 30), // 3000 SC per TiB per month - HostBlockHeightLeeway: 6, // 6 blocks - MinPriceTableValidity: 5 * time.Minute, // 5 minutes - MinAccountExpiry: 24 * time.Hour, // 1 day - MinMaxEphemeralAccountBalance: types.Siacoins(1), // 1 SC - MigrationSurchargeMultiplier: 10, // 10x - } - - // DefaultPricePinSettings define the default price pin settings the bus is - // configured with on startup. These values can be adjusted using the - // settings API. - DefaultPricePinSettings = api.PricePinSettings{ - Enabled: false, - } - - // DefaultUploadPackingSettings define the default upload packing settings - // the bus is configured with on startup. - DefaultUploadPackingSettings = api.UploadPackingSettings{ - Enabled: true, - SlabBufferMaxSizeSoft: 1 << 32, // 4 GiB - } - - // DefaultRedundancySettings define the default redundancy settings the bus - // is configured with on startup. These values can be adjusted using the - // settings API. - // - // NOTE: default redundancy settings for testnet are different from mainnet. - DefaultRedundancySettings = api.RedundancySettings{ - MinShards: 2, - TotalShards: 6, - } -) diff --git a/build/network.go b/build/network.go deleted file mode 100644 index a0a452189..000000000 --- a/build/network.go +++ /dev/null @@ -1,33 +0,0 @@ -package build - -//go:generate go run gen.go - -import ( - "go.sia.tech/core/consensus" - "go.sia.tech/core/types" - "go.sia.tech/coreutils/chain" -) - -// Network returns the Sia network consts and genesis block for the current build. -func Network() (*consensus.Network, types.Block) { - switch network { - case "mainnet": - return chain.Mainnet() - case "zen": - return chain.TestnetZen() - default: - panic("unknown network: " + network) - } -} - -func NetworkName() string { - n, _ := Network() - switch n.Name { - case "mainnet": - return "Mainnet" - case "zen": - return "Zen Testnet" - default: - return n.Name - } -} diff --git a/bus/bus.go b/bus/bus.go index 029aa2c97..cbf4ec5df 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -2236,12 +2236,12 @@ func (b *bus) stateHandlerGET(jc jape.Context) { jc.Encode(api.BusStateResponse{ StartTime: api.TimeRFC3339(b.startTime), BuildState: api.BuildState{ - Network: build.NetworkName(), Version: build.Version(), Commit: build.Commit(), OS: runtime.GOOS, BuildTime: api.TimeRFC3339(build.BuildTime()), }, + Network: b.cm.TipState().Network.Name, }) } @@ -2605,12 +2605,18 @@ func New(am *alerts.Manager, whm *webhooks.Manager, cm ChainManager, cs ChainSto ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() + // testnets have different redundancy settings + defaultRedundancySettings := api.DefaultRedundancySettings + if mn, _ := chain.Mainnet(); mn.Name != b.cm.TipState().Network.Name { + defaultRedundancySettings = api.DefaultRedundancySettingsTestnet + } + // load default settings if the setting is not already set for key, value := range map[string]interface{}{ - api.SettingGouging: build.DefaultGougingSettings, - api.SettingPricePinning: build.DefaultPricePinSettings, - api.SettingRedundancy: build.DefaultRedundancySettings, - api.SettingUploadPacking: build.DefaultUploadPackingSettings, + api.SettingGouging: api.DefaultGougingSettings, + api.SettingPricePinning: api.DefaultPricePinSettings, + api.SettingRedundancy: defaultRedundancySettings, + api.SettingUploadPacking: api.DefaultUploadPackingSettings, } { if _, err := b.ss.Setting(ctx, key); errors.Is(err, api.ErrSettingNotFound) { if bytes, err := json.Marshal(value); err != nil { @@ -2629,7 +2635,7 @@ func New(am *alerts.Manager, whm *webhooks.Manager, cm ChainManager, cs ChainSto return nil, err } else if err := rs.Validate(); err != nil { l.Warn(fmt.Sprintf("invalid redundancy setting found '%v', overwriting the redundancy settings with the default settings", rss)) - bytes, _ := json.Marshal(build.DefaultRedundancySettings) + bytes, _ := json.Marshal(defaultRedundancySettings) if err := b.ss.UpdateSetting(ctx, api.SettingRedundancy, string(bytes)); err != nil { return nil, err } @@ -2643,9 +2649,9 @@ func New(am *alerts.Manager, whm *webhooks.Manager, cm ChainManager, cs ChainSto return nil, err } else if err := gs.Validate(); err != nil { // compat: apply default EA gouging settings - gs.MinMaxEphemeralAccountBalance = build.DefaultGougingSettings.MinMaxEphemeralAccountBalance - gs.MinPriceTableValidity = build.DefaultGougingSettings.MinPriceTableValidity - gs.MinAccountExpiry = build.DefaultGougingSettings.MinAccountExpiry + gs.MinMaxEphemeralAccountBalance = api.DefaultGougingSettings.MinMaxEphemeralAccountBalance + gs.MinPriceTableValidity = api.DefaultGougingSettings.MinPriceTableValidity + gs.MinAccountExpiry = api.DefaultGougingSettings.MinAccountExpiry if err := gs.Validate(); err == nil { l.Info(fmt.Sprintf("updating gouging settings with default EA settings: %+v", gs)) bytes, _ := json.Marshal(gs) @@ -2654,7 +2660,7 @@ func New(am *alerts.Manager, whm *webhooks.Manager, cm ChainManager, cs ChainSto } } else { // compat: apply default host block leeway settings - gs.HostBlockHeightLeeway = build.DefaultGougingSettings.HostBlockHeightLeeway + gs.HostBlockHeightLeeway = api.DefaultGougingSettings.HostBlockHeightLeeway if err := gs.Validate(); err == nil { l.Info(fmt.Sprintf("updating gouging settings with default HostBlockHeightLeeway settings: %v", gs)) bytes, _ := json.Marshal(gs) @@ -2663,7 +2669,7 @@ func New(am *alerts.Manager, whm *webhooks.Manager, cm ChainManager, cs ChainSto } } else { l.Warn(fmt.Sprintf("invalid gouging setting found '%v', overwriting the gouging settings with the default settings", gss)) - bytes, _ := json.Marshal(build.DefaultGougingSettings) + bytes, _ := json.Marshal(api.DefaultGougingSettings) if err := b.ss.UpdateSetting(ctx, api.SettingGouging, string(bytes)); err != nil { return nil, err } diff --git a/bus/client/client_test.go b/bus/client/client_test.go index bd459a6f8..92ad50651 100644 --- a/bus/client/client_test.go +++ b/bus/client/client_test.go @@ -12,9 +12,9 @@ import ( "go.sia.tech/core/types" "go.sia.tech/jape" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/build" "go.sia.tech/renterd/bus/client" "go.sia.tech/renterd/config" + "go.sia.tech/renterd/internal/chain" "go.sia.tech/renterd/internal/node" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -56,7 +56,7 @@ func TestClient(t *testing.T) { // fetch redundancy settings and assert they're configured to the default values if rs, err := c.RedundancySettings(ctx); err != nil { t.Fatal(err) - } else if rs.MinShards != build.DefaultRedundancySettings.MinShards || rs.TotalShards != build.DefaultRedundancySettings.TotalShards { + } else if rs.MinShards != api.DefaultRedundancySettings.MinShards || rs.TotalShards != api.DefaultRedundancySettings.TotalShards { t.Fatal("unexpected redundancy settings", rs) } } @@ -69,7 +69,7 @@ func newTestClient(dir string) (*client.Client, func() error, func(context.Conte } // create bus - network, genesis := build.Network() + network, genesis := chain.Mainnet() b, _, shutdown, _, _, err := node.NewBus(node.BusConfig{ Bus: config.Bus{ AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year diff --git a/cmd/renterd/commands.go b/cmd/renterd/commands.go index 96c798abf..63898cfe1 100644 --- a/cmd/renterd/commands.go +++ b/cmd/renterd/commands.go @@ -78,9 +78,9 @@ func cmdSeed() { fmt.Println("Address", types.StandardUnlockHash(key.PublicKey())) } -func cmdVersion() { +func cmdVersion(network string) { fmt.Println("renterd", build.Version()) - fmt.Println("Network", build.NetworkName()) + fmt.Println("Network", network) fmt.Println("Commit:", build.Commit()) fmt.Println("Build Date:", build.BuildTime()) } diff --git a/cmd/renterd/main.go b/cmd/renterd/main.go index cd4099091..7b140c504 100644 --- a/cmd/renterd/main.go +++ b/cmd/renterd/main.go @@ -18,7 +18,9 @@ import ( "syscall" "time" + "go.sia.tech/core/consensus" "go.sia.tech/core/types" + "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/wallet" "go.sia.tech/jape" "go.sia.tech/renterd/api" @@ -77,8 +79,9 @@ var ( Directory: ".", Seed: os.Getenv("RENTERD_SEED"), AutoOpenWebUI: true, + Network: "mainnet", HTTP: config.HTTP{ - Address: build.DefaultAPIAddress, + Address: "", // default determined by network Password: os.Getenv("RENTERD_API_PASSWORD"), }, ShutdownTimeout: 5 * time.Minute, @@ -111,7 +114,7 @@ var ( Bus: config.Bus{ AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year Bootstrap: true, - GatewayAddr: build.DefaultGatewayAddress, + GatewayAddr: "", // default determined by network UsedUTXOExpiry: 24 * time.Hour, SlabBufferCompletionThreshold: 1 << 12, }, @@ -143,7 +146,7 @@ var ( MigratorParallelSlabsPerWorker: 1, }, S3: config.S3{ - Address: build.DefaultS3Address, + Address: "", // default determined by network Enabled: true, DisableAuth: false, KeypairsV4: nil, @@ -240,6 +243,7 @@ func main() { flag.StringVar(&cfg.Directory, "dir", cfg.Directory, "Directory for storing node state") flag.BoolVar(&disableStdin, "env", false, "disable stdin prompts for environment variables (default false)") flag.BoolVar(&cfg.AutoOpenWebUI, "openui", cfg.AutoOpenWebUI, "automatically open the web UI on startup") + flag.StringVar(&cfg.Network, "network", cfg.Network, "Network to connect to (mainnet|zen|anagami)") // logger flag.StringVar(&cfg.Log.Level, "log.level", cfg.Log.Level, "Global logger level (debug|info|warn|error). Defaults to 'info' (overrides with RENTERD_LOG_LEVEL)") @@ -311,21 +315,6 @@ func main() { flag.Parse() - // NOTE: update the usage header when adding new commands - if flag.Arg(0) == "version" { - cmdVersion() - return - } else if flag.Arg(0) == "seed" { - cmdSeed() - return - } else if flag.Arg(0) == "config" { - cmdBuildConfig() - return - } else if flag.Arg(0) != "" { - flag.Usage() - return - } - // Overwrite flags from environment if set. parseEnvVar("RENTERD_BUS_REMOTE_ADDR", &cfg.Bus.RemoteAddr) parseEnvVar("RENTERD_BUS_API_PASSWORD", &cfg.Bus.RemotePassword) @@ -372,6 +361,64 @@ func main() { parseEnvVar("RENTERD_LOG_DATABASE_IGNORE_RECORD_NOT_FOUND_ERROR", &cfg.Log.Database.IgnoreRecordNotFoundError) parseEnvVar("RENTERD_LOG_DATABASE_SLOW_THRESHOLD", &cfg.Log.Database.SlowThreshold) + // check network + var network *consensus.Network + var genesis types.Block + switch cfg.Network { + case "mainnet": + network, genesis = chain.Mainnet() + case "zen": + network, genesis = chain.TestnetZen() + default: + log.Fatalf("unknown network '%s'", cfg.Network) + } + + // enforce network-specific defaults unless manually specified by user + if cfg.HTTP.Address == "" { + switch cfg.Network { + case "zen": + cfg.HTTP.Address = "localhost:9880" + default: + // mainnet and unknown networks + cfg.HTTP.Address = "localhost:9980" + } + } + + if cfg.Bus.GatewayAddr == "" { + switch cfg.Network { + case "zen": + cfg.HTTP.Address = ":9881" + default: + // mainnet and unknown networks + cfg.HTTP.Address = ":9981" + } + } + + if cfg.S3.Address == "" { + switch cfg.Network { + case "zen": + cfg.HTTP.Address = "localhost:7070" + default: + // mainnet and unknown networks + cfg.HTTP.Address = "localhost:8080" + } + } + + // NOTE: update the usage header when adding new commands + if flag.Arg(0) == "version" { + cmdVersion(network.Name) + return + } else if flag.Arg(0) == "seed" { + cmdSeed() + return + } else if flag.Arg(0) == "config" { + cmdBuildConfig() + return + } else if flag.Arg(0) != "" { + flag.Usage() + return + } + // parse remotes var workerRemotePassStr string var workerRemoteAddrsStr string @@ -444,7 +491,7 @@ func main() { } defer closeFn(context.Background()) - logger.Info("renterd", zap.String("version", build.Version()), zap.String("network", build.NetworkName()), zap.String("commit", build.Commit()), zap.Time("buildDate", build.BuildTime())) + logger.Info("renterd", zap.String("version", build.Version()), zap.String("network", network.Name), zap.String("commit", build.Commit()), zap.Time("buildDate", build.BuildTime())) if runtime.GOARCH == "amd64" && !cpu.X86.HasAVX2 { logger.Warn("renterd is running on a system without AVX2 support, performance may be degraded") } @@ -453,7 +500,6 @@ func main() { cfg.Log.Database.Level = cfg.Log.Level } - network, genesis := build.Network() busCfg := node.BusConfig{ Bus: cfg.Bus, Database: cfg.Database, diff --git a/config/config.go b/config/config.go index 2ada6f8d6..3951ea1d8 100644 --- a/config/config.go +++ b/config/config.go @@ -11,6 +11,7 @@ type ( Seed string `yaml:"seed,omitempty"` Directory string `yaml:"directory,omitempty"` AutoOpenWebUI bool `yaml:"autoOpenWebUI,omitempty"` + Network string `yaml:"network,omitempty"` ShutdownTimeout time.Duration `yaml:"shutdownTimeout,omitempty"` diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index 66e1c4867..34f70ff5a 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -16,7 +16,6 @@ import ( "go.sia.tech/hostd/host/settings/pin" "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" - "go.sia.tech/renterd/build" "go.sia.tech/renterd/webhooks" "go.uber.org/zap" ) @@ -123,9 +122,9 @@ func newTestStore() *mockStore { } // add default price pin - and gouging settings - b, _ := json.Marshal(build.DefaultPricePinSettings) + b, _ := json.Marshal(api.DefaultPricePinSettings) s.settings[api.SettingPricePinning] = string(b) - b, _ = json.Marshal(build.DefaultGougingSettings) + b, _ = json.Marshal(api.DefaultGougingSettings) s.settings[api.SettingGouging] = string(b) // add default autopilot @@ -218,7 +217,7 @@ func TestPinManager(t *testing.T) { } // enable price pinning - pps := build.DefaultPricePinSettings + pps := api.DefaultPricePinSettings pps.Enabled = true pps.Currency = "usd" pps.Threshold = 0.5 diff --git a/internal/chain/chain.go b/internal/chain/chain.go index 6b285c9c4..140fd5b7d 100644 --- a/internal/chain/chain.go +++ b/internal/chain/chain.go @@ -35,6 +35,10 @@ func TestnetZen() (*consensus.Network, types.Block) { return chain.TestnetZen() } +func Mainnet() (*consensus.Network, types.Block) { + return chain.Mainnet() +} + func NewDBStore(db chain.DB, n *consensus.Network, genesisBlock types.Block) (_ *chain.DBStore, _ consensus.State, err error) { return chain.NewDBStore(db, n, genesisBlock) } diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 9d5792cae..7db3657a5 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -20,7 +20,6 @@ import ( "go.sia.tech/jape" "go.sia.tech/renterd/api" "go.sia.tech/renterd/autopilot" - "go.sia.tech/renterd/build" "go.sia.tech/renterd/bus" "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/chain" @@ -444,7 +443,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { })) tt.OK(busClient.UpdateSetting(ctx, api.SettingUploadPacking, api.UploadPackingSettings{ Enabled: enableUploadPacking, - SlabBufferMaxSizeSoft: build.DefaultUploadPackingSettings.SlabBufferMaxSizeSoft, + SlabBufferMaxSizeSoft: api.DefaultUploadPackingSettings.SlabBufferMaxSizeSoft, })) // Fund the bus. diff --git a/worker/upload.go b/worker/upload.go index 4e94bedf4..3522584a1 100644 --- a/worker/upload.go +++ b/worker/upload.go @@ -155,9 +155,9 @@ func (w *worker) initUploadManager(maxMemory, maxOverdrive uint64, overdriveTime w.uploadManager = newUploadManager(w.shutdownCtx, w, mm, w.bus, w.bus, w.bus, maxOverdrive, overdriveTimeout, w.contractLockingDuration, logger) } -func (w *worker) upload(ctx context.Context, bucket, path string, r io.Reader, contracts []api.ContractMetadata, opts ...UploadOption) (_ string, err error) { +func (w *worker) upload(ctx context.Context, bucket, path string, rs api.RedundancySettings, r io.Reader, contracts []api.ContractMetadata, opts ...UploadOption) (_ string, err error) { // apply the options - up := defaultParameters(bucket, path) + up := defaultParameters(bucket, path, rs) for _, opt := range opts { opt(&up) } diff --git a/worker/upload_params.go b/worker/upload_params.go index ae8baa8d0..109488bb9 100644 --- a/worker/upload_params.go +++ b/worker/upload_params.go @@ -2,7 +2,6 @@ package worker import ( "go.sia.tech/renterd/api" - "go.sia.tech/renterd/build" "go.sia.tech/renterd/object" ) @@ -26,7 +25,7 @@ type uploadParameters struct { metadata api.ObjectUserMetadata } -func defaultParameters(bucket, path string) uploadParameters { +func defaultParameters(bucket, path string, rs api.RedundancySettings) uploadParameters { return uploadParameters{ bucket: bucket, path: path, @@ -34,7 +33,7 @@ func defaultParameters(bucket, path string) uploadParameters { ec: object.GenerateEncryptionKey(), // random key encryptionOffset: 0, // from the beginning - rs: build.DefaultRedundancySettings, + rs: rs, } } @@ -82,12 +81,6 @@ func WithPartNumber(partNumber int) UploadOption { } } -func WithRedundancySettings(rs api.RedundancySettings) UploadOption { - return func(up *uploadParameters) { - up.rs = rs - } -} - func WithUploadID(uploadID string) UploadOption { return func(up *uploadParameters) { up.uploadID = uploadID diff --git a/worker/upload_test.go b/worker/upload_test.go index 320c71736..5bad0941a 100644 --- a/worker/upload_test.go +++ b/worker/upload_test.go @@ -222,7 +222,7 @@ func TestUploadPackedSlab(t *testing.T) { uploadBytes := func(n int) { t.Helper() params.path = fmt.Sprintf("%s_%d", t.Name(), c) - _, err := w.upload(context.Background(), params.bucket, params.path, bytes.NewReader(frand.Bytes(n)), w.Contracts(), opts...) + _, err := w.upload(context.Background(), params.bucket, params.path, testRedundancySettings, bytes.NewReader(frand.Bytes(n)), w.Contracts(), opts...) if err != nil { t.Fatal(err) } @@ -599,7 +599,7 @@ func TestUploadRegression(t *testing.T) { // upload data ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - _, err := w.upload(ctx, params.bucket, params.path, bytes.NewReader(data), w.Contracts(), testOpts()...) + _, err := w.upload(ctx, params.bucket, params.path, testRedundancySettings, bytes.NewReader(data), w.Contracts(), testOpts()...) if !errors.Is(err, errUploadInterrupted) { t.Fatal(err) } @@ -608,7 +608,7 @@ func TestUploadRegression(t *testing.T) { unblock() // upload data - _, err = w.upload(context.Background(), params.bucket, params.path, bytes.NewReader(data), w.Contracts(), testOpts()...) + _, err = w.upload(context.Background(), params.bucket, params.path, testRedundancySettings, bytes.NewReader(data), w.Contracts(), testOpts()...) if err != nil { t.Fatal(err) } @@ -676,6 +676,5 @@ func testParameters(path string) uploadParameters { func testOpts() []UploadOption { return []UploadOption{ WithContractSet(testContractSet), - WithRedundancySettings(testRedundancySettings), } } diff --git a/worker/worker.go b/worker/worker.go index a6bfefbd4..fb4f8c49d 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -1268,7 +1268,6 @@ func (w *worker) stateHandlerGET(jc jape.Context) { ID: w.id, StartTime: api.TimeRFC3339(w.startTime), BuildState: api.BuildState{ - Network: build.NetworkName(), Version: build.Version(), Commit: build.Commit(), OS: runtime.GOOS, @@ -1658,12 +1657,11 @@ func (w *worker) UploadObject(ctx context.Context, r io.Reader, bucket, path str } // upload - eTag, err := w.upload(ctx, bucket, path, r, contracts, + eTag, err := w.upload(ctx, bucket, path, up.RedundancySettings, r, contracts, WithBlockHeight(up.CurrentHeight), WithContractSet(up.ContractSet), WithMimeType(opts.MimeType), WithPacking(up.UploadPacking), - WithRedundancySettings(up.RedundancySettings), WithObjectUserMetadata(opts.Metadata), ) if err != nil { @@ -1699,7 +1697,6 @@ func (w *worker) UploadMultipartUploadPart(ctx context.Context, r io.Reader, buc WithBlockHeight(up.CurrentHeight), WithContractSet(up.ContractSet), WithPacking(up.UploadPacking), - WithRedundancySettings(up.RedundancySettings), WithCustomKey(upload.Key), WithPartNumber(partNumber), WithUploadID(uploadID), @@ -1721,7 +1718,7 @@ func (w *worker) UploadMultipartUploadPart(ctx context.Context, r io.Reader, buc } // upload - eTag, err := w.upload(ctx, bucket, path, r, contracts, uploadOpts...) + eTag, err := w.upload(ctx, bucket, path, up.RedundancySettings, r, contracts, uploadOpts...) if err != nil { w.logger.With(zap.Error(err)).With("path", path).With("bucket", bucket).Error("failed to upload object") if !errors.Is(err, ErrShuttingDown) && !errors.Is(err, errUploadInterrupted) && !errors.Is(err, context.Canceled) { From e3deedd3e17545729fd83e1faa18c7c5b402c075 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 8 Aug 2024 14:18:56 +0200 Subject: [PATCH 02/12] publish.yml: remove network from matrix --- .github/workflows/publish.yml | 79 +++++++---------------------------- cmd/renterd/main.go | 4 +- docker/Dockerfile | 9 ++-- 3 files changed, 22 insertions(+), 70 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3abecd311..8e38c4e36 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,9 +21,6 @@ env: jobs: docker: runs-on: ubuntu-latest - strategy: - matrix: - network: ["mainnet" , "zen"] permissions: packages: write steps: @@ -37,13 +34,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Prepare environment variables run: | - if [[ "${{ matrix.network }}" == "zen" ]]; then - echo "BUILD_TAGS=testnet netgo" >> $GITHUB_ENV - echo "DOCKER_METADATA_SUFFIX=-zen" >> $GITHUB_ENV - else - echo "BUILD_TAGS=netgo" >> $GITHUB_ENV - echo "DOCKER_METADATA_SUFFIX=" >> $GITHUB_ENV - fi + echo "BUILD_TAGS=netgo" >> $GITHUB_ENV + echo "DOCKER_METADATA_SUFFIX=" >> $GITHUB_ENV - uses: docker/metadata-action@v4 name: Generate tags id: meta @@ -68,7 +60,6 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - network: ["mainnet" , "zen"] arch: [ amd64, arm64 ] steps: - uses: actions/checkout@v4 @@ -85,13 +76,7 @@ jobs: fi - name: Set build tag environment variable run: | - if [[ "${{ matrix.network }}" == "zen" ]]; then - echo "BUILD_TAGS=testnet netgo" >> $GITHUB_ENV - echo "ZIP_OUTPUT_SUFFIX=_zen" >> $GITHUB_ENV - else - echo "BUILD_TAGS=netgo" >> $GITHUB_ENV - echo "ZIP_OUTPUT_SUFFIX=" >> $GITHUB_ENV - fi + echo "BUILD_TAGS=netgo" >> $GITHUB_ENV - name: Build ${{ matrix.arch }} env: CGO_ENABLED: 1 @@ -99,7 +84,7 @@ jobs: GOARCH: ${{ matrix.arch }} run: | mkdir -p release - ZIP_OUTPUT=release/renterd${{ env.ZIP_OUTPUT_SUFFIX }}_${GOOS}_${GOARCH}.zip + ZIP_OUTPUT=release/renterd_${GOOS}_${GOARCH}.zip go build -tags="$BUILD_TAGS" -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/renterd cp README.md LICENSE bin/ zip -qj $ZIP_OUTPUT bin/* @@ -108,19 +93,11 @@ jobs: id: get_release_asset env: ARCH: ${{ matrix.arch }} - NETWORK: ${{ matrix.network }} with: result-encoding: string script: | - const arch = process.env.ARCH, - network = process.env.NETWORK; - - switch (network) { - case 'mainnet': - return `renterd_linux_${arch}`; - default: - return `renterd_${network}_linux_${arch}`; - } + const arch = process.env.ARCH; + return `renterd_linux_${arch}`; - uses: actions/upload-artifact@v4 with: name: ${{ steps.get_release_asset.outputs.result }} @@ -129,7 +106,6 @@ jobs: runs-on: macos-latest strategy: matrix: - network: ["mainnet" , "zen"] arch: [ amd64, arm64 ] steps: - uses: actions/checkout@v4 @@ -171,13 +147,8 @@ jobs: go generate ./... - name: Set build tag environment variable run: | - if [[ "${{ matrix.network }}" == "zen" ]]; then - echo "BUILD_TAGS=testnet netgo" >> $GITHUB_ENV - echo "ZIP_OUTPUT_SUFFIX=_zen" >> $GITHUB_ENV - else - echo "BUILD_TAGS=netgo" >> $GITHUB_ENV - echo "ZIP_OUTPUT_SUFFIX=" >> $GITHUB_ENV - fi + echo "BUILD_TAGS=netgo" >> $GITHUB_ENV + echo "ZIP_OUTPUT_SUFFIX=" >> $GITHUB_ENV - name: Build ${{ matrix.arch }} env: APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} @@ -203,19 +174,11 @@ jobs: id: get_release_asset env: ARCH: ${{ matrix.arch }} - NETWORK: ${{ matrix.network }} with: result-encoding: string script: | - const arch = process.env.ARCH, - network = process.env.NETWORK; - - switch (network) { - case 'mainnet': - return `renterd_darwin_${arch}`; - default: - return `renterd_${network}_darwin_${arch}`; - } + const arch = process.env.ARCH; + return `renterd_darwin_${arch}`; - uses: actions/upload-artifact@v4 with: name: ${{ steps.get_release_asset.outputs.result }} @@ -224,7 +187,6 @@ jobs: runs-on: windows-latest strategy: matrix: - network: ["mainnet" , "zen"] arch: [ amd64 ] steps: - uses: actions/checkout@v4 @@ -233,13 +195,8 @@ jobs: go-version: 'stable' - name: Set build tag environment variable run: | - if ( "${{ matrix.network }}" -eq "zen" ) { - "BUILD_TAGS=testnet netgo" >> $env:GITHUB_ENV - "ZIP_OUTPUT_SUFFIX=_zen" >> $env:GITHUB_ENV - } else { - "BUILD_TAGS=netgo" >> $env:GITHUB_ENV - "ZIP_OUTPUT_SUFFIX=" >> $env:GITHUB_ENV - } + "BUILD_TAGS=netgo" >> $env:GITHUB_ENV + "ZIP_OUTPUT_SUFFIX=" >> $env:GITHUB_ENV - name: Setup shell: bash run: | @@ -263,19 +220,11 @@ jobs: id: get_release_asset env: ARCH: ${{ matrix.arch }} - NETWORK: ${{ matrix.network }} with: result-encoding: string script: | - const arch = process.env.ARCH, - network = process.env.NETWORK; - - switch (network) { - case 'mainnet': - return `renterd_windows_${arch}`; - default: - return `renterd_${network}_windows_${arch}`; - } + const arch = process.env.ARCH; + return `renterd_windows_${arch}`; - uses: actions/upload-artifact@v4 with: name: ${{ steps.get_release_asset.outputs.result }} diff --git a/cmd/renterd/main.go b/cmd/renterd/main.go index 7b140c504..b8b7e5f52 100644 --- a/cmd/renterd/main.go +++ b/cmd/renterd/main.go @@ -243,7 +243,7 @@ func main() { flag.StringVar(&cfg.Directory, "dir", cfg.Directory, "Directory for storing node state") flag.BoolVar(&disableStdin, "env", false, "disable stdin prompts for environment variables (default false)") flag.BoolVar(&cfg.AutoOpenWebUI, "openui", cfg.AutoOpenWebUI, "automatically open the web UI on startup") - flag.StringVar(&cfg.Network, "network", cfg.Network, "Network to connect to (mainnet|zen|anagami)") + flag.StringVar(&cfg.Network, "network", cfg.Network, "Network to connect to (mainnet|zen|anagami). Defaults to 'mainnet' (overrides with RENTERD_NETWORK)") // logger flag.StringVar(&cfg.Log.Level, "log.level", cfg.Log.Level, "Global logger level (debug|info|warn|error). Defaults to 'info' (overrides with RENTERD_LOG_LEVEL)") @@ -316,6 +316,8 @@ func main() { flag.Parse() // Overwrite flags from environment if set. + parseEnvVar("RENTERD_NETWORK", &cfg.Network) + parseEnvVar("RENTERD_BUS_REMOTE_ADDR", &cfg.Bus.RemoteAddr) parseEnvVar("RENTERD_BUS_API_PASSWORD", &cfg.Bus.RemotePassword) parseEnvVar("RENTERD_BUS_GATEWAY_ADDR", &cfg.Bus.GatewayAddr) diff --git a/docker/Dockerfile b/docker/Dockerfile index c97cd5299..bd6e00866 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -28,10 +28,10 @@ RUN --mount=type=cache,target=/root/go/pkg/mod \ # Build image that will be used to run renterd. FROM alpine:3 LABEL maintainer="The Sia Foundation " \ - org.opencontainers.image.description.vendor="The Sia Foundation" \ - org.opencontainers.image.description="A renterd container - next-generation Sia renter" \ - org.opencontainers.image.source="https://github.com/SiaFoundation/renterd" \ - org.opencontainers.image.licenses=MIT + org.opencontainers.image.description.vendor="The Sia Foundation" \ + org.opencontainers.image.description="A renterd container - next-generation Sia renter" \ + org.opencontainers.image.source="https://github.com/SiaFoundation/renterd" \ + org.opencontainers.image.licenses=MIT # User to run renterd as. Defaults to root. ENV PUID=0 @@ -45,6 +45,7 @@ ENV BUILD_TAGS=$BUILD_TAGS ENV RENTERD_API_PASSWORD= ENV RENTERD_SEED= ENV RENTERD_CONFIG_FILE=/data/renterd.yml +ENV RENTERD_NETWORK= # Copy binary and prepare data dir. COPY --from=builder /renterd/renterd /usr/bin/renterd From 240842ecdd3cd73f1b9204efcb4e7dd2bc313bd4 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 8 Aug 2024 15:12:20 +0200 Subject: [PATCH 03/12] dockerfile: add RENTERD_NETWORK ARG --- .github/workflows/publish.yml | 9 ++++++--- docker/Dockerfile | 7 +++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e38c4e36..031d889f1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,6 +21,9 @@ env: jobs: docker: runs-on: ubuntu-latest + strategy: + matrix: + network: ["mainnet" , "zen"] permissions: packages: write steps: @@ -34,8 +37,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Prepare environment variables run: | - echo "BUILD_TAGS=netgo" >> $GITHUB_ENV - echo "DOCKER_METADATA_SUFFIX=" >> $GITHUB_ENV + echo "RENTERD_NETWORK=${{ matrix.network }}" >> $GITHUB_ENV + echo "DOCKER_METADATA_SUFFIX=-${{ matrix.network }}" >> $GITHUB_ENV - uses: docker/metadata-action@v4 name: Generate tags id: meta @@ -51,7 +54,7 @@ jobs: with: context: . build-args: | - BUILD_TAGS=${{ env.BUILD_TAGS }} + RENTERD_NETWORK=${{ env.RENTERD_NETWORK }} file: ./docker/Dockerfile platforms: linux/amd64,linux/arm64 push: true diff --git a/docker/Dockerfile b/docker/Dockerfile index bd6e00866..5d20721c6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,15 +37,14 @@ LABEL maintainer="The Sia Foundation " \ ENV PUID=0 ENV PGID=0 -# Entrypoint env args -ARG BUILD_TAGS -ENV BUILD_TAGS=$BUILD_TAGS +# Default network of container +ARG RENTERD_NETWORK= # Renterd env args ENV RENTERD_API_PASSWORD= ENV RENTERD_SEED= ENV RENTERD_CONFIG_FILE=/data/renterd.yml -ENV RENTERD_NETWORK= +ENV RENTERD_NETWORK=${RENTERD_NETWORK} # Copy binary and prepare data dir. COPY --from=builder /renterd/renterd /usr/bin/renterd From c5494ace595a1fd11ef8b487d5ba573a8d60ed3f Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 8 Aug 2024 15:27:34 +0200 Subject: [PATCH 04/12] docker: update entrypoint.sh --- docker/entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index da8b4d8ce..c54e982ea 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh -if [[ "$BUILD_TAGS" == *'testnet'* ]]; then - exec renterd -env -http=':9880' -s3.address=':7070' "$@" -else +if [[ "$RENTERD_NETWORK" == *'mainnet'* ]]; then exec renterd -env -http=':9980' -s3.address=':8080' "$@" +else + exec renterd -env -http=':9880' -s3.address=':7070' "$@" fi From d7a0f83bf734bb8a264efff534117335fe56ff2f Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 8 Aug 2024 15:51:06 +0200 Subject: [PATCH 05/12] docker: default RENTERD_NETWORK to 'mainnet' --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5d20721c6..bca2ef654 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,7 +38,7 @@ ENV PUID=0 ENV PGID=0 # Default network of container -ARG RENTERD_NETWORK= +ARG RENTERD_NETWORK='mainnet' # Renterd env args ENV RENTERD_API_PASSWORD= From 915e82ce01261d0303066fb222e71ff1021d71cc Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 8 Aug 2024 15:57:58 +0200 Subject: [PATCH 06/12] publish.yml: remove DOCKER_METADATA_SUFFIX --- .github/workflows/publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 031d889f1..9deff678c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,14 +38,13 @@ jobs: - name: Prepare environment variables run: | echo "RENTERD_NETWORK=${{ matrix.network }}" >> $GITHUB_ENV - echo "DOCKER_METADATA_SUFFIX=-${{ matrix.network }}" >> $GITHUB_ENV - uses: docker/metadata-action@v4 name: Generate tags id: meta with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} flavor: | - suffix=${{ env.DOCKER_METADATA_SUFFIX }},onlatest=true + suffix=-${{ env.RENTERD_NETWORK }},onlatest=true tags: | type=ref,event=branch type=sha,prefix= From 049ddb5552ebe1c554f6176312d37f1d1b4a0839 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 8 Aug 2024 17:50:22 +0200 Subject: [PATCH 07/12] remove testnet container --- .github/workflows/publish.yml | 10 +--------- docker/Dockerfile | 11 +++-------- docker/entrypoint.sh | 7 ------- 3 files changed, 4 insertions(+), 24 deletions(-) delete mode 100644 docker/entrypoint.sh diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9deff678c..6ce2bb28a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,9 +21,6 @@ env: jobs: docker: runs-on: ubuntu-latest - strategy: - matrix: - network: ["mainnet" , "zen"] permissions: packages: write steps: @@ -35,16 +32,13 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Prepare environment variables - run: | - echo "RENTERD_NETWORK=${{ matrix.network }}" >> $GITHUB_ENV - uses: docker/metadata-action@v4 name: Generate tags id: meta with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} flavor: | - suffix=-${{ env.RENTERD_NETWORK }},onlatest=true + onlatest=true tags: | type=ref,event=branch type=sha,prefix= @@ -52,8 +46,6 @@ jobs: - uses: docker/build-push-action@v4 with: context: . - build-args: | - RENTERD_NETWORK=${{ env.RENTERD_NETWORK }} file: ./docker/Dockerfile platforms: linux/amd64,linux/arm64 push: true diff --git a/docker/Dockerfile b/docker/Dockerfile index bca2ef654..5ecc5f99b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,7 +26,7 @@ RUN --mount=type=cache,target=/root/go/pkg/mod \ CGO_ENABLED=1 go build -ldflags='-s -w -linkmode external -extldflags "-static"' -tags="${BUILD_TAGS}" ./cmd/renterd # Build image that will be used to run renterd. -FROM alpine:3 +FROM scratch LABEL maintainer="The Sia Foundation " \ org.opencontainers.image.description.vendor="The Sia Foundation" \ org.opencontainers.image.description="A renterd container - next-generation Sia renter" \ @@ -37,14 +37,11 @@ LABEL maintainer="The Sia Foundation " \ ENV PUID=0 ENV PGID=0 -# Default network of container -ARG RENTERD_NETWORK='mainnet' - # Renterd env args ENV RENTERD_API_PASSWORD= ENV RENTERD_SEED= ENV RENTERD_CONFIG_FILE=/data/renterd.yml -ENV RENTERD_NETWORK=${RENTERD_NETWORK} +ENV RENTERD_NETWORK='mainnet' # Copy binary and prepare data dir. COPY --from=builder /renterd/renterd /usr/bin/renterd @@ -53,6 +50,4 @@ VOLUME [ "/data" ] USER ${PUID}:${PGID} # Copy the script and set it as the entrypoint. -COPY docker/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh", "-dir", "./data"] +ENTRYPOINT ["renterd", "-env", "-http", ":9980", "-s3.address", ":8080", "-dir", "./data"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100644 index c54e982ea..000000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -if [[ "$RENTERD_NETWORK" == *'mainnet'* ]]; then - exec renterd -env -http=':9980' -s3.address=':8080' "$@" -else - exec renterd -env -http=':9880' -s3.address=':7070' "$@" -fi From b545ea6bb3b5f1e8bc4f08330a18bef859577f0d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 8 Aug 2024 18:10:12 +0200 Subject: [PATCH 08/12] cmd: add anagami --- cmd/renterd/main.go | 39 +++++---------------------------------- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 8 insertions(+), 37 deletions(-) diff --git a/cmd/renterd/main.go b/cmd/renterd/main.go index b8b7e5f52..d186ab0ba 100644 --- a/cmd/renterd/main.go +++ b/cmd/renterd/main.go @@ -81,7 +81,7 @@ var ( AutoOpenWebUI: true, Network: "mainnet", HTTP: config.HTTP{ - Address: "", // default determined by network + Address: "localhost:9980", Password: os.Getenv("RENTERD_API_PASSWORD"), }, ShutdownTimeout: 5 * time.Minute, @@ -114,7 +114,7 @@ var ( Bus: config.Bus{ AnnouncementMaxAgeHours: 24 * 7 * 52, // 1 year Bootstrap: true, - GatewayAddr: "", // default determined by network + GatewayAddr: ":9981", UsedUTXOExpiry: 24 * time.Hour, SlabBufferCompletionThreshold: 1 << 12, }, @@ -146,7 +146,7 @@ var ( MigratorParallelSlabsPerWorker: 1, }, S3: config.S3{ - Address: "", // default determined by network + Address: "localhost:8080", Enabled: true, DisableAuth: false, KeypairsV4: nil, @@ -367,6 +367,8 @@ func main() { var network *consensus.Network var genesis types.Block switch cfg.Network { + case "anagami": + network, genesis = chain.TestnetAnagami() case "mainnet": network, genesis = chain.Mainnet() case "zen": @@ -375,37 +377,6 @@ func main() { log.Fatalf("unknown network '%s'", cfg.Network) } - // enforce network-specific defaults unless manually specified by user - if cfg.HTTP.Address == "" { - switch cfg.Network { - case "zen": - cfg.HTTP.Address = "localhost:9880" - default: - // mainnet and unknown networks - cfg.HTTP.Address = "localhost:9980" - } - } - - if cfg.Bus.GatewayAddr == "" { - switch cfg.Network { - case "zen": - cfg.HTTP.Address = ":9881" - default: - // mainnet and unknown networks - cfg.HTTP.Address = ":9981" - } - } - - if cfg.S3.Address == "" { - switch cfg.Network { - case "zen": - cfg.HTTP.Address = "localhost:7070" - default: - // mainnet and unknown networks - cfg.HTTP.Address = "localhost:8080" - } - } - // NOTE: update the usage header when adding new commands if flag.Arg(0) == "version" { cmdVersion(network.Name) diff --git a/go.mod b/go.mod index d6d3b7e1f..c063aab1c 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/montanaflynn/stats v0.7.1 github.com/shopspring/decimal v1.4.0 go.sia.tech/core v0.4.2 - go.sia.tech/coreutils v0.2.2 + go.sia.tech/coreutils v0.2.3-0.20240808145455-bf9cb549c4f2 go.sia.tech/gofakes3 v0.0.4 go.sia.tech/hostd v1.1.3-0.20240807214810-c2d8ed84dc45 go.sia.tech/jape v0.12.0 diff --git a/go.sum b/go.sum index 212cadff0..5ee41984a 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.sia.tech/core v0.4.2 h1:5VCRuRJAOy0cWwG32IGB0BXQAviXgKRfNXOiU0zSViM= go.sia.tech/core v0.4.2/go.mod h1:cGfGNcyAq1k4oIOsrNpJV/Z/p+20/IMS6vIaofE8nr8= -go.sia.tech/coreutils v0.2.2 h1:WWbuk4lEd5SVB/LoBsPG6+heZN2olZT4yOkRZd3pvpg= -go.sia.tech/coreutils v0.2.2/go.mod h1:0D0NLh0c0pBUNKPoO/rDtyyRapB5j4/gfATNyQO67Rs= +go.sia.tech/coreutils v0.2.3-0.20240808145455-bf9cb549c4f2 h1:9z10MuKi77RtaBJvCdKIwbMmoe+HMTE3uAAdy75yD9g= +go.sia.tech/coreutils v0.2.3-0.20240808145455-bf9cb549c4f2/go.mod h1:0D0NLh0c0pBUNKPoO/rDtyyRapB5j4/gfATNyQO67Rs= go.sia.tech/gofakes3 v0.0.4 h1:Kvo8j5cVdJRBXvV1KBJ69bocY23twG8ao/HCdwuPMeI= go.sia.tech/gofakes3 v0.0.4/go.mod h1:6hh4lETCMbyFFNWp3FRE838geY6vh1Aeas7LtYDpQdc= go.sia.tech/hostd v1.1.3-0.20240807214810-c2d8ed84dc45 h1:yq8n3leZWAeEwbAa3sbqe5mS5LgG5IH23aM8tefSuUo= From f7efc888f4950aa7ae912b0f1e44c5d1a9ca0b60 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 9 Aug 2024 09:26:47 +0200 Subject: [PATCH 09/12] publish.yml: address comments --- .github/workflows/publish.yml | 61 ++++++----------------------------- 1 file changed, 9 insertions(+), 52 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6ce2bb28a..202276f81 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -37,8 +37,6 @@ jobs: id: meta with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - flavor: | - onlatest=true tags: | type=ref,event=branch type=sha,prefix= @@ -68,9 +66,6 @@ jobs: sudo apt install -y gcc-aarch64-linux-gnu echo "CC=aarch64-linux-gnu-gcc" >> $GITHUB_ENV fi - - name: Set build tag environment variable - run: | - echo "BUILD_TAGS=netgo" >> $GITHUB_ENV - name: Build ${{ matrix.arch }} env: CGO_ENABLED: 1 @@ -79,23 +74,13 @@ jobs: run: | mkdir -p release ZIP_OUTPUT=release/renterd_${GOOS}_${GOARCH}.zip - go build -tags="$BUILD_TAGS" -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/renterd + go build -tags="netgo" -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/renterd cp README.md LICENSE bin/ zip -qj $ZIP_OUTPUT bin/* - - name: Get Release Asset - uses: actions/github-script@v7 - id: get_release_asset - env: - ARCH: ${{ matrix.arch }} - with: - result-encoding: string - script: | - const arch = process.env.ARCH; - return `renterd_linux_${arch}`; - uses: actions/upload-artifact@v4 with: - name: ${{ steps.get_release_asset.outputs.result }} - path: release/ + name: renterd_linux_${{ matrix.arch }} + path: release/* build-mac: runs-on: macos-latest strategy: @@ -139,10 +124,6 @@ jobs: # generate go generate ./... - - name: Set build tag environment variable - run: | - echo "BUILD_TAGS=netgo" >> $GITHUB_ENV - echo "ZIP_OUTPUT_SUFFIX=" >> $GITHUB_ENV - name: Build ${{ matrix.arch }} env: APPLE_CERT_ID: ${{ secrets.APPLE_CERT_ID }} @@ -158,25 +139,15 @@ jobs: run: | mkdir -p release ZIP_OUTPUT=release/renterd${{ env.ZIP_OUTPUT_SUFFIX }}_${GOOS}_${GOARCH}.zip - go build -tags="$BUILD_TAGS" -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/renterd + go build -tags="netgo" -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/renterd cp README.md LICENSE bin/ /usr/bin/codesign --deep -f -v --timestamp -o runtime,library -s $APPLE_CERT_ID bin/renterd ditto -ck bin $ZIP_OUTPUT xcrun notarytool submit -k ~/private_keys/AuthKey_$APPLE_API_KEY.p8 -d $APPLE_API_KEY -i $APPLE_API_ISSUER --wait --timeout 10m $ZIP_OUTPUT - - name: Get Release Asset - uses: actions/github-script@v7 - id: get_release_asset - env: - ARCH: ${{ matrix.arch }} - with: - result-encoding: string - script: | - const arch = process.env.ARCH; - return `renterd_darwin_${arch}`; - uses: actions/upload-artifact@v4 with: - name: ${{ steps.get_release_asset.outputs.result }} - path: release/ + name: renterd_darwin_${{ matrix.arch }} + path: release/* build-windows: runs-on: windows-latest strategy: @@ -187,10 +158,6 @@ jobs: - uses: actions/setup-go@v5 with: go-version: 'stable' - - name: Set build tag environment variable - run: | - "BUILD_TAGS=netgo" >> $env:GITHUB_ENV - "ZIP_OUTPUT_SUFFIX=" >> $env:GITHUB_ENV - name: Setup shell: bash run: | @@ -205,24 +172,14 @@ jobs: run: | mkdir -p release ZIP_OUTPUT=release/renterd${{ env.ZIP_OUTPUT_SUFFIX }}_${GOOS}_${GOARCH}.zip - go build -tags="$BUILD_TAGS" -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/renterd + go build -tags="netgo" -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/renterd azuresigntool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v bin/renterd.exe cp README.md LICENSE bin/ 7z a $ZIP_OUTPUT bin/* - - name: Get Release Asset - uses: actions/github-script@v7 - id: get_release_asset - env: - ARCH: ${{ matrix.arch }} - with: - result-encoding: string - script: | - const arch = process.env.ARCH; - return `renterd_windows_${arch}`; - uses: actions/upload-artifact@v4 with: - name: ${{ steps.get_release_asset.outputs.result }} - path: release/ + name: renterd_windows_${{ matrix.arch }} + path: release/* combine-release-assets: runs-on: ubuntu-latest From 897d45bc7620c79d9180ff3de0fee93191a6749e Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 9 Aug 2024 09:37:24 +0200 Subject: [PATCH 10/12] remove netgo --- .github/workflows/publish.yml | 6 +++--- docker/Dockerfile | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 202276f81..e40df6930 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -74,7 +74,7 @@ jobs: run: | mkdir -p release ZIP_OUTPUT=release/renterd_${GOOS}_${GOARCH}.zip - go build -tags="netgo" -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/renterd + go build -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/renterd cp README.md LICENSE bin/ zip -qj $ZIP_OUTPUT bin/* - uses: actions/upload-artifact@v4 @@ -139,7 +139,7 @@ jobs: run: | mkdir -p release ZIP_OUTPUT=release/renterd${{ env.ZIP_OUTPUT_SUFFIX }}_${GOOS}_${GOARCH}.zip - go build -tags="netgo" -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/renterd + go build -trimpath -o bin/ -a -ldflags '-s -w' ./cmd/renterd cp README.md LICENSE bin/ /usr/bin/codesign --deep -f -v --timestamp -o runtime,library -s $APPLE_CERT_ID bin/renterd ditto -ck bin $ZIP_OUTPUT @@ -172,7 +172,7 @@ jobs: run: | mkdir -p release ZIP_OUTPUT=release/renterd${{ env.ZIP_OUTPUT_SUFFIX }}_${GOOS}_${GOARCH}.zip - go build -tags="netgo" -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/renterd + go build -trimpath -o bin/ -a -ldflags '-s -w -linkmode external -extldflags "-static"' ./cmd/renterd azuresigntool sign -kvu "${{ secrets.AZURE_KEY_VAULT_URI }}" -kvi "${{ secrets.AZURE_CLIENT_ID }}" -kvt "${{ secrets.AZURE_TENANT_ID }}" -kvs "${{ secrets.AZURE_CLIENT_SECRET }}" -kvc ${{ secrets.AZURE_CERT_NAME }} -tr http://timestamp.digicert.com -v bin/renterd.exe cp README.md LICENSE bin/ 7z a $ZIP_OUTPUT bin/* diff --git a/docker/Dockerfile b/docker/Dockerfile index 5ecc5f99b..fb8fccf0e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,8 +2,7 @@ FROM golang:1.22 AS builder # Define arguments for build tags and to skip running go generate. -ARG BUILD_TAGS='netgo' \ - BUILD_RUN_GO_GENERATE='true' +ARG BUILD_RUN_GO_GENERATE='true' # Set the working directory. WORKDIR /renterd @@ -23,7 +22,7 @@ RUN if [ "$BUILD_RUN_GO_GENERATE" = "true" ] ; then go generate ./... ; fi # Build renterd. RUN --mount=type=cache,target=/root/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=1 go build -ldflags='-s -w -linkmode external -extldflags "-static"' -tags="${BUILD_TAGS}" ./cmd/renterd + CGO_ENABLED=1 go build -ldflags='-s -w -linkmode external -extldflags "-static"' ./cmd/renterd # Build image that will be used to run renterd. FROM scratch From 186a895b95053b14e9f1bacc292bf1a23eb9897e Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 9 Aug 2024 09:47:24 +0200 Subject: [PATCH 11/12] go.mod: update coreutils --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c063aab1c..f028f32c1 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/montanaflynn/stats v0.7.1 github.com/shopspring/decimal v1.4.0 go.sia.tech/core v0.4.2 - go.sia.tech/coreutils v0.2.3-0.20240808145455-bf9cb549c4f2 + go.sia.tech/coreutils v0.2.3 go.sia.tech/gofakes3 v0.0.4 go.sia.tech/hostd v1.1.3-0.20240807214810-c2d8ed84dc45 go.sia.tech/jape v0.12.0 diff --git a/go.sum b/go.sum index 5ee41984a..7e8b2c527 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.sia.tech/core v0.4.2 h1:5VCRuRJAOy0cWwG32IGB0BXQAviXgKRfNXOiU0zSViM= go.sia.tech/core v0.4.2/go.mod h1:cGfGNcyAq1k4oIOsrNpJV/Z/p+20/IMS6vIaofE8nr8= -go.sia.tech/coreutils v0.2.3-0.20240808145455-bf9cb549c4f2 h1:9z10MuKi77RtaBJvCdKIwbMmoe+HMTE3uAAdy75yD9g= -go.sia.tech/coreutils v0.2.3-0.20240808145455-bf9cb549c4f2/go.mod h1:0D0NLh0c0pBUNKPoO/rDtyyRapB5j4/gfATNyQO67Rs= +go.sia.tech/coreutils v0.2.3 h1:wbCJI3Tti/uvDg9H7knUOPt6imNTZU+XjOuyHhtaRtQ= +go.sia.tech/coreutils v0.2.3/go.mod h1:0D0NLh0c0pBUNKPoO/rDtyyRapB5j4/gfATNyQO67Rs= go.sia.tech/gofakes3 v0.0.4 h1:Kvo8j5cVdJRBXvV1KBJ69bocY23twG8ao/HCdwuPMeI= go.sia.tech/gofakes3 v0.0.4/go.mod h1:6hh4lETCMbyFFNWp3FRE838geY6vh1Aeas7LtYDpQdc= go.sia.tech/hostd v1.1.3-0.20240807214810-c2d8ed84dc45 h1:yq8n3leZWAeEwbAa3sbqe5mS5LgG5IH23aM8tefSuUo= From 2f54bc4e31e63ddfa5345f69e7ab053e0d50b536 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Mon, 12 Aug 2024 07:52:49 +0200 Subject: [PATCH 12/12] README.md: update --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index dc51edf86..0267d7264 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ overview of all settings configurable through the CLI. | `Directory` | Directory for storing node state | `.` | `--dir` | - | `directory` | | `Seed` | Seed for the node | - | - | `RENTERD_SEED` | `seed` | | `AutoOpenWebUI` | Automatically open the web UI on startup | `true` | `--openui` | - | `autoOpenWebUI` | +| `Network` | Network to run on (mainnet/zen/anagami) | `mainnet` | `--network` | `RENTERD_NETWORK` | `network` | | `ShutdownTimeout` | Timeout for node shutdown | `5m` | `--node.shutdownTimeout` | - | `shutdownTimeout` | | `Log.Level` | Global logger level (debug\|info\|warn\|error). Defaults to 'info' | `info` | `--log.level` | `RENTERD_LOG_LEVEL` | `log.level` | | `Log.File.Enabled` | Enables logging to disk. Defaults to 'true' | `true` | `--log.file.enabled` | `RENTERD_LOG_FILE_ENABLED` | `log.file.enabled` | @@ -408,14 +409,14 @@ ghcr.io/siafoundation/renterd. version: "3.9" services: renterd: - image: ghcr.io/siafoundation/renterd:master-zen + image: ghcr.io/siafoundation/renterd:master environment: - RENTERD_SEED=put your seed here - RENTERD_API_PASSWORD=test ports: - - 9880:9880 - - 9881:9881 - - 7070:7070 + - 9980:9980 + - 9981:9981 + - 8080:8080 volumes: - ./data:/data restart: unless-stopped @@ -427,23 +428,16 @@ services: From within the root of the repo run the following command to build an image of `renterd` tagged `renterd`. -#### Mainnet - ```sh docker build -t renterd:master -f ./docker/Dockerfile . ``` -#### Testnet - -```sh -docker build --build-arg BUILD_TAGS='netgo testnet' -t renterd:master-zen -f ./docker/Dockerfile . -``` - ### Run Container Run `renterd` in the background as a container named `renterd` that exposes its API to the host system and the gateway to the world. + #### Mainnet ```bash @@ -452,10 +446,16 @@ docker run -d --name renterd -e RENTERD_API_PASSWORD="" -e RENTERD_SEE #### Testnet +To run `renterd` on testnet use the `RENTERD_NETWORK` environment variable. + ```bash -docker run -d --name renterd-testnet -e RENTERD_API_PASSWORD="" -e RENTERD_SEED="" -p 127.0.0.1:9880:9880/tcp -p :9881:9881/tcp ghcr.io/siafoundation/renterd:master-zen +docker run -d --name renterd -e RENTERD_API_PASSWORD="" -e RENTERD_NETWORK="" -e RENTERD_SEED="" -p 127.0.0.1:9980:9980/tcp -p :9981:9981/tcp ghcr.io/siafoundation/renterd:master ``` +Currently available values for `` are: +- `zen` +- `anagami` + ## Architecture `renterd` distinguishes itself from `siad` through a unique architecture