Add support for rewriting only specifc host bases (#1328)

On `gompa` rclone wouldn't work via `localhost` anymore since
`HostBucketEnabled` was set to `true`.
Problem is without this setting, access per proper domain (e.g.
`bucket.example.com`) doesn't work anymore but with it IPs and
`localhost` will be rewritten incorrectly.

The solution is to allow for configuring `HostBucketBases` which is
already supported by `gofakes3` and allows for configuring only specific
domains for which the bucket is extracted from the `host`. That way
`localhost` and `127.0.0.1` still work but if specified
`bucket.example.com` will also work.

README.md

@@ -96,6 +96,7 @@ overview of all settings configurable through the CLI.
 | `S3.Address`                         | Address for serving S3 API                           | `:9982`                          | `--s3.address`                     | `RENTERD_S3_ADDRESS`                           | `s3.address`                        |
 | `S3.DisableAuth`                     | Disables authentication for S3 API                   | `false`                           | `--s3.disableAuth`                 | `RENTERD_S3_DISABLE_AUTH`                      | `s3.disableAuth`                    |
 | `S3.Enabled`                         | Enables/disables S3 API                              | `true`                            | `--s3.enabled`                     | `RENTERD_S3_ENABLED`                           | `s3.enabled`                        |
+| `S3.HostBucketBases`       | Enables bucket rewriting in the router for the provided bases  | -                                 | `--s3.hostBucketBases`           | `RENTERD_S3_HOST_BUCKET_BASES`               | `s3.hostBucketBases`              |
 | `S3.HostBucketEnabled`               | Enables bucket rewriting in the router               | -                                 | `--s3.hostBucketEnabled`           | `RENTERD_S3_HOST_BUCKET_ENABLED`               | `s3.hostBucketEnabled`              |
 | `S3.KeypairsV4 (DEPRECATED)`                      | V4 keypairs for S3                                   | -                                 | -                                  | -            | `s3.keypairsV4`                     |
 

cmd/renterd/main.go

@@ -310,10 +310,12 @@ func main() {
 	flag.DurationVar(&cfg.ShutdownTimeout, "node.shutdownTimeout", cfg.ShutdownTimeout, "Timeout for node shutdown")
 
 	// s3
+	var hostBasesStr string
 	flag.StringVar(&cfg.S3.Address, "s3.address", cfg.S3.Address, "Address for serving S3 API (overrides with RENTERD_S3_ADDRESS)")
 	flag.BoolVar(&cfg.S3.DisableAuth, "s3.disableAuth", cfg.S3.DisableAuth, "Disables authentication for S3 API (overrides with RENTERD_S3_DISABLE_AUTH)")
 	flag.BoolVar(&cfg.S3.Enabled, "s3.enabled", cfg.S3.Enabled, "Enables/disables S3 API (requires worker.enabled to be 'true', overrides with RENTERD_S3_ENABLED)")
-	flag.BoolVar(&cfg.S3.HostBucketEnabled, "s3.hostBucketEnabled", cfg.S3.HostBucketEnabled, "Enables bucket rewriting in the router (overrides with RENTERD_S3_HOST_BUCKET_ENABLED)")
+	flag.StringVar(&hostBasesStr, "s3.hostBases", "", "Enables bucket rewriting in the router for specific hosts provided via comma-separated list (overrides with RENTERD_S3_HOST_BUCKET_BASES)")
+	flag.BoolVar(&cfg.S3.HostBucketEnabled, "s3.hostBucketEnabled", cfg.S3.HostBucketEnabled, "Enables bucket rewriting in the router for all hosts (overrides with RENTERD_S3_HOST_BUCKET_ENABLED)")
 
 	// custom usage
 	flag.Usage = func() {
@@ -376,6 +378,7 @@ func main() {
 	parseEnvVar("RENTERD_S3_ENABLED", &cfg.S3.Enabled)
 	parseEnvVar("RENTERD_S3_DISABLE_AUTH", &cfg.S3.DisableAuth)
 	parseEnvVar("RENTERD_S3_HOST_BUCKET_ENABLED", &cfg.S3.HostBucketEnabled)
+	parseEnvVar("RENTERD_S3_HOST_BUCKET_BASES", &cfg.S3.HostBucketBases)
 
 	parseEnvVar("RENTERD_LOG_LEVEL", &cfg.Log.Level)
 	parseEnvVar("RENTERD_LOG_FILE_ENABLED", &cfg.Log.File.Enabled)
@@ -389,6 +392,13 @@ func main() {
 	parseEnvVar("RENTERD_LOG_DATABASE_IGNORE_RECORD_NOT_FOUND_ERROR", &cfg.Log.Database.IgnoreRecordNotFoundError)
 	parseEnvVar("RENTERD_LOG_DATABASE_SLOW_THRESHOLD", &cfg.Log.Database.SlowThreshold)
 
+	// combine host bucket bases
+	for _, base := range strings.Split(hostBasesStr, ",") {
+		if trimmed := strings.TrimSpace(base); trimmed != "" {
+			cfg.S3.HostBucketBases = append(cfg.S3.HostBucketBases, base)
+		}
+	}
+
 	// check that the API password is set
 	if cfg.HTTP.Password == "" {
 		if disableStdin {
@@ -535,6 +545,7 @@ func main() {
 			var shutdownFn node.ShutdownFn
 			w, s3Handler, setupFn, shutdownFn, err := node.NewWorker(cfg.Worker, s3.Opts{
 				AuthDisabled:      cfg.S3.DisableAuth,
+				HostBucketBases:   cfg.S3.HostBucketBases,
 				HostBucketEnabled: cfg.S3.HostBucketEnabled,
 			}, bc, seed, logger)
 			if err != nil {

config/config.go

@@ -107,6 +107,7 @@ type (
 		Enabled           bool              `yaml:"enabled,omitempty"`
 		KeypairsV4        map[string]string `yaml:"keypairsV4,omitempty"` // deprecated. included for compatibility.
 		HostBucketEnabled bool              `yaml:"hostBucketEnabled,omitempty"`
+		HostBucketBases   []string          `yaml:"hostBucketBases,omitempty"`
 	}
 
 	// Worker contains the configuration for a worker.

go.mod

@@ -15,7 +15,7 @@ require (
 	gitlab.com/NebulousLabs/encoding v0.0.0-20200604091946-456c3dc907fe
 	go.sia.tech/core v0.2.8
 	go.sia.tech/coreutils v0.0.7
-	go.sia.tech/gofakes3 v0.0.3
+	go.sia.tech/gofakes3 v0.0.4
 	go.sia.tech/hostd v1.1.1-beta.1.0.20240618072747-b3f430b4d272
 	go.sia.tech/jape v0.11.2-0.20240306154058-9832414a5385
 	go.sia.tech/mux v1.2.0
@@ -35,7 +35,7 @@ require (
 
 require (
 	github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
-	github.com/aws/aws-sdk-go v1.54.3 // indirect
+	github.com/aws/aws-sdk-go v1.54.6 // indirect
 	github.com/cloudflare/cloudflare-go v0.97.0 // indirect
 	github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect