Plug S3 plugin changes into multi-export S3 origins

cmd/origin.go

@@ -48,6 +48,25 @@ var (
 		Short:        "Start the origin service",
 		RunE:         serveOrigin,
 		SilenceUsage: true,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			// Checking these values has to happen here and not in init() because init
+			// doesn't have access to the actual values passed on the command line.
+			if ost := viper.GetString("Origin.StorageType"); ost == "s3" {
+				if !viper.IsSet("Origin.S3Region") || !viper.IsSet("Origin.S3ServiceUrl") {
+					cmd.PrintErrln("The --region, --service-url flags or equivalent config file entries are required when the origin is launched in S3 mode.")
+					os.Exit(1)
+				}
+			} else if ost == "posix" {
+				// We specifically DON'T want the region, service-url, and url-style flags if the mode is posix
+				if viper.IsSet("Origin.S3Region") || viper.IsSet("Origin.S3ServiceUrl") || viper.IsSet("Origin.S3UrlStyle") {
+					cmd.PrintErrln("The --region, --service-url, and --url-style flags are only used when the origin is launched in S3 mode.")
+					os.Exit(1)
+				}
+			} else {
+				cmd.PrintErrln(fmt.Sprintf("The --mode flag must be either 'posix' or 's3', but you provided '%s'", ost))
+				os.Exit(1)
+			}
+		},
 	}
 
 	originUiCmd = &cobra.Command{
@@ -127,21 +146,20 @@ func init() {
 
 	// A variety of flags we add for S3 mode. These are ultimately required for configuring the S3 xrootd plugin
 	originServeCmd.Flags().String("service-name", "", "Specify the S3 service-name. Only used when an origin is launched in S3 mode.")
+	_ = originServeCmd.Flags().MarkDeprecated("service-name", "It no longer has any effect and will be removed in a future version.")
 	originServeCmd.Flags().String("region", "", "Specify the S3 region. Only used when an origin is launched in S3 mode.")
 	originServeCmd.Flags().String("bucket", "", "Specify the S3 bucket. Only used when an origin is launched in S3 mode.")
+	_ = originServeCmd.Flags().MarkDeprecated("bucket", `It no longer has any effect and will be removed in a future version. To set an S3 export use
+	-v bucket:/federation/prefix
+instead.
+	`)
 	originServeCmd.Flags().String("service-url", "", "Specify the S3 service-url. Only used when an origin is launched in S3 mode.")
 	originServeCmd.Flags().String("bucket-access-keyfile", "", "Specify a filepath to use for configuring the bucket's access key.")
 	originServeCmd.Flags().String("bucket-secret-keyfile", "", "Specify a filepath to use for configuring the bucket's access key.")
 	originServeCmd.Flags().String("url-style", "", "Specify the S3 url-style. Only used when an origin is launched in S3 mode, and can be either 'path' (default) or 'virtual.")
-	if err := viper.BindPFlag("Origin.S3ServiceName", originServeCmd.Flags().Lookup("service-name")); err != nil {
-		panic(err)
-	}
 	if err := viper.BindPFlag("Origin.S3Region", originServeCmd.Flags().Lookup("region")); err != nil {
 		panic(err)
 	}
-	if err := viper.BindPFlag("Origin.S3Bucket", originServeCmd.Flags().Lookup("bucket")); err != nil {
-		panic(err)
-	}
 	if err := viper.BindPFlag("Origin.S3ServiceUrl", originServeCmd.Flags().Lookup("service-url")); err != nil {
 		panic(err)
 	}
@@ -158,18 +176,8 @@ func init() {
 		panic("The --url-style flag must be either 'path' or 'virtual'")
 	}
 
-	// Would be nice to make these mutually exclusive to mode=posix instead of to --volume, but cobra
-	// doesn't seem to have something that can make the value of a flag exclusive to other flags
-	// Anyway, we never want to run the S3 flags with the -v flag.
-	originServeCmd.MarkFlagsMutuallyExclusive("volume", "service-name")
-	originServeCmd.MarkFlagsMutuallyExclusive("volume", "region")
-	originServeCmd.MarkFlagsMutuallyExclusive("volume", "bucket")
-	originServeCmd.MarkFlagsMutuallyExclusive("volume", "service-url")
-	originServeCmd.MarkFlagsMutuallyExclusive("volume", "bucket-access-keyfile")
-	originServeCmd.MarkFlagsMutuallyExclusive("volume", "bucket-secret-keyfile")
-	originServeCmd.MarkFlagsMutuallyExclusive("volume", "url-style")
-	// We don't require the bucket access and secret keyfiles as they're not needed for unauthenticated buckets
-	originServeCmd.MarkFlagsRequiredTogether("service-name", "region", "service-url")
+	// We don't require the bucket access and secret keyfiles as they're not needed for unauthenticated buckets.
+	// However, if you give us one, you've got to give us both.
 	originServeCmd.MarkFlagsRequiredTogether("bucket-access-keyfile", "bucket-secret-keyfile")
 
 	// The port any web UI stuff will be served on

docs/parameters.yaml

@@ -460,8 +460,9 @@ components: ["origin"]
 ---
 name: Origin.StoragePrefix
 description: |+
-  A string indicating the path to the volume exported by an origin's underlying storage. For example, if the origin is configured in posix
-  mode, this constitutes the path on disk exported by the origin for the federation.
+  A string indicating the path to the volume exported by an origin's underlying storage. For example, if the origin has a StorageType
+  of "posix", this constitutes the path on disk exported by the origin for the federation. If the origin has a StorageType of "s3",
+  this value is not currently used.
 
   NOTE: This config option is incompatible with multiple exports defined via `Origin.Exports` and requires that the origin exports
   only a single path.
@@ -717,42 +718,69 @@ components: ["origin"]
 ---
 name: Origin.S3ServiceName
 description: |+
-  The S3 Service Name to be used by the XRootD plugin.
+  [Deprecated] Origin.S3ServiceName was previously used in part to determine an export's FederationPrefix, but
+  upstream changes no longer rely on this value. As of Pelican 7.7.0, setting this value no longer has any effect.
+  AWSv4 signatures used by S3 servers to handle authentication now hardcode "s3" as their service name.
+
+  When constructing signed URLs for S3, this value is used as a part of the signature. It is almost always "s3". For more
+  information about S3 service names, see https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html
 type: string
+deprecated: true
+replacedby: none
 default: none
 components: ["origin"]
 ---
 name: Origin.S3Region
 description: |+
-  The S3 region to be used by the XRootD plugin.
+  Objects in S3 are associated with a "region", which is specifically a part of AWS's infrastructure. Often, S3 endpoints that are not
+  provided by Amazon use "us-east-1" as their region. This value is used when constructing signed URLs for getting authenticated objects
+  from a bucket.
+
+  For more information about how Amazon uses regions, see https://docs.aws.amazon.com/general/latest/gr/s3.html
+
+  This value is REQUIRED for S3 origins.
 type: string
 default: none
 components: ["origin"]
 ---
 name: Origin.S3Bucket
 description: |+
-  The S3 bucket to be used by the XRootD plugin.
+  Objects in S3 are stored in "buckets", which have unique names at each S3 service URL (ie the URL that provides access to your objects).
+  Setting a bucket restricts the origin to only serving objects from that bucket.
+
+  However, if the origin is meant to export all of the buckets associated with a given service URL, this value can be left unset *IF* all
+  of those buckets are public and the origin is using path-style URLS. When this is the case, objects can be fetched from the origin at the
+  path `/federation/prefix/bucket-name/object-name`.
 type: string
 default: none
 components: ["origin"]
 ---
 name: Origin.S3ServiceUrl
 description: |+
-  The S3 service URL to be used by the XRootD plugin.
+  The URL that provides API access your objects. When the S3 instance is hosted by Amazon, this is often "https://s3.us-east-1.amazonaws.com".
+
+  This value is REQUIRED for S3 origins.
 type: string
 default: none
 components: ["origin"]
 ---
 name: Origin.S3AccessKeyfile
 description: |+
-  A path to a file containing an S3 access keyfile for authenticated buckets when an origin is run in S3 mode.
+  A path to a file containing an S3 access keyfile (also sometimes called an API key) for authenticated buckets when an origin
+  is run in S3 mode.
+
+  This value is OPTIONAL for S3 origins, and only applies when an exported bucket requires authentication. It should not be used
+  if the bucket is public or if the origin is meant to export all public buckets from the S3 service URL.
 type: filename
 default: none
 components: ["origin"]
 ---
 name: Origin.S3SecretKeyfile
 description: |+
   A path to a file containing an S3 secret keyfile for authenticated buckets when an origin is run in S3 mode.
+
+  This value is OPTIONAL for S3 origins, and only applies when an exported bucket requires authentication. It should not be used
+  if the bucket is public or if the origin is meant to export all public buckets from the S3 service URL.
 type: filename
 default: none
 components: ["origin"]
@@ -761,6 +789,8 @@ name: Origin.S3UrlStyle
 description: |+
   The style of S3 urls used by the service URL host. This can be either "path" if objects are fetched at <service-url>/<bucket>/<object>
   or "virtual" if objects are fetched at <bucket>.<service-url>/<object>.
+
+  This value is REQUIRED for S3 origins, but defaults to "path" if not set.
 type: string
 default: path
 components: ["origin"]

github_scripts/osx_install.sh

@@ -39,6 +39,7 @@ popd
 
 git clone --depth=1 https://github.com/PelicanPlatform/xrootd-s3-http.git
 pushd xrootd-s3-http
+git checkout v0.1.1
 mkdir build
 cd build
 cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=$PWD/release_dir

go.mod

@@ -21,7 +21,6 @@ require (
 	github.com/jellydator/ttlcache/v3 v3.1.0
 	github.com/jsipprell/keyctl v1.0.4-0.20211208153515-36ca02672b6c
 	github.com/lestrrat-go/jwx/v2 v2.0.21
-	github.com/minio/minio-go/v7 v7.0.65
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
 	github.com/oklog/run v1.1.0
 	github.com/opensaucerer/grab/v3 v3.0.1
@@ -147,8 +146,6 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -166,7 +163,6 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/rs/xid v1.5.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
 	github.com/spf13/afero v1.11.0 // indirect

go.sum

@@ -476,7 +476,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
 github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@@ -539,12 +538,6 @@ github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6B
 github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
 github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
 github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
-github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
-github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.65 h1:sOlB8T3nQK+TApTpuN3k4WD5KasvZIE3vVFzyyCa0go=
-github.com/minio/minio-go/v7 v7.0.65/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc=
-github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
-github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -653,8 +646,6 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
-github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
-github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
 github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=

images/Dockerfile

@@ -82,7 +82,7 @@ ADD https://api.github.com/repos/PelicanPlatform/xrootd-s3-http/git/refs/heads/m
 RUN \
     git clone https://github.com/PelicanPlatform/xrootd-s3-http.git && \
     cd xrootd-s3-http && \
-    git reset a0eb944 --hard && \
+    git checkout v0.1.1 && \
     mkdir build && cd build && \
     cmake -DLIB_INSTALL_DIR=/usr/lib64 .. && \
     make install

images/dev.Dockerfile

@@ -104,7 +104,7 @@ ADD https://api.github.com/repos/PelicanPlatform/xrootd-s3-http/git/refs/heads/m
 RUN \
     git clone https://github.com/PelicanPlatform/xrootd-s3-http.git && \
     cd xrootd-s3-http && \
-    git reset a0eb944 --hard && \
+    git checkout v0.1.1 && \
     mkdir build && cd build && \
     cmake -DLIB_INSTALL_DIR=/usr/lib64 .. && \
     make install

launchers/launcher.go

@@ -171,27 +171,50 @@ func LaunchModules(ctx context.Context, modules config.ServerType) (servers []se
 
 	Origin:
 		Exports:
-		- StoragePrefix: /mnt/foo
-		  FederationPrefix: /bar
-		  Capabilities: ["PublicReads", "Writes", "Listings"]
-		- StoragePrefix: /mnt/test
-		  FederationPrefix: /baz
-		  Capabilities: ["Writes"]
+		  - StoragePrefix: /mnt/foo
+		    FederationPrefix: /bar
+		    Capabilities: ["PublicReads", "Writes", "Listings"]
+		  - StoragePrefix: /mnt/test
+		    FederationPrefix: /baz
+		    Capabilities: ["Writes"]
 
 	to export the directories /mnt/foo and /mnt/test under the namespace prefixes /bar and /baz, respectively (with listed permissions).
 	`)
 				return
 			}
 		case "s3":
-			if param.Origin_S3Region.GetString() == "" || param.Origin_S3ServiceName.GetString() == "" ||
-				param.Origin_S3ServiceUrl.GetString() == "" {
-				err = errors.Errorf("The S3 origin is missing configuration options to run properly." +
-					" You must specify a region, a service name and a service URL via the command line or via" +
-					" your configuration file.")
+			if len(*originExports) == 0 {
+				err = errors.Errorf(`
+	Export information was not provided.
+	To specify exports via the command line, use:
+
+				-v my-bucket:/my/prefix (REQUIRED --service-url https://my-s3-url.com) (REQUIRED --url-style <path or virtual>) \
+						(REQUIRED --region "my-region") (OPTIONAL --bucket-access-keyfile /path/to/access.key) \
+						(OPTIONAL --bucket-secret-keyfile /path/to/secret.key)
+
+
+	to export the S3 bucket under the namespace prefix /my/prefix.
+
+	Alternatively, specify Origin.Exports in the parameters.yaml file:
+
+	Origin:
+		StorageType: s3
+		S3UrlStyle: <path or virtual>
+		S3ServiceUrl: https://my-s3-url.com
+		S3Region: my-region
+		Exports:
+		  - FederationPrefix: /my/prefix
+			S3Bucket: my-bucket
+			S3AccessKeyfile: /path/to/access.key
+			S3SecretKeyfile: /path/to/secret.key
+			Capabilities: ["PublicReads", "Writes", "Listings"]
+
+	to export the S3 bucket my-bucket from https://my-s3-url.com under the namespace prefix /my/prefix (with listed permissions).
+				`)
 				return
 			}
 		default:
-			err = errors.Errorf("Currently-supported origin modes include posix and s3.")
+			err = errors.Errorf("Currently-supported origin modes include posix and s3, but you provided %s.", mode)
 			return
 		}