Replace os.stat sentinel file check with XRootD http GET

launchers/launcher.go

@@ -156,6 +156,13 @@ func LaunchModules(ctx context.Context, modules server_structs.ServerType) (serv
 
 	if modules.IsEnabled(server_structs.OriginType) {
 
+		var server server_structs.XRootDServer
+		server, err = OriginServe(ctx, engine, egrp, modules)
+		if err != nil {
+			return
+		}
+		servers = append(servers, server)
+
 		var originExports []server_utils.OriginExport
 		originExports, err = server_utils.GetOriginExports()
 		if err != nil {
@@ -167,13 +174,6 @@ func LaunchModules(ctx context.Context, modules server_structs.ServerType) (serv
 			return
 		}
 
-		var server server_structs.XRootDServer
-		server, err = OriginServe(ctx, engine, egrp, modules)
-		if err != nil {
-			return
-		}
-		servers = append(servers, server)
-
 		// Ordering: `LaunchBrokerListener` depends on the "right" value of Origin.FederationPrefix
 		// which is possibly not set until `OriginServe` is called.
 		// NOTE: Until the Broker supports multi-export origins, we've made the assumption that there

server_utils/origin.go

@@ -20,19 +20,23 @@ package server_utils
 
 import (
 	"fmt"
-	"os"
+	"net/http"
+	"net/url"
 	"path"
 	"path/filepath"
 	"reflect"
 	"strings"
+	"time"
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 
+	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/param"
 	"github.com/pelicanplatform/pelican/server_structs"
+	"github.com/pelicanplatform/pelican/token"
 )
 
 var originExports []OriginExport
@@ -750,18 +754,64 @@ from S3 service URL. In this configuration, objects can be accessed at /federati
 	return originExports, nil
 }
 
+// Generate a minimally scoped auth token that allows the origin
+// to query itself for its sentinel file
+func generateSentinelReadToken(resourceScope string) (string, error) {
+	issuerUrl := param.Server_ExternalWebUrl.GetString()
+	if issuerUrl == "" { // if both are empty, then error
+		return "", errors.New("failed to create a sentinel check auth token because required configuration 'Server.ExternalWebUrl' is empty")
+	}
+	fTestTokenCfg := token.NewWLCGToken()
+	fTestTokenCfg.Lifetime = time.Minute
+	fTestTokenCfg.Issuer = issuerUrl
+	fTestTokenCfg.Subject = "origin"
+	fTestTokenCfg.Claims = map[string]string{"scope": fmt.Sprintf("storage.read:/%v", resourceScope)}
+	// For self-tests, the audience is the server itself
+	fTestTokenCfg.AddAudienceAny()
+
+	// CreateToken also handles validation for us
+	tok, err := fTestTokenCfg.CreateToken()
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create sentinel check auth token")
+	}
+
+	return tok, nil
+}
+
 // Check the sentinel files from Origin.Exports
 func CheckOriginSentinelLocations(exports []OriginExport) (ok bool, err error) {
 	for _, export := range exports {
 		if export.SentinelLocation != "" {
+			log.Infof("Checking that sentinel object %v is present for federation prefix %s", export.SentinelLocation, export.FederationPrefix)
 			sentinelPath := path.Clean(export.SentinelLocation)
 			if path.Base(sentinelPath) != sentinelPath {
-				return false, errors.Errorf("invalid SentinelLocation path for StoragePrefix %s, file must not contain a directory. Got %s", export.StoragePrefix, export.SentinelLocation)
+				return false, errors.Errorf("invalid SentinelLocation path for federation prefix %s, path must not contain a directory. Got %s", export.FederationPrefix, export.SentinelLocation)
 			}
-			fullPath := filepath.Join(export.StoragePrefix, sentinelPath)
-			_, err := os.Stat(fullPath)
+
+			fullPath := filepath.Join(export.FederationPrefix, sentinelPath)
+			tkn, err := generateSentinelReadToken(sentinelPath)
 			if err != nil {
-				return false, errors.Wrapf(err, "fail to open SentinelLocation %s for StoragePrefix %s. Collection check failed", export.SentinelLocation, export.StoragePrefix)
+				return false, errors.Wrap(err, "failed to generate self-auth token for sentinel object check")
+			}
+
+			sentinelUrl, err := url.JoinPath(param.Origin_Url.GetString(), fullPath)
+			if err != nil {
+				return false, errors.Wrapf(err, "unable fo form sentinel URL for Origin.Url %v, sentinel path %v", param.Origin_Url.GetString(), fullPath)
+			}
+			req, err := http.NewRequest(http.MethodGet, sentinelUrl, nil)
+			if err != nil {
+				return false, errors.Wrap(err, "failed to create GET request for sentinel object check")
+			}
+			req.Header.Set("Authorization", "Bearer "+tkn)
+
+			client := http.Client{Transport: config.GetTransport()}
+			resp, err := client.Do(req)
+			if err != nil {
+				return false, errors.Wrapf(err, "fail to open sentinel object %s for federation prefix %s.", export.SentinelLocation, export.FederationPrefix)
+			}
+
+			if resp.StatusCode != 200 {
+				return false, errors.New(fmt.Sprintf("got non-200 response code %v when checking sentinel object %s for federation prefix %s", resp.StatusCode, export.SentinelLocation, export.FederationPrefix))
 			}
 		}
 	}

server_utils/origin_test.go

@@ -23,8 +23,6 @@ package server_utils
 import (
 	_ "embed"
 	"fmt"
-	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -395,64 +393,6 @@ func TestGetExports(t *testing.T) {
 	})
 }
 
-func TestCheckOriginSentinelLocation(t *testing.T) {
-	tmpDir := t.TempDir()
-	tempStn := filepath.Join(tmpDir, "mock_sentinel")
-	file, err := os.Create(tempStn)
-	require.NoError(t, err)
-	err = file.Close()
-	require.NoError(t, err)
-
-	mockExportNoStn := OriginExport{
-		StoragePrefix:    "/foo/bar",
-		FederationPrefix: "/demo/foo/bar",
-		Capabilities:     server_structs.Capabilities{Reads: true},
-	}
-	mockExportValidStn := OriginExport{
-		StoragePrefix:    tmpDir,
-		FederationPrefix: "/demo/foo/bar",
-		Capabilities:     server_structs.Capabilities{Reads: true},
-		SentinelLocation: "mock_sentinel",
-	}
-	mockExportInvalidStn := OriginExport{
-		StoragePrefix:    tmpDir,
-		FederationPrefix: "/demo/foo/bar",
-		Capabilities:     server_structs.Capabilities{Reads: true},
-		SentinelLocation: "sentinel_dne",
-	}
-
-	t.Run("empty-sentinel-return-ok", func(t *testing.T) {
-		exports := make([]OriginExport, 0)
-		exports = append(exports, mockExportNoStn)
-		exports = append(exports, mockExportNoStn)
-
-		ok, err := CheckOriginSentinelLocations(exports)
-		assert.NoError(t, err)
-		assert.True(t, ok)
-	})
-
-	t.Run("valid-sentinel-return-ok", func(t *testing.T) {
-		exports := make([]OriginExport, 0)
-		exports = append(exports, mockExportNoStn)
-		exports = append(exports, mockExportValidStn)
-
-		ok, err := CheckOriginSentinelLocations(exports)
-		assert.NoError(t, err)
-		assert.True(t, ok)
-	})
-
-	t.Run("invalid-sentinel-return-error", func(t *testing.T) {
-		exports := make([]OriginExport, 0)
-		exports = append(exports, mockExportNoStn)
-		exports = append(exports, mockExportValidStn)
-		exports = append(exports, mockExportInvalidStn)
-
-		ok, err := CheckOriginSentinelLocations(exports)
-		assert.Error(t, err)
-		assert.False(t, ok)
-	})
-}
-
 func runBucketNameTest(t *testing.T, name string, valid bool) {
 	t.Run(fmt.Sprintf("testBucketNameValidation-%s", name), func(t *testing.T) {
 		err := validateBucketName(name)

xrootd/origin_test.go

@@ -231,16 +231,7 @@ func TestMultiExportOrigin(t *testing.T) {
 	require.True(t, ok)
 }
 
-func runS3Test(t *testing.T, bucketName, urlStyle, objectName string) {
-	ctx, cancel, egrp := test_utils.TestContext(context.Background(), t)
-	defer func() { require.NoError(t, egrp.Wait()) }()
-	defer cancel()
-
-	server_utils.ResetTestState()
-
-	defer server_utils.ResetTestState()
-
-	federationPrefix := "/test"
+func mockupS3Origin(ctx context.Context, egrp *errgroup.Group, t *testing.T, federationPrefix, bucketName, urlStyle string) context.CancelFunc {
 	regionName := "us-east-1"
 	serviceUrl := "https://s3.amazonaws.com"
 	viper.Set("Origin.FederationPrefix", federationPrefix)
@@ -260,7 +251,19 @@ func runS3Test(t *testing.T, bucketName, urlStyle, objectName string) {
 	viper.Set("Server.WebPort", 0)
 	viper.Set("TLSSkipVerify", true)
 
-	mockupCancel := originMockup(ctx, egrp, t)
+	return originMockup(ctx, egrp, t)
+}
+
+func runS3Test(t *testing.T, bucketName, urlStyle, objectName string) {
+	ctx, cancel, egrp := test_utils.TestContext(context.Background(), t)
+	defer func() { require.NoError(t, egrp.Wait()) }()
+	defer cancel()
+	server_utils.ResetTestState()
+	defer server_utils.ResetTestState()
+
+	federationPrefix := "/test"
+
+	mockupCancel := mockupS3Origin(ctx, egrp, t, federationPrefix, bucketName, urlStyle)
 	defer mockupCancel()
 
 	originEndpoint := param.Origin_Url.GetString()
@@ -304,3 +307,149 @@ func TestS3OriginConfig(t *testing.T) {
 		runS3Test(t, "", "path", "noaa-wod-pds/MD5SUMS")
 	})
 }
+
+func TestS3OriginWithSentinel(t *testing.T) {
+	ctx, cancel, egrp := test_utils.TestContext(context.Background(), t)
+	defer func() { require.NoError(t, egrp.Wait()) }()
+	defer cancel()
+	server_utils.ResetTestState()
+	defer server_utils.ResetTestState()
+
+	federationPrefix := "/test"
+	bucketName := "noaa-wod-pds"
+
+	mockupCancel := mockupS3Origin(ctx, egrp, t, federationPrefix, bucketName, "path")
+	defer mockupCancel()
+
+	mockExportValidStn := server_utils.OriginExport{
+		StoragePrefix:    viper.GetString("Origin.StoragePrefix"),
+		FederationPrefix: viper.GetString("Origin.FederationPrefix"),
+		Capabilities:     server_structs.Capabilities{Reads: true},
+		SentinelLocation: "MD5SUMS",
+	}
+
+	originEndpoint := param.Origin_Url.GetString()
+	// At this point, a 403 means the server is running, which means its ready to grab objects from
+	err := server_utils.WaitUntilWorking(ctx, "GET", originEndpoint, "xrootd", 403, true)
+	if err != nil {
+		t.Fatalf("Unsuccessful test: Server encountered an error: %v", err)
+	}
+
+	// mock export with no sentinel
+	mockExportNoStn := server_utils.OriginExport{
+		StoragePrefix:    viper.GetString("Origin.StoragePrefix"),
+		FederationPrefix: viper.GetString("Origin.FederationPrefix"),
+		Capabilities:     server_structs.Capabilities{Reads: true},
+	}
+
+	// mock export with an invalid sentinel
+	mockExportInvalidStn := server_utils.OriginExport{
+		StoragePrefix:    viper.GetString("Origin.StoragePrefix"),
+		FederationPrefix: viper.GetString("Origin.FederationPrefix"),
+		Capabilities:     server_structs.Capabilities{Reads: true},
+		SentinelLocation: "MD5SUMS_dne",
+	}
+
+	t.Run("valid-sentinel-return-ok", func(t *testing.T) {
+		ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportValidStn})
+		require.NoError(t, err)
+		require.True(t, ok)
+	})
+	t.Run("empty-sentinel-return-ok", func(t *testing.T) {
+		ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportNoStn})
+		require.NoError(t, err)
+		require.True(t, ok)
+	})
+
+	t.Run("invalid-sentinel-return-error", func(t *testing.T) {
+		ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportInvalidStn})
+		require.Error(t, err)
+		require.False(t, ok)
+	})
+}
+
+func TestPosixOriginWithSentinel(t *testing.T) {
+	ctx, cancel, egrp := test_utils.TestContext(context.Background(), t)
+	defer func() { require.NoError(t, egrp.Wait()) }()
+	defer cancel()
+
+	server_utils.ResetTestState()
+
+	defer server_utils.ResetTestState()
+
+	// Create a test temp dir, ensure it's readable by XRootD
+	tmpPathPattern := "XRD-Tst_Orgn*"
+	tmpPath, err := os.MkdirTemp("", tmpPathPattern)
+	require.NoError(t, err)
+	err = os.Chmod(tmpPath, 0755)
+	require.NoError(t, err)
+
+	viper.Set("Origin.StoragePrefix", tmpPath)
+	viper.Set("Origin.FederationPrefix", "/test")
+	viper.Set("Origin.StorageType", "posix")
+	// Disable functionality we're not using (and is difficult to make work on Mac)
+	viper.Set("Origin.EnableCmsd", false)
+	viper.Set("Origin.EnableMacaroons", false)
+	viper.Set("Origin.EnableVoms", false)
+	viper.Set("Origin.Port", 0)
+	viper.Set("Server.WebPort", 0)
+	viper.Set("TLSSkipVerify", true)
+	viper.Set("Logging.Origin.Scitokens", "trace")
+
+	mockupCancel := originMockup(ctx, egrp, t)
+	defer mockupCancel()
+
+	// mock export with a valid sentinel
+	mockExportValidStn := server_utils.OriginExport{
+		StoragePrefix:    viper.GetString("Origin.StoragePrefix"),
+		FederationPrefix: viper.GetString("Origin.FederationPrefix"),
+		Capabilities:     server_structs.Capabilities{Reads: true},
+		SentinelLocation: "mock_sentinel",
+	}
+	// mock export with no sentinel
+	mockExportNoStn := server_utils.OriginExport{
+		StoragePrefix:    viper.GetString("Origin.StoragePrefix"),
+		FederationPrefix: viper.GetString("Origin.FederationPrefix"),
+		Capabilities:     server_structs.Capabilities{Reads: true},
+	}
+	// mock export with an invalid sentinel
+	mockExportInvalidStn := server_utils.OriginExport{
+		StoragePrefix:    viper.GetString("Origin.StoragePrefix"),
+		FederationPrefix: viper.GetString("Origin.FederationPrefix"),
+		Capabilities:     server_structs.Capabilities{Reads: true},
+		SentinelLocation: "sentinel_dne",
+	}
+
+	// Create a sentinel file, ensure it's readable by XRootD
+	tempStn := filepath.Join(mockExportValidStn.StoragePrefix, mockExportValidStn.SentinelLocation)
+	file, err := os.Create(tempStn)
+	require.NoError(t, err)
+	err = file.Close()
+	require.NoError(t, err)
+	err = os.Chmod(tempStn, 0755)
+	require.NoError(t, err)
+
+	err = server_utils.WaitUntilWorking(ctx, "GET", param.Origin_Url.GetString(), "xrootd", 403, false)
+	if err != nil {
+		t.Fatalf("Unsuccessful test: Server encountered an error: %v", err)
+	}
+	require.NoError(t, err)
+
+	t.Run("valid-sentinel-return-ok", func(t *testing.T) {
+		ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportValidStn})
+		require.NoError(t, err)
+		require.True(t, ok)
+	})
+
+	t.Run("empty-sentinel-return-ok", func(t *testing.T) {
+		ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportNoStn})
+		require.NoError(t, err)
+		require.True(t, ok)
+	})
+
+	t.Run("invalid-sentinel-return-error", func(t *testing.T) {
+		ok, err := server_utils.CheckOriginSentinelLocations([]server_utils.OriginExport{mockExportInvalidStn})
+		require.Error(t, err)
+		require.False(t, ok)
+	})
+}