diff --git a/go.mod b/go.mod index bb56128ba..f37e9588e 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/jellydator/ttlcache/v3 v3.1.0 github.com/jsipprell/keyctl v1.0.4-0.20211208153515-36ca02672b6c github.com/lestrrat-go/jwx/v2 v2.0.21 - github.com/minio/minio-go/v7 v7.0.65 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/oklog/run v1.1.0 github.com/opensaucerer/grab/v3 v3.0.1 @@ -147,8 +146,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -166,7 +163,6 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/rs/xid v1.5.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect github.com/spf13/afero v1.11.0 // indirect diff --git a/go.sum b/go.sum index 9f71ea6bb..24426ecf2 100644 --- a/go.sum +++ b/go.sum @@ -476,7 +476,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -539,12 +538,6 @@ github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6B github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.65 h1:sOlB8T3nQK+TApTpuN3k4WD5KasvZIE3vVFzyyCa0go= -github.com/minio/minio-go/v7 v7.0.65/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -653,8 +646,6 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= diff --git a/xrootd/origin_test.go b/xrootd/origin_test.go index d9c56e19c..5809fd686 100644 --- a/xrootd/origin_test.go +++ b/xrootd/origin_test.go @@ -21,20 +21,17 @@ package xrootd import ( - "bytes" "context" "crypto/elliptic" _ "embed" "fmt" + "io" "net" + "net/http" "os" - "os/exec" "path/filepath" - "strings" "testing" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" log "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stretchr/testify/require" @@ -136,91 +133,60 @@ func originMockup(ctx context.Context, egrp *errgroup.Group, t *testing.T) conte return shutdownCancel } -func TestOrigin(t *testing.T) { - ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - defer func() { require.NoError(t, egrp.Wait()) }() - defer cancel() - +func runS3Test(t *testing.T, ctx context.Context, egrp *errgroup.Group, bucketName, urlStyle, objectName string) { viper.Reset() server_utils.ResetOriginExports() defer viper.Reset() defer server_utils.ResetOriginExports() + federationPrefix := "/test" + regionName := "us-east-1" + serviceUrl := "https://s3.amazonaws.com" + viper.Set("Origin.FederationPrefix", federationPrefix) + viper.Set("Origin.S3Bucket", bucketName) + viper.Set("Origin.S3Region", regionName) + viper.Set("Origin.S3ServiceUrl", serviceUrl) + viper.Set("Origin.S3UrlStyle", urlStyle) + viper.Set("Origin.StorageType", "s3") + viper.Set("Origin.EnablePublicReads", true) - viper.Set("Origin.StoragePrefix", t.TempDir()) - viper.Set("Origin.FederationPrefix", "/test") - viper.Set("Origin.StorageType", "posix") // Disable functionality we're not using (and is difficult to make work on Mac) viper.Set("Origin.EnableCmsd", false) viper.Set("Origin.EnableMacaroons", false) viper.Set("Origin.EnableVoms", false) + viper.Set("Origin.SelfTest", false) viper.Set("Origin.Port", 0) viper.Set("Server.WebPort", 0) viper.Set("TLSSkipVerify", true) - viper.Set("Logging.Origin.Scitokens", "debug") mockupCancel := originMockup(ctx, egrp, t) defer mockupCancel() - // In this case a 403 means its running - err := server_utils.WaitUntilWorking(ctx, "GET", param.Origin_Url.GetString(), "xrootd", 403) + originEndpoint := fmt.Sprintf("%s/%s", param.Origin_Url.GetString(), federationPrefix) + // At this point, a 404 means the server is running, which means its ready to grab objects from + err := server_utils.WaitUntilWorking(ctx, "GET", originEndpoint, "xrootd", 404) if err != nil { - t.Fatalf("Unsuccessful test: Server encountered an error: %v", err) + t.Fatalf("Unsucessful test: Server encountered an error: %v", err) } - fileTests := server_utils.TestFileTransferImpl{} - issuerUrl, err := config.GetServerIssuerURL() - require.NoError(t, err) - ok, err := fileTests.RunTests(ctx, param.Origin_Url.GetString(), config.GetServerAudience(), issuerUrl, server_utils.OriginSelfFileTest) + // Now try to get the object + originEndpoint = fmt.Sprintf("%s%s/%s", param.Origin_Url.GetString(), federationPrefix, objectName) + // Set up an HTTP client to request the object from originEndpoint + transport := config.GetTransport() + client := &http.Client{Transport: transport} + req, err := http.NewRequest("GET", originEndpoint, nil) require.NoError(t, err) - require.True(t, ok) -} - -func TestMultiExportOrigin(t *testing.T) { - ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - defer func() { require.NoError(t, egrp.Wait()) }() - defer cancel() - - viper.Reset() - defer viper.Reset() - server_utils.ResetOriginExports() - defer server_utils.ResetOriginExports() - - viper.SetConfigType("yaml") - // Use viper to read in the embedded config - err := viper.ReadConfig(strings.NewReader(multiExportOriginConfig)) - require.NoError(t, err, "error reading config") - - exports, err := server_utils.GetOriginExports() + resp, err := client.Do(req) require.NoError(t, err) - require.Len(t, *exports, 2) - // Override the object store prefix to a temp directory - (*exports)[0].StoragePrefix = t.TempDir() - (*exports)[1].StoragePrefix = t.TempDir() + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) - // Disable functionality we're not using (and is difficult to make work on Mac) - viper.Set("Origin.EnableCmsd", false) - viper.Set("Origin.EnableMacaroons", false) - viper.Set("Origin.EnableVoms", false) - viper.Set("Origin.Port", 0) - viper.Set("Server.WebPort", 0) - viper.Set("TLSSkipVerify", true) - viper.Set("Logging.Origin.Scitokens", "debug") - - mockupCancel := originMockup(ctx, egrp, t) - defer mockupCancel() - - // In this case a 403 means its running - err = server_utils.WaitUntilWorking(ctx, "GET", param.Origin_Url.GetString(), "xrootd", 403) - if err != nil { - t.Fatalf("Unsuccessful test: Server encountered an error: %v", err) - } - fileTests := server_utils.TestFileTransferImpl{} - issuerUrl, err := config.GetServerIssuerURL() + // Luckily, this is historic data and there's no reason to believe it will change, so we can + // grab the body and check for a known value indicating the file correctly downloaded (there is a + // time travel paradox baked into this -- if someone goes back to 1800 and alters the measurement, + // will this code be updated automatically?) + body, err := io.ReadAll(resp.Body) require.NoError(t, err) - - ok, err := fileTests.RunTests(ctx, param.Origin_Url.GetString(), config.GetServerAudience(), issuerUrl, server_utils.OriginSelfFileTest) - require.NoError(t, err) - require.True(t, ok) + require.Contains(t, string(body), "36e5f1edfe5d403b2da163bd50669f66 1800/wod_osd_1800.nc") } func TestS3OriginConfig(t *testing.T) { @@ -228,128 +194,15 @@ func TestS3OriginConfig(t *testing.T) { defer func() { require.NoError(t, egrp.Wait()) }() defer cancel() - viper.Reset() - server_utils.ResetOriginExports() - defer viper.Reset() - defer server_utils.ResetOriginExports() - tmpDir := t.TempDir() - - // We need to start up a minio server, which is how we emulate S3. Open to better ways to do this! - minIOServerCmd := exec.CommandContext(ctx, "minio", "server", "--quiet", tmpDir) - minIOServerCmd.Env = []string{fmt.Sprintf("HOME=%s", tmpDir)} - - // Create a few buffers to grab outputs - var outb, errb bytes.Buffer - minIOServerCmd.Stdout = &outb - minIOServerCmd.Stderr = &errb - - err := minIOServerCmd.Start() - require.NoError(t, err) - // Check for any other errors in the outputs - if strings.Contains(strings.ToLower(outb.String()), "error") { - t.Fatalf("Could not start the MinIO server: %s", outb.String()) - } else if errb.String() != "" { - t.Fatalf("Could not start the MinIO server: %s", errb.String()) - } - - // Check if MinIO is running (by default at localhost:9000) - endpoint := "localhost:9000" - // Expect a 403 from this endpoint -- that means it's running - err = server_utils.WaitUntilWorking(ctx, "GET", fmt.Sprintf("http://%s", endpoint), "xrootd", 403) - if err != nil { - t.Fatalf("Unsuccessful test: Server encountered an error: %v", err) - } - - defer func() { - err = minIOServerCmd.Process.Kill() - require.NoError(t, err) - }() - - // Let's create an unauthenticated bucket. First we set up a client instance - // By default, the endpoint will require access/secret key with these values. Note that this doesn't - // necessarily mean the bucket we create needs those keys, as the bucket will have its own IAM - accessKey := "minioadmin" - secretKey := "minioadmin" - useSSL := false - - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKey, secretKey, ""), - Secure: useSSL, + t.Run("S3OriginPathBucket", func(t *testing.T) { + runS3Test(t, ctx, egrp, "noaa-wod-pds", "path", "MD5SUMS") }) - require.NoError(t, err) - - // Create a new unauthenticated bucket. Under the hood, this will access the minio server endpoint - // and do a PUT - bucketName := "test-bucket" - regionName := "test-region" - federationPrefix := "/test" - - err = minioClient.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{}) - require.NoError(t, err) - - // Set bucket policy for public read access - policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::` + bucketName + `/*"]}]}` - err = minioClient.SetBucketPolicy(context.Background(), bucketName, policy) - require.NoError(t, err) - - // Upload a test file to the bucket - testFilePath := filepath.Join(tmpDir, "test-file.txt") - content := []byte("This is the content of the test file.") - err = os.WriteFile(testFilePath, content, 0644) - require.NoError(t, err) - - objectName := "test-file.txt" - contentType := "application/octet-stream" - _, err = minioClient.FPutObject(context.Background(), bucketName, objectName, testFilePath, minio.PutObjectOptions{ContentType: contentType}) - require.NoError(t, err) - - // All the setup to create the S3 server, add a bucket with a publicly-readable object is done. Now onto Pelican stuff - // Set up the origin and try to pull a file - viper.Set("Origin.StorageType", "s3") - viper.Set("Origin.S3Region", regionName) - viper.Set("Origin.S3Bucket", bucketName) - viper.Set("Origin.S3ServiceUrl", fmt.Sprintf("http://%s", endpoint)) - viper.Set("Origin.S3UrlStyle", "path") - viper.Set("Origin.FederationPrefix", federationPrefix) - - // Disable functionality we're not using (and is difficult to make work on Mac) - viper.Set("Origin.EnableCmsd", false) - viper.Set("Origin.EnableMacaroons", false) - viper.Set("Origin.EnableVoms", false) - viper.Set("Origin.SelfTest", false) - viper.Set("Origin.Port", 0) - viper.Set("Server.WebPort", 0) - viper.Set("TLSSkipVerify", true) - - mockupCancel := originMockup(ctx, egrp, t) - defer mockupCancel() - - // FOR NOW, we consider the test a success if the origin's xrootd server boots. - // TODO: When we've made it easier to come back and test whole pieces of this process by disentangling our - // *serve* commands, come back and make this e2e. The reason I'm punting is that in S3 mode, we also need all - // of the web_ui stuff initialized to export the public signing keys (as we can't export via XRootD) and we - // need a real token. These become much easier when we have an internally workable set of commands to do so. - originEndpoint := fmt.Sprintf("%s/%s/%s", param.Origin_Url.GetString(), federationPrefix, objectName) - // Until we sort out the things we mentioned above, we should expect a 403 because we don't try to pass tokens - // and we don't actually export any keys for token validation. - err = server_utils.WaitUntilWorking(ctx, "GET", originEndpoint, "xrootd", 403) - if err != nil { - t.Fatalf("Unsucessful test: Server encountered an error: %v", err) - } - - cancel() - mockupCancel() - - // Test virtual-style hosting - viper.Set("Origin.S3UrlStyle", "virtual") - ctx, cancel, egrp = test_utils.TestContext(context.Background(), t) - mockupCancel = originMockup(ctx, egrp, t) - defer cancel() - defer mockupCancel() + t.Run("S3OriginVirtualBucket", func(t *testing.T) { + runS3Test(t, ctx, egrp, "noaa-wod-pds", "virtual", "MD5SUMS") + }) - err = server_utils.WaitUntilWorking(ctx, "GET", originEndpoint, "xrootd", 403) - if err != nil { - t.Fatalf("Unsucessful test: Server encountered an error: %v", err) - } + t.Run("S3OriginNoBucket", func(t *testing.T) { + runS3Test(t, ctx, egrp, "", "path", "noaa-wod-pds/MD5SUMS") + }) }