Skip to content

Commit

Permalink
Update S3 tests to point at AWS
Browse files Browse the repository at this point in the history
  • Loading branch information
@jhiemstrawisc
jhiemstrawisc committed Apr 5, 2024
1 parent d03c8d1 commit 5383a4b
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 201 deletions.
4 changes: 0 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ require (
github.com/jellydator/ttlcache/v3 v3.1.0
github.com/jsipprell/keyctl v1.0.4-0.20211208153515-36ca02672b6c
github.com/lestrrat-go/jwx/v2 v2.0.21
github.com/minio/minio-go/v7 v7.0.65
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
github.com/oklog/run v1.1.0
github.com/opensaucerer/grab/v3 v3.0.1
Expand Down Expand Up @@ -147,8 +146,6 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
Expand All @@ -166,7 +163,6 @@ require (
github.com/prometheus/procfs v0.12.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
github.com/spf13/afero v1.11.0 // indirect
Expand Down
9 changes: 0 additions & 9 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
Expand Down Expand Up @@ -539,12 +538,6 @@ github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6B
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.65 h1:sOlB8T3nQK+TApTpuN3k4WD5KasvZIE3vVFzyyCa0go=
github.com/minio/minio-go/v7 v7.0.65/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
Expand Down Expand Up @@ -653,8 +646,6 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
Expand Down
229 changes: 41 additions & 188 deletions xrootd/origin_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,20 +21,17 @@
package xrootd

import (
"bytes"
"context"
"crypto/elliptic"
_ "embed"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"

"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -136,220 +133,76 @@ func originMockup(ctx context.Context, egrp *errgroup.Group, t *testing.T) conte
return shutdownCancel
}

func TestOrigin(t *testing.T) {
ctx, cancel, egrp := test_utils.TestContext(context.Background(), t)
defer func() { require.NoError(t, egrp.Wait()) }()
defer cancel()

func runS3Test(t *testing.T, ctx context.Context, egrp *errgroup.Group, bucketName, urlStyle, objectName string) {
viper.Reset()
server_utils.ResetOriginExports()
defer viper.Reset()
defer server_utils.ResetOriginExports()
federationPrefix := "/test"
regionName := "us-east-1"
serviceUrl := "https://s3.amazonaws.com"
viper.Set("Origin.FederationPrefix", federationPrefix)
viper.Set("Origin.S3Bucket", bucketName)
viper.Set("Origin.S3Region", regionName)
viper.Set("Origin.S3ServiceUrl", serviceUrl)
viper.Set("Origin.S3UrlStyle", urlStyle)
viper.Set("Origin.StorageType", "s3")
viper.Set("Origin.EnablePublicReads", true)

viper.Set("Origin.StoragePrefix", t.TempDir())
viper.Set("Origin.FederationPrefix", "/test")
viper.Set("Origin.StorageType", "posix")
// Disable functionality we're not using (and is difficult to make work on Mac)
viper.Set("Origin.EnableCmsd", false)
viper.Set("Origin.EnableMacaroons", false)
viper.Set("Origin.EnableVoms", false)
viper.Set("Origin.SelfTest", false)
viper.Set("Origin.Port", 0)
viper.Set("Server.WebPort", 0)
viper.Set("TLSSkipVerify", true)
viper.Set("Logging.Origin.Scitokens", "debug")

mockupCancel := originMockup(ctx, egrp, t)
defer mockupCancel()

// In this case a 403 means its running
err := server_utils.WaitUntilWorking(ctx, "GET", param.Origin_Url.GetString(), "xrootd", 403)
originEndpoint := fmt.Sprintf("%s/%s", param.Origin_Url.GetString(), federationPrefix)
// At this point, a 404 means the server is running, which means its ready to grab objects from
err := server_utils.WaitUntilWorking(ctx, "GET", originEndpoint, "xrootd", 404)
if err != nil {
t.Fatalf("Unsuccessful test: Server encountered an error: %v", err)
t.Fatalf("Unsucessful test: Server encountered an error: %v", err)
}
fileTests := server_utils.TestFileTransferImpl{}
issuerUrl, err := config.GetServerIssuerURL()
require.NoError(t, err)

ok, err := fileTests.RunTests(ctx, param.Origin_Url.GetString(), config.GetServerAudience(), issuerUrl, server_utils.OriginSelfFileTest)
// Now try to get the object
originEndpoint = fmt.Sprintf("%s%s/%s", param.Origin_Url.GetString(), federationPrefix, objectName)
// Set up an HTTP client to request the object from originEndpoint
transport := config.GetTransport()
client := &http.Client{Transport: transport}
req, err := http.NewRequest("GET", originEndpoint, nil)
require.NoError(t, err)
require.True(t, ok)
}

func TestMultiExportOrigin(t *testing.T) {
ctx, cancel, egrp := test_utils.TestContext(context.Background(), t)
defer func() { require.NoError(t, egrp.Wait()) }()
defer cancel()

viper.Reset()
defer viper.Reset()
server_utils.ResetOriginExports()
defer server_utils.ResetOriginExports()

viper.SetConfigType("yaml")
// Use viper to read in the embedded config
err := viper.ReadConfig(strings.NewReader(multiExportOriginConfig))
require.NoError(t, err, "error reading config")

exports, err := server_utils.GetOriginExports()
resp, err := client.Do(req)
require.NoError(t, err)
require.Len(t, *exports, 2)
// Override the object store prefix to a temp directory
(*exports)[0].StoragePrefix = t.TempDir()
(*exports)[1].StoragePrefix = t.TempDir()
defer resp.Body.Close()
require.Equal(t, 200, resp.StatusCode)

// Disable functionality we're not using (and is difficult to make work on Mac)
viper.Set("Origin.EnableCmsd", false)
viper.Set("Origin.EnableMacaroons", false)
viper.Set("Origin.EnableVoms", false)
viper.Set("Origin.Port", 0)
viper.Set("Server.WebPort", 0)
viper.Set("TLSSkipVerify", true)
viper.Set("Logging.Origin.Scitokens", "debug")

mockupCancel := originMockup(ctx, egrp, t)
defer mockupCancel()

// In this case a 403 means its running
err = server_utils.WaitUntilWorking(ctx, "GET", param.Origin_Url.GetString(), "xrootd", 403)
if err != nil {
t.Fatalf("Unsuccessful test: Server encountered an error: %v", err)
}
fileTests := server_utils.TestFileTransferImpl{}
issuerUrl, err := config.GetServerIssuerURL()
// Luckily, this is historic data and there's no reason to believe it will change, so we can
// grab the body and check for a known value indicating the file correctly downloaded (there is a
// time travel paradox baked into this -- if someone goes back to 1800 and alters the measurement,
// will this code be updated automatically?)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)

ok, err := fileTests.RunTests(ctx, param.Origin_Url.GetString(), config.GetServerAudience(), issuerUrl, server_utils.OriginSelfFileTest)
require.NoError(t, err)
require.True(t, ok)
require.Contains(t, string(body), "36e5f1edfe5d403b2da163bd50669f66 1800/wod_osd_1800.nc")
}

func TestS3OriginConfig(t *testing.T) {
ctx, cancel, egrp := test_utils.TestContext(context.Background(), t)
defer func() { require.NoError(t, egrp.Wait()) }()
defer cancel()

viper.Reset()
server_utils.ResetOriginExports()
defer viper.Reset()
defer server_utils.ResetOriginExports()
tmpDir := t.TempDir()

// We need to start up a minio server, which is how we emulate S3. Open to better ways to do this!
minIOServerCmd := exec.CommandContext(ctx, "minio", "server", "--quiet", tmpDir)
minIOServerCmd.Env = []string{fmt.Sprintf("HOME=%s", tmpDir)}

// Create a few buffers to grab outputs
var outb, errb bytes.Buffer
minIOServerCmd.Stdout = &outb
minIOServerCmd.Stderr = &errb

err := minIOServerCmd.Start()
require.NoError(t, err)
// Check for any other errors in the outputs
if strings.Contains(strings.ToLower(outb.String()), "error") {
t.Fatalf("Could not start the MinIO server: %s", outb.String())
} else if errb.String() != "" {
t.Fatalf("Could not start the MinIO server: %s", errb.String())
}

// Check if MinIO is running (by default at localhost:9000)
endpoint := "localhost:9000"
// Expect a 403 from this endpoint -- that means it's running
err = server_utils.WaitUntilWorking(ctx, "GET", fmt.Sprintf("http://%s", endpoint), "xrootd", 403)
if err != nil {
t.Fatalf("Unsuccessful test: Server encountered an error: %v", err)
}

defer func() {
err = minIOServerCmd.Process.Kill()
require.NoError(t, err)
}()

// Let's create an unauthenticated bucket. First we set up a client instance
// By default, the endpoint will require access/secret key with these values. Note that this doesn't
// necessarily mean the bucket we create needs those keys, as the bucket will have its own IAM
accessKey := "minioadmin"
secretKey := "minioadmin"
useSSL := false

minioClient, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: useSSL,
t.Run("S3OriginPathBucket", func(t *testing.T) {
runS3Test(t, ctx, egrp, "noaa-wod-pds", "path", "MD5SUMS")
})
require.NoError(t, err)

// Create a new unauthenticated bucket. Under the hood, this will access the minio server endpoint
// and do a PUT
bucketName := "test-bucket"
regionName := "test-region"
federationPrefix := "/test"

err = minioClient.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{})
require.NoError(t, err)

// Set bucket policy for public read access
policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::` + bucketName + `/*"]}]}`
err = minioClient.SetBucketPolicy(context.Background(), bucketName, policy)
require.NoError(t, err)

// Upload a test file to the bucket
testFilePath := filepath.Join(tmpDir, "test-file.txt")
content := []byte("This is the content of the test file.")
err = os.WriteFile(testFilePath, content, 0644)
require.NoError(t, err)

objectName := "test-file.txt"
contentType := "application/octet-stream"
_, err = minioClient.FPutObject(context.Background(), bucketName, objectName, testFilePath, minio.PutObjectOptions{ContentType: contentType})
require.NoError(t, err)

// All the setup to create the S3 server, add a bucket with a publicly-readable object is done. Now onto Pelican stuff
// Set up the origin and try to pull a file
viper.Set("Origin.StorageType", "s3")
viper.Set("Origin.S3Region", regionName)
viper.Set("Origin.S3Bucket", bucketName)
viper.Set("Origin.S3ServiceUrl", fmt.Sprintf("http://%s", endpoint))
viper.Set("Origin.S3UrlStyle", "path")
viper.Set("Origin.FederationPrefix", federationPrefix)

// Disable functionality we're not using (and is difficult to make work on Mac)
viper.Set("Origin.EnableCmsd", false)
viper.Set("Origin.EnableMacaroons", false)
viper.Set("Origin.EnableVoms", false)
viper.Set("Origin.SelfTest", false)
viper.Set("Origin.Port", 0)
viper.Set("Server.WebPort", 0)
viper.Set("TLSSkipVerify", true)

mockupCancel := originMockup(ctx, egrp, t)
defer mockupCancel()

// FOR NOW, we consider the test a success if the origin's xrootd server boots.
// TODO: When we've made it easier to come back and test whole pieces of this process by disentangling our
// *serve* commands, come back and make this e2e. The reason I'm punting is that in S3 mode, we also need all
// of the web_ui stuff initialized to export the public signing keys (as we can't export via XRootD) and we
// need a real token. These become much easier when we have an internally workable set of commands to do so.

originEndpoint := fmt.Sprintf("%s/%s/%s", param.Origin_Url.GetString(), federationPrefix, objectName)
// Until we sort out the things we mentioned above, we should expect a 403 because we don't try to pass tokens
// and we don't actually export any keys for token validation.
err = server_utils.WaitUntilWorking(ctx, "GET", originEndpoint, "xrootd", 403)
if err != nil {
t.Fatalf("Unsucessful test: Server encountered an error: %v", err)
}

cancel()
mockupCancel()

// Test virtual-style hosting
viper.Set("Origin.S3UrlStyle", "virtual")
ctx, cancel, egrp = test_utils.TestContext(context.Background(), t)
mockupCancel = originMockup(ctx, egrp, t)
defer cancel()
defer mockupCancel()
t.Run("S3OriginVirtualBucket", func(t *testing.T) {
runS3Test(t, ctx, egrp, "noaa-wod-pds", "virtual", "MD5SUMS")
})

err = server_utils.WaitUntilWorking(ctx, "GET", originEndpoint, "xrootd", 403)
if err != nil {
t.Fatalf("Unsucessful test: Server encountered an error: %v", err)
}
t.Run("S3OriginNoBucket", func(t *testing.T) {
runS3Test(t, ctx, egrp, "", "path", "noaa-wod-pds/MD5SUMS")
})
}

0 comments on commit 5383a4b

Please sign in to comment.