Skip to content

Commit

Permalink
Bypassing slow recursive tests for macos runner
Browse files Browse the repository at this point in the history
  • Loading branch information
@jhiemstrawisc @turetske
jhiemstrawisc authored and turetske committed Mar 13, 2024
1 parent b05521b commit fcc923f
Show file tree
Hide file tree
Showing 2 changed files with 408 additions and 366 deletions.
366 changes: 0 additions & 366 deletions client/fed_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,7 @@ import (
"io"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"testing"
"time"

Expand Down Expand Up @@ -409,367 +407,3 @@ func TestGetPublicRead(t *testing.T) {
}
})
}

func TestRecursiveUploadsAndDownloads(t *testing.T) {
// Create instance of test federation
ctx, _, _ := test_utils.TestContext(context.Background(), t)
viper.Reset()
common.ResetOriginExports()

fed := fed_test_utils.NewFedTest(t, mixedAuthOriginCfg)

//////////////////////////SETUP///////////////////////////
// Create a token file
issuer, err := config.GetServerIssuerURL()
require.NoError(t, err)
audience := config.GetServerAudience()

tokenConfig := token.NewWLCGToken()
tokenConfig.Lifetime = time.Minute
tokenConfig.Issuer = issuer
tokenConfig.Subject = "origin"
tokenConfig.AddAudiences(audience)
tokenConfig.AddResourceScopes(token_scopes.NewResourceScope(token_scopes.Storage_Read, "/"),
token_scopes.NewResourceScope(token_scopes.Storage_Modify, "/"))
token, err := tokenConfig.CreateToken()
assert.NoError(t, err)
tempToken, err := os.CreateTemp(t.TempDir(), "token")
assert.NoError(t, err, "Error creating temp token file")
defer os.Remove(tempToken.Name())
_, err = tempToken.WriteString(token)
assert.NoError(t, err, "Error writing to temp token file")
tempToken.Close()

// Disable progress bars to not reuse the same mpb instance
viper.Set("Logging.DisableProgressBars", true)

// Make our test directories and files
tempDir, err := os.MkdirTemp("", "UploadDir")
assert.NoError(t, err)
defer os.RemoveAll(tempDir)
permissions := os.FileMode(0777)
err = os.Chmod(tempDir, permissions)
require.NoError(t, err)

testFileContent1 := "test file content"
testFileContent2 := "more test file content!"
tempFile1, err := os.CreateTemp(tempDir, "test1")
assert.NoError(t, err, "Error creating temp1 file")
tempFile2, err := os.CreateTemp(tempDir, "test1")
assert.NoError(t, err, "Error creating temp2 file")
defer os.Remove(tempFile1.Name())
defer os.Remove(tempFile2.Name())
_, err = tempFile1.WriteString(testFileContent1)
assert.NoError(t, err, "Error writing to temp1 file")
tempFile1.Close()
_, err = tempFile2.WriteString(testFileContent2)
assert.NoError(t, err, "Error writing to temp2 file")
tempFile2.Close()

config.SetPreferredPrefix("pelican")
for _, export := range *fed.Exports {
// Set path for object to upload/download
tempPath := tempDir
dirName := filepath.Base(tempPath)
// Note: minimally fixing this test as it is soon to be replaced
uploadURL := fmt.Sprintf("pelican://%s:%s%s/%s", param.Server_Hostname.GetString(), strconv.Itoa(param.Server_WebPort.GetInt()),
export.FederationPrefix, dirName)

//////////////////////////////////////////////////////////

// Upload the file with PUT
transferDetailsUpload, err := client.DoPut(ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
require.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytes17 := 0
countBytes23 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
for _, transfer := range transferDetailsUpload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case int64(17):
countBytes17++
continue
case int64(23):
countBytes23++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not upload proper amount of bytes")
}
}
if countBytes17 != 1 || countBytes23 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not uploaded correctly")
}
} else if len(transferDetailsUpload) != 2 {
t.Fatalf("Amount of transfers results returned for upload was not correct. Transfer details returned: %d", len(transferDetailsUpload))
}

// Download the files we just uploaded
var transferDetailsDownload []client.TransferResults
if export.Capabilities.PublicReads {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true)
} else {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true, client.WithTokenLocation(tempToken.Name()))
}
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytesUploadIdx0 := 0
countBytesUploadIdx1 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
// In this case, we want to match them to the sizes of the uploaded files
for _, transfer := range transferDetailsUpload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case transferDetailsUpload[0].TransferredBytes:
countBytesUploadIdx0++
continue
case transferDetailsUpload[1].TransferredBytes:
countBytesUploadIdx1++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not download proper amount of bytes")
}
}
if countBytesUploadIdx0 != 1 || countBytesUploadIdx1 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not downloaded correctly")
} else if len(transferDetailsDownload) != 2 {
t.Fatalf("Amount of transfers results returned for download was not correct. Transfer details returned: %d", len(transferDetailsDownload))
}
}
}

config.SetPreferredPrefix("pelican")
for _, export := range *fed.Exports {
// Set path for object to upload/download
tempPath := tempDir
dirName := filepath.Base(tempPath)
uploadURL := fmt.Sprintf("pelican://%s/%s", export.FederationPrefix, dirName)

//////////////////////////////////////////////////////////

// Upload the file with PUT
transferDetailsUpload, err := client.DoPut(ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytes17 := 0
countBytes23 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
for _, transfer := range transferDetailsUpload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case int64(17):
countBytes17++
continue
case int64(23):
countBytes23++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not upload proper amount of bytes")
}
}
if countBytes17 != 1 || countBytes23 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not uploaded correctly")
}
} else if len(transferDetailsUpload) != 2 {
t.Fatalf("Amount of transfers results returned for upload was not correct. Transfer details returned: %d", len(transferDetailsUpload))
}

// Download the files we just uploaded
var transferDetailsDownload []client.TransferResults
if export.Capabilities.PublicReads {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true)
} else {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true, client.WithTokenLocation(tempToken.Name()))
}
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytesUploadIdx0 := 0
countBytesUploadIdx1 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
// In this case, we want to match them to the sizes of the uploaded files
for _, transfer := range transferDetailsUpload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case transferDetailsUpload[0].TransferredBytes:
countBytesUploadIdx0++
continue
case transferDetailsUpload[1].TransferredBytes:
countBytesUploadIdx1++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not download proper amount of bytes")
}
}
if countBytesUploadIdx0 != 1 || countBytesUploadIdx1 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not downloaded correctly")
} else if len(transferDetailsDownload) != 2 {
t.Fatalf("Amount of transfers results returned for download was not correct. Transfer details returned: %d", len(transferDetailsDownload))
}
}
}

config.SetPreferredPrefix("osdf")
for _, export := range *fed.Exports {
// Set path for object to upload/download
tempPath := tempDir
dirName := filepath.Base(tempPath)
// Note: minimally fixing this test as it is soon to be replaced
uploadURL := fmt.Sprintf("pelican://%s:%s%s/%s", param.Server_Hostname.GetString(), strconv.Itoa(param.Server_WebPort.GetInt()),
export.FederationPrefix, dirName)

//////////////////////////////////////////////////////////

// Upload the file with PUT
transferDetailsUpload, err := client.DoPut(ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytes17 := 0
countBytes23 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
for _, transfer := range transferDetailsUpload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case int64(17):
countBytes17++
continue
case int64(23):
countBytes23++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not upload proper amount of bytes")
}
}
if countBytes17 != 1 || countBytes23 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not uploaded correctly")
}
} else if len(transferDetailsUpload) != 2 {
t.Fatalf("Amount of transfers results returned for upload was not correct. Transfer details returned: %d", len(transferDetailsUpload))
}

// Download the files we just uploaded
tmpDir := t.TempDir()
var transferDetailsDownload []client.TransferResults
if export.Capabilities.PublicReads {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, tmpDir, true)
} else {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, tmpDir, true, client.WithTokenLocation(tempToken.Name()))
}
assert.NoError(t, err)
if err == nil && len(transferDetailsDownload) == 2 {
countBytesUploadIdx0 := 0
countBytesUploadIdx1 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
// In this case, we want to match them to the sizes of the uploaded files
for _, transfer := range transferDetailsDownload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case transferDetailsUpload[0].TransferredBytes:
countBytesUploadIdx0++
continue
case transferDetailsUpload[1].TransferredBytes:
countBytesUploadIdx1++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not download proper amount of bytes")
}
}
if countBytesUploadIdx0 != 1 || countBytesUploadIdx1 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not downloaded correctly")
}
contents, err := os.ReadFile(filepath.Join(tmpDir, path.Join(dirName, path.Base(tempFile2.Name()))))
assert.NoError(t, err)
assert.Equal(t, testFileContent2, string(contents))
contents, err = os.ReadFile(filepath.Join(tmpDir, path.Join(dirName, path.Base(tempFile1.Name()))))
assert.NoError(t, err)
assert.Equal(t, testFileContent1, string(contents))
} else if err == nil && len(transferDetailsDownload) != 2 {
t.Fatalf("Number of transfers results returned for download was not correct. Transfer details returned: %d", len(transferDetailsDownload))
}
}

config.SetPreferredPrefix("osdf")
for _, export := range *fed.Exports {
// Set path for object to upload/download
tempPath := tempDir
dirName := filepath.Base(tempPath)
uploadURL := fmt.Sprintf("pelican://%s/%s", export.FederationPrefix, dirName)

//////////////////////////////////////////////////////////

// Upload the file with PUT
transferDetailsUpload, err := client.DoPut(ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytes17 := 0
countBytes23 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
for _, transfer := range transferDetailsUpload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case int64(17):
countBytes17++
continue
case int64(23):
countBytes23++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not upload proper amount of bytes")
}
}
if countBytes17 != 1 || countBytes23 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not uploaded correctly")
}
} else if len(transferDetailsUpload) != 2 {
t.Fatalf("Amount of transfers results returned for upload was not correct. Transfer details returned: %d", len(transferDetailsUpload))
}

// Download the files we just uploaded
var transferDetailsDownload []client.TransferResults
if export.Capabilities.PublicReads {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true)
} else {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true, client.WithTokenLocation(tempToken.Name()))
}
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytesUploadIdx0 := 0
countBytesUploadIdx1 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
// In this case, we want to match them to the sizes of the uploaded files
for _, transfer := range transferDetailsUpload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case transferDetailsUpload[0].TransferredBytes:
countBytesUploadIdx0++
continue
case transferDetailsUpload[1].TransferredBytes:
countBytesUploadIdx1++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not download proper amount of bytes")
}
}
if countBytesUploadIdx0 != 1 || countBytesUploadIdx1 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not downloaded correctly")
} else if len(transferDetailsDownload) != 2 {
t.Fatalf("Amount of transfers results returned for download was not correct. Transfer details returned: %d", len(transferDetailsDownload))
}
}
}
}
Loading

0 comments on commit fcc923f

Please sign in to comment.