diff --git a/.goreleaser.yml b/.goreleaser.yml index b37d6c16f..5ed036612 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -22,6 +22,8 @@ builds: binary: pelican tags: - forceposix + ldflags: + - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.builtBy=goreleaser -X config.version={{.Version}} ignore: - goos: windows goarch: arm64 diff --git a/broker/broker_test.go b/broker/broker_test.go index 7e0a0b3d6..306cddf9a 100644 --- a/broker/broker_test.go +++ b/broker/broker_test.go @@ -139,7 +139,7 @@ func doRetrieveRequest(t *testing.T, ctx context.Context, dur time.Duration) (*h req.Header.Set("X-Pelican-Timeout", dur.String()) req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pelican-origin/"+config.PelicanVersion) + req.Header.Set("User-Agent", "pelican-origin/"+config.GetVersion()) req.Header.Set("Authorization", "Bearer "+token) diff --git a/broker/client.go b/broker/client.go index 00a4722c8..bb4d4962b 100644 --- a/broker/client.go +++ b/broker/client.go @@ -203,7 +203,7 @@ func ConnectToOrigin(ctx context.Context, brokerUrl, prefix, originName string) // Send a request to the broker for a connection reversal req, err := http.NewRequestWithContext(ctx, "POST", brokerUrl, reqReader) req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pelican-cache/"+config.PelicanVersion) + req.Header.Set("User-Agent", "pelican-cache/"+config.GetVersion()) brokerAud, err := url.Parse(brokerUrl) if err != nil { @@ -404,7 +404,7 @@ func doCallback(ctx context.Context, brokerResp reversalRequest) (listener net.L dur := time.Duration(5*time.Second - time.Duration(mrand.Intn(500))*time.Millisecond) req.Header.Set("X-Pelican-Timeout", dur.String()) req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pelican-origin/"+config.PelicanVersion) + req.Header.Set("User-Agent", "pelican-origin/"+config.GetVersion()) cacheAud, err := url.Parse(brokerResp.CallbackUrl) if err != nil { @@ -585,7 +585,7 @@ func LaunchRequestMonitor(ctx context.Context, egrp *errgroup.Group, resultChan dur := param.Transport_ResponseHeaderTimeout.GetDuration() - time.Duration(mrand.Intn(500))*time.Millisecond req.Header.Set("X-Pelican-Timeout", dur.String()) req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "pelican-origin/"+config.PelicanVersion) + req.Header.Set("User-Agent", "pelican-origin/"+config.GetVersion()) brokerAud, err := url.Parse(param.Federation_BrokerUrl.GetString()) if err != nil { diff --git a/client/director.go b/client/director.go index 2168b316e..a27c8cade 100644 --- a/client/director.go +++ b/client/director.go @@ -155,8 +155,7 @@ func queryDirector(verb, source, directorUrl string) (resp *http.Response, err e // Include the Client's version as a User-Agent header. The Director will decide // if it supports the version, and provide an error message in the case that it // cannot. - userAgent := "pelican-client/" + ObjectClientOptions.Version - req.Header.Set("User-Agent", userAgent) + req.Header.Set("User-Agent", getUserAgent("")) // Perform the HTTP request resp, err = client.Do(req) @@ -235,8 +234,8 @@ func GetCachesFromDirectorResponse(resp *http.Response, needsToken bool) (caches } // NewTransferDetails creates the TransferDetails struct with the given cache -func NewTransferDetailsUsingDirector(cache namespaces.DirectorCache, opts TransferDetailsOptions) []TransferDetails { - details := make([]TransferDetails, 0) +func NewTransferDetailsUsingDirector(cache namespaces.DirectorCache, opts transferDetailsOptions) []transferAttemptDetails { + details := make([]transferAttemptDetails, 0) cacheEndpoint := cache.EndpointUrl // Form the URL @@ -252,40 +251,41 @@ func NewTransferDetailsUsingDirector(cache namespaces.DirectorCache, opts Transf cacheURL.Scheme = "" cacheURL.Opaque = "" } - log.Debugf("Parsed Cache: %s\n", cacheURL.String()) + log.Debugf("Parsed Cache: %s", cacheURL.String()) if opts.NeedsToken { cacheURL.Scheme = "https" - if !HasPort(cacheURL.Host) { + if !hasPort(cacheURL.Host) { // Add port 8444 and 8443 - cacheURL.Host += ":8444" - details = append(details, TransferDetails{ - Url: *cacheURL, + urlCopy := *cacheURL + urlCopy.Host += ":8444" + details = append(details, transferAttemptDetails{ + Url: &urlCopy, Proxy: false, PackOption: opts.PackOption, }) // Strip the port off and add 8443 - cacheURL.Host = cacheURL.Host[:len(cacheURL.Host)-5] + ":8443" + cacheURL.Host = cacheURL.Host + ":8443" } // Whether port is specified or not, add a transfer without proxy - details = append(details, TransferDetails{ - Url: *cacheURL, + details = append(details, transferAttemptDetails{ + Url: cacheURL, Proxy: false, PackOption: opts.PackOption, }) } else { cacheURL.Scheme = "http" - if !HasPort(cacheURL.Host) { + if !hasPort(cacheURL.Host) { cacheURL.Host += ":8000" } - isProxyEnabled := IsProxyEnabled() - details = append(details, TransferDetails{ - Url: *cacheURL, + isProxyEnabled := isProxyEnabled() + details = append(details, transferAttemptDetails{ + Url: cacheURL, Proxy: isProxyEnabled, PackOption: opts.PackOption, }) if isProxyEnabled && CanDisableProxy() { - details = append(details, TransferDetails{ - Url: *cacheURL, + details = append(details, transferAttemptDetails{ + Url: cacheURL, Proxy: false, PackOption: opts.PackOption, }) diff --git a/client/director_test.go b/client/director_test.go index 6f552ea5c..f37d1c8c7 100644 --- a/client/director_test.go +++ b/client/director_test.go @@ -155,7 +155,7 @@ func TestNewTransferDetailsUsingDirector(t *testing.T) { // Case 1: cache with http - transfers := NewTransferDetailsUsingDirector(nonAuthCache, TransferDetailsOptions{nonAuthCache.AuthedReq, ""}) + transfers := NewTransferDetailsUsingDirector(nonAuthCache, transferDetailsOptions{nonAuthCache.AuthedReq, ""}) assert.Equal(t, 2, len(transfers)) assert.Equal(t, "my-cache-url:8000", transfers[0].Url.Host) assert.Equal(t, "http", transfers[0].Url.Scheme) @@ -166,7 +166,7 @@ func TestNewTransferDetailsUsingDirector(t *testing.T) { assert.Equal(t, false, transfers[1].Proxy) // Case 2: cache with https - transfers = NewTransferDetailsUsingDirector(authCache, TransferDetailsOptions{authCache.AuthedReq, ""}) + transfers = NewTransferDetailsUsingDirector(authCache, transferDetailsOptions{authCache.AuthedReq, ""}) assert.Equal(t, 1, len(transfers)) assert.Equal(t, "my-cache-url:8443", transfers[0].Url.Host) assert.Equal(t, "https", transfers[0].Url.Scheme) @@ -174,7 +174,7 @@ func TestNewTransferDetailsUsingDirector(t *testing.T) { // Case 3: cache without port with http nonAuthCache.EndpointUrl = "my-cache-url" - transfers = NewTransferDetailsUsingDirector(nonAuthCache, TransferDetailsOptions{nonAuthCache.AuthedReq, ""}) + transfers = NewTransferDetailsUsingDirector(nonAuthCache, transferDetailsOptions{nonAuthCache.AuthedReq, ""}) assert.Equal(t, 2, len(transfers)) assert.Equal(t, "my-cache-url:8000", transfers[0].Url.Host) assert.Equal(t, "http", transfers[0].Url.Scheme) @@ -185,7 +185,7 @@ func TestNewTransferDetailsUsingDirector(t *testing.T) { // Case 4. cache without port with https authCache.EndpointUrl = "my-cache-url" - transfers = NewTransferDetailsUsingDirector(authCache, TransferDetailsOptions{authCache.AuthedReq, ""}) + transfers = NewTransferDetailsUsingDirector(authCache, transferDetailsOptions{authCache.AuthedReq, ""}) assert.Equal(t, 2, len(transfers)) assert.Equal(t, "my-cache-url:8444", transfers[0].Url.Host) assert.Equal(t, "https", transfers[0].Url.Scheme) diff --git a/client/errorAccum.go b/client/errorAccum.go index 38f58b810..b5f660c61 100644 --- a/client/errorAccum.go +++ b/client/errorAccum.go @@ -22,46 +22,75 @@ import ( "errors" "fmt" "net/http" - "sync" + "strings" "time" grab "github.com/opensaucerer/grab/v3" ) -type TimestampedError struct { - err error - timestamp time.Time -} +type ( + TimestampedError struct { + err error + timestamp time.Time + } -var ( - bunchOfErrors []TimestampedError - mu sync.Mutex - // We will generate an error string including the time since startup - startup time.Time = time.Now() + // A container object for multiple sub-errors representing transfer failures. + TransferErrors struct { + start time.Time + errors []error + } ) -// AddError will add an accumulated error to the error stack -func AddError(err error) bool { - mu.Lock() - defer mu.Unlock() - bunchOfErrors = append(bunchOfErrors, TimestampedError{err, time.Now()}) - return true +func (te *TimestampedError) Error() string { + return te.err.Error() +} + +func (te *TimestampedError) Unwrap() error { + return te.err +} + +// Create a new transfer error object +func NewTransferErrors() *TransferErrors { + return &TransferErrors{ + start: time.Now(), + errors: make([]error, 0), + } +} + +func (te *TransferErrors) AddError(err error) { + if te.errors == nil { + te.errors = make([]error, 0) + } + if err != nil { + te.errors = append(te.errors, &TimestampedError{err: err, timestamp: time.Now()}) + } } -func ClearErrors() { - mu.Lock() - defer mu.Unlock() +func (te *TransferErrors) Unwrap() []error { + return te.errors +} - bunchOfErrors = make([]TimestampedError, 0) +func (te *TransferErrors) Error() string { + if te.errors == nil { + return "transfer error unknown" + } + if len(te.errors) == 1 { + return "transfer error: " + te.errors[0].Error() + } + errors := make([]string, len(te.errors)) + for idx, err := range te.errors { + errors[idx] = err.Error() + } + return "transfer errors: [" + strings.Join(errors, ", ") + "]" } -func GetErrors() string { - mu.Lock() - defer mu.Unlock() +// Return a more refined, user-friendly error string +func (te *TransferErrors) UserError() string { first := true - lastError := startup + lastError := te.start var errorsFormatted []string - for idx, theError := range bunchOfErrors { + for idx, err := range te.errors { + theError := err.(*TimestampedError) errFmt := fmt.Sprintf("Attempt #%v: %s", idx+1, theError.err.Error()) timeElapsed := theError.timestamp.Sub(lastError) timeFormat := timeElapsed.Truncate(100 * time.Millisecond).String() @@ -69,7 +98,7 @@ func GetErrors() string { if first { errFmt += " since start)" } else { - timeSinceStart := theError.timestamp.Sub(startup) + timeSinceStart := theError.timestamp.Sub(te.start) timeSinceStartFormat := timeSinceStart.Truncate(100 * time.Millisecond).String() errFmt += " elapsed, " + timeSinceStartFormat + " since start)" } @@ -127,15 +156,24 @@ func IsRetryable(err error) bool { return false } -// ErrorsRetryable returns if the errors in the stack are retryable later -func ErrorsRetryable() bool { - mu.Lock() - defer mu.Unlock() - // Loop through the errors and see if all of them are retryable - for _, theError := range bunchOfErrors { - if !IsRetryable(theError.err) { +// Returns true if all errors are retryable. +// If no errors are present, then returns true +func (te *TransferErrors) AllErrorsRetryable() bool { + if te.errors == nil { + return true + } + for _, err := range te.errors { + if !IsRetryable(err) { return false } } return true } + +func ShouldRetry(err error) bool { + var te *TransferErrors + if errors.As(err, &te) { + return te.AllErrorsRetryable() + } + return IsRetryable(err) +} diff --git a/client/errorAccum_test.go b/client/errorAccum_test.go index f623395e1..6c2eadff3 100644 --- a/client/errorAccum_test.go +++ b/client/errorAccum_test.go @@ -28,52 +28,45 @@ import ( // TestErrorAccum tests simple adding and removing from the accumulator func TestErrorAccum(t *testing.T) { - bunchOfErrors = make([]TimestampedError, 0) - defer func() { - bunchOfErrors = make([]TimestampedError, 0) - }() + te := NewTransferErrors() // Case 1: cache with http err := errors.New("error1") err2 := errors.New("error2") - AddError(err) - AddError(err2) + te.AddError(err) + te.AddError(err2) - errStr := GetErrors() + errStr := te.UserError() assert.Regexp(t, `Attempt\ \#2:\ error2\ \(0s\ elapsed,\ [0-9]+m?s\ since\ start\);\ Attempt\ \#1:\ error1\ \([0-9]+m?s\ since\ start\)`, errStr) } // TestErrorsRetryableFalse tests that errors are not retryable func TestErrorsRetryableFalse(t *testing.T) { - bunchOfErrors = make([]TimestampedError, 0) - defer func() { - bunchOfErrors = make([]TimestampedError, 0) - }() + te := NewTransferErrors() + // Case 2: cache with http - AddError(&SlowTransferError{}) - AddError(&SlowTransferError{}) - assert.True(t, ErrorsRetryable(), "ErrorsRetryable should be true") + te.AddError(&SlowTransferError{}) + te.AddError(&SlowTransferError{}) + assert.True(t, te.AllErrorsRetryable(), "ErrorsRetryable should be true") - AddError(&ConnectionSetupError{}) - assert.True(t, ErrorsRetryable(), "ErrorsRetryable should be true") + te.AddError(&ConnectionSetupError{}) + assert.True(t, te.AllErrorsRetryable(), "ErrorsRetryable should be true") // Now add a non-retryable error - AddError(errors.New("Non retryable error")) - assert.False(t, ErrorsRetryable(), "ErrorsRetryable should be false") + te.AddError(errors.New("Non retryable error")) + assert.False(t, te.AllErrorsRetryable(), "ErrorsRetryable should be false") } // TestErrorsRetryableTrue tests that errors are retryable func TestErrorsRetryableTrue(t *testing.T) { - bunchOfErrors = make([]TimestampedError, 0) - defer func() { - bunchOfErrors = make([]TimestampedError, 0) - }() + te := NewTransferErrors() + // Try with a retryable error nested error - AddError(&url.Error{Err: &SlowTransferError{}}) - assert.True(t, ErrorsRetryable(), "ErrorsRetryable should be true") + te.AddError(&url.Error{Err: &SlowTransferError{}}) + assert.True(t, te.AllErrorsRetryable(), "ErrorsRetryable should be true") - AddError(&ConnectionSetupError{}) - assert.True(t, ErrorsRetryable(), "ErrorsRetryable should be true") + te.AddError(&ConnectionSetupError{}) + assert.True(t, te.AllErrorsRetryable(), "ErrorsRetryable should be true") } diff --git a/client/get_best_cache.go b/client/get_best_cache.go index 62d476aa3..6455db8ce 100644 --- a/client/get_best_cache.go +++ b/client/get_best_cache.go @@ -26,7 +26,6 @@ import ( "net" "net/http" "net/url" - "os" "strconv" "strings" @@ -49,43 +48,15 @@ func GetBestCache(cacheListName string) ([]string, error) { }) var caches_list []string + //Use Stashservers.dat api - // Check if the user provided a caches json file location - if CachesJsonLocation != "" { - if _, err := os.Stat(CachesJsonLocation); os.IsNotExist(err) { - // path does not exist - log.Errorln(CachesJsonLocation, "does not exist") + //api_text = "stashservers.dat" + GeoIpUrl.Path = "stashservers.dat" - return nil, errors.New("Unable to open caches json file at: " + CachesJsonLocation) - } - - //Use geo ip api on caches in provided json file - //caches_list := get_json_caches(caches_json_location) - var caches_string string = "" - - for _, cache := range caches_list { - parsed_url, err := url.Parse(cache) - if err != nil { - log.Errorln("Could not parse URL") - } - - caches_string = caches_string + parsed_url.Host - - // Remove the first comma - caches_string = string([]rune(caches_string)[1:]) - GeoIpUrl.Path = "api/v1.0/geo/stashcp/" + caches_string - } - } else { - //Use Stashservers.dat api - - //api_text = "stashservers.dat" - GeoIpUrl.Path = "stashservers.dat" - - if cacheListName != "" { - queryParams := GeoIpUrl.Query() - queryParams.Set("list", cacheListName) - GeoIpUrl.RawQuery = queryParams.Encode() - } + if cacheListName != "" { + queryParams := GeoIpUrl.Query() + queryParams.Set("list", cacheListName) + GeoIpUrl.RawQuery = queryParams.Encode() } var responselines_b [][]byte @@ -128,7 +99,7 @@ func GetBestCache(cacheListName string) ([]string, error) { break } req.Header.Add("Cache-control", "max-age=0") - req.Header.Add("User-Agent", "pelican/"+ObjectClientOptions.Version) + req.Header.Add("User-Agent", getUserAgent("")) resp, err = client.Do(req) if err == nil { break @@ -186,7 +157,6 @@ func GetBestCache(cacheListName string) ([]string, error) { caches_list[i], caches_list[j] = caches_list[j], caches_list[i] }) minsite := caches_list[0] - NearestCacheList = caches_list log.Debugf("Unable to use Geoip to find closest cache! Returning random cache %s", minsite) log.Debugf("Randomized list of nearest caches: %s", strings.Join(caches_list, ",")) return caches_list, nil @@ -214,13 +184,14 @@ func GetBestCache(cacheListName string) ([]string, error) { minsite := cachesList[cacheListName][minIndex-1] log.Debugln("Closest cache:", minsite) + finalCacheList := make([]string, 0, len(ordered_list)) for _, ordered_index := range ordered_list { orderedIndex, _ := strconv.Atoi(ordered_index) - NearestCacheList = append(NearestCacheList, cachesList[cacheListName][orderedIndex-1]) + finalCacheList = append(finalCacheList, cachesList[cacheListName][orderedIndex-1]) } log.Debugf("Returning closest cache: %s", minsite) - log.Debugf("Ordered list of nearest caches: %s", NearestCacheList) - return NearestCacheList, nil + log.Debugf("Ordered list of nearest caches: %s", finalCacheList) + return finalCacheList, nil } } diff --git a/client/handle_http.go b/client/handle_http.go index 3844f166d..830890496 100644 --- a/client/handle_http.go +++ b/client/handle_http.go @@ -19,6 +19,7 @@ package client import ( + "bytes" "context" "fmt" "io" @@ -28,7 +29,9 @@ import ( "net/url" "os" "path" + "reflect" "regexp" + "slices" "strconv" "strings" "sync" @@ -36,12 +39,16 @@ import ( "syscall" "time" - grab "github.com/opensaucerer/grab/v3" + "github.com/VividCortex/ewma" + "github.com/google/uuid" + "github.com/lestrrat-go/option" + "github.com/opensaucerer/grab/v3" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/spf13/viper" "github.com/studio-b12/gowebdav" "github.com/vbauerster/mpb/v8" - "github.com/vbauerster/mpb/v8/decor" + "golang.org/x/sync/errgroup" "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/namespaces" @@ -53,9 +60,176 @@ var ( progressCtr *mpb.Progress ) -type StoppedTransferError struct { - Err string -} +type ( + // Error type for when the transfer started to return data then completely stopped + StoppedTransferError struct { + Err string + } + + // SlowTransferError is an error that is returned when a transfer takes longer than the configured timeout + SlowTransferError struct { + BytesTransferred int64 + BytesPerSecond int64 + BytesTotal int64 + Duration time.Duration + } + + // ConnectionSetupError is an error that is returned when a connection to the remote server fails + ConnectionSetupError struct { + URL string + Err error + } + + // Represents the results of a single object transfer, + // potentially across multiple attempts / retries. + TransferResults struct { + JobId uuid.UUID // The job ID this result corresponds to + job *TransferJob + Error error + TransferredBytes int64 + Attempts []TransferResult + } + + TransferResult struct { + Number int // indicates which attempt this is + TransferFileBytes int64 // how much each attempt downloaded + TimeToFirstByte int64 // how long it took to download the first byte + TransferEndTime int64 // when the transfer ends + Endpoint string // which origin did it use + ServerVersion string // TODO: figure out how to get this??? + Error error // what error the attempt returned (if any) + } + + clientTransferResults struct { + id uuid.UUID // ID of the client that created the job + results TransferResults // Actual transfer results + } + + // A structure representing a single endpoint we will attempt a transfer against. + transferAttemptDetails struct { + // Url of the server's hostname and port + Url *url.URL + + // Proxy specifies if a proxy should be used + Proxy bool + + // Specifies the pack option in the transfer URL + PackOption string + } + + // A structure representing a single file to transfer. + transferFile struct { + ctx context.Context + engine *TransferEngine + job *TransferJob + callback TransferCallbackFunc + remoteURL *url.URL + localPath string + token string + upload bool + packOption string + attempts []transferAttemptDetails + accounting payloadStruct + project string + err error + } + + // A representation of a "transfer job". The job + // can be submitted to the client library, resulting + // in one or more transfers (if recursive is true). + // We assume the transfer job is potentially queued for a + // long time and all the transfers generated by this job will + // use the same namespace and token. + TransferJob struct { + ctx context.Context + cancel context.CancelFunc + callback TransferCallbackFunc + uuid uuid.UUID + remoteURL *url.URL + lookupDone atomic.Bool + lookupErr error + activeXfer atomic.Int64 + totalXfer int + localPath string + upload bool + recursive bool + skipAcquire bool + caches []*url.URL + useDirector bool + tokenLocation string + token string + project string + namespace namespaces.Namespace + } + + // A TransferJob associated with a client's request + clientTransferJob struct { + uuid uuid.UUID + job *TransferJob + } + + // A transferFile associated with a client request + clientTransferFile struct { + uuid uuid.UUID + jobId uuid.UUID + file *transferFile + } + + // An object able to process transfer jobs. + TransferEngine struct { + ctx context.Context // The context provided upon creation of the engine. + cancel context.CancelFunc + egrp *errgroup.Group // The errgroup for the worker goroutines + work chan *clientTransferJob + files chan *clientTransferFile + results chan *clientTransferResults + jobLookupDone chan *clientTransferJob // Indicates the job lookup handler is done with the job + workersActive int + resultsMap map[uuid.UUID]chan *TransferResults + workMap map[uuid.UUID]chan *TransferJob + notifyChan chan bool + closeChan chan bool + closeDoneChan chan bool + ewmaTick *time.Ticker + ewma ewma.MovingAverage + ewmaVal atomic.Int64 + ewmaCtr atomic.Int64 + clientLock sync.RWMutex + } + + TransferCallbackFunc = func(path string, downloaded int64, totalSize int64, completed bool) + + // A client to the transfer engine. + TransferClient struct { + id uuid.UUID + ctx context.Context + cancel context.CancelFunc + callback TransferCallbackFunc + skipAcquire bool // Enable/disable the token acquisition logic. Defaults to acquiring a token + tokenLocation string + work chan *TransferJob + closed bool + caches []*url.URL + results chan *TransferResults + finalResults chan TransferResults + setupResults sync.Once + } + + TransferOption = option.Interface + identTransferOptionCaches struct{} + identTransferOptionCallback struct{} + identTransferOptionTokenLocation struct{} + identTransferOptionAcquireToken struct{} + + transferDetailsOptions struct { + NeedsToken bool + PackOption string + } +) + +const ( + ewmaInterval = 15 * time.Second +) // The progress container object creates several // background goroutines. Instead of creating the object @@ -81,14 +255,6 @@ func (e *HttpErrResp) Error() string { return e.Err } -// SlowTransferError is an error that is returned when a transfer takes longer than the configured timeout -type SlowTransferError struct { - BytesTransferred int64 - BytesPerSecond int64 - BytesTotal int64 - Duration time.Duration -} - func (e *SlowTransferError) Error() string { return "cancelled transfer, too slow. Detected speed: " + ByteCountSI(e.BytesPerSecond) + @@ -103,21 +269,8 @@ func (e *SlowTransferError) Is(target error) bool { return ok } -type FileDownloadError struct { - Text string - Err error -} - -func (e *FileDownloadError) Error() string { - return e.Text -} - -func (e *FileDownloadError) Unwrap() error { - return e.Err -} - // Determines whether or not we can interact with the site HTTP proxy -func IsProxyEnabled() bool { +func isProxyEnabled() bool { if _, isSet := os.LookupEnv("http_proxy"); !isSet { return false } @@ -132,12 +285,6 @@ func CanDisableProxy() bool { return !param.Client_DisableProxyFallback.GetBool() } -// ConnectionSetupError is an error that is returned when a connection to the remote server fails -type ConnectionSetupError struct { - URL string - Err error -} - func (e *ConnectionSetupError) Error() string { if e.Err != nil { if len(e.URL) > 0 { @@ -160,26 +307,575 @@ func (e *ConnectionSetupError) Is(target error) bool { return ok } -// HasPort test the host if it includes a port -func HasPort(host string) bool { +// hasPort test the host if it includes a port +func hasPort(host string) bool { var checkPort = regexp.MustCompile("^.*:[0-9]+$") return checkPort.MatchString(host) } -type TransferDetails struct { - // Url is the url.URL of the cache and port - Url url.URL +// Create a new transfer results object +func newTransferResults(job *TransferJob) TransferResults { + return TransferResults{ + job: job, + JobId: job.uuid, + Attempts: make([]TransferResult, 0), + } +} + +// Returns a new transfer engine object whose lifetime is tied +// to the provided context. Will launcher worker goroutines to +// handle the underlying transfers +func NewTransferEngine(ctx context.Context) *TransferEngine { + ctx, cancel := context.WithCancel(ctx) + egrp, _ := errgroup.WithContext(ctx) + work := make(chan *clientTransferJob) + files := make(chan *clientTransferFile) + results := make(chan *clientTransferResults, 5) + te := &TransferEngine{ + ctx: ctx, + cancel: cancel, + egrp: egrp, + work: work, + files: files, + results: results, + resultsMap: make(map[uuid.UUID]chan *TransferResults), + workMap: make(map[uuid.UUID]chan *TransferJob), + jobLookupDone: make(chan *clientTransferJob), + notifyChan: make(chan bool), + closeChan: make(chan bool), + closeDoneChan: make(chan bool), + ewmaTick: time.NewTicker(ewmaInterval), + ewma: ewma.NewMovingAverage(), + } + workerCount := param.Client_WorkerCount.GetInt() + if workerCount <= 0 { + log.Errorln("Worker count must be a positive integer") + return nil + } + for idx := 0; idx < workerCount; idx++ { + egrp.Go(func() error { + return runTransferWorker(ctx, te.files, te.results) + }) + } + te.workersActive = workerCount + egrp.Go(te.runMux) + egrp.Go(te.runJobHandler) + return te +} + +// Create an option that provides a callback for a TransferClient +// +// The callback is invoked periodically by one of the transfer workers, +// with inputs of the local path (e.g., source on upload), the current +// bytes transferred, and the total object size +func WithCallback(callback TransferCallbackFunc) TransferOption { + return option.New(identTransferOptionCallback{}, callback) +} + +// Create an option to override the cache list +func WithCaches(caches ...*url.URL) TransferOption { + return option.New(identTransferOptionCaches{}, caches) +} + +// Create an option to override the token locating logic +// +// This will force the transfer to use a specific file for the token +// contents instead of doing any sort of auto-detection +func WithTokenLocation(location string) TransferOption { + return option.New(identTransferOptionTokenLocation{}, location) +} + +// Create an option to specify the token acquisition logic +// +// Token acquisition (e.g., using OAuth2 to get a token when one +// isn't found in the environment) defaults to `true` but can be +// disabled with this options +func WithAcquireToken(enable bool) TransferOption { + return option.New(identTransferOptionAcquireToken{}, enable) +} + +// Create a new client to work with an engine +func (te *TransferEngine) NewClient(options ...TransferOption) (client *TransferClient, err error) { + log.Debugln("Making new clients") + id, err := uuid.NewV7() + if err != nil { + err = errors.Wrap(err, "Unable to create new UUID for client") + return + } + client = &TransferClient{ + id: id, + results: make(chan *TransferResults), + work: make(chan *TransferJob), + } + client.ctx, client.cancel = context.WithCancel(te.ctx) + + for _, option := range options { + switch option.Ident() { + case identTransferOptionCaches{}: + client.caches = option.Value().([]*url.URL) + case identTransferOptionCallback{}: + client.callback = option.Value().(TransferCallbackFunc) + case identTransferOptionTokenLocation{}: + client.tokenLocation = option.Value().(string) + case identTransferOptionAcquireToken{}: + client.skipAcquire = !option.Value().(bool) + } + } + func() { + te.clientLock.Lock() + defer te.clientLock.Unlock() + te.resultsMap[id] = client.results + te.workMap[id] = client.work + }() + log.Debugln("Inserted work map for client", id.String()) + te.notifyChan <- true + return +} + +// Initiates a shutdown of the transfer engine. +// Waits until all workers have finished +func (te *TransferEngine) Shutdown() error { + te.Close() + <-te.closeDoneChan + te.ewmaTick.Stop() + te.cancel() + + err := te.egrp.Wait() + if err != nil && err != context.Canceled { + return err + } + return nil +} + +// Closes the TransferEngine. No new work may +// be submitted. Any ongoing work will continue +func (te *TransferEngine) Close() { + te.closeChan <- true +} + +// Launches a helper goroutine that ensures completed +// transfer results are routed back to their requesting +// channels +func (te *TransferEngine) runMux() error { + tmpResults := make(map[uuid.UUID][]*TransferResults) + activeJobs := make(map[uuid.UUID][]*TransferJob) + closing := false + closedWorkChan := false + // The main body of the routine; continuously select on one of the channels, + // which indicate some event occurs, until an exit condition is met. + for { + // The channels we interact with on depend on how many clients and how many results we have. + // Since this is dynamic, we can't do a fixed-size case statement and instead need to use reflect.Select. + // This helper function iterates through the TransferEngine's internals with a read-lock held, building up + // the list of work. + cases, workMap, workKeys, resultsMap, resultsKeys := func() (cases []reflect.SelectCase, workMap map[uuid.UUID]chan *TransferJob, workKeys []uuid.UUID, resultsMap map[uuid.UUID]chan *TransferResults, resultsKeys []uuid.UUID) { + te.clientLock.RLock() + defer te.clientLock.RUnlock() + workMap = make(map[uuid.UUID]chan *TransferJob, len(te.workMap)) + ctr := 0 + workKeys = make(uuid.UUIDs, 0) + for key, val := range te.workMap { + if val != nil { + workKeys = append(workKeys, key) + } + } + cases = make([]reflect.SelectCase, len(workKeys)+len(tmpResults)+7) + sortFunc := func(a, b uuid.UUID) int { + return bytes.Compare(a[:], b[:]) + } + slices.SortFunc(workKeys, sortFunc) + for _, key := range workKeys { + workMap[key] = te.workMap[key] + cases[ctr].Dir = reflect.SelectRecv + cases[ctr].Chan = reflect.ValueOf(workMap[key]) + ctr++ + } + resultsMap = make(map[uuid.UUID]chan *TransferResults, len(tmpResults)) + resultsKeys = make([]uuid.UUID, 0) + for key := range tmpResults { + resultsKeys = append(resultsKeys, key) + } + slices.SortFunc(resultsKeys, sortFunc) + for _, key := range resultsKeys { + resultsMap[key] = te.resultsMap[key] + cases[ctr].Dir = reflect.SelectSend + cases[ctr].Chan = reflect.ValueOf(resultsMap[key]) + cases[ctr].Send = reflect.ValueOf(tmpResults[key][0]) + ctr++ + } + // Notification a new client has been started; recompute the channels + cases[ctr].Dir = reflect.SelectRecv + cases[ctr].Chan = reflect.ValueOf(te.notifyChan) + // Notification that a transfer has finished. + cases[ctr+1].Dir = reflect.SelectRecv + cases[ctr+1].Chan = reflect.ValueOf(te.results) + // Placeholder; never used. + cases[ctr+2].Dir = reflect.SelectRecv + cases[ctr+2].Chan = reflect.ValueOf(nil) + // Notification that the TransferEngine has been cancelled; shutdown immediately + cases[ctr+3].Dir = reflect.SelectRecv + cases[ctr+3].Chan = reflect.ValueOf(te.ctx.Done()) + // Notification the translation from job-to-file has been completed. + cases[ctr+4].Dir = reflect.SelectRecv + cases[ctr+4].Chan = reflect.ValueOf(te.jobLookupDone) + // Notification the transfer engine has been "closed". No more jobs will come in + // and shutdown can start. + cases[ctr+5].Dir = reflect.SelectRecv + cases[ctr+5].Chan = reflect.ValueOf(te.closeChan) + // The transfer engine keeps statistics on the number of concurrent transfers occur + // (this is later used to normalize the minimum transfer rate); this ticker periodically + // will recalculate the average. + cases[ctr+6].Dir = reflect.SelectRecv + cases[ctr+6].Chan = reflect.ValueOf(te.ewmaTick.C) + return + }() + if closing && len(workMap) == 0 && !closedWorkChan { + // If there's no more incoming work, we can safely close the work channel + // which will cause the job-to-file worker to shutdown. + close(te.work) + closedWorkChan = true + } + // Statement purposely left commented out; too heavyweight/noisy to leave in at runtime but useful for developer debugging. + //log.Debugf("runMux running with %d active client channels and sending %d client responses", len(workMap), len(resultsMap)) + chosen, recv, ok := reflect.Select(cases) + if chosen < len(workMap) { + // One of the clients has produced work. Send it to the central queue. + id := workKeys[chosen] + if !ok { + // Client has closed its input channels. See if we're done. + func() { + te.clientLock.Lock() + defer te.clientLock.Unlock() + te.workMap[id] = nil + }() + if activeJobs[id] == nil { + close(te.resultsMap[id]) + } + continue + } + job := recv.Interface().(*TransferJob) + clientJob := &clientTransferJob{job: job, uuid: id} + clientJobs := activeJobs[id] + if clientJobs == nil { + clientJobs = make([]*TransferJob, 0) + } + clientJobs = append(clientJobs, job) + activeJobs[id] = clientJobs + te.work <- clientJob + } else if chosen < len(workMap)+len(resultsMap) { + // One of the "write" channels has been sent some results. + id := resultsKeys[chosen-len(workMap)] + clientJob := tmpResults[id][0] + job := clientJob.job + job.activeXfer.Add(-1) + if len(tmpResults[id]) == 1 { + // The last result back to this client has been sent; delete the + // slice from the map and check if the overall job is done. + delete(tmpResults, id) + // Test to see if the transfer job is done (true if job-to-file translation + // has completed and there are no remaining active transfers) + if job.lookupDone.Load() && job.activeXfer.Load() == 0 { + log.Debugln("Job is done") + if len(activeJobs[id]) == 1 { + // Delete the job from the list of active jobs + delete(activeJobs, id) + func() { + te.clientLock.Lock() + defer te.clientLock.Unlock() + // If the client is closed and there are no remaining + // jobs for that client, we can close the results channel + // for the client -- a clean shutdown of the client. + if te.workMap[id] == nil { + close(te.resultsMap[id]) + } + }() + } else { + // Scan through the list of active jobs, removing the recently + // completed one and saving the updated list. + newJobList := make([]*TransferJob, 0, len(activeJobs[id])) + for _, oldJob := range activeJobs[id] { + if oldJob.uuid != job.uuid { + newJobList = append(newJobList, oldJob) + } + } + activeJobs[id] = newJobList + } + } + } else { + tmpResults[id] = tmpResults[id][1:] + } + } else if chosen == len(workMap)+len(resultsMap) { + // The notify channel is meant to let the engine know + // a new client has joined. We should restart the for loop, + // recalculating the channels with the new entry. + continue + } else if chosen == len(workMap)+len(resultsMap)+1 { + // Receive transfer results from one of the engine's workers + result := recv.Interface().(*clientTransferResults) + if result == nil { + te.workersActive-- + if te.workersActive == 0 { + te.closeDoneChan <- true + close(te.closeDoneChan) + return nil + } + } else { + resultBuffer := tmpResults[result.id] + if resultBuffer == nil { + resultBuffer = make([]*TransferResults, 0) + } + tmpResults[result.id] = append(resultBuffer, &result.results) + } + } else if chosen == len(workMap)+len(resultsMap)+3 { + // Engine's context has been cancelled; immediately exit. + log.Debugln("Transfer engine has been cancelled") + close(te.closeDoneChan) + return te.ctx.Err() + } else if chosen == len(workMap)+len(resultsMap)+4 { + // Notification that a job has been processed into files (or failed) + job := recv.Interface().(*clientTransferJob) + job.job.lookupDone.Store(true) + // If no transfers were created and we have an error, the job is no + // longer active + if job.job.lookupErr != nil && job.job.totalXfer == 0 { + // Remove this job from the list of active jobs for the client. + activeJobs[job.uuid] = slices.DeleteFunc(activeJobs[job.uuid], func(oldJob *TransferJob) bool { + return oldJob.uuid == job.job.uuid + }) + if len(activeJobs[job.uuid]) == 0 { + func() { + te.clientLock.Lock() + defer te.clientLock.Unlock() + // If the client is closed and there are no remaining + // jobs for that client, we can close the results channel. + if te.workMap[job.uuid] == nil { + close(te.resultsMap[job.uuid]) + } + }() + } + } + } else if chosen == len(workMap)+len(resultsMap)+5 { + // Notification that the engine should shut down + closing = true + log.Debugln("Shutting down transfer engine") + func() { + te.clientLock.Lock() + defer te.clientLock.Unlock() + for _, channel := range te.workMap { + if channel != nil { + close(channel) + } + } + }() + } else { + // EWMA tick. + newVals := te.ewmaCtr.Swap(0) + te.ewma.Add(float64(newVals)) + te.ewmaVal.Store(int64(te.ewma.Value())) + } + } +} + +// Listen for new jobs on the engine's work queue and +// turn them into transferFile objects. +// +// Meant to be run as a standalone goroutine +func (te *TransferEngine) runJobHandler() error { + for { + select { + case <-te.ctx.Done(): + log.Debugln("Job handler has been cancelled") + return te.ctx.Err() + case job, ok := <-te.work: + if !ok { + log.Debugln("Job handler has been shutdown") + close(te.files) + return nil + } + if job.job.ctx.Err() == context.Canceled { + job.job.lookupErr = job.job.ctx.Err() + } else { + err := te.createTransferFiles(job) + job.job.lookupErr = err + } + te.jobLookupDone <- job + } + } +} + +// Create a new transfer job for the client +// +// The returned object can be further customized as desired. +// This function does not "submit" the job for execution. +func (tc *TransferClient) NewTransferJob(remoteUrl *url.URL, localPath string, upload bool, recursive bool, options ...TransferOption) (tj *TransferJob, err error) { + + id, err := uuid.NewV7() + if err != nil { + return + } - // Proxy specifies if a proxy should be used - Proxy bool + copyUrl := *remoteUrl // Make a copy of the input URL to avoid concurrent issues. + tj = &TransferJob{ + caches: tc.caches, + recursive: recursive, + localPath: localPath, + remoteURL: ©Url, + callback: tc.callback, + skipAcquire: tc.skipAcquire, + tokenLocation: tc.tokenLocation, + upload: upload, + uuid: id, + } + tj.ctx, tj.cancel = context.WithCancel(tc.ctx) + + for _, option := range options { + switch option.Ident() { + case identTransferOptionCaches{}: + tc.caches = option.Value().([]*url.URL) + case identTransferOptionCallback{}: + tc.callback = option.Value().(TransferCallbackFunc) + case identTransferOptionTokenLocation{}: + tc.tokenLocation = option.Value().(string) + case identTransferOptionAcquireToken{}: + tc.skipAcquire = !option.Value().(bool) + } + } - // Specifies the pack option in the transfer URL - PackOption string + if remoteUrl.Scheme == "pelican" && remoteUrl.Host != "" { + fd := config.GetFederation() + defer config.SetFederation(fd) + config.SetFederation(config.FederationDiscovery{}) + fedUrlCopy := *remoteUrl + fedUrlCopy.Scheme = "https" + fedUrlCopy.Path = "" + fedUrlCopy.RawFragment = "" + fedUrlCopy.RawQuery = "" + viper.Set("Federation.DiscoveryUrl", fedUrlCopy.String()) + if err = config.DiscoverFederation(); err != nil { + return + } + } else if remoteUrl.Scheme == "osdf" { + if remoteUrl.Host != "" { + remoteUrl.Path = path.Clean(path.Join("/", remoteUrl.Host, remoteUrl.Path)) + } + fd := config.GetFederation() + defer config.SetFederation(fd) + config.SetFederation(config.FederationDiscovery{}) + fedUrl := &url.URL{} + fedUrl.Scheme = "https" + fedUrl.Host = "osg-htc.org" + viper.Set("Federation.DiscoveryUrl", fedUrl.String()) + if err = config.DiscoverFederation(); err != nil { + return + } + } + + tj.useDirector = param.Federation_DirectorUrl.GetString() != "" + ns, err := getNamespaceInfo(remoteUrl.Path, param.Federation_DirectorUrl.GetString(), upload) + if err != nil { + log.Errorln(err) + err = errors.Wrapf(err, "failed to get namespace information for remote URL %s", remoteUrl) + } + tj.namespace = ns + + if upload || ns.UseTokenOnRead { + tj.token, err = getToken(remoteUrl, ns, true, "", tc.tokenLocation, !tj.skipAcquire) + if err != nil { + return nil, fmt.Errorf("failed to get token for transfer: %v", err) + } + } + return +} + +// Submit the transfer job to the client for processing +func (tc *TransferClient) Submit(tj *TransferJob) error { + // Ensure that a tj.Wait() immediately after Submit will always block. + log.Debugln("Submiting transfer job", tj.uuid.String()) + select { + case <-tc.ctx.Done(): + return tc.ctx.Err() + case tc.work <- tj: + return nil + } +} + +// Close the transfer client object +// +// Any subsequent job submissions will cause a panic +func (tc *TransferClient) Close() { + if !tc.closed { + close(tc.work) + tc.closed = true + } } -// NewTransferDetails creates the TransferDetails struct with the given cache -func NewTransferDetails(cache namespaces.Cache, opts TransferDetailsOptions) []TransferDetails { - details := make([]TransferDetails, 0) +// Shutdown the transfer client +// +// Closes the client and waits for all jobs to exit cleanly. Returns +// any results that were pending when Shutdown was called +func (tc *TransferClient) Shutdown() (results []TransferResults, err error) { + tc.Close() + results = make([]TransferResults, 0) + resultsChan := tc.Results() + for { + select { + case <-tc.ctx.Done(): + err = tc.ctx.Err() + return + case result, ok := <-resultsChan: + if !ok { + return + } + results = append(results, result) + } + } +} + +// Return a channel containing the results from the client +func (tc *TransferClient) Results() chan TransferResults { + tc.setupResults.Do(func() { + tc.finalResults = make(chan TransferResults) + go func() { + for { + select { + case <-tc.ctx.Done(): + return + case result, ok := <-tc.results: + if !ok { + close(tc.finalResults) + return + } + newResult := *result + newResult.job = nil + tc.finalResults <- newResult + } + } + }() + }) + return tc.finalResults +} + +// Cancel a client +// +// When cancelled, all channels and goroutines associated with +// the client will close/exit immediately. +func (tc *TransferClient) Cancel() { + tc.cancel() +} + +// Cancel the transfer job +func (tj *TransferJob) Cancel() { + tj.cancel() +} + +// Get the transfer's ID +func (tj *TransferJob) ID() string { + return tj.uuid.String() +} + +// newTransferDetails creates the TransferDetails struct with the given cache +func newTransferDetails(cache namespaces.Cache, opts transferDetailsOptions) []transferAttemptDetails { + details := make([]transferAttemptDetails, 0) var cacheEndpoint string if opts.NeedsToken { cacheEndpoint = cache.AuthEndpoint @@ -200,40 +896,41 @@ func NewTransferDetails(cache namespaces.Cache, opts TransferDetailsOptions) []T cacheURL.Scheme = "" cacheURL.Opaque = "" } - log.Debugf("Parsed Cache: %s\n", cacheURL.String()) + log.Debugf("Parsed Cache: %s", cacheURL.String()) if opts.NeedsToken { cacheURL.Scheme = "https" - if !HasPort(cacheURL.Host) { + if !hasPort(cacheURL.Host) { // Add port 8444 and 8443 - cacheURL.Host += ":8444" - details = append(details, TransferDetails{ - Url: *cacheURL, + urlCopy := *cacheURL + urlCopy.Host += ":8444" + details = append(details, transferAttemptDetails{ + Url: &urlCopy, Proxy: false, PackOption: opts.PackOption, }) // Strip the port off and add 8443 - cacheURL.Host = cacheURL.Host[:len(cacheURL.Host)-5] + ":8443" + cacheURL.Host = cacheURL.Host + ":8443" } // Whether port is specified or not, add a transfer without proxy - details = append(details, TransferDetails{ - Url: *cacheURL, + details = append(details, transferAttemptDetails{ + Url: cacheURL, Proxy: false, PackOption: opts.PackOption, }) } else { cacheURL.Scheme = "http" - if !HasPort(cacheURL.Host) { + if !hasPort(cacheURL.Host) { cacheURL.Host += ":8000" } - isProxyEnabled := IsProxyEnabled() - details = append(details, TransferDetails{ - Url: *cacheURL, + isProxyEnabled := isProxyEnabled() + details = append(details, transferAttemptDetails{ + Url: cacheURL, Proxy: isProxyEnabled, PackOption: opts.PackOption, }) if isProxyEnabled && CanDisableProxy() { - details = append(details, TransferDetails{ - Url: *cacheURL, + details = append(details, transferAttemptDetails{ + Url: cacheURL, Proxy: false, PackOption: opts.PackOption, }) @@ -243,246 +940,248 @@ func NewTransferDetails(cache namespaces.Cache, opts TransferDetailsOptions) []T return details } -type TransferResults struct { - Error error - TransferredBytes int64 - Attempts []Attempt -} - -type Attempt struct { - Number int // indicates which attempt this is - TransferFileBytes int64 // how much each attempt downloaded - TimeToFirstByte int64 // how long it took to download the first byte - TransferEndTime int64 // when the transfer ends - Endpoint string // which origin did it use - ServerVersion string // TODO: figure out how to get this??? - Error error // what error the attempt returned (if any) -} - -type TransferDetailsOptions struct { - NeedsToken bool - PackOption string -} - type CacheInterface interface{} -func GenerateTransferDetailsUsingCache(cache CacheInterface, opts TransferDetailsOptions) []TransferDetails { +func GenerateTransferDetailsUsingCache(cache CacheInterface, opts transferDetailsOptions) []transferAttemptDetails { if directorCache, ok := cache.(namespaces.DirectorCache); ok { return NewTransferDetailsUsingDirector(directorCache, opts) } else if cache, ok := cache.(namespaces.Cache); ok { - return NewTransferDetails(cache, opts) + return newTransferDetails(cache, opts) } return nil } -func download_http(sourceUrl *url.URL, destination string, payload *payloadStruct, namespace namespaces.Namespace, recursive bool, tokenName string) (transferResults []TransferResults, err error) { +// Take a transfer job and produce one or more transfer file requests. +// The transfer file requests are sent to be processed via the engine +func (te *TransferEngine) createTransferFiles(job *clientTransferJob) (err error) { // First, create a handler for any panics that occur defer func() { if r := recover(); r != nil { - log.Errorln("Panic occurred in download_http:", r) - ret := fmt.Sprintf("Unrecoverable error (panic) occurred in download_http: %v", r) + log.Errorln("Panic occurred in createTransferFiles:", r) + ret := fmt.Sprintf("Unrecoverable error (panic) occurred in createTransferFiles: %v", r) err = errors.New(ret) - - // Attempt to add the panic to the error accumulator - AddError(errors.New(ret)) } }() - packOption := sourceUrl.Query().Get("pack") + packOption := job.job.remoteURL.Query().Get("pack") if packOption != "" { log.Debugln("Will use unpack option value", packOption) } - sourceUrl = &url.URL{Path: sourceUrl.Path} - - var token string - if namespace.UseTokenOnRead { - var err error - token, err = getToken(sourceUrl, namespace, false, tokenName) - if err != nil { - log.Errorln("Failed to get token though required to read from this namespace:", err) - return nil, err - } - } - - // Check the env var "USE_OSDF_DIRECTOR" and decide if ordered caches should come from director - var transfers []TransferDetails - var files []string - directorUrl := param.Federation_DirectorUrl.GetString() - closestNamespaceCaches, err := GetCachesFromNamespace(namespace, directorUrl != "") - if err != nil { - log.Errorln("Failed to get namespaced caches (treated as non-fatal):", err) - } - - log.Debugln("Matched caches:", closestNamespaceCaches) - - // Make sure we only try as many caches as we have - cachesToTry := CachesToTry - if cachesToTry > len(closestNamespaceCaches) { - cachesToTry = len(closestNamespaceCaches) - } - log.Debugln("Trying the caches:", closestNamespaceCaches[:cachesToTry]) + remoteUrl := &url.URL{Path: job.job.remoteURL.Path} - if recursive { - var err error - files, err = walkDavDir(sourceUrl, namespace, token, "", false) + var transfers []transferAttemptDetails + if job.job.upload { // Uploads use the redirected endpoint directly + endpoint, err := url.Parse(job.job.namespace.WriteBackHost) if err != nil { - log.Errorln("Error from walkDavDir", err) - return nil, err + return errors.Wrap(err, "Invalid URL returned by director for PUT") } - } else { - files = append(files, sourceUrl.Path) - } - - for _, cache := range closestNamespaceCaches[:cachesToTry] { - // Parse the cache URL - log.Debugln("Cache:", cache) - td := TransferDetailsOptions{ - NeedsToken: namespace.ReadHTTPS || namespace.UseTokenOnRead, + transfers = append(transfers, transferAttemptDetails{ + Url: endpoint, PackOption: packOption, + }) + } else { + var closestNamespaceCaches []CacheInterface + closestNamespaceCaches, err = getCachesFromNamespace(job.job.namespace, job.job.useDirector, job.job.caches) + if err != nil { + log.Errorln("Failed to get namespaced caches (treated as non-fatal):", err) } - transfers = append(transfers, GenerateTransferDetailsUsingCache(cache, td)...) - } - if len(transfers) > 0 { - log.Debugln("Transfers:", transfers[0].Url.Opaque) - } else { - log.Debugln("No transfers possible as no caches are found") - return nil, errors.New("No transfers possible as no caches are found") - } - // Create the wait group and the transfer files - var wg sync.WaitGroup + log.Debugln("Matched caches:", closestNamespaceCaches) - workChan := make(chan string) - results := make(chan TransferResults, len(files)) - //tf := TransferFiles{files: files} + // Make sure we only try as many caches as we have + cachesToTry := CachesToTry + if cachesToTry > len(closestNamespaceCaches) { + cachesToTry = len(closestNamespaceCaches) + } + log.Debugln("Trying the caches:", closestNamespaceCaches[:cachesToTry]) + + for _, cache := range closestNamespaceCaches[:cachesToTry] { + // Parse the cache URL + log.Debugln("Cache:", cache) + td := transferDetailsOptions{ + NeedsToken: job.job.namespace.ReadHTTPS || job.job.namespace.UseTokenOnRead, + PackOption: packOption, + } + transfers = append(transfers, GenerateTransferDetailsUsingCache(cache, td)...) + } - if ObjectClientOptions.Recursive && ObjectClientOptions.ProgressBars { - log.SetOutput(getProgressContainer()) - } - // Start the workers - for i := 1; i <= 5; i++ { - wg.Add(1) - go startDownloadWorker(sourceUrl.Path, destination, token, transfers, payload, &wg, workChan, results) + if len(transfers) > 0 { + log.Debugln("Transfers:", transfers[0].Url) + } else { + log.Debugln("No transfers possible as no caches are found") + err = errors.New("No transfers possible as no caches are found") + return + } } - // For each file, send it to the worker - for _, file := range files { - workChan <- file + if job.job.recursive { + if job.job.upload { + return te.walkDirUpload(job, transfers, te.files, job.job.localPath) + } else { + return te.walkDirDownload(job, transfers, te.files, remoteUrl) + } } - close(workChan) - // Wait for all the transfers to complete - wg.Wait() + job.job.totalXfer += 1 + job.job.activeXfer.Add(1) + te.files <- &clientTransferFile{ + uuid: job.uuid, + jobId: job.job.uuid, + file: &transferFile{ + ctx: job.job.ctx, + callback: job.job.callback, + job: job.job, + engine: te, + remoteURL: remoteUrl, + packOption: packOption, + localPath: job.job.localPath, + upload: job.job.upload, + token: job.job.token, + attempts: transfers, + project: job.job.project, + }, + } + + return +} - var downloadError error = nil - // Every transfer should send a TransferResults to the results channel - for i := 0; i < len(files); i++ { +// Start a transfer worker in the current goroutine. +// The transfer workers read in transfers to perform on `workChan` and write out +// the results of the transfer attempt on `results`. +func runTransferWorker(ctx context.Context, workChan <-chan *clientTransferFile, results chan<- *clientTransferResults) error { + for { select { - case result := <-results: - transferResults = append(transferResults, result) - if result.Error != nil { - downloadError = result.Error + case <-ctx.Done(): + return ctx.Err() + case file, ok := <-workChan: + if !ok { + // If the transfer engine is cancelled while a shutdown is occurring, the + // write to the results channel may block. Hence, we should see if we're + // cancelled while the write is pending. + select { + case <-ctx.Done(): + return ctx.Err() + case results <- nil: + return nil + } + } + if file.file.ctx.Err() == context.Canceled { + results <- &clientTransferResults{ + id: file.uuid, + results: TransferResults{ + JobId: file.jobId, + Error: file.file.ctx.Err(), + }, + } + break + } + if file.file.err != nil { + results <- &clientTransferResults{ + id: file.uuid, + results: TransferResults{ + JobId: file.jobId, + Error: file.file.err, + }, + } + break } - default: - // Didn't get a result, that's weird - downloadError = errors.New("failed to get outputs from one of the transfers") + var err error + var transferResults TransferResults + if file.file.upload { + transferResults, err = uploadObject(file.file) + } else { + transferResults, err = downloadObject(file.file) + } + transferResults.JobId = file.jobId + if err != nil { + log.Errorf("Error when attempting to transfer object %s for client %s", file.file.remoteURL, file.uuid.String()) + transferResults = newTransferResults(file.file.job) + transferResults.Error = err + } else if transferResults.Error == nil { + xferErrors := NewTransferErrors() + lastXferGood := false + for _, attempt := range transferResults.Attempts { + if attempt.Error == nil { + lastXferGood = true + } else { + xferErrors.AddError(attempt.Error) + } + } + if !lastXferGood { + transferResults.Error = xferErrors + } + } + results <- &clientTransferResults{id: file.uuid, results: transferResults} } } - // Make sure to close the progressContainer after all download complete - if ObjectClientOptions.Recursive && ObjectClientOptions.ProgressBars { - getProgressContainer().Wait() - log.SetOutput(os.Stdout) - } - return transferResults, downloadError - } -func startDownloadWorker(source string, destination string, token string, transfers []TransferDetails, payload *payloadStruct, wg *sync.WaitGroup, workChan <-chan string, results chan<- TransferResults) { - - defer wg.Done() - var success bool - var attempts []Attempt - for file := range workChan { - // Remove the source from the file path - newFile := strings.Replace(file, source, "", 1) - finalDest := path.Join(destination, newFile) - directory := path.Dir(finalDest) - var downloaded int64 - err := os.MkdirAll(directory, 0700) - if err != nil { - results <- TransferResults{Error: errors.New("Failed to make directory:" + directory)} - continue - } - for idx, transfer := range transfers { // For each transfer (usually 3), populate each attempt given - var attempt Attempt - var timeToFirstByte int64 - var serverVersion string - attempt.Number = idx // Start with 0 - attempt.Endpoint = transfer.Url.Host - transfer.Url.Path = file - log.Debugln("Constructed URL:", transfer.Url.String()) - if downloaded, timeToFirstByte, serverVersion, err = DownloadHTTP(transfer, finalDest, token, payload); err != nil { - log.Debugln("Failed to download:", err) - transferEndTime := time.Now().Unix() - var ope *net.OpError - var cse *ConnectionSetupError - errorString := "Failed to download from " + transfer.Url.Hostname() + ":" + - transfer.Url.Port() + " " - if errors.As(err, &ope) && ope.Op == "proxyconnect" { - log.Debugln(ope) - AddrString, _ := os.LookupEnv("http_proxy") - if ope.Addr != nil { - AddrString = " " + ope.Addr.String() - } - errorString += "due to proxy " + AddrString + " error: " + ope.Unwrap().Error() - } else if errors.As(err, &cse) { - errorString += "+ proxy=" + strconv.FormatBool(transfer.Proxy) + ": " - if sce, ok := cse.Unwrap().(grab.StatusCodeError); ok { - errorString += sce.Error() - } else { - errorString += err.Error() - } +func downloadObject(transfer *transferFile) (transferResults TransferResults, err error) { + log.Debugln("Downloading file from", transfer.remoteURL) + // Remove the source from the file path + directory := path.Dir(transfer.localPath) + var downloaded int64 + if err = os.MkdirAll(directory, 0700); err != nil { + return + } + transferResults = newTransferResults(transfer.job) + success := false + for idx, transferEndpoint := range transfer.attempts { // For each transfer (usually 3), populate each attempt given + var attempt TransferResult + var timeToFirstByte int64 + var serverVersion string + attempt.Number = idx // Start with 0 + attempt.Endpoint = transferEndpoint.Url.Host + transferEndpoint.Url.Path = transfer.remoteURL.Path + if downloaded, timeToFirstByte, serverVersion, err = downloadHTTP(transfer.ctx, transfer.engine, transfer.callback, transferEndpoint, transfer.localPath, transfer.token, &transfer.accounting); err != nil { + log.Debugln("Failed to download:", err) + transferEndTime := time.Now().Unix() + var ope *net.OpError + var cse *ConnectionSetupError + errorString := "Failed to download from " + transferEndpoint.Url.Hostname() + ":" + + transferEndpoint.Url.Port() + " " + if errors.As(err, &ope) && ope.Op == "proxyconnect" { + log.Debugln(ope) + AddrString, _ := os.LookupEnv("http_proxy") + if ope.Addr != nil { + AddrString = " " + ope.Addr.String() + } + errorString += "due to proxy " + AddrString + " error: " + ope.Unwrap().Error() + } else if errors.As(err, &cse) { + errorString += "+ proxy=" + strconv.FormatBool(transferEndpoint.Proxy) + ": " + if sce, ok := cse.Unwrap().(grab.StatusCodeError); ok { + errorString += sce.Error() } else { - errorString += "+ proxy=" + strconv.FormatBool(transfer.Proxy) + - ": " + err.Error() + errorString += err.Error() } - AddError(&FileDownloadError{errorString, err}) - attempt.TransferFileBytes = downloaded - attempt.TimeToFirstByte = timeToFirstByte - attempt.Error = errors.New(errorString) - attempt.TransferEndTime = int64(transferEndTime) - attempt.ServerVersion = serverVersion - attempts = append(attempts, attempt) - continue } else { - transferEndTime := time.Now().Unix() - attempt.TransferEndTime = int64(transferEndTime) - attempt.TimeToFirstByte = timeToFirstByte - attempt.TransferFileBytes = downloaded - attempt.ServerVersion = serverVersion - log.Debugln("Downloaded bytes:", downloaded) - attempts = append(attempts, attempt) - success = true - break - } - - } - if !success { - log.Debugln("Failed to download with HTTP") - results <- TransferResults{ - TransferredBytes: downloaded, - Error: errors.New("failed to download with HTTP"), - Attempts: attempts, - } - return - } else { - results <- TransferResults{ - TransferredBytes: downloaded, - Error: nil, - Attempts: attempts, + errorString += "+ proxy=" + strconv.FormatBool(transferEndpoint.Proxy) + + ": " + err.Error() } + attempt.TransferFileBytes = downloaded + attempt.TimeToFirstByte = timeToFirstByte + attempt.Error = errors.New(errorString) + attempt.TransferEndTime = int64(transferEndTime) + attempt.ServerVersion = serverVersion + transferResults.Attempts = append(transferResults.Attempts, attempt) + } else { // Success + transferEndTime := time.Now().Unix() + attempt.TransferEndTime = int64(transferEndTime) + attempt.TimeToFirstByte = timeToFirstByte + attempt.TransferFileBytes = downloaded + attempt.ServerVersion = serverVersion + log.Debugln("Downloaded bytes:", downloaded) + transferResults.Attempts = append(transferResults.Attempts, attempt) + success = true + break } } + transferResults.TransferredBytes = downloaded + if !success { + log.Debugln("Failed to download with HTTP") + transferResults.Error = errors.New("failed to download with HTTP") + } + return } func parseTransferStatus(status string) (int, string) { @@ -499,9 +1198,32 @@ func parseTransferStatus(status string) (int, string) { return statusCode, strings.TrimSpace(parts[1]) } -// DownloadHTTP - Perform the actual download of the file -// Returns: downloaded size, time to 1st byte downloaded, serverVersion and an error if there is one -func DownloadHTTP(transfer TransferDetails, dest string, token string, payload *payloadStruct) (int64, int64, string, error) { +// Perform the actual download of the file +// +// Returns the downloaded size, time to 1st byte downloaded, serverVersion and an error if there is one +func downloadHTTP(ctx context.Context, te *TransferEngine, callback TransferCallbackFunc, transfer transferAttemptDetails, dest string, token string, payload *payloadStruct) (downloaded int64, timeToFirstByte int64, serverVersion string, err error) { + defer func() { + if r := recover(); r != nil { + log.Errorln("Panic occurred in downloadHTTP:", r) + ret := fmt.Sprintf("Unrecoverable error (panic) occurred in downloadHTTP: %v", r) + err = errors.New(ret) + } + }() + + var totalSize int64 = 0 + + lastUpdate := time.Now() + if callback != nil { + callback(dest, 0, 0, false) + } + defer func() { + if callback != nil { + callback(dest, downloaded, totalSize, true) + } + if te != nil { + te.ewmaCtr.Add(int64(time.Since(lastUpdate))) + } + }() // Create the client, request, and context client := grab.NewClient() @@ -515,11 +1237,10 @@ func DownloadHTTP(transfer TransferDetails, dest string, token string, payload * } httpClient.Transport = transport - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) defer cancel() log.Debugln("Transfer URL String:", transfer.Url.String()) var req *grab.Request - var err error var unpacker *autoUnpacker if transfer.PackOption != "" { behavior, err := GetBehavior(transfer.PackOption) @@ -547,120 +1268,101 @@ func DownloadHTTP(transfer TransferDetails, dest string, token string, payload * req.HTTPRequest.Header.Set("X-Transfer-Status", "true") req.HTTPRequest.Header.Set("TE", "trailers") if payload != nil && payload.ProjectName != "" { - req.HTTPRequest.Header.Set("User-Agent", payload.ProjectName) + req.HTTPRequest.Header.Set("User-Agent", getUserAgent(payload.ProjectName)) } - req.WithContext(ctx) + req = req.WithContext(ctx) // Test the transfer speed every 5 seconds t := time.NewTicker(5000 * time.Millisecond) defer t.Stop() // Progress ticker - progressTicker := time.NewTicker(500 * time.Millisecond) + progressTicker := time.NewTicker(100 * time.Millisecond) defer progressTicker.Stop() downloadLimit := param.Client_MinimumDownloadSpeed.GetInt() - // If we are doing a recursive, decrease the download limit by the number of likely workers ~5 - if ObjectClientOptions.Recursive { - downloadLimit /= 5 - } - // Start the transfer log.Debugln("Starting the HTTP transfer...") - filename := path.Base(dest) resp := client.Do(req) downloadStart := time.Now() // Check the error real quick if resp.IsComplete() { - if err := resp.Err(); err != nil { + if err = resp.Err(); err != nil { if errors.Is(err, grab.ErrBadLength) { err = fmt.Errorf("Local copy of file is larger than remote copy %w", grab.ErrBadLength) } log.Errorln("Failed to download:", err) - return 0, 0, "", &ConnectionSetupError{Err: err} + err = &ConnectionSetupError{Err: err} + return } } - serverVersion := resp.HTTPResponse.Header.Get("Server") + serverVersion = resp.HTTPResponse.Header.Get("Server") // Size of the download - contentLength := resp.Size() + totalSize = resp.Size() // Do a head request for content length if resp.Size is unknown - if contentLength <= 0 && ObjectClientOptions.ProgressBars { + if totalSize <= 0 { headClient := &http.Client{Transport: config.GetTransport()} headRequest, _ := http.NewRequest("HEAD", transfer.Url.String(), nil) - headResponse, err := headClient.Do(headRequest) + var headResponse *http.Response + headResponse, err = headClient.Do(headRequest) if err != nil { log.Errorln("Could not successfully get response for HEAD request") - return 0, 0, serverVersion, errors.Wrap(err, "Could not determine the size of the remote object") + err = errors.Wrap(err, "Could not determine the size of the remote object") + return } - defer headResponse.Body.Close() + headResponse.Body.Close() contentLengthStr := headResponse.Header.Get("Content-Length") - contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64) - if err != nil { - log.Errorln("problem converting content-length to an int", err) - contentLength = resp.Size() + if contentLengthStr != "" { + totalSize, err = strconv.ParseInt(contentLengthStr, 10, 64) + if err != nil { + log.Errorln("problem converting content-length to an int:", err) + totalSize = 0 + } } } - - var progressBar *mpb.Bar - if ObjectClientOptions.ProgressBars { - progressBar = getProgressContainer().AddBar(0, - mpb.PrependDecorators( - decor.Name(filename, decor.WCSyncSpaceR), - decor.CountersKibiByte("% .2f / % .2f"), - ), - mpb.AppendDecorators( - decor.OnComplete(decor.EwmaETA(decor.ET_STYLE_GO, 90), ""), - decor.OnComplete(decor.Name(" ] "), ""), - decor.OnComplete(decor.EwmaSpeed(decor.SizeB1024(0), "% .2f", 5), "Done!"), - ), - ) + if callback != nil { + callback(dest, 0, totalSize, false) } stoppedTransferTimeout := int64(param.Client_StoppedTransferTimeout.GetInt()) slowTransferRampupTime := int64(param.Client_SlowTransferRampupTime.GetInt()) slowTransferWindow := int64(param.Client_SlowTransferWindow.GetInt()) - var previousCompletedBytes int64 = 0 var startBelowLimit int64 = 0 - var previousCompletedTime = time.Now() var noProgressStartTime time.Time var lastBytesComplete int64 - var timeToFirstByte int64 timeToFirstByteRecorded := false // Loop of the download Loop: for { select { case <-progressTicker.C: - if !timeToFirstByteRecorded && resp.BytesComplete() > 1 { + downloaded = resp.BytesComplete() + if !timeToFirstByteRecorded && downloaded > 1 { timeToFirstByte = int64(time.Since(downloadStart)) } - if ObjectClientOptions.ProgressBars { - progressBar.SetTotal(contentLength, false) - currentCompletedBytes := resp.BytesComplete() - bytesDelta := currentCompletedBytes - previousCompletedBytes - previousCompletedBytes = currentCompletedBytes - currentCompletedTime := time.Now() - timeElapsed := currentCompletedTime.Sub(previousCompletedTime) - progressBar.EwmaIncrInt64(bytesDelta, timeElapsed) - previousCompletedTime = currentCompletedTime + currentTime := time.Now() + if te != nil { + te.ewmaCtr.Add(int64(currentTime.Sub(lastUpdate))) + } + lastUpdate = currentTime + if callback != nil { + callback(dest, downloaded, totalSize, false) } case <-t.C: // Check that progress is being made and that it is not too slow - if resp.BytesComplete() == lastBytesComplete { + downloaded = resp.BytesComplete() + if downloaded == lastBytesComplete { if noProgressStartTime.IsZero() { noProgressStartTime = time.Now() } else if time.Since(noProgressStartTime) > time.Duration(stoppedTransferTimeout)*time.Second { errMsg := "No progress for more than " + time.Since(noProgressStartTime).Truncate(time.Millisecond).String() log.Errorln(errMsg) - if ObjectClientOptions.ProgressBars { - progressBar.Abort(true) - progressBar.Wait() - } - return 5, timeToFirstByte, serverVersion, &StoppedTransferError{ + err = &StoppedTransferError{ Err: errMsg, } + return } } else { noProgressStartTime = time.Time{} @@ -668,7 +1370,15 @@ Loop: lastBytesComplete = resp.BytesComplete() // Check if we are downloading fast enough - if resp.BytesPerSecond() < float64(downloadLimit) { + limit := float64(downloadLimit) + var concurrency float64 = 1 + if te != nil { + concurrency = float64(te.ewmaVal.Load()) / float64(ewmaInterval) + } + if concurrency > 1 { + limit /= concurrency + } + if resp.BytesPerSecond() < limit { // Give the download `slowTransferRampupTime` (default 120) seconds to start if resp.Duration() < time.Second*time.Duration(slowTransferRampupTime) { continue @@ -687,19 +1397,16 @@ Loop: } // The download is below the threshold for more than `SlowTransferWindow` seconds, cancel the download cancel() - if ObjectClientOptions.ProgressBars { - progressBar.Abort(true) - progressBar.Wait() - } log.Errorln("Cancelled: Download speed of ", resp.BytesPerSecond(), "bytes/s", " is below the limit of", downloadLimit, "bytes/s") - return 0, timeToFirstByte, serverVersion, &SlowTransferError{ + err = &SlowTransferError{ BytesTransferred: resp.BytesComplete(), BytesPerSecond: int64(resp.BytesPerSecond()), Duration: resp.Duration(), - BytesTotal: contentLength, + BytesTotal: totalSize, } + return } else { // The download is fast enough, reset the startBelowLimit @@ -707,38 +1414,21 @@ Loop: } case <-resp.Done: - // download is complete - if ObjectClientOptions.ProgressBars { - downloadError := resp.Err() - if downloadError != nil { - log.Errorln(downloadError.Error()) - progressBar.Abort(true) - progressBar.Wait() - } else { - progressBar.SetTotal(contentLength, true) - // call wait here for the bar to complete and flush - // If recursive, we still want to use container so keep it open - if ObjectClientOptions.Recursive { - progressBar.Wait() - } else { // Otherwise just close it - getProgressContainer().Wait() - } - } - } + downloaded = resp.BytesComplete() break Loop } } - //fmt.Printf("\nDownload saved to", resp.Filename) err = resp.Err() if err != nil { // Connection errors if errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNABORTED) { - return 0, 0, "", &ConnectionSetupError{URL: resp.Request.URL().String()} + err = &ConnectionSetupError{URL: resp.Request.URL().String()} + return } log.Debugln("Got error from HTTP download", err) - return 0, 0, serverVersion, err + return } else { // Check the trailers for any error information trailer := resp.HTTPResponse.Trailer @@ -746,7 +1436,8 @@ Loop: statusCode, statusText := parseTransferStatus(errorStatus) if statusCode != 200 { log.Debugln("Got error from file transfer") - return 0, 0, serverVersion, errors.New("transfer error: " + statusText) + err = errors.New("transfer error: " + statusText) + return } } } @@ -760,13 +1451,13 @@ Loop: if unpacker != nil { unpacker.Close() - if err := unpacker.Error(); err != nil { - return 0, 0, serverVersion, err + if err = unpacker.Error(); err != nil { + return } } log.Debugln("HTTP Transfer was successful") - return resp.BytesComplete(), timeToFirstByte, serverVersion, nil + return } type Sizer interface { @@ -820,82 +1511,35 @@ func (pr *ProgressReader) Size() int64 { return pr.sizer.Size() } -// Recursively uploads a directory with all files and nested dirs, keeping file structure on server side -func UploadDirectory(src string, dest *url.URL, token string, namespace namespaces.Namespace, projectName string) (transferResults []TransferResults, err error) { - var files []string - srcUrl := url.URL{Path: src} - // Get the list of files as well as make any directories on the server end - files, err = walkDavDir(&srcUrl, namespace, token, dest.Path, true) - if err != nil { - return nil, err - } - - if ObjectClientOptions.ProgressBars { - log.SetOutput(getProgressContainer()) - } - var transfer TransferResults - - // Check if there is a directory specified in dest, if not then use the local one - cleanDest := path.Clean(dest.Path) - destComponents := strings.Split(cleanDest, "/") - destDirSpecified := false - - if len(destComponents) == 2 { - log.Debugln("No directory specified in destination, using local directory name") - } else { - destDirSpecified = true - } +// Upload a single object to the origin +func uploadObject(transfer *transferFile) (transferResult TransferResults, err error) { + log.Debugln("Uploading file to destination", transfer.remoteURL) + transferResult.job = transfer.job - // Upload all of our files within the proper directories - for _, file := range files { - tempDest := url.URL{} - - if destDirSpecified { - destFile := strings.TrimPrefix(file, src) - tempDest.Path, err = url.JoinPath(dest.Path, destFile) - } else { - destFile := strings.TrimPrefix(file, src) - tempDest.Path, err = url.JoinPath(dest.Path, path.Base(src), destFile) - } - - if err != nil { - return nil, err - } - transfer, err = UploadFile(file, &tempDest, token, namespace, projectName) - if err != nil { - return nil, err - } - // Add info from each transfer to transferResults - transferResults = append(transferResults, transfer) - } - // Close progress bar container - if ObjectClientOptions.ProgressBars { - getProgressContainer().Wait() - log.SetOutput(os.Stdout) + var sizer Sizer = &ConstantSizer{size: 0} + var downloaded int64 = 0 + if transfer.callback != nil { + transfer.callback(transfer.localPath, downloaded, sizer.Size(), false) + defer func() { + transfer.callback(transfer.localPath, downloaded, sizer.Size(), true) + }() } - return transferResults, err -} -// UploadFile Uploads a file using HTTP -func UploadFile(src string, origDest *url.URL, token string, namespace namespaces.Namespace, projectName string) (transferResult TransferResults, err error) { - log.Debugln("In UploadFile") - log.Debugln("Dest", origDest.String()) - var attempt Attempt + var attempt TransferResult // Stat the file to get the size (for progress bar) - fileInfo, err := os.Stat(src) + fileInfo, err := os.Stat(transfer.localPath) if err != nil { - log.Errorln("Error checking local file ", src, ":", err) + log.Errorln("Error checking local file ", transfer.localPath, ":", err) transferResult.Error = err return transferResult, err } var ioreader io.ReadCloser - var sizer Sizer - pack := origDest.Query().Get("pack") nonZeroSize := true + pack := transfer.packOption if pack != "" { if !fileInfo.IsDir() { - err = errors.Errorf("Upload with pack=%v only works when input (%v) is a directory", pack, src) + err = errors.Errorf("Upload with pack=%v only works when input (%v) is a directory", pack, transfer.localPath) transferResult.Error = err return transferResult, err } @@ -907,12 +1551,12 @@ func UploadFile(src string, origDest *url.URL, token string, namespace namespace if behavior == autoBehavior { behavior = defaultBehavior } - ap := newAutoPacker(src, behavior) + ap := newAutoPacker(transfer.localPath, behavior) ioreader = ap sizer = ap } else { // Try opening the file to send - file, err := os.Open(src) + file, err := os.Open(transfer.localPath) if err != nil { log.Errorln("Error opening local file:", err) transferResult.Error = err @@ -922,18 +1566,17 @@ func UploadFile(src string, origDest *url.URL, token string, namespace namespace sizer = &ConstantSizer{size: fileInfo.Size()} nonZeroSize = fileInfo.Size() > 0 } + if transfer.callback != nil { + transfer.callback(transfer.localPath, downloaded, sizer.Size(), false) + } // Parse the writeback host as a URL - writebackhostUrl, err := url.Parse(namespace.WriteBackHost) - if err != nil { - transferResult.Error = err - return transferResult, err - } + writebackhostUrl := transfer.attempts[0].Url dest := &url.URL{ Host: writebackhostUrl.Host, Scheme: "https", - Path: origDest.Path, + Path: transfer.remoteURL.Path, } attempt.Endpoint = dest.Host // Create the wrapped reader and send it to the request @@ -941,7 +1584,7 @@ func UploadFile(src string, origDest *url.URL, token string, namespace namespace errorChan := make(chan error, 1) responseChan := make(chan *http.Response) reader := &ProgressReader{ioreader, sizer, closed} - putContext, cancel := context.WithCancel(context.Background()) + putContext, cancel := context.WithCancel(transfer.ctx) defer cancel() log.Debugln("Full destination URL:", dest.String()) var request *http.Request @@ -957,46 +1600,16 @@ func UploadFile(src string, origDest *url.URL, token string, namespace namespace return transferResult, err } // Set the authorization header - request.Header.Set("Authorization", "Bearer "+token) - if projectName != "" { - request.Header.Set("User-Agent", projectName) - } + request.Header.Set("Authorization", "Bearer "+transfer.token) + request.Header.Set("User-Agent", getUserAgent(transfer.project)) var lastKnownWritten int64 t := time.NewTicker(20 * time.Second) defer t.Stop() - go doPut(request, responseChan, errorChan) + go runPut(request, responseChan, errorChan) uploadStart := time.Now() var lastError error = nil - var progressBar *mpb.Bar - if ObjectClientOptions.ProgressBars { - progressBar = getProgressContainer().AddBar(0, - mpb.PrependDecorators( - decor.Name(src, decor.WCSyncSpaceR), - decor.CountersKibiByte("% .2f / % .2f"), - ), - mpb.AppendDecorators( - decor.OnComplete(decor.EwmaETA(decor.ET_STYLE_GO, 90), ""), - decor.OnComplete(decor.Name(" ] "), ""), - decor.OnComplete(decor.EwmaSpeed(decor.SizeB1024(0), "% .2f", 5), "Done!"), - ), - ) - // Shutdown progress bar at the end of the function - defer func() { - if lastError == nil { - progressBar.SetTotal(reader.Size(), true) - } else { - progressBar.Abort(true) - } - // If it is recursive, we need to reuse the mpb instance. Closed later - if ObjectClientOptions.Recursive { - progressBar.Wait() - } else { // If not recursive, go ahead and close it - getProgressContainer().Wait() - } - }() - } - tickerDuration := 500 * time.Millisecond + tickerDuration := 100 * time.Millisecond progressTicker := time.NewTicker(tickerDuration) firstByteRecorded := false defer progressTicker.Stop() @@ -1006,14 +1619,13 @@ Loop: for { select { case <-progressTicker.C: + if transfer.callback != nil { + transfer.callback(transfer.localPath, reader.BytesComplete(), sizer.Size(), false) + } if !firstByteRecorded && reader.BytesComplete() > 0 { attempt.TimeToFirstByte = int64(time.Since(uploadStart)) firstByteRecorded = true } - if progressBar != nil { - progressBar.SetTotal(reader.Size(), false) - progressBar.EwmaSetCurrent(reader.BytesComplete(), tickerDuration) - } case <-t.C: // If we are not making any progress, if we haven't written 1MB in the last 5 seconds @@ -1070,8 +1682,11 @@ Loop: } -// Actually perform the Put request to the server -func doPut(request *http.Request, responseChan chan<- *http.Response, errorChan chan<- error) { +// Actually perform the HTTP PUT request to the server. +// +// This is executed in a separate goroutine to allow periodic progress callbacks +// to be created within the main goroutine. +func runPut(request *http.Request, responseChan chan<- *http.Response, errorChan chan<- error) { var UploadClient = &http.Client{Transport: config.GetTransport()} client := UploadClient dump, _ := httputil.DumpRequestOut(request, false) @@ -1099,111 +1714,148 @@ func doPut(request *http.Request, responseChan chan<- *http.Response, errorChan } -func walkDavDir(url *url.URL, namespace namespaces.Namespace, token string, destPath string, upload bool) ([]string, error) { - +// Walk a remote directory in a WebDAV server, emitting the files discovered +func (te *TransferEngine) walkDirDownload(job *clientTransferJob, transfers []transferAttemptDetails, files chan *clientTransferFile, url *url.URL) error { // Create the client to walk the filesystem rootUrl := *url - if namespace.DirListHost != "" { + if job.job.namespace.DirListHost != "" { // Parse the dir list host - dirListURL, err := url.Parse(namespace.DirListHost) + dirListURL, err := url.Parse(job.job.namespace.DirListHost) if err != nil { log.Errorln("Failed to parse dirlisthost from namespaces into URL:", err) - return nil, err + return err } rootUrl = *dirListURL } else { log.Errorln("Host for directory listings is unknown") - return nil, errors.New("Host for directory listings is unknown") + return errors.New("Host for directory listings is unknown") } log.Debugln("Dir list host: ", rootUrl.String()) - auth := &bearerAuth{token: token} - c := gowebdav.NewAuthClient(rootUrl.String(), auth) + auth := &bearerAuth{token: job.job.token} + client := gowebdav.NewAuthClient(rootUrl.String(), auth) // XRootD does not like keep alives and kills things, so turn them off. transport := config.GetTransport() - c.SetTransport(transport) - var files []string - var err error - if upload { - files, err = walkDirUpload(url.Path, c, destPath) - } else { - files, err = walkDir(url.Path, c) - } - log.Debugln("Found files:", files) - return files, err - + client.SetTransport(transport) + return te.walkDirDownloadHelper(job, transfers, files, url.Path, client) } -// For uploads, we want to make directories on the server end -func walkDirUpload(path string, client *gowebdav.Client, destPath string) ([]string, error) { - // List of files to return - var files []string - - // Get our list of files - infos, err := os.ReadDir(path) +// Helper function for the `walkDirDownload`. +// +// Recursively walks through the remote server directory, emitting transfer files +// for the engine to process. +func (te *TransferEngine) walkDirDownloadHelper(job *clientTransferJob, transfers []transferAttemptDetails, files chan *clientTransferFile, path string, client *gowebdav.Client) error { + log.Debugln("Reading directory: ", path) + // Check for cancelation since the client does not respect the context + if err := job.job.ctx.Err(); err != nil { + return err + } + infos, err := client.ReadDir(path) if err != nil { - return nil, err + return err } for _, info := range infos { newPath := path + "/" + info.Name() - newDestPath := destPath + "/" + info.Name() //TODO make path.Join if info.IsDir() { - // Recursively call this function to create any nested dir's as well as list their files - returnedFiles, err := walkDirUpload(newPath, client, newDestPath) + err := te.walkDirDownloadHelper(job, transfers, files, newPath, client) if err != nil { - return nil, err + return err } - files = append(files, returnedFiles...) } else { - // It is a normal file - files = append(files, newPath) + job.job.activeXfer.Add(1) + select { + case <-job.job.ctx.Done(): + return job.job.ctx.Err() + case files <- &clientTransferFile{ + uuid: job.uuid, + jobId: job.job.uuid, + file: &transferFile{ + ctx: job.job.ctx, + callback: job.job.callback, + job: job.job, + engine: te, + remoteURL: &url.URL{Path: newPath}, + packOption: transfers[0].PackOption, + localPath: job.job.localPath, + upload: job.job.upload, + token: job.job.token, + attempts: transfers, + }, + }: + } } } - return files, err + return nil } -func walkDir(path string, client *gowebdav.Client) ([]string, error) { - var files []string - log.Debugln("Reading directory: ", path) - infos, err := client.ReadDir(path) +// Walk a local directory structure, writing all discovered files to the files channel +func (te *TransferEngine) walkDirUpload(job *clientTransferJob, transfers []transferAttemptDetails, files chan *clientTransferFile, localPath string) error { + if job.job.ctx.Err() != nil { + return job.job.ctx.Err() + } + + // Get our list of directory entries + infos, err := os.ReadDir(localPath) if err != nil { - return nil, err + return err } + for _, info := range infos { - newPath := path + "/" + info.Name() + newPath := localPath + "/" + info.Name() if info.IsDir() { - returnedFiles, err := walkDir(newPath, client) + // Recursively call this function to create any nested dir's as well as list their files + err := te.walkDirUpload(job, transfers, files, newPath) if err != nil { - return nil, err + return err + } + } else if info.Type().IsRegular() { + // It is a normal file; strip off the path prefix and append the destination prefix + remotePath := path.Join(job.job.remoteURL.Path, strings.TrimPrefix(newPath, job.job.localPath)) + job.job.activeXfer.Add(1) + select { + case <-job.job.ctx.Done(): + return job.job.ctx.Err() + case files <- &clientTransferFile{ + uuid: job.uuid, + jobId: job.job.uuid, + file: &transferFile{ + ctx: job.job.ctx, + callback: job.job.callback, + job: job.job, + engine: te, + remoteURL: &url.URL{Path: remotePath}, + packOption: transfers[0].PackOption, + localPath: newPath, + upload: job.job.upload, + token: job.job.token, + attempts: transfers, + }, + }: } - files = append(files, returnedFiles...) - } else { - // It is a normal file - files = append(files, newPath) } } - return files, nil + return err } -func StatHttp(dest *url.URL, namespace namespaces.Namespace) (uint64, error) { +func statHttp(ctx context.Context, dest *url.URL, namespace namespaces.Namespace, tokenLocation string, acquire bool) (size uint64, err error) { - scitoken_contents, err := getToken(dest, namespace, false, "") + token, err := getToken(dest, namespace, false, "", tokenLocation, acquire) if err != nil { - return 0, err + return } // Parse the writeback host as a URL writebackhostUrl, err := url.Parse(namespace.WriteBackHost) if err != nil { - return 0, err + return } dest.Host = writebackhostUrl.Host dest.Scheme = "https" canDisableProxy := CanDisableProxy() - disableProxy := !IsProxyEnabled() + disableProxy := !isProxyEnabled() var resp *http.Response for { @@ -1216,14 +1868,15 @@ func StatHttp(dest *url.URL, namespace namespaces.Namespace) (uint64, error) { } client := &http.Client{Transport: transport} - req, err := http.NewRequest("HEAD", dest.String(), nil) + var req *http.Request + req, err = http.NewRequestWithContext(ctx, "HEAD", dest.String(), nil) if err != nil { log.Errorln("Failed to create HTTP request:", err) - return 0, err + return } - if scitoken_contents != "" { - req.Header.Set("Authorization", "Bearer "+scitoken_contents) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) } resp, err = client.Do(req) @@ -1238,7 +1891,7 @@ func StatHttp(dest *url.URL, namespace namespaces.Namespace) (uint64, error) { } } log.Errorln("Failed to get HTTP response:", err) - return 0, err + return } if resp.StatusCode == 200 { @@ -1246,21 +1899,25 @@ func StatHttp(dest *url.URL, namespace namespaces.Namespace) (uint64, error) { contentLengthStr := resp.Header.Get("Content-Length") if len(contentLengthStr) == 0 { log.Errorln("HEAD response did not include Content-Length header") - return 0, errors.New("HEAD response did not include Content-Length header") + err = errors.New("HEAD response did not include Content-Length header") + return } - contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) + var contentLength int64 + contentLength, err = strconv.ParseInt(contentLengthStr, 10, 64) if err != nil { log.Errorf("Unable to parse Content-Length header value (%s) as integer: %s", contentLengthStr, err) - return 0, err + return } return uint64(contentLength), nil } else { - response_b, err := io.ReadAll(resp.Body) + var respB []byte + respB, err = io.ReadAll(resp.Body) if err != nil { log.Errorln("Failed to read error message:", err) - return 0, err + return } defer resp.Body.Close() - return 0, &HttpErrResp{resp.StatusCode, fmt.Sprintf("Request failed (HTTP status %d): %s", resp.StatusCode, string(response_b))} + err = &HttpErrResp{resp.StatusCode, fmt.Sprintf("Request failed (HTTP status %d): %s", resp.StatusCode, string(respB))} + return } } diff --git a/client/handle_http_test.go b/client/handle_http_test.go index 774cbef82..a28557eb4 100644 --- a/client/handle_http_test.go +++ b/client/handle_http_test.go @@ -32,6 +32,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -64,15 +65,15 @@ func TestMain(m *testing.M) { // for a valid return value. func TestIsPort(t *testing.T) { - if HasPort("blah.not.port:") { + if hasPort("blah.not.port:") { t.Fatal("Failed to parse port when : at end") } - if !HasPort("host:1") { + if !hasPort("host:1") { t.Fatal("Failed to parse with port = 1") } - if HasPort("https://example.com") { + if hasPort("https://example.com") { t.Fatal("Failed when scheme is specified") } } @@ -87,7 +88,7 @@ func TestNewTransferDetails(t *testing.T) { Endpoint: "cache.edu:8000", Resource: "Cache", } - transfers := NewTransferDetails(testCache, TransferDetailsOptions{false, ""}) + transfers := newTransferDetails(testCache, transferDetailsOptions{false, ""}) assert.Equal(t, 2, len(transfers)) assert.Equal(t, "cache.edu:8000", transfers[0].Url.Host) assert.Equal(t, "http", transfers[0].Url.Scheme) @@ -97,7 +98,7 @@ func TestNewTransferDetails(t *testing.T) { assert.Equal(t, false, transfers[1].Proxy) // Case 2: cache with https - transfers = NewTransferDetails(testCache, TransferDetailsOptions{true, ""}) + transfers = newTransferDetails(testCache, transferDetailsOptions{true, ""}) assert.Equal(t, 1, len(transfers)) assert.Equal(t, "cache.edu:8443", transfers[0].Url.Host) assert.Equal(t, "https", transfers[0].Url.Scheme) @@ -105,7 +106,7 @@ func TestNewTransferDetails(t *testing.T) { testCache.Endpoint = "cache.edu" // Case 3: cache without port with http - transfers = NewTransferDetails(testCache, TransferDetailsOptions{false, ""}) + transfers = newTransferDetails(testCache, transferDetailsOptions{false, ""}) assert.Equal(t, 2, len(transfers)) assert.Equal(t, "cache.edu:8000", transfers[0].Url.Host) assert.Equal(t, "http", transfers[0].Url.Scheme) @@ -116,7 +117,7 @@ func TestNewTransferDetails(t *testing.T) { // Case 4. cache without port with https testCache.AuthEndpoint = "cache.edu" - transfers = NewTransferDetails(testCache, TransferDetailsOptions{true, ""}) + transfers = newTransferDetails(testCache, transferDetailsOptions{true, ""}) assert.Equal(t, 2, len(transfers)) assert.Equal(t, "cache.edu:8444", transfers[0].Url.Host) assert.Equal(t, "https", transfers[0].Url.Scheme) @@ -137,11 +138,11 @@ func TestNewTransferDetailsEnv(t *testing.T) { os.Setenv("OSG_DISABLE_PROXY_FALLBACK", "") err := config.InitClient() assert.Nil(t, err) - transfers := NewTransferDetails(testCache, TransferDetailsOptions{false, ""}) + transfers := newTransferDetails(testCache, transferDetailsOptions{false, ""}) assert.Equal(t, 1, len(transfers)) assert.Equal(t, true, transfers[0].Proxy) - transfers = NewTransferDetails(testCache, TransferDetailsOptions{true, ""}) + transfers = newTransferDetails(testCache, transferDetailsOptions{true, ""}) assert.Equal(t, 1, len(transfers)) assert.Equal(t, "https", transfers[0].Url.Scheme) assert.Equal(t, false, transfers[0].Proxy) @@ -152,6 +153,8 @@ func TestNewTransferDetailsEnv(t *testing.T) { } func TestSlowTransfers(t *testing.T) { + ctx, _, _ := test_utils.TestContext(context.Background(), t) + // Adjust down some timeouts to speed up the test viper.Set("Client.SlowTransferWindow", 5) viper.Set("Client.SlowTransferRampupTime", 10) @@ -159,6 +162,11 @@ func TestSlowTransfers(t *testing.T) { channel := make(chan bool) slowDownload := 1024 * 10 // 10 KiB/s < 100 KiB/s svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Add("Content-Length", "1024000") + w.WriteHeader(http.StatusOK) + return + } buffer := make([]byte, slowDownload) for { select { @@ -183,7 +191,7 @@ func TestSlowTransfers(t *testing.T) { Endpoint: svr.URL, Resource: "Cache", } - transfers := NewTransferDetails(testCache, TransferDetailsOptions{false, ""}) + transfers := newTransferDetails(testCache, transferDetailsOptions{false, ""}) assert.Equal(t, 2, len(transfers)) assert.Equal(t, svr.URL, transfers[0].Url.String()) @@ -191,7 +199,7 @@ func TestSlowTransfers(t *testing.T) { var err error // Do a quick timeout go func() { - _, _, _, err = DownloadHTTP(transfers[0], filepath.Join(t.TempDir(), "test.txt"), "", nil) + _, _, _, err = downloadHTTP(ctx, nil, nil, transfers[0], filepath.Join(t.TempDir(), "test.txt"), "", nil) finishedChannel <- true }() @@ -215,12 +223,20 @@ func TestSlowTransfers(t *testing.T) { // Test stopped transfer func TestStoppedTransfer(t *testing.T) { + os.Setenv("http_proxy", "http://proxy.edu:3128") + ctx, _, _ := test_utils.TestContext(context.Background(), t) + // Adjust down the timeouts viper.Set("Client.StoppedTransferTimeout", 3) viper.Set("Client.SlowTransferRampupTime", 100) channel := make(chan bool) svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Add("Content-Length", "102400") + w.WriteHeader(http.StatusOK) + return + } buffer := make([]byte, 1024*100) for { select { @@ -246,7 +262,7 @@ func TestStoppedTransfer(t *testing.T) { Endpoint: svr.URL, Resource: "Cache", } - transfers := NewTransferDetails(testCache, TransferDetailsOptions{false, ""}) + transfers := newTransferDetails(testCache, transferDetailsOptions{false, ""}) assert.Equal(t, 2, len(transfers)) assert.Equal(t, svr.URL, transfers[0].Url.String()) @@ -254,7 +270,7 @@ func TestStoppedTransfer(t *testing.T) { var err error go func() { - _, _, _, err = DownloadHTTP(transfers[0], filepath.Join(t.TempDir(), "test.txt"), "", nil) + _, _, _, err = downloadHTTP(ctx, nil, nil, transfers[0], filepath.Join(t.TempDir(), "test.txt"), "", nil) finishedChannel <- true }() @@ -277,6 +293,8 @@ func TestStoppedTransfer(t *testing.T) { // Test connection error func TestConnectionError(t *testing.T) { + ctx, _, _ := test_utils.TestContext(context.Background(), t) + l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("dialClosedPort: Listen failed: %v", err) @@ -284,13 +302,15 @@ func TestConnectionError(t *testing.T) { addr := l.Addr().String() l.Close() - _, _, _, err = DownloadHTTP(TransferDetails{Url: url.URL{Host: addr, Scheme: "http"}, Proxy: false}, filepath.Join(t.TempDir(), "test.txt"), "", nil) + _, _, _, err = downloadHTTP(ctx, nil, nil, transferAttemptDetails{Url: &url.URL{Host: addr, Scheme: "http"}, Proxy: false}, filepath.Join(t.TempDir(), "test.txt"), "", nil) assert.IsType(t, &ConnectionSetupError{}, err) } func TestTrailerError(t *testing.T) { + ctx, _, _ := test_utils.TestContext(context.Background(), t) + // Set up an HTTP server that returns an error trailer svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Trailer", "X-Transfer-Status") @@ -312,12 +332,12 @@ func TestTrailerError(t *testing.T) { Endpoint: svr.URL, Resource: "Cache", } - transfers := NewTransferDetails(testCache, TransferDetailsOptions{false, ""}) + transfers := newTransferDetails(testCache, transferDetailsOptions{false, ""}) assert.Equal(t, 2, len(transfers)) assert.Equal(t, svr.URL, transfers[0].Url.String()) // Call DownloadHTTP and check if the error is returned correctly - _, _, _, err := DownloadHTTP(transfers[0], filepath.Join(t.TempDir(), "test.txt"), "", nil) + _, _, _, err := downloadHTTP(ctx, nil, nil, transfers[0], filepath.Join(t.TempDir(), "test.txt"), "", nil) assert.NotNil(t, err) assert.EqualError(t, err, "transfer error: Unable to read test.txt; input/output error") @@ -340,7 +360,7 @@ func TestUploadZeroLengthFile(t *testing.T) { request.Header.Set("Authorization", "Bearer test") errorChan := make(chan error, 1) responseChan := make(chan *http.Response) - go doPut(request, responseChan, errorChan) + go runPut(request, responseChan, errorChan) select { case err := <-errorChan: assert.NoError(t, err) @@ -370,7 +390,7 @@ func TestFailedUpload(t *testing.T) { request.Header.Set("Authorization", "Bearer test") errorChan := make(chan error, 1) responseChan := make(chan *http.Response) - go doPut(request, responseChan, errorChan) + go runPut(request, responseChan, errorChan) select { case err := <-errorChan: assert.Error(t, err) @@ -407,7 +427,7 @@ func generateFileTestScitoken() (string, error) { TokenProfile: utils.WLCG, Lifetime: time.Minute, Issuer: issuerUrl, - Audience: []string{param.Origin_Url.GetString()}, + Audience: []string{config.GetServerAudience()}, Version: "1.0", Subject: "origin", } @@ -473,6 +493,7 @@ func TestFullUpload(t *testing.T) { viper.Set("Registry.RequireOriginApproval", false) viper.Set("Registry.RequireCacheApproval", false) viper.Set("Logging.Origin.Scitokens", "debug") + viper.Set("Origin.Port", 0) err = config.InitServer(ctx, modules) require.NoError(t, err) @@ -526,27 +547,24 @@ func TestFullUpload(t *testing.T) { _, err = tempToken.WriteString(token) assert.NoError(t, err, "Error writing to temp token file") tempToken.Close() - ObjectClientOptions.Token = tempToken.Name() // Upload the file tempPath := tempFile.Name() fileName := filepath.Base(tempPath) uploadURL := "stash:///test/" + fileName - methods := []string{"http"} - transferResults, err := DoStashCPSingle(tempFile.Name(), uploadURL, methods, false) + transferResults, err := DoCopy(ctx, tempFile.Name(), uploadURL, false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err, "Error uploading file") assert.Equal(t, int64(len(testFileContent)), transferResults[0].TransferredBytes, "Uploaded file size does not match") // Upload an osdf file - uploadURL = "osdf:///test/stuff/blah.txt" + uploadURL = "pelican:///test/stuff/blah.txt" assert.NoError(t, err, "Error parsing upload URL") - transferResults, err = DoStashCPSingle(tempFile.Name(), uploadURL, methods, false) + transferResults, err = DoCopy(ctx, tempFile.Name(), uploadURL, false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err, "Error uploading file") assert.Equal(t, int64(len(testFileContent)), transferResults[0].TransferredBytes, "Uploaded file size does not match") }) t.Cleanup(func() { - ObjectClientOptions.Token = "" os.RemoveAll(tmpPath) os.RemoveAll(originDir) }) @@ -667,6 +685,8 @@ func (f *FedTest) Teardown() { // A test that spins up a federation, and tests object get and put func TestGetAndPutAuth(t *testing.T) { // Create instance of test federation + ctx, _, _ := test_utils.TestContext(context.Background(), t) + viper.Reset() fed := FedTest{T: t} fed.Spinup() @@ -723,20 +743,18 @@ func TestGetAndPutAuth(t *testing.T) { uploadURL := "pelican:///test/" + fileName // Upload the file with PUT - ObjectClientOptions.Token = tempToken.Name() - transferResultsUpload, err := DoPut(tempFile.Name(), uploadURL, false) + transferResultsUpload, err := DoPut(ctx, tempFile.Name(), uploadURL, false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil { assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) } // Download that same file with GET - transferResultsDownload, err := DoGet(uploadURL, t.TempDir(), false) + transferResultsDownload, err := DoGet(ctx, uploadURL, t.TempDir(), false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil { assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) } - ObjectClientOptions.Token = "" }) // This tests pelican object get/put with an osdf url @@ -745,23 +763,22 @@ func TestGetAndPutAuth(t *testing.T) { // Set path for object to upload/download tempPath := tempFile.Name() fileName := filepath.Base(tempPath) - uploadURL := "osdf:///test/" + fileName + // Minimal fix of test as it is soon to be replaced + uploadURL := "pelican:///test/" + fileName // Upload the file with PUT - ObjectClientOptions.Token = tempToken.Name() - transferResultsUpload, err := DoPut(tempFile.Name(), uploadURL, false) + transferResultsUpload, err := DoPut(ctx, tempFile.Name(), uploadURL, false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil { assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) } // Download that same file with GET - transferResultsDownload, err := DoGet(uploadURL, t.TempDir(), false) + transferResultsDownload, err := DoGet(ctx, uploadURL, t.TempDir(), false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil { assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) } - ObjectClientOptions.Token = "" }) // This tests object get/put with a pelican:// url @@ -773,20 +790,18 @@ func TestGetAndPutAuth(t *testing.T) { uploadURL := "pelican:///test/" + fileName // Upload the file with PUT - ObjectClientOptions.Token = tempToken.Name() - transferResultsUpload, err := DoPut(tempFile.Name(), uploadURL, false) + transferResultsUpload, err := DoPut(ctx, tempFile.Name(), uploadURL, false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil { assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) } // Download that same file with GET - transferResultsDownload, err := DoGet(uploadURL, t.TempDir(), false) + transferResultsDownload, err := DoGet(ctx, uploadURL, t.TempDir(), false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil { assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) } - ObjectClientOptions.Token = "" }) // This tests pelican object get/put with an osdf url @@ -795,28 +810,28 @@ func TestGetAndPutAuth(t *testing.T) { // Set path for object to upload/download tempPath := tempFile.Name() fileName := filepath.Base(tempPath) - uploadURL := "osdf:///test/" + fileName + // Minimal fix of test as it is soon to be replaced + uploadURL := "pelican:///test/" + fileName // Upload the file with PUT - ObjectClientOptions.Token = tempToken.Name() - transferResultsUpload, err := DoPut(tempFile.Name(), uploadURL, false) + transferResultsUpload, err := DoPut(ctx, tempFile.Name(), uploadURL, false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil { assert.Equal(t, transferResultsUpload[0].TransferredBytes, int64(17)) } // Download that same file with GET - transferResultsDownload, err := DoGet(uploadURL, t.TempDir(), false) + transferResultsDownload, err := DoGet(ctx, uploadURL, t.TempDir(), false, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil { assert.Equal(t, transferResultsDownload[0].TransferredBytes, transferResultsUpload[0].TransferredBytes) } - ObjectClientOptions.Token = "" }) } // A test that spins up the federation, where the origin is in EnablePublicReads mode. Then GET a file from the origin without a token func TestGetPublicRead(t *testing.T) { + ctx, _, _ := test_utils.TestContext(context.Background(), t) viper.Reset() viper.Set("Origin.EnablePublicReads", true) fed := FedTest{T: t} @@ -837,10 +852,10 @@ func TestGetPublicRead(t *testing.T) { // Set path for object to upload/download tempPath := tempFile.Name() fileName := filepath.Base(tempPath) - uploadURL := "osdf:///test/" + fileName + uploadURL := "pelican:///test/" + fileName // Download the file with GET. Shouldn't need a token to succeed - transferResults, err := DoGet(uploadURL, t.TempDir(), false) + transferResults, err := DoGet(ctx, uploadURL, t.TempDir(), false) assert.NoError(t, err) if err == nil { assert.Equal(t, transferResults[0].TransferredBytes, int64(17)) @@ -850,6 +865,8 @@ func TestGetPublicRead(t *testing.T) { func TestRecursiveUploadsAndDownloads(t *testing.T) { // Create instance of test federation + ctx, _, _ := test_utils.TestContext(context.Background(), t) + viper.Reset() fed := FedTest{T: t} fed.Spinup() @@ -912,18 +929,18 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { tempFile2.Close() t.Run("testPelicanRecursiveGetAndPutOsdfURL", func(t *testing.T) { - ObjectClientOptions.Token = tempToken.Name() config.SetPreferredPrefix("pelican") // Set path for object to upload/download tempPath := tempDir dirName := filepath.Base(tempPath) - uploadURL := "osdf:///test/" + dirName + // Note: minimally fixing this test as it is soon to be replaced + uploadURL := "pelican://" + param.Server_Hostname.GetString() + ":" + strconv.Itoa(param.Server_WebPort.GetInt()) + "/test/" + dirName ////////////////////////////////////////////////////////// // Upload the file with PUT - transferDetailsUpload, err := DoPut(tempDir, uploadURL, true) - assert.NoError(t, err) + transferDetailsUpload, err := DoPut(ctx, tempDir, uploadURL, true, WithTokenLocation(tempToken.Name())) + require.NoError(t, err) if err == nil && len(transferDetailsUpload) == 2 { countBytes17 := 0 countBytes23 := 0 @@ -951,7 +968,7 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { } // Download the files we just uploaded - transferDetailsDownload, err := DoGet(uploadURL, t.TempDir(), true) + transferDetailsDownload, err := DoGet(ctx, uploadURL, t.TempDir(), true, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil && len(transferDetailsUpload) == 2 { countBytesUploadIdx0 := 0 @@ -979,11 +996,9 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { t.Fatalf("Amount of transfers results returned for download was not correct. Transfer details returned: %d", len(transferDetailsDownload)) } } - ObjectClientOptions.Token = "" }) t.Run("testPelicanRecursiveGetAndPutPelicanURL", func(t *testing.T) { - ObjectClientOptions.Token = tempToken.Name() config.SetPreferredPrefix("pelican") // Set path for object to upload/download tempPath := tempDir @@ -993,7 +1008,7 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { ////////////////////////////////////////////////////////// // Upload the file with PUT - transferDetailsUpload, err := DoPut(tempDir, uploadURL, true) + transferDetailsUpload, err := DoPut(ctx, tempDir, uploadURL, true, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil && len(transferDetailsUpload) == 2 { countBytes17 := 0 @@ -1022,7 +1037,7 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { } // Download the files we just uploaded - transferDetailsDownload, err := DoGet(uploadURL, t.TempDir(), true) + transferDetailsDownload, err := DoGet(ctx, uploadURL, t.TempDir(), true, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil && len(transferDetailsUpload) == 2 { countBytesUploadIdx0 := 0 @@ -1050,21 +1065,20 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { t.Fatalf("Amount of transfers results returned for download was not correct. Transfer details returned: %d", len(transferDetailsDownload)) } } - ObjectClientOptions.Token = "" }) t.Run("testOsdfRecursiveGetAndPutOsdfURL", func(t *testing.T) { - ObjectClientOptions.Token = tempToken.Name() config.SetPreferredPrefix("osdf") // Set path for object to upload/download tempPath := tempDir dirName := filepath.Base(tempPath) - uploadURL := "osdf:///test/" + dirName + // Note: minimally fixing this test as it is soon to be replaced + uploadURL := "pelican://" + param.Server_Hostname.GetString() + ":" + strconv.Itoa(param.Server_WebPort.GetInt()) + "/test/" + dirName ////////////////////////////////////////////////////////// // Upload the file with PUT - transferDetailsUpload, err := DoPut(tempDir, uploadURL, true) + transferDetailsUpload, err := DoPut(ctx, tempDir, uploadURL, true, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil && len(transferDetailsUpload) == 2 { countBytes17 := 0 @@ -1093,7 +1107,7 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { } // Download the files we just uploaded - transferDetailsDownload, err := DoGet(uploadURL, t.TempDir(), true) + transferDetailsDownload, err := DoGet(ctx, uploadURL, t.TempDir(), true, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil && len(transferDetailsUpload) == 2 { countBytesUploadIdx0 := 0 @@ -1121,11 +1135,9 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { t.Fatalf("Amount of transfers results returned for download was not correct. Transfer details returned: %d", len(transferDetailsDownload)) } } - ObjectClientOptions.Token = "" }) t.Run("testOsdfRecursiveGetAndPutPelicanURL", func(t *testing.T) { - ObjectClientOptions.Token = tempToken.Name() config.SetPreferredPrefix("osdf") // Set path for object to upload/download tempPath := tempDir @@ -1135,7 +1147,7 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { ////////////////////////////////////////////////////////// // Upload the file with PUT - transferDetailsUpload, err := DoPut(tempDir, uploadURL, true) + transferDetailsUpload, err := DoPut(ctx, tempDir, uploadURL, true, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil && len(transferDetailsUpload) == 2 { countBytes17 := 0 @@ -1164,7 +1176,7 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { } // Download the files we just uploaded - transferDetailsDownload, err := DoGet(uploadURL, t.TempDir(), true) + transferDetailsDownload, err := DoGet(ctx, uploadURL, t.TempDir(), true, WithTokenLocation(tempToken.Name())) assert.NoError(t, err) if err == nil && len(transferDetailsUpload) == 2 { countBytesUploadIdx0 := 0 @@ -1192,7 +1204,6 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) { t.Fatalf("Amount of transfers results returned for download was not correct. Transfer details returned: %d", len(transferDetailsDownload)) } } - ObjectClientOptions.Token = "" }) } diff --git a/client/handle_ingest.go b/client/handle_ingest.go index 936740565..f14a396f5 100644 --- a/client/handle_ingest.go +++ b/client/handle_ingest.go @@ -19,6 +19,7 @@ package client import ( + "context" "errors" "fmt" "path" @@ -39,7 +40,7 @@ func version_status(filePath string) (string, uint64, error) { return path.Join(dir, fmt.Sprintf("%s.%s", base, hash)), localSize, nil } -func generate_destination(filePath string, originPrefix string, shadowOriginPrefix string) (string, uint64, error) { +func generateDestination(filePath string, originPrefix string, shadowOriginPrefix string) (string, uint64, error) { hashRaw, localSize, err := version_status(filePath) if err != nil { return "", 0, err @@ -52,22 +53,21 @@ func generate_destination(filePath string, originPrefix string, shadowOriginPref return "", 0, errors.New("File path must have the origin prefix") } -func DoShadowIngest(sourceFile string, originPrefix string, shadowOriginPrefix string) (int64, string, error) { +func DoShadowIngest(ctx context.Context, sourceFile string, originPrefix string, shadowOriginPrefix string, options ...TransferOption) (int64, string, error) { // After each transfer attempt, we'll check to see if the local file was modified. If so, we'll re-upload. for idx := 0; idx < 10; idx++ { - shadowFile, localSize, err := generate_destination(sourceFile, originPrefix, shadowOriginPrefix) + shadowFile, localSize, err := generateDestination(sourceFile, originPrefix, shadowOriginPrefix) log.Debugln("Resulting shadow URL:", shadowFile) if err != nil { return 0, "", err } - methods := []string{"http"} lastRemoteSize := uint64(0) lastUpdateTime := time.Now() startTime := lastUpdateTime maxRuntime := float64(localSize/10*1024*1024) + 300 for { - remoteSize, err := CheckOSDF(shadowFile, methods) + remoteSize, err := DoStat(ctx, shadowFile, options...) if httpErr, ok := err.(*HttpErrResp); ok { if httpErr.Code == 404 { break @@ -103,13 +103,13 @@ func DoShadowIngest(sourceFile string, originPrefix string, shadowOriginPrefix s time.Sleep(5 * time.Second) } - transferResults, err := DoStashCPSingle(sourceFile, shadowFile, methods, false) + transferResults, err := DoCopy(ctx, sourceFile, shadowFile, false, options...) if err != nil { return 0, "", err } // See if the file was modified while we were uploading; if not, we'll return success - shadowFilePost, _, err := generate_destination(sourceFile, originPrefix, shadowOriginPrefix) + shadowFilePost, _, err := generateDestination(sourceFile, originPrefix, shadowOriginPrefix) if err != nil { return 0, "", err } diff --git a/client/main.go b/client/main.go index eebf02c9f..7f53ceaf6 100644 --- a/client/main.go +++ b/client/main.go @@ -19,8 +19,8 @@ package client import ( + "context" "encoding/json" - "errors" "fmt" "net" "net/http" @@ -36,6 +36,7 @@ import ( "path/filepath" "time" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/pelicanplatform/pelican/config" @@ -44,47 +45,25 @@ import ( "github.com/spf13/viper" ) -type OptionsStruct struct { - ProgressBars bool - Recursive bool - Plugin bool - Token string - Version string -} - -var ObjectClientOptions OptionsStruct - -var ( - version string +type ( + payloadStruct struct { + filename string + status string + Owner string + ProjectName string + version string + start1 int64 + end1 int64 + timestamp int64 + downloadTime int64 + fileSize int64 + downloadSize int64 + } ) -// Nearest cache -var NearestCache string - -// List of caches, in order from closest to furthest -var NearestCacheList []string -var CachesJsonLocation string - // Number of caches to attempt to use in any invocation var CachesToTry int = 3 -// CacheOverride -var CacheOverride bool - -type payloadStruct struct { - filename string - status string - Owner string - ProjectName string - version string - start1 int64 - end1 int64 - timestamp int64 - downloadTime int64 - fileSize int64 - downloadSize int64 -} - // Determine the token name if it is embedded in the scheme, Condor-style func getTokenName(destination *url.URL) (scheme, tokenName string) { schemePieces := strings.Split(destination.Scheme, "+") @@ -98,29 +77,13 @@ func getTokenName(destination *url.URL) (scheme, tokenName string) { return } -// Do writeback to stash using SciTokens -func doWriteBack(source string, destination *url.URL, namespace namespaces.Namespace, recursive bool, projectName string) (transferResults []TransferResults, err error) { - - scitoken_contents, err := getToken(destination, namespace, true, "") - if err != nil { - return nil, fmt.Errorf("Failed to get token for write-back: %v", err) - } - if recursive { - return UploadDirectory(source, destination, scitoken_contents, namespace, projectName) - } else { - transferResult, err := UploadFile(source, destination, scitoken_contents, namespace, projectName) - transferResults = append(transferResults, transferResult) - return transferResults, err - } -} - // getToken returns the token to use for the given destination // -// If token_name is not empty, it will be used as the token name. -// If token_name is empty, the token name will be determined from the destination URL (if possible) using getTokenName -func getToken(destination *url.URL, namespace namespaces.Namespace, isWrite bool, token_name string) (string, error) { - if token_name == "" { - _, token_name = getTokenName(destination) +// If tokenName is not empty, it will be used as the token name. +// If tokenName is empty, the token name will be determined from the destination URL (if possible) using getTokenName +func getToken(destination *url.URL, namespace namespaces.Namespace, isWrite bool, tokenName string, tokenLocation string, acquireToken bool) (string, error) { + if tokenName == "" { + _, tokenName = getTokenName(destination) } type tokenJson struct { @@ -132,82 +95,73 @@ func getToken(destination *url.URL, namespace namespaces.Namespace, isWrite bool with the environment variable "TOKEN", or it can be searched in the standard HTCondor directory pointed to by the environment variable "_CONDOR_CREDS". */ - var token_location string - if ObjectClientOptions.Token != "" { - token_location = ObjectClientOptions.Token - log.Debugln("Getting token location from command line:", ObjectClientOptions.Token) - } else { - - // WLCG Token Discovery - if bearerToken, isBearerTokenSet := os.LookupEnv("BEARER_TOKEN"); isBearerTokenSet { - return bearerToken, nil - } else if bearerTokenFile, isBearerTokenFileSet := os.LookupEnv("BEARER_TOKEN_FILE"); isBearerTokenFileSet { - if _, err := os.Stat(bearerTokenFile); err != nil { - log.Warningln("Environment variable BEARER_TOKEN_FILE is set, but file being point to does not exist:", err) - } else { - token_location = bearerTokenFile - } + // WLCG Token Discovery + if bearerToken, isBearerTokenSet := os.LookupEnv("BEARER_TOKEN"); tokenLocation == "" && isBearerTokenSet { + return bearerToken, nil + } else if bearerTokenFile, isBearerTokenFileSet := os.LookupEnv("BEARER_TOKEN_FILE"); tokenLocation == "" && isBearerTokenFileSet { + if _, err := os.Stat(bearerTokenFile); err != nil { + log.Warningln("Environment variable BEARER_TOKEN_FILE is set, but file being point to does not exist:", err) + } else { + tokenLocation = bearerTokenFile } - if xdgRuntimeDir, xdgRuntimeDirSet := os.LookupEnv("XDG_RUNTIME_DIR"); token_location == "" && xdgRuntimeDirSet { - // Get the uid - uid := os.Getuid() - tmpTokenPath := filepath.Join(xdgRuntimeDir, "bt_u"+strconv.Itoa(uid)) - if _, err := os.Stat(tmpTokenPath); err == nil { - token_location = tmpTokenPath - } + } + if xdgRuntimeDir, xdgRuntimeDirSet := os.LookupEnv("XDG_RUNTIME_DIR"); tokenLocation == "" && xdgRuntimeDirSet { + // Get the uid + uid := os.Getuid() + tmpTokenPath := filepath.Join(xdgRuntimeDir, "bt_u"+strconv.Itoa(uid)) + if _, err := os.Stat(tmpTokenPath); err == nil { + tokenLocation = tmpTokenPath } + } - // Check for /tmp/bt_u - if token_location == "" { - uid := os.Getuid() - tmpTokenPath := "/tmp/bt_u" + strconv.Itoa(uid) - if _, err := os.Stat(tmpTokenPath); err == nil { - token_location = tmpTokenPath - } + // Check for /tmp/bt_u + if tokenLocation == "" { + uid := os.Getuid() + tmpTokenPath := "/tmp/bt_u" + strconv.Itoa(uid) + if _, err := os.Stat(tmpTokenPath); err == nil { + tokenLocation = tmpTokenPath } + } - // Backwards compatibility for getting scitokens - // If TOKEN is not set in environment, and _CONDOR_CREDS is set, then... - if tokenFile, isTokenSet := os.LookupEnv("TOKEN"); isTokenSet && token_location == "" { - if _, err := os.Stat(tokenFile); err != nil { - log.Warningln("Environment variable TOKEN is set, but file being point to does not exist:", err) - } else { - token_location = tokenFile - } + // Backwards compatibility for getting scitokens + // If TOKEN is not set in environment, and _CONDOR_CREDS is set, then... + if tokenFile, isTokenSet := os.LookupEnv("TOKEN"); isTokenSet && tokenLocation == "" { + if _, err := os.Stat(tokenFile); err != nil { + log.Warningln("Environment variable TOKEN is set, but file being point to does not exist:", err) + } else { + tokenLocation = tokenFile } + } - // Finally, look in the HTCondor runtime - if token_location == "" { - token_location = discoverHTCondorToken(token_name) - } - - if token_location == "" { - if !ObjectClientOptions.Plugin { - opts := config.TokenGenerationOpts{Operation: config.TokenSharedRead} - if isWrite { - opts.Operation = config.TokenSharedWrite - } - value, err := AcquireToken(destination, namespace, opts) - if err == nil { - return value, nil - } - log.Errorln("Failed to generate a new authorization token for this transfer: ", err) - log.Errorln("This transfer requires authorization to complete and no token is available") - err = errors.New("failed to find or generate a token as required for " + destination.String()) - AddError(err) - return "", err - } else { - log.Errorln("Credential is required, but currently mssing") - err := errors.New("Credential is required for " + destination.String() + " but is currently missing") - AddError(err) - return "", err + // Finally, look in the HTCondor runtime + if tokenLocation == "" { + tokenLocation = discoverHTCondorToken(tokenName) + } + + if tokenLocation == "" { + if acquireToken { + opts := config.TokenGenerationOpts{Operation: config.TokenSharedRead} + if isWrite { + opts.Operation = config.TokenSharedWrite } + value, err := AcquireToken(destination, namespace, opts) + if err == nil { + return value, nil + } + log.Errorln("Failed to generate a new authorization token for this transfer: ", err) + log.Errorln("This transfer requires authorization to complete and no token is available") + err = errors.New("failed to find or generate a token as required for " + destination.String()) + return "", err + } else { + log.Errorln("Credential is required, but currently mssing") + err := errors.New("Credential is required for " + destination.String() + " but is currently missing") + return "", err } } //Read in the JSON - log.Debug("Opening token file: " + token_location) - tokenContents, err := os.ReadFile(token_location) + log.Debug("Opening token file: " + tokenLocation) + tokenContents, err := os.ReadFile(tokenLocation) if err != nil { log.Errorln("Error reading token file:", err) return "", err @@ -223,7 +177,7 @@ func getToken(destination *url.URL, namespace namespaces.Namespace, isWrite bool } // Check the size of a remote file in an origin -func CheckOSDF(destination string, methods []string) (remoteSize uint64, err error) { +func DoStat(ctx context.Context, destination string, options ...TransferOption) (remoteSize uint64, err error) { defer func() { if r := recover(); r != nil { @@ -272,19 +226,20 @@ func CheckOSDF(destination string, methods []string) (remoteSize uint64, err err return 0, err } - for _, method := range methods { - - switch method { - case "http": - log.Info("Trying HTTP...") - if remoteSize, err = StatHttp(dest_uri, ns); err == nil { - return remoteSize, nil - } - default: - log.Errorf("Unknown transfer method: %s", method) - return 0, errors.New("Unknown transfer method") + tokenLocation := "" + acquire := true + for _, option := range options { + switch option.Ident() { + case identTransferOptionTokenLocation{}: + tokenLocation = option.Value().(string) + case identTransferOptionAcquireToken{}: + acquire = option.Value().(bool) } } + + if remoteSize, err = statHttp(ctx, dest_uri, ns, tokenLocation, acquire); err == nil { + return remoteSize, nil + } return 0, err } @@ -296,7 +251,7 @@ func GetCacheHostnames(testFile string) (urls []string, err error) { return } - caches, err := GetCachesFromNamespace(ns, directorUrl != "") + caches, err := getCachesFromNamespace(ns, directorUrl != "", make([]*url.URL, 0)) if err != nil { return } @@ -319,15 +274,27 @@ func GetCacheHostnames(testFile string) (urls []string, err error) { return } -func GetCachesFromNamespace(namespace namespaces.Namespace, useDirector bool) (caches []CacheInterface, err error) { +func getUserAgent(project string) (agent string) { + agent = "pelican-client/" + config.GetVersion() + if project != "" { + agent += " project/" + project + } + return +} + +func getCachesFromNamespace(namespace namespaces.Namespace, useDirector bool, preferredCaches []*url.URL) (caches []CacheInterface, err error) { // The global cache override is set - if CacheOverride { - log.Debugf("Using the cache (%s) from the config override\n", NearestCache) + if len(preferredCaches) > 0 { + if preferredCaches[0].String() == "" { + err = errors.New("Preferred cache was specified as an empty string") + return + } + log.Debugf("Using the cache (%s) from the config override\n", preferredCaches[0]) cache := namespaces.Cache{ - Endpoint: NearestCache, - AuthEndpoint: NearestCache, - Resource: NearestCache, + Endpoint: preferredCaches[0].String(), + AuthEndpoint: preferredCaches[0].String(), + Resource: preferredCaches[0].String(), } caches = []CacheInterface{cache} return @@ -343,23 +310,23 @@ func GetCachesFromNamespace(namespace namespaces.Namespace, useDirector bool) (c return } - if len(NearestCacheList) == 0 { + var bestCaches []string + if len(preferredCaches) == 0 { cacheListName := "xroot" if namespace.ReadHTTPS || namespace.UseTokenOnRead { cacheListName = "xroots" } - // FIXME: GetBestCache, for some reason, sets the NearestCacheList global? - _, err = GetBestCache(cacheListName) + bestCaches, err = GetBestCache(cacheListName) if err != nil { log.Errorln("Failed to get best caches:", err) return } } - log.Debugln("Nearest cache list:", NearestCacheList) + log.Debugln("Nearest cache list:", bestCaches) log.Debugln("Cache list name:", namespace.Caches) - matchedCaches := namespace.MatchCaches(NearestCacheList) + matchedCaches := namespace.MatchCaches(bestCaches) log.Debugln("Matched caches:", matchedCaches) caches = make([]CacheInterface, len(matchedCaches)) for idx, val := range matchedCaches { @@ -431,17 +398,14 @@ func getNamespaceInfo(resourcePath, OSDFDirectorUrl string, isPut bool) (ns name if err != nil { if isPut && dirResp != nil && dirResp.StatusCode == 405 { err = errors.New("Error 405: No writeable origins were found") - AddError(err) return } else { log.Errorln("Error while querying the Director:", err) - AddError(err) return } } ns, err = CreateNsFromDirectorResp(dirResp) if err != nil { - AddError(err) return } @@ -460,7 +424,6 @@ func getNamespaceInfo(resourcePath, OSDFDirectorUrl string, isPut bool) (ns name } else { ns, err = namespaces.MatchNamespace(resourcePath) if err != nil { - AddError(err) return } return @@ -474,8 +437,7 @@ localObject: the source file/directory you would like to upload remoteDestination: the end location of the upload recursive: a boolean indicating if the source is a directory or not */ -func DoPut(localObject string, remoteDestination string, recursive bool) (transferResults []TransferResults, err error) { - isPut := true +func DoPut(ctx context.Context, localObject string, remoteDestination string, recursive bool, options ...TransferOption) (transferResults []TransferResults, err error) { // First, create a handler for any panics that occur defer func() { if r := recover(); r != nil { @@ -483,19 +445,9 @@ func DoPut(localObject string, remoteDestination string, recursive bool) (transf log.Debugln("Panic caused by the following", string(debug.Stack())) ret := fmt.Sprintf("Unrecoverable error (panic) captured in DoPut: %v", r) err = errors.New(ret) - - // Attempt to add the panic to the error accumulator - AddError(errors.New(ret)) } }() - // Parse the source and destination with URL parse - localObjectUrl, err := url.Parse(localObject) - if err != nil { - log.Errorln("Failed to parse source URL:", err) - return nil, err - } - remoteDestination, remoteDestScheme := correctURLWithUnderscore(remoteDestination) remoteDestUrl, err := url.Parse(remoteDestination) if err != nil { @@ -536,27 +488,28 @@ func DoPut(localObject string, remoteDestination string, recursive bool) (transf remoteDestUrl.Scheme, strings.Join(understoodSchemes, ", ")) } - directorUrl := param.Federation_DirectorUrl.GetString() - - if remoteDestScheme == "osdf" || remoteDestScheme == "pelican" { - remoteDestination = remoteDestUrl.Path - } - - // Get the namespace of the remote filesystem - // For write back, it will be the destination - if !strings.HasPrefix(remoteDestination, "/") { - remoteDestination = strings.TrimPrefix(remoteDestination, remoteDestScheme+"://") + te := NewTransferEngine(ctx) + defer func() { + if err := te.Shutdown(); err != nil { + log.Errorln("Failure when shutting down transfer engine:", err) + } + }() + client, err := te.NewClient(options...) + if err != nil { + return } - - ns, err := getNamespaceInfo(remoteDestination, directorUrl, isPut) + tj, err := client.NewTransferJob(remoteDestUrl, localObject, true, recursive) if err != nil { - log.Errorln(err) - return nil, errors.New("Failed to get namespace information from source") + return } - uploadedBytes, err := doWriteBack(localObjectUrl.Path, remoteDestUrl, ns, recursive, "") - AddError(err) - return uploadedBytes, err - + if err = client.Submit(tj); err != nil { + return + } + transferResults, err = client.Shutdown() + if tj.lookupErr != nil { + err = tj.lookupErr + } + return } /* @@ -566,8 +519,7 @@ remoteObject: the source file/directory you would like to upload localDestination: the end location of the upload recursive: a boolean indicating if the source is a directory or not */ -func DoGet(remoteObject string, localDestination string, recursive bool) (transferResults []TransferResults, err error) { - isPut := false +func DoGet(ctx context.Context, remoteObject string, localDestination string, recursive bool, options ...TransferOption) (transferResults []TransferResults, err error) { // First, create a handler for any panics that occur defer func() { if r := recover(); r != nil { @@ -575,9 +527,6 @@ func DoGet(remoteObject string, localDestination string, recursive bool) (transf log.Debugln("Panic caused by the following", string(debug.Stack())) ret := fmt.Sprintf("Unrecoverable error (panic) captured in DoGet: %v", r) err = errors.New(ret) - - // Attempt to add the panic to the error accumulator - AddError(errors.New(ret)) } }() @@ -632,14 +581,6 @@ func DoGet(remoteObject string, localDestination string, recursive bool) (transf remoteObject = "/" + remoteObject } - directorUrl := param.Federation_DirectorUrl.GetString() - - ns, err := getNamespaceInfo(remoteObject, directorUrl, isPut) - if err != nil { - log.Errorln(err) - return nil, errors.New("Failed to get namespace information from source") - } - // get absolute path localDestPath, _ := filepath.Abs(localDestination) @@ -654,28 +595,55 @@ func DoGet(remoteObject string, localDestination string, recursive bool) (transf } payload := payloadStruct{} - payload.version = version + payload.version = config.GetVersion() //Fill out the payload as much as possible payload.filename = remoteObjectUrl.Path - parse_job_ad(&payload) + parseJobAd(&payload) - payload.start1 = time.Now().Unix() + start := time.Now() + payload.start1 = start.Unix() success := false - _, token_name := getTokenName(remoteObjectUrl) + te := NewTransferEngine(ctx) + defer func() { + if err := te.Shutdown(); err != nil { + log.Errorln("Failure when shutting down transfer engine:", err) + } + }() + tc, err := te.NewClient(options...) + if err != nil { + return + } + tj, err := tc.NewTransferJob(remoteObjectUrl, localDestination, false, recursive) + if err != nil { + return + } + err = tc.Submit(tj) + if err != nil { + return + } - var downloaded int64 - if transferResults, err = download_http(remoteObjectUrl, localDestination, &payload, ns, recursive, token_name); err == nil { - success = true + transferResults, err = tc.Shutdown() + end := time.Now() + if err == nil { + if tj.lookupErr == nil { + success = true + } else { + err = tj.lookupErr + } + } + var downloaded int64 = 0 + for _, result := range transferResults { + downloaded += result.TransferredBytes } - payload.end1 = time.Now().Unix() + payload.end1 = end.Unix() payload.timestamp = payload.end1 - payload.downloadTime = (payload.end1 - payload.start1) + payload.downloadTime = int64(end.Sub(start).Seconds()) if success { payload.status = "Success" @@ -684,19 +652,19 @@ func DoGet(remoteObject string, localDestination string, recursive bool) (transf payload.fileSize = downloaded payload.downloadSize = downloaded } else { - log.Error("Http GET failed! Unable to download file.") + log.Error("Http GET failed! Unable to download file:", err) payload.status = "Fail" } if !success { - return nil, errors.New("failed to download file") + return nil, errors.Wrap(err, "failed to download file") } else { return transferResults, err } } // Start the transfer, whether read or write back. Primarily used for backwards compatibility -func DoStashCPSingle(sourceFile string, destination string, methods []string, recursive bool) (transferResults []TransferResults, err error) { +func DoCopy(ctx context.Context, sourceFile string, destination string, recursive bool, options ...TransferOption) (transferResults []TransferResults, err error) { // First, create a handler for any panics that occur defer func() { @@ -705,20 +673,17 @@ func DoStashCPSingle(sourceFile string, destination string, methods []string, re log.Debugln("Panic caused by the following", string(debug.Stack())) ret := fmt.Sprintf("Unrecoverable error (panic) captured in DoStashCPSingle: %v", r) err = errors.New(ret) - - // Attempt to add the panic to the error accumulator - AddError(errors.New(ret)) } }() // Parse the source and destination with URL parse sourceFile, source_scheme := correctURLWithUnderscore(sourceFile) - source_url, err := url.Parse(sourceFile) + sourceURL, err := url.Parse(sourceFile) if err != nil { log.Errorln("Failed to parse source URL:", err) return nil, err } - source_url.Scheme = source_scheme + sourceURL.Scheme = source_scheme destination, dest_scheme := correctURLWithUnderscore(destination) dest_url, err := url.Parse(destination) @@ -731,12 +696,12 @@ func DoStashCPSingle(sourceFile string, destination string, methods []string, re defer config.SetFederation(fd) // If there is a host specified, prepend it to the path in the osdf case - if source_url.Host != "" { - if source_url.Scheme == "osdf" || source_url.Scheme == "stash" { - source_url.Path = "/" + path.Join(source_url.Host, source_url.Path) - } else if source_url.Scheme == "pelican" { + if sourceURL.Host != "" { + if sourceURL.Scheme == "osdf" || sourceURL.Scheme == "stash" { + sourceURL.Path = "/" + path.Join(sourceURL.Host, sourceURL.Path) + } else if sourceURL.Scheme == "pelican" { config.SetFederation(config.FederationDiscovery{}) - federationUrl, _ := url.Parse(source_url.String()) + federationUrl, _ := url.Parse(sourceURL.String()) federationUrl.Scheme = "https" federationUrl.Path = "" viper.Set("Federation.DiscoveryUrl", federationUrl.String()) @@ -763,115 +728,110 @@ func DoStashCPSingle(sourceFile string, destination string, methods []string, re } } - sourceScheme, _ := getTokenName(source_url) + sourceScheme, _ := getTokenName(sourceURL) destScheme, _ := getTokenName(dest_url) understoodSchemes := []string{"stash", "file", "osdf", "pelican", ""} _, foundSource := Find(understoodSchemes, sourceScheme) if !foundSource { - log.Errorln("Do not understand source scheme:", source_url.Scheme) + log.Errorln("Do not understand source scheme:", sourceURL.Scheme) return nil, errors.New("Do not understand source scheme") } _, foundDest := Find(understoodSchemes, destScheme) if !foundDest { - log.Errorln("Do not understand destination scheme:", source_url.Scheme) + log.Errorln("Do not understand destination scheme:", sourceURL.Scheme) return nil, errors.New("Do not understand destination scheme") } payload := payloadStruct{} - parse_job_ad(&payload) + parseJobAd(&payload) - // Get the namespace of the remote filesystem - // For write back, it will be the destination - // For read it will be the source. - - OSDFDirectorUrl := param.Federation_DirectorUrl.GetString() isPut := destScheme == "stash" || destScheme == "osdf" || destScheme == "pelican" + var localPath string + var remoteURL *url.URL if isPut { log.Debugln("Detected object write to remote federation object", dest_url.Path) - ns, err := getNamespaceInfo(dest_url.Path, OSDFDirectorUrl, isPut) - if err != nil { - log.Errorln(err) - return nil, errors.New("Failed to get namespace information from destination") - } - transferResults, err := doWriteBack(source_url.Path, dest_url, ns, recursive, payload.ProjectName) //TODO dowriteback transferResults!!!!! - AddError(err) - return transferResults, err - } - - if dest_url.Scheme == "file" { - destination = dest_url.Path - } + localPath = sourceFile + remoteURL = dest_url + } else { - if sourceScheme == "stash" || sourceScheme == "osdf" || sourceScheme == "pelican" { - sourceFile = source_url.Path - } + if dest_url.Scheme == "file" { + destination = dest_url.Path + } - if string(sourceFile[0]) != "/" { - sourceFile = "/" + sourceFile - } + if sourceScheme == "stash" || sourceScheme == "osdf" || sourceScheme == "pelican" { + sourceFile = sourceURL.Path + } - ns, err := getNamespaceInfo(sourceFile, OSDFDirectorUrl, isPut) - if err != nil { - log.Errorln(err) - return nil, errors.New("Failed to get namespace information from source") - } + if string(sourceFile[0]) != "/" { + sourceFile = "/" + sourceFile + } - // get absolute path - destPath, _ := filepath.Abs(destination) + // get absolute path + destPath, _ := filepath.Abs(destination) - //Check if path exists or if its in a folder - if destStat, err := os.Stat(destPath); os.IsNotExist(err) { - destination = destPath - } else if destStat.IsDir() && source_url.Query().Get("pack") == "" { - // If we have an auto-pack request, it's OK for the destination to be a directory - // Otherwise, get the base name of the source and append it to the destination dir. - sourceFilename := path.Base(sourceFile) - destination = path.Join(destPath, sourceFilename) + //Check if path exists or if its in a folder + if destStat, err := os.Stat(destPath); os.IsNotExist(err) { + destination = destPath + } else if destStat.IsDir() && sourceURL.Query().Get("pack") == "" { + // If we have an auto-pack request, it's OK for the destination to be a directory + // Otherwise, get the base name of the source and append it to the destination dir. + sourceFilename := path.Base(sourceFile) + destination = path.Join(destPath, sourceFilename) + } + localPath = destination + remoteURL = sourceURL } - payload.version = version + payload.version = config.GetVersion() //Fill out the payload as much as possible - payload.filename = source_url.Path + payload.filename = sourceURL.Path - payload.start1 = time.Now().Unix() + start := time.Now() + payload.start1 = start.Unix() // Go thru the download methods success := false - // If recursive, only do http method to guarantee freshest directory contents - if ObjectClientOptions.Recursive { - methods = []string{"http"} - } - - _, token_name := getTokenName(source_url) - // switch statement? var downloaded int64 = 0 -Loop: - for _, method := range methods { - - switch method { - case "http": - log.Info("Trying HTTP...") - if transferResults, err = download_http(source_url, destination, &payload, ns, recursive, token_name); err == nil { - success = true - break Loop - } - default: - log.Errorf("Unknown transfer method: %s", method) + te := NewTransferEngine(ctx) + defer func() { + if err := te.Shutdown(); err != nil { + log.Errorln("Failure when shutting down transfer engine:", err) } + }() + tc, err := te.NewClient(options...) + if err != nil { + return + } + tj, err := tc.NewTransferJob(remoteURL, localPath, isPut, recursive) + if err != nil { + return + } + if err = tc.Submit(tj); err != nil { + return + } + transferResults, err = tc.Shutdown() + if err == nil { + success = true } - payload.end1 = time.Now().Unix() + end := time.Now() + + for _, result := range transferResults { + downloaded += result.TransferredBytes + } + + payload.end1 = end.Unix() payload.timestamp = payload.end1 - payload.downloadTime = (payload.end1 - payload.start1) + payload.downloadTime = int64(end.Sub(start).Seconds()) if success { payload.status = "Success" @@ -882,7 +842,7 @@ Loop: return transferResults, nil } else { payload.status = "Fail" - return transferResults, errors.New("All methods failed! Unable to download file.") + return transferResults, err } } @@ -932,7 +892,7 @@ func get_ips(name string) []string { } -func parse_job_ad(payload *payloadStruct) { +func parseJobAd(payload *payloadStruct) { //Parse the .job.ad file for the Owner (username) and ProjectName of the callee. diff --git a/client/main_test.go b/client/main_test.go index 46f5d5144..738bff150 100644 --- a/client/main_test.go +++ b/client/main_test.go @@ -74,7 +74,7 @@ func TestGetToken(t *testing.T) { // ENVs to test: BEARER_TOKEN, BEARER_TOKEN_FILE, XDG_RUNTIME_DIR/bt_u, TOKEN, _CONDOR_CREDS/scitoken.use, .condor_creds/scitokens.use os.Setenv("BEARER_TOKEN", "bearer_token_contents") - token, err := getToken(url, namespace, true, "") + token, err := getToken(url, namespace, true, "", "", false) assert.NoError(t, err) assert.Equal(t, "bearer_token_contents", token) os.Unsetenv("BEARER_TOKEN") @@ -87,7 +87,7 @@ func TestGetToken(t *testing.T) { err = os.WriteFile(bearer_token_file, tmpFile, 0644) assert.NoError(t, err) os.Setenv("BEARER_TOKEN_FILE", bearer_token_file) - token, err = getToken(url, namespace, true, "") + token, err = getToken(url, namespace, true, "", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) os.Unsetenv("BEARER_TOKEN_FILE") @@ -99,7 +99,7 @@ func TestGetToken(t *testing.T) { err = os.WriteFile(bearer_token_file, tmpFile, 0644) assert.NoError(t, err) os.Setenv("XDG_RUNTIME_DIR", tmpDir) - token, err = getToken(url, namespace, true, "") + token, err = getToken(url, namespace, true, "", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) os.Unsetenv("XDG_RUNTIME_DIR") @@ -111,7 +111,7 @@ func TestGetToken(t *testing.T) { err = os.WriteFile(bearer_token_file, tmpFile, 0644) assert.NoError(t, err) os.Setenv("TOKEN", bearer_token_file) - token, err = getToken(url, namespace, true, "") + token, err = getToken(url, namespace, true, "", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) os.Unsetenv("TOKEN") @@ -123,7 +123,7 @@ func TestGetToken(t *testing.T) { err = os.WriteFile(bearer_token_file, tmpFile, 0644) assert.NoError(t, err) os.Setenv("_CONDOR_CREDS", tmpDir) - token, err = getToken(url, namespace, true, "") + token, err = getToken(url, namespace, true, "", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) os.Unsetenv("_CONDOR_CREDS") @@ -140,7 +140,7 @@ func TestGetToken(t *testing.T) { assert.NoError(t, err) renamedNamespace, err := namespaces.MatchNamespace("/user/ligo/frames") assert.NoError(t, err) - token, err = getToken(renamedUrl, renamedNamespace, false, "") + token, err = getToken(renamedUrl, renamedNamespace, false, "", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) os.Unsetenv("_CONDOR_CREDS") @@ -159,7 +159,7 @@ func TestGetToken(t *testing.T) { assert.NoError(t, err) renamedNamespace, err = namespaces.MatchNamespace("/user/ligo/frames") assert.NoError(t, err) - token, err = getToken(renamedUrl, renamedNamespace, false, "") + token, err = getToken(renamedUrl, renamedNamespace, false, "", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) os.Unsetenv("_CONDOR_CREDS") @@ -176,7 +176,7 @@ func TestGetToken(t *testing.T) { assert.NoError(t, err) renamedNamespace, err = namespaces.MatchNamespace("/user/ligo/frames") assert.NoError(t, err) - token, err = getToken(renamedUrl, renamedNamespace, false, "") + token, err = getToken(renamedUrl, renamedNamespace, false, "", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) os.Unsetenv("_CONDOR_CREDS") @@ -193,7 +193,7 @@ func TestGetToken(t *testing.T) { assert.NoError(t, err) renamedNamespace, err = namespaces.MatchNamespace("/user/ligo/frames") assert.NoError(t, err) - token, err = getToken(renamedUrl, renamedNamespace, false, "") + token, err = getToken(renamedUrl, renamedNamespace, false, "", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) os.Unsetenv("_CONDOR_CREDS") @@ -210,7 +210,7 @@ func TestGetToken(t *testing.T) { assert.NoError(t, err) renamedNamespace, err = namespaces.MatchNamespace("/user/ligo/frames") assert.NoError(t, err) - token, err = getToken(renamedUrl, renamedNamespace, false, "renamed") + token, err = getToken(renamedUrl, renamedNamespace, false, "renamed", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) os.Unsetenv("_CONDOR_CREDS") @@ -227,17 +227,14 @@ func TestGetToken(t *testing.T) { assert.NoError(t, err) err = os.Chdir(tmpDir) assert.NoError(t, err) - token, err = getToken(url, namespace, true, "") + token, err = getToken(url, namespace, true, "", "", false) assert.NoError(t, err) assert.Equal(t, token_contents, token) err = os.Chdir(currentDir) assert.NoError(t, err) - ObjectClientOptions.Plugin = true - _, err = getToken(url, namespace, true, "") + _, err = getToken(url, namespace, true, "", "", false) assert.EqualError(t, err, "Credential is required for osdf:///user/foo but is currently missing") - ObjectClientOptions.Plugin = false - } // TestGetTokenName tests getTokenName @@ -331,5 +328,5 @@ func TestParseNoJobAd(t *testing.T) { os.Setenv("_CONDOR_JOB_AD", path) payload := payloadStruct{} - parse_job_ad(&payload) + parseJobAd(&payload) } diff --git a/cmd/main.go b/cmd/main.go index 9c0b378e7..c2e7a352d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -23,6 +23,8 @@ import ( "os" "path/filepath" "strings" + + "github.com/pelicanplatform/pelican/config" ) func main() { @@ -54,7 +56,7 @@ func handleCLI(args []string) error { // version info regardless of the commands and whether they are defined // * Remove the -v shorthand since in "origin serve" flagset it's already used for "volume" flag if args[len(args)-1] == "--version" { - fmt.Println("Version:", version) + fmt.Println("Version:", config.GetVersion()) fmt.Println("Build Date:", date) fmt.Println("Build Commit:", commit) fmt.Println("Built By:", builtBy) diff --git a/cmd/main_test.go b/cmd/main_test.go index e025b8ae5..9733ffaa5 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -12,13 +12,14 @@ import ( "testing" + "github.com/pelicanplatform/pelican/config" "github.com/stretchr/testify/assert" ) func TestHandleCLIVersionFlag(t *testing.T) { // Save the current version to reset this variable - var currentVersion = version - version = "0.0.1" + var currentVersion = config.GetVersion() + config.SetVersion("0.0.1") date = "2023-10-06T15:26:50Z" commit = "f0f94a3edf6641c2472345819a0d5453fc9e68d1" builtBy = "goreleaser" @@ -29,7 +30,7 @@ func TestHandleCLIVersionFlag(t *testing.T) { mockVersionOutput := fmt.Sprintf( "Version: %s\nBuild Date: %s\nBuild Commit: %s\nBuilt By: %s", - version, date, commit, builtBy, + config.GetVersion(), date, commit, builtBy, ) testCases := []struct { @@ -106,7 +107,7 @@ func TestHandleCLIVersionFlag(t *testing.T) { os.Args = oldArgs // Set the version back to what it was - version = currentVersion + config.SetVersion(currentVersion) } func TestHandleCLIExecutableAlias(t *testing.T) { diff --git a/cmd/object_copy.go b/cmd/object_copy.go index e8ca4ce5b..254882812 100644 --- a/cmd/object_copy.go +++ b/cmd/object_copy.go @@ -20,6 +20,7 @@ package main import ( "fmt" + "net/url" "os" "path/filepath" "strings" @@ -28,6 +29,7 @@ import ( "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/namespaces" "github.com/pelicanplatform/pelican/param" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -78,8 +80,7 @@ func init() { } func copyMain(cmd *cobra.Command, args []string) { - - client.ObjectClientOptions.Version = config.PelicanVersion + ctx := cmd.Context() // Need to check just stashcp since it does not go through root, the other modes get checked there if strings.HasPrefix(execName, "stashcp") { @@ -97,22 +98,22 @@ func copyMain(cmd *cobra.Command, args []string) { } if val, err := cmd.Flags().GetBool("version"); err == nil && val { - fmt.Println("Version:", version) + fmt.Println("Version:", config.GetVersion()) fmt.Println("Build Date:", date) fmt.Println("Build Commit:", commit) fmt.Println("Built By:", builtBy) os.Exit(0) } - // Set the progress bars to the command line option - client.ObjectClientOptions.Token, _ = cmd.Flags().GetString("token") + pb := newProgressBar() + defer pb.shutdown() + + tokenLocation, _ := cmd.Flags().GetString("token") // Check if the program was executed from a terminal and does not specify a log location // https://rosettacode.org/wiki/Check_output_device_is_a_terminal#Go if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode()&os.ModeCharDevice) != 0 && param.Logging_LogLocation.GetString() == "" && !param.Logging_DisableProgressBars.GetBool() { - client.ObjectClientOptions.ProgressBars = true - } else { - client.ObjectClientOptions.ProgressBars = false + pb.launchDisplay(ctx) } if val, err := cmd.Flags().GetBool("namespaces"); err == nil && val { @@ -168,26 +169,22 @@ func copyMain(cmd *cobra.Command, args []string) { log.Debugln("Sources:", source) log.Debugln("Destination:", dest) - // Check for manually entered cache to use ?? - nearestCache, nearestCacheIsPresent := os.LookupEnv("NEAREST_CACHE") - - if nearestCacheIsPresent { - client.NearestCache = nearestCache - client.NearestCacheList = append(client.NearestCacheList, client.NearestCache) - client.CacheOverride = true + // Check for manually entered cache to use + var preferredCache string + if nearestCache, ok := os.LookupEnv("NEAREST_CACHE"); ok { + preferredCache = nearestCache } else if cache, _ := cmd.Flags().GetString("cache"); cache != "" { - client.NearestCache = cache - client.NearestCacheList = append(client.NearestCacheList, cache) - client.CacheOverride = true + preferredCache = cache } - - // Convert the methods - methodNames, _ := cmd.Flags().GetString("methods") - splitMethods := strings.Split(methodNames, ",") - - // If the user overrides the cache, then only use HTTP - if client.CacheOverride { - splitMethods = []string{"http"} + caches := make([]*url.URL, 0, 1) + if preferredCache != "" { + if preferredCacheURL, err := url.Parse(preferredCache); err != nil { + log.Errorf("Unable to parse preferred cache (%s) as URL: %s", preferredCache, err.Error()) + os.Exit(1) + } else { + caches = append(caches, preferredCacheURL) + log.Debugln("Preferred cache for transfer:", preferredCacheURL) + } } if len(source) > 1 { @@ -201,25 +198,23 @@ func copyMain(cmd *cobra.Command, args []string) { lastSrc := "" for _, src := range source { isRecursive, _ := cmd.Flags().GetBool("recursive") - client.ObjectClientOptions.Recursive = isRecursive - _, result = client.DoStashCPSingle(src, dest, splitMethods, isRecursive) + _, result = client.DoCopy(ctx, src, dest, isRecursive, client.WithCallback(pb.callback), client.WithTokenLocation(tokenLocation), client.WithCaches(caches...)) if result != nil { lastSrc = src break - } else { - client.ClearErrors() } } // Exit with failure if result != nil { // Print the list of errors - errMsg := client.GetErrors() - if errMsg == "" { - errMsg = result.Error() + errMsg := result.Error() + var te *client.TransferErrors + if errors.As(result, &te) { + errMsg = te.UserError() } log.Errorln("Failure transferring " + lastSrc + ": " + errMsg) - if client.ErrorsRetryable() { + if client.ShouldRetry(err) { log.Errorln("Errors are retryable") os.Exit(11) } diff --git a/cmd/object_get.go b/cmd/object_get.go index dbac14c08..4ba143231 100644 --- a/cmd/object_get.go +++ b/cmd/object_get.go @@ -19,11 +19,13 @@ package main import ( + "net/url" "os" "github.com/pelicanplatform/pelican/client" "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -48,8 +50,7 @@ func init() { } func getMain(cmd *cobra.Command, args []string) { - - client.ObjectClientOptions.Version = version + ctx := cmd.Context() err := config.InitClient() if err != nil { @@ -57,15 +58,15 @@ func getMain(cmd *cobra.Command, args []string) { os.Exit(1) } - // Set the progress bars to the command line option - client.ObjectClientOptions.Token, _ = cmd.Flags().GetString("token") + tokenLocation, _ := cmd.Flags().GetString("token") + + pb := newProgressBar() + defer pb.shutdown() // Check if the program was executed from a terminal // https://rosettacode.org/wiki/Check_output_device_is_a_terminal#Go if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode()&os.ModeCharDevice) != 0 && param.Logging_LogLocation.GetString() == "" && !param.Logging_DisableProgressBars.GetBool() { - client.ObjectClientOptions.ProgressBars = true - } else { - client.ObjectClientOptions.ProgressBars = false + pb.launchDisplay(ctx) } log.Debugln("Len of source:", len(args)) @@ -83,17 +84,22 @@ func getMain(cmd *cobra.Command, args []string) { log.Debugln("Sources:", source) log.Debugln("Destination:", dest) - // Check for manually entered cache to use ?? - nearestCache, nearestCacheIsPresent := os.LookupEnv("NEAREST_CACHE") - - if nearestCacheIsPresent { - client.NearestCache = nearestCache - client.NearestCacheList = append(client.NearestCacheList, client.NearestCache) - client.CacheOverride = true + // Check for manually entered cache to use + var preferredCache string + if nearestCache, ok := os.LookupEnv("NEAREST_CACHE"); ok { + preferredCache = nearestCache } else if cache, _ := cmd.Flags().GetString("cache"); cache != "" { - client.NearestCache = cache - client.NearestCacheList = append(client.NearestCacheList, cache) - client.CacheOverride = true + preferredCache = cache + } + caches := make([]*url.URL, 0, 1) + if preferredCache != "" { + if preferredCacheURL, err := url.Parse(preferredCache); err != nil { + log.Errorf("Unable to parse preferred cache (%s) as URL: %s", preferredCache, err.Error()) + os.Exit(1) + } else { + caches = append(caches, preferredCacheURL) + log.Debugln("Preferred cache for transfer:", preferredCacheURL) + } } if len(source) > 1 { @@ -107,25 +113,23 @@ func getMain(cmd *cobra.Command, args []string) { lastSrc := "" for _, src := range source { isRecursive, _ := cmd.Flags().GetBool("recursive") - client.ObjectClientOptions.Recursive = isRecursive - _, result = client.DoGet(src, dest, isRecursive) + _, result = client.DoGet(ctx, src, dest, isRecursive, client.WithCallback(pb.callback), client.WithTokenLocation(tokenLocation), client.WithCaches(caches...)) if result != nil { lastSrc = src break - } else { - client.ClearErrors() } } // Exit with failure if result != nil { // Print the list of errors - errMsg := client.GetErrors() - if errMsg == "" { - errMsg = result.Error() + errMsg := result.Error() + var te *client.TransferErrors + if errors.As(result, &te) { + errMsg = te.UserError() } log.Errorln("Failure getting " + lastSrc + ": " + errMsg) - if client.ErrorsRetryable() { + if client.ShouldRetry(result) { log.Errorln("Errors are retryable") os.Exit(11) } diff --git a/cmd/object_put.go b/cmd/object_put.go index dba0fd8fa..331278291 100644 --- a/cmd/object_put.go +++ b/cmd/object_put.go @@ -24,6 +24,7 @@ import ( "github.com/pelicanplatform/pelican/client" "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -44,8 +45,7 @@ func init() { } func putMain(cmd *cobra.Command, args []string) { - - client.ObjectClientOptions.Version = version + ctx := cmd.Context() err := config.InitClient() if err != nil { @@ -54,14 +54,15 @@ func putMain(cmd *cobra.Command, args []string) { } // Set the progress bars to the command line option - client.ObjectClientOptions.Token, _ = cmd.Flags().GetString("token") + tokenLocation, _ := cmd.Flags().GetString("token") + + pb := newProgressBar() + defer pb.shutdown() // Check if the program was executed from a terminal // https://rosettacode.org/wiki/Check_output_device_is_a_terminal#Go if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode()&os.ModeCharDevice) != 0 && param.Logging_LogLocation.GetString() == "" && !param.Logging_DisableProgressBars.GetBool() { - client.ObjectClientOptions.ProgressBars = true - } else { - client.ObjectClientOptions.ProgressBars = false + pb.launchDisplay(ctx) } log.Debugln("Len of source:", len(args)) @@ -83,25 +84,23 @@ func putMain(cmd *cobra.Command, args []string) { lastSrc := "" for _, src := range source { isRecursive, _ := cmd.Flags().GetBool("recursive") - client.ObjectClientOptions.Recursive = isRecursive - _, result = client.DoPut(src, dest, isRecursive) + _, result = client.DoPut(ctx, src, dest, isRecursive, client.WithCallback(pb.callback), client.WithTokenLocation(tokenLocation)) if result != nil { lastSrc = src break - } else { - client.ClearErrors() } } // Exit with failure if result != nil { // Print the list of errors - errMsg := client.GetErrors() - if errMsg == "" { - errMsg = result.Error() + errMsg := result.Error() + var te *client.TransferErrors + if errors.As(result, &te) { + errMsg = te.UserError() } log.Errorln("Failure putting " + lastSrc + ": " + errMsg) - if client.ErrorsRetryable() { + if client.ShouldRetry(result) { log.Errorln("Errors are retryable") os.Exit(11) } diff --git a/cmd/plugin.go b/cmd/plugin.go index f2069e3eb..d899222e7 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -20,6 +20,7 @@ package main import ( "bufio" + "context" "fmt" "io/fs" "net/url" @@ -27,7 +28,6 @@ import ( "path/filepath" "runtime/debug" "strings" - "sync" "syscall" "time" @@ -37,10 +37,10 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" ) var ( - version = "dev" commit = "none" date = "unknown" builtBy = "unknown" @@ -56,8 +56,8 @@ var ( outfile string ) -type Transfer struct { - url string +type PluginTransfer struct { + url *url.URL localFile string } @@ -92,19 +92,12 @@ func stashPluginMain(args []string) { log.Warningln("Panic caused by the following", string(debug.Stack())) ret := fmt.Sprintf("Unrecoverable error (panic) captured in stashPluginMain(): %v", r) - debugStack := strings.ReplaceAll(string(debug.Stack()), "\n", ";") - client.AddError(errors.New(debugStack)) - - // Attempt to add the panic to the error accumulator - client.AddError(errors.New(ret)) - - // Write our important classAds resultAd := classads.NewClassAd() var resultAds []*classads.ClassAd // Set as failure and add errors resultAd.Set("TransferSuccess", false) - resultAd.Set("TransferError", client.GetErrors()) + resultAd.Set("TransferError", ret+";"+strings.ReplaceAll(string(debug.Stack()), "\n", ";")) resultAds = append(resultAds, resultAd) // Attempt to write our file and bail @@ -120,18 +113,12 @@ func stashPluginMain(args []string) { if err != nil { log.Errorf("Problem initializing the Pelican client config: %v", err) err = errors.Wrap(err, "Problem initializing the Pelican Client configuration") - client.AddError(err) isConfigErr = true } // Parse command line arguments var upload bool = false // Set the options - client.ObjectClientOptions.Recursive = false - client.ObjectClientOptions.ProgressBars = false - client.ObjectClientOptions.Version = version - client.ObjectClientOptions.Plugin = true - methods := []string{"http"} var infile, testCachePath string var getCaches bool = false @@ -140,12 +127,12 @@ func stashPluginMain(args []string) { if args[0] == "-classad" { // Print classad and exit fmt.Println("MultipleFileSupport = true") - fmt.Println("PluginVersion = \"" + version + "\"") + fmt.Println("PluginVersion = \"" + config.GetVersion() + "\"") fmt.Println("PluginType = \"FileTransfer\"") fmt.Println("SupportedMethods = \"stash, osdf\"") os.Exit(0) } else if args[0] == "-version" || args[0] == "-v" { - fmt.Println("Version:", version) + fmt.Println("Version:", config.GetVersion()) fmt.Println("Build Date:", date) fmt.Println("Build Commit:", commit) fmt.Println("Built By:", builtBy) @@ -191,7 +178,7 @@ func stashPluginMain(args []string) { // Set as failure and add errors resultAd.Set("TransferSuccess", false) - resultAd.Set("TransferError", client.GetErrors()) + resultAd.Set("TransferError", err.Error()) resultAds = append(resultAds, resultAd) // Attempt to write our file and bail @@ -220,13 +207,14 @@ func stashPluginMain(args []string) { var source []string var dest string - var transfers []Transfer + var transfers []PluginTransfer if len(args) == 0 && (infile == "" || outfile == "") { fmt.Fprint(os.Stderr, "No source or destination specified\n") os.Exit(1) } + var workChan chan PluginTransfer if len(args) == 0 { // Open the input and output files infileFile, err := os.Open(infile) @@ -241,13 +229,26 @@ func stashPluginMain(args []string) { log.Errorln("Failed to read in from stdin:", err) os.Exit(1) } - } else { + workChan = make(chan PluginTransfer, len(transfers)) + for _, transfer := range transfers { + workChan <- transfer + } + } else if len(args) > 1 { source = args[:len(args)-1] dest = args[len(args)-1] + workChan = make(chan PluginTransfer, len(args)-1) for _, src := range source { - transfers = append(transfers, Transfer{url: src, localFile: dest}) + srcUrl, err := url.Parse(src) + if err != nil { + log.Errorf("Failed to parse input URL (%s): %s", src, err) + } + workChan <- PluginTransfer{url: srcUrl, localFile: dest} } + } else { + log.Errorln("Must provide both source and destination as argument") + os.Exit(1) } + close(workChan) // NOTE: HTCondor 23.3.0 and before would reuse the outfile names for multiple // transfers, meaning the results of prior plugin invocations would be present @@ -269,33 +270,34 @@ func stashPluginMain(args []string) { defer outputFile.Close() } - var wg sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + egrp, _ := errgroup.WithContext(ctx) + defer func() { + err = egrp.Wait() + if err != context.Canceled { + log.Errorln("Error when shutting down worker:", err) + } + }() + defer cancel() - workChan := make(chan Transfer, len(transfers)) - results := make(chan *classads.ClassAd, len(transfers)) + results := make(chan *classads.ClassAd, 5) - // Start workers - for i := 1; i <= 5; i++ { - wg.Add(1) - go moveObjects(source, methods, upload, &wg, workChan, results) - } + egrp.Go(func() error { + return runPluginWorker(ctx, upload, workChan, results) + }) success := true var resultAds []*classads.ClassAd - counter := 0 done := false for !done { - // Send to work channel the amount of transfers we have - if counter < len(transfers) { - workChan <- transfers[counter] - counter++ - } else if counter == len(transfers) { // Once we sent all the work, close the channel - close(workChan) - // Increment counter so we no longer hit this case - counter++ - } select { - case resultAd := <-results: + case <-ctx.Done(): + done = true + case resultAd, ok := <-results: + if !ok { + done = true + break + } // Process results as soon as we get them transferSuccess, err := resultAd.Get("TransferSuccess") if err != nil { @@ -308,28 +310,22 @@ func stashPluginMain(args []string) { // Add the final (failed) result to the resultAds resultAds = append(resultAds, resultAd) done = true - break } else { // Otherwise, we add to end result ads resultAds = append(resultAds, resultAd) } - default: - // We are either done or still downloading/uploading - if len(resultAds) == len(transfers) { - log.Debugln("Finished transfering objects! :)") - done = true - break - } } } - // Wait for transfers only if successful - if success { - wg.Wait() + // Ensure all our workers are shut down. + cancel() + if waitErr := egrp.Wait(); waitErr != nil && waitErr != context.Canceled { + log.Errorln("Error when shutting down worker:", waitErr) + success = false + err = waitErr } - close(results) - - success, retryable := writeOutfile(resultAds, outputFile) + tmpSuccess, retryable := writeOutfile(err, resultAds, outputFile) + success = tmpSuccess && success if success { os.Exit(0) @@ -358,49 +354,81 @@ func writeClassadOutputAndBail(exitCode int, resultAds []*classads.ClassAd) { } // We'll exit 3 in here if anything fails to write the file - writeOutfile(resultAds, outputFile) + writeOutfile(nil, resultAds, outputFile) log.Errorln("Failure with pelican plugin. Exiting...") os.Exit(exitCode) } -// moveObjects performs the appropriate download or upload functions for the plugin as well as +// runPluginWorker performs the appropriate download or upload functions for the plugin as well as // writes the resultAds for each transfer // Returns: resultAds and if an error given is retryable -func moveObjects(source []string, methods []string, upload bool, wg *sync.WaitGroup, workChan <-chan Transfer, results chan<- *classads.ClassAd) { - defer wg.Done() - var result error - for transfer := range workChan { - var transferResults []client.TransferResults - if upload { - source = append(source, transfer.localFile) - log.Debugln("Uploading:", transfer.localFile, "to", transfer.url) - transferResults, result = client.DoStashCPSingle(transfer.localFile, transfer.url, methods, false) - } else { - source = append(source, transfer.url) - log.Debugln("Downloading:", transfer.url, "to", transfer.localFile) +func runPluginWorker(ctx context.Context, upload bool, workChan <-chan PluginTransfer, results chan<- *classads.ClassAd) (err error) { + te := client.NewTransferEngine(ctx) + defer func() { + if shutdownErr := te.Shutdown(); shutdownErr != nil && err == nil { + err = shutdownErr + } + }() + tc, err := te.NewClient(client.WithAcquireToken(false)) + if err != nil { + return + } + defer func() { + if err != nil { + tc.Cancel() + } + }() + defer close(results) - // When we want to auto-unpack files, we should do this to the containing directory, not the destination - // file which HTCondor prepares - url, err := url.Parse(transfer.url) - if err != nil { - result = errors.Wrap(err, "Unable to parse transfer source as a URL") + jobMap := make(map[string]PluginTransfer) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case transfer, ok := <-workChan: + if !ok { + tc.Close() + workChan = nil + break + } + if upload { + log.Debugln("Uploading:", transfer.localFile, "to", transfer.url) } else { - localFile := transfer.localFile - if url.Query().Get("pack") != "" { - localFile = filepath.Dir(localFile) + log.Debugln("Downloading:", transfer.url, "to", transfer.localFile) + + // When we want to auto-unpack files, we should do this to the containing directory, not the destination + // file which HTCondor prepares + if transfer.url.Query().Get("pack") != "" { + transfer.localFile = filepath.Dir(transfer.localFile) } - transferResults, result = client.DoStashCPSingle(transfer.url, localFile, methods, false) } - } - startTime := time.Now().Unix() - resultAd := classads.NewClassAd() - // Set our DeveloperData: - developerData := make(map[string]interface{}) - developerData["PelicanClientVersion"] = version - if len(transferResults) != 0 && !upload { - developerData["Attempts"] = len(transferResults[0].Attempts) - for _, attempt := range transferResults[0].Attempts { + + var tj *client.TransferJob + urlCopy := *transfer.url + tj, err = tc.NewTransferJob(&urlCopy, transfer.localFile, upload, false, client.WithAcquireToken(false)) + jobMap[tj.ID()] = transfer + if err != nil { + return errors.Wrap(err, "Failed to create new transfer job") + } + + if err = tc.Submit(tj); err != nil { + return err + } + case result, ok := <-tc.Results(): + if !ok { + log.Debugln("Client has no more results") + return + } + log.Debugln("Got result from transfer client") + startTime := time.Now().Unix() + resultAd := classads.NewClassAd() + // Set our DeveloperData: + developerData := make(map[string]interface{}) + developerData["PelicanClientVersion"] = config.GetVersion() + developerData["Attempts"] = len(result.Attempts) + for _, attempt := range result.Attempts { developerData[fmt.Sprintf("TransferFileBytes%d", attempt.Number)] = attempt.TransferFileBytes developerData[fmt.Sprintf("TimeToFirstByte%d", attempt.Number)] = attempt.TimeToFirstByte developerData[fmt.Sprintf("Endpoint%d", attempt.Number)] = attempt.Endpoint @@ -410,61 +438,51 @@ func moveObjects(source []string, methods []string, upload bool, wg *sync.WaitGr developerData[fmt.Sprintf("TransferError%d", attempt.Number)] = attempt.Error } } - } else if len(transferResults) != 0 && upload { // For uploads, we only care about idx 0 since there is only 1 Attempt and 1 TransferResult - developerData["TransferFileBytes"] = transferResults[0].TransferredBytes - if len(transferResults[0].Attempts) != 0 { // Should be fine but check to be sure so we don't go out of bounds - developerData["Endpoint"] = transferResults[0].Attempts[0].Endpoint - developerData["TransferEndTime"] = transferResults[0].Attempts[0].TransferEndTime - developerData["ServerVersion"] = transferResults[0].Attempts[0].ServerVersion - developerData["TimeToFirstByte"] = transferResults[0].Attempts[0].TimeToFirstByte - } - if transferResults[0].Error != nil { - developerData["TransferError"] = transferResults[0].Error - } - } - resultAd.Set("DeveloperData", developerData) - - resultAd.Set("TransferStartTime", startTime) - resultAd.Set("TransferEndTime", time.Now().Unix()) - hostname, _ := os.Hostname() - resultAd.Set("TransferLocalMachineName", hostname) - resultAd.Set("TransferProtocol", "stash") - resultAd.Set("TransferUrl", transfer.url) - resultAd.Set("TransferFileName", transfer.localFile) - if upload { - resultAd.Set("TransferType", "upload") - } else { - resultAd.Set("TransferType", "download") - } - if result == nil { - resultAd.Set("TransferSuccess", true) - resultAd.Set("TransferFileBytes", transferResults[0].TransferredBytes) - resultAd.Set("TransferTotalBytes", transferResults[0].TransferredBytes) // idx 0 since we are not using recursive uploads/downloads - } else { - resultAd.Set("TransferSuccess", false) - if client.GetErrors() == "" { - resultAd.Set("TransferError", result.Error()) + resultAd.Set("DeveloperData", developerData) + + resultAd.Set("TransferStartTime", startTime) + resultAd.Set("TransferEndTime", time.Now().Unix()) + hostname, _ := os.Hostname() + resultAd.Set("TransferLocalMachineName", hostname) + resultAd.Set("TransferProtocol", "stash") + transfer := jobMap[result.JobId.String()] + resultAd.Set("TransferUrl", transfer.url.String()) + resultAd.Set("TransferFileName", transfer.localFile) + if upload { + resultAd.Set("TransferType", "upload") + } else { + resultAd.Set("TransferType", "download") + } + if result.Error == nil { + resultAd.Set("TransferSuccess", true) + resultAd.Set("TransferFileBytes", result.Attempts[len(result.Attempts)-1].TransferFileBytes) + resultAd.Set("TransferTotalBytes", result.Attempts[len(result.Attempts)-1].TransferFileBytes) } else { + resultAd.Set("TransferSuccess", false) + var te *client.TransferErrors + errMsgInternal := result.Error.Error() + if errors.As(result.Error, &te) { + errMsgInternal = te.UserError() + } errMsg := " Failure " if upload { errMsg += "uploading " } else { errMsg += "downloading " } - errMsg += transfer.url + ": " + client.GetErrors() + errMsg += transfer.url.String() + ": " + errMsgInternal resultAd.Set("TransferError", errMsg) + resultAd.Set("TransferFileBytes", 0) + resultAd.Set("TransferTotalBytes", 0) + if client.ShouldRetry(result.Error) { + resultAd.Set("TransferRetryable", true) + } else { + resultAd.Set("TransferRetryable", false) + } } - resultAd.Set("TransferFileBytes", 0) - resultAd.Set("TransferTotalBytes", 0) - if client.ErrorsRetryable() { - resultAd.Set("TransferRetryable", true) - } else { - resultAd.Set("TransferRetryable", false) - } - client.ClearErrors() + results <- resultAd } - results <- resultAd } } @@ -472,7 +490,24 @@ func moveObjects(source []string, methods []string, upload bool, wg *sync.WaitGr // true: all result ads indicate transfer success // false: at least one result ad has failed // As well as a boolean letting us know if errors are retryable -func writeOutfile(resultAds []*classads.ClassAd, outputFile *os.File) (bool, bool) { +func writeOutfile(err error, resultAds []*classads.ClassAd, outputFile *os.File) (bool, bool) { + + if err != nil { + alreadyFailed := false + for _, ad := range resultAds { + failed, getErr := ad.Get("TransferSuccess") + if getErr != nil || failed.(bool) { + alreadyFailed = true + break + } + } + if !alreadyFailed { + resultAd := classads.NewClassAd() + resultAd.Set("TransferSuccess", false) + resultAd.Set("TransferError", err.Error()) + resultAds = append(resultAds, resultAd) + } + } success := true retryable := false for _, resultAd := range resultAds { @@ -503,14 +538,14 @@ func writeOutfile(resultAds []*classads.ClassAd, outputFile *os.File) (bool, boo var serr syscall.Errno // Error code 1 (serr) is ERROR_INVALID_FUNCTION, the expected Windows syscall error // Error code EINVAL is returned on Linux - // Error code ENODEV is returned on Mac OS X - if errors.As(err, &perr) && errors.As(perr.Unwrap(), &serr) && (int(serr) == 1 || serr == syscall.EINVAL || serr == syscall.ENODEV) { + // Error code ENODEV (/dev/null) or ENOTTY (/dev/stdout) is returned on Mac OS X + if errors.As(err, &perr) && errors.As(perr.Unwrap(), &serr) && (int(serr) == 1 || serr == syscall.EINVAL || serr == syscall.ENODEV || serr == syscall.ENOTTY) { log.Debugf("Error when syncing: %s; can be ignored\n", perr) } else { if errors.As(err, &perr) && errors.As(perr.Unwrap(), &serr) { - log.Errorf("Failed to sync output file: %s (errno %d)", serr, int(serr)) + log.Errorf("Failed to sync output file (%s): %s (errno %d)", outputFile.Name(), serr, int(serr)) } else { - log.Errorln("Failed to sync output file:", err) + log.Errorf("Failed to sync output file (%s): %s", outputFile.Name(), err) } os.Exit(FailedOutfile) // Unique error code to let us know the outfile could not be created } @@ -519,7 +554,7 @@ func writeOutfile(resultAds []*classads.ClassAd, outputFile *os.File) (bool, boo } // readMultiTransfers reads the transfers from a Reader, such as stdin -func readMultiTransfers(stdin bufio.Reader) (transfers []Transfer, err error) { +func readMultiTransfers(stdin bufio.Reader) (transfers []PluginTransfer, err error) { // Check stdin for a list of transfers ads, err := classads.ReadClassAd(&stdin) if err != nil { @@ -529,7 +564,11 @@ func readMultiTransfers(stdin bufio.Reader) (transfers []Transfer, err error) { return nil, errors.New("No transfers found") } for _, ad := range ads { - adUrl, err := ad.Get("Url") + adUrlStr, err := ad.Get("Url") + if err != nil { + return nil, err + } + adUrl, err := url.Parse(adUrlStr.(string)) if err != nil { return nil, err } @@ -538,7 +577,7 @@ func readMultiTransfers(stdin bufio.Reader) (transfers []Transfer, err error) { if err != nil { return nil, err } - transfers = append(transfers, Transfer{url: adUrl.(string), localFile: destination.(string)}) + transfers = append(transfers, PluginTransfer{url: adUrl, localFile: destination.(string)}) } return transfers, nil diff --git a/cmd/plugin_stage.go b/cmd/plugin_stage.go index 1fdd4b970..fa7c5eeb2 100644 --- a/cmd/plugin_stage.go +++ b/cmd/plugin_stage.go @@ -19,6 +19,7 @@ package main import ( + "context" "fmt" "net/url" "os" @@ -85,6 +86,7 @@ Terminology: } func stagePluginMain(cmd *cobra.Command, args []string) { + ctx := cmd.Context() originPrefixStr := param.StagePlugin_OriginPrefix.GetString() if len(originPrefixStr) == 0 { @@ -115,16 +117,15 @@ func stagePluginMain(cmd *cobra.Command, args []string) { os.Exit(1) } - // Set the progress bars to the command line option - client.ObjectClientOptions.Token = param.Plugin_Token.GetString() - client.ObjectClientOptions.Plugin = true + tokenLocation := param.Plugin_Token.GetString() + + pb := newProgressBar() + defer pb.shutdown() // Check if the program was executed from a terminal // https://rosettacode.org/wiki/Check_output_device_is_a_terminal#Go if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) != 0 { - client.ObjectClientOptions.ProgressBars = true - } else { - client.ObjectClientOptions.ProgressBars = false + pb.launchDisplay(ctx) } var sources []string @@ -187,7 +188,7 @@ func stagePluginMain(cmd *cobra.Command, args []string) { var result error var xformSources []string for _, src := range sources { - _, newSource, result := client.DoShadowIngest(src, mountPrefixStr, shadowOriginPrefixStr) + _, newSource, result := client.DoShadowIngest(context.Background(), src, mountPrefixStr, shadowOriginPrefixStr, client.WithTokenLocation(tokenLocation), client.WithAcquireToken(false)) if result != nil { // What's the correct behavior on failure? For now, we silently put the transfer // back on the original list. This is arguably the wrong approach as it might @@ -204,8 +205,8 @@ func stagePluginMain(cmd *cobra.Command, args []string) { // Exit with failure if result != nil { // Print the list of errors - log.Errorln(client.GetErrors()) - if client.ErrorsRetryable() { + log.Errorln("Failure in staging files:", result) + if client.ShouldRetry(result) { log.Errorln("Errors are retryable") os.Exit(11) } diff --git a/cmd/plugin_test.go b/cmd/plugin_test.go index 55f74befb..4860a8baf 100644 --- a/cmd/plugin_test.go +++ b/cmd/plugin_test.go @@ -42,18 +42,18 @@ func TestReadMultiTransfer(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 3, len(transfers)) assert.Equal(t, "/path/to/local/copy/of/foo", transfers[0].localFile) - assert.Equal(t, "url://server/some/directory//foo", transfers[0].url) + assert.Equal(t, "url://server/some/directory//foo", transfers[0].url.String()) assert.Equal(t, "/path/to/local/copy/of/bar", transfers[1].localFile) - assert.Equal(t, "url://server/some/directory//bar", transfers[1].url) + assert.Equal(t, "url://server/some/directory//bar", transfers[1].url.String()) assert.Equal(t, "/path/to/local/copy/of/qux", transfers[2].localFile) - assert.Equal(t, "url://server/some/directory//qux", transfers[2].url) + assert.Equal(t, "url://server/some/directory//qux", transfers[2].url.String()) // Test with single transfers stdin = "[ LocalFileName = \"/path/to/local/copy/of/blah\"; Url = \"url://server/some/directory//blah\" ]" transfers, err = readMultiTransfers(*bufio.NewReader(strings.NewReader(stdin))) assert.NoError(t, err) assert.Equal(t, 1, len(transfers)) - assert.Equal(t, "url://server/some/directory//blah", transfers[0].url) + assert.Equal(t, "url://server/some/directory//blah", transfers[0].url.String()) assert.Equal(t, "/path/to/local/copy/of/blah", transfers[0].localFile) } diff --git a/cmd/progress_bars.go b/cmd/progress_bars.go new file mode 100644 index 000000000..2e485a5f9 --- /dev/null +++ b/cmd/progress_bars.go @@ -0,0 +1,152 @@ +/*************************************************************** + * + * Copyright (C) 2023, University of Nebraska-Lincoln + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +package main + +import ( + "context" + "os" + "path/filepath" + "sync" + "time" + + log "github.com/sirupsen/logrus" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" + "golang.org/x/sync/errgroup" +) + +type ( + progressStatus struct { + xfer int64 // Number of bytes transferred + size int64 // Total size of object to move + completed bool // Whether the object is complete + } + + progressBar struct { + progressStatus + bar *mpb.Bar + } + + progressBars struct { + lock sync.RWMutex + done chan bool + status map[string]progressStatus + egrp *errgroup.Group + } +) + +func newProgressBar() *progressBars { + return &progressBars{ + done: make(chan bool), + status: make(map[string]progressStatus), + } +} + +func (pb *progressBars) callback(path string, xfer int64, size int64, completed bool) { + pb.lock.Lock() + defer pb.lock.Unlock() + stat := pb.status[path] + stat.completed = completed + stat.size = size + stat.xfer = xfer + pb.status[path] = stat +} + +func (pb *progressBars) shutdown() { + if pb.egrp != nil { + pb.done <- true + if err := pb.egrp.Wait(); err != nil { + log.Debugln("Failure to shut down progress bar:", err) + } + } +} + +func (pb *progressBars) launchDisplay(ctx context.Context) { + progressCtr := mpb.NewWithContext(ctx) + pb.egrp, _ = errgroup.WithContext(ctx) + log.Debugln("Launch progress bars display") + + pb.egrp.Go(func() error { + defer func() { + log.SetOutput(os.Stdout) + progressCtr.Wait() + }() + + tickDuration := 200 * time.Millisecond + ticker := time.NewTicker(tickDuration) + pbMap := make(map[string]*progressBar) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-pb.done: + for path := range pbMap { + pbMap[path].bar.Abort(true) + pbMap[path].bar.Wait() + } + return nil + case <-ticker.C: + func() { + pb.lock.RLock() + defer pb.lock.RUnlock() + for path := range pbMap { + pbMap[path].xfer = -1 + } + for path := range pb.status { + if pbMap[path] == nil { + pbMap[path] = &progressBar{ + bar: progressCtr.AddBar(0, + mpb.PrependDecorators( + decor.Name(filepath.Base(path), decor.WCSyncSpaceR), + decor.CountersKibiByte("% .2f / % .2f"), + ), + mpb.AppendDecorators( + decor.OnComplete(decor.EwmaETA(decor.ET_STYLE_GO, 15), ""), + decor.OnComplete(decor.Name(" ] "), ""), + decor.OnComplete(decor.EwmaSpeed(decor.SizeB1024(0), "% .2f", 15), "Done!"), + ), + ), + } + } + oldStatus := pbMap[path].progressStatus + newStatus := pb.status[path] + if oldStatus.size == 0 && newStatus.size > 0 { + pbMap[path].bar.SetTotal(newStatus.size, false) + } + pbMap[path].bar.EwmaSetCurrent(newStatus.xfer, tickDuration) + pbMap[path].progressStatus = newStatus + } + toDelete := make([]string, 0) + for path := range pbMap { + if pbMap[path].xfer == -1 { + toDelete = append(toDelete, path) + } + } + for _, path := range toDelete { + bar := pbMap[path].bar + bar.Abort(true) + bar.Wait() + delete(pbMap, path) + } + }() + } + } + + }) +} diff --git a/cmd/root.go b/cmd/root.go index a16a74cb1..a7f98c0ae 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -121,7 +121,6 @@ func restartProgram() error { } func init() { - config.PelicanVersion = version cobra.OnInitialize(config.InitConfig) rootCmd.AddCommand(objectCmd) objectCmd.CompletionOptions.DisableDefaultCmd = true diff --git a/config/config.go b/config/config.go index 81760c39d..219ea2540 100644 --- a/config/config.go +++ b/config/config.go @@ -135,7 +135,7 @@ var ( RestartFlag = make(chan any) // A channel flag to restart the server instance that launcher listens to (including cache) // Pelican version - PelicanVersion string + version string = "dev" ) func init() { @@ -184,6 +184,18 @@ func IsServerEnabled(testServer ServerType) bool { return enabledServers.IsEnabled(testServer) } +// Returns the version of the current binary +func GetVersion() string { + return version +} + +// Overrides the version of the current binary +// +// Intended mainly for use in unit tests +func SetVersion(newVersion string) { + version = newVersion +} + // Get a string slice of currently enabled servers, sorted by alphabetical order. // By default, it calls String method of each enabled server. // To get strings in lowerCase, set lowerCase = true. @@ -365,7 +377,7 @@ func DiscoverFederation() error { if err != nil { return errors.Wrapf(err, "Failure when doing federation metadata request creation for %s", discoveryUrl) } - req.Header.Set("User-Agent", "pelican/"+PelicanVersion) + req.Header.Set("User-Agent", "pelican/"+version) result, err := httpClient.Do(req) if err != nil { diff --git a/config/resources/defaults.yaml b/config/resources/defaults.yaml index 101ad4162..496a2e72f 100644 --- a/config/resources/defaults.yaml +++ b/config/resources/defaults.yaml @@ -27,6 +27,8 @@ Logging: Cms: error Pfc: info Xrootd: emsg login stall redirect +Client: + WorkerCount: 5 Server: WebPort: 8444 WebHost: "0.0.0.0" diff --git a/docs/parameters.yaml b/docs/parameters.yaml index e4f84392e..1e25917b9 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -329,6 +329,14 @@ type: bool default: false components: ["client"] --- +name: Client.WorkerCount +description: >- + An integer indicating the number of file transfer tasks that should be + executed in parallel +type: int +default: 5 +components: ["client"] +--- name: DisableHttpProxy description: >- A legacy configuration for disabling the client's HTTP proxy. See Client.DisableHttpProxy for new config. diff --git a/github_scripts/get_put_test.sh b/github_scripts/get_put_test.sh index 9f4cd1c63..4d81fc0e8 100755 --- a/github_scripts/get_put_test.sh +++ b/github_scripts/get_put_test.sh @@ -96,7 +96,7 @@ do done # Run pelican object put -./pelican object put input.txt osdf:///test/input.txt -d -t token -l putOutput.txt +./pelican object put input.txt pelican:///test/input.txt -d -t token -l putOutput.txt # Check output of command if grep -q "Uploaded bytes: 47" putOutput.txt; then @@ -107,7 +107,7 @@ else to_exit=1 fi -./pelican object get osdf:///test/input.txt output.txt -d -t token -l getOutput.txt +./pelican object get pelican:///test/input.txt output.txt -d -t token -l getOutput.txt # Check output of command if grep -q "Downloaded bytes: 47" getOutput.txt; then diff --git a/go.mod b/go.mod index ab6d27dc0..450cf368d 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 // indirect - github.com/VividCortex/ewma v1.2.0 // indirect + github.com/VividCortex/ewma v1.2.0 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go v1.45.25 // indirect @@ -124,7 +124,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.5.0 // indirect + github.com/google/uuid v1.5.0 github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -139,7 +139,7 @@ require ( github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect - github.com/lestrrat-go/option v1.0.1 // indirect + github.com/lestrrat-go/option v1.0.1 github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/param/parameters.go b/param/parameters.go index e8e5b189a..2f18dc75f 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -176,6 +176,7 @@ var ( Client_SlowTransferRampupTime = IntParam{"Client.SlowTransferRampupTime"} Client_SlowTransferWindow = IntParam{"Client.SlowTransferWindow"} Client_StoppedTransferTimeout = IntParam{"Client.StoppedTransferTimeout"} + Client_WorkerCount = IntParam{"Client.WorkerCount"} Director_MaxStatResponse = IntParam{"Director.MaxStatResponse"} Director_MinStatResponse = IntParam{"Director.MinStatResponse"} Director_StatConcurrencyLimit = IntParam{"Director.StatConcurrencyLimit"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index a6114391e..811989681 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -22,6 +22,7 @@ type Config struct { SlowTransferRampupTime int SlowTransferWindow int StoppedTransferTimeout int + WorkerCount int } ConfigDir string Debug bool @@ -241,6 +242,7 @@ type configWithType struct { SlowTransferRampupTime struct { Type string; Value int } SlowTransferWindow struct { Type string; Value int } StoppedTransferTimeout struct { Type string; Value int } + WorkerCount struct { Type string; Value int } } ConfigDir struct { Type string; Value string } Debug struct { Type string; Value bool } diff --git a/server_ui/advertise.go b/server_ui/advertise.go index bec3ce40c..42f5a42ba 100644 --- a/server_ui/advertise.go +++ b/server_ui/advertise.go @@ -155,7 +155,7 @@ func advertiseInternal(ctx context.Context, server server_utils.XRootDServer) er req.Header.Set("Content-Type", "application/json") req.Header.Set("Authorization", "Bearer "+tok) - userAgent := "pelican-" + strings.ToLower(server.GetServerType().String()) + "/" + config.PelicanVersion + userAgent := "pelican-" + strings.ToLower(server.GetServerType().String()) + "/" + config.GetVersion() req.Header.Set("User-Agent", userAgent) // We should switch this over to use the common transport, but for that to happen diff --git a/web_ui/oauth2_client.go b/web_ui/oauth2_client.go index 8e3ee11a5..eb2be0655 100644 --- a/web_ui/oauth2_client.go +++ b/web_ui/oauth2_client.go @@ -164,9 +164,9 @@ func handleOAuthCallback(ctx *gin.Context) { ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprint("Error requesting user info from CILogon: ", err)}) return } - body, _ := io.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { - ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprint("Error parsing user info from CILogon: ", err)}) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprint("Failed to get OAuth2 user info response: ", err)}) return }