From d84ba7d302bd8bacfef72e8e898a32e5847f9b6a Mon Sep 17 00:00:00 2001 From: Ioannis Sfakianakis Date: Mon, 8 Jan 2024 16:24:15 +0200 Subject: [PATCH] Added error handling when a chuck is not downloaded from an Azure blob - To download a file from an azure blob we use the libraries "github.com/Azure/azure-pipeline-go/pipeline" and "github.com/Azure/azure-storage-blob-go/azblob". We divide the file into chunks and download it chunk by chunk until we download the whole file. For each chunk, we can retry up to 20 times. However, if the download of a chunk fails in all of 20 times (for example due to connectivity issues), we do not handle and report the error. This patch stores all the errors when downloading a chunk and in case all 20 retries have failed it stops the download and reports the error. Signed-off-by: Ioannis Sfakianakis --- zedUpload/azureutil/azure.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/zedUpload/azureutil/azure.go b/zedUpload/azureutil/azure.go index ff58d2c..c962bef 100644 --- a/zedUpload/azureutil/azure.go +++ b/zedUpload/azureutil/azure.go @@ -312,7 +312,20 @@ func DownloadAzureBlob(accountURL, accountName, accountKey, containerName, remot if err != nil { return err } - body := dr.Body(azblob.RetryReaderOptions{MaxRetryRequests: maxRetries}) + var retryErrorStr []string + body := dr.Body(azblob.RetryReaderOptions{ + MaxRetryRequests: maxRetries, + NotifyFailedRead: func(failureCount int, lastError error, offset int64, count int64, willRetry bool) { + retryError := fmt.Errorf("total read failures: %d, offset: %d, error: %v", failureCount, offset, lastError) + retryErrorStr = append(retryErrorStr, retryError.Error()) + if !willRetry { + err = fmt.Errorf(strings.Join(retryErrorStr, "\n")) + } + }, + }) + if err != nil { + return err + } rangeProgress := int64(0) body = pipeline.NewResponseBodyProgress( body,