diff --git a/Examples/AddDataSetFiles/AddDataSetFiles.csproj b/Examples/AddDataSetFiles/AddDataSetFiles.csproj index 2f2785b..617653c 100644 --- a/Examples/AddDataSetFiles/AddDataSetFiles.csproj +++ b/Examples/AddDataSetFiles/AddDataSetFiles.csproj @@ -2,7 +2,7 @@ Exe - net461 + net6.0 diff --git a/IndicoV2.Abstractions/IndicoV2.Abstractions.csproj b/IndicoV2.Abstractions/IndicoV2.Abstractions.csproj index 2234226..cc8a52b 100644 --- a/IndicoV2.Abstractions/IndicoV2.Abstractions.csproj +++ b/IndicoV2.Abstractions/IndicoV2.Abstractions.csproj @@ -1,7 +1,7 @@  - netstandard2.0;net461 + net6.0 IndicoV2 diff --git a/IndicoV2.Abstractions/Storage/IStorageClient.cs b/IndicoV2.Abstractions/Storage/IStorageClient.cs index fd37787..b72b6c6 100644 --- a/IndicoV2.Abstractions/Storage/IStorageClient.cs +++ b/IndicoV2.Abstractions/Storage/IStorageClient.cs @@ -16,7 +16,7 @@ public interface IStorageClient Task> UploadAsync(IEnumerable filePaths, CancellationToken cancellationToken); Task<(string Name, string Meta)[]> UploadAsync(IEnumerable<(string Path, Stream Content)> files, - CancellationToken cancellationToken); + CancellationToken cancellationToken, int batchSize = 20); JArray Serialize(IEnumerable filesMetadata); } diff --git a/IndicoV2.Abstractions/Submissions/ISubmissionsClient.cs b/IndicoV2.Abstractions/Submissions/ISubmissionsClient.cs index 667cc5e..fbb8610 100644 --- a/IndicoV2.Abstractions/Submissions/ISubmissionsClient.cs +++ b/IndicoV2.Abstractions/Submissions/ISubmissionsClient.cs @@ -34,7 +34,7 @@ public interface ISubmissionsClient /// Optional. Specifies version to use for the results file. /// Optional. Flag to batch all files under a single submission id. /// IEnumerable of submissions ids. - Task> CreateAsync(int workflowId, IEnumerable<(string Name, Stream Content)> files, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false); + Task> CreateAsync(int workflowId, IEnumerable<(string Name, Stream Content)> files, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false, int batchSize = 20); /// /// Method creates . @@ -56,7 +56,7 @@ public interface ISubmissionsClient /// Optional. Specifies version to use for the results file. /// Optional. Flag to batch all files under a single submission id. /// of submissions ids. - Task> CreateAsync(int workflowId, IEnumerable paths, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false); + Task> CreateAsync(int workflowId, IEnumerable paths, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false, int batchSize = 20); /// /// Method lists . diff --git a/IndicoV2/IndicoV2.csproj b/IndicoV2/IndicoV2.csproj index e9be698..49c5beb 100644 --- a/IndicoV2/IndicoV2.csproj +++ b/IndicoV2/IndicoV2.csproj @@ -1,7 +1,7 @@  - netstandard2.0;net461 + net6.0 V2 diff --git a/IndicoV2/Storage/StorageClient.cs b/IndicoV2/Storage/StorageClient.cs index edddc23..5cb0e8e 100644 --- a/IndicoV2/Storage/StorageClient.cs +++ b/IndicoV2/Storage/StorageClient.cs @@ -40,7 +40,7 @@ public async Task GetAsync(Uri uri, CancellationToken cancellationToken) public async Task> UploadAsync(IEnumerable filePaths, CancellationToken cancellationToken) { - var metadata = await new UploadFile(_indicoClient) {Files = filePaths.ToList()}.Call(cancellationToken); + var metadata = await new UploadFile(_indicoClient) { Files = filePaths.ToList() }.Call(cancellationToken); return DeserializeMetadata(metadata); } @@ -63,25 +63,36 @@ public JArray Serialize(IEnumerable filesMetadata) public async Task<(string Name, string Meta)> UploadAsync(Stream content, string filePath, - CancellationToken cancellationToken) => - (await UploadAsync(new[] {(filePath, content)}, cancellationToken)).SingleOrDefault(); + CancellationToken cancellationToken, int batchSize = 20) => + (await UploadAsync(new[] { (filePath, content) }, cancellationToken, batchSize)).SingleOrDefault(); public async Task<(string Name, string Meta)[]> UploadAsync(IEnumerable<(string Path, Stream Content)> files, - CancellationToken cancellationToken) + CancellationToken cancellationToken, int batchSize = 20) { - var content = await CreateRequest(files, cancellationToken); - var response = await _indicoClient.HttpClient.PostAsync(UploadUri, content, cancellationToken); - - using (var reader = new JsonTextReader(new StreamReader(await response.Content.ReadAsStreamAsync()))) + if (batchSize <= 0) + { + throw new ArgumentException("Batch size must be greater than 0."); + } + var results = new (string Name, string Meta)[] { }; + var batches = files.Chunk(batchSize); + foreach (var batch in batches) { - var uploadResult = await JArray.LoadAsync(reader, cancellationToken); + var content = await CreateRequest(files, cancellationToken); + var response = await _indicoClient.HttpClient.PostAsync(UploadUri, content, cancellationToken); - return uploadResult - .Select(t => ( - Name: t.Value("name"), - Meta: t.ToString())) - .ToArray(); + using (var reader = new JsonTextReader(new StreamReader(await response.Content.ReadAsStreamAsync()))) + { + var uploadResult = await JArray.LoadAsync(reader, cancellationToken); + + var result = uploadResult + .Select(t => ( + Name: t.Value("name"), + Meta: t.ToString())) + .ToArray(); + results = results.Concat(result).ToArray(); + } } + return results; } private async Task CreateRequest(IEnumerable<(string Path, Stream Content)> files, CancellationToken cancellationToken) diff --git a/IndicoV2/Storage/UploadStream.cs b/IndicoV2/Storage/UploadStream.cs index d2f0ad3..bfc7645 100644 --- a/IndicoV2/Storage/UploadStream.cs +++ b/IndicoV2/Storage/UploadStream.cs @@ -59,6 +59,7 @@ public async Task Call(CancellationToken cancellationToken = default) return uploadResult; } + } } diff --git a/IndicoV2/Submissions/SubmissionsClient.cs b/IndicoV2/Submissions/SubmissionsClient.cs index a878469..7a2fbba 100644 --- a/IndicoV2/Submissions/SubmissionsClient.cs +++ b/IndicoV2/Submissions/SubmissionsClient.cs @@ -51,9 +51,9 @@ public async Task> CreateAsync(int workflowId, IEnumerable)files, cancellationToken, (SubmissionResultVersion?) resultsFileVersion, bundle); } - public async Task> CreateAsync(int workflowId, IEnumerable<(string Name, Stream Content)> filesToUpload, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false) + public async Task> CreateAsync(int workflowId, IEnumerable<(string Name, Stream Content)> filesToUpload, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false, int batchSize = 20) { - var filesUploaded = await _indicoClient.Storage().UploadAsync(filesToUpload, cancellationToken); + var filesUploaded = await _indicoClient.Storage().UploadAsync(filesToUpload, cancellationToken, batchSize: batchSize); return await _strawberryShakeClient.Submissions().Create(workflowId, filesUploaded, cancellationToken, (SubmissionResultVersion?)resultsFileVersion, bundle); } @@ -68,7 +68,7 @@ public async Task> CreateAsync(int workflowId, IEnumerable await _strawberryShakeClient.Submissions().CreateUri(workflowId, uris, cancellationToken); public async Task> CreateAsync(int workflowId, IEnumerable paths, - CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false) + CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false, int batchSize = 20) { var filesToUpload = new List<(string Name, Stream content)>(); foreach (var path in paths) @@ -84,7 +84,7 @@ public async Task> CreateAsync(int workflowId, IEnumerable