diff --git a/Examples/AddDataSetFiles/AddDataSetFiles.csproj b/Examples/AddDataSetFiles/AddDataSetFiles.csproj
index 2f2785b..617653c 100644
--- a/Examples/AddDataSetFiles/AddDataSetFiles.csproj
+++ b/Examples/AddDataSetFiles/AddDataSetFiles.csproj
@@ -2,7 +2,7 @@
Exe
- net461
+ net6.0
diff --git a/IndicoV2.Abstractions/IndicoV2.Abstractions.csproj b/IndicoV2.Abstractions/IndicoV2.Abstractions.csproj
index 2234226..cc8a52b 100644
--- a/IndicoV2.Abstractions/IndicoV2.Abstractions.csproj
+++ b/IndicoV2.Abstractions/IndicoV2.Abstractions.csproj
@@ -1,7 +1,7 @@
- netstandard2.0;net461
+ net6.0
IndicoV2
diff --git a/IndicoV2.Abstractions/Storage/IStorageClient.cs b/IndicoV2.Abstractions/Storage/IStorageClient.cs
index fd37787..b72b6c6 100644
--- a/IndicoV2.Abstractions/Storage/IStorageClient.cs
+++ b/IndicoV2.Abstractions/Storage/IStorageClient.cs
@@ -16,7 +16,7 @@ public interface IStorageClient
Task> UploadAsync(IEnumerable filePaths, CancellationToken cancellationToken);
Task<(string Name, string Meta)[]> UploadAsync(IEnumerable<(string Path, Stream Content)> files,
- CancellationToken cancellationToken);
+ CancellationToken cancellationToken, int batchSize = 20);
JArray Serialize(IEnumerable filesMetadata);
}
diff --git a/IndicoV2.Abstractions/Submissions/ISubmissionsClient.cs b/IndicoV2.Abstractions/Submissions/ISubmissionsClient.cs
index 667cc5e..fbb8610 100644
--- a/IndicoV2.Abstractions/Submissions/ISubmissionsClient.cs
+++ b/IndicoV2.Abstractions/Submissions/ISubmissionsClient.cs
@@ -34,7 +34,7 @@ public interface ISubmissionsClient
/// Optional. Specifies version to use for the results file.
/// Optional. Flag to batch all files under a single submission id.
/// IEnumerable of submissions ids.
- Task> CreateAsync(int workflowId, IEnumerable<(string Name, Stream Content)> files, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false);
+ Task> CreateAsync(int workflowId, IEnumerable<(string Name, Stream Content)> files, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false, int batchSize = 20);
///
/// Method creates .
@@ -56,7 +56,7 @@ public interface ISubmissionsClient
/// Optional. Specifies version to use for the results file.
/// Optional. Flag to batch all files under a single submission id.
/// of submissions ids.
- Task> CreateAsync(int workflowId, IEnumerable paths, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false);
+ Task> CreateAsync(int workflowId, IEnumerable paths, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false, int batchSize = 20);
///
/// Method lists .
diff --git a/IndicoV2/IndicoV2.csproj b/IndicoV2/IndicoV2.csproj
index e9be698..49c5beb 100644
--- a/IndicoV2/IndicoV2.csproj
+++ b/IndicoV2/IndicoV2.csproj
@@ -1,7 +1,7 @@
- netstandard2.0;net461
+ net6.0
V2
diff --git a/IndicoV2/Storage/StorageClient.cs b/IndicoV2/Storage/StorageClient.cs
index edddc23..5cb0e8e 100644
--- a/IndicoV2/Storage/StorageClient.cs
+++ b/IndicoV2/Storage/StorageClient.cs
@@ -40,7 +40,7 @@ public async Task GetAsync(Uri uri, CancellationToken cancellationToken)
public async Task> UploadAsync(IEnumerable filePaths, CancellationToken cancellationToken)
{
- var metadata = await new UploadFile(_indicoClient) {Files = filePaths.ToList()}.Call(cancellationToken);
+ var metadata = await new UploadFile(_indicoClient) { Files = filePaths.ToList() }.Call(cancellationToken);
return DeserializeMetadata(metadata);
}
@@ -63,25 +63,36 @@ public JArray Serialize(IEnumerable filesMetadata)
public async Task<(string Name, string Meta)> UploadAsync(Stream content,
string filePath,
- CancellationToken cancellationToken) =>
- (await UploadAsync(new[] {(filePath, content)}, cancellationToken)).SingleOrDefault();
+ CancellationToken cancellationToken, int batchSize = 20) =>
+ (await UploadAsync(new[] { (filePath, content) }, cancellationToken, batchSize)).SingleOrDefault();
public async Task<(string Name, string Meta)[]> UploadAsync(IEnumerable<(string Path, Stream Content)> files,
- CancellationToken cancellationToken)
+ CancellationToken cancellationToken, int batchSize = 20)
{
- var content = await CreateRequest(files, cancellationToken);
- var response = await _indicoClient.HttpClient.PostAsync(UploadUri, content, cancellationToken);
-
- using (var reader = new JsonTextReader(new StreamReader(await response.Content.ReadAsStreamAsync())))
+ if (batchSize <= 0)
+ {
+ throw new ArgumentException("Batch size must be greater than 0.");
+ }
+ var results = new (string Name, string Meta)[] { };
+ var batches = files.Chunk(batchSize);
+ foreach (var batch in batches)
{
- var uploadResult = await JArray.LoadAsync(reader, cancellationToken);
+ var content = await CreateRequest(files, cancellationToken);
+ var response = await _indicoClient.HttpClient.PostAsync(UploadUri, content, cancellationToken);
- return uploadResult
- .Select(t => (
- Name: t.Value("name"),
- Meta: t.ToString()))
- .ToArray();
+ using (var reader = new JsonTextReader(new StreamReader(await response.Content.ReadAsStreamAsync())))
+ {
+ var uploadResult = await JArray.LoadAsync(reader, cancellationToken);
+
+ var result = uploadResult
+ .Select(t => (
+ Name: t.Value("name"),
+ Meta: t.ToString()))
+ .ToArray();
+ results = results.Concat(result).ToArray();
+ }
}
+ return results;
}
private async Task CreateRequest(IEnumerable<(string Path, Stream Content)> files, CancellationToken cancellationToken)
diff --git a/IndicoV2/Storage/UploadStream.cs b/IndicoV2/Storage/UploadStream.cs
index d2f0ad3..bfc7645 100644
--- a/IndicoV2/Storage/UploadStream.cs
+++ b/IndicoV2/Storage/UploadStream.cs
@@ -59,6 +59,7 @@ public async Task Call(CancellationToken cancellationToken = default)
return uploadResult;
}
+
}
}
diff --git a/IndicoV2/Submissions/SubmissionsClient.cs b/IndicoV2/Submissions/SubmissionsClient.cs
index de282e1..c75bc5a 100644
--- a/IndicoV2/Submissions/SubmissionsClient.cs
+++ b/IndicoV2/Submissions/SubmissionsClient.cs
@@ -51,9 +51,9 @@ public async Task> CreateAsync(int workflowId, IEnumerable)files, cancellationToken, (SubmissionResultVersion?)resultsFileVersion, bundle);
}
- public async Task> CreateAsync(int workflowId, IEnumerable<(string Name, Stream Content)> filesToUpload, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false)
+ public async Task> CreateAsync(int workflowId, IEnumerable<(string Name, Stream Content)> filesToUpload, CancellationToken cancellationToken = default, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false, int batchSize = 20)
{
- var filesUploaded = await _indicoClient.Storage().UploadAsync(filesToUpload, cancellationToken);
+ var filesUploaded = await _indicoClient.Storage().UploadAsync(filesToUpload, cancellationToken, batchSize: batchSize);
return await _strawberryShakeClient.Submissions().Create(workflowId, filesUploaded, cancellationToken, (SubmissionResultVersion?)resultsFileVersion, bundle);
}
@@ -68,7 +68,7 @@ public async Task> CreateAsync(int workflowId, IEnumerable
await _strawberryShakeClient.Submissions().CreateUri(workflowId, uris, cancellationToken);
public async Task> CreateAsync(int workflowId, IEnumerable paths,
- CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false)
+ CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null, bool bundle = false, int batchSize = 20)
{
var filesToUpload = new List<(string Name, Stream content)>();
foreach (var path in paths)
@@ -84,7 +84,7 @@ public async Task> CreateAsync(int workflowId, IEnumerable