[DEV-11443] Use Batch Upload for workflow submission call

Examples/AddDataSetFiles/AddDataSetFiles.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net461</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

IndicoV2.Abstractions/IndicoV2.Abstractions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
+    <TargetFramework>net6.0</TargetFramework>
     <RootNamespace>IndicoV2</RootNamespace>
   </PropertyGroup>
 

IndicoV2.Abstractions/Storage/IStorageClient.cs

@@ -13,7 +13,7 @@ public interface IStorageClient
         Task<Stream> GetAsync(Uri uri, CancellationToken cancellationToken);
 
         [Obsolete("On this level it's preferable to use streams instead of paths")]
-        Task<IEnumerable<IFileMetadata>> UploadAsync(IEnumerable<string> filePaths, CancellationToken cancellationToken);
+        Task<IEnumerable<IFileMetadata>> UploadAsync(IEnumerable<string> filePaths, CancellationToken cancellationToken, int batchSize);
 
         Task<(string Name, string Meta)[]> UploadAsync(IEnumerable<(string Path, Stream Content)> files,
             CancellationToken cancellationToken);

IndicoV2.IntegrationTests/Storage/StorageClientTests.cs

@@ -31,7 +31,7 @@ public async Task UploadAsync_ShouldReturnUploadedFileInfo()
             var filePath = _dataHelper.Files().GetSampleFilePath();
 
             // Act
-            var uploadedFiles = (await _storageClient.UploadAsync(new[] {filePath}, default)).ToArray();
+            var uploadedFiles = (await _storageClient.UploadAsync(new[] {filePath}, default, batchSize: 20)).ToArray();
 
             // Assert
             uploadedFiles.Should().HaveCount(1);

IndicoV2/DataSets/DataSetClient.cs

@@ -38,7 +38,7 @@ public async Task<IEnumerable<IDataSetFull>> ListFullAsync(int? limit = null, Ca
         public async Task<IDataSetAddFilesResult> AddFilesAsync(int dataSetId, IEnumerable<string> filePaths,
             CancellationToken cancellationToken)
         {
-            var uploadedFiles = await _storage.UploadAsync(filePaths, cancellationToken);
+            var uploadedFiles = await _storage.UploadAsync(filePaths, cancellationToken, batchSize: 20);
             var metadata = _storage.Serialize(uploadedFiles);
             var result = await _dataSetSsClient.AddFiles(dataSetId, metadata.ToString(), cancellationToken);
 

IndicoV2/IndicoV2.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
+    <TargetFramework>net6.0</TargetFramework>
 	  <BuildType>V2</BuildType>
   </PropertyGroup>
 

IndicoV2/Storage/StorageClient.cs

@@ -38,9 +38,9 @@ public async Task<Stream> GetAsync(Uri uri, CancellationToken cancellationToken)
             return result;
         }
 
-        public async Task<IEnumerable<IFileMetadata>> UploadAsync(IEnumerable<string> filePaths, CancellationToken cancellationToken)
+        public async Task<IEnumerable<IFileMetadata>> UploadAsync(IEnumerable<string> filePaths, CancellationToken cancellationToken, int batchSize = 20)
         {
-            var metadata = await new UploadFile(_indicoClient) {Files = filePaths.ToList()}.Call(cancellationToken);
+            var metadata = await new UploadFile(_indicoClient) {Files = filePaths.ToList()}.CallBatched(batchSize, cancellationToken);
 
             return DeserializeMetadata(metadata);
         }

IndicoV2/Storage/UploadFile.cs

@@ -5,6 +5,7 @@
 using System.Threading;
 using System;
 using IndicoV2.Exception;
+using System.Linq;
 
 namespace IndicoV2.Storage
 {
@@ -93,5 +94,17 @@ public async Task<JArray> Call(CancellationToken cancellationToken = default)
 
             return uploadResult;
         }
+
+        public async Task<JArray> CallBatched(int batchSize, CancellationToken cancellationToken = default)
+        {
+            var results = new JArray();
+            var batches = Files.Chunk(batchSize);
+            foreach (var batch in batches)
+            {
+                var result = await new UploadFile(_client) {Files = batch.ToList()}.Call(cancellationToken);
+                results.Merge(result);
+            }
+            return results;
+        }
     }
 }

IndicoV2/Storage/UploadStream.cs

@@ -59,6 +59,7 @@ public async Task<JArray> Call(CancellationToken cancellationToken = default)
 
             return uploadResult;
         }
+
     }
 }