[DEV-11443] Use Batch Upload for workflow submission call

Examples/AddDataSetFiles/AddDataSetFiles.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net461</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

IndicoV2.Abstractions/IndicoV2.Abstractions.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
+    <TargetFramework>net6.0</TargetFramework>
     <RootNamespace>IndicoV2</RootNamespace>
   </PropertyGroup>
 

IndicoV2.Abstractions/Storage/IStorageClient.cs

@@ -16,7 +16,7 @@ public interface IStorageClient
         Task<IEnumerable<IFileMetadata>> UploadAsync(IEnumerable<string> filePaths, CancellationToken cancellationToken);
 
         Task<(string Name, string Meta)[]> UploadAsync(IEnumerable<(string Path, Stream Content)> files,
-            CancellationToken cancellationToken);
+            CancellationToken cancellationToken, int batchSize = 20);
 
         JArray Serialize(IEnumerable<IFileMetadata> filesMetadata);
     }

IndicoV2/IndicoV2.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
+    <TargetFramework>net6.0</TargetFramework>
 	  <BuildType>V2</BuildType>
   </PropertyGroup>
 

IndicoV2/Storage/StorageClient.cs

@@ -40,7 +40,7 @@ public async Task<Stream> GetAsync(Uri uri, CancellationToken cancellationToken)
 
         public async Task<IEnumerable<IFileMetadata>> UploadAsync(IEnumerable<string> filePaths, CancellationToken cancellationToken)
         {
-            var metadata = await new UploadFile(_indicoClient) {Files = filePaths.ToList()}.Call(cancellationToken);
+            var metadata = await new UploadFile(_indicoClient) { Files = filePaths.ToList() }.Call(cancellationToken);
 
             return DeserializeMetadata(metadata);
         }
@@ -63,25 +63,32 @@ public JArray Serialize(IEnumerable<IFileMetadata> filesMetadata)
 
         public async Task<(string Name, string Meta)> UploadAsync(Stream content,
             string filePath,
-            CancellationToken cancellationToken) =>
-            (await UploadAsync(new[] {(filePath, content)}, cancellationToken)).SingleOrDefault();
+            CancellationToken cancellationToken, int batchSize = 20) =>
+            (await UploadAsync(new[] { (filePath, content) }, cancellationToken, batchSize)).SingleOrDefault();
 
         public async Task<(string Name, string Meta)[]> UploadAsync(IEnumerable<(string Path, Stream Content)> files,
-            CancellationToken cancellationToken)
+            CancellationToken cancellationToken, int batchSize = 20)
         {
-            var content = await CreateRequest(files, cancellationToken);
-            var response = await _indicoClient.HttpClient.PostAsync(UploadUri, content, cancellationToken);
-
-            using (var reader = new JsonTextReader(new StreamReader(await response.Content.ReadAsStreamAsync())))
+            var results = new (string Name, string Meta)[] { };
+            var batches = files.Chunk(batchSize);
+            foreach (var batch in batches)
             {
-                var uploadResult = await JArray.LoadAsync(reader, cancellationToken);
+                var content = await CreateRequest(files, cancellationToken);
+                var response = await _indicoClient.HttpClient.PostAsync(UploadUri, content, cancellationToken);
 
-                return uploadResult
-                    .Select(t => (
-                        Name: t.Value<string>("name"),
-                        Meta: t.ToString()))
-                    .ToArray();
+                using (var reader = new JsonTextReader(new StreamReader(await response.Content.ReadAsStreamAsync())))
+                {
+                    var uploadResult = await JArray.LoadAsync(reader, cancellationToken);
+
+                    var result = uploadResult
+                        .Select(t => (
+                            Name: t.Value<string>("name"),
+                            Meta: t.ToString()))
+                        .ToArray();
+                    results = results.Concat(result).ToArray();
+                }
             }
+            return results;
         }
 
         private async Task<HttpContent> CreateRequest(IEnumerable<(string Path, Stream Content)> files, CancellationToken cancellationToken)

IndicoV2/Storage/UploadFile.cs

@@ -5,6 +5,7 @@
 using System.Threading;
 using System;
 using IndicoV2.Exception;
+using System.Linq;
 
 namespace IndicoV2.Storage
 {

IndicoV2/Storage/UploadStream.cs

@@ -59,6 +59,7 @@ public async Task<JArray> Call(CancellationToken cancellationToken = default)
 
             return uploadResult;
         }
+
     }
 }