Attempt to download tentacles multiple times

source/Octopus.Tentacle.Tests.Integration/Support/OctopusPackageDownloader.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
@@ -17,56 +18,99 @@ public class OctopusPackageDownloader
     {
         public static async Task DownloadPackage(string downloadUrl, string filePath, ILogger logger)
         {
+            var exceptions = new List<Exception>();
+            for (int i = 0; i < 5; i++)
+            {
+                try
+                {
+                    await AttemptToDownloadPackage(downloadUrl, filePath, logger);
+                    return;
+                }
+                catch (Exception e)
+                {
+                    exceptions.Add(e);
+                }
+            }
+
+            throw new AggregateException(exceptions);
+        }
+        static async Task AttemptToDownloadPackage(string downloadUrl, string filePath, ILogger logger)
+        {
+            var totalTime = Stopwatch.StartNew();
+            var totalRead = 0L;
             string expectedHash = null;
-            using (var client = new HttpClient())
+            try
             {
-                client.Timeout = TimeSpan.FromSeconds(150);
-                using (var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead))
+                using (var client = new HttpClient())
                 {
-                    response.EnsureSuccessStatusCode();
-                    var totalLength = response.Content.Headers.ContentLength;
-                    if (response.Headers.TryGetValues("x-amz-meta-sha256", out var expectedHashs))
+                    // This appears to be the time it takes to do a single read/write, not the entire download.
+                    client.Timeout = TimeSpan.FromSeconds(20);
+                    using (var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead))
                     {
-                        expectedHash = expectedHashs.FirstOrDefault();
-                    }
+                        response.EnsureSuccessStatusCode();
+                        var totalLength = response.Content.Headers.ContentLength;
+                        expectedHash = TryGetExpectedHashFromHeaders(response, expectedHash);
 
-                    logger.Information($"Downloading {downloadUrl} ({totalLength} bytes)");
-                    var sw = new Stopwatch();
-                    sw.Start();
-                    using (Stream contentStream = await response.Content.ReadAsStreamAsync(),
-                           fileStream = new FileStream(
-                               filePath,
-                               FileMode.Create,
-                               FileAccess.Write,
-                               FileShare.None,
-                               8192,
-                               true))
-                    {
-                        var totalRead = 0L;
-                        var buffer = new byte[8192];
+                        logger.Information($"Downloading {downloadUrl} ({totalLength} bytes)");
 
-                        var read = await contentStream.ReadAsync(buffer, 0, buffer.Length);
-                        while (read != 0)
+                        var sw = new Stopwatch();
+                        sw.Start();
+                        using (Stream contentStream = await response.Content.ReadAsStreamAsync(),
+                               fileStream = new FileStream(
+                                   filePath,
+                                   FileMode.Create,
+                                   FileAccess.Write,
+                                   FileShare.None,
+                                   8192,
+                                   true))
                         {
-                            await fileStream.WriteAsync(buffer, 0, read);
 
-                            if (totalLength.HasValue && sw.ElapsedMilliseconds >= TimeSpan.FromSeconds(7).TotalMilliseconds)
+                            var buffer = new byte[8192];
+
+                            var read = await contentStream.ReadAsync(buffer, 0, buffer.Length);
+                            while (read != 0)
                             {
-                                var percentRead = totalRead * 1.0 / totalLength.Value * 100;
-                                logger.Information($"Downloading Completed {percentRead}%");
-                                sw.Reset();
-                                sw.Start();
+                                await fileStream.WriteAsync(buffer, 0, read);
+
+                                if (totalLength.HasValue && sw.ElapsedMilliseconds >= TimeSpan.FromSeconds(7).TotalMilliseconds)
+                                {
+                                    var percentRead = totalRead * 1.0 / totalLength.Value * 100;
+                                    logger.Information($"Downloading Completed {percentRead}%");
+                                    sw.Reset();
+                                    sw.Start();
+                                }
+
+                                read = await contentStream.ReadAsync(buffer, 0, buffer.Length);
+                                totalRead += read;
                             }
+
+                            totalTime.Stop();
 
-                            read = await contentStream.ReadAsync(buffer, 0, buffer.Length);
-                            totalRead += read;
+                            logger.Information("Download Finished in {totalTime}ms", totalTime.ElapsedMilliseconds);
                         }
-
-                        logger.Information("Download Finished");
                     }
                 }
             }
+            catch (Exception e)
+            {
+                throw new Exception($"Failure to download: {downloadUrl}. After {totalTime.Elapsed.TotalSeconds} seconds we only downloaded, {totalRead}", e);
+            }
 
+            ValidateDownload(filePath, expectedHash);
+        }
+
+        static string TryGetExpectedHashFromHeaders(HttpResponseMessage response, string expectedHash)
+        {
+            if (response.Headers.TryGetValues("x-amz-meta-sha256", out var expectedHashs))
+            {
+                expectedHash = expectedHashs.FirstOrDefault();
+            }
+
+            return expectedHash;
+        }
+
+        static void ValidateDownload(string filePath, string expectedHash)
+        {
             if (!expectedHash.IsNullOrEmpty())
             {
                 using (var sha256 = SHA256.Create())