Skip to content

Commit

Permalink
More logging to determine disk space issue
Browse files Browse the repository at this point in the history
  • Loading branch information
samdanaei committed Nov 7, 2024
1 parent 6f8e8c9 commit 777d6f2
Showing 1 changed file with 23 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@
using System.Collections;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using FluentAssertions;
using Halibut;
using NUnit.Framework;
using Octopus.Client.Extensions;
using Octopus.Tentacle.CommonTestUtils.Diagnostics;
using Octopus.Tentacle.Contracts.ClientServices;
using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
Expand Down Expand Up @@ -146,6 +148,10 @@ public async Task WhenUploadFileFails_AndTakesLongerThanTheRetryDuration_TheCall
[TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None, additionalParameterTypes: new object[] { typeof(StopPortForwarderAfterFirstCallValues) })]
public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled(TentacleConfigurationTestCase tentacleConfigurationTestCase, bool stopPortForwarderAfterFirstCall)
{
var driveInfos = DriveInfo.GetDrives();

Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}");

PortForwarder portForwarder = null!;
await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder()
// Set a short retry duration so we cancel fairly quickly
Expand Down Expand Up @@ -187,14 +193,18 @@ public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled
.Build(CancellationToken);

portForwarder = clientAndTentacle.PortForwarder;
Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after CreateLegacyBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}");

using var tempFile = new RandomTemporaryFileBuilder().Build();
Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after RandomTemporaryFileBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}");

var inMemoryLog = new InMemoryLog();

// Start the script which will wait for a file to exist
var duration = Stopwatch.StartNew();
var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog);

Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after TentacleClient.DownloadFile: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}");

var expectedException = new ExceptionContractAssertionBuilder(FailureScenario.ConnectionFaulted, tentacleConfigurationTestCase.TentacleType, clientAndTentacle).Build();

Expand All @@ -204,6 +214,8 @@ public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled

recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().BeGreaterOrEqualTo(2);
recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0);

Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after recordedUsages: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}");

// Ensure we actually waited and retried until the timeout policy kicked in
duration.Elapsed.Should().BeGreaterOrEqualTo(clientAndTentacle.RpcRetrySettings.RetryDuration - retryIfRemainingDurationAtLeastBuffer - retryBackoffBuffer);
Expand All @@ -215,6 +227,10 @@ public async Task WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled
[TentacleConfigurations(scriptServiceToTest: ScriptServiceVersionToTest.None)]
public async Task WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut(TentacleConfigurationTestCase tentacleConfigurationTestCase)
{
var driveInfos = DriveInfo.GetDrives();

Logger.Information($"WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space before starting: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}");

var retryDuration = TimeSpan.FromSeconds(15);

await using var clientAndTentacle = await tentacleConfigurationTestCase.CreateBuilder()
Expand All @@ -240,14 +256,20 @@ public async Task WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCa
.Build(CancellationToken);

using var tempFile = new RandomTemporaryFileBuilder().Build();
Logger.Information($"WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space after RandomTemporaryFileBuilder: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}");

var inMemoryLog = new InMemoryLog();
var executeScriptTask = clientAndTentacle.TentacleClient.DownloadFile(tempFile.File.FullName, CancellationToken, inMemoryLog);

Logger.Information($"WhenRpcRetriesTimeOut_DuringDownloadFile_TheRpcCallIsCancelled Available Disk space after TentacleClient.DownloadFile: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}");

Func<Task<DataStream>> action = async () => await executeScriptTask;
await action.Should().ThrowAsync<HalibutClientException>();

recordedUsages.For(nameof(IAsyncClientFileTransferService.DownloadFileAsync)).Started.Should().Be(1);
recordedUsages.For(nameof(IAsyncClientFileTransferService.UploadFileAsync)).Started.Should().Be(0);

Logger.Information($"WhenDownloadFileFails_AndTakesLongerThanTheRetryDuration_TheCallIsNotRetried_AndTimesOut Available Disk space after recordedUsages: {driveInfos.Select(d => $"{d.Name}: {d.AvailableFreeSpace}").ToList().StringJoin(", ")}");

inMemoryLog.ShouldHaveLoggedRetryFailureAndNoRetryAttempts();
}
Expand Down

0 comments on commit 777d6f2

Please sign in to comment.