diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/PollingTentacleBuilder.cs b/source/Octopus.Tentacle.Tests.Integration/Support/PollingTentacleBuilder.cs
index ebb91ebe1..bc0146f32 100644
--- a/source/Octopus.Tentacle.Tests.Integration/Support/PollingTentacleBuilder.cs
+++ b/source/Octopus.Tentacle.Tests.Integration/Support/PollingTentacleBuilder.cs
@@ -19,7 +19,7 @@ public PollingTentacleBuilder(int pollingPort, string serverThumbprint)
 
             ServerThumbprint = serverThumbprint;
         }
-
+        
         internal async Task<RunningTentacle> Build(ILogger log, CancellationToken cancellationToken)
         {
             var instanceName = InstanceNameGenerator();
@@ -46,7 +46,7 @@ internal async Task<RunningTentacle> Build(ILogger log, CancellationToken cancel
                 cancellationToken);
         }
 
-        private void ConfigureTentacleToPollOctopusServer(string configFilePath, Uri subscriptionId, string applicationDirectory)
+        void ConfigureTentacleToPollOctopusServer(string configFilePath, Uri subscriptionId, string applicationDirectory)
         {
             WithWritableTentacleConfiguration(configFilePath, writableTentacleConfiguration =>
             {
diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TentacleNLogFile.cs b/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TentacleNLogFile.cs
new file mode 100644
index 000000000..60b427b22
--- /dev/null
+++ b/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TentacleNLogFile.cs
@@ -0,0 +1,27 @@
+using System.IO;
+using System;
+using Serilog;
+
+namespace Octopus.Tentacle.Tests.Integration.Support.SetupFixtures
+{
+    public static class TentacleNLogFile
+    {
+        public static void TryAdjustForRuntime(TentacleRuntime runtime, ILogger logger, Func<string, string> adjustLogContents)
+        {
+            try
+            {
+                var exePath = TentacleExeFinder.FindTentacleExe(runtime);
+                var exeFileInfo = new FileInfo(exePath);
+                var nlogFileInfo = new FileInfo(Path.Combine(exeFileInfo.DirectoryName!, "Tentacle.exe.nlog"));
+
+                var content = File.ReadAllText(nlogFileInfo.FullName);
+                content = adjustLogContents(content);
+                File.WriteAllText(nlogFileInfo.FullName, content);
+            }
+            catch (Exception e)
+            {
+                logger.Error(e, $"Unable to turn on Trace logging for {runtime}");
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TurnOnExceptionThrowingForLogFileForLatestTentacle.cs b/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TurnOnExceptionThrowingForLogFileForLatestTentacle.cs
new file mode 100644
index 000000000..82c2e7960
--- /dev/null
+++ b/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TurnOnExceptionThrowingForLogFileForLatestTentacle.cs
@@ -0,0 +1,27 @@
+using System;
+using Serilog;
+
+namespace Octopus.Tentacle.Tests.Integration.Support.SetupFixtures
+{
+    public class TurnOnExceptionThrowingForLogFileForLatestTentacle : ISetupFixture
+    {
+        public void OneTimeSetUp(ILogger logger)
+        {
+            // This will help us find errors that normally get swallowed, and become hard to find.
+            TryTurnOnExceptionThrowingForTentacleRuntime(TentacleRuntime.DotNet6, logger);
+            TryTurnOnExceptionThrowingForTentacleRuntime(TentacleRuntime.Framework48, logger);
+        }
+
+        static void TryTurnOnExceptionThrowingForTentacleRuntime(TentacleRuntime runtime, ILogger logger)
+        {
+            TentacleNLogFile.TryAdjustForRuntime(
+                runtime, 
+                logger, 
+                content => content.Replace("throwExceptions=\"false\"", "throwExceptions=\"true\""));
+        }
+
+        public void OneTimeTearDown(ILogger logger)
+        {
+        }
+    }
+}
diff --git a/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TurnOnTraceLoggingForLogFileForLatestTentacle.cs b/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TurnOnTraceLoggingForLogFileForLatestTentacle.cs
index 87897de92..c7b1e4e6e 100644
--- a/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TurnOnTraceLoggingForLogFileForLatestTentacle.cs
+++ b/source/Octopus.Tentacle.Tests.Integration/Support/SetupFixtures/TurnOnTraceLoggingForLogFileForLatestTentacle.cs
@@ -7,36 +7,22 @@ namespace Octopus.Tentacle.Tests.Integration.Support.SetupFixtures
 {
     public class TurnOnTraceLoggingForLogFileForLatestTentacle : ISetupFixture
     {
-        private CancellationTokenSource cts = new();
-
         public void OneTimeSetUp(ILogger logger)
         {
             TryTurnOnTraceLoggingForTentacleRuntime(TentacleRuntime.DotNet6, logger);
             TryTurnOnTraceLoggingForTentacleRuntime(TentacleRuntime.Framework48, logger);
         }
 
-        void TryTurnOnTraceLoggingForTentacleRuntime(TentacleRuntime runtime, ILogger logger)
+        static void TryTurnOnTraceLoggingForTentacleRuntime(TentacleRuntime runtime, ILogger logger)
         {
-            try
-            {
-                var exePath = TentacleExeFinder.FindTentacleExe(runtime);
-                var exeFileInfo = new FileInfo(exePath);
-                var nlogFileInfo = new FileInfo(Path.Combine(exeFileInfo.DirectoryName!, "Tentacle.exe.nlog"));
-
-                var content = File.ReadAllText(nlogFileInfo.FullName);
-                content = content.Replace("<logger name=\"*\" minlevel=\"Info\" writeTo=\"octopus-log-file\" />", "<logger name=\"*\" minlevel=\"Trace\" writeTo=\"octopus-log-file\" />");
-                File.WriteAllText(nlogFileInfo.FullName, content);
-            }
-            catch (Exception e)
-            {
-                logger.Error(e, $"Unable to turn on Trace logging for {runtime}");
-            }
+            TentacleNLogFile.TryAdjustForRuntime(
+                runtime, 
+                logger, 
+                content => content.Replace("<logger name=\"*\" minlevel=\"Info\" writeTo=\"octopus-log-file\" />", "<logger name=\"*\" minlevel=\"Trace\" writeTo=\"octopus-log-file\" />"));
         }
-        
+
         public void OneTimeTearDown(ILogger logger)
         {
-            cts.Cancel();
-            cts.Dispose();
         }
     }
 }
diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs
index 33b1aae28..87e119870 100644
--- a/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs
+++ b/source/Octopus.Tentacle.Tests.Integration/TentacleCommandLineTests.cs
@@ -405,9 +405,16 @@ public async Task ShouldLogStartupDiagnosticsToInstanceLogFileOnly(TentacleConfi
         {
             await using var clientAndTentacle = await tc.CreateBuilder().Build(CancellationToken);
             var startingLogText = clientAndTentacle.RunningTentacle.ReadAllLogFileText();
+            
+            
+            var fs = new FileStream(clientAndTentacle.RunningTentacle.LogFilePath, FileMode.Open,
+                FileAccess.ReadWrite, FileShare.None);
+
 
             var (exitCode, stdout, stderr) = await RunCommandAndAssertExitsWithSuccessExitCode(tc, "show-thumbprint", $"--instance={clientAndTentacle.RunningTentacle.InstanceName}");
 
+            fs.Dispose();
+
             try
             {
                 var logFileText = Policy
@@ -425,7 +432,6 @@ public async Task ShouldLogStartupDiagnosticsToInstanceLogFileOnly(TentacleConfi
                             {
                                 throw new NotLoggedYetException();
                             }
-
                             return newLog;
                         });
 
diff --git a/source/Octopus.Tentacle.Tests.Integration/TentacleIntegrationSetupFixtures.cs b/source/Octopus.Tentacle.Tests.Integration/TentacleIntegrationSetupFixtures.cs
index 69f34b0f6..251bf7b72 100644
--- a/source/Octopus.Tentacle.Tests.Integration/TentacleIntegrationSetupFixtures.cs
+++ b/source/Octopus.Tentacle.Tests.Integration/TentacleIntegrationSetupFixtures.cs
@@ -17,9 +17,10 @@ namespace Octopus.Tentacle.Tests.Integration
     [SetUpFixture] // Must be the one and only. 
     public class TentacleIntegrationSetupFixtures
     {
-        private ISetupFixture[] setupFixtures = new ISetupFixture[]
+        readonly ISetupFixture[] setupFixtures =
         {
             new TurnOnTraceLoggingForLogFileForLatestTentacle(),
+            new TurnOnExceptionThrowingForLogFileForLatestTentacle(),
             new BumpThreadPoolForAllTests(),
             new WarmTentacleCache()
         };