diff --git a/source/Octopus.Tentacle.Client/AggregateScriptExecutor.cs b/source/Octopus.Tentacle.Client/AggregateScriptExecutor.cs new file mode 100644 index 000000000..533b06fbb --- /dev/null +++ b/source/Octopus.Tentacle.Client/AggregateScriptExecutor.cs @@ -0,0 +1,137 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Halibut; +using Octopus.Tentacle.Client.EventDriven; +using Octopus.Tentacle.Client.Execution; +using Octopus.Tentacle.Client.Observability; +using Octopus.Tentacle.Client.Scripts; +using Octopus.Tentacle.Client.Scripts.Models; +using Octopus.Tentacle.Client.ServiceHelpers; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.Logging; +using Octopus.Tentacle.Contracts.Observability; + +namespace Octopus.Tentacle.Client +{ + /// + /// Executes scripts, on the best available script service. + /// + public class AggregateScriptExecutor : IScriptExecutor + { + readonly ITentacleClientTaskLog logger; + readonly ITentacleClientObserver tentacleClientObserver; + readonly TentacleClientOptions clientOptions; + readonly ClientsHolder clientsHolder; + readonly RpcCallExecutor rpcCallExecutor; + readonly TimeSpan onCancellationAbandonCompleteScriptAfter; + + public AggregateScriptExecutor(ITentacleClientTaskLog logger, + ITentacleClientObserver tentacleClientObserver, + TentacleClientOptions clientOptions, + IHalibutRuntime halibutRuntime, + ServiceEndPoint serviceEndPoint, TimeSpan onCancellationAbandonCompleteScriptAfter) : this(logger, tentacleClientObserver, clientOptions, halibutRuntime, serviceEndPoint, null, onCancellationAbandonCompleteScriptAfter) + { + } + + internal AggregateScriptExecutor(ITentacleClientTaskLog logger, + ITentacleClientObserver tentacleClientObserver, + TentacleClientOptions clientOptions, + IHalibutRuntime halibutRuntime, + ServiceEndPoint serviceEndPoint, + ITentacleServiceDecoratorFactory? tentacleServicesDecoratorFactory, + TimeSpan onCancellationAbandonCompleteScriptAfter) + { + this.logger = logger; + this.tentacleClientObserver = tentacleClientObserver; + this.clientOptions = clientOptions; + this.onCancellationAbandonCompleteScriptAfter = onCancellationAbandonCompleteScriptAfter; + clientsHolder = new ClientsHolder(halibutRuntime, serviceEndPoint, tentacleServicesDecoratorFactory); + rpcCallExecutor = RpcCallExecutorFactory.Create(this.clientOptions.RpcRetrySettings.RetryDuration, this.tentacleClientObserver); + } + + public async Task<(ScriptStatus, ICommandContext)> StartScript(ExecuteScriptCommand executeScriptCommand, + StartScriptIsBeingReAttempted startScriptIsBeingReAttempted, + CancellationToken cancellationToken) + { + var operationMetricsBuilder = ClientOperationMetricsBuilder.Start(); + + // Pick what service to use. + var scriptServiceToUse = await DetermineScriptServiceVersionToUse(cancellationToken, operationMetricsBuilder); + + + var scriptOrchestratorFactory = GetNewScriptOrchestratorFactory(operationMetricsBuilder); + + var orchestrator = scriptOrchestratorFactory.CreateScriptExecutor(scriptServiceToUse); + return await orchestrator.StartScript(executeScriptCommand, startScriptIsBeingReAttempted, cancellationToken); + } + + async Task DetermineScriptServiceVersionToUse(CancellationToken cancellationToken, ClientOperationMetricsBuilder operationMetricsBuilder) + { + try + { + return await new ScriptServicePicker(clientsHolder.CapabilitiesServiceV2, logger, rpcCallExecutor, clientOptions, operationMetricsBuilder) + .DetermineScriptServiceVersionToUse(cancellationToken); + } + catch (Exception ex) when (cancellationToken.IsCancellationRequested) + { + throw new OperationCanceledException("Script execution was cancelled", ex); + } + } + + public async Task<(ScriptStatus, ICommandContext)> GetStatus(ICommandContext ticketForNextNextStatus, CancellationToken cancellationToken) + { + var operationMetricsBuilder = ClientOperationMetricsBuilder.Start(); + + var scriptOrchestratorFactory = GetNewScriptOrchestratorFactory(operationMetricsBuilder); + + var orchestrator = scriptOrchestratorFactory.CreateScriptExecutor(ticketForNextNextStatus.WhichService); + + return await orchestrator.GetStatus(ticketForNextNextStatus, cancellationToken); + } + + public async Task<(ScriptStatus, ICommandContext)> CancelScript(ICommandContext ticketForNextNextStatus, CancellationToken cancellationToken) + { + var operationMetricsBuilder = ClientOperationMetricsBuilder.Start(); + + var scriptOrchestratorFactory = GetNewScriptOrchestratorFactory(operationMetricsBuilder); + + var orchestrator = scriptOrchestratorFactory.CreateScriptExecutor(ticketForNextNextStatus.WhichService); + + return await orchestrator.CancelScript(ticketForNextNextStatus, cancellationToken); + } + + public Task Finish(ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken) + { + throw new NotImplementedException(); + } + + public Task<(ScriptStatus, ICommandContext)> CancelScript(ScriptTicket scriptTicket, CancellationToken cancellationToken) + { + var operationMetricsBuilder = ClientOperationMetricsBuilder.Start(); + throw new System.NotImplementedException(); + } + + public async Task CleanUpScript(ICommandContext ticketForNextNextStatus, CancellationToken cancellationToken) + { + var operationMetricsBuilder = ClientOperationMetricsBuilder.Start(); + + var scriptOrchestratorFactory = GetNewScriptOrchestratorFactory(operationMetricsBuilder); + + var orchestrator = scriptOrchestratorFactory.CreateScriptExecutor(ticketForNextNextStatus.WhichService); + + return await orchestrator.CleanUpScript(ticketForNextNextStatus, cancellationToken); + } + + ScriptOrchestratorFactory GetNewScriptOrchestratorFactory(ClientOperationMetricsBuilder operationMetricsBuilder) + { + return new ScriptOrchestratorFactory(clientsHolder, + rpcCallExecutor, + operationMetricsBuilder, + onCancellationAbandonCompleteScriptAfter, + clientOptions, + logger); + } + + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/EventDriven/ICommandContext.cs b/source/Octopus.Tentacle.Client/EventDriven/ICommandContext.cs new file mode 100644 index 000000000..beeee8887 --- /dev/null +++ b/source/Octopus.Tentacle.Client/EventDriven/ICommandContext.cs @@ -0,0 +1,33 @@ +using Octopus.Tentacle.Client.Scripts; +using Octopus.Tentacle.Contracts; + +namespace Octopus.Tentacle.Client.EventDriven +{ + public interface ICommandContext + { + + ScriptTicket ScriptTicket { get; } + long NextLogSequence { get; } + public ScriptServiceVersion WhichService { get; } + + // TODO does it actually make sense to all of these properties? Perhaps instead we should just expose this concept of + // serialize the entire thing. That way the driver only needs to know how to save something down for the next time it + // wants to continue script execution. + } + + public class DefaultCommandContext : ICommandContext + { + public DefaultCommandContext(ScriptTicket scriptTicket, + long nextLogSequence, + ScriptServiceVersion whichService) + { + ScriptTicket = scriptTicket; + NextLogSequence = nextLogSequence; + WhichService = whichService; + } + + public ScriptTicket ScriptTicket { get; } + public long NextLogSequence { get; } + public ScriptServiceVersion WhichService { get; } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/IScriptExecutor.cs b/source/Octopus.Tentacle.Client/Scripts/IScriptExecutor.cs new file mode 100644 index 000000000..03c53a14b --- /dev/null +++ b/source/Octopus.Tentacle.Client/Scripts/IScriptExecutor.cs @@ -0,0 +1,37 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Octopus.Tentacle.Client.EventDriven; +using Octopus.Tentacle.Client.Scripts.Models; +using Octopus.Tentacle.Contracts; + +namespace Octopus.Tentacle.Client.Scripts +{ + public interface IScriptExecutor { + Task<(ScriptStatus, ICommandContext)> StartScript(ExecuteScriptCommand command, + StartScriptIsBeingReAttempted startScriptIsBeingReAttempted, + CancellationToken scriptExecutionCancellationToken); + + /// + /// Returns a status or null when scriptExecutionCancellationToken is null. + /// + /// + /// + /// + Task<(ScriptStatus, ICommandContext)> GetStatus(ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken); + + Task<(ScriptStatus, ICommandContext)> CancelScript(ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken); + + /// + /// Use this cancel method if only the ScriptTicket is known, e.g. we called StartScript but never got a response. + /// Inheritors + /// + /// + /// + /// + // Task<(ScriptStatus, ICommandContext)> CancelScript(ScriptTicket scriptTicket, CancellationToken cancellationToken); + + + Task CleanUpScript(ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken); + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/IScriptOrchestrator.cs b/source/Octopus.Tentacle.Client/Scripts/IScriptOrchestrator.cs index c0026d809..9bf1ce458 100644 --- a/source/Octopus.Tentacle.Client/Scripts/IScriptOrchestrator.cs +++ b/source/Octopus.Tentacle.Client/Scripts/IScriptOrchestrator.cs @@ -5,7 +5,7 @@ namespace Octopus.Tentacle.Client.Scripts { - interface IScriptOrchestrator + public interface IScriptOrchestrator { Task ExecuteScript(ExecuteScriptCommand command, CancellationToken scriptExecutionCancellationToken); } diff --git a/source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1AlphaOrchestrator.cs b/source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1AlphaOrchestrator.cs index b35234d4a..f0c1b2de4 100644 --- a/source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1AlphaOrchestrator.cs +++ b/source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1AlphaOrchestrator.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Halibut; using Halibut.ServiceModel; +using Octopus.Tentacle.Client.EventDriven; using Octopus.Tentacle.Client.Execution; using Octopus.Tentacle.Client.Observability; using Octopus.Tentacle.Client.Scripts.Models; @@ -15,37 +16,32 @@ namespace Octopus.Tentacle.Client.Scripts { - class KubernetesScriptServiceV1AlphaOrchestrator : ObservingScriptOrchestrator + class KubernetesScriptServiceV1AlphaExecutor : IScriptExecutor { readonly IAsyncClientKubernetesScriptServiceV1Alpha clientKubernetesScriptServiceV1Alpha; readonly RpcCallExecutor rpcCallExecutor; readonly ClientOperationMetricsBuilder clientOperationMetricsBuilder; readonly TimeSpan onCancellationAbandonCompleteScriptAfter; readonly ITentacleClientTaskLog logger; + readonly TentacleClientOptions clientOptions; - public KubernetesScriptServiceV1AlphaOrchestrator( + public KubernetesScriptServiceV1AlphaExecutor( IAsyncClientKubernetesScriptServiceV1Alpha clientKubernetesScriptServiceV1Alpha, - IScriptObserverBackoffStrategy scriptObserverBackOffStrategy, RpcCallExecutor rpcCallExecutor, ClientOperationMetricsBuilder clientOperationMetricsBuilder, - OnScriptStatusResponseReceived onScriptStatusResponseReceived, - OnScriptCompleted onScriptCompleted, TimeSpan onCancellationAbandonCompleteScriptAfter, TentacleClientOptions clientOptions, ITentacleClientTaskLog logger) - : base(scriptObserverBackOffStrategy, - onScriptStatusResponseReceived, - onScriptCompleted, - clientOptions) { this.clientKubernetesScriptServiceV1Alpha = clientKubernetesScriptServiceV1Alpha; this.rpcCallExecutor = rpcCallExecutor; this.clientOperationMetricsBuilder = clientOperationMetricsBuilder; this.onCancellationAbandonCompleteScriptAfter = onCancellationAbandonCompleteScriptAfter; this.logger = logger; + this.clientOptions = clientOptions; } - protected override StartKubernetesScriptCommandV1Alpha Map(ExecuteScriptCommand command) + StartKubernetesScriptCommandV1Alpha Map(ExecuteScriptCommand command) { if (command is not ExecuteKubernetesScriptCommand kubernetesScriptCommand) throw new InvalidOperationException($"Invalid execute script command received. Expected {nameof(ExecuteKubernetesScriptCommand)}, but received {command.GetType().Name}."); @@ -72,17 +68,35 @@ protected override StartKubernetesScriptCommandV1Alpha Map(ExecuteScriptCommand kubernetesScriptCommand.Files.ToArray()); } - protected override ScriptExecutionStatus MapToStatus(KubernetesScriptStatusResponseV1Alpha response) + public ScriptExecutionStatus MapToStatus(KubernetesScriptStatusResponseV1Alpha response) => new(response.Logs); - protected override ScriptExecutionResult MapToResult(KubernetesScriptStatusResponseV1Alpha response) + public ScriptExecutionResult MapToResult(KubernetesScriptStatusResponseV1Alpha response) => new(response.State, response.ExitCode); - protected override ProcessState GetState(KubernetesScriptStatusResponseV1Alpha response) => response.State; + public ProcessState GetState(KubernetesScriptStatusResponseV1Alpha response) => response.State; + + + private ScriptStatus MapToScriptStatus(KubernetesScriptStatusResponseV1Alpha scriptStatusResponse) + { + return new ScriptStatus(scriptStatusResponse.State, scriptStatusResponse.ExitCode, scriptStatusResponse.Logs); + } + + private ICommandContext MapToNextStatus(KubernetesScriptStatusResponseV1Alpha scriptStatusResponse) + { + return new DefaultCommandContext(scriptStatusResponse.ScriptTicket, scriptStatusResponse.NextLogSequence, ScriptServiceVersion.KubernetesScriptServiceVersion1Alpha); + } + + (ScriptStatus, ICommandContext) Map(KubernetesScriptStatusResponseV1Alpha r) + { + return (MapToScriptStatus(r), MapToNextStatus(r)); + } - protected override async Task StartScript(StartKubernetesScriptCommandV1Alpha command, CancellationToken scriptExecutionCancellationToken) + public async Task<(ScriptStatus, ICommandContext)> StartScript(ExecuteScriptCommand executeScriptCommand, + StartScriptIsBeingReAttempted startScriptIsBeingReAttempted, + CancellationToken scriptExecutionCancellationToken) { - KubernetesScriptStatusResponseV1Alpha scriptStatusResponse; + var command = Map(executeScriptCommand); var startScriptCallsConnectedCount = 0; try { @@ -103,14 +117,16 @@ void OnErrorAction(Exception ex) } } - scriptStatusResponse = await rpcCallExecutor.Execute( - retriesEnabled: ClientOptions.RpcRetrySettings.RetriesEnabled, + var scriptStatusResponse = await rpcCallExecutor.Execute( + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, RpcCall.Create(nameof(IKubernetesScriptServiceV1Alpha.StartScript)), StartScriptAction, OnErrorAction, logger, clientOperationMetricsBuilder, scriptExecutionCancellationToken).ConfigureAwait(false); + + return (MapToScriptStatus(scriptStatusResponse), MapToNextStatus(scriptStatusResponse)); } catch (Exception ex) when (scriptExecutionCancellationToken.IsCancellationRequested) { @@ -124,65 +140,47 @@ void OnErrorAction(Exception ex) if (!startScriptCallIsConnecting || startScriptCallIsBeingRetried) { - // We have to assume the script started executing and call CancelScript and CompleteScript - // We don't have a response so we need to create one to continue the execution flow - scriptStatusResponse = new KubernetesScriptStatusResponseV1Alpha( - command.ScriptTicket, - ProcessState.Pending, - ScriptExitCodes.RunningExitCode, - new List(), - 0); - - try - { - await ObserveUntilCompleteThenFinish(scriptStatusResponse, scriptExecutionCancellationToken).ConfigureAwait(false); - } - catch (Exception observerUntilCompleteException) - { - // Throw an error so the caller knows that execution of the script was cancelled - throw new OperationCanceledException("Script execution was cancelled", observerUntilCompleteException); - } - - // Throw an error so the caller knows that execution of the script was cancelled - throw new OperationCanceledException("Script execution was cancelled"); + var scriptStatus = new ScriptStatus(ProcessState.Pending, null, new List()); + var defaultTicketForNextStatus = new DefaultCommandContext(command.ScriptTicket, 0, ScriptServiceVersion.KubernetesScriptServiceVersion1Alpha); + return (scriptStatus, defaultTicketForNextStatus); } // If the StartScript call was not in-flight or being retries then we know the script has not started executing on Tentacle // So can exit without calling CancelScript or CompleteScript throw new OperationCanceledException("Script execution was cancelled", ex); } + } - return scriptStatusResponse; + public async Task<(ScriptStatus, ICommandContext)> GetStatus(ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken) + { + return Map(await _GetStatus(commandContext, scriptExecutionCancellationToken)); } - protected override async Task GetStatus(KubernetesScriptStatusResponseV1Alpha lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + async Task _GetStatus(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { - try + async Task GetStatusAction(CancellationToken ct) { - async Task GetStatusAction(CancellationToken ct) - { - var request = new KubernetesScriptStatusRequestV1Alpha(lastStatusResponse.ScriptTicket, lastStatusResponse.NextLogSequence); - var result = await clientKubernetesScriptServiceV1Alpha.GetStatusAsync(request, new HalibutProxyRequestOptions(ct)); - - return result; - } + var request = new KubernetesScriptStatusRequestV1Alpha(lastStatusResponse.ScriptTicket, lastStatusResponse.NextLogSequence); + var result = await clientKubernetesScriptServiceV1Alpha.GetStatusAsync(request, new HalibutProxyRequestOptions(ct)); - return await rpcCallExecutor.Execute( - retriesEnabled: ClientOptions.RpcRetrySettings.RetriesEnabled, - RpcCall.Create(nameof(IKubernetesScriptServiceV1Alpha.GetStatus)), - GetStatusAction, - logger, - clientOperationMetricsBuilder, - scriptExecutionCancellationToken).ConfigureAwait(false); - } - catch (Exception e) when (e is OperationCanceledException && scriptExecutionCancellationToken.IsCancellationRequested) - { - // Return the last known response without logs when cancellation occurs and let the script execution go into the CancelScript and CompleteScript flow - return new KubernetesScriptStatusResponseV1Alpha(lastStatusResponse.ScriptTicket, lastStatusResponse.State, lastStatusResponse.ExitCode, new List(), lastStatusResponse.NextLogSequence); + return result; } + + return await rpcCallExecutor.Execute( + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, + RpcCall.Create(nameof(IKubernetesScriptServiceV1Alpha.GetStatus)), + GetStatusAction, + logger, + clientOperationMetricsBuilder, + scriptExecutionCancellationToken).ConfigureAwait(false); } - protected override async Task Cancel(KubernetesScriptStatusResponseV1Alpha lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + public async Task<(ScriptStatus, ICommandContext)> CancelScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + { + return Map(await _Cancel(lastStatusResponse, scriptExecutionCancellationToken)); + } + + async Task _Cancel(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { async Task CancelScriptAction(CancellationToken ct) { @@ -197,7 +195,7 @@ async Task CancelScriptAction(Cancellatio // We could potentially reduce the time to failure by not retrying the cancel RPC Call if the previous RPC call was already triggering RPC Retries. return await rpcCallExecutor.Execute( - retriesEnabled: ClientOptions.RpcRetrySettings.RetriesEnabled, + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, RpcCall.Create(nameof(IKubernetesScriptServiceV1Alpha.CancelScript)), CancelScriptAction, logger, @@ -206,7 +204,8 @@ async Task CancelScriptAction(Cancellatio CancellationToken.None).ConfigureAwait(false); } - protected override async Task Finish(KubernetesScriptStatusResponseV1Alpha lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + + public async Task CleanUpScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { try { @@ -233,7 +232,7 @@ await rpcCallExecutor.ExecuteWithNoRetries( logger.Verbose(ex); } - return lastStatusResponse; + return null; } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1Orchestrator.cs b/source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1Orchestrator.cs index b4e614c57..92cb7c4a1 100644 --- a/source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1Orchestrator.cs +++ b/source/Octopus.Tentacle.Client/Scripts/KubernetesScriptServiceV1Orchestrator.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Halibut; using Halibut.ServiceModel; +using Octopus.Tentacle.Client.EventDriven; using Octopus.Tentacle.Client.Execution; using Octopus.Tentacle.Client.Observability; using Octopus.Tentacle.Client.Scripts.Models; @@ -15,37 +16,32 @@ namespace Octopus.Tentacle.Client.Scripts { - class KubernetesScriptServiceV1Orchestrator : ObservingScriptOrchestrator + class KubernetesScriptServiceV1Executor : IScriptExecutor { readonly IAsyncClientKubernetesScriptServiceV1 clientKubernetesScriptServiceV1; readonly RpcCallExecutor rpcCallExecutor; readonly ClientOperationMetricsBuilder clientOperationMetricsBuilder; readonly TimeSpan onCancellationAbandonCompleteScriptAfter; readonly ITentacleClientTaskLog logger; + readonly TentacleClientOptions clientOptions; - public KubernetesScriptServiceV1Orchestrator( + public KubernetesScriptServiceV1Executor( IAsyncClientKubernetesScriptServiceV1 clientKubernetesScriptServiceV1, - IScriptObserverBackoffStrategy scriptObserverBackOffStrategy, RpcCallExecutor rpcCallExecutor, ClientOperationMetricsBuilder clientOperationMetricsBuilder, - OnScriptStatusResponseReceived onScriptStatusResponseReceived, - OnScriptCompleted onScriptCompleted, TimeSpan onCancellationAbandonCompleteScriptAfter, TentacleClientOptions clientOptions, ITentacleClientTaskLog logger) - : base(scriptObserverBackOffStrategy, - onScriptStatusResponseReceived, - onScriptCompleted, - clientOptions) { this.clientKubernetesScriptServiceV1 = clientKubernetesScriptServiceV1; this.rpcCallExecutor = rpcCallExecutor; this.clientOperationMetricsBuilder = clientOperationMetricsBuilder; this.onCancellationAbandonCompleteScriptAfter = onCancellationAbandonCompleteScriptAfter; + this.clientOptions = clientOptions; this.logger = logger; } - protected override StartKubernetesScriptCommandV1 Map(ExecuteScriptCommand command) + StartKubernetesScriptCommandV1 Map(ExecuteScriptCommand command) { if (command is not ExecuteKubernetesScriptCommand kubernetesScriptCommand) throw new InvalidOperationException($"Invalid execute script command received. Expected {nameof(ExecuteKubernetesScriptCommand)}, but received {command.GetType().Name}."); @@ -72,18 +68,26 @@ protected override StartKubernetesScriptCommandV1 Map(ExecuteScriptCommand comma kubernetesScriptCommand.Files.ToArray(), kubernetesScriptCommand.IsRawScript); } + (ScriptStatus, ICommandContext) Map(KubernetesScriptStatusResponseV1 r) + { + return (MapToScriptStatus(r), MapToNextStatus(r)); + } + + private ScriptStatus MapToScriptStatus(KubernetesScriptStatusResponseV1 scriptStatusResponse) + { + return new ScriptStatus(scriptStatusResponse.State, scriptStatusResponse.ExitCode, scriptStatusResponse.Logs); + } - protected override ScriptExecutionStatus MapToStatus(KubernetesScriptStatusResponseV1 response) - => new(response.Logs); - - protected override ScriptExecutionResult MapToResult(KubernetesScriptStatusResponseV1 response) - => new(response.State, response.ExitCode); - - protected override ProcessState GetState(KubernetesScriptStatusResponseV1 response) => response.State; + private ICommandContext MapToNextStatus(KubernetesScriptStatusResponseV1 scriptStatusResponse) + { + return new DefaultCommandContext(scriptStatusResponse.ScriptTicket, scriptStatusResponse.NextLogSequence, ScriptServiceVersion.KubernetesScriptServiceVersion1); + } - protected override async Task StartScript(StartKubernetesScriptCommandV1 command, CancellationToken scriptExecutionCancellationToken) + public async Task<(ScriptStatus, ICommandContext)> StartScript(ExecuteScriptCommand executeScriptCommand, + StartScriptIsBeingReAttempted startScriptIsBeingReAttempted, + CancellationToken scriptExecutionCancellationToken) { - KubernetesScriptStatusResponseV1 scriptStatusResponse; + var command = Map(executeScriptCommand); var startScriptCallsConnectedCount = 0; try { @@ -104,14 +108,16 @@ void OnErrorAction(Exception ex) } } - scriptStatusResponse = await rpcCallExecutor.Execute( - retriesEnabled: ClientOptions.RpcRetrySettings.RetriesEnabled, + var scriptStatusResponse = await rpcCallExecutor.Execute( + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, RpcCall.Create(nameof(IKubernetesScriptServiceV1.StartScript)), StartScriptAction, OnErrorAction, logger, clientOperationMetricsBuilder, scriptExecutionCancellationToken).ConfigureAwait(false); + + return (MapToScriptStatus(scriptStatusResponse), MapToNextStatus(scriptStatusResponse)); } catch (Exception ex) when (scriptExecutionCancellationToken.IsCancellationRequested) { @@ -125,65 +131,47 @@ void OnErrorAction(Exception ex) if (!startScriptCallIsConnecting || startScriptCallIsBeingRetried) { - // We have to assume the script started executing and call CancelScript and CompleteScript - // We don't have a response so we need to create one to continue the execution flow - scriptStatusResponse = new KubernetesScriptStatusResponseV1( - command.ScriptTicket, - ProcessState.Pending, - ScriptExitCodes.RunningExitCode, - new List(), - 0); - - try - { - await ObserveUntilCompleteThenFinish(scriptStatusResponse, scriptExecutionCancellationToken).ConfigureAwait(false); - } - catch (Exception observerUntilCompleteException) - { - // Throw an error so the caller knows that execution of the script was cancelled - throw new OperationCanceledException("Script execution was cancelled", observerUntilCompleteException); - } - - // Throw an error so the caller knows that execution of the script was cancelled - throw new OperationCanceledException("Script execution was cancelled"); + var scriptStatus = new ScriptStatus(ProcessState.Pending, null, new List()); + var defaultTicketForNextStatus = new DefaultCommandContext(command.ScriptTicket, 0, ScriptServiceVersion.KubernetesScriptServiceVersion1); + return (scriptStatus, defaultTicketForNextStatus); } // If the StartScript call was not in-flight or being retries then we know the script has not started executing on Tentacle // So can exit without calling CancelScript or CompleteScript throw new OperationCanceledException("Script execution was cancelled", ex); } + } - return scriptStatusResponse; + public async Task<(ScriptStatus, ICommandContext)> GetStatus(ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken) + { + return Map(await _GetStatus(commandContext, scriptExecutionCancellationToken)); } - protected override async Task GetStatus(KubernetesScriptStatusResponseV1 lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + private async Task _GetStatus(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { - try + async Task GetStatusAction(CancellationToken ct) { - async Task GetStatusAction(CancellationToken ct) - { - var request = new KubernetesScriptStatusRequestV1(lastStatusResponse.ScriptTicket, lastStatusResponse.NextLogSequence); - var result = await clientKubernetesScriptServiceV1.GetStatusAsync(request, new HalibutProxyRequestOptions(ct)); + var request = new KubernetesScriptStatusRequestV1(lastStatusResponse.ScriptTicket, lastStatusResponse.NextLogSequence); + var result = await clientKubernetesScriptServiceV1.GetStatusAsync(request, new HalibutProxyRequestOptions(ct)); - return result; - } - - return await rpcCallExecutor.Execute( - retriesEnabled: ClientOptions.RpcRetrySettings.RetriesEnabled, - RpcCall.Create(nameof(IKubernetesScriptServiceV1.GetStatus)), - GetStatusAction, - logger, - clientOperationMetricsBuilder, - scriptExecutionCancellationToken).ConfigureAwait(false); - } - catch (Exception e) when (e is OperationCanceledException && scriptExecutionCancellationToken.IsCancellationRequested) - { - // Return the last known response without logs when cancellation occurs and let the script execution go into the CancelScript and CompleteScript flow - return new KubernetesScriptStatusResponseV1(lastStatusResponse.ScriptTicket, lastStatusResponse.State, lastStatusResponse.ExitCode, new List(), lastStatusResponse.NextLogSequence); + return result; } + + return await rpcCallExecutor.Execute( + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, + RpcCall.Create(nameof(IKubernetesScriptServiceV1.GetStatus)), + GetStatusAction, + logger, + clientOperationMetricsBuilder, + scriptExecutionCancellationToken).ConfigureAwait(false); + } + + public async Task<(ScriptStatus, ICommandContext)> CancelScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + { + return Map(await _GetStatus(lastStatusResponse, scriptExecutionCancellationToken)); } - protected override async Task Cancel(KubernetesScriptStatusResponseV1 lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + async Task _CancelScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { async Task CancelScriptAction(CancellationToken ct) { @@ -198,7 +186,7 @@ async Task CancelScriptAction(CancellationToke // We could potentially reduce the time to failure by not retrying the cancel RPC Call if the previous RPC call was already triggering RPC Retries. return await rpcCallExecutor.Execute( - retriesEnabled: ClientOptions.RpcRetrySettings.RetriesEnabled, + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, RpcCall.Create(nameof(IKubernetesScriptServiceV1.CancelScript)), CancelScriptAction, logger, @@ -207,7 +195,7 @@ async Task CancelScriptAction(CancellationToke CancellationToken.None).ConfigureAwait(false); } - protected override async Task Finish(KubernetesScriptStatusResponseV1 lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + public async Task CleanUpScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { try { @@ -234,7 +222,7 @@ await rpcCallExecutor.ExecuteWithNoRetries( logger.Verbose(ex); } - return lastStatusResponse; + return null; } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/ObservingScriptOrchestrator.cs b/source/Octopus.Tentacle.Client/Scripts/ObservingScriptOrchestrator.cs index 47d7fd64c..c00f5fcaa 100644 --- a/source/Octopus.Tentacle.Client/Scripts/ObservingScriptOrchestrator.cs +++ b/source/Octopus.Tentacle.Client/Scripts/ObservingScriptOrchestrator.cs @@ -1,26 +1,31 @@ using System; using System.Threading; using System.Threading.Tasks; +using Octopus.Tentacle.Client.EventDriven; using Octopus.Tentacle.Client.Scripts.Models; using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.Logging; namespace Octopus.Tentacle.Client.Scripts { - abstract class ObservingScriptOrchestrator : IScriptOrchestrator + public sealed class ObservingScriptOrchestrator : IScriptOrchestrator { readonly IScriptObserverBackoffStrategy scriptObserverBackOffStrategy; readonly OnScriptStatusResponseReceived onScriptStatusResponseReceived; readonly OnScriptCompleted onScriptCompleted; + readonly ITentacleClientTaskLog logger; - protected TentacleClientOptions ClientOptions { get; } + IScriptExecutor scriptExecutor; - protected ObservingScriptOrchestrator( + public ObservingScriptOrchestrator( IScriptObserverBackoffStrategy scriptObserverBackOffStrategy, OnScriptStatusResponseReceived onScriptStatusResponseReceived, OnScriptCompleted onScriptCompleted, - TentacleClientOptions clientOptions) + IScriptExecutor scriptExecutor, + ITentacleClientTaskLog logger) { - ClientOptions = clientOptions; + this.scriptExecutor = scriptExecutor; + this.logger = logger; this.scriptObserverBackOffStrategy = scriptObserverBackOffStrategy; this.onScriptStatusResponseReceived = onScriptStatusResponseReceived; this.onScriptCompleted = onScriptCompleted; @@ -28,11 +33,11 @@ protected ObservingScriptOrchestrator( public async Task ExecuteScript(ExecuteScriptCommand command, CancellationToken scriptExecutionCancellationToken) { - var mappedStartCommand = Map(command); + var (scriptStatus, ticketForNextStatus) = await scriptExecutor.StartScript(command, + StartScriptIsBeingReAttempted.FirstAttempt, // This is not re-entrant so this should be true. + scriptExecutionCancellationToken).ConfigureAwait(false); - var scriptStatusResponse = await StartScript(mappedStartCommand, scriptExecutionCancellationToken).ConfigureAwait(false); - - scriptStatusResponse = await ObserveUntilCompleteThenFinish(scriptStatusResponse, scriptExecutionCancellationToken).ConfigureAwait(false); + (scriptStatus, _) = await ObserveUntilCompleteThenFinish(scriptStatus, ticketForNextStatus, scriptExecutionCancellationToken).ConfigureAwait(false); if (scriptExecutionCancellationToken.IsCancellationRequested) { @@ -40,45 +45,49 @@ public async Task ExecuteScript(ExecuteScriptCommand comm throw new OperationCanceledException("Script execution was cancelled"); } - var mappedResponse = MapToResult(scriptStatusResponse); - - return new ScriptExecutionResult(mappedResponse.State, mappedResponse.ExitCode); + + return new ScriptExecutionResult(scriptStatus.State, scriptStatus.ExitCode!.Value); } - protected async Task ObserveUntilCompleteThenFinish( - TScriptStatusResponse scriptStatusResponse, + async Task<(ScriptStatus, ICommandContext)> ObserveUntilCompleteThenFinish( + ScriptStatus scriptStatus, + ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken) { - OnScriptStatusResponseReceived(scriptStatusResponse); + OnScriptStatusResponseReceived(scriptStatus); - var lastScriptStatus = await ObserveUntilComplete(scriptStatusResponse, scriptExecutionCancellationToken).ConfigureAwait(false); + var (lastStatusResponse, lastTicketForNextStatus) = await ObserveUntilComplete(scriptStatus, commandContext, scriptExecutionCancellationToken).ConfigureAwait(false); await onScriptCompleted(scriptExecutionCancellationToken).ConfigureAwait(false); + + lastStatusResponse = await scriptExecutor.CleanUpScript(lastTicketForNextStatus, scriptExecutionCancellationToken).ConfigureAwait(false) ?? lastStatusResponse; - lastScriptStatus = await Finish(lastScriptStatus, scriptExecutionCancellationToken).ConfigureAwait(false); + OnScriptStatusResponseReceived(lastStatusResponse); - return lastScriptStatus; + return (lastStatusResponse, lastTicketForNextStatus); } - async Task ObserveUntilComplete( - TScriptStatusResponse scriptStatusResponse, + async Task<(ScriptStatus lastStatusResponse, ICommandContext lastTicketForNextStatus)> ObserveUntilComplete( + ScriptStatus scriptStatus, + ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken) { - var lastStatusResponse = scriptStatusResponse; + var lastTicketForNextStatus = commandContext; + var lastStatusResponse = scriptStatus; var iteration = 0; var cancellationIteration = 0; - while (GetState(lastStatusResponse) != ProcessState.Complete) + while (lastStatusResponse.State != ProcessState.Complete) { if (scriptExecutionCancellationToken.IsCancellationRequested) { - lastStatusResponse = await Cancel(lastStatusResponse, scriptExecutionCancellationToken).ConfigureAwait(false); + (lastStatusResponse, lastTicketForNextStatus) = await scriptExecutor.CancelScript(lastTicketForNextStatus, scriptExecutionCancellationToken).ConfigureAwait(false); } else { try { - lastStatusResponse = await GetStatus(lastStatusResponse, scriptExecutionCancellationToken).ConfigureAwait(false); + (lastStatusResponse, lastTicketForNextStatus) = await scriptExecutor.GetStatus(lastTicketForNextStatus, scriptExecutionCancellationToken).ConfigureAwait(false); } catch (Exception) { @@ -93,7 +102,7 @@ async Task ObserveUntilComplete( OnScriptStatusResponseReceived(lastStatusResponse); - if (GetState(lastStatusResponse) == ProcessState.Complete) + if (lastStatusResponse.State == ProcessState.Complete) { continue; } @@ -112,28 +121,14 @@ await Task.Delay(scriptObserverBackOffStrategy.GetBackoff(++iteration), scriptEx } } - return lastStatusResponse; + new ShortCutTakenHere(); + return (lastStatusResponse, lastTicketForNextStatus); } - protected abstract TStartCommand Map(ExecuteScriptCommand command); - - protected abstract ScriptExecutionStatus MapToStatus(TScriptStatusResponse response); - - protected abstract ScriptExecutionResult MapToResult(TScriptStatusResponse response); - - protected abstract ProcessState GetState(TScriptStatusResponse response); - - protected abstract Task StartScript(TStartCommand command, CancellationToken scriptExecutionCancellationToken); - - protected abstract Task GetStatus(TScriptStatusResponse lastStatusResponse, CancellationToken scriptExecutionCancellationToken); - - protected abstract Task Cancel(TScriptStatusResponse lastStatusResponse, CancellationToken scriptExecutionCancellationToken); - - protected abstract Task Finish(TScriptStatusResponse lastStatusResponse, CancellationToken scriptExecutionCancellationToken); - - protected void OnScriptStatusResponseReceived(TScriptStatusResponse scriptStatusResponse) + void OnScriptStatusResponseReceived(ScriptStatus scriptStatusResponse) { - onScriptStatusResponseReceived(MapToStatus(scriptStatusResponse)); + var scriptExecutionStatus = new ScriptExecutionStatus(scriptStatusResponse.Logs); + onScriptStatusResponseReceived(scriptExecutionStatus); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/ScriptOrchestratorFactory.cs b/source/Octopus.Tentacle.Client/Scripts/ScriptOrchestratorFactory.cs index 211db51f2..55d1cd5fe 100644 --- a/source/Octopus.Tentacle.Client/Scripts/ScriptOrchestratorFactory.cs +++ b/source/Octopus.Tentacle.Client/Scripts/ScriptOrchestratorFactory.cs @@ -1,99 +1,55 @@ using System; -using System.Threading; -using System.Threading.Tasks; -using Halibut.ServiceModel; -using Octopus.Tentacle.Client.Capabilities; using Octopus.Tentacle.Client.Execution; -using Octopus.Tentacle.Client.Kubernetes; using Octopus.Tentacle.Client.Observability; -using Octopus.Tentacle.Contracts.Capabilities; -using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Client.ServiceHelpers; using Octopus.Tentacle.Contracts.Logging; -using Octopus.Tentacle.Contracts.Observability; namespace Octopus.Tentacle.Client.Scripts { - class ScriptOrchestratorFactory : IScriptOrchestratorFactory + // TODO: this is not an orchestrator factory. + class ScriptOrchestratorFactory { - readonly IScriptObserverBackoffStrategy scriptObserverBackOffStrategy; readonly RpcCallExecutor rpcCallExecutor; readonly ClientOperationMetricsBuilder clientOperationMetricsBuilder; - readonly OnScriptStatusResponseReceived onScriptStatusResponseReceived; - readonly OnScriptCompleted onScriptCompleted; readonly TimeSpan onCancellationAbandonCompleteScriptAfter; readonly ITentacleClientTaskLog logger; - readonly IAsyncClientScriptService clientScriptServiceV1; - readonly IAsyncClientScriptServiceV2 clientScriptServiceV2; - readonly IAsyncClientKubernetesScriptServiceV1Alpha clientKubernetesScriptServiceV1Alpha; - readonly IAsyncClientKubernetesScriptServiceV1 clientKubernetesScriptServiceV1; - readonly IAsyncClientCapabilitiesServiceV2 clientCapabilitiesServiceV2; + readonly ClientsHolder clientsHolder; readonly TentacleClientOptions clientOptions; public ScriptOrchestratorFactory( - IAsyncClientScriptService clientScriptServiceV1, - IAsyncClientScriptServiceV2 clientScriptServiceV2, - IAsyncClientKubernetesScriptServiceV1Alpha clientKubernetesScriptServiceV1Alpha, - IAsyncClientKubernetesScriptServiceV1 clientKubernetesScriptServiceV1, - IAsyncClientCapabilitiesServiceV2 clientCapabilitiesServiceV2, - IScriptObserverBackoffStrategy scriptObserverBackOffStrategy, + ClientsHolder clientsHolder, RpcCallExecutor rpcCallExecutor, ClientOperationMetricsBuilder clientOperationMetricsBuilder, - OnScriptStatusResponseReceived onScriptStatusResponseReceived, - OnScriptCompleted onScriptCompleted, TimeSpan onCancellationAbandonCompleteScriptAfter, TentacleClientOptions clientOptions, ITentacleClientTaskLog logger) { - this.clientScriptServiceV1 = clientScriptServiceV1; - this.clientScriptServiceV2 = clientScriptServiceV2; - this.clientKubernetesScriptServiceV1Alpha = clientKubernetesScriptServiceV1Alpha; - this.clientKubernetesScriptServiceV1 = clientKubernetesScriptServiceV1; - this.clientCapabilitiesServiceV2 = clientCapabilitiesServiceV2; - this.scriptObserverBackOffStrategy = scriptObserverBackOffStrategy; + this.clientsHolder = clientsHolder; this.rpcCallExecutor = rpcCallExecutor; this.clientOperationMetricsBuilder = clientOperationMetricsBuilder; - this.onScriptStatusResponseReceived = onScriptStatusResponseReceived; - this.onScriptCompleted = onScriptCompleted; this.onCancellationAbandonCompleteScriptAfter = onCancellationAbandonCompleteScriptAfter; this.clientOptions = clientOptions; this.logger = logger; } - public async Task CreateOrchestrator(CancellationToken cancellationToken) + public IScriptExecutor CreateScriptExecutor(ScriptServiceVersion scriptServiceToUse) { - ScriptServiceVersion scriptServiceToUse; - try - { - scriptServiceToUse = await DetermineScriptServiceVersionToUse(cancellationToken); - } - catch (Exception ex) when (cancellationToken.IsCancellationRequested) - { - throw new OperationCanceledException("Script execution was cancelled", ex); - } - if (scriptServiceToUse == ScriptServiceVersion.ScriptServiceVersion1) { - return new ScriptServiceV1Orchestrator( - clientScriptServiceV1, - scriptObserverBackOffStrategy, + return new ScriptServiceV1Executor( + clientsHolder.ScriptServiceV1, rpcCallExecutor, clientOperationMetricsBuilder, - onScriptStatusResponseReceived, - onScriptCompleted, - clientOptions, logger); } if (scriptServiceToUse == ScriptServiceVersion.ScriptServiceVersion2) { - return new ScriptServiceV2Orchestrator( - clientScriptServiceV2, - scriptObserverBackOffStrategy, + return new ScriptServiceV2Executor( + clientsHolder.ScriptServiceV2, rpcCallExecutor, clientOperationMetricsBuilder, - onScriptStatusResponseReceived, - onScriptCompleted, onCancellationAbandonCompleteScriptAfter, clientOptions, logger); @@ -101,13 +57,10 @@ public async Task CreateOrchestrator(CancellationToken canc if (scriptServiceToUse == ScriptServiceVersion.KubernetesScriptServiceVersion1Alpha) { - return new KubernetesScriptServiceV1AlphaOrchestrator( - clientKubernetesScriptServiceV1Alpha, - scriptObserverBackOffStrategy, + return new KubernetesScriptServiceV1AlphaExecutor( + clientsHolder.KubernetesScriptServiceV1Alpha, rpcCallExecutor, clientOperationMetricsBuilder, - onScriptStatusResponseReceived, - onScriptCompleted, onCancellationAbandonCompleteScriptAfter, clientOptions, logger); @@ -115,13 +68,10 @@ public async Task CreateOrchestrator(CancellationToken canc if (scriptServiceToUse == ScriptServiceVersion.KubernetesScriptServiceVersion1) { - return new KubernetesScriptServiceV1Orchestrator( - clientKubernetesScriptServiceV1, - scriptObserverBackOffStrategy, + return new KubernetesScriptServiceV1Executor( + clientsHolder.KubernetesScriptServiceV1, rpcCallExecutor, clientOperationMetricsBuilder, - onScriptStatusResponseReceived, - onScriptCompleted, onCancellationAbandonCompleteScriptAfter, clientOptions, logger); @@ -130,72 +80,6 @@ public async Task CreateOrchestrator(CancellationToken canc throw new InvalidOperationException($"Unknown ScriptServiceVersion {scriptServiceToUse}"); } - async Task DetermineScriptServiceVersionToUse(CancellationToken cancellationToken) - { - logger.Verbose("Determining ScriptService version to use"); - - async Task GetCapabilitiesFunc(CancellationToken ct) - { - var result = await clientCapabilitiesServiceV2.GetCapabilitiesAsync(new HalibutProxyRequestOptions(ct)); - - return result; - } - - var tentacleCapabilities = await rpcCallExecutor.Execute( - retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, - RpcCall.Create(nameof(ICapabilitiesServiceV2.GetCapabilities)), - GetCapabilitiesFunc, - logger, - clientOperationMetricsBuilder, - cancellationToken); - - logger.Verbose($"Discovered Tentacle capabilities: {string.Join(",", tentacleCapabilities.SupportedCapabilities)}"); - - // Check if we support any kubernetes script service. - // It's implied (and tested) that GetCapabilities will only return Kubernetes or non-Kubernetes script services, never a mix - if (tentacleCapabilities.HasAnyKubernetesScriptService()) - { - return DetermineKubernetesScriptServiceVersionToUse(tentacleCapabilities); - } - - return DetermineShellScriptServiceVersionToUse(tentacleCapabilities); - } - - ScriptServiceVersion DetermineShellScriptServiceVersionToUse(CapabilitiesResponseV2 tentacleCapabilities) - { - if (tentacleCapabilities.HasScriptServiceV2()) - { - logger.Verbose("Using ScriptServiceV2"); - logger.Verbose(clientOptions.RpcRetrySettings.RetriesEnabled - ? $"RPC call retries are enabled. Retry timeout {rpcCallExecutor.RetryTimeout.TotalSeconds} seconds" - : "RPC call retries are disabled."); - return ScriptServiceVersion.ScriptServiceVersion2; - } - - logger.Verbose("RPC call retries are enabled but will not be used for Script Execution as a compatible ScriptService was not found. Please upgrade Tentacle to enable this feature."); - logger.Verbose("Using ScriptServiceV1"); - return ScriptServiceVersion.ScriptServiceVersion1; - } - - ScriptServiceVersion DetermineKubernetesScriptServiceVersionToUse(CapabilitiesResponseV2 tentacleCapabilities) - { - if (tentacleCapabilities.HasKubernetesScriptServiceV1()) - { - logger.Verbose($"Using KubernetesScriptServiceV1"); - logger.Verbose(clientOptions.RpcRetrySettings.RetriesEnabled - ? $"RPC call retries are enabled. Retry timeout {rpcCallExecutor.RetryTimeout.TotalSeconds} seconds" - : "RPC call retries are disabled."); - - return ScriptServiceVersion.KubernetesScriptServiceVersion1; - } - - logger.Verbose($"Using KubernetesScriptServiceV1Alpha"); - logger.Verbose(clientOptions.RpcRetrySettings.RetriesEnabled - ? $"RPC call retries are enabled. Retry timeout {rpcCallExecutor.RetryTimeout.TotalSeconds} seconds" - : "RPC call retries are disabled."); - - //this is the only supported kubernetes script service - return ScriptServiceVersion.KubernetesScriptServiceVersion1Alpha; - } + } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/ScriptServicePicker.cs b/source/Octopus.Tentacle.Client/Scripts/ScriptServicePicker.cs new file mode 100644 index 000000000..672391fe1 --- /dev/null +++ b/source/Octopus.Tentacle.Client/Scripts/ScriptServicePicker.cs @@ -0,0 +1,100 @@ +using System.Threading; +using System.Threading.Tasks; +using Halibut.ServiceModel; +using Octopus.Tentacle.Client.Capabilities; +using Octopus.Tentacle.Client.Execution; +using Octopus.Tentacle.Client.Kubernetes; +using Octopus.Tentacle.Client.Observability; +using Octopus.Tentacle.Contracts.Capabilities; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Contracts.Logging; +using Octopus.Tentacle.Contracts.Observability; + +namespace Octopus.Tentacle.Client.Scripts +{ + public class ScriptServicePicker + { + readonly ITentacleClientTaskLog logger; + readonly IAsyncClientCapabilitiesServiceV2 clientCapabilitiesServiceV2; + readonly RpcCallExecutor rpcCallExecutor; + readonly TentacleClientOptions clientOptions; + readonly ClientOperationMetricsBuilder clientOperationMetricsBuilder; + + internal ScriptServicePicker(IAsyncClientCapabilitiesServiceV2 clientCapabilitiesServiceV2, ITentacleClientTaskLog logger, RpcCallExecutor rpcCallExecutor, TentacleClientOptions clientOptions, ClientOperationMetricsBuilder clientOperationMetricsBuilder) + { + this.clientCapabilitiesServiceV2 = clientCapabilitiesServiceV2; + this.logger = logger; + this.rpcCallExecutor = rpcCallExecutor; + this.clientOptions = clientOptions; + this.clientOperationMetricsBuilder = clientOperationMetricsBuilder; + } + + public async Task DetermineScriptServiceVersionToUse(CancellationToken cancellationToken) + { + logger.Verbose("Determining ScriptService version to use"); + + async Task GetCapabilitiesFunc(CancellationToken ct) + { + var result = await clientCapabilitiesServiceV2.GetCapabilitiesAsync(new HalibutProxyRequestOptions(ct)); + + return result; + } + + var tentacleCapabilities = await rpcCallExecutor.Execute( + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, + RpcCall.Create(nameof(ICapabilitiesServiceV2.GetCapabilities)), + GetCapabilitiesFunc, + logger, + clientOperationMetricsBuilder, + cancellationToken); + + logger.Verbose($"Discovered Tentacle capabilities: {string.Join(",", tentacleCapabilities.SupportedCapabilities)}"); + + // Check if we support any kubernetes script service. + // It's implied (and tested) that GetCapabilities will only return Kubernetes or non-Kubernetes script services, never a mix + if (tentacleCapabilities.HasAnyKubernetesScriptService()) + { + return DetermineKubernetesScriptServiceVersionToUse(tentacleCapabilities); + } + + return DetermineShellScriptServiceVersionToUse(tentacleCapabilities); + } + + ScriptServiceVersion DetermineShellScriptServiceVersionToUse(CapabilitiesResponseV2 tentacleCapabilities) + { + if (tentacleCapabilities.HasScriptServiceV2()) + { + logger.Verbose("Using ScriptServiceV2"); + logger.Verbose(clientOptions.RpcRetrySettings.RetriesEnabled + ? $"RPC call retries are enabled. Retry timeout {rpcCallExecutor.RetryTimeout.TotalSeconds} seconds" + : "RPC call retries are disabled."); + return ScriptServiceVersion.ScriptServiceVersion2; + } + + logger.Verbose("RPC call retries are enabled but will not be used for Script Execution as a compatible ScriptService was not found. Please upgrade Tentacle to enable this feature."); + logger.Verbose("Using ScriptServiceV1"); + return ScriptServiceVersion.ScriptServiceVersion1; + } + + ScriptServiceVersion DetermineKubernetesScriptServiceVersionToUse(CapabilitiesResponseV2 tentacleCapabilities) + { + if (tentacleCapabilities.HasKubernetesScriptServiceV1()) + { + logger.Verbose($"Using KubernetesScriptServiceV1"); + logger.Verbose(clientOptions.RpcRetrySettings.RetriesEnabled + ? $"RPC call retries are enabled. Retry timeout {rpcCallExecutor.RetryTimeout.TotalSeconds} seconds" + : "RPC call retries are disabled."); + + return ScriptServiceVersion.KubernetesScriptServiceVersion1; + } + + logger.Verbose($"Using KubernetesScriptServiceV1Alpha"); + logger.Verbose(clientOptions.RpcRetrySettings.RetriesEnabled + ? $"RPC call retries are enabled. Retry timeout {rpcCallExecutor.RetryTimeout.TotalSeconds} seconds" + : "RPC call retries are disabled."); + + //this is the only supported kubernetes script service + return ScriptServiceVersion.KubernetesScriptServiceVersion1Alpha; + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV1Orchestrator.cs b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV1Executor.cs similarity index 55% rename from source/Octopus.Tentacle.Client/Scripts/ScriptServiceV1Orchestrator.cs rename to source/Octopus.Tentacle.Client/Scripts/ScriptServiceV1Executor.cs index 12b42f64f..f1a7b873c 100644 --- a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV1Orchestrator.cs +++ b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV1Executor.cs @@ -3,6 +3,7 @@ using System.Threading; using System.Threading.Tasks; using Halibut.ServiceModel; +using Octopus.Tentacle.Client.EventDriven; using Octopus.Tentacle.Client.Execution; using Octopus.Tentacle.Client.Observability; using Octopus.Tentacle.Client.Scripts.Models; @@ -13,28 +14,19 @@ namespace Octopus.Tentacle.Client.Scripts { - class ScriptServiceV1Orchestrator : ObservingScriptOrchestrator + class ScriptServiceV1Executor : IScriptExecutor { - readonly RpcCallExecutor rpcCallExecutor; readonly ClientOperationMetricsBuilder clientOperationMetricsBuilder; readonly ITentacleClientTaskLog logger; readonly IAsyncClientScriptService clientScriptServiceV1; - public ScriptServiceV1Orchestrator( + public ScriptServiceV1Executor( IAsyncClientScriptService clientScriptServiceV1, - IScriptObserverBackoffStrategy scriptObserverBackOffStrategy, RpcCallExecutor rpcCallExecutor, ClientOperationMetricsBuilder clientOperationMetricsBuilder, - OnScriptStatusResponseReceived onScriptStatusResponseReceived, - OnScriptCompleted onScriptCompleted, - TentacleClientOptions clientOptions, ITentacleClientTaskLog logger) - : base(scriptObserverBackOffStrategy, - onScriptStatusResponseReceived, - onScriptCompleted, - clientOptions) { this.clientScriptServiceV1 = clientScriptServiceV1; this.rpcCallExecutor = rpcCallExecutor; @@ -42,10 +34,10 @@ public ScriptServiceV1Orchestrator( this.logger = logger; } - protected override StartScriptCommand Map(ExecuteScriptCommand command) + StartScriptCommand Map(ExecuteScriptCommand command) { if (command is not ExecuteShellScriptCommand shellScriptCommand) - throw new InvalidOperationException($"{nameof(ScriptServiceV2Orchestrator)} only supports commands of type {nameof(ExecuteShellScriptCommand)}."); + throw new InvalidOperationException($"{nameof(ScriptServiceV2Executor)} only supports commands of type {nameof(ExecuteShellScriptCommand)}."); return new StartScriptCommand( shellScriptCommand.ScriptBody, @@ -58,16 +50,37 @@ protected override StartScriptCommand Map(ExecuteScriptCommand command) shellScriptCommand.Files.ToArray()); } - protected override ScriptExecutionStatus MapToStatus(ScriptStatusResponse response) - => new(response.Logs); + ScriptStatus MapToScriptStatus(ScriptStatusResponse scriptStatusResponse) + { + return new ScriptStatus(scriptStatusResponse.State, scriptStatusResponse.ExitCode, scriptStatusResponse.Logs); + } - protected override ScriptExecutionResult MapToResult(ScriptStatusResponse response) - => new(response.State, response.ExitCode); + ICommandContext MapToNextStatus(ScriptStatusResponse scriptStatusResponse) + { + return new DefaultCommandContext(scriptStatusResponse.Ticket, scriptStatusResponse.NextLogSequence, ScriptServiceVersion.ScriptServiceVersion1); + } - protected override ProcessState GetState(ScriptStatusResponse response) => response.State; + (ScriptStatus, ICommandContext) Map(ScriptStatusResponse r) + { + return (MapToScriptStatus(r), MapToNextStatus(r)); + } - protected override async Task StartScript(StartScriptCommand command, CancellationToken scriptExecutionCancellationToken) + public async Task<(ScriptStatus, ICommandContext)> StartScript(ExecuteScriptCommand executeScriptCommand, + StartScriptIsBeingReAttempted startScriptIsBeingReAttempted, + CancellationToken scriptExecutionCancellationToken) { + // Script Service v1 is not idempotent, do not allow it to be re-attempted as it may run a second time. + if (startScriptIsBeingReAttempted == StartScriptIsBeingReAttempted.PossiblyBeingReAttempted) + { + return (new ScriptStatus(ProcessState.Complete, + ScriptExitCodes.UnknownScriptExitCode, + new List()), + // TODO: We should try to encourage a DefaultCommandContext which will do nothing perhaps set the ScriptServiceVersion to + // one that always returns the above exit code. + new DefaultCommandContext(new ScriptTicket(Guid.NewGuid().ToString()), 0, ScriptServiceVersion.ScriptServiceVersion1)); + } + + var command = Map(executeScriptCommand); var scriptTicket = await rpcCallExecutor.ExecuteWithNoRetries( RpcCall.Create(nameof(IScriptService.StartScript)), async ct => @@ -80,16 +93,21 @@ protected override async Task StartScript(StartScriptComma clientOperationMetricsBuilder, scriptExecutionCancellationToken).ConfigureAwait(false); - return new ScriptStatusResponse(scriptTicket, ProcessState.Pending, 0, new List(), 0); + return Map(new ScriptStatusResponse(scriptTicket, ProcessState.Pending, 0, new List(), 0)); } - protected override async Task GetStatus(ScriptStatusResponse lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + public async Task<(ScriptStatus, ICommandContext)> GetStatus(ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken) + { + return Map(await _GetStatus(commandContext, scriptExecutionCancellationToken)); + } + + async Task _GetStatus(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { var scriptStatusResponseV1 = await rpcCallExecutor.ExecuteWithNoRetries( RpcCall.Create(nameof(IScriptService.GetStatus)), async ct => { - var request = new ScriptStatusRequest(lastStatusResponse.Ticket, lastStatusResponse.NextLogSequence); + var request = new ScriptStatusRequest(lastStatusResponse.ScriptTicket, lastStatusResponse.NextLogSequence); var result = await clientScriptServiceV1.GetStatusAsync(request, new HalibutProxyRequestOptions(ct)); return result; @@ -101,13 +119,18 @@ protected override async Task GetStatus(ScriptStatusRespon return scriptStatusResponseV1; } - protected override async Task Cancel(ScriptStatusResponse lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + public async Task<(ScriptStatus, ICommandContext)> CancelScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + { + return Map(await _Cancel(lastStatusResponse, scriptExecutionCancellationToken)); + } + + async Task _Cancel(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { var response = await rpcCallExecutor.ExecuteWithNoRetries( RpcCall.Create(nameof(IScriptService.CancelScript)), async ct => { - var request = new CancelScriptCommand(lastStatusResponse.Ticket, lastStatusResponse.NextLogSequence); + var request = new CancelScriptCommand(lastStatusResponse.ScriptTicket, lastStatusResponse.NextLogSequence); var result = await clientScriptServiceV1.CancelScriptAsync(request, new HalibutProxyRequestOptions(ct)); return result; @@ -119,13 +142,18 @@ protected override async Task Cancel(ScriptStatusResponse return response; } - protected override async Task Finish(ScriptStatusResponse lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + public async Task CleanUpScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + { + return MapToScriptStatus(await _CleanUpScript(lastStatusResponse, scriptExecutionCancellationToken)); + } + + async Task _CleanUpScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { var response = await rpcCallExecutor.ExecuteWithNoRetries( RpcCall.Create(nameof(IScriptService.CompleteScript)), async ct => { - var request = new CompleteScriptCommand(lastStatusResponse.Ticket, lastStatusResponse.NextLogSequence); + var request = new CompleteScriptCommand(lastStatusResponse.ScriptTicket, lastStatusResponse.NextLogSequence); var result = await clientScriptServiceV1.CompleteScriptAsync(request, new HalibutProxyRequestOptions(ct)); return result; @@ -134,8 +162,6 @@ protected override async Task Finish(ScriptStatusResponse clientOperationMetricsBuilder, CancellationToken.None).ConfigureAwait(false); - OnScriptStatusResponseReceived(response); - return response; } } diff --git a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Orchestrator.cs b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Orchestrator.cs index 149f7a6ae..6e2d238ae 100644 --- a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Orchestrator.cs +++ b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceV2Orchestrator.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Halibut; using Halibut.ServiceModel; +using Octopus.Tentacle.Client.EventDriven; using Octopus.Tentacle.Client.Execution; using Octopus.Tentacle.Client.Observability; using Octopus.Tentacle.Client.Scripts.Models; @@ -15,40 +16,35 @@ namespace Octopus.Tentacle.Client.Scripts { - class ScriptServiceV2Orchestrator : ObservingScriptOrchestrator + class ScriptServiceV2Executor : IScriptExecutor { readonly IAsyncClientScriptServiceV2 clientScriptServiceV2; readonly RpcCallExecutor rpcCallExecutor; readonly ClientOperationMetricsBuilder clientOperationMetricsBuilder; readonly TimeSpan onCancellationAbandonCompleteScriptAfter; readonly ITentacleClientTaskLog logger; + readonly TentacleClientOptions clientOptions; - public ScriptServiceV2Orchestrator( + public ScriptServiceV2Executor( IAsyncClientScriptServiceV2 clientScriptServiceV2, - IScriptObserverBackoffStrategy scriptObserverBackOffStrategy, RpcCallExecutor rpcCallExecutor, ClientOperationMetricsBuilder clientOperationMetricsBuilder, - OnScriptStatusResponseReceived onScriptStatusResponseReceived, - OnScriptCompleted onScriptCompleted, TimeSpan onCancellationAbandonCompleteScriptAfter, TentacleClientOptions clientOptions, ITentacleClientTaskLog logger) - : base(scriptObserverBackOffStrategy, - onScriptStatusResponseReceived, - onScriptCompleted, - clientOptions) { this.clientScriptServiceV2 = clientScriptServiceV2; this.rpcCallExecutor = rpcCallExecutor; this.clientOperationMetricsBuilder = clientOperationMetricsBuilder; this.onCancellationAbandonCompleteScriptAfter = onCancellationAbandonCompleteScriptAfter; + this.clientOptions = clientOptions; this.logger = logger; } - protected override StartScriptCommandV2 Map(ExecuteScriptCommand command) + StartScriptCommandV2 Map(ExecuteScriptCommand command) { if (command is not ExecuteShellScriptCommand shellScriptCommand) - throw new InvalidOperationException($"{nameof(ScriptServiceV2Orchestrator)} only supports commands of type {nameof(ExecuteShellScriptCommand)}."); + throw new InvalidOperationException($"{nameof(ScriptServiceV2Executor)} only supports commands of type {nameof(ExecuteShellScriptCommand)}."); return new StartScriptCommandV2( shellScriptCommand.ScriptBody, @@ -62,17 +58,28 @@ protected override StartScriptCommandV2 Map(ExecuteScriptCommand command) shellScriptCommand.Scripts, shellScriptCommand.Files.ToArray()); } + + (ScriptStatus, ICommandContext) Map(ScriptStatusResponseV2 r) + { + return (MapToScriptStatus(r), MapToNextStatus(r)); + } + + private ScriptStatus MapToScriptStatus(ScriptStatusResponseV2 scriptStatusResponse) + { + return new ScriptStatus(scriptStatusResponse.State, scriptStatusResponse.ExitCode, scriptStatusResponse.Logs); + } - protected override ScriptExecutionStatus MapToStatus(ScriptStatusResponseV2 response) - => new(response.Logs); - - protected override ScriptExecutionResult MapToResult(ScriptStatusResponseV2 response) - => new(response.State, response.ExitCode); - - protected override ProcessState GetState(ScriptStatusResponseV2 response) => response.State; + private ICommandContext MapToNextStatus(ScriptStatusResponseV2 scriptStatusResponse) + { + return new DefaultCommandContext(scriptStatusResponse.Ticket, scriptStatusResponse.NextLogSequence, ScriptServiceVersion.ScriptServiceVersion2); + } + public ProcessState GetState(ScriptStatusResponseV2 response) => response.State; - protected override async Task StartScript(StartScriptCommandV2 command, CancellationToken scriptExecutionCancellationToken) + public async Task<(ScriptStatus, ICommandContext)> StartScript(ExecuteScriptCommand executeScriptCommand, + StartScriptIsBeingReAttempted startScriptIsBeingReAttempted, + CancellationToken scriptExecutionCancellationToken) { + var command = Map(executeScriptCommand); ScriptStatusResponseV2 scriptStatusResponse; var startScriptCallsConnectedCount = 0; try @@ -95,13 +102,15 @@ void OnErrorAction(Exception ex) } scriptStatusResponse = await rpcCallExecutor.Execute( - retriesEnabled: ClientOptions.RpcRetrySettings.RetriesEnabled, + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, RpcCall.Create(nameof(IScriptServiceV2.StartScript)), StartScriptAction, OnErrorAction, logger, clientOperationMetricsBuilder, scriptExecutionCancellationToken).ConfigureAwait(false); + + return (MapToScriptStatus(scriptStatusResponse), MapToNextStatus(scriptStatusResponse)); } catch (Exception ex) when (scriptExecutionCancellationToken.IsCancellationRequested) { @@ -115,69 +124,53 @@ void OnErrorAction(Exception ex) if (!startScriptCallIsConnecting || startScriptCallIsBeingRetried) { - // We have to assume the script started executing and call CancelScript and CompleteScript - // We don't have a response so we need to create one to continue the execution flow - scriptStatusResponse = new ScriptStatusResponseV2( - command.ScriptTicket, - ProcessState.Pending, - ScriptExitCodes.RunningExitCode, - new List(), - 0); - - try - { - await ObserveUntilCompleteThenFinish(scriptStatusResponse, scriptExecutionCancellationToken).ConfigureAwait(false); - } - catch (Exception observerUntilCompleteException) - { - // Throw an error so the caller knows that execution of the script was cancelled - throw new OperationCanceledException("Script execution was cancelled", observerUntilCompleteException); - } - - // Throw an error so the caller knows that execution of the script was cancelled - throw new OperationCanceledException("Script execution was cancelled"); + var scriptStatus = new ScriptStatus(ProcessState.Pending, null, new List()); + var defaultTicketForNextStatus = new DefaultCommandContext(command.ScriptTicket, 0, ScriptServiceVersion.ScriptServiceVersion2); + return (scriptStatus, defaultTicketForNextStatus); } // If the StartScript call was not in-flight or being retries then we know the script has not started executing on Tentacle // So can exit without calling CancelScript or CompleteScript throw new OperationCanceledException("Script execution was cancelled", ex); } - - return scriptStatusResponse; } - protected override async Task GetStatus(ScriptStatusResponseV2 lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + public async Task<(ScriptStatus, ICommandContext)> GetStatus(ICommandContext commandContext, CancellationToken scriptExecutionCancellationToken) { - try - { - async Task GetStatusAction(CancellationToken ct) - { - var request = new ScriptStatusRequestV2(lastStatusResponse.Ticket, lastStatusResponse.NextLogSequence); - var result = await clientScriptServiceV2.GetStatusAsync(request, new HalibutProxyRequestOptions(ct)); + return Map(await _GetStatus(commandContext, scriptExecutionCancellationToken)); - return result; - } - - return await rpcCallExecutor.Execute( - retriesEnabled: ClientOptions.RpcRetrySettings.RetriesEnabled, - RpcCall.Create(nameof(IScriptServiceV2.GetStatus)), - GetStatusAction, - logger, - clientOperationMetricsBuilder, - scriptExecutionCancellationToken).ConfigureAwait(false); - } - catch (Exception e) when (e is OperationCanceledException && scriptExecutionCancellationToken.IsCancellationRequested) + } + + private async Task _GetStatus(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + { + + async Task GetStatusAction(CancellationToken ct) { - // Return the last known response without logs when cancellation occurs and let the script execution go into the CancelScript and CompleteScript flow - return new ScriptStatusResponseV2(lastStatusResponse.Ticket, lastStatusResponse.State, lastStatusResponse.ExitCode, new List(), lastStatusResponse.NextLogSequence); + var request = new ScriptStatusRequestV2(lastStatusResponse.ScriptTicket, lastStatusResponse.NextLogSequence); + var result = await clientScriptServiceV2.GetStatusAsync(request, new HalibutProxyRequestOptions(ct)); + + return result; } + + return await rpcCallExecutor.Execute( + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, + RpcCall.Create(nameof(IScriptServiceV2.GetStatus)), + GetStatusAction, + logger, + clientOperationMetricsBuilder, + scriptExecutionCancellationToken).ConfigureAwait(false); + } + + public async Task<(ScriptStatus, ICommandContext)> CancelScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + { + return Map(await _Cancel(lastStatusResponse, scriptExecutionCancellationToken)); } - protected override async Task Cancel(ScriptStatusResponseV2 lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + private async Task _Cancel(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { async Task CancelScriptAction(CancellationToken ct) { - var request = new CancelScriptCommandV2(lastStatusResponse.Ticket, lastStatusResponse.NextLogSequence); + var request = new CancelScriptCommandV2(lastStatusResponse.ScriptTicket, lastStatusResponse.NextLogSequence); var result = await clientScriptServiceV2.CancelScriptAsync(request, new HalibutProxyRequestOptions(ct)); return result; @@ -188,7 +181,7 @@ async Task CancelScriptAction(CancellationToken ct) // We could potentially reduce the time to failure by not retrying the cancel RPC Call if the previous RPC call was already triggering RPC Retries. return await rpcCallExecutor.Execute( - retriesEnabled: ClientOptions.RpcRetrySettings.RetriesEnabled, + retriesEnabled: clientOptions.RpcRetrySettings.RetriesEnabled, RpcCall.Create(nameof(IScriptServiceV2.CancelScript)), CancelScriptAction, logger, @@ -197,7 +190,7 @@ async Task CancelScriptAction(CancellationToken ct) CancellationToken.None).ConfigureAwait(false); } - protected override async Task Finish(ScriptStatusResponseV2 lastStatusResponse, CancellationToken scriptExecutionCancellationToken) + public async Task CleanUpScript(ICommandContext lastStatusResponse, CancellationToken scriptExecutionCancellationToken) { try { @@ -212,7 +205,7 @@ await rpcCallExecutor.ExecuteWithNoRetries( RpcCall.Create(nameof(IScriptServiceV2.CompleteScript)), async ct => { - var request = new CompleteScriptCommandV2(lastStatusResponse.Ticket); + var request = new CompleteScriptCommandV2(lastStatusResponse.ScriptTicket); await clientScriptServiceV2.CompleteScriptAsync(request, new HalibutProxyRequestOptions(ct)); }, logger, @@ -225,7 +218,7 @@ await rpcCallExecutor.ExecuteWithNoRetries( logger.Verbose(ex); } - return lastStatusResponse; + return null; } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceVersion.cs b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceVersion.cs index 035ea0f53..4be59fbed 100644 --- a/source/Octopus.Tentacle.Client/Scripts/ScriptServiceVersion.cs +++ b/source/Octopus.Tentacle.Client/Scripts/ScriptServiceVersion.cs @@ -1,6 +1,7 @@ namespace Octopus.Tentacle.Client.Scripts { - record ScriptServiceVersion(string Value) + // TODO should not be public. + public record ScriptServiceVersion(string Value) { public static ScriptServiceVersion ScriptServiceVersion1 = new(nameof(ScriptServiceVersion1)); public static ScriptServiceVersion ScriptServiceVersion2 = new(nameof(ScriptServiceVersion2)); diff --git a/source/Octopus.Tentacle.Client/ServiceHelpers/ClientsHolder.cs b/source/Octopus.Tentacle.Client/ServiceHelpers/ClientsHolder.cs new file mode 100644 index 000000000..a178edb57 --- /dev/null +++ b/source/Octopus.Tentacle.Client/ServiceHelpers/ClientsHolder.cs @@ -0,0 +1,40 @@ +using Halibut; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.Capabilities; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; +using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1Alpha; +using Octopus.Tentacle.Contracts.ScriptServiceV2; + +namespace Octopus.Tentacle.Client.ServiceHelpers +{ + public class ClientsHolder + { + public IAsyncClientScriptService ScriptServiceV1 { get; } + public IAsyncClientScriptServiceV2 ScriptServiceV2 { get; } + public IAsyncClientKubernetesScriptServiceV1Alpha KubernetesScriptServiceV1Alpha { get; } + public IAsyncClientKubernetesScriptServiceV1 KubernetesScriptServiceV1 { get; } + public IAsyncClientFileTransferService ClientFileTransferServiceV1 { get; } + public IAsyncClientCapabilitiesServiceV2 CapabilitiesServiceV2 { get; } + + internal ClientsHolder(IHalibutRuntime halibutRuntime, ServiceEndPoint serviceEndPoint, ITentacleServiceDecoratorFactory? tentacleServicesDecoratorFactory) + { + ScriptServiceV1 = halibutRuntime.CreateAsyncClient(serviceEndPoint); + ScriptServiceV2 = halibutRuntime.CreateAsyncClient(serviceEndPoint); + KubernetesScriptServiceV1Alpha = halibutRuntime.CreateAsyncClient(serviceEndPoint); + KubernetesScriptServiceV1 = halibutRuntime.CreateAsyncClient(serviceEndPoint); + ClientFileTransferServiceV1 = halibutRuntime.CreateAsyncClient(serviceEndPoint); + CapabilitiesServiceV2 = halibutRuntime.CreateAsyncClient(serviceEndPoint).WithBackwardsCompatability(); + + if (tentacleServicesDecoratorFactory != null) + { + ScriptServiceV1 = tentacleServicesDecoratorFactory.Decorate(ScriptServiceV1); + ScriptServiceV2 = tentacleServicesDecoratorFactory.Decorate(ScriptServiceV2); + KubernetesScriptServiceV1Alpha = tentacleServicesDecoratorFactory.Decorate(KubernetesScriptServiceV1Alpha); + KubernetesScriptServiceV1 = tentacleServicesDecoratorFactory.Decorate(KubernetesScriptServiceV1); + ClientFileTransferServiceV1 = tentacleServicesDecoratorFactory.Decorate(ClientFileTransferServiceV1); + CapabilitiesServiceV2 = tentacleServicesDecoratorFactory.Decorate(CapabilitiesServiceV2); + } + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/ShortCutTakenHere.cs b/source/Octopus.Tentacle.Client/ShortCutTakenHere.cs new file mode 100644 index 000000000..4a94e8f92 --- /dev/null +++ b/source/Octopus.Tentacle.Client/ShortCutTakenHere.cs @@ -0,0 +1,7 @@ +namespace Octopus.Tentacle.Client +{ + public class ShortCutTakenHere + { + + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/StartScriptIsBeingReAttempted.cs b/source/Octopus.Tentacle.Client/StartScriptIsBeingReAttempted.cs new file mode 100644 index 000000000..e9ab7ac44 --- /dev/null +++ b/source/Octopus.Tentacle.Client/StartScriptIsBeingReAttempted.cs @@ -0,0 +1,10 @@ +using System; + +namespace Octopus.Tentacle.Client +{ + public enum StartScriptIsBeingReAttempted + { + FirstAttempt, + PossiblyBeingReAttempted + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Client/TentacleClient.cs b/source/Octopus.Tentacle.Client/TentacleClient.cs index bbd340ccf..66d114a01 100644 --- a/source/Octopus.Tentacle.Client/TentacleClient.cs +++ b/source/Octopus.Tentacle.Client/TentacleClient.cs @@ -8,31 +8,26 @@ using Octopus.Tentacle.Client.Observability; using Octopus.Tentacle.Client.Scripts; using Octopus.Tentacle.Client.Scripts.Models; +using Octopus.Tentacle.Client.ServiceHelpers; using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Contracts.Capabilities; -using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; -using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1Alpha; using Octopus.Tentacle.Contracts.Logging; using Octopus.Tentacle.Contracts.Observability; -using Octopus.Tentacle.Contracts.ScriptServiceV2; using ITentacleClientObserver = Octopus.Tentacle.Contracts.Observability.ITentacleClientObserver; namespace Octopus.Tentacle.Client { public class TentacleClient : ITentacleClient { + readonly ServiceEndPoint serviceEndPoint; + readonly IHalibutRuntime halibutRuntime; readonly IScriptObserverBackoffStrategy scriptObserverBackOffStrategy; readonly ITentacleClientObserver tentacleClientObserver; readonly RpcCallExecutor rpcCallExecutor; - - readonly IAsyncClientScriptService scriptServiceV1; - readonly IAsyncClientScriptServiceV2 scriptServiceV2; - readonly IAsyncClientKubernetesScriptServiceV1Alpha kubernetesScriptServiceV1Alpha; - readonly IAsyncClientKubernetesScriptServiceV1 kubernetesScriptServiceV1; - readonly IAsyncClientFileTransferService clientFileTransferServiceV1; - readonly IAsyncClientCapabilitiesServiceV2 capabilitiesServiceV2; + readonly TentacleClientOptions clientOptions; + readonly ITentacleServiceDecoratorFactory? tentacleServicesDecoratorFactory; + readonly ClientsHolder clientsHolder; public static void CacheServiceWasNotFoundResponseMessages(IHalibutRuntime halibutRuntime) { @@ -67,10 +62,13 @@ internal TentacleClient( TentacleClientOptions clientOptions, ITentacleServiceDecoratorFactory? tentacleServicesDecoratorFactory) { + this.serviceEndPoint = serviceEndPoint; + this.halibutRuntime = halibutRuntime; this.scriptObserverBackOffStrategy = scriptObserverBackOffStrategy; this.tentacleClientObserver = tentacleClientObserver.DecorateWithNonThrowingTentacleClientObserver(); this.clientOptions = clientOptions; + this.tentacleServicesDecoratorFactory = tentacleServicesDecoratorFactory; if (halibutRuntime.OverrideErrorResponseMessageCaching == null) { @@ -79,22 +77,7 @@ internal TentacleClient( throw new ArgumentException("Ensure that TentacleClient.CacheServiceWasNotFoundResponseMessages has been called for the HalibutRuntime", nameof(halibutRuntime)); } - scriptServiceV1 = halibutRuntime.CreateAsyncClient(serviceEndPoint); - scriptServiceV2 = halibutRuntime.CreateAsyncClient(serviceEndPoint); - kubernetesScriptServiceV1Alpha = halibutRuntime.CreateAsyncClient(serviceEndPoint); - kubernetesScriptServiceV1 = halibutRuntime.CreateAsyncClient(serviceEndPoint); - clientFileTransferServiceV1 = halibutRuntime.CreateAsyncClient(serviceEndPoint); - capabilitiesServiceV2 = halibutRuntime.CreateAsyncClient(serviceEndPoint).WithBackwardsCompatability(); - - if (tentacleServicesDecoratorFactory != null) - { - scriptServiceV1 = tentacleServicesDecoratorFactory.Decorate(scriptServiceV1); - scriptServiceV2 = tentacleServicesDecoratorFactory.Decorate(scriptServiceV2); - kubernetesScriptServiceV1Alpha = tentacleServicesDecoratorFactory.Decorate(kubernetesScriptServiceV1Alpha); - kubernetesScriptServiceV1 = tentacleServicesDecoratorFactory.Decorate(kubernetesScriptServiceV1); - clientFileTransferServiceV1 = tentacleServicesDecoratorFactory.Decorate(clientFileTransferServiceV1); - capabilitiesServiceV2 = tentacleServicesDecoratorFactory.Decorate(capabilitiesServiceV2); - } + clientsHolder = new ClientsHolder(halibutRuntime, serviceEndPoint, tentacleServicesDecoratorFactory); rpcCallExecutor = RpcCallExecutorFactory.Create(this.clientOptions.RpcRetrySettings.RetryDuration, this.tentacleClientObserver); } @@ -108,7 +91,7 @@ public async Task UploadFile(string fileName, string path, DataStr async Task UploadFileAction(CancellationToken ct) { logger.Info($"Beginning upload of {fileName} to Tentacle"); - var result = await clientFileTransferServiceV1.UploadFileAsync(path, package, new HalibutProxyRequestOptions(ct)); + var result = await clientsHolder.ClientFileTransferServiceV1.UploadFileAsync(path, package, new HalibutProxyRequestOptions(ct)); logger.Info("Upload complete"); return result; @@ -143,7 +126,7 @@ async Task UploadFileAction(CancellationToken ct) async Task DownloadFileAction(CancellationToken ct) { logger.Info($"Beginning download of {Path.GetFileName(remotePath)} from Tentacle"); - var result = await clientFileTransferServiceV1.DownloadFileAsync(remotePath, new HalibutProxyRequestOptions(ct)); + var result = await clientsHolder.ClientFileTransferServiceV1.DownloadFileAsync(remotePath, new HalibutProxyRequestOptions(ct)); logger.Info("Download complete"); return result; @@ -181,23 +164,21 @@ public async Task ExecuteScript(ExecuteScriptCommand exec try { - var factory = new ScriptOrchestratorFactory( - scriptServiceV1, - scriptServiceV2, - kubernetesScriptServiceV1Alpha, - kubernetesScriptServiceV1, - capabilitiesServiceV2, - scriptObserverBackOffStrategy, - rpcCallExecutor, - operationMetricsBuilder, + + var eventDrivenScriptExecutor = new AggregateScriptExecutor(logger, + tentacleClientObserver, + clientOptions, + halibutRuntime, + serviceEndPoint, + tentacleServicesDecoratorFactory, + OnCancellationAbandonCompleteScriptAfter); + + var orchestrator = new ObservingScriptOrchestrator(scriptObserverBackOffStrategy, onScriptStatusResponseReceived, onScriptCompleted, - OnCancellationAbandonCompleteScriptAfter, - clientOptions, + eventDrivenScriptExecutor, logger); - var orchestrator = await factory.CreateOrchestrator(scriptExecutionCancellationToken); - var result = await orchestrator.ExecuteScript(executeScriptCommand, scriptExecutionCancellationToken); return new ScriptExecutionResult(result.State, result.ExitCode); @@ -209,6 +190,7 @@ public async Task ExecuteScript(ExecuteScriptCommand exec } finally { + // TODO handle this in the new pipeline. var operationMetrics = operationMetricsBuilder.Build(); tentacleClientObserver.ExecuteScriptCompleted(operationMetrics, logger); } diff --git a/source/Octopus.Tentacle.Contracts/ScriptStatus.cs b/source/Octopus.Tentacle.Contracts/ScriptStatus.cs new file mode 100644 index 000000000..5ce9f80cf --- /dev/null +++ b/source/Octopus.Tentacle.Contracts/ScriptStatus.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace Octopus.Tentacle.Contracts +{ + public class ScriptStatus + { + public ProcessState State; + public int? ExitCode; + public List Logs; + + public ScriptStatus(ProcessState state, int? exitCode, List logs) + { + this.State = state; + this.ExitCode = exitCode; + this.Logs = logs; + } + } +} \ No newline at end of file