diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/HelmVersionsTestFixtureAttributes.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/HelmVersionsTestFixtureAttributes.cs new file mode 100644 index 000000000..6e0227ce0 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/HelmVersionsTestFixtureAttributes.cs @@ -0,0 +1,15 @@ +namespace Octopus.Tentacle.Kubernetes.Tests.Integration; + +public class HelmVersion1TestFixtureAttribute : TestFixtureAttribute +{ + public HelmVersion1TestFixtureAttribute(): + base("1.*.*") + { } +} + +public class HelmVersion2AlphaTestFixtureAttribute : TestFixtureAttribute +{ + public HelmVersion2AlphaTestFixtureAttribute(): + base("2.*.*-alpha") + { } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs index 5874cdd5a..8f148594e 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs @@ -16,9 +16,8 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration; public abstract class KubernetesAgentIntegrationTest { - protected KubernetesAgentInstaller kubernetesAgentInstaller; - TraceLogFileLogger? traceLogFileLogger; - CancellationTokenSource cancellationTokenSource; + readonly string? helmChartVersion; + protected ILogger Logger { get; private set; } protected HalibutRuntime ServerHalibutRuntime { get; private set; } = null!; @@ -27,31 +26,41 @@ public abstract class KubernetesAgentIntegrationTest protected CancellationToken CancellationToken { get; private set; } + protected KubernetesAgentInstaller KubernetesAgentInstaller { get; private set; } = null!; + protected KubeCtlTool KubeCtl { get; private set; } string agentThumbprint; + TraceLogFileLogger? traceLogFileLogger; + CancellationTokenSource cancellationTokenSource; + + protected KubernetesAgentIntegrationTest(string? helmChartVersion) + { + this.helmChartVersion = helmChartVersion; + } [OneTimeSetUp] public async Task OneTimeSetUp() { - kubernetesAgentInstaller = new KubernetesAgentInstaller( + KubernetesAgentInstaller = new KubernetesAgentInstaller( KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.HelmExePath, KubernetesTestsGlobalContext.Instance.KubeCtlExePath, KubernetesTestsGlobalContext.Instance.KubeConfigPath, - KubernetesTestsGlobalContext.Instance.Logger); + KubernetesTestsGlobalContext.Instance.Logger, + helmChartVersion); KubeCtl = new KubeCtlTool( KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.KubeCtlExePath, KubernetesTestsGlobalContext.Instance.KubeConfigPath, - kubernetesAgentInstaller.Namespace, + KubernetesAgentInstaller.Namespace, KubernetesTestsGlobalContext.Instance.Logger); //create a new server halibut runtime var listeningPort = BuildServerHalibutRuntimeAndListen(); - agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag); + agentThumbprint = await KubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag); //trust the generated cert thumbprint ServerHalibutRuntime.Trust(agentThumbprint); @@ -90,7 +99,7 @@ public async Task TearDown() void BuildTentacleClient() { - var endpoint = new ServiceEndPoint(kubernetesAgentInstaller.SubscriptionId, agentThumbprint, ServerHalibutRuntime.TimeoutsAndLimits); + var endpoint = new ServiceEndPoint(KubernetesAgentInstaller.SubscriptionId, agentThumbprint, ServerHalibutRuntime.TimeoutsAndLimits); var retrySettings = new RpcRetrySettings(true, TimeSpan.FromMinutes(2)); var clientOptions = new TentacleClientOptions(retrySettings); @@ -125,6 +134,6 @@ int BuildServerHalibutRuntimeAndListen() public async Task OneTimeTearDown() { await ServerHalibutRuntime.DisposeAsync(); - kubernetesAgentInstaller?.Dispose(); + KubernetesAgentInstaller?.Dispose(); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs index e1f6d86ad..52c021001 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs @@ -2,16 +2,24 @@ using FluentAssertions; using k8s; using Newtonsoft.Json; +using NSubstitute; using Octopus.Diagnostics; using Octopus.Tentacle.Diagnostics; using Octopus.Tentacle.Kubernetes.Diagnostics; namespace Octopus.Tentacle.Kubernetes.Tests.Integration; +[HelmVersion1TestFixture] +[HelmVersion2AlphaTestFixture] public class KubernetesAgentMetricsIntegrationTest : KubernetesAgentIntegrationTest { readonly ISystemLog systemLog = new SystemLog(); + + public KubernetesAgentMetricsIntegrationTest(string? helmChartVersion) + : base(helmChartVersion) + { } + class KubernetesFileWrappedProvider : IKubernetesClientConfigProvider { readonly string filename; @@ -26,19 +34,20 @@ public KubernetesClientConfiguration Get() return KubernetesClientConfiguration.BuildConfigFromConfigFile(filename); } } - + [Test] public async Task FetchingTimestampFromEmptyConfigMapEntryShouldBeMinValue() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, kubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", configMapService); + var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, config, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog); //Act var result = await metrics.GetLatestEventTimestamp(CancellationToken.None); - + //Assert result.Should().Be(DateTimeOffset.MinValue); } @@ -47,14 +56,15 @@ public async Task FetchingTimestampFromEmptyConfigMapEntryShouldBeMinValue() public async Task FetchingLatestEventTimestampFromNonexistentConfigMapThrowsException() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, kubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("nonexistent-config-map", configMapService); + var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, config, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("nonexistent-config-map", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, "metrics", systemLog); //Act Func func = async () => await metrics.GetLatestEventTimestamp(CancellationToken.None); - + //Assert await func.Should().ThrowAsync(); } @@ -63,9 +73,10 @@ public async Task FetchingLatestEventTimestampFromNonexistentConfigMapThrowsExce public async Task WritingEventToNonExistentConfigMapShouldFailSilently() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, kubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("nonexistent-config-map", configMapService); + var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, config, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("nonexistent-config-map", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog); //Act @@ -79,15 +90,16 @@ public async Task WritingEventToNonExistentConfigMapShouldFailSilently() public async Task WritingEventToExistingConfigMapShouldPersistJsonEntry() { //Arrange + var config = Substitute.For(); var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath); - var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, kubernetesAgentInstaller.Namespace); - var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", configMapService); + var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, config, systemLog, KubernetesAgentInstaller.Namespace); + var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", config, configMapService); var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog); //Act var eventTimestamp = DateTimeOffset.Now; await metrics.TrackEvent("reason", "source", eventTimestamp, CancellationToken.None); - + //Assert var persistedDictionary = await persistenceProvider.ReadValues(CancellationToken.None); var metricsData = persistedDictionary[ConfigMapNames.AgentMetricsConfigMapKey]; diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1AlphaIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1AlphaIntegrationTest.cs index 863f31907..234712c56 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1AlphaIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1AlphaIntegrationTest.cs @@ -10,14 +10,20 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration; -[TestFixture] +[HelmVersion1TestFixture] +[HelmVersion2AlphaTestFixture] public class KubernetesScriptServiceV1AlphaIntegrationTest : KubernetesAgentIntegrationTest { + public KubernetesScriptServiceV1AlphaIntegrationTest(string? helmChartVersion) + : base(helmChartVersion) + { + } + protected override TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) { return builder .DecorateCapabilitiesServiceV2With(d => d - .DecorateGetCapabilitiesWith((inner, options) => Task.FromResult(new CapabilitiesResponseV2(new List { nameof(IFileTransferService), nameof(IKubernetesScriptServiceV1Alpha) })))); + .DecorateGetCapabilitiesWith((inner, options) => Task.FromResult(new CapabilitiesResponseV2(new List { nameof(IFileTransferService), nameof(IKubernetesScriptServiceV1Alpha) })))); } [Test] diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs index e1d5acdb6..047efc0e2 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs @@ -14,11 +14,17 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration; -[TestFixture] +[HelmVersion1TestFixture] +[HelmVersion2AlphaTestFixture] public class KubernetesScriptServiceV1IntegrationTest : KubernetesAgentIntegrationTest { IRecordedMethodUsages recordedMethodUsages = null!; + public KubernetesScriptServiceV1IntegrationTest(string? helmChartVersion) + : base(helmChartVersion) + { + } + protected override TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) { builder.RecordMethodUsages(out var recordedUsages) @@ -78,7 +84,7 @@ Task ScriptCompleted(CancellationToken ct) return Task.CompletedTask; } } - + [Test] [TestCase(ScriptType.Normal)] [TestCase(ScriptType.Raw)] @@ -122,7 +128,7 @@ Task ScriptCompleted(CancellationToken ct) return Task.CompletedTask; } } - + [Test] [TestCase(ScriptType.Normal)] [TestCase(ScriptType.Raw)] diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj index 0f9d444b1..365574828 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj @@ -13,6 +13,7 @@ + diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs index ac1e2c7dd..97ff91957 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs @@ -19,17 +19,19 @@ public class KubernetesAgentInstaller readonly string kubeCtlExePath; readonly TemporaryDirectory temporaryDirectory; readonly ILogger logger; + readonly string helmChartVersion; readonly string kubeConfigPath; bool isAgentInstalled; - public KubernetesAgentInstaller(TemporaryDirectory temporaryDirectory, string helmExePath, string kubeCtlExePath, string kubeConfigPath, ILogger logger) + public KubernetesAgentInstaller(TemporaryDirectory temporaryDirectory, string helmExePath, string kubeCtlExePath, string kubeConfigPath, ILogger logger, string? helmChartVersion) { this.temporaryDirectory = temporaryDirectory; this.helmExePath = helmExePath; this.kubeCtlExePath = kubeCtlExePath; this.kubeConfigPath = kubeConfigPath; this.logger = logger; + this.helmChartVersion = helmChartVersion ?? "1.*.*"; AgentName = Guid.NewGuid().ToString("N"); } @@ -128,11 +130,13 @@ string BuildAgentInstallArguments(string valuesFilePath, string? tentacleImageAn return string.Join(" ", args.WhereNotNull()); } - static string GetChartVersion() + string GetChartVersion() { var customHelmChartVersion = Environment.GetEnvironmentVariable("KubernetesIntegrationTests_HelmChartVersion"); - - return !string.IsNullOrWhiteSpace(customHelmChartVersion) ? customHelmChartVersion : "1.*.*"; + + return !string.IsNullOrWhiteSpace(customHelmChartVersion) + ? customHelmChartVersion + : helmChartVersion; } static string? GetImageAndRepository(string? tentacleImageAndTag) @@ -161,7 +165,7 @@ async Task GetAgentThumbprint() .WriteTo.StringBuilder(sb) .MinimumLevel.Debug() .CreateLogger(); - + var exitCode = SilentProcessRunner.ExecuteCommand( kubeCtlExePath, //get the generated thumbprint from the config map diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs index af9b58fda..7b61ac718 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Support/TestSupportConfigMapService.cs @@ -14,8 +14,8 @@ public class TestSupportConfigMapService : KubernetesService, IKubernetesConfigM { readonly string targetNamespace; - public TestSupportConfigMapService(IKubernetesClientConfigProvider configProvider, ISystemLog log, string targetNamespace) - : base(configProvider, log) + public TestSupportConfigMapService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log, string targetNamespace) + : base(configProvider, kubernetesConfiguration, log) { this.targetNamespace = targetNamespace; } diff --git a/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs b/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs index 8580ea0a0..2a5e23a1a 100644 --- a/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs +++ b/source/Octopus.Tentacle.Tests/Capabilities/CapabilitiesServiceV2Fixture.cs @@ -2,6 +2,7 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; +using NSubstitute; using NUnit.Framework; using Octopus.Tentacle.Contracts; using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; @@ -17,7 +18,10 @@ public class CapabilitiesServiceV2Fixture [Test] public async Task CapabilitiesAreReturned() { - var capabilities = (await new CapabilitiesServiceV2() + var k8sDetection = Substitute.For(); + k8sDetection.IsRunningAsKubernetesAgent.Returns(false); + + var capabilities = (await new CapabilitiesServiceV2(k8sDetection) .GetCapabilitiesAsync(CancellationToken.None)) .SupportedCapabilities; @@ -30,9 +34,10 @@ public async Task CapabilitiesAreReturned() [Test] public async Task OnlyKubernetesScriptServicesAreReturnedWhenRunningAsKubernetesAgent() { - Environment.SetEnvironmentVariable(KubernetesConfig.NamespaceVariableName, "ABC"); + var k8sDetection = Substitute.For(); + k8sDetection.IsRunningAsKubernetesAgent.Returns(true); - var capabilities = (await new CapabilitiesServiceV2() + var capabilities = (await new CapabilitiesServiceV2(k8sDetection) .GetCapabilitiesAsync(CancellationToken.None)) .SupportedCapabilities; @@ -40,8 +45,6 @@ public async Task OnlyKubernetesScriptServicesAreReturnedWhenRunningAsKubernetes capabilities.Count.Should().Be(3); capabilities.Should().NotContainMatch("IScriptService*"); - - Environment.SetEnvironmentVariable(KubernetesConfig.NamespaceVariableName, null); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs b/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs index e473ef60f..52c8de126 100644 --- a/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs +++ b/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs @@ -8,6 +8,7 @@ using Octopus.Tentacle.Configuration.Crypto; using Octopus.Tentacle.Configuration.Instances; using Octopus.Tentacle.Kubernetes; +using Octopus.Tentacle.Kubernetes.Configuration; using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Tests.Configuration @@ -189,10 +190,11 @@ ApplicationInstanceSelector CreateApplicationInstanceSelector(StartUpInstanceReq return new ApplicationInstanceSelector(ApplicationName.Tentacle, applicationInstanceStore, instanceRequest ?? new StartUpDynamicInstanceRequest(), - additionalConfigurations ?? new IApplicationConfigurationContributor[0], - new Lazy(() => new ConfigMapKeyValueStore(Substitute.For(), Substitute.For())), + additionalConfigurations ?? Array.Empty(), + new Lazy(() => new ConfigMapKeyValueStore( Substitute.For(),Substitute.For(), Substitute.For())), octopusFileSystem, - log); + log, + Substitute.For()); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs index 5f89b8126..183b9e9a5 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesDirectoryInformationProviderFixture.cs @@ -18,7 +18,7 @@ public class KubernetesDirectoryInformationProviderFixture { // Sizes const ulong Megabyte = 1000 * 1000; - + [Test] public void DuOutputParses() { @@ -30,10 +30,10 @@ public void DuOutputParses() x.ArgAt>(3).Invoke($"{usedSize}\t/octopus"); }); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } - + [Test] public void DuOutputParsesWithMultipleLines() { @@ -44,10 +44,10 @@ public void DuOutputParsesWithMultipleLines() { x.ArgAt>(3).Invoke($"500\t/octopus/extradir"); x.ArgAt>(3).Invoke($"{usedSize}\t/octopus"); - x.ArgAt>(3).Invoke($"{usedSize+1000}\tTotal"); + x.ArgAt>(3).Invoke($"{usedSize + 1000}\tTotal"); }); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } @@ -64,10 +64,10 @@ public void IfDuFailsWeStillGetData() }); spr.ReturnsForAll(1); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } - + [Test] public void IfDuFailsWeLogCorrectly() { @@ -81,16 +81,16 @@ public void IfDuFailsWeLogCorrectly() // stdout x.ArgAt>(3).Invoke("500\t/octopus"); x.ArgAt>(3).Invoke($"{usedSize}\t/octopus"); - + // stderr x.ArgAt>(4).Invoke("no permission for foo"); x.ArgAt>(4).Invoke("also no permission for bar"); }); spr.ReturnsForAll(1); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(systemLog, spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), systemLog, spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); - + systemLog.GetLogsForCategory(LogCategory.Warning).Should().Contain("Could not reliably get disk space using du. Getting best approximation..."); systemLog.GetLogsForCategory(LogCategory.Info).Should().Contain($"Du stdout returned 500\t/octopus, {usedSize}\t/octopus"); systemLog.GetLogsForCategory(LogCategory.Info).Should().Contain("Du stderr returned no permission for foo, also no permission for bar"); @@ -102,10 +102,10 @@ public void IfDuFailsCompletelyReturnNull() var spr = Substitute.For(); spr.ReturnsForAll(1); var memoryCache = new MemoryCache(new MemoryCacheOptions()); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(null); } - + [Test] public void ReturnedValueShouldBeCached() { @@ -113,10 +113,10 @@ public void ReturnedValueShouldBeCached() spr.ReturnsForAll(1); var baseTime = DateTimeOffset.UtcNow; var clock = new TestClock(baseTime); - var memoryCache = new MemoryCache(new MemoryCacheOptions(){ Clock = clock}); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var memoryCache = new MemoryCache(new MemoryCacheOptions() { Clock = clock }); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(null); - + const ulong usedSize = 500 * Megabyte; spr.When(x => x.ExecuteCommand("du", "-s -B 1 /octopus", "/", Arg.Any>(), Arg.Any>())) .Do(x => @@ -128,7 +128,7 @@ public void ReturnedValueShouldBeCached() sut.GetPathUsedBytes("/octopus").Should().Be(null); } - + [Test] public void DuCacheExpiresAfter30Seconds() { @@ -136,10 +136,10 @@ public void DuCacheExpiresAfter30Seconds() spr.ReturnsForAll(1); var baseTime = DateTimeOffset.UtcNow; var clock = new TestClock(baseTime); - var memoryCache = new MemoryCache(new MemoryCacheOptions(){ Clock = clock}); - var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), spr, memoryCache); + var memoryCache = new MemoryCache(new MemoryCacheOptions() { Clock = clock }); + var sut = new KubernetesDirectoryInformationProvider(Substitute.For(), Substitute.For(), spr, memoryCache); sut.GetPathUsedBytes("/octopus").Should().Be(null); - + const ulong usedSize = 500 * Megabyte; spr.When(x => x.ExecuteCommand("du", "-s -B 1 /octopus", "/", Arg.Any>(), Arg.Any>())) .Do(x => @@ -149,10 +149,9 @@ public void DuCacheExpiresAfter30Seconds() }); clock.UtcNow = baseTime + TimeSpan.FromSeconds(30); - + sut.GetPathUsedBytes("/octopus").Should().Be(usedSize); } - } public class TestClock : ISystemClock diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs index 8c980203a..f25eedb9d 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesEventMonitorFixture.cs @@ -56,9 +56,10 @@ public class KubernetesEventMonitorFixture public async Task NoEntriesAreSentToMetricsWhenEventListIsEmpty() { var agentMetrics = Substitute.For(); + var config = Substitute.For(); agentMetrics.GetLatestEventTimestamp(Arg.Any()).ReturnsForAnyArgs(testEpoch); var eventService = Substitute.For(); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(config, agentMetrics, eventService, new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); await sut.CacheNewEvents(tokenSource.Token); @@ -71,7 +72,8 @@ public async Task NfsPodKillingEventsAreTrackedInMetrics() //Arrange var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + var config = Substitute.For(); + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -85,7 +87,7 @@ public async Task NfsPodKillingEventsAreTrackedInMetrics() LastTimestamp = testEpoch.DateTime.AddMinutes(1) } })); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(config,agentMetrics, eventService, new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); //Act await sut.CacheNewEvents(tokenSource.Token); @@ -104,7 +106,8 @@ public async Task NfsWatchDogEventsAreTrackedInMetrics() var podName = "octopus-script-123412341234.123412341234"; var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + var config = Substitute.For(); + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -119,7 +122,7 @@ public async Task NfsWatchDogEventsAreTrackedInMetrics() } })); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(config,agentMetrics, eventService, new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); //Act await sut.CacheNewEvents(tokenSource.Token); @@ -138,7 +141,8 @@ public async Task EventsOlderThanOrEqualToMetricsTimestampCursorAreNotAddedToMet var podName = "octopus-script-123412341234.123412341234"; var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + var config = Substitute.For(); + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -153,7 +157,7 @@ public async Task EventsOlderThanOrEqualToMetricsTimestampCursorAreNotAddedToMet } })); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(config,agentMetrics, eventService, new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); //Act await sut.CacheNewEvents(tokenSource.Token); @@ -168,7 +172,8 @@ public async Task NewestTimeStampInEventIsUsedToDetermineAgeAndAsMetricsValue() var podName = "octopus-script-123412341234.123412341234"; var agentMetrics = new StubbedAgentMetrics(testEpoch); var eventService = Substitute.For(); - eventService.FetchAllEventsAsync(Arg.Any(), Arg.Any()).ReturnsForAnyArgs( + var config = Substitute.For(); + eventService.FetchAllEventsAsync(Arg.Any()).ReturnsForAnyArgs( new Corev1EventList(new List { new() @@ -184,7 +189,7 @@ public async Task NewestTimeStampInEventIsUsedToDetermineAgeAndAsMetricsValue() } })); - var sut = new KubernetesEventMonitor(agentMetrics, eventService, "arbitraryNamespace", new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); + var sut = new KubernetesEventMonitor(config, agentMetrics, eventService, new IEventMapper[]{new NfsPodRestarted(), new TentacleKilledEventMapper(), new NfsStaleEventMapper()}, log); //Act await sut.CacheNewEvents(tokenSource.Token); diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs index 994b394d9..b368ff007 100644 --- a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs +++ b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs @@ -26,6 +26,7 @@ public class KubernetesOrphanedPodCleanerTests DateTimeOffset startTime; ITentacleScriptLogProvider scriptLogProvider; IScriptPodSinceTimeStore scriptPodSinceTimeStore; + IKubernetesConfiguration config; [SetUp] public void Setup() @@ -40,17 +41,14 @@ public void Setup() monitor = Substitute.For(); scriptTicket = new ScriptTicket(Guid.NewGuid().ToString()); - cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore); + config = Substitute.For(); + config.PodsConsideredOrphanedAfterTimeSpan.Returns(TimeSpan.FromMinutes(10)); + config.DisableAutomaticPodCleanup.Returns(false); - overCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan + 1.Minutes(); - underCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan - 1.Minutes(); - } + cleaner = new KubernetesOrphanedPodCleaner(config, monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore); - [TearDown] - public void TearDown() - { - Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__DISABLEAUTOPODCLEANUP", null); - Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__PODSCONSIDEREDORPHANEDAFTERMINUTES", null); + overCutoff = config.PodsConsideredOrphanedAfterTimeSpan + 1.Minutes(); + underCutoff = config.PodsConsideredOrphanedAfterTimeSpan - 1.Minutes(); } [Test] @@ -142,7 +140,7 @@ public async Task OrphanedPodNotCleanedUpIfOnly9MinutesHavePassed() public async Task OrphanedPodNotCleanedUpIfPodCleanupIsDisabled() { //Arrange - Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__DISABLEAUTOPODCLEANUP", "true"); + config.DisableAutomaticPodCleanup.Returns(true); var pods = new List { CreatePod(TrackedScriptPodState.Succeeded(0, startTime)) @@ -159,40 +157,6 @@ public async Task OrphanedPodNotCleanedUpIfPodCleanupIsDisabled() scriptPodSinceTimeStore.Received().Delete(scriptTicket); } - [TestCase(1, false)] - [TestCase(3, true)] - public async Task EnvironmentVariableDictatesWhenPodsAreConsideredOrphaned(int checkAfterMinutes, bool shouldDelete) - { - //Arrange - Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__PODSCONSIDEREDORPHANEDAFTERMINUTES", "2"); - - // We need to reinitialise the sut after changing the env var value - cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore); - var pods = new List - { - CreatePod(TrackedScriptPodState.Succeeded(0, startTime)) - }; - monitor.GetAllTrackedScriptPods().Returns(pods); - clock.WindForward(TimeSpan.FromMinutes(checkAfterMinutes)); - - //Act - await cleaner.CheckForOrphanedPods(CancellationToken.None); - - //Assert - if (shouldDelete) - { - await podService.Received().DeleteIfExists(scriptTicket, Arg.Any()); - scriptLogProvider.Received().Delete(scriptTicket); - scriptPodSinceTimeStore.Received().Delete(scriptTicket); - } - else - { - await podService.DidNotReceiveWithAnyArgs().DeleteIfExists(scriptTicket, Arg.Any()); - scriptLogProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket); - scriptPodSinceTimeStore.DidNotReceiveWithAnyArgs().Delete(scriptTicket); - } - } - ITrackedScriptPod CreatePod(TrackedScriptPodState state) { var trackedScriptPod = Substitute.For(); diff --git a/source/Octopus.Tentacle/Configuration/ConfigurationModule.cs b/source/Octopus.Tentacle/Configuration/ConfigurationModule.cs index f508c2434..893ab32f3 100644 --- a/source/Octopus.Tentacle/Configuration/ConfigurationModule.cs +++ b/source/Octopus.Tentacle/Configuration/ConfigurationModule.cs @@ -3,6 +3,7 @@ using Octopus.Tentacle.Configuration.Crypto; using Octopus.Tentacle.Configuration.EnvironmentVariableMappings; using Octopus.Tentacle.Configuration.Instances; +using Octopus.Tentacle.Kubernetes.Configuration; using Octopus.Tentacle.Startup; using Octopus.Tentacle.Util; using Octopus.Tentacle.Watchdog; diff --git a/source/Octopus.Tentacle/Configuration/Crypto/KubernetesMachineKeyEncryptor.cs b/source/Octopus.Tentacle/Configuration/Crypto/KubernetesMachineKeyEncryptor.cs index f7d8f889e..ed968775e 100644 --- a/source/Octopus.Tentacle/Configuration/Crypto/KubernetesMachineKeyEncryptor.cs +++ b/source/Octopus.Tentacle/Configuration/Crypto/KubernetesMachineKeyEncryptor.cs @@ -52,7 +52,8 @@ public string Decrypt(string encrypted) } V1Secret GetSecret() { - return kubernetesSecretService.TryGetSecretAsync(SecretName, CancellationToken.None).GetAwaiter().GetResult() ?? throw new InvalidOperationException($"Unable to retrieve MachineKey from secret for namespace {KubernetesConfig.Namespace}"); + return kubernetesSecretService.TryGetSecretAsync(SecretName, CancellationToken.None).GetAwaiter().GetResult() + ?? throw new InvalidOperationException($"Unable to retrieve MachineKey from secret for namespace {KubernetesAgentDetection.Namespace}"); } (byte[] key, byte[] iv) GetMachineKey() diff --git a/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs b/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs index b5324d983..6088e8ca4 100644 --- a/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs +++ b/source/Octopus.Tentacle/Configuration/Crypto/LinuxGeneratedMachineKey.cs @@ -2,6 +2,7 @@ using System.IO; using System.Security.Cryptography; using Octopus.Diagnostics; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Util; using Octopus.Tentacle.Variables; @@ -13,7 +14,7 @@ public class LinuxGeneratedMachineKey: ICryptoKeyNixSource readonly IOctopusFileSystem fileSystem; static string FileName => - PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent + KubernetesAgentDetection.IsRunningAsKubernetesAgent //if we are running in K8S, we want to save the machine key to the home directory, which is likely on a network drives ? Path.Combine(Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleHome)!, "machinekey") : "/etc/octopus/machinekey"; diff --git a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs index ae5482e0f..555dc99fd 100644 --- a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs +++ b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceManager.cs @@ -11,6 +11,7 @@ class ApplicationInstanceManager : IApplicationInstanceManager readonly IOctopusFileSystem fileSystem; readonly ISystemLog log; readonly IApplicationInstanceStore instanceStore; + readonly IKubernetesAgentDetection kubernetesAgentDetection; readonly Lazy homeConfiguration; readonly ApplicationName applicationName; readonly StartUpInstanceRequest startUpInstanceRequest; @@ -21,6 +22,7 @@ public ApplicationInstanceManager( IOctopusFileSystem fileSystem, ISystemLog log, IApplicationInstanceStore instanceStore, + IKubernetesAgentDetection kubernetesAgentDetection, Lazy homeConfiguration) { this.applicationName = applicationName; @@ -28,6 +30,7 @@ public ApplicationInstanceManager( this.fileSystem = fileSystem; this.log = log; this.instanceStore = instanceStore; + this.kubernetesAgentDetection = kubernetesAgentDetection; this.homeConfiguration = homeConfiguration; } @@ -68,10 +71,13 @@ void WriteHomeDirectory(string homeDirectory) void EnsureConfigurationFileExists(string configurationFile, string homeDirectory) { //Skip this step if we're running on Kubernetes - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) return; + if (kubernetesAgentDetection.IsRunningAsKubernetesAgent) + { + return; + } // Ensure we can write configuration file - string configurationDirectory = Path.GetDirectoryName(configurationFile) ?? homeDirectory; + var configurationDirectory = Path.GetDirectoryName(configurationFile) ?? homeDirectory; fileSystem.EnsureDirectoryExists(configurationDirectory); if (!fileSystem.FileExists(configurationFile)) { @@ -87,9 +93,7 @@ void EnsureConfigurationFileExists(string configurationFile, string homeDirector (string ConfigurationFilePath, string HomeDirectory) ValidateConfigDirectory(string? configurationFile, string? homeDirectory) { // If home directory is not provided, we should try use the config file path if provided, otherwise fallback to cwd - homeDirectory ??= (string.IsNullOrEmpty(configurationFile) ? - "." : - Path.GetDirectoryName(fileSystem.GetFullPath(configurationFile!)) ?? "."); + homeDirectory ??= (string.IsNullOrEmpty(configurationFile) ? "." : Path.GetDirectoryName(fileSystem.GetFullPath(configurationFile!)) ?? "."); // Current "Indexed" installs require configuration file to be provided. // We can therefore assume that if its missing, it will end up being created in the cwd diff --git a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs index 1f12ac1c4..649f6efa6 100644 --- a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs +++ b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceSelector.cs @@ -3,6 +3,7 @@ using System.Linq; using Octopus.Diagnostics; using Octopus.Tentacle.Kubernetes; +using Octopus.Tentacle.Kubernetes.Configuration; using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Configuration.Instances @@ -14,9 +15,10 @@ class ApplicationInstanceSelector : IApplicationInstanceSelector readonly IApplicationConfigurationContributor[] instanceStrategies; readonly IOctopusFileSystem fileSystem; readonly ISystemLog log; + readonly IKubernetesAgentDetection kubernetesAgentDetection; readonly object @lock = new object(); ApplicationInstanceConfiguration? current; - Lazy configMapStoreFactory; + readonly Lazy configMapStoreFactory; public ApplicationInstanceSelector( ApplicationName applicationName, @@ -25,13 +27,15 @@ public ApplicationInstanceSelector( IApplicationConfigurationContributor[] instanceStrategies, Lazy configMapStoreFactory, IOctopusFileSystem fileSystem, - ISystemLog log) + ISystemLog log, + IKubernetesAgentDetection kubernetesAgentDetection) { this.applicationInstanceStore = applicationInstanceStore; this.startUpInstanceRequest = startUpInstanceRequest; this.instanceStrategies = instanceStrategies; this.fileSystem = fileSystem; this.log = log; + this.kubernetesAgentDetection = kubernetesAgentDetection; ApplicationName = applicationName; this.configMapStoreFactory = configMapStoreFactory; } @@ -77,10 +81,9 @@ ApplicationInstanceConfiguration LoadInstance() (IKeyValueStore, IWritableKeyValueStore) LoadConfigurationStore((string? instanceName, string? configurationpath) appInstance) { - if (appInstance is { instanceName: not null, configurationpath: null } && - PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (appInstance is { instanceName: not null, configurationpath: null } && kubernetesAgentDetection.IsRunningAsKubernetesAgent) { - log.Verbose($"Loading configuration from ConfigMap for namespace {KubernetesConfig.Namespace}"); + log.Verbose($"Loading configuration from ConfigMap for namespace {kubernetesAgentDetection.Namespace}"); var configMapWritableStore = configMapStoreFactory.Value; return (ContributeAdditionalConfiguration(configMapWritableStore), configMapWritableStore); } diff --git a/source/Octopus.Tentacle/Configuration/Instances/ConfigMapKeyValueStore.cs b/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs similarity index 90% rename from source/Octopus.Tentacle/Configuration/Instances/ConfigMapKeyValueStore.cs rename to source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs index 39660de48..4343b52f3 100644 --- a/source/Octopus.Tentacle/Configuration/Instances/ConfigMapKeyValueStore.cs +++ b/source/Octopus.Tentacle/Kubernetes/Configuration/ConfigMapKeyValueStore.cs @@ -3,14 +3,14 @@ using System.Threading; using k8s.Models; using Newtonsoft.Json; -using Octopus.Tentacle.Kubernetes; +using Octopus.Tentacle.Configuration; using Octopus.Tentacle.Configuration.Crypto; +using Octopus.Tentacle.Configuration.Instances; -namespace Octopus.Tentacle.Configuration.Instances +namespace Octopus.Tentacle.Kubernetes.Configuration { class ConfigMapKeyValueStore : IWritableKeyValueStore, IAggregatableKeyValueStore { - readonly IKubernetesConfigMapService configMapService; readonly IKubernetesMachineKeyEncryptor encryptor; const string Name = ConfigMapNames.TentacleConfig; @@ -18,12 +18,12 @@ class ConfigMapKeyValueStore : IWritableKeyValueStore, IAggregatableKeyValueStor readonly Lazy configMap; IDictionary ConfigMapData => configMap.Value.Data ??= new Dictionary(); - public ConfigMapKeyValueStore(IKubernetesConfigMapService configMapService, IKubernetesMachineKeyEncryptor encryptor) + public ConfigMapKeyValueStore(IKubernetesConfiguration kubernetesConfiguration, IKubernetesConfigMapService configMapService, IKubernetesMachineKeyEncryptor encryptor) { this.configMapService = configMapService; this.encryptor = encryptor; configMap = new Lazy(() => configMapService.TryGet(Name, CancellationToken.None).GetAwaiter().GetResult() - ?? throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {KubernetesConfig.Namespace}")); + ?? throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {kubernetesConfiguration.Namespace}")); } public string? Get(string name, ProtectionLevel protectionLevel = ProtectionLevel.None) diff --git a/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs b/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs index 5784d06f0..47c7f4403 100644 --- a/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs +++ b/source/Octopus.Tentacle/Kubernetes/Diagnostics/PersistenceProvider.cs @@ -18,12 +18,14 @@ public class PersistenceProvider : IPersistenceProvider public delegate PersistenceProvider Factory(string configMapName); readonly string configMapName; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesConfigMapService configMapService; - public PersistenceProvider(string configMapName, IKubernetesConfigMapService configMapService) + public PersistenceProvider(string configMapName, IKubernetesConfiguration kubernetesConfiguration, IKubernetesConfigMapService configMapService) { this.configMapService = configMapService; this.configMapName = configMapName; + this.kubernetesConfiguration = kubernetesConfiguration; } public async Task GetValue(string key, CancellationToken cancellationToken) @@ -35,7 +37,7 @@ public PersistenceProvider(string configMapName, IKubernetesConfigMapService con public async Task PersistValue(string key, string value, CancellationToken cancellationToken) { var configMapData = await LoadConfigMapData(cancellationToken); - if (configMapData is null) throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {KubernetesConfig.Namespace}"); + if (configMapData is null) throw new InvalidOperationException($"Unable to retrieve Tentacle Configuration from config map for namespace {kubernetesConfiguration.Namespace}"); configMapData[key] = value; await configMapService.Patch(configMapName, configMapData, cancellationToken); diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs new file mode 100644 index 000000000..db6bfa505 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesAgentDetection.cs @@ -0,0 +1,31 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Octopus.Tentacle.Kubernetes +{ + public interface IKubernetesAgentDetection + { + /// + /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent + /// + bool IsRunningAsKubernetesAgent { get; } + + /// + /// The Kubernetes namespace the agent is running under, null if not running as a Kubernetes agent + /// + [MemberNotNullWhen(true, nameof(Namespace))] + string? Namespace { get; } + } + + public class KubernetesAgentDetection : IKubernetesAgentDetection + { + public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Namespace); + public static string? Namespace => Environment.GetEnvironmentVariable(KubernetesEnvironmentVariableNames.Namespace); + + /// + bool IKubernetesAgentDetection.IsRunningAsKubernetesAgent => IsRunningAsKubernetesAgent; + + /// + string? IKubernetesAgentDetection.Namespace => Namespace; + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs new file mode 100644 index 000000000..ae3b44105 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesConfiguration.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; + +namespace Octopus.Tentacle.Kubernetes +{ + public interface IKubernetesConfiguration + { + string Namespace { get;} + string BootstrapRunnerExecutablePath { get; } + string ScriptPodServiceAccountName { get; } + IEnumerable ScriptPodImagePullSecretNames { get; } + string ScriptPodVolumeClaimName { get; } + string? ScriptPodResourceJson { get; } + string? NfsWatchdogImage { get; } + string HelmReleaseName { get; } + string HelmChartVersion { get; } + string[] ServerCommsAddresses { get; } + string PodVolumeClaimName { get; } + int? PodMonitorTimeoutSeconds { get; } + TimeSpan PodsConsideredOrphanedAfterTimeSpan { get; } + bool DisableAutomaticPodCleanup { get; } + string PersistentVolumeSize { get; } + bool IsMetricsEnabled { get; } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesPodLogService.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesPodLogService.cs index 94bc274f6..93cd0ec2e 100644 --- a/source/Octopus.Tentacle/Kubernetes/IKubernetesPodLogService.cs +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesPodLogService.cs @@ -22,8 +22,8 @@ class KubernetesPodLogService : KubernetesService, IKubernetesPodLogService readonly ITentacleScriptLogProvider scriptLogProvider; readonly IScriptPodSinceTimeStore scriptPodSinceTimeStore; - public KubernetesPodLogService(IKubernetesClientConfigProvider configProvider, IKubernetesPodMonitor podMonitor, ITentacleScriptLogProvider scriptLogProvider, IScriptPodSinceTimeStore scriptPodSinceTimeStore, ISystemLog log) - : base(configProvider, log) + public KubernetesPodLogService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodMonitor podMonitor, ITentacleScriptLogProvider scriptLogProvider, IScriptPodSinceTimeStore scriptPodSinceTimeStore, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { this.podMonitor = podMonitor; this.scriptLogProvider = scriptLogProvider; @@ -93,7 +93,7 @@ public KubernetesPodLogService(IKubernetesClientConfigProvider configProvider, I { try { - return await Client.GetNamespacedPodLogsAsync(podName, KubernetesConfig.Namespace, podName, sinceTime, cancellationToken: cancellationToken); + return await Client.GetNamespacedPodLogsAsync(podName, Namespace, podName, sinceTime, cancellationToken: cancellationToken); } catch (HttpOperationException ex) { diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs index 8d51ea7a9..220d04a00 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs @@ -14,8 +14,8 @@ public interface IKubernetesClusterService public class KubernetesClusterService : KubernetesService, IKubernetesClusterService { readonly AsyncLazy lazyVersion; - public KubernetesClusterService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesClusterService(IKubernetesClientConfigProvider configProvider,IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { //As the cluster version isn't going to change without restarting, we just cache the version in an AsyncLazy lazyVersion = new AsyncLazy(async () => diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs deleted file mode 100644 index 66d49bd7c..000000000 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs +++ /dev/null @@ -1,84 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Octopus.Tentacle.Util; - -namespace Octopus.Tentacle.Kubernetes -{ - public static class KubernetesConfig - { - const string ServerCommsAddressVariableName = "ServerCommsAddress"; - const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; - - public static string NamespaceVariableName => $"{EnvVarPrefix}__NAMESPACE"; - public static string Namespace => GetRequiredEnvVar(NamespaceVariableName, "Unable to determine Kubernetes namespace."); - public static string PodServiceAccountName => GetRequiredEnvVar($"{EnvVarPrefix}__PODSERVICEACCOUNTNAME", "Unable to determine Kubernetes Pod service account name."); - public static string PodVolumeClaimName => GetRequiredEnvVar($"{EnvVarPrefix}__PODVOLUMECLAIMNAME", "Unable to determine Kubernetes Pod persistent volume claim name."); - - public static int PodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODMONITORTIMEOUT"), out var podMonitorTimeout) ? podMonitorTimeout : 10*60; //10min - public static string NfsWatchdogImageVariableName => $"{EnvVarPrefix}__NFSWATCHDOGIMAGE"; - public static string? NfsWatchdogImage => Environment.GetEnvironmentVariable(NfsWatchdogImageVariableName); - - public static TimeSpan PodsConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODSCONSIDEREDORPHANEDAFTERMINUTES"), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); - public static bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable($"{EnvVarPrefix}__DISABLEAUTOPODCLEANUP"), out var disableAutoCleanup) && disableAutoCleanup; - - public static string HelmReleaseNameVariableName => $"{EnvVarPrefix}__HELMRELEASENAME"; - public static string HelmReleaseName => GetRequiredEnvVar(HelmReleaseNameVariableName, "Unable to determine Helm release name."); - - public static string HelmChartVersionVariableName => $"{EnvVarPrefix}__HELMCHARTVERSION"; - public static string HelmChartVersion => GetRequiredEnvVar(HelmChartVersionVariableName, "Unable to determine Helm chart version."); - - public static string BootstrapRunnerExecutablePath => GetRequiredEnvVar("BOOTSTRAPRUNNEREXECUTABLEPATH", "Unable to determine Bootstrap Runner Executable Path"); - - public static string PersistentVolumeSizeVariableName => $"{EnvVarPrefix}__PERSISTENTVOLUMESIZE"; - public static string PersistentVolumeSize => GetRequiredEnvVar(PersistentVolumeSizeVariableName, "Unable to determine Persistent Volume Size"); - - public static string PersistentVolumeSizeBytesVariableName => $"{EnvVarPrefix}__PERSISTENTVOLUMETOTALBYTES"; - public static string PersistentVolumeFreeBytesVariableName => $"{EnvVarPrefix}__PERSISTENTVOLUMEFREEBYTES"; - - public const string ServerCommsAddressesVariableName = "ServerCommsAddresses"; - - public static IEnumerable PodImagePullSecretNames => Environment.GetEnvironmentVariable($"{EnvVarPrefix}__PODIMAGEPULLSECRETNAMES") - ?.Split(',') - .Select(str => str.Trim()) - .WhereNotNullOrWhiteSpace() - .ToArray() ?? Array.Empty(); - - public static readonly string PodResourceJsonVariableName = $"{EnvVarPrefix}__PODRESOURCEJSON"; - public static string? PodResourceJson => Environment.GetEnvironmentVariable(PodResourceJsonVariableName); - - public static string MetricsEnableVariableName => $"{EnvVarPrefix}__ENABLEMETRICSCAPTURE"; - public static bool MetricsIsEnabled - { - get - { - var envContent = Environment.GetEnvironmentVariable(MetricsEnableVariableName); - if(bool.TryParse(envContent, out var result)) - { - return result; - } - return true; - } - } - - public static string[] ServerCommsAddresses { - get { - var addresses = new List(); - if (Environment.GetEnvironmentVariable(ServerCommsAddressVariableName) is { Length: > 0 } addressString) - { - addresses.Add(addressString); - } - if (Environment.GetEnvironmentVariable(ServerCommsAddressesVariableName) is {} addressesString) - { - addresses.AddRange(addressesString.Split(',').Where(a => !a.IsNullOrEmpty())); - } - return addresses.ToArray(); - } - - } - - static string GetRequiredEnvVar(string variable, string errorMessage) - => Environment.GetEnvironmentVariable(variable) - ?? throw new InvalidOperationException($"{errorMessage} The environment variable '{variable}' must be defined with a non-null value."); - } -} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs index 19292a205..b5c22a6ce 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesConfigMapService.cs @@ -18,8 +18,8 @@ public interface IKubernetesConfigMapService public class KubernetesConfigMapService : KubernetesService, IKubernetesConfigMapService { - public KubernetesConfigMapService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesConfigMapService(IKubernetesClientConfigProvider configProvider,IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration,log) { } @@ -29,7 +29,7 @@ public KubernetesConfigMapService(IKubernetesClientConfigProvider configProvider { try { - return await Client.CoreV1.ReadNamespacedConfigMapAsync(name, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + return await Client.CoreV1.ReadNamespacedConfigMapAsync(name, Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) when (opException.Response.StatusCode == HttpStatusCode.NotFound) @@ -49,7 +49,7 @@ public async Task Patch(string name, IDictionary da var configMapJson = KubernetesJson.Serialize(configMap); return await RetryPolicy.ExecuteAsync(async () => - await Client.CoreV1.PatchNamespacedConfigMapAsync(new V1Patch(configMapJson, V1Patch.PatchType.MergePatch), name, KubernetesConfig.Namespace, cancellationToken: cancellationToken)); + await Client.CoreV1.PatchNamespacedConfigMapAsync(new V1Patch(configMapJson, V1Patch.PatchType.MergePatch), name, Namespace, cancellationToken: cancellationToken)); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfiguration.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfiguration.cs new file mode 100644 index 000000000..2ce8de88d --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesConfiguration.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Linq;using Octopus.Tentacle.Kubernetes; +using Octopus.Tentacle.Util; +using Names = Octopus.Tentacle.Kubernetes.KubernetesEnvironmentVariableNames; + +namespace Octopus.Tentacle.Kubernetes +{ + public class KubernetesConfiguration : IKubernetesConfiguration + { + public string Namespace => GetRequiredEnvVar(Names.Namespace, "Unable to determine Kubernetes namespace."); + public string BootstrapRunnerExecutablePath => GetRequiredEnvVar(Names.BootstrapRunnerExecutablePath, "Unable to determine Bootstrap Runner Executable Path"); + public string ScriptPodServiceAccountName => GetRequiredEnvVar(Names.ScriptPodServiceAccountName, "Unable to determine Kubernetes Pod service account name."); + + public IEnumerable ScriptPodImagePullSecretNames => Environment.GetEnvironmentVariable(Names.ScriptPodImagePullSecretNames) + ?.Split(',') + .Select(str => str.Trim()) + .WhereNotNullOrWhiteSpace() + .ToArray() ?? Array.Empty(); + + public string ScriptPodVolumeClaimName => GetRequiredEnvVar(Names.ScriptPodVolumeClaimName, "Unable to determine Kubernetes Pod persistent volume claim name."); + public string? ScriptPodResourceJson => Environment.GetEnvironmentVariable(Names.ScriptPodResourceJson); + + public string? NfsWatchdogImage => Environment.GetEnvironmentVariable(Names.NfsWatchdogImage); + + + public string HelmReleaseName => GetRequiredEnvVar(Names.HelmReleaseName, "Unable to determine Helm release name."); + public string HelmChartVersion => GetRequiredEnvVar(Names.HelmChartVersion, "Unable to determine Helm chart version."); + + public string[] ServerCommsAddresses { + get { + var addresses = new List(); + if (Environment.GetEnvironmentVariable(Names.ServerCommsAddress) is { Length: > 0 } addressString) + { + addresses.Add(addressString); + } + if (Environment.GetEnvironmentVariable(Names.ServerCommsAddresses) is {} addressesString) + { + addresses.AddRange(addressesString.Split(',').Where(a => !a.IsNullOrEmpty())); + } + return addresses.ToArray(); + } + + } + + public string PodVolumeClaimName => GetRequiredEnvVar(Names.ScriptPodVolumeClaimName, "Unable to determine Kubernetes Pod persistent volume claim name."); + public int? PodMonitorTimeoutSeconds => int.TryParse(Environment.GetEnvironmentVariable(Names.ScriptPodMonitorTimeoutSeconds), out var podMonitorTimeout) ? podMonitorTimeout : 10*60; //10min + public TimeSpan PodsConsideredOrphanedAfterTimeSpan => TimeSpan.FromMinutes(int.TryParse(Environment.GetEnvironmentVariable(Names.ScriptPodsConsideredOrphanedAfter), out var podsConsideredOrphanedAfterTimeSpan) ? podsConsideredOrphanedAfterTimeSpan : 10); + public bool DisableAutomaticPodCleanup => bool.TryParse(Environment.GetEnvironmentVariable(Names.DisableAutomaticPodCleanup), out var disableAutoCleanup) && disableAutoCleanup; + + public string PersistentVolumeSize => GetRequiredEnvVar(Names.PersistentVolumeSize, "Unable to determine Persistent Volume Size"); + + public bool IsMetricsEnabled => !bool.TryParse(Environment.GetEnvironmentVariable(Names.EnableMetricsCapture), out var enableMetrics) || enableMetrics; + + static string GetRequiredEnvVar(string variable, string errorMessage) + => Environment.GetEnvironmentVariable(variable) + ?? throw new InvalidOperationException($"{errorMessage} The environment variable '{variable}' must be defined with a non-null value."); + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs index 2a1a5be53..3b628efe4 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesDirectoryInformationProvider.cs @@ -16,6 +16,7 @@ public interface IKubernetesDirectoryInformationProvider public class KubernetesDirectoryInformationProvider : IKubernetesDirectoryInformationProvider { + readonly IKubernetesConfiguration kubernetesConfiguration; readonly ISystemLog log; readonly ISilentProcessRunner silentProcessRunner; readonly IMemoryCache directoryInformationCache; @@ -29,8 +30,9 @@ public class KubernetesDirectoryInformationProvider : IKubernetesDirectoryInform //No calls to `du` at all: 8min ea. static readonly TimeSpan CacheExpiry = TimeSpan.FromSeconds(30); - public KubernetesDirectoryInformationProvider(ISystemLog log, ISilentProcessRunner silentProcessRunner, IMemoryCache directoryInformationCache) + public KubernetesDirectoryInformationProvider(IKubernetesConfiguration kubernetesConfiguration, ISystemLog log, ISilentProcessRunner silentProcessRunner, IMemoryCache directoryInformationCache) { + this.kubernetesConfiguration = kubernetesConfiguration; this.log = log; this.silentProcessRunner = silentProcessRunner; this.directoryInformationCache = directoryInformationCache; @@ -47,7 +49,7 @@ public KubernetesDirectoryInformationProvider(ISystemLog log, ISilentProcessRunn public ulong? GetPathTotalBytes() { - return KubernetesUtilities.GetResourceBytes(KubernetesConfig.PersistentVolumeSize); + return KubernetesUtilities.GetResourceBytes(kubernetesConfiguration.PersistentVolumeSize); } diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEnvironmentVariableNames.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEnvironmentVariableNames.cs new file mode 100644 index 000000000..d79baec66 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEnvironmentVariableNames.cs @@ -0,0 +1,38 @@ +namespace Octopus.Tentacle.Kubernetes +{ + public static class KubernetesEnvironmentVariableNames + { + const string EnvVarPrefix = "OCTOPUS__K8STENTACLE"; + + public static string GetPrefixedName(string suffix) => $"{EnvVarPrefix}__{suffix}".ToUpperInvariant(); + + public static string Namespace => GetPrefixedName("NAMESPACE"); + public static string BootstrapRunnerExecutablePath => "BOOTSTRAPRUNNEREXECUTABLEPATH"; + + public static string ScriptPodServiceAccountName => GetPrefixedName("PODSERVICEACCOUNTNAME"); + public static string ScriptPodImagePullSecretNames => GetPrefixedName("PODIMAGEPULLSECRETNAMES"); + public static string ScriptPodVolumeClaimName => GetPrefixedName("PODVOLUMECLAIMNAME"); + public static string ScriptPodResourceJson => GetPrefixedName("PODRESOURCEJSON"); + + + public static string NfsWatchdogImage => GetPrefixedName("NFSWATCHDOGIMAGE"); + + + + public static string HelmReleaseName => GetPrefixedName("HELMRELEASENAME"); + public static string HelmChartVersion => GetPrefixedName("HELMCHARTVERSION"); + + public const string ServerCommsAddresses = "ServerCommsAddresses"; + public const string ServerCommsAddress = "ServerCommsAddress"; + + public static string PersistentVolumeSizeBytes => GetPrefixedName("PERSISTENTVOLUMETOTALBYTES"); + public static string PersistentVolumeFreeBytes => GetPrefixedName("PERSISTENTVOLUMEFREEBYTES"); + public static string PersistentVolumeSize => GetPrefixedName("PERSISTENTVOLUMESIZE"); + public static string EnableMetricsCapture => GetPrefixedName("ENABLEMETRICSCAPTURE"); + + public static string ScriptPodMonitorTimeoutSeconds => GetPrefixedName("PODMONITORTIMEOUT"); + public static string ScriptPodsConsideredOrphanedAfter => GetPrefixedName("PODSCONSIDEREDORPHANEDAFTERMINUTES"); + public static string DisableAutomaticPodCleanup => GetPrefixedName("DISABLEAUTOPODCLEANUP"); + + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs index 7342416b8..6ab281d23 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitor.cs @@ -18,27 +18,25 @@ public interface IKubernetesEventMonitor public class KubernetesEventMonitor : IKubernetesEventMonitor { - public delegate KubernetesEventMonitor Factory(string kubernetesNamespace); - + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesAgentMetrics agentMetrics; readonly IKubernetesEventService eventService; - readonly string kubernetesNamespace; readonly IEventMapper[] eventMappers; readonly ISystemLog log; - public KubernetesEventMonitor(IKubernetesAgentMetrics agentMetrics, IKubernetesEventService eventService, string kubernetesNamespace, IEventMapper[] eventMappers, ISystemLog log) + public KubernetesEventMonitor(IKubernetesConfiguration kubernetesConfiguration,IKubernetesAgentMetrics agentMetrics, IKubernetesEventService eventService, IEventMapper[] eventMappers, ISystemLog log) { + this.kubernetesConfiguration = kubernetesConfiguration; this.agentMetrics = agentMetrics; this.eventService = eventService; - this.kubernetesNamespace = kubernetesNamespace; this.eventMappers = eventMappers; this.log = log; } public async Task CacheNewEvents(CancellationToken cancellationToken) { - log.Info($"Parsing kubernetes event list for namespace {kubernetesNamespace}."); - var allEvents = await eventService.FetchAllEventsAsync(kubernetesNamespace, cancellationToken) ?? new Corev1EventList(new List()); + log.Info($"Parsing kubernetes event list for namespace {kubernetesConfiguration.Namespace}."); + var allEvents = await eventService.FetchAllEventsAsync(cancellationToken) ?? new Corev1EventList(new List()); var lastCachedEventTimeStamp = await agentMetrics.GetLatestEventTimestamp(cancellationToken); diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs index fe2d910a6..6a4bbb21a 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventMonitorTask.cs @@ -9,20 +9,21 @@ namespace Octopus.Tentacle.Kubernetes { public class KubernetesEventMonitorTask : BackgroundTask { - public delegate KubernetesEventMonitorTask Factory(IKubernetesEventMonitor eventMonitor); - readonly IKubernetesEventMonitor eventMonitor; + readonly IKubernetesConfiguration kubernetesConfiguration; readonly ISystemLog log; readonly TimeSpan taskInterval = TimeSpan.FromMinutes(10); - public KubernetesEventMonitorTask(ISystemLog log, IKubernetesEventMonitor eventMonitor) : base(log, TimeSpan.FromSeconds(30)) + + public KubernetesEventMonitorTask(IKubernetesConfiguration kubernetesConfiguration, ISystemLog log, IKubernetesEventMonitor eventMonitor) : base(log, TimeSpan.FromSeconds(30)) { + this.kubernetesConfiguration = kubernetesConfiguration; this.log = log; this.eventMonitor = eventMonitor; } protected override async Task RunTask(CancellationToken cancellationToken) { - if (!KubernetesConfig.MetricsIsEnabled) + if (!kubernetesConfiguration.IsMetricsEnabled) { log.Info("Event monitoring for agent metrics is not enabled."); return; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs index b1b255984..57c355249 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesEventService.cs @@ -11,22 +11,23 @@ namespace Octopus.Tentacle.Kubernetes { public interface IKubernetesEventService { - Task FetchAllEventsAsync(string kubernetesNamespace, CancellationToken cancellationToken); + Task FetchAllEventsAsync(CancellationToken cancellationToken); } public class KubernetesEventService : KubernetesService, IKubernetesEventService { - public KubernetesEventService(IKubernetesClientConfigProvider configProvider, ISystemLog log) : base(configProvider, log) + public KubernetesEventService(IKubernetesClientConfigProvider configProvider,IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration,log) { } - public async Task FetchAllEventsAsync(string kubernetesNamespace, CancellationToken cancellationToken) + public async Task FetchAllEventsAsync(CancellationToken cancellationToken) { return await RetryPolicy.ExecuteAsync(async () => { try { - return await Client.CoreV1.ListNamespacedEventAsync(kubernetesNamespace, cancellationToken: cancellationToken); + return await Client.CoreV1.ListNamespacedEventAsync(Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) when (opException.Response.StatusCode == HttpStatusCode.NotFound) diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs index d7d8e7bc1..ededbf029 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs @@ -47,8 +47,7 @@ protected override void Load(ContainerBuilder builder) .As(); builder.RegisterType(); - builder.Register(ctx => ctx.Resolve().Invoke(KubernetesConfig.Namespace)) - .As(); + builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs index e67285db8..291097a6d 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesOrphanedPodCleaner.cs @@ -16,6 +16,7 @@ public interface IKubernetesOrphanedPodCleaner public class KubernetesOrphanedPodCleaner : IKubernetesOrphanedPodCleaner { + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesPodStatusProvider podStatusProvider; readonly IKubernetesPodService podService; readonly ISystemLog log; @@ -24,11 +25,11 @@ public class KubernetesOrphanedPodCleaner : IKubernetesOrphanedPodCleaner readonly IScriptPodSinceTimeStore scriptPodSinceTimeStore; readonly TimeSpan initialDelay = TimeSpan.FromMinutes(1); - internal readonly TimeSpan CompletedPodConsideredOrphanedAfterTimeSpan = KubernetesConfig.PodsConsideredOrphanedAfterTimeSpan; - public KubernetesOrphanedPodCleaner(IKubernetesPodStatusProvider podStatusProvider, IKubernetesPodService podService, ISystemLog log, IClock clock, + public KubernetesOrphanedPodCleaner(IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodStatusProvider podStatusProvider, IKubernetesPodService podService, ISystemLog log, IClock clock, ITentacleScriptLogProvider scriptLogProvider, IScriptPodSinceTimeStore scriptPodSinceTimeStore) { + this.kubernetesConfiguration = kubernetesConfiguration; this.podStatusProvider = podStatusProvider; this.podService = podService; this.log = log; @@ -63,16 +64,16 @@ async Task CleanupOrphanedPodsLoop(CancellationToken cancellationToken) log.Verbose("OrphanedPodCleaner: Checking for orphaned pods"); await CheckForOrphanedPods(cancellationToken); - var nextCheckTime = clock.GetUtcTime() + CompletedPodConsideredOrphanedAfterTimeSpan; + var nextCheckTime = clock.GetUtcTime() + kubernetesConfiguration.PodsConsideredOrphanedAfterTimeSpan; log.Verbose($"OrphanedPodCleaner: Next check will happen at {nextCheckTime:O}"); - await Task.Delay(CompletedPodConsideredOrphanedAfterTimeSpan, cancellationToken); + await Task.Delay(kubernetesConfiguration.PodsConsideredOrphanedAfterTimeSpan, cancellationToken); } } internal async Task CheckForOrphanedPods(CancellationToken cancellationToken) { - var cutOffDateTime = clock.GetUtcTime() - CompletedPodConsideredOrphanedAfterTimeSpan; + var cutOffDateTime = clock.GetUtcTime() - kubernetesConfiguration.PodsConsideredOrphanedAfterTimeSpan; var allPods = podStatusProvider.GetAllTrackedScriptPods(); var orphanedPods = allPods.Where(p => { @@ -94,7 +95,7 @@ state.FinishedAt is not null && scriptLogProvider.Delete(pod.ScriptTicket); scriptPodSinceTimeStore.Delete(pod.ScriptTicket); - if (KubernetesConfig.DisableAutomaticPodCleanup) + if (kubernetesConfiguration.DisableAutomaticPodCleanup) { log.Verbose($"OrphanedPodCleaner: Not deleting orphaned pod {pod.ScriptTicket} as automatic cleanup is disabled"); continue; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs index 9b52ac063..5e18b6c70 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodService.cs @@ -19,14 +19,14 @@ public interface IKubernetesPodService public class KubernetesPodService : KubernetesService, IKubernetesPodService { - public KubernetesPodService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesPodService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { } public async Task ListAllPods(CancellationToken cancellationToken) { - return await Client.ListNamespacedPodAsync(KubernetesConfig.Namespace, + return await Client.ListNamespacedPodAsync(Namespace, labelSelector: OctopusLabels.ScriptTicketId, cancellationToken: cancellationToken); } @@ -34,11 +34,11 @@ public async Task ListAllPods(CancellationToken cancellationToken) public async Task WatchAllPods(string initialResourceVersion, Func onChange, Action onError, CancellationToken cancellationToken) { using var response = Client.CoreV1.ListNamespacedPodWithHttpMessagesAsync( - KubernetesConfig.Namespace, + Namespace, labelSelector: OctopusLabels.ScriptTicketId, resourceVersion: initialResourceVersion, watch: true, - timeoutSeconds: KubernetesConfig.PodMonitorTimeoutSeconds, + timeoutSeconds: KubernetesConfiguration.PodMonitorTimeoutSeconds, cancellationToken: cancellationToken); var watchErrorCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -77,10 +77,10 @@ public async Task WatchAllPods(string initialResourceVersion, Func Create(V1Pod pod, CancellationToken cancellationToken) { AddStandardMetadata(pod); - return await Client.CreateNamespacedPodAsync(pod, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + return await Client.CreateNamespacedPodAsync(pod, Namespace, cancellationToken: cancellationToken); } public async Task DeleteIfExists(ScriptTicket scriptTicket, CancellationToken cancellationToken) - => await TryExecuteAsync(async () => await Client.DeleteNamespacedPodAsync(scriptTicket.ToKubernetesScriptPodName(), KubernetesConfig.Namespace, cancellationToken: cancellationToken)); + => await TryExecuteAsync(async () => await Client.DeleteNamespacedPodAsync(scriptTicket.ToKubernetesScriptPodName(), Namespace, cancellationToken: cancellationToken)); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs index 8d8ad3d42..c18be2ba1 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs @@ -19,6 +19,7 @@ public class KubernetesRawScriptPodCreator : KubernetesScriptPodCreator, IKubern readonly IKubernetesPodContainerResolver containerResolver; public KubernetesRawScriptPodCreator( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodService podService, IKubernetesPodMonitor podMonitor, IKubernetesSecretService secretService, @@ -28,7 +29,7 @@ public KubernetesRawScriptPodCreator( ITentacleScriptLogProvider scriptLogProvider, IHomeConfiguration homeConfiguration, KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem) - : base(podService, podMonitor, secretService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem) + : base(kubernetesConfiguration, podService, podMonitor, secretService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem) { this.containerResolver = containerResolver; } @@ -76,7 +77,7 @@ protected override IList CreateVolumes(StartKubernetesScriptCommandV1 Name = "init-nfs-volume", PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource { - ClaimName = KubernetesConfig.PodVolumeClaimName + ClaimName = KubernetesConfiguration.PodVolumeClaimName } } }; diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs index 213bc475b..c275e2713 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs @@ -26,6 +26,8 @@ public interface IKubernetesScriptPodCreator public class KubernetesScriptPodCreator : IKubernetesScriptPodCreator { + protected IKubernetesConfiguration KubernetesConfiguration { get; } + readonly IKubernetesPodService podService; readonly IKubernetesPodMonitor podMonitor; readonly IKubernetesSecretService secretService; @@ -37,6 +39,7 @@ public class KubernetesScriptPodCreator : IKubernetesScriptPodCreator readonly KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem; public KubernetesScriptPodCreator( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodService podService, IKubernetesPodMonitor podMonitor, IKubernetesSecretService secretService, @@ -47,6 +50,7 @@ public KubernetesScriptPodCreator( IHomeConfiguration homeConfiguration, KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem) { + this.KubernetesConfiguration = kubernetesConfiguration; this.podService = podService; this.podMonitor = podMonitor; this.secretService = secretService; @@ -120,7 +124,7 @@ public async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorks Metadata = new V1ObjectMeta { Name = secretName, - NamespaceProperty = KubernetesConfig.Namespace + NamespaceProperty = KubernetesConfiguration.Namespace }, Data = new Dictionary { @@ -164,19 +168,19 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo LogVerboseToBothLogs($"Creating Kubernetes Pod '{podName}'.", tentacleScriptLog); - workspace.CopyFile(KubernetesConfig.BootstrapRunnerExecutablePath, "bootstrapRunner", true); + workspace.CopyFile(KubernetesConfiguration.BootstrapRunnerExecutablePath, "bootstrapRunner", true); var scriptName = Path.GetFileName(workspace.BootstrapScriptFilePath); var workspacePath = Path.Combine("Work", workspace.ScriptTicket.TaskId); var serviceAccountName = !string.IsNullOrWhiteSpace(command.ScriptPodServiceAccountName) ? command.ScriptPodServiceAccountName - : KubernetesConfig.PodServiceAccountName; + : KubernetesConfiguration.ScriptPodServiceAccountName; // image pull secrets may have been defined in the helm chart (e.g. to avoid docker hub rate limiting) // we put any specified secret name first so it's resolved first var imagePullSecretNames = new[] { imagePullSecretName } - .Concat(KubernetesConfig.PodImagePullSecretNames) + .Concat(KubernetesConfiguration.ScriptPodImagePullSecretNames) .WhereNotNull() .Select(secretName => new V1LocalObjectReference(secretName)) .ToList(); @@ -186,7 +190,7 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo Metadata = new V1ObjectMeta { Name = podName, - NamespaceProperty = KubernetesConfig.Namespace, + NamespaceProperty = KubernetesConfiguration.Namespace, Labels = new Dictionary { ["octopus.com/serverTaskId"] = command.TaskId, @@ -243,7 +247,7 @@ protected virtual IList CreateVolumes(StartKubernetesScriptCommandV1 c Name = "tentacle-home", PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource { - ClaimName = KubernetesConfig.PodVolumeClaimName + ClaimName = KubernetesConfiguration.ScriptPodVolumeClaimName } } }; @@ -275,12 +279,12 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom VolumeMounts = new List { new(homeDir, "tentacle-home") }, Env = new List { - new(KubernetesConfig.NamespaceVariableName, KubernetesConfig.Namespace), - new(KubernetesConfig.HelmReleaseNameVariableName, KubernetesConfig.HelmReleaseName), - new(KubernetesConfig.HelmChartVersionVariableName, KubernetesConfig.HelmChartVersion), - new(KubernetesConfig.ServerCommsAddressesVariableName, string.Join(",", KubernetesConfig.ServerCommsAddresses)), - new(KubernetesConfig.PersistentVolumeFreeBytesVariableName, spaceInformation?.freeSpaceBytes.ToString()), - new(KubernetesConfig.PersistentVolumeSizeBytesVariableName, spaceInformation?.totalSpaceBytes.ToString()), + new(KubernetesEnvironmentVariableNames.Namespace, KubernetesConfiguration.Namespace), + new(KubernetesEnvironmentVariableNames.HelmReleaseName, KubernetesConfiguration.HelmReleaseName), + new(KubernetesEnvironmentVariableNames.HelmChartVersion, KubernetesConfiguration.HelmChartVersion), + new(KubernetesEnvironmentVariableNames.ServerCommsAddresses, string.Join(",", KubernetesConfiguration.ServerCommsAddresses)), + new(KubernetesEnvironmentVariableNames.PersistentVolumeFreeBytes, spaceInformation?.freeSpaceBytes.ToString()), + new(KubernetesEnvironmentVariableNames.PersistentVolumeSizeBytes, spaceInformation?.totalSpaceBytes.ToString()), new(EnvironmentVariables.TentacleHome, homeDir), new(EnvironmentVariables.TentacleInstanceName, appInstanceSelector.Current.InstanceName), new(EnvironmentVariables.TentacleVersion, Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleVersion)), @@ -295,7 +299,7 @@ protected async Task CreateScriptContainer(StartKubernetesScriptCom V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLog tentacleScriptLog) { - var json = KubernetesConfig.PodResourceJson; + var json = KubernetesConfiguration.ScriptPodResourceJson; if (!string.IsNullOrWhiteSpace(json)) { try @@ -304,7 +308,7 @@ V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLo } catch (Exception e) { - var message = $"Failed to deserialize env.{KubernetesConfig.PodResourceJsonVariableName} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod."; + var message = $"Failed to deserialize env.{KubernetesEnvironmentVariableNames.ScriptPodResourceJson} into valid pod resource requirements.{Environment.NewLine}JSON value: {json}{Environment.NewLine}Using default resource requests for script pod."; //if we can't parse the JSON, fall back to the defaults below and warn the user log.WarnFormat(e, message); //write a verbose message to the script log. @@ -323,9 +327,9 @@ V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLo }; } - static V1Container? CreateWatchdogContainer(string homeDir) + V1Container? CreateWatchdogContainer(string homeDir) { - if (KubernetesConfig.NfsWatchdogImage is null) + if (KubernetesConfiguration.NfsWatchdogImage is null) { return null; } @@ -333,7 +337,7 @@ V1ResourceRequirements GetScriptPodResourceRequirements(InMemoryTentacleScriptLo return new V1Container { Name = "nfs-watchdog", - Image = KubernetesConfig.NfsWatchdogImage, + Image = KubernetesConfiguration.NfsWatchdogImage, VolumeMounts = new List { new(homeDir, "tentacle-home"), diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs index c94c84f13..bc5a6b8c7 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesSecretService.cs @@ -19,8 +19,8 @@ public interface IKubernetesSecretService public class KubernetesSecretService : KubernetesService, IKubernetesSecretService { - public KubernetesSecretService(IKubernetesClientConfigProvider configProvider, ISystemLog log) - : base(configProvider, log) + public KubernetesSecretService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) + : base(configProvider, kubernetesConfiguration, log) { } @@ -30,7 +30,7 @@ public KubernetesSecretService(IKubernetesClientConfigProvider configProvider, I { try { - return await Client.ReadNamespacedSecretAsync(name, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + return await Client.ReadNamespacedSecretAsync(name, Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) when (opException.Response.StatusCode == HttpStatusCode.NotFound) @@ -45,7 +45,7 @@ public async Task CreateSecretAsync(V1Secret secret, CancellationToken cancellat AddStandardMetadata(secret); //We only want to retry read/modify operations for now (since they are idempotent) - await Client.CreateNamespacedSecretAsync(secret, KubernetesConfig.Namespace, cancellationToken: cancellationToken); + await Client.CreateNamespacedSecretAsync(secret, Namespace, cancellationToken: cancellationToken); } public async Task UpdateSecretDataAsync(string secretName, IDictionary secretData, CancellationToken cancellationToken) @@ -58,7 +58,7 @@ public async Task UpdateSecretDataAsync(string secretName, IDictionary - await Client.PatchNamespacedSecretAsync(new V1Patch(patchYaml, V1Patch.PatchType.MergePatch), secretName, KubernetesConfig.Namespace, cancellationToken: cancellationToken)); + await Client.PatchNamespacedSecretAsync(new V1Patch(patchYaml, V1Patch.PatchType.MergePatch), secretName, Namespace, cancellationToken: cancellationToken)); } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs index 27614adee..a26e78422 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs @@ -20,9 +20,14 @@ public abstract class KubernetesService protected ISystemLog Log { get; } protected AsyncRetryPolicy RetryPolicy { get; } protected k8sClient Client { get; } + + protected IKubernetesConfiguration KubernetesConfiguration { get; } + + protected string Namespace => KubernetesConfiguration.Namespace; - protected KubernetesService(IKubernetesClientConfigProvider configProvider, ISystemLog log) + protected KubernetesService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log) { + KubernetesConfiguration = kubernetesConfiguration; Log = log; Client = new k8sClient(configProvider.Get()); RetryPolicy = Policy.Handle().WaitAndRetryAsync(5, @@ -37,12 +42,12 @@ protected KubernetesService(IKubernetesClientConfigProvider configProvider, ISys protected void AddStandardMetadata(IKubernetesObject k8sObject) { //Everything should be in the main namespace - k8sObject.Metadata.NamespaceProperty = KubernetesConfig.Namespace; + k8sObject.Metadata.NamespaceProperty = Namespace; - //Add helm specific metadata so it's removed if the helm release is uninstalled + //Add helm specific metadata, so it's removed if the helm release is uninstalled k8sObject.Metadata.Annotations ??= new Dictionary(); - k8sObject.Metadata.Annotations["meta.helm.sh/release-name"] = KubernetesConfig.HelmReleaseName; - k8sObject.Metadata.Annotations["meta.helm.sh/release-namespace"] = KubernetesConfig.Namespace; + k8sObject.Metadata.Annotations["meta.helm.sh/release-name"] = KubernetesConfiguration.HelmReleaseName; + k8sObject.Metadata.Annotations["meta.helm.sh/release-namespace"] = Namespace; k8sObject.Metadata.Labels ??= new Dictionary(); k8sObject.Metadata.Labels["app.kubernetes.io/managed-by"] = "Helm"; diff --git a/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs b/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs index 25ed995d8..6b3d75541 100644 --- a/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs +++ b/source/Octopus.Tentacle/NullableReferenceTypeAttributes.cs @@ -5,7 +5,7 @@ /// These attributes replicate the ones from System.Diagnostics.CodeAnalysis, and are here so we can still compile against the older frameworks. /// -namespace Octopus.Tentacle +namespace System.Diagnostics.CodeAnalysis { [AttributeUsage(AttributeTargets.Property | AttributeTargets.Parameter | AttributeTargets.ReturnValue, AllowMultiple = true)] public sealed class NotNullIfNotNullAttribute : Attribute @@ -44,5 +44,12 @@ public NotNullWhenAttribute(bool returnValue) /// Gets the return value condition. public bool ReturnValue { get; } } + + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, AllowMultiple = true)] + public sealed class MemberNotNullWhenAttribute : Attribute + { + public MemberNotNullWhenAttribute(bool returnValue, params string[] members) { } + public MemberNotNullWhenAttribute(bool returnValue, string member) { } + } } #endif \ No newline at end of file diff --git a/source/Octopus.Tentacle/Program.cs b/source/Octopus.Tentacle/Program.cs index a9a05a74f..0df94e010 100644 --- a/source/Octopus.Tentacle/Program.cs +++ b/source/Octopus.Tentacle/Program.cs @@ -57,8 +57,9 @@ public override IContainer BuildContainer(StartUpInstanceRequest startUpInstance builder.RegisterModule(new ServicesModule()); builder.RegisterModule(new VersioningModule(GetType().Assembly)); builder.RegisterModule(new MaintenanceModule()); - - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + + builder.RegisterType().As().SingleInstance(); + if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { builder.RegisterModule(); } diff --git a/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs b/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs index f65e7f116..8a810abdf 100644 --- a/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs +++ b/source/Octopus.Tentacle/Services/Capabilities/CapabilitiesServiceV2.cs @@ -6,6 +6,7 @@ using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1Alpha; using Octopus.Tentacle.Contracts.ScriptServiceV2; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Services.Capabilities @@ -13,12 +14,19 @@ namespace Octopus.Tentacle.Services.Capabilities [Service(typeof(ICapabilitiesServiceV2))] public class CapabilitiesServiceV2 : IAsyncCapabilitiesServiceV2 { + readonly IKubernetesAgentDetection kubernetesAgentDetection; + + public CapabilitiesServiceV2(IKubernetesAgentDetection kubernetesAgentDetection) + { + this.kubernetesAgentDetection = kubernetesAgentDetection; + } + public async Task GetCapabilitiesAsync(CancellationToken cancellationToken) { await Task.CompletedTask; //the kubernetes agent only supports the kubernetes script services - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (kubernetesAgentDetection.IsRunningAsKubernetesAgent) { return new CapabilitiesResponseV2(new List { nameof(IFileTransferService), nameof(IKubernetesScriptServiceV1Alpha), nameof(IKubernetesScriptServiceV1) }); } diff --git a/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs b/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs index bae1084b7..7bc6d8627 100644 --- a/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs +++ b/source/Octopus.Tentacle/Services/Scripts/Kubernetes/KubernetesScriptServiceV1.cs @@ -18,6 +18,7 @@ namespace Octopus.Tentacle.Services.Scripts.Kubernetes [KubernetesService(typeof(IKubernetesScriptServiceV1))] public class KubernetesScriptServiceV1 : IAsyncKubernetesScriptServiceV1Alpha, IAsyncKubernetesScriptServiceV1, IRunningScriptReporter { + readonly IKubernetesConfiguration kubernetesConfiguration; readonly IKubernetesPodService podService; readonly IScriptWorkspaceFactory workspaceFactory; readonly IKubernetesPodStatusProvider podStatusProvider; @@ -29,6 +30,7 @@ public class KubernetesScriptServiceV1 : IAsyncKubernetesScriptServiceV1Alpha, I readonly IKeyedSemaphore keyedSemaphore; public KubernetesScriptServiceV1( + IKubernetesConfiguration kubernetesConfiguration, IKubernetesPodService podService, IScriptWorkspaceFactory workspaceFactory, IKubernetesPodStatusProvider podStatusProvider, @@ -39,6 +41,7 @@ public KubernetesScriptServiceV1( IScriptPodSinceTimeStore scriptPodSinceTimeStore, IKeyedSemaphore keyedSemaphore) { + this.kubernetesConfiguration = kubernetesConfiguration; this.podService = podService; this.workspaceFactory = workspaceFactory; this.podStatusProvider = podStatusProvider; @@ -117,7 +120,7 @@ public async Task CompleteScriptAsync(CompleteKubernetesScriptCommandV1 command, scriptLogProvider.Delete(command.ScriptTicket); scriptPodSinceTimeStore.Delete(command.ScriptTicket); - if (!KubernetesConfig.DisableAutomaticPodCleanup) + if (!kubernetesConfiguration.DisableAutomaticPodCleanup) await podService.DeleteIfExists(command.ScriptTicket, cancellationToken); } diff --git a/source/Octopus.Tentacle/Services/ServicesModule.cs b/source/Octopus.Tentacle/Services/ServicesModule.cs index 4e24c14e4..e4157130f 100644 --- a/source/Octopus.Tentacle/Services/ServicesModule.cs +++ b/source/Octopus.Tentacle/Services/ServicesModule.cs @@ -4,6 +4,7 @@ using System.Reflection; using Autofac; using Octopus.Tentacle.Communications; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Packages; using Octopus.Tentacle.Scripts; using Octopus.Tentacle.Util; @@ -27,7 +28,7 @@ protected override void Load(ContainerBuilder builder) RegisterHalibutServices(builder, allTypes); //only register kubernetes services when - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { RegisterHalibutServices(builder, allTypes); } diff --git a/source/Octopus.Tentacle/Startup/OctopusProgram.cs b/source/Octopus.Tentacle/Startup/OctopusProgram.cs index bc7b6687f..5c4f5c5d4 100644 --- a/source/Octopus.Tentacle/Startup/OctopusProgram.cs +++ b/source/Octopus.Tentacle/Startup/OctopusProgram.cs @@ -308,7 +308,7 @@ void InitializeLogging() Target.Register("EventLog"); #endif #if REQUIRES_EXPLICIT_LOG_CONFIG - var nLogFileExtension = !PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent + var nLogFileExtension = !KubernetesAgentDetection.IsRunningAsKubernetesAgent ? "exe.nlog" : "exe.k8s.nlog"; @@ -384,7 +384,7 @@ StartUpInstanceRequest TryLoadInstanceNameFromCommandLineArguments(string[] comm if (!string.IsNullOrWhiteSpace(instanceName)) { - return PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent + return KubernetesAgentDetection.IsRunningAsKubernetesAgent ? new StartUpKubernetesConfigMapInstanceRequest(instanceName) : new StartUpRegistryInstanceRequest(instanceName); } diff --git a/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs b/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs index 53ab25598..d98451dfd 100644 --- a/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs +++ b/source/Octopus.Tentacle/Util/OctopusFileSystemModule.cs @@ -9,7 +9,7 @@ public class OctopusFileSystemModule : Module protected override void Load(ContainerBuilder builder) { base.Load(builder); - if (PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) + if (KubernetesAgentDetection.IsRunningAsKubernetesAgent) { builder.RegisterType().AsSelf().As(); } diff --git a/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs b/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs index fe5f93a6b..5d37b9470 100644 --- a/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs +++ b/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs @@ -265,10 +265,6 @@ public virtual void EnsureDiskHasEnoughFreeSpace(string directoryPath, long requ if (!Path.IsPathRooted(directoryPath)) return; - //We can't perform this check in Kubernetes due to how drives are mounted and reported (always returns 0 byte sized drives) - if(PlatformDetection.Kubernetes.IsRunningAsKubernetesAgent) - return; - var driveInfo = SafelyGetDriveInfo(directoryPath); var required = requiredSpaceInBytes < 0 ? 0 : (ulong)requiredSpaceInBytes; diff --git a/source/Octopus.Tentacle/Util/PlatformDetection.cs b/source/Octopus.Tentacle/Util/PlatformDetection.cs index 801e10bfe..3d7b219d7 100644 --- a/source/Octopus.Tentacle/Util/PlatformDetection.cs +++ b/source/Octopus.Tentacle/Util/PlatformDetection.cs @@ -1,6 +1,5 @@ using System; using System.Runtime.InteropServices; -using Octopus.Tentacle.Kubernetes; namespace Octopus.Tentacle.Util { @@ -9,13 +8,5 @@ public static class PlatformDetection public static bool IsRunningOnNix => RuntimeInformation.IsOSPlatform(OSPlatform.Linux); public static bool IsRunningOnWindows => RuntimeInformation.IsOSPlatform(OSPlatform.Windows); public static bool IsRunningOnMac => RuntimeInformation.IsOSPlatform(OSPlatform.OSX); - - public static class Kubernetes - { - /// - /// Indicates if the Tentacle is running inside a Kubernetes cluster as the Kubernetes Agent. This is done by checking if the namespace environment variable is set - /// - public static bool IsRunningAsKubernetesAgent => !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(KubernetesConfig.NamespaceVariableName)); - } } } \ No newline at end of file