Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move to non-static IKubernetesConfiguration #984

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
namespace Octopus.Tentacle.Kubernetes.Tests.Integration;

public class HelmVersion1TestFixtureAttribute : TestFixtureAttribute
{
public HelmVersion1TestFixtureAttribute():
base("1.*.*")
{ }
}

public class HelmVersion2AlphaTestFixtureAttribute : TestFixtureAttribute
{
public HelmVersion2AlphaTestFixtureAttribute():
base("2.*.*-alpha")
{ }
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration;

public abstract class KubernetesAgentIntegrationTest
{
protected KubernetesAgentInstaller kubernetesAgentInstaller;
TraceLogFileLogger? traceLogFileLogger;
CancellationTokenSource cancellationTokenSource;
readonly string? helmChartVersion;

protected ILogger Logger { get; private set; }

protected HalibutRuntime ServerHalibutRuntime { get; private set; } = null!;
Expand All @@ -27,31 +26,41 @@ public abstract class KubernetesAgentIntegrationTest

protected CancellationToken CancellationToken { get; private set; }

protected KubernetesAgentInstaller KubernetesAgentInstaller { get; private set; } = null!;

protected KubeCtlTool KubeCtl { get; private set; }

string agentThumbprint;
TraceLogFileLogger? traceLogFileLogger;
CancellationTokenSource cancellationTokenSource;

protected KubernetesAgentIntegrationTest(string? helmChartVersion)
{
this.helmChartVersion = helmChartVersion;
}

[OneTimeSetUp]
public async Task OneTimeSetUp()
{
kubernetesAgentInstaller = new KubernetesAgentInstaller(
KubernetesAgentInstaller = new KubernetesAgentInstaller(
KubernetesTestsGlobalContext.Instance.TemporaryDirectory,
KubernetesTestsGlobalContext.Instance.HelmExePath,
KubernetesTestsGlobalContext.Instance.KubeCtlExePath,
KubernetesTestsGlobalContext.Instance.KubeConfigPath,
KubernetesTestsGlobalContext.Instance.Logger);
KubernetesTestsGlobalContext.Instance.Logger,
helmChartVersion);

KubeCtl = new KubeCtlTool(
KubernetesTestsGlobalContext.Instance.TemporaryDirectory,
KubernetesTestsGlobalContext.Instance.KubeCtlExePath,
KubernetesTestsGlobalContext.Instance.KubeConfigPath,
kubernetesAgentInstaller.Namespace,
KubernetesAgentInstaller.Namespace,
KubernetesTestsGlobalContext.Instance.Logger);

//create a new server halibut runtime
var listeningPort = BuildServerHalibutRuntimeAndListen();

agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag);
agentThumbprint = await KubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag);

//trust the generated cert thumbprint
ServerHalibutRuntime.Trust(agentThumbprint);
Expand Down Expand Up @@ -90,7 +99,7 @@ public async Task TearDown()

void BuildTentacleClient()
{
var endpoint = new ServiceEndPoint(kubernetesAgentInstaller.SubscriptionId, agentThumbprint, ServerHalibutRuntime.TimeoutsAndLimits);
var endpoint = new ServiceEndPoint(KubernetesAgentInstaller.SubscriptionId, agentThumbprint, ServerHalibutRuntime.TimeoutsAndLimits);

var retrySettings = new RpcRetrySettings(true, TimeSpan.FromMinutes(2));
var clientOptions = new TentacleClientOptions(retrySettings);
Expand Down Expand Up @@ -125,6 +134,6 @@ int BuildServerHalibutRuntimeAndListen()
public async Task OneTimeTearDown()
{
await ServerHalibutRuntime.DisposeAsync();
kubernetesAgentInstaller?.Dispose();
KubernetesAgentInstaller?.Dispose();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,24 @@
using FluentAssertions;
using k8s;
using Newtonsoft.Json;
using NSubstitute;
using Octopus.Diagnostics;
using Octopus.Tentacle.Diagnostics;
using Octopus.Tentacle.Kubernetes.Diagnostics;

namespace Octopus.Tentacle.Kubernetes.Tests.Integration;

[HelmVersion1TestFixture]
[HelmVersion2AlphaTestFixture]
public class KubernetesAgentMetricsIntegrationTest : KubernetesAgentIntegrationTest
{
readonly ISystemLog systemLog = new SystemLog();


public KubernetesAgentMetricsIntegrationTest(string? helmChartVersion)
: base(helmChartVersion)
{ }

class KubernetesFileWrappedProvider : IKubernetesClientConfigProvider
{
readonly string filename;
Expand All @@ -26,19 +34,20 @@ public KubernetesClientConfiguration Get()
return KubernetesClientConfiguration.BuildConfigFromConfigFile(filename);
}
}

[Test]
public async Task FetchingTimestampFromEmptyConfigMapEntryShouldBeMinValue()
{
//Arrange
var config = Substitute.For<IKubernetesConfiguration>();
var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath);
var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, kubernetesAgentInstaller.Namespace);
var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", configMapService);
var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, config, systemLog, KubernetesAgentInstaller.Namespace);
var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", config, configMapService);
var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog);

//Act
var result = await metrics.GetLatestEventTimestamp(CancellationToken.None);

//Assert
result.Should().Be(DateTimeOffset.MinValue);
}
Expand All @@ -47,14 +56,15 @@ public async Task FetchingTimestampFromEmptyConfigMapEntryShouldBeMinValue()
public async Task FetchingLatestEventTimestampFromNonexistentConfigMapThrowsException()
{
//Arrange
var config = Substitute.For<IKubernetesConfiguration>();
var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath);
var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, kubernetesAgentInstaller.Namespace);
var persistenceProvider = new PersistenceProvider("nonexistent-config-map", configMapService);
var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, config, systemLog, KubernetesAgentInstaller.Namespace);
var persistenceProvider = new PersistenceProvider("nonexistent-config-map", config, configMapService);
var metrics = new KubernetesAgentMetrics(persistenceProvider, "metrics", systemLog);

//Act
Func<Task> func = async () => await metrics.GetLatestEventTimestamp(CancellationToken.None);

//Assert
await func.Should().ThrowAsync<Exception>();
}
Expand All @@ -63,9 +73,10 @@ public async Task FetchingLatestEventTimestampFromNonexistentConfigMapThrowsExce
public async Task WritingEventToNonExistentConfigMapShouldFailSilently()
{
//Arrange
var config = Substitute.For<IKubernetesConfiguration>();
var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath);
var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, kubernetesAgentInstaller.Namespace);
var persistenceProvider = new PersistenceProvider("nonexistent-config-map", configMapService);
var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, config, systemLog, KubernetesAgentInstaller.Namespace);
var persistenceProvider = new PersistenceProvider("nonexistent-config-map", config, configMapService);
var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog);

//Act
Expand All @@ -79,15 +90,16 @@ public async Task WritingEventToNonExistentConfigMapShouldFailSilently()
public async Task WritingEventToExistingConfigMapShouldPersistJsonEntry()
{
//Arrange
var config = Substitute.For<IKubernetesConfiguration>();
var kubernetesConfigClient = new KubernetesFileWrappedProvider(KubernetesTestsGlobalContext.Instance.KubeConfigPath);
var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, systemLog, kubernetesAgentInstaller.Namespace);
var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", configMapService);
var configMapService = new Support.TestSupportConfigMapService(kubernetesConfigClient, config, systemLog, KubernetesAgentInstaller.Namespace);
var persistenceProvider = new PersistenceProvider("kubernetes-agent-metrics", config, configMapService);
var metrics = new KubernetesAgentMetrics(persistenceProvider, ConfigMapNames.AgentMetricsConfigMapKey, systemLog);

//Act
var eventTimestamp = DateTimeOffset.Now;
await metrics.TrackEvent("reason", "source", eventTimestamp, CancellationToken.None);

//Assert
var persistedDictionary = await persistenceProvider.ReadValues(CancellationToken.None);
var metricsData = persistedDictionary[ConfigMapNames.AgentMetricsConfigMapKey];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,20 @@

namespace Octopus.Tentacle.Kubernetes.Tests.Integration;

[TestFixture]
[HelmVersion1TestFixture]
[HelmVersion2AlphaTestFixture]
public class KubernetesScriptServiceV1AlphaIntegrationTest : KubernetesAgentIntegrationTest
{
public KubernetesScriptServiceV1AlphaIntegrationTest(string? helmChartVersion)
: base(helmChartVersion)
{
}

protected override TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder)
{
return builder
.DecorateCapabilitiesServiceV2With(d => d
.DecorateGetCapabilitiesWith((inner, options) => Task.FromResult(new CapabilitiesResponseV2(new List<string> { nameof(IFileTransferService), nameof(IKubernetesScriptServiceV1Alpha) }))));
.DecorateGetCapabilitiesWith((inner, options) => Task.FromResult(new CapabilitiesResponseV2(new List<string> { nameof(IFileTransferService), nameof(IKubernetesScriptServiceV1Alpha) }))));
}

[Test]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,17 @@

namespace Octopus.Tentacle.Kubernetes.Tests.Integration;

[TestFixture]
[HelmVersion1TestFixture]
[HelmVersion2AlphaTestFixture]
public class KubernetesScriptServiceV1IntegrationTest : KubernetesAgentIntegrationTest
{
IRecordedMethodUsages recordedMethodUsages = null!;

public KubernetesScriptServiceV1IntegrationTest(string? helmChartVersion)
: base(helmChartVersion)
{
}

protected override TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder)
{
builder.RecordMethodUsages<IAsyncClientKubernetesScriptServiceV1>(out var recordedUsages)
Expand Down Expand Up @@ -78,7 +84,7 @@ Task ScriptCompleted(CancellationToken ct)
return Task.CompletedTask;
}
}

[Test]
[TestCase(ScriptType.Normal)]
[TestCase(ScriptType.Raw)]
Expand Down Expand Up @@ -122,7 +128,7 @@ Task ScriptCompleted(CancellationToken ct)
return Task.CompletedTask;
}
}

[Test]
[TestCase(ScriptType.Normal)]
[TestCase(ScriptType.Raw)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0"/>
<PackageReference Include="NSubstitute" Version="4.4.0" />
<PackageReference Include="NUnit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
<PackageReference Include="Octopus.TestPortForwarder" Version="7.0.539" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,19 @@ public class KubernetesAgentInstaller
readonly string kubeCtlExePath;
readonly TemporaryDirectory temporaryDirectory;
readonly ILogger logger;
readonly string helmChartVersion;
readonly string kubeConfigPath;

bool isAgentInstalled;

public KubernetesAgentInstaller(TemporaryDirectory temporaryDirectory, string helmExePath, string kubeCtlExePath, string kubeConfigPath, ILogger logger)
public KubernetesAgentInstaller(TemporaryDirectory temporaryDirectory, string helmExePath, string kubeCtlExePath, string kubeConfigPath, ILogger logger, string? helmChartVersion)
{
this.temporaryDirectory = temporaryDirectory;
this.helmExePath = helmExePath;
this.kubeCtlExePath = kubeCtlExePath;
this.kubeConfigPath = kubeConfigPath;
this.logger = logger;
this.helmChartVersion = helmChartVersion ?? "1.*.*";

AgentName = Guid.NewGuid().ToString("N");
}
Expand Down Expand Up @@ -128,11 +130,13 @@ string BuildAgentInstallArguments(string valuesFilePath, string? tentacleImageAn
return string.Join(" ", args.WhereNotNull());
}

static string GetChartVersion()
string GetChartVersion()
{
var customHelmChartVersion = Environment.GetEnvironmentVariable("KubernetesIntegrationTests_HelmChartVersion");

return !string.IsNullOrWhiteSpace(customHelmChartVersion) ? customHelmChartVersion : "1.*.*";

return !string.IsNullOrWhiteSpace(customHelmChartVersion)
? customHelmChartVersion
: helmChartVersion;
}

static string? GetImageAndRepository(string? tentacleImageAndTag)
Expand Down Expand Up @@ -161,7 +165,7 @@ async Task<string> GetAgentThumbprint()
.WriteTo.StringBuilder(sb)
.MinimumLevel.Debug()
.CreateLogger();

var exitCode = SilentProcessRunner.ExecuteCommand(
kubeCtlExePath,
//get the generated thumbprint from the config map
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ public class TestSupportConfigMapService : KubernetesService, IKubernetesConfigM
{
readonly string targetNamespace;

public TestSupportConfigMapService(IKubernetesClientConfigProvider configProvider, ISystemLog log, string targetNamespace)
: base(configProvider, log)
public TestSupportConfigMapService(IKubernetesClientConfigProvider configProvider, IKubernetesConfiguration kubernetesConfiguration, ISystemLog log, string targetNamespace)
: base(configProvider, kubernetesConfiguration, log)
{
this.targetNamespace = targetNamespace;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using NSubstitute;
using NUnit.Framework;
using Octopus.Tentacle.Contracts;
using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1;
Expand All @@ -17,7 +18,10 @@ public class CapabilitiesServiceV2Fixture
[Test]
public async Task CapabilitiesAreReturned()
{
var capabilities = (await new CapabilitiesServiceV2()
var k8sDetection = Substitute.For<IKubernetesAgentDetection>();
k8sDetection.IsRunningAsKubernetesAgent.Returns(false);

var capabilities = (await new CapabilitiesServiceV2(k8sDetection)
.GetCapabilitiesAsync(CancellationToken.None))
.SupportedCapabilities;

Expand All @@ -30,18 +34,17 @@ public async Task CapabilitiesAreReturned()
[Test]
public async Task OnlyKubernetesScriptServicesAreReturnedWhenRunningAsKubernetesAgent()
{
Environment.SetEnvironmentVariable(KubernetesConfig.NamespaceVariableName, "ABC");
var k8sDetection = Substitute.For<IKubernetesAgentDetection>();
k8sDetection.IsRunningAsKubernetesAgent.Returns(true);

var capabilities = (await new CapabilitiesServiceV2()
var capabilities = (await new CapabilitiesServiceV2(k8sDetection)
.GetCapabilitiesAsync(CancellationToken.None))
.SupportedCapabilities;

capabilities.Should().BeEquivalentTo(nameof(IFileTransferService), nameof(IKubernetesScriptServiceV1Alpha), nameof(IKubernetesScriptServiceV1));
capabilities.Count.Should().Be(3);

capabilities.Should().NotContainMatch("IScriptService*");

Environment.SetEnvironmentVariable(KubernetesConfig.NamespaceVariableName, null);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Octopus.Tentacle.Configuration.Crypto;
using Octopus.Tentacle.Configuration.Instances;
using Octopus.Tentacle.Kubernetes;
using Octopus.Tentacle.Kubernetes.Configuration;
using Octopus.Tentacle.Util;

namespace Octopus.Tentacle.Tests.Configuration
Expand Down Expand Up @@ -189,10 +190,11 @@ ApplicationInstanceSelector CreateApplicationInstanceSelector(StartUpInstanceReq
return new ApplicationInstanceSelector(ApplicationName.Tentacle,
applicationInstanceStore,
instanceRequest ?? new StartUpDynamicInstanceRequest(),
additionalConfigurations ?? new IApplicationConfigurationContributor[0],
new Lazy<ConfigMapKeyValueStore>(() => new ConfigMapKeyValueStore(Substitute.For<IKubernetesConfigMapService>(), Substitute.For<IKubernetesMachineKeyEncryptor>())),
additionalConfigurations ?? Array.Empty<IApplicationConfigurationContributor>(),
new Lazy<ConfigMapKeyValueStore>(() => new ConfigMapKeyValueStore( Substitute.For<IKubernetesConfiguration>(),Substitute.For<IKubernetesConfigMapService>(), Substitute.For<IKubernetesMachineKeyEncryptor>())),
octopusFileSystem,
log);
log,
Substitute.For<IKubernetesAgentDetection>());
}
}
}
Loading