Encrypt pod logs in bootstrap runner and decrypt in Tentacle (#1047)

* Encrypt pod logs in bootstrap runner and decrypt in Tentacle * Fix decoding key * Change to factory created provide so we can inject the interface * If we can't read the encryption key, don't try and read the logs * Generate the encryption key based on the machine key and the script ticket * Move certain k8s types back into configurationmodule * Support custom tags with sha's * lock key generation and also make sure secret is updated
OctopusDeploy · Dec 1, 2024 · f4ed2ae · f4ed2ae
1 parent 3aeb42e
commit f4ed2ae
Show file tree

Hide file tree

Showing 33 changed files with 690 additions and 183 deletions.
diff --git a/docker/kubernetes-agent-tentacle/bootstrapRunner/bootstrapRunner.go b/docker/kubernetes-agent-tentacle/bootstrapRunner/bootstrapRunner.go
@@ -2,10 +2,16 @@ package main
 
 import (
 	"bufio"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
+	"path"
 	"sync"
 )
 
@@ -30,6 +36,12 @@ func main() {
 	args := os.Args[2:]
 	cmd := exec.Command("bash", args[0:]...)
 	cmd.Dir = workspacePath
+
+	gcm, err := CreateCipher(workspacePath)
+	if err != nil {
+		panic(err)
+	}
+
 	stdOutCmdReader, _ := cmd.StdoutPipe()
 	stdErrCmdReader, _ := cmd.StderrPipe()
 
@@ -39,14 +51,14 @@ func main() {
 	doneStd := make(chan bool)
 	doneErr := make(chan bool)
 
-	go reader(stdOutScanner, "stdout", &doneStd, &lineCounter)
-	go reader(stdErrScanner, "stderr", &doneErr, &lineCounter)
+	go reader(stdOutScanner, "stdout", &doneStd, &lineCounter, gcm)
+	go reader(stdErrScanner, "stderr", &doneErr, &lineCounter, gcm)
 
-	Write("stdout", "##octopus[stdout-verbose]", &lineCounter)
-	Write("stdout", "Kubernetes Script Pod started", &lineCounter)
-	Write("stdout", "##octopus[stdout-default]", &lineCounter)
+	Write("stdout", "##octopus[stdout-verbose]", &lineCounter, gcm)
+	Write("stdout", "Kubernetes Script Pod started", &lineCounter, gcm)
+	Write("stdout", "##octopus[stdout-default]", &lineCounter, gcm)
 
-	err := cmd.Start()
+	err = cmd.Start()
 
 	// Wait for output buffering first
 	<-doneStd
@@ -66,29 +78,73 @@ func main() {
 
 	exitCode := cmd.ProcessState.ExitCode()
 
-	Write("stdout", "##octopus[stdout-verbose]", &lineCounter)
-	Write("stdout", "Kubernetes Script Pod completed", &lineCounter)
-	Write("stdout", "##octopus[stdout-default]", &lineCounter)
+	Write("stdout", "##octopus[stdout-verbose]", &lineCounter, gcm)
+	Write("stdout", "Kubernetes Script Pod completed", &lineCounter, gcm)
+	Write("stdout", "##octopus[stdout-default]", &lineCounter, gcm)
 
-	Write("debug", fmt.Sprintf("EOS-075CD4F0-8C76-491D-BA76-0879D35E9CFE<<>>%d", exitCode), &lineCounter)
+	Write("debug", fmt.Sprintf("EOS-075CD4F0-8C76-491D-BA76-0879D35E9CFE<<>>%d", exitCode), &lineCounter, gcm)
 
 	os.Exit(exitCode)
 }
 
-func reader(scanner *bufio.Scanner, stream string, done *chan bool, counter *SafeCounter) {
+func reader(scanner *bufio.Scanner, stream string, done *chan bool, counter *SafeCounter, gcm cipher.AEAD) {
 	for scanner.Scan() {
-		Write(stream, scanner.Text(), counter)
+		Write(stream, scanner.Text(), counter, gcm)
 	}
 	*done <- true
 }
 
-func Write(stream string, text string, counter *SafeCounter) {
+func Write(stream string, text string, counter *SafeCounter, gcm cipher.AEAD) {
 	//Use a mutex to prevent race conditions updating the line number
 	//https://go.dev/tour/concurrency/9
 	counter.Mutex.Lock()
 
-	fmt.Printf("|%d|%s|%s\n", counter.Value, stream, text)
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+		panic(err)
+	}
+
+	ciphertext := gcm.Seal(nonce, nonce, []byte(text), nil)
+
+	fmt.Printf("|%d|%s|%x\n", counter.Value, stream, ciphertext)
 	counter.Value++
 
 	counter.Mutex.Unlock()
 }
+
+func CreateCipher(workspaceDir string) (cipher.AEAD, error) {
+	// Read the key from the file
+	fileBytes, err := os.ReadFile(path.Join(workspaceDir, "keyfile"))
+	if err != nil {
+		return nil, err
+	}
+
+	//the key is encoded in the file in Base64
+	key := make([]byte, base64.StdEncoding.DecodedLen(len(fileBytes)))
+	length, err := base64.StdEncoding.Decode(key, fileBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	// use the decoded length to slice the array to the correct length (removes padding bytes)
+	key = key[:length]
+
+	// Ensure the key length is valid for AES (16, 24, or 32 bytes for AES-128, AES-192, or AES-256)
+	keyLength := len(key)
+	if keyLength != 16 && keyLength != 24 && keyLength != 32 {
+		return nil, fmt.Errorf("invalid key size: %d bytes. Key must be 16, 24, or 32 bytes", keyLength)
+	}
+	// Create the AES cipher
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	//we specify a known 12 byte nonce size so we can easily retrieve it in Tentacle
+	gcm, err := cipher.NewGCMWithNonceSize(block, 12)
+	if err != nil {
+		return nil, err
+	}
+
+	return gcm, nil
+}
diff --git a/source/Octopus.Manager.Tentacle/Octopus.Manager.Tentacle.csproj b/source/Octopus.Manager.Tentacle/Octopus.Manager.Tentacle.csproj
@@ -132,7 +132,7 @@
     <PackageReference Include="NuGet.Packaging.Core" Version="3.6.0-octopus-58692" />
     <PackageReference Include="NuGet.Packaging.Core.Types" Version="3.6.0-octopus-58692" />
     <PackageReference Include="NuGet.Versioning" Version="3.6.0-octopus-58692" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
+    <PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
     <PackageReference Include="TaskScheduler" Version="2.7.2" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' != 'net8.0-windows'">

diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClusterOneTimeSetUp.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClusterOneTimeSetUp.cs
@@ -36,6 +36,10 @@ public async Task OneTimeSetUp()
             var tag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageTag");
             imageAndTag = $"docker.packages.octopushq.com/octopusdeploy/kubernetes-agent-tentacle:{tag}";
         }
+        else if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag")))
+        {
+            imageAndTag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag");
+        }
         else if(bool.TryParse(Environment.GetEnvironmentVariable("KubernetesAgentTests_UseLatestLocalImage"), out var useLocal) && useLocal)
         {
             //if we should use the latest locally build image, load the tag from docker and load it into kind

diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs
@@ -146,7 +146,7 @@ static string GetChartVersion()
         if (tentacleImageAndTag is null)
             return null;
 
-        var parts = tentacleImageAndTag.Split(":");
+        var parts = tentacleImageAndTag.Split(':',2,StringSplitOptions.TrimEntries);
         var repo = parts[0];
         var tag = parts[1];
 

diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/HelmDownloader.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/HelmDownloader.cs
@@ -9,7 +9,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup.Tooling;
 
 public class HelmDownloader : ToolDownloader
 {
-    const string LatestVersion = "v3.14.3";
+    const string LatestVersion = "v3.16.3";
     public HelmDownloader( ILogger logger)
         : base("helm", logger)
     {

diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/KindDownloader.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/KindDownloader.cs
@@ -6,7 +6,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup.Tooling
 {
     public class KindDownloader : ToolDownloader
     {
-        const string LatestKindVersion = "v0.22.0";
+        const string LatestKindVersion = "v0.25.0";
 
         public KindDownloader(ILogger logger)
             : base("kind", logger)

diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/KubeCtlDownloader.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/KubeCtlDownloader.cs
@@ -4,7 +4,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup.Tooling;
 
 public class KubeCtlDownloader : ToolDownloader
 {
-    public const string LatestKubeCtlVersion = "v1.29.3";
+    public const string LatestKubeCtlVersion = "v1.30.6";
 
     public KubeCtlDownloader(ILogger logger) 
         : base("kubectl", logger)

diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/kind-config.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/kind-config.yaml
@@ -3,6 +3,6 @@ apiVersion: kind.x-k8s.io/v1alpha4
 
 nodes:
   - role: control-plane
-    image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
+    image: kindest/node:v1.30.6@sha256:b6d08db72079ba5ae1f4a88a09025c0a904af3b52387643c285442afb05ab994
   - role: worker
-    image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
+    image: kindest/node:v1.30.6@sha256:b6d08db72079ba5ae1f4a88a09025c0a904af3b52387643c285442afb05ab994
diff --git a/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs b/source/Octopus.Tentacle.Tests/Configuration/ApplicationInstanceSelectorFixture.cs
@@ -8,6 +8,8 @@
 using Octopus.Tentacle.Configuration.Crypto;
 using Octopus.Tentacle.Configuration.Instances;
 using Octopus.Tentacle.Kubernetes;
+using Octopus.Tentacle.Kubernetes.Configuration;
+using Octopus.Tentacle.Kubernetes.Crypto;
 using Octopus.Tentacle.Util;
 
 namespace Octopus.Tentacle.Tests.Configuration

diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs b/source/Octopus.Tentacle.Tests/Kubernetes/KubernetesOrphanedPodCleanerTests.cs
@@ -7,6 +7,7 @@
 using NUnit.Framework;
 using Octopus.Tentacle.Contracts;
 using Octopus.Tentacle.Kubernetes;
+using Octopus.Tentacle.Kubernetes.Crypto;
 using Octopus.Tentacle.Tests.Support;
 using Octopus.Time;
 
@@ -26,6 +27,7 @@ public class KubernetesOrphanedPodCleanerTests
         DateTimeOffset startTime;
         ITentacleScriptLogProvider scriptLogProvider;
         IScriptPodSinceTimeStore scriptPodSinceTimeStore;
+        IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider;
 
         [SetUp]
         public void Setup()
@@ -37,10 +39,11 @@ public void Setup()
             clock = new FixedClock(startTime);
             scriptLogProvider = Substitute.For<ITentacleScriptLogProvider>();
             scriptPodSinceTimeStore = Substitute.For<IScriptPodSinceTimeStore>();
+            scriptPodLogEncryptionKeyProvider = Substitute.For<IScriptPodLogEncryptionKeyProvider>();
             monitor = Substitute.For<IKubernetesPodStatusProvider>();
             scriptTicket = new ScriptTicket(Guid.NewGuid().ToString());
 
-            cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore);
+            cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider);
 
             overCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan + 1.Minutes();
             underCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan - 1.Minutes();
@@ -71,6 +74,7 @@ public async Task OrphanedPodCleanedUpIfOver10MinutesHavePassed()
             await podService.Received().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
             scriptLogProvider.Received().Delete(scriptTicket);
             scriptPodSinceTimeStore.Received().Delete(scriptTicket);
+            scriptPodLogEncryptionKeyProvider.Received().Delete(scriptTicket);
         }
 
         [TestCase(TrackedScriptPodPhase.Succeeded, true)]
@@ -95,12 +99,14 @@ public async Task OrphanedPodOnlyCleanedUpWhenNotRunning(TrackedScriptPodPhase p
                 await podService.Received().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
                 scriptLogProvider.Received().Delete(scriptTicket);
                 scriptPodSinceTimeStore.Received().Delete(scriptTicket);
+                scriptPodLogEncryptionKeyProvider.Received().Delete(scriptTicket);
             }
             else
             {
                 await podService.DidNotReceiveWithAnyArgs().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
                 scriptLogProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
                 scriptPodSinceTimeStore.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
+                scriptPodLogEncryptionKeyProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
             }
 
             TrackedScriptPodState CreateState(TrackedScriptPodPhase phase)
@@ -136,6 +142,7 @@ public async Task OrphanedPodNotCleanedUpIfOnly9MinutesHavePassed()
             //Assert
             await podService.DidNotReceiveWithAnyArgs().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
             scriptLogProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
+            scriptPodLogEncryptionKeyProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
         }
 
         [Test]
@@ -157,6 +164,7 @@ public async Task OrphanedPodNotCleanedUpIfPodCleanupIsDisabled()
             await podService.DidNotReceive().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
             scriptLogProvider.Received().Delete(scriptTicket);
             scriptPodSinceTimeStore.Received().Delete(scriptTicket);
+            scriptPodLogEncryptionKeyProvider.Received().Delete(scriptTicket);
         }
 
         [TestCase(1, false)]
@@ -167,7 +175,7 @@ public async Task EnvironmentVariableDictatesWhenPodsAreConsideredOrphaned(int c
             Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__PODSCONSIDEREDORPHANEDAFTERMINUTES", "2");
 
             // We need to reinitialise the sut after changing the env var value
-            cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore);
+            cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider);
             var pods = new List<ITrackedScriptPod>
             {
                 CreatePod(TrackedScriptPodState.Succeeded(0, startTime))
@@ -184,12 +192,14 @@ public async Task EnvironmentVariableDictatesWhenPodsAreConsideredOrphaned(int c
                 await podService.Received().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
                 scriptLogProvider.Received().Delete(scriptTicket);
                 scriptPodSinceTimeStore.Received().Delete(scriptTicket);
+                scriptPodLogEncryptionKeyProvider.Received().Delete(scriptTicket);
             }
             else
             {
                 await podService.DidNotReceiveWithAnyArgs().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
                 scriptLogProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
                 scriptPodSinceTimeStore.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
+                scriptPodLogEncryptionKeyProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
             }
         }
 

diff --git a/source/Octopus.Tentacle.Tests/Kubernetes/PodLogEncryptionProviderTests.cs b/source/Octopus.Tentacle.Tests/Kubernetes/PodLogEncryptionProviderTests.cs
@@ -0,0 +1,45 @@
+using System.Text;
+using FluentAssertions;
+using NUnit.Framework;
+using Octopus.Tentacle.Kubernetes;
+using Octopus.Tentacle.Kubernetes.Crypto;
+
+namespace Octopus.Tentacle.Tests.Kubernetes
+{
+    [TestFixture]
+    public class PodLogEncryptionProviderTests
+    {
+        IPodLogEncryptionProvider sut;
+
+        public static readonly byte[] Nonce =
+        {
+            12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1
+        };
+
+        [SetUp]
+        public void SetUp()
+        {
+            var key = Encoding.UTF8.GetBytes("qwertyuioplkjhgfdsazxcvbnmqwertd");
+            sut = PodLogEncryptionProvider.Create(key);
+        }
+
+        [TestCase("531f69408fcc09129a42b46b93c7c14fe7c36fec74ac77929b4e1f29b6b0c1e7cfe78055ceee24fbca5f9097501b5cb548f78928b5", "##octopus[stdout-verbose]")]
+        [TestCase("4463ba9b0672e65fdabcd99681ee20e5f003a59ce2c1b1751303e6399d5515ae4a715ed47ce5b7c5727ca66a127e485cd22de2d1dad76d8f704922dae0036d99c4a7e151498043b9d8", "EOS-075CD4F0-8C76-491D-BA76-0879D35E9CFE<<>>0")]
+        public void Decrypt_ShouldProduceCorrectOutput(string encryptedMessage, string expectedDecryptedMessage)
+        {
+            var result = sut.Decrypt(encryptedMessage);
+
+            result.Should().Be(expectedDecryptedMessage);
+        }
+
+        [TestCase("a cool message", "0c0b0a090807060504030201416ce819a896623cee784f92807857b9be36bd6075461789031a3d29aef1")]
+        [TestCase("##octopus[stdout-verbose]", "0c0b0a090807060504030201036fe415b3953224f8504f8783723ca65dca38cd3eab365c4d10d4efad161112e96cd449c6a52348ef")]
+        [TestCase("EOS-075CD4F0-8C76-491D-BA76-0879D35E9CFE<<>>0", "0c0b0a0908070605040302016503d85bf7cd7712cf3f7ac3ca250ae5469169866d80687b51b3cede4af296732090b9a9577055be3ced266ceb9eecd094a74f8df65c95ab33d9f5170c")]
+        public void Encrypt_ShouldProduceCorrectOutput(string plaintextMessage, string expectedEncryptedMessage)
+        {
+            var result = sut.Encrypt(plaintextMessage, Nonce);
+
+            result.Should().Be(expectedEncryptedMessage);
+        }
+    }
+}