diff --git a/juju/client/proxy/kubernetes/proxy.py b/juju/client/proxy/kubernetes/proxy.py
index 0c77ef62..3eb53c5c 100644
--- a/juju/client/proxy/kubernetes/proxy.py
+++ b/juju/client/proxy/kubernetes/proxy.py
@@ -1,12 +1,15 @@
 # Copyright 2023 Canonical Ltd.
 # Licensed under the Apache V2, see LICENCE file for details.
-
+import os
 import tempfile
+import logging
 
 from juju.client.proxy.proxy import Proxy, ProxyNotConnectedError
 from kubernetes import client
 from kubernetes.stream import portforward
 
+log = logging.getLogger('juju.client.connection')
+
 
 class KubernetesProxy(Proxy):
     def __init__(
@@ -33,7 +36,7 @@ def __init__(
             raise ValueError("Invalid port number: {}".format(remote_port))
 
         if ca_cert:
-            self.temp_ca_file = tempfile.NamedTemporaryFile()
+            self.temp_ca_file = tempfile.NamedTemporaryFile(delete=False)
             self.temp_ca_file.write(bytes(ca_cert, 'utf-8'))
             self.temp_ca_file.flush()
             config.ssl_ca_cert = self.temp_ca_file.name
@@ -60,6 +63,10 @@ def connect(self):
 
     def __del__(self):
         self.close()
+        try:
+            os.unlink(self.temp_ca_file.name)
+        except FileNotFoundError:
+            log.debug(f"file {self.temp_ca_file.name} not found")
 
     def close(self):
         try: