From e8964afa799c005f0140a2c6618808d984230c7c Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Wed, 3 Jan 2024 15:37:29 -0500 Subject: [PATCH 01/15] Allows runtime configuration of vault provider retry delay and tries. --- gestalt/vault.py | 55 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index ae8f843..0bbbe02 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -9,11 +9,11 @@ from queue import Queue import os from threading import Thread -from retry import retry +from retry.api import retry_call class Vault(Provider): - @retry((RuntimeError, Timeout), delay=2, tries=5) # type: ignore + def __init__(self, cert: Optional[Tuple[str, str]] = None, role: Optional[str] = None, @@ -21,9 +21,26 @@ def __init__(self, url: Optional[str] = os.environ.get("VAULT_ADDR"), token: Optional[str] = os.environ.get("VAULT_TOKEN"), verify: Optional[bool] = True, - scheme: str = "ref+vault://") -> None: + scheme: str = "ref+vault://", + delay=2, + tries=5) -> None: + + self.delay = delay + self.tries = tries + + retry_call(f=Vault.__do_init, fargs=[self, cert, role, jwt, url, token, verify], + exceptions=(RuntimeError, Timeout), delay=self.delay, + tries=self.tries) + + def __do_init(self, + cert: Optional[Tuple[str, str]], + role: Optional[str], + jwt: Optional[str], + url: Optional[str], + token: Optional[str], + verify: Optional[bool], + scheme: str) -> None: """Initialized vault client and authenticates vault - Args: client_config (HVAC_ClientConfig): initializes vault. URL can be set in VAULT_ADDR environment variable, token can be set to VAULT_TOKEN environment variable. @@ -42,7 +59,7 @@ def __init__(self, verify=verify) self._secret_expiry_times: Dict[str, datetime] = dict() self._secret_values: Dict[str, Union[str, int, float, bool, - List[Any]]] = dict() + List[Any]]] = dict() try: self.vault_client.is_authenticated() @@ -72,11 +89,11 @@ def __init__(self, name='dynamic-token-renew', target=self.worker, daemon=True, - args=(self.dynamic_token_queue, )) # noqa: F841 + args=(self.dynamic_token_queue,)) # noqa: F841 kubernetes_ttl_renew = Thread(name="kubes-token-renew", target=self.worker, daemon=True, - args=(self.kubes_token_queue, )) + args=(self.kubes_token_queue,)) kubernetes_ttl_renew.start() def stop(self) -> None: @@ -85,13 +102,23 @@ def stop(self) -> None: def __del__(self) -> None: self.stop() - @retry((RuntimeError, Timeout), delay=3, tries=3) # type: ignore - def get( - self, - key: str, - path: str, - filter: str, - sep: Optional[str] = "." + def get( + self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." + ) -> Union[str, int, float, bool, List[Any]]: + + return retry_call(f=Vault.__do_get, fargs=[self, key, path, filter, sep], exceptions=(RuntimeError, Timeout), + delay=self.delay, tries=self.tries) + + def __do_get( + self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." ) -> Union[str, int, float, bool, List[Any]]: """Gets secret from vault Args: From 40dc82a342dbd972d8f7b3bd1a998bc20daa956a Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 4 Jan 2024 09:49:59 -0500 Subject: [PATCH 02/15] Fixes formatting. --- gestalt/vault.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 0bbbe02..b88ed87 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -1,16 +1,18 @@ +import os from datetime import datetime, timedelta +from queue import Queue +from threading import Thread from time import sleep -from gestalt.provider import Provider +from typing import Any, Dict, List, Optional, Tuple, Union + +import hvac # type: ignore import requests -from requests.exceptions import Timeout from jsonpath_ng import parse # type: ignore -from typing import Optional, Tuple, Any, Dict, Union, List -import hvac # type: ignore -from queue import Queue -import os -from threading import Thread +from requests.exceptions import Timeout from retry.api import retry_call +from gestalt.provider import Provider + class Vault(Provider): @@ -28,7 +30,7 @@ def __init__(self, self.delay = delay self.tries = tries - retry_call(f=Vault.__do_init, fargs=[self, cert, role, jwt, url, token, verify], + retry_call(f=Vault.__do_init, fargs=[self, cert, role, jwt, url, token, verify, scheme], exceptions=(RuntimeError, Timeout), delay=self.delay, tries=self.tries) @@ -102,13 +104,13 @@ def stop(self) -> None: def __del__(self) -> None: self.stop() - def get( - self, - key: str, - path: str, - filter: str, - sep: Optional[str] = "." - ) -> Union[str, int, float, bool, List[Any]]: + def get( + self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." + ) -> Union[str, int, float, bool, List[Any]]: return retry_call(f=Vault.__do_get, fargs=[self, key, path, filter, sep], exceptions=(RuntimeError, Timeout), delay=self.delay, tries=self.tries) From 12036ed50808003bb45a112996bc6cce4b4a02fb Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 4 Jan 2024 09:52:49 -0500 Subject: [PATCH 03/15] Adds missing type info. --- gestalt/vault.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index b88ed87..44adc6e 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -24,8 +24,8 @@ def __init__(self, token: Optional[str] = os.environ.get("VAULT_TOKEN"), verify: Optional[bool] = True, scheme: str = "ref+vault://", - delay=2, - tries=5) -> None: + delay: int = 2, + tries: int = 5) -> None: self.delay = delay self.tries = tries From abe7ce53838875a99eec150dec9a19be94e4dd52 Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 4 Jan 2024 09:58:03 -0500 Subject: [PATCH 04/15] Black formatted. --- gestalt/vault.py | 152 ++++++++++++++++++++++++----------------------- 1 file changed, 79 insertions(+), 73 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 44adc6e..6c5b8f0 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -15,33 +15,39 @@ class Vault(Provider): - - def __init__(self, - cert: Optional[Tuple[str, str]] = None, - role: Optional[str] = None, - jwt: Optional[str] = None, - url: Optional[str] = os.environ.get("VAULT_ADDR"), - token: Optional[str] = os.environ.get("VAULT_TOKEN"), - verify: Optional[bool] = True, - scheme: str = "ref+vault://", - delay: int = 2, - tries: int = 5) -> None: - + def __init__( + self, + cert: Optional[Tuple[str, str]] = None, + role: Optional[str] = None, + jwt: Optional[str] = None, + url: Optional[str] = os.environ.get("VAULT_ADDR"), + token: Optional[str] = os.environ.get("VAULT_TOKEN"), + verify: Optional[bool] = True, + scheme: str = "ref+vault://", + delay: int = 2, + tries: int = 5, + ) -> None: self.delay = delay self.tries = tries - retry_call(f=Vault.__do_init, fargs=[self, cert, role, jwt, url, token, verify, scheme], - exceptions=(RuntimeError, Timeout), delay=self.delay, - tries=self.tries) - - def __do_init(self, - cert: Optional[Tuple[str, str]], - role: Optional[str], - jwt: Optional[str], - url: Optional[str], - token: Optional[str], - verify: Optional[bool], - scheme: str) -> None: + retry_call( + f=Vault.__do_init, + fargs=[self, cert, role, jwt, url, token, verify, scheme], + exceptions=(RuntimeError, Timeout), + delay=self.delay, + tries=self.tries, + ) + + def __do_init( + self, + cert: Optional[Tuple[str, str]], + role: Optional[str], + jwt: Optional[str], + url: Optional[str], + token: Optional[str], + verify: Optional[bool], + scheme: str, + ) -> None: """Initialized vault client and authenticates vault Args: client_config (HVAC_ClientConfig): initializes vault. URL can be set in VAULT_ADDR @@ -55,13 +61,9 @@ def __do_init(self, self.dynamic_token_queue: Queue[Tuple[str, str, str]] = Queue() self.kubes_token_queue: Queue[Tuple[str, str, str]] = Queue() - self.vault_client = hvac.Client(url=url, - token=token, - cert=cert, - verify=verify) + self.vault_client = hvac.Client(url=url, token=token, cert=cert, verify=verify) self._secret_expiry_times: Dict[str, datetime] = dict() - self._secret_values: Dict[str, Union[str, int, float, bool, - List[Any]]] = dict() + self._secret_values: Dict[str, Union[str, int, float, bool, List[Any]]] = dict() try: self.vault_client.is_authenticated() @@ -72,30 +74,36 @@ def __do_init(self, if role and jwt: try: - hvac.api.auth_methods.Kubernetes( - self.vault_client.adapter).login(role=role, jwt=jwt) + hvac.api.auth_methods.Kubernetes(self.vault_client.adapter).login( + role=role, jwt=jwt + ) token = self.vault_client.auth.token.lookup_self() if token is not None: kubes_token = ( "kubernetes", - token['data']['id'], # type: ignore - token['data']['ttl']) # type: ignore + token["data"]["id"], # type: ignore + token["data"]["ttl"], + ) # type: ignore self.kubes_token_queue.put(kubes_token) except hvac.exceptions.InvalidPath: raise RuntimeError( - "Gestalt Error: Kubernetes auth couldn't be performed") + "Gestalt Error: Kubernetes auth couldn't be performed" + ) except requests.exceptions.ConnectionError: raise RuntimeError("Gestalt Error: Couldn't connect to Vault") dynamic_ttl_renew = Thread( - name='dynamic-token-renew', + name="dynamic-token-renew", target=self.worker, daemon=True, - args=(self.dynamic_token_queue,)) # noqa: F841 - kubernetes_ttl_renew = Thread(name="kubes-token-renew", - target=self.worker, - daemon=True, - args=(self.kubes_token_queue,)) + args=(self.dynamic_token_queue,), + ) # noqa: F841 + kubernetes_ttl_renew = Thread( + name="kubes-token-renew", + target=self.worker, + daemon=True, + args=(self.kubes_token_queue,), + ) kubernetes_ttl_renew.start() def stop(self) -> None: @@ -105,22 +113,18 @@ def __del__(self) -> None: self.stop() def get( - self, - key: str, - path: str, - filter: str, - sep: Optional[str] = "." + self, key: str, path: str, filter: str, sep: Optional[str] = "." ) -> Union[str, int, float, bool, List[Any]]: - - return retry_call(f=Vault.__do_get, fargs=[self, key, path, filter, sep], exceptions=(RuntimeError, Timeout), - delay=self.delay, tries=self.tries) + return retry_call( + f=Vault.__do_get, + fargs=[self, key, path, filter, sep], + exceptions=(RuntimeError, Timeout), + delay=self.delay, + tries=self.tries, + ) def __do_get( - self, - key: str, - path: str, - filter: str, - sep: Optional[str] = "." + self, key: str, path: str, filter: str, sep: Optional[str] = "." ) -> Union[str, int, float, bool, List[Any]]: """Gets secret from vault Args: @@ -136,25 +140,27 @@ def __do_get( return self._secret_values[key] # if the secret can expire but hasn't expired yet - if key in self._secret_expiry_times and not self._is_secret_expired( - key): + if key in self._secret_expiry_times and not self._is_secret_expired(key): return self._secret_values[key] try: response = self.vault_client.read(path) if response is None: raise RuntimeError("Gestalt Error: No secrets found") - if response['lease_id']: - dynamic_token = ("dynamic", response['lease_id'], - response['lease_duration']) + if response["lease_id"]: + dynamic_token = ( + "dynamic", + response["lease_id"], + response["lease_duration"], + ) self.dynamic_token_queue.put_nowait(dynamic_token) requested_data = response["data"].get("data", response["data"]) except hvac.exceptions.InvalidPath: raise RuntimeError( - "Gestalt Error: The secret path or mount is set incorrectly") + "Gestalt Error: The secret path or mount is set incorrectly" + ) except requests.exceptions.ConnectionError: - raise RuntimeError( - "Gestalt Error: Gestalt couldn't connect to Vault") + raise RuntimeError("Gestalt Error: Gestalt couldn't connect to Vault") except Exception as err: raise RuntimeError(f"Gestalt Error: {err}") if filter is None: @@ -180,12 +186,13 @@ def _is_secret_expired(self, key: str) -> bool: is_expired = now >= secret_expires_dt return is_expired - def _set_secrets_ttl(self, requested_data: Dict[str, Any], - key: str) -> None: - last_vault_rotation_str = requested_data["last_vault_rotation"].split( - ".")[0] # to the nearest second - last_vault_rotation_dt = datetime.strptime(last_vault_rotation_str, - '%Y-%m-%dT%H:%M:%S') + def _set_secrets_ttl(self, requested_data: Dict[str, Any], key: str) -> None: + last_vault_rotation_str = requested_data["last_vault_rotation"].split(".")[ + 0 + ] # to the nearest second + last_vault_rotation_dt = datetime.strptime( + last_vault_rotation_str, "%Y-%m-%dT%H:%M:%S" + ) ttl = requested_data["ttl"] secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) self._secret_expiry_times[key] = secret_expires_dt @@ -198,8 +205,7 @@ def worker(self, token_queue: Queue) -> None: # type: ignore try: while self._run_worker: if not token_queue.empty(): - token_type, token_id, token_duration = token = token_queue.get( - ) + token_type, token_id, token_duration = token = token_queue.get() if token_type == "kubernetes": self.vault_client.auth.token.renew(token_id) print("kubernetes token for the app has been renewed") @@ -211,10 +217,10 @@ def worker(self, token_queue: Queue) -> None: # type: ignore sleep((token_duration / 3) * 2) except hvac.exceptions.InvalidPath: raise RuntimeError( - "Gestalt Error: The lease path or mount is set incorrectly") + "Gestalt Error: The lease path or mount is set incorrectly" + ) except requests.exceptions.ConnectionError: - raise RuntimeError( - "Gestalt Error: Gestalt couldn't connect to Vault") + raise RuntimeError("Gestalt Error: Gestalt couldn't connect to Vault") except Exception as err: raise RuntimeError(f"Gestalt Error: {err}") From 74d200db4f83d638e6ec37da922e04f4f9442330 Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 4 Jan 2024 10:01:53 -0500 Subject: [PATCH 05/15] Fixes type errors. --- gestalt/vault.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 6c5b8f0..e579e0b 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -82,8 +82,8 @@ def __do_init( kubes_token = ( "kubernetes", token["data"]["id"], # type: ignore - token["data"]["ttl"], - ) # type: ignore + token["data"]["ttl"], # type: ignore + ) self.kubes_token_queue.put(kubes_token) except hvac.exceptions.InvalidPath: raise RuntimeError( From aa7790944525f258b0377ebeede42a294765f489 Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 4 Jan 2024 10:06:45 -0500 Subject: [PATCH 06/15] yapf formatting. --- gestalt/vault.py | 70 +++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index e579e0b..09e6f82 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -15,6 +15,7 @@ class Vault(Provider): + def __init__( self, cert: Optional[Tuple[str, str]] = None, @@ -61,9 +62,13 @@ def __do_init( self.dynamic_token_queue: Queue[Tuple[str, str, str]] = Queue() self.kubes_token_queue: Queue[Tuple[str, str, str]] = Queue() - self.vault_client = hvac.Client(url=url, token=token, cert=cert, verify=verify) + self.vault_client = hvac.Client(url=url, + token=token, + cert=cert, + verify=verify) self._secret_expiry_times: Dict[str, datetime] = dict() - self._secret_values: Dict[str, Union[str, int, float, bool, List[Any]]] = dict() + self._secret_values: Dict[str, Union[str, int, float, bool, + List[Any]]] = dict() try: self.vault_client.is_authenticated() @@ -74,9 +79,8 @@ def __do_init( if role and jwt: try: - hvac.api.auth_methods.Kubernetes(self.vault_client.adapter).login( - role=role, jwt=jwt - ) + hvac.api.auth_methods.Kubernetes( + self.vault_client.adapter).login(role=role, jwt=jwt) token = self.vault_client.auth.token.lookup_self() if token is not None: kubes_token = ( @@ -87,8 +91,7 @@ def __do_init( self.kubes_token_queue.put(kubes_token) except hvac.exceptions.InvalidPath: raise RuntimeError( - "Gestalt Error: Kubernetes auth couldn't be performed" - ) + "Gestalt Error: Kubernetes auth couldn't be performed") except requests.exceptions.ConnectionError: raise RuntimeError("Gestalt Error: Couldn't connect to Vault") @@ -96,13 +99,13 @@ def __do_init( name="dynamic-token-renew", target=self.worker, daemon=True, - args=(self.dynamic_token_queue,), + args=(self.dynamic_token_queue, ), ) # noqa: F841 kubernetes_ttl_renew = Thread( name="kubes-token-renew", target=self.worker, daemon=True, - args=(self.kubes_token_queue,), + args=(self.kubes_token_queue, ), ) kubernetes_ttl_renew.start() @@ -112,9 +115,12 @@ def stop(self) -> None: def __del__(self) -> None: self.stop() - def get( - self, key: str, path: str, filter: str, sep: Optional[str] = "." - ) -> Union[str, int, float, bool, List[Any]]: + def get(self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." + ) -> Union[str, int, float, bool, List[Any]]: return retry_call( f=Vault.__do_get, fargs=[self, key, path, filter, sep], @@ -123,9 +129,12 @@ def get( tries=self.tries, ) - def __do_get( - self, key: str, path: str, filter: str, sep: Optional[str] = "." - ) -> Union[str, int, float, bool, List[Any]]: + def __do_get(self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." + ) -> Union[str, int, float, bool, List[Any]]: """Gets secret from vault Args: key (str): key to get secret from @@ -140,7 +149,8 @@ def __do_get( return self._secret_values[key] # if the secret can expire but hasn't expired yet - if key in self._secret_expiry_times and not self._is_secret_expired(key): + if key in self._secret_expiry_times and not self._is_secret_expired( + key): return self._secret_values[key] try: @@ -157,10 +167,10 @@ def __do_get( requested_data = response["data"].get("data", response["data"]) except hvac.exceptions.InvalidPath: raise RuntimeError( - "Gestalt Error: The secret path or mount is set incorrectly" - ) + "Gestalt Error: The secret path or mount is set incorrectly") except requests.exceptions.ConnectionError: - raise RuntimeError("Gestalt Error: Gestalt couldn't connect to Vault") + raise RuntimeError( + "Gestalt Error: Gestalt couldn't connect to Vault") except Exception as err: raise RuntimeError(f"Gestalt Error: {err}") if filter is None: @@ -186,13 +196,12 @@ def _is_secret_expired(self, key: str) -> bool: is_expired = now >= secret_expires_dt return is_expired - def _set_secrets_ttl(self, requested_data: Dict[str, Any], key: str) -> None: - last_vault_rotation_str = requested_data["last_vault_rotation"].split(".")[ - 0 - ] # to the nearest second - last_vault_rotation_dt = datetime.strptime( - last_vault_rotation_str, "%Y-%m-%dT%H:%M:%S" - ) + def _set_secrets_ttl(self, requested_data: Dict[str, Any], + key: str) -> None: + last_vault_rotation_str = requested_data["last_vault_rotation"].split( + ".")[0] # to the nearest second + last_vault_rotation_dt = datetime.strptime(last_vault_rotation_str, + "%Y-%m-%dT%H:%M:%S") ttl = requested_data["ttl"] secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) self._secret_expiry_times[key] = secret_expires_dt @@ -205,7 +214,8 @@ def worker(self, token_queue: Queue) -> None: # type: ignore try: while self._run_worker: if not token_queue.empty(): - token_type, token_id, token_duration = token = token_queue.get() + token_type, token_id, token_duration = token = token_queue.get( + ) if token_type == "kubernetes": self.vault_client.auth.token.renew(token_id) print("kubernetes token for the app has been renewed") @@ -217,10 +227,10 @@ def worker(self, token_queue: Queue) -> None: # type: ignore sleep((token_duration / 3) * 2) except hvac.exceptions.InvalidPath: raise RuntimeError( - "Gestalt Error: The lease path or mount is set incorrectly" - ) + "Gestalt Error: The lease path or mount is set incorrectly") except requests.exceptions.ConnectionError: - raise RuntimeError("Gestalt Error: Gestalt couldn't connect to Vault") + raise RuntimeError( + "Gestalt Error: Gestalt couldn't connect to Vault") except Exception as err: raise RuntimeError(f"Gestalt Error: {err}") From 4d3cb9fe74ae2b64fa38f5a30c7b3e5a0c088bfa Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 4 Jan 2024 10:15:02 -0500 Subject: [PATCH 07/15] Adds comment for test. --- gestalt/vault.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gestalt/vault.py b/gestalt/vault.py index 09e6f82..2f41f85 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -39,6 +39,7 @@ def __init__( tries=self.tries, ) + # __init__ impl for retry_call def __do_init( self, cert: Optional[Tuple[str, str]], From 0a50e031df58b6fd7e6602f71b15ee84e2a473fc Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 4 Jan 2024 10:28:41 -0500 Subject: [PATCH 08/15] Re-applies yapf formatting with correct yapf version. --- gestalt/vault.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 2f41f85..cd15696 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -15,7 +15,6 @@ class Vault(Provider): - def __init__( self, cert: Optional[Tuple[str, str]] = None, @@ -41,14 +40,14 @@ def __init__( # __init__ impl for retry_call def __do_init( - self, - cert: Optional[Tuple[str, str]], - role: Optional[str], - jwt: Optional[str], - url: Optional[str], - token: Optional[str], - verify: Optional[bool], - scheme: str, + self, + cert: Optional[Tuple[str, str]], + role: Optional[str], + jwt: Optional[str], + url: Optional[str], + token: Optional[str], + verify: Optional[bool], + scheme: str, ) -> None: """Initialized vault client and authenticates vault Args: @@ -116,12 +115,13 @@ def stop(self) -> None: def __del__(self) -> None: self.stop() - def get(self, - key: str, - path: str, - filter: str, - sep: Optional[str] = "." - ) -> Union[str, int, float, bool, List[Any]]: + def get( + self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." + ) -> Union[str, int, float, bool, List[Any]]: return retry_call( f=Vault.__do_get, fargs=[self, key, path, filter, sep], @@ -130,12 +130,13 @@ def get(self, tries=self.tries, ) - def __do_get(self, - key: str, - path: str, - filter: str, - sep: Optional[str] = "." - ) -> Union[str, int, float, bool, List[Any]]: + def __do_get( + self, + key: str, + path: str, + filter: str, + sep: Optional[str] = "." + ) -> Union[str, int, float, bool, List[Any]]: """Gets secret from vault Args: key (str): key to get secret from From c7a125904de57bc8470a8a9a67d263bc7db50f86 Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 4 Jan 2024 10:29:46 -0500 Subject: [PATCH 09/15] Fixes formatting. --- gestalt/vault.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gestalt/vault.py b/gestalt/vault.py index cd15696..331ba12 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -130,6 +130,7 @@ def get( tries=self.tries, ) + # get impl for retry_call def __do_get( self, key: str, From cd9ba6137ffbddd57519d98f3bda78b16181b0ce Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Mon, 8 Jan 2024 13:26:40 -0500 Subject: [PATCH 10/15] Removes private methods for init and get. Retries only on vault_client calls. Timeout set to 60 seconds, retries to 5. --- gestalt/vault.py | 66 ++++++++++++++++++------------------------------ 1 file changed, 24 insertions(+), 42 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index 331ba12..e1a914b 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -24,30 +24,8 @@ def __init__( token: Optional[str] = os.environ.get("VAULT_TOKEN"), verify: Optional[bool] = True, scheme: str = "ref+vault://", - delay: int = 2, + delay: int = 60, tries: int = 5, - ) -> None: - self.delay = delay - self.tries = tries - - retry_call( - f=Vault.__do_init, - fargs=[self, cert, role, jwt, url, token, verify, scheme], - exceptions=(RuntimeError, Timeout), - delay=self.delay, - tries=self.tries, - ) - - # __init__ impl for retry_call - def __do_init( - self, - cert: Optional[Tuple[str, str]], - role: Optional[str], - jwt: Optional[str], - url: Optional[str], - token: Optional[str], - verify: Optional[bool], - scheme: str, ) -> None: """Initialized vault client and authenticates vault Args: @@ -70,8 +48,16 @@ def __do_init( self._secret_values: Dict[str, Union[str, int, float, bool, List[Any]]] = dict() + self.delay = delay + self.tries = tries + try: - self.vault_client.is_authenticated() + retry_call( + self.vault_client.is_authenticated, + exceptions=(RuntimeError, Timeout), + delay=self.delay, + tries=self.tries, + ) except requests.exceptions.MissingSchema: raise RuntimeError( "Gestalt Error: Unable to connect to vault with the given configuration" @@ -81,7 +67,13 @@ def __do_init( try: hvac.api.auth_methods.Kubernetes( self.vault_client.adapter).login(role=role, jwt=jwt) - token = self.vault_client.auth.token.lookup_self() + token = retry_call( + self.vault_client.auth.token.lookup_self, + exceptions=(RuntimeError, Timeout), + delay=self.delay, + tries=self.tries, + ) + if token is not None: kubes_token = ( "kubernetes", @@ -121,22 +113,6 @@ def get( path: str, filter: str, sep: Optional[str] = "." - ) -> Union[str, int, float, bool, List[Any]]: - return retry_call( - f=Vault.__do_get, - fargs=[self, key, path, filter, sep], - exceptions=(RuntimeError, Timeout), - delay=self.delay, - tries=self.tries, - ) - - # get impl for retry_call - def __do_get( - self, - key: str, - path: str, - filter: str, - sep: Optional[str] = "." ) -> Union[str, int, float, bool, List[Any]]: """Gets secret from vault Args: @@ -157,7 +133,13 @@ def __do_get( return self._secret_values[key] try: - response = self.vault_client.read(path) + response = retry_call( + self.vault_client.read, + fargs=[path], + exceptions=(RuntimeError, Timeout), + delay=self.delay, + tries=self.tries, + ) if response is None: raise RuntimeError("Gestalt Error: No secrets found") if response["lease_id"]: From 8e6175ac0967fbcc7058a5353f9cbf3251316f2c Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Mon, 8 Jan 2024 15:48:54 -0500 Subject: [PATCH 11/15] Reverts black formatting single quotes as double. --- gestalt/vault.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/gestalt/vault.py b/gestalt/vault.py index e1a914b..6c0a76b 100644 --- a/gestalt/vault.py +++ b/gestalt/vault.py @@ -28,6 +28,7 @@ def __init__( tries: int = 5, ) -> None: """Initialized vault client and authenticates vault + Args: client_config (HVAC_ClientConfig): initializes vault. URL can be set in VAULT_ADDR environment variable, token can be set to VAULT_TOKEN environment variable. @@ -77,9 +78,8 @@ def __init__( if token is not None: kubes_token = ( "kubernetes", - token["data"]["id"], # type: ignore - token["data"]["ttl"], # type: ignore - ) + token['data']['id'], # type: ignore + token['data']['ttl']) # type: ignore self.kubes_token_queue.put(kubes_token) except hvac.exceptions.InvalidPath: raise RuntimeError( @@ -88,17 +88,14 @@ def __init__( raise RuntimeError("Gestalt Error: Couldn't connect to Vault") dynamic_ttl_renew = Thread( - name="dynamic-token-renew", - target=self.worker, - daemon=True, - args=(self.dynamic_token_queue, ), - ) # noqa: F841 - kubernetes_ttl_renew = Thread( - name="kubes-token-renew", + name='dynamic-token-renew', target=self.worker, daemon=True, - args=(self.kubes_token_queue, ), - ) + args=(self.dynamic_token_queue, )) # noqa: F841 + kubernetes_ttl_renew = Thread(name="kubes-token-renew", + target=self.worker, + daemon=True, + args=(self.kubes_token_queue, )) kubernetes_ttl_renew.start() def stop(self) -> None: @@ -142,12 +139,9 @@ def get( ) if response is None: raise RuntimeError("Gestalt Error: No secrets found") - if response["lease_id"]: - dynamic_token = ( - "dynamic", - response["lease_id"], - response["lease_duration"], - ) + if response['lease_id']: + dynamic_token = ("dynamic", response['lease_id'], + response['lease_duration']) self.dynamic_token_queue.put_nowait(dynamic_token) requested_data = response["data"].get("data", response["data"]) except hvac.exceptions.InvalidPath: @@ -186,7 +180,7 @@ def _set_secrets_ttl(self, requested_data: Dict[str, Any], last_vault_rotation_str = requested_data["last_vault_rotation"].split( ".")[0] # to the nearest second last_vault_rotation_dt = datetime.strptime(last_vault_rotation_str, - "%Y-%m-%dT%H:%M:%S") + '%Y-%m-%dT%H:%M:%S') ttl = requested_data["ttl"] secret_expires_dt = last_vault_rotation_dt + timedelta(seconds=ttl) self._secret_expiry_times[key] = secret_expires_dt From f6cbd9896d6d67843224fea58d37e06b41f429d8 Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 11 Jan 2024 11:22:59 -0500 Subject: [PATCH 12/15] Bumps version to 3.3.6. Adds CHANGELOG.md. --- CHANGELOG.md | 18 ++++++++++++++++++ setup.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..978f648 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) + +## Unreleased + +## [3.3.6] - 2023-06-11 + +### Changes +- Removed `retry` decorator usage in `vault.py` +- Invokes `vault_client` calls through `retry_call` instead +- Adds`delay` and `tries` to `Vault` constructor for runtime configuration of `retry_call` +- Sets `delay` default to 60 seconds and `tries` to 5 + + + diff --git a/setup.py b/setup.py index d84dd0f..e30325d 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ def readme(): reqs_list = list(map(lambda x: x.rstrip(), reqs)) setup(name='gestalt-cfg', - version='3.3.5', + version='3.3.6', description='A sensible configuration library for Python', long_description=readme(), long_description_content_type="text/markdown", From 629edd557da6bb3fb3debfa30d77d45d91a68471 Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 11 Jan 2024 11:24:43 -0500 Subject: [PATCH 13/15] Re-arranged changelog. --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 978f648..c103dcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [3.3.6] - 2023-06-11 -### Changes +### Added +- `delay` and `tries` to `Vault` constructor for runtime configuration of `retry_call` +- `delay` default to 60 seconds and `tries` to 5 + +### Changed - Removed `retry` decorator usage in `vault.py` - Invokes `vault_client` calls through `retry_call` instead -- Adds`delay` and `tries` to `Vault` constructor for runtime configuration of `retry_call` -- Sets `delay` default to 60 seconds and `tries` to 5 + From 20998673d5a55c758c4be79dc9f8e98c3250e7a4 Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 11 Jan 2024 11:27:09 -0500 Subject: [PATCH 14/15] Adds Released section for backfill. Minor formatting changes. --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c103dcc..66bcc30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,11 @@ # Changelog - +*** All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## Unreleased - +*** ## [3.3.6] - 2023-06-11 ### Added @@ -17,5 +17,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Invokes `vault_client` calls through `retry_call` instead +## Released +*** +# TODO From cd435bde12a73043222c750ac8fb3a471105262b Mon Sep 17 00:00:00 2001 From: Samson Sebastian Date: Thu, 11 Jan 2024 11:28:39 -0500 Subject: [PATCH 15/15] Updates keepchangelog version. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66bcc30..8824684 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ *** All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ## Unreleased ***