diff --git a/tests/integration/flows/test_sap_rfc_to_adls.py b/tests/integration/flows/test_sap_rfc_to_adls.py index 5503b4684..dc07d6160 100644 --- a/tests/integration/flows/test_sap_rfc_to_adls.py +++ b/tests/integration/flows/test_sap_rfc_to_adls.py @@ -19,7 +19,7 @@ def test_sap_rfc_to_adls_query(): name="test flow", query="SELECT MATNR, MATKL FROM MARA WHERE LAEDA LIKE '2022%' LIMIT 5", func="BBP_RFC_READ_TABLE", - sap_credentials=SAP_TEST_CREDS, + credentials=SAP_TEST_CREDS, local_file_path=FILE_NAME, adls_path=ADLS_PATH, overwrite=True, @@ -37,7 +37,7 @@ def test_sap_rfc_to_adls_validation_fail(): name="test flow", query="SELECT MATNR, MATKL FROM MARA WHERE LAEDA LIKE '2022%' LIMIT 5", func="BBP_RFC_READ_TABLE", - sap_credentials=SAP_TEST_CREDS, + credentials=SAP_TEST_CREDS, local_file_path=FILE_NAME, adls_path=ADLS_PATH, overwrite=True, @@ -54,7 +54,7 @@ def test_sap_rfc_to_adls_validation_success(): name="test flow", query="SELECT MATNR, MATKL FROM MARA WHERE LAEDA LIKE '2022%' LIMIT 5", func="BBP_RFC_READ_TABLE", - sap_credentials=SAP_TEST_CREDS, + credentials=SAP_TEST_CREDS, local_file_path=FILE_NAME, adls_path=ADLS_PATH, overwrite=True, diff --git a/tests/integration/test_sap_rfc.py b/tests/integration/test_sap_rfc.py index 34d5106ed..bfdb47bbb 100644 --- a/tests/integration/test_sap_rfc.py +++ b/tests/integration/test_sap_rfc.py @@ -210,7 +210,7 @@ def test_default_credentials_warning_SAPRFCV2(caplog): def test_credentials_dictionary_wrong_key_warning_SAPRFC(caplog): _ = SAPRFC( - sap_credentials={ + credentials={ "sysnr_test": "test", "user": "test", "passwd": "test", @@ -218,7 +218,7 @@ def test_credentials_dictionary_wrong_key_warning_SAPRFC(caplog): } ) assert ( - f"Required key 'sysnr' not found in your 'sap_credentials' dictionary!" + f"Required key 'sysnr' not found in your 'credentials' dictionary!" in caplog.text ) assert ( @@ -229,7 +229,7 @@ def test_credentials_dictionary_wrong_key_warning_SAPRFC(caplog): def test_credentials_dictionary_wrong_key_warning_SAPRFCV2(caplog): _ = SAPRFCV2( - sap_credentials={ + credentials={ "sysnr_test": "test", "user": "test", "passwd": "test", @@ -237,7 +237,7 @@ def test_credentials_dictionary_wrong_key_warning_SAPRFCV2(caplog): } ) assert ( - f"Required key 'sysnr' not found in your 'sap_credentials' dictionary!" + f"Required key 'sysnr' not found in your 'credentials' dictionary!" in caplog.text ) assert ( diff --git a/viadot/flows/sap_rfc_to_adls.py b/viadot/flows/sap_rfc_to_adls.py index 5a380bbaa..a72716b0f 100644 --- a/viadot/flows/sap_rfc_to_adls.py +++ b/viadot/flows/sap_rfc_to_adls.py @@ -16,7 +16,7 @@ def __init__( func: str = "RFC_READ_TABLE", rfc_total_col_width_character_limit: int = 400, rfc_unique_id: List[str] = None, - sap_credentials: dict = None, + credentials: dict = None, sap_credentials_key: str = "SAP", env: str = "DEV", output_file_extension: str = ".parquet", @@ -68,7 +68,7 @@ def __init__( rfc_unique_id=["VBELN", "LPRIO"], ... ) - sap_credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. + credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV" output_file_extension (str, optional): Output file extension - to allow selection of .csv for data which is not easy to handle with parquet. Defaults to ".parquet". @@ -94,7 +94,7 @@ def __init__( self.func = func self.rfc_total_col_width_character_limit = rfc_total_col_width_character_limit self.rfc_unique_id = rfc_unique_id - self.sap_credentials = sap_credentials + self.credentials = credentials self.sap_credentials_key = sap_credentials_key self.env = env self.output_file_extension = output_file_extension @@ -126,7 +126,7 @@ def gen_flow(self) -> Flow: rfc_total_col_width_character_limit=self.rfc_total_col_width_character_limit, rfc_unique_id=self.rfc_unique_id, alternative_version=self.alternative_version, - sap_credentials=self.sap_credentials, + credentials=self.credentials, sap_credentials_key=self.sap_credentials_key, env=self.env, flow=self, diff --git a/viadot/sources/sap_rfc.py b/viadot/sources/sap_rfc.py index 94d1b1f07..99524a0d0 100644 --- a/viadot/sources/sap_rfc.py +++ b/viadot/sources/sap_rfc.py @@ -237,7 +237,7 @@ def __init__( sep: str = None, func: str = "RFC_READ_TABLE", rfc_total_col_width_character_limit: int = 400, - sap_credentials: dict = None, + credentials: dict = None, sap_credentials_key: str = "SAP", env: str = "DEV", *args, @@ -253,7 +253,7 @@ def __init__( in case of too many columns for RFC function. According to SAP documentation, the limit is 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. - sap_credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. + credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". @@ -265,27 +265,27 @@ def __init__( self.sap_credentials_key = sap_credentials_key self.env = env - if isinstance(sap_credentials, dict): - credentials_keys = list(sap_credentials.keys()) + if isinstance(credentials, dict): + credentials_keys = list(credentials.keys()) required_credentials_params = ["sysnr", "user", "passwd", "ashost"] for key in required_credentials_params: if key not in credentials_keys: logger.warning( - f"Required key '{key}' not found in your 'sap_credentials' dictionary!" + f"Required key '{key}' not found in your 'credentials' dictionary!" ) - sap_credentials = None + credentials = None - if sap_credentials is None: + if credentials is None: logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in env parameter." ) try: - sap_credentials = local_config.get(sap_credentials_key).get(env) + credentials = local_config.get(sap_credentials_key).get(env) except AttributeError: raise CredentialError( f"sap_credentials_key: {sap_credentials_key} is not stored neither in KeyVault or local config!" ) - if sap_credentials is None: + if credentials is None: raise CredentialError(f"Missing {env} credentials!") super().__init__( @@ -293,7 +293,7 @@ def __init__( **kwargs, ) - self.sap_credentials = sap_credentials + self.credentials = credentials self.sep = sep self.client_side_filters = None self.func = func @@ -303,7 +303,7 @@ def __init__( def con(self) -> pyrfc.Connection: if self._con is not None: return self._con - con = pyrfc.Connection(**self.sap_credentials) + con = pyrfc.Connection(**self.credentials) self._con = con return con @@ -704,7 +704,7 @@ def __init__( func: str = "RFC_READ_TABLE", rfc_total_col_width_character_limit: int = 400, rfc_unique_id: List[str] = None, - sap_credentials: dict = None, + credentials: dict = None, sap_credentials_key: str = "SAP", env: str = "DEV", *args, @@ -723,7 +723,7 @@ def __init__( 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. rfc_unique_id (List[str], optional): Reference columns to merge chunks Data Frames. These columns must to be unique. Defaults to None. - sap_credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. + credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". @@ -735,32 +735,32 @@ def __init__( self.sap_credentials_key = sap_credentials_key self.env = env - if isinstance(sap_credentials, dict): - credentials_keys = list(sap_credentials.keys()) + if isinstance(credentials, dict): + credentials_keys = list(credentials.keys()) required_credentials_params = ["sysnr", "user", "passwd", "ashost"] for key in required_credentials_params: if key not in credentials_keys: logger.warning( - f"Required key '{key}' not found in your 'sap_credentials' dictionary!" + f"Required key '{key}' not found in your 'credentials' dictionary!" ) - sap_credentials = None + credentials = None - if sap_credentials is None: + if credentials is None: logger.warning( f"Your credentials will use {env} environment from local config. If you would like to use different one - please specified it in env parameter." ) try: - sap_credentials = local_config.get(sap_credentials_key).get(env) + credentials = local_config.get(sap_credentials_key).get(env) except AttributeError: raise CredentialError( f"sap_credentials_key: {sap_credentials_key} is not stored neither in KeyVault or local config!" ) - if sap_credentials is None: + if credentials is None: raise CredentialError(f"Missing {env} credentials!") super().__init__(*args, **kwargs) - self.sap_credentials = sap_credentials + self.credentials = credentials self.sep = sep self.replacement = replacement self.client_side_filters = None diff --git a/viadot/tasks/sap_rfc.py b/viadot/tasks/sap_rfc.py index 2d47e573a..3527e4632 100644 --- a/viadot/tasks/sap_rfc.py +++ b/viadot/tasks/sap_rfc.py @@ -24,7 +24,7 @@ def __init__( replacement: str = "-", func: str = None, rfc_total_col_width_character_limit: int = 400, - sap_credentials: dict = None, + credentials: dict = None, sap_credentials_key: str = "SAP", env: str = "DEV", max_retries: int = 3, @@ -58,7 +58,7 @@ def __init__( in case of too many columns for RFC function. According to SAP documentation, the limit is 512 characters. However, we observed SAP raising an exception even on a slightly lower number of characters, so we add a safety margin. Defaults to 400. - sap_credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. + credentials (dict, optional): The credentials to use to authenticate with SAP. By default, they're taken from the local viadot config. sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". By default, they're taken from the local viadot config. @@ -66,7 +66,7 @@ def __init__( self.query = query self.sep = sep self.replacement = replacement - self.sap_credentials = sap_credentials + self.credentials = credentials self.sap_credentials_key = sap_credentials_key self.env = env self.func = func @@ -87,14 +87,14 @@ def __init__( "replacement", "func", "rfc_total_col_width_character_limit", - "sap_credentials", + "credentials", ) def run( self, query: str, sep: str = None, replacement: str = "-", - sap_credentials: dict = None, + credentials: dict = None, sap_credentials_key: str = "SAP", env: str = "DEV", func: str = None, @@ -110,7 +110,7 @@ def run( multiple options are automatically tried. Defaults to None. replacement (str, optional): In case of sep is on a columns, set up a new character to replace inside the string to avoid flow breakdowns. Defaults to "-". - sap_credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. + credentials (dict, optional): The credentials to use to authenticate with SAP. Defaults to None. sap_credentials_key (str, optional): The key for sap credentials located in the local config or Azure Key Vault. Defaults to "SAP". env (str, optional): The key for sap_credentials_key pointing to the SAP environment. Defaults to "DEV". func (str, optional): SAP RFC function to use. Defaults to None. @@ -133,12 +133,12 @@ def run( pd.DataFrame: DataFrame with SAP data. """ - if sap_credentials is None: + if credentials is None: try: credentials_str = AzureKeyVaultSecret( secret=sap_credentials_key, ).run() - sap_credentials = json.loads(credentials_str).get(env) + credentials = json.loads(credentials_str).get(env) except: logger.warning( f"Getting credentials from Azure Key Vault was not possible. Either there is no key: {sap_credentials_key} or env: {env} or there is not Key Vault in your environment." @@ -152,7 +152,7 @@ def run( sap = SAPRFCV2( sep=sep, replacement=replacement, - sap_credentials=sap_credentials, + credentials=credentials, sap_credentials_key=sap_credentials_key, env=env, func=func, @@ -162,7 +162,7 @@ def run( else: sap = SAPRFC( sep=sep, - sap_credentials=sap_credentials, + credentials=credentials, sap_credentials_key=sap_credentials_key, env=env, func=func,