From d2352618764f26da29fe9a1624490479aef96095 Mon Sep 17 00:00:00 2001 From: marcinpurtak Date: Fri, 8 Dec 2023 12:17:38 +0100 Subject: [PATCH] HOTFIX for issues with sharepoint_list non dict item --- CHANGELOG.md | 2 +- .../flows/test_sharepoint_to_adls.py | 25 +++++++++++-------- tests/unit/test_utils.py | 6 +++++ viadot/utils.py | 15 ++++++----- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0fe463f2..e9490299a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Fixed - +- `task_utils/get_nested_value` fixed issue with non dict parameter passed without level(1st workflow) ### Changed ## [0.4.23] - 2023-12-07 diff --git a/tests/integration/flows/test_sharepoint_to_adls.py b/tests/integration/flows/test_sharepoint_to_adls.py index 198704aa5..0411c80a9 100644 --- a/tests/integration/flows/test_sharepoint_to_adls.py +++ b/tests/integration/flows/test_sharepoint_to_adls.py @@ -1,6 +1,7 @@ import os from unittest import mock +import logging import pandas as pd import pendulum import pytest @@ -232,15 +233,17 @@ def test_sharepoint_list_to_adls_run_flow_success_warn_on_no_data_returned( CSV file should not be generated! """ # Get prefect client instance - flow = SharepointListToADLS( - "test_sharepoint_to_adls_run_flow", - output_file_extension=".csv", - adls_sp_credentials_secret=CREDENTIALS_SECRET, - adls_dir_path=ADLS_DIR_PATH, - file_name=ADLS_FILE_NAME_LIST, - list_title="", - site_url="", - if_no_data_returned="warn", - ) - result = flow.run() + with caplog.at_level(logging.WARNING): + flow = SharepointListToADLS( + "test_sharepoint_to_adls_run_flow", + output_file_extension=".csv", + adls_sp_credentials_secret=CREDENTIALS_SECRET, + adls_dir_path=ADLS_DIR_PATH, + file_name=ADLS_FILE_NAME_LIST, + list_title="", + site_url="", + if_no_data_returned="warn", + ) + result = flow.run() + assert "No data in the source response. Df empty." in caplog.text assert result.is_successful() diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 38564ed9e..2d6567da8 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -315,3 +315,9 @@ def test_get_nested_value_without_levels(nested_dict): "searched_phrase_3": "Found it!", } assert result_2 == {"searched_phrase_2": "Found it_2!"} + + +def test_get_nested_value_non_dict_passed(): + """Sample test checking the correctness of the function when non dict value (int) is provided.""" + + assert get_nested_value(nested_dict=5) == None diff --git a/viadot/utils.py b/viadot/utils.py index 654a408d1..0c819ccff 100644 --- a/viadot/utils.py +++ b/viadot/utils.py @@ -491,13 +491,16 @@ def get_nested_value( else: return nested_dict[lvl] else: - for lvl in nested_dict.values(): - if isinstance(lvl, dict): - return get_nested_value(nested_dict=lvl) - else: - return nested_dict + if isinstance(nested_dict, dict): + for lvl in nested_dict.values(): + if isinstance(lvl, dict): + return get_nested_value(nested_dict=lvl) + else: + return nested_dict + else: + return None except KeyError as e: return None - except TypeError as e: + except (TypeError, AttributeError) as e: logger.error(f"The 'nested_dict' must be a dictionary. {e}") return None