Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hotfix/check_nested_value non dict value handling #830

Merged
merged 3 commits into from
Dec 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added

### Fixed

- `task_utils/get_nested_value` fixed issue with non dict parameter passed without level(1st workflow)
### Changed

## [0.4.23] - 2023-12-07
Expand Down
25 changes: 14 additions & 11 deletions tests/integration/flows/test_sharepoint_to_adls.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
from unittest import mock

import logging
import pandas as pd
import pendulum
import pytest
Expand Down Expand Up @@ -232,15 +233,17 @@ def test_sharepoint_list_to_adls_run_flow_success_warn_on_no_data_returned(
CSV file should not be generated!
"""
# Get prefect client instance
flow = SharepointListToADLS(
"test_sharepoint_to_adls_run_flow",
output_file_extension=".csv",
adls_sp_credentials_secret=CREDENTIALS_SECRET,
adls_dir_path=ADLS_DIR_PATH,
file_name=ADLS_FILE_NAME_LIST,
list_title="",
site_url="",
if_no_data_returned="warn",
)
result = flow.run()
with caplog.at_level(logging.WARNING):
flow = SharepointListToADLS(
"test_sharepoint_to_adls_run_flow",
output_file_extension=".csv",
adls_sp_credentials_secret=CREDENTIALS_SECRET,
adls_dir_path=ADLS_DIR_PATH,
file_name=ADLS_FILE_NAME_LIST,
list_title="",
site_url="",
if_no_data_returned="warn",
)
result = flow.run()
assert "No data in the source response. Df empty." in caplog.text
assert result.is_successful()
6 changes: 6 additions & 0 deletions tests/unit/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -315,3 +315,9 @@ def test_get_nested_value_without_levels(nested_dict):
"searched_phrase_3": "Found it!",
}
assert result_2 == {"searched_phrase_2": "Found it_2!"}


def test_get_nested_value_non_dict_passed():
"""Sample test checking the correctness of the function when non dict value (int) is provided."""

assert get_nested_value(nested_dict=5) == None
15 changes: 9 additions & 6 deletions viadot/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -491,13 +491,16 @@ def get_nested_value(
else:
return nested_dict[lvl]
else:
for lvl in nested_dict.values():
if isinstance(lvl, dict):
return get_nested_value(nested_dict=lvl)
else:
return nested_dict
if isinstance(nested_dict, dict):
for lvl in nested_dict.values():
if isinstance(lvl, dict):
return get_nested_value(nested_dict=lvl)
else:
return nested_dict
else:
return None
except KeyError as e:
return None
except TypeError as e:
except (TypeError, AttributeError) as e:
logger.error(f"The 'nested_dict' must be a dictionary. {e}")
return None
Loading