Merge pull request #816 from dyvenia/dev

Release 0.4.22 PR

CHANGELOG.md

@@ -5,6 +5,50 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+
+### Fixed
+
+### Changed
+
+
+## [0.4.22] - 2023-11-15
+### Added
+- Added `TM1` source class.
+- Added `TM1ToDF` task class.
+- Added `set_prefect_kv` parameter to `BigQueryToADLS` with `False` as a default. If there is a need to create new pair in KV Store the parameter can be changed to `True`.
+- Added `_rename_duplicated_fields` method to `SharepointListToDF` task class for finding and rename duplicated columns.
+- Added new view type `agent_interaction_view_type` in `Genesys`source.
+- Added new logic for endpoint `users` in `Genesys`task.
+- Added libraries `nltk` and `sklearn` to `requirements`.
+
+### Fixed
+- Fixed bug for endpoint `conversations` in GET method in `Genesys` Task.
+
+### Changed
+- Splitted test for `Eurostat` on source tests and task tests.
+- Modified `SharepointList` source class: 
+  -> docstrings update.
+- Modified `SharepointToADLS` flow class:
+  -> docstrings update.
+  -> changed set_prefect_kv: bool = False to prevent forced KV store append.
+- Modified `SharepointListToADLS` flow class:
+  -> changed set_prefect_kv: bool = False to prevent forced KV store append.
+- Modified `SharepointList` source class:
+  -> docstrings update.
+  -> Changed `_unpack_fields` method to handle Sharepoint MultiChoiceField type + small improvements.
+  -> Changed `get_fields` method to handle special characters - different approach to call get() and execute_query().
+  -> Renamed method from `select_expandable_user_fields` to `select_fields` + update for MultiChoiceField type.
+  -> Changed `check_filters` method errors messages and more checks added.
+  -> Changed `operators_mapping` method errors messages.
+  -> Changed `make_filter_for_df` method errors messages.
+- Modified `SharepointListToDF` task class:
+  -> docstrings update
+- Splitted test for Eurostat on source tests and task tests.
+- Modified `CustomerGauge` source class with simplified logic to return json structure.
+- Expanded `CustomerGaugeToDF` task class with separate cleaning functions and handling nested json structure flattening with two new methods `_field_reference_unpacker` and `_nested_dict_transformer`.
+- Changed `CustomerGaugeToADLS` to containing new arguments.
+
 
 ## [0.4.21] - 2023-10-26
 ### Added
@@ -13,13 +57,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `SharepointListToDF` task class.
 - Added `SharepointListToADLS` flow class.
 - Added tests for `SharepointList`.
-- Added `get_nested_dict` to untils.py.
+- Added `get_nested_dict` to utils.py.
 
 ### Fixed
 
 ### Changed
 - Changed `GenesysToCSV` logic for end_point == "conversations". Added new fields to extraction.
 
+
 ## [0.4.20] - 2023-10-12
 ### Added
 - Added `Office365-REST-Python-Client` library to `requirements`.
@@ -626,4 +671,4 @@ specified in the `SUPERMETRICS_DEFAULT_USER` secret
 - Moved from poetry to pip
 
 ### Fixed
-- Fix `AzureBlobStorage`'s `to_storage()` method is missing the final upload blob part
+- Fix `AzureBlobStorage`'s `to_storage()` method is missing the final upload blob part

requirements.txt

@@ -43,3 +43,6 @@ dbt-core==1.3.2
 dbt-sqlserver==1.3.1
 lumaCLI==0.0.19
 Office365-REST-Python-Client==2.4.4
+TM1py==1.11.3
+nltk==3.8.1
+scikit-learn==1.3.2

tests/integration/flows/test_bigquery_to_adls.py

@@ -1,15 +1,14 @@
 import os
+from unittest import mock
 
+import pandas as pd
 import pendulum
 import pytest
-from unittest import mock
-import pandas as pd
-
 from prefect.tasks.secrets import PrefectSecret
-from viadot.flows import BigQueryToADLS
-from viadot.tasks import AzureDataLakeRemove
 
 from viadot.exceptions import ValidationError
+from viadot.flows import BigQueryToADLS
+from viadot.tasks import AzureDataLakeRemove
 
 ADLS_DIR_PATH = "raw/tests/"
 ADLS_FILE_NAME = str(pendulum.now("utc")) + ".parquet"

tests/integration/flows/test_cloud_for_customers_report_to_adls.py

@@ -1,6 +1,6 @@
 from viadot.config import local_config
-from viadot.flows import CloudForCustomersReportToADLS
 from viadot.exceptions import ValidationError
+from viadot.flows import CloudForCustomersReportToADLS
 
 
 def test_cloud_for_customers_report_to_adls():

tests/integration/flows/test_customer_gauge_to_adls.py

@@ -4,8 +4,8 @@
 import pandas as pd
 import pytest
 
-from viadot.flows import CustomerGaugeToADLS
 from viadot.exceptions import ValidationError
+from viadot.flows import CustomerGaugeToADLS
 
 DATA = {
     "user_name": ["Jane", "Bob"],
@@ -90,6 +90,3 @@ def test_customer_gauge_to_adls_run_flow_validation_failure(mocked_class):
         flow.run()
     except ValidationError:
         pass
-
-    os.remove("test_customer_gauge_to_adls_run_flow_validation_failure.parquet")
-    os.remove("test_customer_gauge_to_adls_run_flow_validation_failure.json")

tests/integration/flows/test_hubspot_to_adls.py

@@ -5,8 +5,8 @@
 import pandas as pd
 import pytest
 
-from viadot.flows import HubspotToADLS
 from viadot.exceptions import ValidationError
+from viadot.flows import HubspotToADLS
 
 DATA = {
     "id": {"0": "820306930"},

tests/integration/flows/test_mediatool_to_adls.py

@@ -4,8 +4,8 @@
 import pandas as pd
 import pytest
 
-from viadot.flows import MediatoolToADLS
 from viadot.exceptions import ValidationError
+from viadot.flows import MediatoolToADLS
 
 DATA = {"country": ["DK", "DE"], "sales": [3, 4]}
 ADLS_FILE_NAME = "test_mediatool.parquet"

tests/integration/flows/test_mysql_to_adls.py

@@ -1,4 +1,5 @@
 from unittest import mock
+
 from viadot.flows.mysql_to_adls import MySqlToADLS
 
 query = """SELECT * FROM `example-views`.`sales`"""

tests/integration/flows/test_salesforce_to_adls.py

@@ -2,9 +2,9 @@
 
 from prefect.tasks.secrets import PrefectSecret
 
+from viadot.exceptions import ValidationError
 from viadot.flows import SalesforceToADLS
 from viadot.tasks import AzureDataLakeRemove
-from viadot.exceptions import ValidationError
 
 ADLS_FILE_NAME = "test_salesforce.parquet"
 ADLS_DIR_PATH = "raw/tests/"

tests/integration/flows/test_sap_bw_to_adls.py

@@ -4,8 +4,8 @@
 import pandas as pd
 import pytest
 
-from viadot.flows import SAPBWToADLS
 from viadot.exceptions import ValidationError
+from viadot.flows import SAPBWToADLS
 
 DATA = {
     "[0CALMONTH].[LEVEL01].[DESCRIPTION]": ["January 2023"],

tests/integration/flows/test_sap_rfc_to_adls.py

@@ -1,8 +1,8 @@
 from viadot.config import local_config
+from viadot.exceptions import ValidationError
 from viadot.flows import SAPRFCToADLS
 from viadot.sources import AzureDataLake
 from viadot.tasks import AzureDataLakeRemove
-from viadot.exceptions import ValidationError
 
 try:
     import pyrfc

tests/integration/flows/test_supermetrics_to_adls.py

@@ -4,8 +4,8 @@
 import pytest
 from prefect.storage import Local
 
-from viadot.flows import SupermetricsToADLS
 from viadot.exceptions import ValidationError
+from viadot.flows import SupermetricsToADLS
 
 CWD = os.getcwd()
 adls_dir_path = "raw/tests/supermetrics"

tests/integration/flows/test_vidclub_to_adls.py

@@ -4,8 +4,8 @@
 import pandas as pd
 import pytest
 
-from viadot.flows import VidClubToADLS
 from viadot.exceptions import ValidationError
+from viadot.flows import VidClubToADLS
 
 DATA = {"col1": ["aaa", "bbb", "ccc"], "col2": [11, 22, 33]}
 ADLS_FILE_NAME = "test_vid_club.parquet"