Merge branch 'dev' into feature/sharepoint_list_extension_KPI5

CHANGELOG.md

@@ -5,6 +5,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+### Added
+- Added `TM1` source class.
+- Added `TM1ToDF` task class.
+- Added `set_prefect_kv` parameter to `BigQueryToADLS` with `False` as a default. If there is a need to create new pair in KV Store the parameter can be changed to `True`.
+
+### Changed
+- Splitted test for Eurostat on source tests and task tests
 - Modified `SharepointList` source class: 
   -> docstrings update
 - Modified `SharepointToADLS` flow class:
@@ -24,6 +32,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   -> docstrings update
   -> Added `_rename_duplicated_fields` method to find and rename duplicated columns
 
+### Added
+- Added new view type `agent_interaction_view_type` in `Genesys`source.
+
 
 ## [0.4.21] - 2023-10-26
 ### Added

requirements.txt

@@ -43,3 +43,4 @@ dbt-core==1.3.2
 dbt-sqlserver==1.3.1
 lumaCLI==0.0.19
 Office365-REST-Python-Client==2.4.4
+TM1py==1.11.3

tests/integration/tasks/test_eurostat.py

@@ -3,139 +3,7 @@
 import pandas as pd
 import pytest
 
-from viadot.tasks import eurostat
-
-
-def test_and_validate_dataset_code_without_params(caplog):
-    """This function is designed to test the accuracy of the data retrieval feature in a program.
-    Specifically, it tests to ensure that the program returns a non-empty DataFrame when a correct
-    dataset code is provided without any parameters. The function is intended to be used in software
-    development to verify that the program is correctly retrieving data from the appropriate dataset.
-    """
-    task = eurostat.EurostatToDF(dataset_code="ILC_DI04").run()
-    assert isinstance(task, pd.DataFrame)
-    assert not task.empty
-    assert caplog.text == ""
-
-
-def test_wrong_dataset_code_logger(caplog):
-    """This function is designed to test the accuracy of the error logging feature in a program.
-    Specifically, it tests to ensure that the program is able to correctly identify and log errors
-    when provided with only incorrect dataset code.
-    The function is intended to be used in software development to identify correct type errors
-    and messages in the program's handling of codes.
-    """
-    task = eurostat.EurostatToDF(dataset_code="ILC_DI04E")
-
-    with pytest.raises(ValueError, match="DataFrame is empty!"):
-        with caplog.at_level(logging.ERROR):
-            task.run()
-    assert (
-        f"Failed to fetch data for ILC_DI04E, please check correctness of dataset code!"
-        in caplog.text
-    )
-
-
-def test_wrong_parameters_codes_logger(caplog):
-    """This function is designed to test the accuracy of the error logging feature in a program.
-    Specifically, it tests to ensure that the program is able to correctly identify and log errors
-    when provided with a correct dataset_code and correct parameters are provided, but both parameters codes are incorrect.
-    The function is intended to be used in software development to identify correct type errors
-    and messages in the program's handling of codes.
-    """
-    task = eurostat.EurostatToDF(
-        dataset_code="ILC_DI04",
-        params={"hhtyp": "total1", "indic_il": "non_existing_code"},
-    )
-
-    with pytest.raises(ValueError, match="DataFrame is empty!"):
-        with caplog.at_level(logging.ERROR):
-            task.run()
-    assert (
-        f"Parameters codes: 'total1 | non_existing_code' are not available. Please check your spelling!"
-        in caplog.text
-    )
-    assert (
-        f"You can find everything via link: https://ec.europa.eu/eurostat/databrowser/view/ILC_DI04/default/table?lang=en"
-        in caplog.text
-    )
-
-
-def test_parameter_codes_as_list_logger(caplog):
-    """This function is designed to test the accuracy of the error logging feature in a program.
-    Specifically, it tests to ensure that the program is able to correctly identify and log errors
-    when provided with a correct dataset code, correct parameters, but incorrect parameters codes structure
-    (as a list with strings, instead of single string).
-    The function is intended to be used in software development to identify correct type errors
-    and messages in the program's handling of codes.
-    """
-
-    task = eurostat.EurostatToDF(
-        dataset_code="ILC_DI04",
-        params={"hhtyp": ["totale", "nottotale"], "indic_il": "med_e"},
-    )
-    with pytest.raises(ValueError, match="Wrong structure of params!"):
-        with caplog.at_level(logging.ERROR):
-            task.run()
-    assert (
-        "You can provide only one code per one parameter as 'str' in params!\n"
-        in caplog.text
-    )
-    assert (
-        "CORRECT: params = {'unit': 'EUR'} | INCORRECT: params = {'unit': ['EUR', 'USD', 'PLN']}"
-        in caplog.text
-    )
-
-
-def test_wrong_parameters(caplog):
-    """This function is designed to test the accuracy of the error logging feature in a program.
-    Specifically, it tests to ensure that the program is able to correctly identify and log errors
-    when provided with a correct dataset_code, but incorrect parameters keys.
-    The function is intended to be used in software development to identify correct type errors
-    and messages in the program's handling of codes.
-    """
-
-    task = eurostat.EurostatToDF(
-        dataset_code="ILC_DI04", params={"hhhtyp": "total", "indic_ilx": "med_e"}
-    )
-    with pytest.raises(ValueError, match="DataFrame is empty!"):
-        with caplog.at_level(logging.ERROR):
-            task.run()
-    assert (
-        f"Parameters: 'hhhtyp | indic_ilx' are not in dataset. Please check your spelling!\n"
-        in caplog.text
-    )
-    assert (
-        f"Possible parameters: freq | hhtyp | indic_il | unit | geo | time"
-        in caplog.text
-    )
-
-
-def test_params_as_list():
-    """This function is designed to test the accuracy of the error logging feature in a program.
-    Specifically, it tests to ensure that the program is able to correctly identify and log error
-    when provided with a correct dataset_code, but incorrect params structure (as list instead of dict).
-    The function is intended to be used in software development to identify correct type errors
-    and messages in the program's handling of codes.
-    """
-    with pytest.raises(TypeError, match="Params should be a dictionary."):
-        eurostat.EurostatToDF(dataset_code="ILC_DI04", params=["total", "med_e"]).run()
-
-
-def test_correct_params_and_dataset_code(caplog):
-    """This function is designed to test the accuracy of the data retrieval feature in a program.
-    Specifically, it tests to ensure that the program returns a non-empty DataFrame when a correct
-    dataset code is provided with correct params. The function is intended to be used in software
-    development to verify that the program is correctly retrieving data from the appropriate dataset.
-    """
-
-    task = eurostat.EurostatToDF(
-        dataset_code="ILC_DI04", params={"hhtyp": "total", "indic_il": "med_e"}
-    ).run()
-
-    assert isinstance(task, pd.DataFrame)
-    assert not task.empty
-    assert caplog.text == ""
+from viadot.tasks import EurostatToDF
 
 
 def task_correct_requested_columns(caplog):
@@ -145,7 +13,7 @@ def task_correct_requested_columns(caplog):
     The function is intended to be used in software development to verify that the program is correctly
     retrieving data from the appropriate dataset.
     """
-    task = eurostat.EurostatToDF(
+    task = EurostatToDF(
         dataset_code="ILC_DI04",
         params={"hhtyp": "total", "indic_il": "med_e"},
         requested_columns=["updated", "geo", "indicator"],
@@ -155,7 +23,7 @@ def task_correct_requested_columns(caplog):
     assert isinstance(task, pd.DataFrame)
     assert not task.empty
     assert caplog.text == ""
-    assert list(task.columns) == task.needed_columns
+    assert list(task.columns) == task.requested_columns
 
 
 def test_wrong_needed_columns_names(caplog):
@@ -165,7 +33,7 @@ def test_wrong_needed_columns_names(caplog):
     The function is intended to be used in software development to identify correct type errors
     and messages in the program's handling of codes.
     """
-    task = eurostat.EurostatToDF(
+    task = EurostatToDF(
         dataset_code="ILC_DI04",
         params={"hhtyp": "total", "indic_il": "med_e"},
         requested_columns=["updated1", "geo1", "indicator1"],
@@ -188,7 +56,7 @@ def test_wrong_params_and_wrong_requested_columns_names(caplog):
     params validation. The function is intended to be used in software development to identify correct type errors
     and messages in the program's handling of codes.
     """
-    task = eurostat.EurostatToDF(
+    task = EurostatToDF(
         dataset_code="ILC_DI04",
         params={"hhhtyp": "total", "indic_ilx": "med_e"},
         requested_columns=["updated1", "geo1", "indicator1"],
@@ -217,8 +85,32 @@ def test_requested_columns_not_in_list():
     with pytest.raises(
         TypeError, match="Requested columns should be provided as list of strings."
     ):
-        eurostat.EurostatToDF(
+        EurostatToDF(
             dataset_code="ILC_DI04",
             params={"hhtyp": "total", "indic_il": "med_e"},
             requested_columns="updated",
         ).run()
+
+
+def test_requested_columns_not_provided(caplog):
+    """Test the behavior when 'requested_columns' are not provided to EurostatToDF.
+
+    This test checks the behavior of the EurostatToDF class when 'requested_columns' are not provided.
+    It ensures that the resulting DataFrame is of the correct type, not empty, and that no error
+    messages are logged using the 'caplog' fixture.
+
+    Parameters:
+    - caplog: pytest fixture for capturing log messages.
+
+    Usage:
+    - Invoke this test function to check the behavior of EurostatToDF when 'requested_columns' are not provided.
+    """
+    task = EurostatToDF(
+        dataset_code="ILC_DI04",
+        params={"hhtyp": "total", "indic_il": "med_e"},
+    )
+    df = task.run()
+
+    assert isinstance(df, pd.DataFrame)
+    assert not df.empty
+    assert caplog.text == ""

tests/integration/tasks/test_tm1.py

@@ -0,0 +1,15 @@
+import pandas as pd
+
+from viadot.tasks import TM1ToDF
+from viadot.config import local_config
+
+CUBE = local_config.get("test_cube")
+VIEW = local_config.get("test_view")
+
+
+def test_tm1_to_df():
+    tm1 = TM1ToDF(CUBE, VIEW)
+    df = tm1.run()
+
+    assert isinstance(df, pd.DataFrame)
+    assert df.empty is False

tests/integration/test_customer_gauge.py

@@ -9,6 +9,16 @@
 CG = CustomerGauge(endpoint=ENDPOINT)
 
 
+def test_wrong_endpoint():
+    with pytest.raises(ValueError, match="Incorrect endpoint name"):
+        CustomerGauge(endpoint=["wrong_endpoint"])
+
+
+def test_endpoint_and_url_not_provided():
+    with pytest.raises(ValueError, match="Provide endpoint name"):
+        CustomerGauge()
+
+
 def test_get_json_content():
     json_response = CG.get_json_response()
     assert isinstance(json_response, dict)
@@ -21,8 +31,10 @@ def test_properties_cleaning():
     json_response = CG.get_json_response()
     data = json_response["data"][2].copy()
     cleaned_data = CG.properties_cleaning(data.copy())
+
     assert isinstance(data["properties"], list)
     assert isinstance(cleaned_data["properties"], dict)
+    assert r"{',':" or "label" or "}" in json_response["drivers"]
 
 
 def test_flatten_json():
@@ -63,6 +75,13 @@ def test_pagesize_and_to_df():
     assert len(df) == 1
 
 
+def test_to_df_with_wrong_json_response():
+    with pytest.raises(
+        ValueError, match="Provided argument doesn't contain 'data' value"
+    ):
+        CG.to_df(json_response={})
+
+
 def test_pass_specific_cursor():
     # for default pagesize=1000 returned cursor value should be bigger than passed
     cur = random.randint(1, 9999)
@@ -71,11 +90,16 @@ def test_pass_specific_cursor():
     assert cur_retrieved > cur
 
 
+def test_cursor_is_not_provided():
+    with pytest.raises(
+        ValueError, match="Provided argument doesn't contain 'cursor' value"
+    ):
+        CG.get_cursor(json_response={})
+
+
 def test_uncomplete_date_arguments():
     with pytest.raises(ValueError, match="Missing date arguments"):
-        json_response = CG.get_json_response(
-            date_field="date_sent", start_date="2012-01-03"
-        )
+        CG.get_json_response(date_field="date_sent", start_date="2012-01-03")
 
 
 def test_endpoint_url_argument():

tests/integration/test_epicor.py

@@ -1,7 +1,8 @@
 import pytest
+import pandas as pd
 
 from viadot.config import local_config
-from viadot.exceptions import DataRangeError
+from viadot.exceptions import CredentialError, DataRangeError
 from viadot.sources import Epicor
 
 
@@ -48,3 +49,25 @@ def test_connection(epicor):
 def test_validate_filter(epicor_error):
     with pytest.raises(DataRangeError):
         epicor_error.validate_filter()
+
+
+def test_credentials_not_provided():
+    with pytest.raises(CredentialError):
+        Epicor(
+            base_url=local_config.get("EPICOR").get("test_url"),
+            credentials={"username": "user12", "port": 1111},
+            filters_xml="""
+            <OrderQuery>
+                <QueryFields>
+                    <CompanyNumber>001</CompanyNumber>
+                    <BegInvoiceDate></BegInvoiceDate>
+                    <EndInvoiceDate>2022-05-16</EndInvoiceDate>
+                    <RecordCount>3</RecordCount>
+                </QueryFields>
+            </OrderQuery>""",
+        )
+
+
+def test_to_df_return_type(epicor):
+    df = epicor.to_df()
+    assert isinstance(df, pd.DataFrame)