Merge pull request #599 from dyvenia/dev

Release 0.4.12 PR

CHANGELOG.md

@@ -7,7 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 
-# [0.4.11] - 2022-12-15
+## [0.4.12] - 2023-01-31
+### Added
+- Added `view_type_time_sleep` to the Genesys `queue_performance_detail_view`.
+
+### Changed
+- Updated `genesys_to_adls.py` flow with the `adls_bulk_upload` task
+- Updated `mindful_to_adls.py` flow with the `adls_bulk_upload` task
+- Changed `MindfulToCSV` task to download surveys info.
+
+
+## [0.4.11] - 2022-12-15
 ### Added
 - Added into `Genesys` the new view type `AGENT`. 
 

tests/integration/flows/test_adls_gen1_to_azure_sql_new.py

@@ -13,7 +13,6 @@
 
 
 def test_adls_gen1_to_azure_sql_new_init_args():
-
     flow = ADLSGen1ToAzureSQLNew(
         name="test_adls_gen1_gen2_flow",
         gen1_path="test_file_1.csv",
@@ -44,27 +43,23 @@ def test_adls_gen1_to_azure_sql_new_mock():
         mock_method.assert_called_with()
 
 
-def test_adls_gen1_to_azure_sql_new_flow_run_mock():
-
-    d = {"country": [1, 2], "sales": [3, 4]}
-    df = pd.DataFrame(data=d)
-
-    with mock.patch(
-        "viadot.flows.adls_gen1_to_azure_sql_new.gen1_to_df_task.bind"
-    ) as gen1_to_df_task_mock_bind_method_mock:
-        gen1_to_df_task_mock_bind_method_mock.return_value = df
-
-        flow = ADLSGen1ToAzureSQLNew(
-            name="test_adls_g1g2",
-            gen1_path="example_path",
-            gen2_path="raw/test/test.csv",
-            dtypes={"country": "VARCHAR(25)", "sales": "INT"},
-            if_exists="replace",
-            table="test",
-            schema="sandbox",
-        )
+@mock.patch(
+    "viadot.tasks.AzureDataLakeToDF.run",
+    return_value=pd.DataFrame(data={"country": [1, 2], "sales": [3, 4]}),
+)
+@pytest.mark.run
+def test_adls_gen1_to_azure_sql_new_flow_run_mock(mocked_class):
+    flow = ADLSGen1ToAzureSQLNew(
+        name="test_adls_g1g2",
+        gen1_path="example_path",
+        gen2_path="raw/test/test.csv",
+        dtypes={"country": "VARCHAR(25)", "sales": "INT"},
+        if_exists="replace",
+        table="test",
+        schema="sandbox",
+    )
 
-        result = flow.run()
+    result = flow.run()
 
-        assert result.is_successful()
-        os.remove("test_adls_g1g2.csv")
+    assert result.is_successful()
+    os.remove("test_adls_g1g2.csv")

tests/integration/flows/test_adls_to_azure_sql.py

@@ -1,9 +1,12 @@
 import os
+import pytest
+from unittest import mock
 
 import pandas as pd
+from prefect.engine import signals
 
 from viadot.flows import ADLSToAzureSQL
-from viadot.flows.adls_to_azure_sql import df_to_csv_task
+from viadot.flows.adls_to_azure_sql import df_to_csv_task, check_dtypes_sort
 
 
 def test_get_promoted_adls_path_csv_file():
@@ -67,3 +70,34 @@ def test_df_to_csv_task_none(caplog):
     task.run(df, path=path, remove_tab=False)
     assert "DataFrame is None" in caplog.text
     assert os.path.isfile(path) == False
+
+
+@pytest.mark.dtypes
+def test_check_dtypes_sort():
+    d = {"col1": ["rat", "cat"], "col2": [3, 4]}
+    df = pd.DataFrame(data=d)
+    dtypes = {
+        "col1": "varchar(6)",
+        "col2": "varchar(6)",
+    }
+    task = check_dtypes_sort
+
+    n_dtypes = task.run(df=df, dtypes=dtypes)
+    assert list(dtypes.keys()) == list(n_dtypes.keys())
+
+    dtypes = {
+        "col2": "varchar(6)",
+        "col1": "varchar(6)",
+    }
+    n_dtypes = task.run(df=df, dtypes=dtypes)
+    assert list(dtypes.keys()) != list(n_dtypes.keys())
+
+    dtypes = {
+        "col1": "varchar(6)",
+        "col3": "varchar(6)",
+    }
+    try:
+        n_dtypes = task.run(df=df, dtypes=dtypes)
+        assert False
+    except signals.FAIL:
+        assert True

tests/integration/flows/test_sharepoint_to_adls.py

@@ -1,6 +1,6 @@
 import os
 from unittest import mock
-
+import pytest
 import pandas as pd
 import pendulum
 from prefect.tasks.secrets import PrefectSecret
@@ -11,75 +11,64 @@
 ADLS_FILE_NAME = str(pendulum.now("utc")) + ".csv"
 ADLS_DIR_PATH = "raw/tests/"
 CREDENTIALS_SECRET = PrefectSecret("AZURE_DEFAULT_ADLS_SERVICE_PRINCIPAL_SECRET").run()
-
-
-def test_sharepoint_to_adls_run_flow():
-
-    d = {"country": [1, 2], "sales": [3, 4]}
-    df = pd.DataFrame(data=d)
-
-    with mock.patch(
-        "viadot.flows.sharepoint_to_adls.excel_to_df_task.bind"
-    ) as excel_to_df_task_mock:
-        excel_to_df_task_mock.return_value = df
-
-        flow = SharepointToADLS(
-            "test_sharepoint_to_adls_run_flow",
-            output_file_extension=".csv",
-            adls_sp_credentials_secret=CREDENTIALS_SECRET,
-            adls_dir_path=ADLS_DIR_PATH,
-            adls_file_name=ADLS_FILE_NAME,
-        )
-        result = flow.run()
-        assert result.is_successful()
-        os.remove("test_sharepoint_to_adls_run_flow.csv")
-        os.remove("test_sharepoint_to_adls_run_flow.json")
-
-
-def test_sharepoint_to_adls_run_flow_overwrite_true():
-
-    d = {"country": [1, 2], "sales": [3, 4]}
-    df = pd.DataFrame(data=d)
-
-    with mock.patch(
-        "viadot.flows.sharepoint_to_adls.excel_to_df_task.bind"
-    ) as excel_to_df_task_mock:
-        excel_to_df_task_mock.return_value = df
-
-        flow = SharepointToADLS(
-            "test_sharepoint_to_adls_run_flow_overwrite_true",
-            output_file_extension=".csv",
-            adls_sp_credentials_secret=CREDENTIALS_SECRET,
-            adls_dir_path=ADLS_DIR_PATH,
-            adls_file_name=ADLS_FILE_NAME,
-            overwrite_adls=True,
-        )
-        result = flow.run()
-        assert result.is_successful()
-        os.remove("test_sharepoint_to_adls_run_flow_overwrite_true.csv")
-        os.remove("test_sharepoint_to_adls_run_flow_overwrite_true.json")
-
-
-def test_sharepoint_to_adls_run_flow_overwrite_false():
-
-    d = {"country": [1, 2], "sales": [3, 4]}
-    df = pd.DataFrame(data=d)
-
-    with mock.patch(
-        "viadot.flows.sharepoint_to_adls.excel_to_df_task.bind"
-    ) as excel_to_df_task_mock:
-        excel_to_df_task_mock.return_value = df
-
-        flow = SharepointToADLS(
-            "test_sharepoint_to_adls_run_flow_overwrite_false",
-            output_file_extension=".csv",
-            adls_sp_credentials_secret=CREDENTIALS_SECRET,
-            adls_dir_path=ADLS_DIR_PATH,
-            adls_file_name=ADLS_FILE_NAME,
-            overwrite_adls=False,
-        )
-        result = flow.run()
-
-        assert result.is_failed()
-        os.remove("test_sharepoint_to_adls_run_flow_overwrite_false.csv")
-        os.remove("test_sharepoint_to_adls_run_flow_overwrite_false.json")
+DATA = {"country": [1, 2], "sales": [3, 4]}
+
+
+@mock.patch(
+    "viadot.tasks.SharepointToDF.run",
+    return_value=pd.DataFrame(data=DATA),
+)
+@pytest.mark.run
+def test_sharepoint_to_adls_run_flow(mocked_class):
+    flow = SharepointToADLS(
+        "test_sharepoint_to_adls_run_flow",
+        output_file_extension=".csv",
+        adls_sp_credentials_secret=CREDENTIALS_SECRET,
+        adls_dir_path=ADLS_DIR_PATH,
+        adls_file_name=ADLS_FILE_NAME,
+    )
+    result = flow.run()
+    assert result.is_successful()
+    os.remove("test_sharepoint_to_adls_run_flow.csv")
+    os.remove("test_sharepoint_to_adls_run_flow.json")
+
+
+@mock.patch(
+    "viadot.tasks.SharepointToDF.run",
+    return_value=pd.DataFrame(data=DATA),
+)
+@pytest.mark.run
+def test_sharepoint_to_adls_run_flow_overwrite_true(mocked_class):
+    flow = SharepointToADLS(
+        "test_sharepoint_to_adls_run_flow_overwrite_true",
+        output_file_extension=".csv",
+        adls_sp_credentials_secret=CREDENTIALS_SECRET,
+        adls_dir_path=ADLS_DIR_PATH,
+        adls_file_name=ADLS_FILE_NAME,
+        overwrite_adls=True,
+    )
+    result = flow.run()
+    assert result.is_successful()
+    os.remove("test_sharepoint_to_adls_run_flow_overwrite_true.csv")
+    os.remove("test_sharepoint_to_adls_run_flow_overwrite_true.json")
+
+
+@mock.patch(
+    "viadot.tasks.SharepointToDF.run",
+    return_value=pd.DataFrame(data=DATA),
+)
+@pytest.mark.run
+def test_sharepoint_to_adls_run_flow_overwrite_false(mocked_class):
+    flow = SharepointToADLS(
+        "test_sharepoint_to_adls_run_flow_overwrite_false",
+        output_file_extension=".csv",
+        adls_sp_credentials_secret=CREDENTIALS_SECRET,
+        adls_dir_path=ADLS_DIR_PATH,
+        adls_file_name=ADLS_FILE_NAME,
+        overwrite_adls=False,
+    )
+    result = flow.run()
+
+    assert result.is_failed()
+    os.remove("test_sharepoint_to_adls_run_flow_overwrite_false.csv")
+    os.remove("test_sharepoint_to_adls_run_flow_overwrite_false.json")

tests/integration/test_mindful.py

@@ -30,6 +30,14 @@ def json():
         return test
 
 
+class MockClass2:
+    status_code = 204
+    content = b""
+
+    def json():
+        return None
+
+
 @pytest.mark.init
 def test_instance_mindful():
     mf = Mindful(header=header)
@@ -67,6 +75,17 @@ def test_mindful_api_response3(mock_api_response):
     assert mf.endpoint == "responses"
 
 
+@mock.patch("viadot.sources.mindful.handle_api_response", return_value=MockClass)
+@pytest.mark.connect
+def test_mindful_api_response4(mock_api_response):
+    mf = Mindful(header=header)
+
+    response = mf.get_survey_list()
+
+    assert response.status_code == 200 and isinstance(response.json(), list)
+    assert mf.endpoint == "surveys"
+
+
 @mock.patch("viadot.sources.Mindful._mindful_api_response", return_value=MockClass)
 @pytest.mark.save
 def test_mindful_interactions(mock_connection):
@@ -89,3 +108,23 @@ def test_mindful_responses(mock_connection):
     assert mf.endpoint == "responses" and isinstance(mf.endpoint, str)
     assert os.path.exists("responses.csv")
     os.remove("responses.csv")
+
+
+@mock.patch("viadot.sources.Mindful._mindful_api_response", return_value=MockClass)
+@pytest.mark.save
+def test_mindful_surveys(mock_connection):
+    mf = Mindful(header=header)
+    response = mf.get_survey_list()
+    mf.response_to_file(response)
+
+    assert mf.endpoint == "surveys" and isinstance(mf.endpoint, str)
+    assert os.path.exists("surveys.csv")
+    os.remove("surveys.csv")
+
+
+@mock.patch("viadot.sources.Mindful._mindful_api_response", return_value=MockClass2)
+@pytest.mark.exception
+def test_file_exception(mock_mindful):
+    mf = MindfulToCSV()
+    response = mf.run(credentials_mindful=credentials_mindful)
+    assert response == None

tests/test_viadot.py

@@ -2,4 +2,4 @@
 
 
 def test_version():
-    assert __version__ == "0.4.11"
+    assert __version__ == "0.4.12"

tests/unit/test_task_utils.py

@@ -5,6 +5,7 @@
 import pandas as pd
 import prefect
 import pytest
+from unittest import mock
 
 from viadot.task_utils import (
     add_ingestion_metadata_task,
@@ -18,9 +19,20 @@
     dtypes_to_json_task,
     union_dfs_task,
     write_to_json,
+    adls_bulk_upload,
 )
 
 
+class MockAzureUploadClass:
+    def run(
+        from_path: str = "",
+        to_path: str = "",
+        sp_credentials_secret: str = "",
+        overwrite: bool = False,
+    ) -> None:
+        pass
+
+
 def count_dtypes(dtypes_dict: dict = None, dtypes_to_count: List[str] = None) -> int:
     dtypes_counter = 0
     for v in dtypes_dict.values():
@@ -221,3 +233,12 @@ def test_df_clean_column_defined():
     df = pd.DataFrame.from_dict(data)
     output = df_clean_column.run(df, ["col_2"]).to_dict()
     assert output == expected_output
+
+
+@mock.patch("viadot.task_utils.AzureDataLakeUpload", return_value=MockAzureUploadClass)
+@pytest.mark.bulk
+def test_adls_bulk_upload(mock_upload):
+    file_names = ["random_1.csv", "random_2.csv"]
+
+    adls_bulk_upload.run(file_names=file_names, adls_file_path="any/at/random")
+    mock_upload.assert_called_once()

viadot/__init__.py

@@ -1 +1 @@
-__version__ = "0.4.11"
+__version__ = "0.4.12"