Merge pull request #796 from angelika233/sharepoint_list_formatted

Sharepoint list formatted

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - Added `validate_df` task to task_utils.
+- Added `SharepointList` source class.
+- Added `SharepointListToDF` task class.
+- Added `SharepointListToADLS` flow class.
+- Added tests for `SharepointList`.
+- Added `get_nested_dict` to untils.py.
+- Added `validate_df` task to `SharepointToADLS` class.
+
 ### Fixed
 
 ### Changed
@@ -618,4 +625,4 @@ specified in the `SUPERMETRICS_DEFAULT_USER` secret
 - Moved from poetry to pip
 
 ### Fixed
-- Fix `AzureBlobStorage`'s `to_storage()` method is missing the final upload blob part
+- Fix `AzureBlobStorage`'s `to_storage()` method is missing the final upload blob part

tests/integration/test_sharepoint.py

@@ -1,6 +1,8 @@
 import os
+import re
 
 import pandas as pd
+from copy import deepcopy
 import pytest
 from prefect.tasks.secrets import PrefectSecret
 
@@ -9,6 +11,7 @@
 from viadot.sources import Sharepoint
 from viadot.task_utils import df_get_data_types_task
 from viadot.tasks.sharepoint import SharepointToDF
+from viadot.sources import SharepointList
 
 
 def get_url() -> str:
@@ -18,7 +21,7 @@ def get_url() -> str:
     Returns:
         str: File URL.
     """
-    return local_config["SHAREPOINT"].get("url")
+    return local_config["SHAREPOINT"].get("file_url")
 
 
 @pytest.fixture(scope="session")
@@ -163,3 +166,191 @@ def test_get_data_types(file_name):
     dtypes = dtypes_map.values()
 
     assert "String" in dtypes
+
+
+@pytest.fixture(scope="session")
+def sharepoint_list():
+    """
+    Fixture for creating a Sharepoint class instance.
+    The class instance can be used within a test functions to interact with Sharepoint.
+    """
+    spl = SharepointList()
+    yield spl
+
+
+def test_valid_filters(sharepoint_list):
+    filters = {
+        "filter1": {"dtype": "int", "operator1": "<", "value1": 10},
+        "filter2": {"dtype": "str", "operator1": "==", "value1": "value"},
+    }
+    result = sharepoint_list.check_filters(filters)
+    assert result is True
+
+
+def test_invalid_dtype(sharepoint_list):
+    filters = {
+        "filter1": {"dtype": "list", "operator1": ">", "value1": 10},
+    }
+    with pytest.raises(ValueError, match="dtype not allowed!"):
+        sharepoint_list.check_filters(filters)
+
+
+def test_missing_operator1(sharepoint_list):
+    filters = {
+        "filter1": {"dtype": "int", "value1": 10},
+    }
+    with pytest.raises(ValueError, match="Operator1 is missing!"):
+        sharepoint_list.check_filters(filters)
+
+
+def test_invalid_operator1(sharepoint_list):
+    filters = {
+        "filter1": {"dtype": "int", "operator1": "*", "value1": 10},
+    }
+    with pytest.raises(ValueError, match="Operator type not allowed!"):
+        sharepoint_list.check_filters(filters)
+
+
+def test_missing_value1(sharepoint_list):
+    filters = {
+        "filter1": {"dtype": "int", "operator1": ">", "value1": None},
+    }
+    with pytest.raises(ValueError, match="Value for operator1 is missing!"):
+        sharepoint_list.check_filters(filters)
+
+
+def test_missing_operators_conjuction(sharepoint_list):
+    filters = {
+        "filter1": {
+            "dtype": "int",
+            "operator1": ">",
+            "value1": 10,
+            "operator2": "<",
+            "value2": 20,
+        },
+    }
+    with pytest.raises(ValueError, match="Operators for conjuction is missing!"):
+        sharepoint_list.check_filters(filters)
+
+
+def test_invalid_operators_conjuction(sharepoint_list):
+    filters = {
+        "filter1": {
+            "dtype": "int",
+            "operator1": ">",
+            "value1": 10,
+            "operator2": "<",
+            "value2": 20,
+            "operators_conjuction": "!",
+        },
+    }
+    with pytest.raises(ValueError, match="Operators for conjuction not allowed!"):
+        sharepoint_list.check_filters(filters)
+
+
+def test_invalid_filters_conjuction(sharepoint_list):
+    filters = {
+        "filter1": {
+            "dtype": "int",
+            "operator1": ">",
+            "value1": 10,
+            "filters_conjuction": "!",
+        },
+    }
+    with pytest.raises(
+        ValueError, match="Filters operators for conjuction not allowed!"
+    ):
+        sharepoint_list.check_filters(filters)
+
+
+def test_valid_mapping(sharepoint_list):
+    filters = {
+        "filter1": {
+            "operator1": ">",
+            "operator2": "<=",
+            "operators_conjuction": "&",
+            "filters_conjuction": "|",
+        },
+        "filter2": {"operator1": "==", "operator2": "!=", "operators_conjuction": "|"},
+    }
+    expected_result = {
+        "filter1": {
+            "operator1": "gt",
+            "operator2": "le",
+            "operators_conjuction": "and",
+            "filters_conjuction": "or",
+        },
+        "filter2": {"operator1": "eq", "operator2": "ne", "operators_conjuction": "or"},
+    }
+    result = sharepoint_list.operators_mapping(deepcopy(filters))
+    assert result == expected_result
+
+
+def test_invalid_comparison_operator(sharepoint_list):
+    filters = {
+        "filter1": {
+            "operator1": "*",
+            "operator2": "<=",
+            "operators_conjuction": "&",
+            "filters_conjuction": "|",
+        },
+    }
+    error_message = "This comparison operator: * is not allowed. Please read the function documentation for details!"
+    with pytest.raises(ValueError, match=re.escape(error_message)):
+        sharepoint_list.operators_mapping(deepcopy(filters))
+
+
+def test_invalid_logical_operator(sharepoint_list):
+    filters = {
+        "filter1": {
+            "operator1": ">",
+            "operator2": "<=",
+            "operators_conjuction": "!",
+            "filters_conjuction": "|",
+        },
+    }
+    error_message = "This conjuction(logical) operator: ! is not allowed. Please read the function documentation for details!"
+    with pytest.raises(ValueError, match=re.escape(error_message)):
+        sharepoint_list.operators_mapping(deepcopy(filters))
+
+
+def test_single_filter_datetime_api(sharepoint_list):
+    filters = {
+        "date_column": {"dtype": "datetime", "operator1": ">", "value1": "2023-01-01"}
+    }
+    result = sharepoint_list.make_filter_for_api(filters)
+    expected_result = "date_column gt datetime'2023-01-01T00:00:00' "
+    assert result == expected_result
+
+
+def test_multiple_filters_api(sharepoint_list):
+    filters = {
+        "int_column": {
+            "dtype": "int",
+            "operator1": ">=",
+            "value1": 10,
+            "operator2": "<",
+            "value2": 20,
+        },
+        "str_column": {"dtype": "str", "operator1": "==", "value1": "example"},
+    }
+    result = sharepoint_list.make_filter_for_api(filters)
+    expected_result = "int_column ge '10'int_column lt '20'str_column eq 'example'"
+    assert result == expected_result
+
+
+def test_single_df_filter(sharepoint_list):
+    filters = {"column1": {"operator1": ">", "value1": 10}}
+    result = sharepoint_list.make_filter_for_df(filters)
+    expected_result = "df.loc[(df.column1 > '10')]"
+    assert result == expected_result
+
+
+def test_multiple_df_filters(sharepoint_list):
+    filters = {
+        "column1": {"operator1": ">", "value1": 10, "filters_conjuction": "&"},
+        "column2": {"operator1": "<", "value1": 20},
+    }
+    result = sharepoint_list.make_filter_for_df(filters)
+    expected_result = "df.loc[(df.column1 > '10')&(df.column2 < '20')]"
+    assert result == expected_result

viadot/flows/__init__.py

@@ -11,7 +11,7 @@
 from .genesys_to_adls import GenesysToADLS
 from .outlook_to_adls import OutlookToADLS
 from .salesforce_to_adls import SalesforceToADLS
-from .sharepoint_to_adls import SharepointToADLS
+from .sharepoint_to_adls import SharepointToADLS, SharepointListToADLS
 from .supermetrics_to_adls import SupermetricsToADLS
 from .supermetrics_to_azure_sql import SupermetricsToAzureSQL