✨ new connector for sharepoint_list

dyvenia · Oct 26, 2023 · b3c8915 · b3c8915
1 parent eb86bdf
commit b3c8915
Show file tree

Hide file tree

Showing 9 changed files with 970 additions and 18 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - Added `validate_df` task to task_utils.
+- Added `SharepointList` source class.
+- Added `SharepointListToDF` task class.
+- Added `SharepointListToADLS` flow class.
+- Added tests for `SharepointList`.
+- Added `get_nested_dict` to untils.py.
+- Added `validate_df` task to `SharepointToADLS` class.
+
 ### Fixed
 
 ### Changed
@@ -618,4 +625,4 @@ specified in the `SUPERMETRICS_DEFAULT_USER` secret
 - Moved from poetry to pip
 
 ### Fixed
-- Fix `AzureBlobStorage`'s `to_storage()` method is missing the final upload blob part
+- Fix `AzureBlobStorage`'s `to_storage()` method is missing the final upload blob part
diff --git a/tests/integration/test_sharepoint.py b/tests/integration/test_sharepoint.py
@@ -1,6 +1,8 @@
 import os
+import re
 
 import pandas as pd
+from copy import deepcopy
 import pytest
 from prefect.tasks.secrets import PrefectSecret
 
@@ -9,6 +11,7 @@
 from viadot.sources import Sharepoint
 from viadot.task_utils import df_get_data_types_task
 from viadot.tasks.sharepoint import SharepointToDF
+from viadot.sources import SharepointList
 
 
 def get_url() -> str:
@@ -18,7 +21,7 @@ def get_url() -> str:
     Returns:
         str: File URL.
     """
-    return local_config["SHAREPOINT"].get("url")
+    return local_config["SHAREPOINT"].get("file_url")
 
 
 @pytest.fixture(scope="session")
@@ -163,3 +166,150 @@ def test_get_data_types(file_name):
     dtypes = dtypes_map.values()
 
     assert "String" in dtypes
+
+
+# testing get_connection function passing invalid credentials and raises AuthenticationContext error.
+def test_get_connection():
+    site_url = "https://velux.sharepoint.com/"
+    credentials ={
+        "SHAREPOINT_CERT":
+            {"TENANT": "xxx",
+            "CLIENT_ID": "123",
+            "SCOPES": "https://velux.sharepoint.com/",
+            "THUMBPRINT": "xyz",
+            "PRIVATE_KEY": "private"}
+            }
+
+    spl = SharepointList(credentials=credentials)
+    with pytest.raises(AttributeError, match="'SharepointList' object has no attribute 'ctx'"):
+        spl.get_connection(site_url=site_url)
+
+
+@pytest.fixture(scope="session")
+def sharepoint_list():
+    """
+    Fixture for creating a Sharepoint class instance.
+    The class instance can be used within a test functions to interact with Sharepoint.
+    """
+    spl = SharepointList()
+    yield spl
+
+
+def test_valid_filters(sharepoint_list):
+    filters = {
+        'filter1': {'dtype': 'int', 'operator1': '<', 'value1': 10},
+        'filter2': {'dtype': 'str', 'operator1': '==', 'value1': 'value'},
+    }
+    result = sharepoint_list.check_filters(filters)
+    assert result is True
+
+def test_invalid_dtype(sharepoint_list):
+    filters = {
+        'filter1': {'dtype': 'list', 'operator1': '>', 'value1': 10},
+    }
+    with pytest.raises(ValueError, match="dtype not allowed!"):
+        sharepoint_list.check_filters(filters)
+
+def test_missing_operator1(sharepoint_list):
+    filters = {
+        'filter1': {'dtype': 'int', 'value1': 10},
+    }
+    with pytest.raises(ValueError, match="Operator1 is missing!"):
+        sharepoint_list.check_filters(filters)
+
+def test_invalid_operator1(sharepoint_list):
+    filters = {
+        'filter1': {'dtype': 'int', 'operator1': '*', 'value1': 10},
+    }
+    with pytest.raises(ValueError, match="Operator type not allowed!"):
+        sharepoint_list.check_filters(filters)
+
+def test_missing_value1(sharepoint_list):
+    filters = {
+        'filter1': {'dtype': 'int', 'operator1': '>', 'value1': None},
+    }
+    with pytest.raises(ValueError, match="Value for operator1 is missing!"):
+        sharepoint_list.check_filters(filters)
+
+def test_missing_operators_conjuction(sharepoint_list):
+    filters = {
+        'filter1': {'dtype': 'int', 'operator1': '>', 'value1': 10, 'operator2': '<', 'value2': 20},
+    }
+    with pytest.raises(ValueError, match="Operators for conjuction is missing!"):
+        sharepoint_list.check_filters(filters)
+
+def test_invalid_operators_conjuction(sharepoint_list):
+    filters = {
+        'filter1': {'dtype': 'int', 'operator1': '>', 'value1': 10, 'operator2': '<', 'value2': 20, 'operators_conjuction': '!'},
+    }
+    with pytest.raises(ValueError, match="Operators for conjuction not allowed!"):
+        sharepoint_list.check_filters(filters)
+
+def test_invalid_filters_conjuction(sharepoint_list):
+    filters = {
+        'filter1': {'dtype': 'int', 'operator1': '>', 'value1': 10, 'filters_conjuction': '!'},
+    }
+    with pytest.raises(ValueError, match="Filters operators for conjuction not allowed!"):
+        sharepoint_list.check_filters(filters)
+
+def test_valid_mapping(sharepoint_list):
+    filters = {
+        'filter1': {'operator1': '>', 'operator2': '<=', 'operators_conjuction': '&', 'filters_conjuction': '|'},
+        'filter2': {'operator1': '==', 'operator2': '!=', 'operators_conjuction': '|'},
+    }
+    expected_result = {
+        'filter1': {'operator1': 'gt', 'operator2': 'le', 'operators_conjuction': 'and', 'filters_conjuction': 'or'},
+        'filter2': {'operator1': 'eq', 'operator2': 'ne', 'operators_conjuction': 'or'},
+    }
+    result = sharepoint_list.operators_mapping(deepcopy(filters))
+    assert result == expected_result
+
+def test_invalid_comparison_operator(sharepoint_list):
+    filters = {
+        'filter1': {'operator1': '*', 'operator2': '<=', 'operators_conjuction': '&', 'filters_conjuction': '|'},
+    }
+    error_message = "This comparison operator: * is not allowed. Please read the function documentation for details!"
+    with pytest.raises(ValueError, match=re.escape(error_message)):
+        sharepoint_list.operators_mapping(deepcopy(filters))
+
+def test_invalid_logical_operator(sharepoint_list):
+    filters = {
+        'filter1': {'operator1': '>', 'operator2': '<=', 'operators_conjuction': '!', 'filters_conjuction': '|'},
+    }
+    error_message = "This conjuction(logical) operator: ! is not allowed. Please read the function documentation for details!"
+    with pytest.raises(ValueError, match=re.escape(error_message)):
+        sharepoint_list.operators_mapping(deepcopy(filters))
+
+def test_single_filter_datetime_api(sharepoint_list):
+    filters = {
+        'date_column': {'dtype': 'datetime', 'operator1': '>', 'value1': '2023-01-01'}
+    }
+    result = sharepoint_list.make_filter_for_api(filters)
+    expected_result = "date_column gt datetime'2023-01-01T00:00:00' "
+    assert result == expected_result
+
+def test_multiple_filters_api(sharepoint_list):
+    filters = {
+        'int_column': {'dtype': 'int', 'operator1': '>=', 'value1': 10, 'operator2': '<', 'value2': 20},
+        'str_column': {'dtype': 'str', 'operator1': '==', 'value1': 'example'}
+    }
+    result = sharepoint_list.make_filter_for_api(filters)
+    expected_result = "int_column ge '10'int_column lt '20'str_column eq 'example'"
+    assert result == expected_result
+
+def test_single_df_filter(sharepoint_list):
+    filters = {
+        'column1': {'operator1': '>', 'value1': 10}
+    }
+    result = sharepoint_list.make_filter_for_df(filters)
+    expected_result = "df.loc[(df.column1 > '10')]"
+    assert result == expected_result
+
+def test_multiple_df_filters(sharepoint_list):
+    filters = {
+        'column1': {'operator1': '>', 'value1': 10, 'filters_conjuction': '&'},
+        'column2': {'operator1': '<', 'value1': 20}
+    }
+    result = sharepoint_list.make_filter_for_df(filters)
+    expected_result = "df.loc[(df.column1 > '10')&(df.column2 < '20')]"
+    assert result == expected_result
diff --git a/viadot/flows/__init__.py b/viadot/flows/__init__.py
@@ -11,7 +11,7 @@
 from .genesys_to_adls import GenesysToADLS
 from .outlook_to_adls import OutlookToADLS
 from .salesforce_to_adls import SalesforceToADLS
-from .sharepoint_to_adls import SharepointToADLS
+from .sharepoint_to_adls import SharepointToADLS, SharepointListToADLS
 from .supermetrics_to_adls import SupermetricsToADLS
 from .supermetrics_to_azure_sql import SupermetricsToAzureSQL
 
@@ -46,4 +46,4 @@
 from .sql_server_to_parquet import SQLServerToParquet
 from .sql_server_transform import SQLServerTransform
 from .transform_and_catalog import TransformAndCatalogToLuma
-from .vid_club_to_adls import VidClubToADLS
+from .vid_club_to_adls import VidClubToADLS