Sharepoint list connector extension for multichoice fields with some …

…small fixes and docstring update
dyvenia · Nov 14, 2023 · 3991b5f · 3991b5f
1 parent ac7f63b
commit 3991b5f
Show file tree

Hide file tree

Showing 5 changed files with 457 additions and 143 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+- Modified `SharepointList` source class: 
+  -> docstrings update
+- Modified `SharepointToADLS` flow class:
+  -> docstrings update
+  -> changed key_value_param: bool = False to prevent forced KV store append
+- Modified `SharepointListToADLS` flow class:
+  -> changed key_value_param: bool = False to prevent forced KV store append
+- Modified `SharepointList` source class:
+  -> docstrings update
+  -> Changed `_unpack_fields` method to handle Sharepoint MultiChoiceField type + small improvements
+  -> Changed `get_fields` method to handle special characters - different approach to call get() and execute_query()
+  -> Renamed method from `select_expandable_user_fields` to `select_fields` + update for MultiChoiceField type
+  -> Changed `check_filters` method errors messages and more checks added
+  -> Changed `operators_mapping` method errors messages
+  -> Changed `make_filter_for_df` method errors messages
+- Modified `SharepointListToDF` task class:
+  -> docstrings update
+  -> Added `_rename_duplicated_fields` method to find and rename duplicated columns
+
 
 ## [0.4.21] - 2023-10-26
 ### Added

diff --git a/tests/integration/test_sharepoint.py b/tests/integration/test_sharepoint.py
@@ -2,15 +2,14 @@
 import re
 
 import pandas as pd
-from copy import deepcopy
 import pytest
 from prefect.tasks.secrets import PrefectSecret
 
 from viadot.config import local_config
 from viadot.exceptions import CredentialError
 from viadot.sources import Sharepoint
 from viadot.task_utils import df_get_data_types_task
-from viadot.tasks.sharepoint import SharepointToDF
+from viadot.tasks.sharepoint import SharepointToDF, SharepointListToDF
 from viadot.sources import SharepointList
 
 
@@ -168,10 +167,11 @@ def test_get_data_types(file_name):
     assert "String" in dtypes
 
 
+### SECTION FOR TESTING SHAREPOINT LIST CONNECTOR ###
 @pytest.fixture(scope="session")
 def sharepoint_list():
     """
-    Fixture for creating a Sharepoint class instance.
+    Fixture for creating a SharepointList class instance.
     The class instance can be used within a test functions to interact with Sharepoint.
     """
     spl = SharepointList()
@@ -187,39 +187,60 @@ def test_valid_filters(sharepoint_list):
     assert result is True
 
 
-def test_invalid_dtype(sharepoint_list):
+def test_filters_missing_dtype(sharepoint_list):
+    filters = {
+        "filter1": {"operator1": ">", "value1": 10},
+    }
+    with pytest.raises(
+        ValueError,
+        match=re.escape("dtype for filter1 is missing!"),
+    ):
+        sharepoint_list.check_filters(filters)
+
+
+def test_filters_invalid_dtype(sharepoint_list):
     filters = {
         "filter1": {"dtype": "list", "operator1": ">", "value1": 10},
     }
-    with pytest.raises(ValueError, match="dtype not allowed!"):
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "dtype not allowed! Expected: ['datetime', 'date', 'bool', 'int', 'float', 'complex', 'str'] got: list ."
+        ),
+    ):
         sharepoint_list.check_filters(filters)
 
 
-def test_missing_operator1(sharepoint_list):
+def test_filters_missing_operator1(sharepoint_list):
     filters = {
         "filter1": {"dtype": "int", "value1": 10},
     }
     with pytest.raises(ValueError, match="Operator1 is missing!"):
         sharepoint_list.check_filters(filters)
 
 
-def test_invalid_operator1(sharepoint_list):
+def test_filters_invalid_operator1(sharepoint_list):
     filters = {
         "filter1": {"dtype": "int", "operator1": "*", "value1": 10},
     }
-    with pytest.raises(ValueError, match="Operator type not allowed!"):
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Operator1 type not allowed! Expected: ['<', '>', '<=', '>=', '==', '!='] got: * ."
+        ),
+    ):
         sharepoint_list.check_filters(filters)
 
 
-def test_missing_value1(sharepoint_list):
+def test_filters_missing_value1(sharepoint_list):
     filters = {
         "filter1": {"dtype": "int", "operator1": ">", "value1": None},
     }
-    with pytest.raises(ValueError, match="Value for operator1 is missing!"):
+    with pytest.raises(ValueError, match="Value1 for operator1 is missing!"):
         sharepoint_list.check_filters(filters)
 
 
-def test_missing_operators_conjuction(sharepoint_list):
+def test_filters_missing_operators_conjuction(sharepoint_list):
     filters = {
         "filter1": {
             "dtype": "int",
@@ -229,11 +250,16 @@ def test_missing_operators_conjuction(sharepoint_list):
             "value2": 20,
         },
     }
-    with pytest.raises(ValueError, match="Operators for conjuction is missing!"):
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Operator for conjuction is missing! Expected: ['&', '|'] got empty."
+        ),
+    ):
         sharepoint_list.check_filters(filters)
 
 
-def test_invalid_operators_conjuction(sharepoint_list):
+def test_filters_invalid_operators_conjuction(sharepoint_list):
     filters = {
         "filter1": {
             "dtype": "int",
@@ -244,11 +270,16 @@ def test_invalid_operators_conjuction(sharepoint_list):
             "operators_conjuction": "!",
         },
     }
-    with pytest.raises(ValueError, match="Operators for conjuction not allowed!"):
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Operator for conjuction not allowed! Expected: ['&', '|'] got ! ."
+        ),
+    ):
         sharepoint_list.check_filters(filters)
 
 
-def test_invalid_filters_conjuction(sharepoint_list):
+def test_filters_conjuction_not_allowed(sharepoint_list):
     filters = {
         "filter1": {
             "dtype": "int",
@@ -258,35 +289,80 @@ def test_invalid_filters_conjuction(sharepoint_list):
         },
     }
     with pytest.raises(
-        ValueError, match="Filters operators for conjuction not allowed!"
+        ValueError,
+        match=re.escape(
+            "Filters conjuction allowed only when more then one filter provided!"
+        ),
+    ):
+        sharepoint_list.check_filters(filters)
+
+
+def test_filters_invalid_conjuction(sharepoint_list):
+    filters = {
+        "filter1": {
+            "dtype": "int",
+            "value1": 10,
+            "operator1": ">",
+            "filters_conjuction": "!",
+        },
+        "filter2": {
+            "dtype": "int",
+            "operator1": "==",
+        },
+    }
+    with pytest.raises(
+        ValueError,
+        match=re.escape(
+            "Filter operator for conjuction not allowed! Expected: ['&', '|'] got ! ."
+        ),
     ):
         sharepoint_list.check_filters(filters)
 
 
 def test_valid_mapping(sharepoint_list):
     filters = {
         "filter1": {
+            "dtype": "int",
+            "value1": 10,
+            "value2": 20,
             "operator1": ">",
             "operator2": "<=",
             "operators_conjuction": "&",
             "filters_conjuction": "|",
         },
-        "filter2": {"operator1": "==", "operator2": "!=", "operators_conjuction": "|"},
+        "filter2": {
+            "dtype": "int",
+            "value1": 30,
+            "value2": 0,
+            "operator1": "==",
+            "operator2": "!=",
+            "operators_conjuction": "|",
+        },
     }
     expected_result = {
         "filter1": {
+            "dtype": "int",
+            "value1": 10,
+            "value2": 20,
             "operator1": "gt",
             "operator2": "le",
             "operators_conjuction": "and",
             "filters_conjuction": "or",
         },
-        "filter2": {"operator1": "eq", "operator2": "ne", "operators_conjuction": "or"},
+        "filter2": {
+            "dtype": "int",
+            "value1": 30,
+            "value2": 0,
+            "operator1": "eq",
+            "operator2": "ne",
+            "operators_conjuction": "or",
+        },
     }
-    result = sharepoint_list.operators_mapping(deepcopy(filters))
+    result = sharepoint_list.operators_mapping(filters)
     assert result == expected_result
 
 
-def test_invalid_comparison_operator(sharepoint_list):
+def test_operators_mapping_invalid_comparison_operator(sharepoint_list):
     filters = {
         "filter1": {
             "operator1": "*",
@@ -297,10 +373,10 @@ def test_invalid_comparison_operator(sharepoint_list):
     }
     error_message = "This comparison operator: * is not allowed. Please read the function documentation for details!"
     with pytest.raises(ValueError, match=re.escape(error_message)):
-        sharepoint_list.operators_mapping(deepcopy(filters))
+        sharepoint_list.operators_mapping(filters)
 
 
-def test_invalid_logical_operator(sharepoint_list):
+def test_operators_mapping_invalid_logical_operator(sharepoint_list):
     filters = {
         "filter1": {
             "operator1": ">",
@@ -309,9 +385,23 @@ def test_invalid_logical_operator(sharepoint_list):
             "filters_conjuction": "|",
         },
     }
-    error_message = "This conjuction(logical) operator: ! is not allowed. Please read the function documentation for details!"
+    error_message = "This conjuction (logical) operator: ! is not allowed. Please read the function documentation for details!"
+    with pytest.raises(ValueError, match=re.escape(error_message)):
+        sharepoint_list.operators_mapping(filters)
+
+
+def test_operators_mapping_invalid_filters_logical_operator(sharepoint_list):
+    filters = {
+        "filter1": {
+            "operator1": ">",
+            "operator2": "<=",
+            "operators_conjuction": "&",
+            "filters_conjuction": "!",
+        },
+    }
+    error_message = "This filters conjuction (logical) operator: ! is not allowed. Please read the function documentation for details!"
     with pytest.raises(ValueError, match=re.escape(error_message)):
-        sharepoint_list.operators_mapping(deepcopy(filters))
+        sharepoint_list.operators_mapping(filters)
 
 
 def test_single_filter_datetime_api(sharepoint_list):