Merge pull request #205 from dyvenia/dev

Release 0.2.12

CHANGELOG.md

@@ -4,8 +4,43 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
 ## [Unreleased]
 
+## [0.2.12]
+### Added
+- Added `Sharepoint` source
+- Added `SharepointToDF` task
+- Added `SharepointToADLS` flow
+- Added `CloudForCustomers` source
+- Added `c4c_report_to_df` taks
+- Added `def c4c_to_df` task
+- Added `CloudForCustomersReportToADLS` flow
+- Added `df_to_csv` task to task_utils.py
+- Added `df_to_parquet` task to task_utils.py
+- Added `dtypes_to_json` task to task_utils.py
+## [0.2.11]
+### Fixed
+- `ADLSToAzureSQL` - fixed path to csv issue. 
+- `SupermetricsToADLS` - fixed local json path issue. 
+
+## [0.2.10] - 2021-10-29
+### Release due to CI/CD error
+
+## [0.2.9] - 2021-10-29
+### Release due to CI/CD error
+
+## [0.2.8] - 2021-10-29
+### Changed
+- CI/CD: `dev` image is now only published on push to the `dev` branch
+- Docker: 
+  - updated registry links to use the new `ghcr.io` domain
+  - `run.sh` now also accepts the `-t` option. When run in standard mode, it will only spin up the `viadot_jupyter_lab` service.
+  When ran with `-t dev`, it will also spin up `viadot_testing` and `viadot_docs` containers.
+
+### Fixed
+- `ADLSToAzureSQL` - fixed path parameter issue.
+
 ## [0.2.11]
 ### Fixed
 - ADLSToAzureSQL - fixed path to csv issue. 

requirements.txt

@@ -24,3 +24,4 @@ PyGithub==1.55
 Shapely==1.8.0
 imagehash==4.2.1
 visions==0.7.4
+sharepy==1.3.0

tests/integration/flows/test_cloud_for_customers_report_to_adls.py

@@ -0,0 +1,29 @@
+from viadot.flows import CloudForCustomersReportToADLS
+from viadot.config import local_config
+
+
+def test_cloud_for_customers_report_to_adls():
+    credentials = local_config.get("CLOUD_FOR_CUSTOMERS")
+    credentials_prod = credentials["Prod"]
+    channels = ["VEL_B_AFS", "VEL_B_ASA"]
+    month = ["01"]
+    year = ["2021"]
+    flow = CloudForCustomersReportToADLS(
+        direct_url=credentials_prod["server"],
+        source_type="Prod",
+        channels=channels,
+        months=month,
+        years=year,
+        name="test_c4c_report_to_adls",
+        local_file_path=f"test_c4c_report_to_adls.csv",
+        adls_sp_credentials_secret=credentials["adls_sp_credentials_secret"],
+        adls_dir_path=credentials["adls_dir_path"],
+    )
+    number_of_urls = len(month) * len(year) * len(channels)
+    assert len(flow.report_urls_with_filters) == number_of_urls
+
+    result = flow.run()
+    assert result.is_successful()
+
+    task_results = result.result.values()
+    assert all([task_result.is_successful() for task_result in task_results])

tests/integration/test_cloud_for_customers.py

@@ -13,7 +13,9 @@
 def cloud_for_customers():
     url = "http://services.odata.org/V2/Northwind/Northwind.svc/"
     endpoint = "Employees"
-    cloud_for_customers = CloudForCustomers(url=url, endpoint=endpoint)
+    cloud_for_customers = CloudForCustomers(
+        url=url, endpoint=endpoint, params={"$top": "2"}
+    )
     yield cloud_for_customers
     os.remove(TEST_FILE_1)
 
@@ -36,10 +38,10 @@ def test_csv(cloud_for_customers):
 
 
 def test_credentials():
-    credentials = local_config.get("CLOUD_FOR_CUSTOMERS")
-    url = credentials["server"]
+    qa_credentials = local_config.get("CLOUD_FOR_CUSTOMERS")["QA"]
+    url = qa_credentials["server"]
     endpoint = "ServiceRequestCollection"
-    c4c = CloudForCustomers(url=url, endpoint=endpoint)
+    c4c = CloudForCustomers(url=url, endpoint=endpoint, params={"$top": "2"})
     df = c4c.to_df(
         fields=["ProductRecipientPartyName", "CreationDateTime", "CreatedBy"]
     )

tests/integration/test_sharepoint.py

@@ -0,0 +1,61 @@
+import pytest
+import os
+import pathlib
+import pandas as pd
+from typing import List
+
+from viadot.sources import Sharepoint
+from viadot.flows import SharepointToADLS as s_flow
+from viadot.config import local_config
+from viadot.task_utils import df_get_data_types_task
+
+s = Sharepoint()
+
+FILE_NAME = "EUL Data.xlsm"
+s.download_file(download_to_path=FILE_NAME)
+DF = pd.read_excel(FILE_NAME, sheet_name=0)
+
+
+def test_credentials():
+    credentials = {"site": "tenant.sharepoint.com", "username": "User"}
+    s = Sharepoint(credentials=credentials)
+    with pytest.raises(ValueError, match="Missing credentials."):
+        s.get_connection()
+
+
+def test_connection():
+    credentials = local_config.get("SHAREPOINT")
+    site = f'https://{credentials["site"]}'
+    conn = s.get_connection()
+    response = conn.get(site)
+    assert response.status_code == 200
+
+
+def test_file_extension():
+    file_ext = [".xlsm", ".xlsx"]
+    assert pathlib.Path(s.download_from_path).suffix in file_ext
+
+
+def test_file_name():
+    assert os.path.basename(s.download_from_path) == FILE_NAME
+
+
+def test_file_download():
+    s.download_file(download_to_path=FILE_NAME)
+    files = []
+    for file in os.listdir():
+        if os.path.isfile(os.path.join(file)):
+            files.append(file)
+    assert FILE_NAME in files
+    os.remove(FILE_NAME)
+
+
+def test_file_to_df():
+    df_test = pd.DataFrame(data={"col1": [1, 2]})
+    assert type(DF) == type(df_test)
+
+
+def test_get_data_types():
+    dtypes_map = df_get_data_types_task.run(DF)
+    dtypes = [v for k, v in dtypes_map.items()]
+    assert "String" in dtypes

tests/test_viadot.py

@@ -2,4 +2,4 @@
 
 
 def test_version():
-    assert __version__ == "0.2.11"
+    assert __version__ == "0.2.12"

tests/unit/test_task_utils.py

@@ -0,0 +1,34 @@
+import pytest
+import pandas as pd
+from typing import List
+
+from viadot.task_utils import df_get_data_types_task, df_map_mixed_dtypes_for_parquet
+
+
+def count_dtypes(dtypes_dict: dict = None, dtypes_to_count: List[str] = None) -> int:
+    dtypes_counter = 0
+    for v in dtypes_dict.values():
+        if v in dtypes_to_count:
+            dtypes_counter += 1
+    return dtypes_counter
+
+
+def test_map_dtypes_for_parquet():
+    df = pd.DataFrame(
+        {
+            "a": {0: 55.7, 1: "Hello", 2: "Hello"},
+            "b": {0: "Start", 1: "Hello", 2: "Hello"},
+            "w": {0: 679, 1: "Hello", 2: "Hello"},
+            "x": {0: 1, 1: 2, 2: 444},
+            "y": {0: 1.5, 1: 11.97, 2: 56.999},
+            "z": {0: "Start", 1: 1, 2: "2021-01-01"},
+        }
+    )
+    dtyps_dict = df_get_data_types_task.run(df)
+    sum_of_dtypes = count_dtypes(dtyps_dict, ["Object", "String"])
+
+    df_map = df_map_mixed_dtypes_for_parquet.run(df, dtyps_dict)
+    dtyps_dict_mapped = df_get_data_types_task.run(df_map)
+    sum_of_mapped_dtypes = count_dtypes(dtyps_dict_mapped, ["String"])
+
+    assert sum_of_dtypes == sum_of_mapped_dtypes

viadot/__init__.py

@@ -1 +1 @@
-__version__ = "0.2.11"
+__version__ = "0.2.12"

viadot/exceptions.py

@@ -0,0 +1,6 @@
+class ValidationError(Exception):
+    pass
+
+
+class APIError(Exception):
+    pass

viadot/flows/__init__.py

@@ -7,4 +7,5 @@
 from .supermetrics_to_adls import SupermetricsToADLS
 from .supermetrics_to_azure_sql import SupermetricsToAzureSQL
 from .adls_container_to_container import ADLSContainerToContainer
-from .cloud_for_customers_to_adls import CloudForCustomersToADLS
+from .sharepoint_to_adls import SharepointToADLS
+from .cloud_for_customers_report_to_adls import CloudForCustomersReportToADLS