From 3c70f10f4457413071aa9167d8bc15908c1e61a6 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Mon, 7 Nov 2022 14:13:54 +0100 Subject: [PATCH 01/27] =?UTF-8?q?=E2=9C=A8=20added=20agent=20summary=20vie?= =?UTF-8?q?w=20in=20Genesys.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 55 ++++++++++++++++-------- viadot/sources/genesys.py | 39 ++++++++++++++--- viadot/tasks/genesys.py | 75 +++++++++++++++++++++++---------- 3 files changed, 123 insertions(+), 46 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 9e71131fb..25472dd03 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -63,6 +63,9 @@ class GenesysToADLS(Flow): def __init__( self, name: str, + view_type: Literal[ + "queue_performance_detail", "agent_performance_summery_view" + ] = "queue_performance_detail", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -87,6 +90,8 @@ def __init__( Args: name (str): The name of the Flow. + view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -108,6 +113,7 @@ def __init__( """ # GenesysToCSV self.flow_name = name + self.view_type = view_type self.media_type_list = media_type_list self.queueIds_list = queueIds_list self.data_to_post = data_to_post_str @@ -134,29 +140,42 @@ def gen_flow(self) -> Flow: to_csv = GenesysToCSV() - file_names = to_csv.bind( - media_type_list=self.media_type_list, - queueIds_list=self.queueIds_list, - data_to_post_str=self.data_to_post, - start_date=self.start_date, - end_date=self.end_date, - days_interval=self.days_interval, - environment=self.environment, - credentials_genesys=self.credentials_genesys, - flow=self, - ) + if self.view_type == "queue_performance_detail": + file_names = to_csv.bind( + view_type=self.view_type, + media_type_list=self.media_type_list, + queueIds_list=self.queueIds_list, + data_to_post_str=self.data_to_post, + start_date=self.start_date, + end_date=self.end_date, + days_interval=self.days_interval, + environment=self.environment, + credentials_genesys=self.credentials_genesys, + flow=self, + ) + elif self.view_type == "agent_performance_summery_view": + file_names = to_csv.bind( + view_type=self.view_type, + media_type_list=self.media_type_list, + queueIds_list=[""], + data_to_post_str=self.data_to_post, + start_date=self.start_date, + environment=self.environment, + credentials_genesys=self.credentials_genesys, + flow=self, + ) add_timestamp.bind(file_names, sep=self.sep, flow=self) - uploader = adls_bulk_upload( - file_names=file_names, - adls_file_path=self.adls_file_path, - adls_sp_credentials_secret=self.adls_sp_credentials_secret, - flow=self, - ) + # uploader = adls_bulk_upload( + # file_names=file_names, + # adls_file_path=self.adls_file_path, + # adls_sp_credentials_secret=self.adls_sp_credentials_secret, + # flow=self, + # ) add_timestamp.set_upstream(file_names, flow=self) - uploader.set_upstream(add_timestamp, flow=self) + # uploader.set_upstream(add_timestamp, flow=self) class GenesysReportToADLS(Flow): diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 27207a1c7..810cc0aa8 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -1,4 +1,4 @@ -import json +import json, sys import base64 import warnings import asyncio @@ -23,6 +23,9 @@ class Genesys(Source): def __init__( self, + view_type: Literal[ + "queue_performance_detail", "agent_performance_summery_view" + ] = "queue_performance_detail", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -44,6 +47,8 @@ def __init__( Genesys connector which allows for reports scheduling, listing and downloading into Data Frame or specified format output. Args: + view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -81,6 +86,7 @@ def __init__( super().__init__(*args, credentials=self.credentials_genesys, **kwargs) + self.view_type = view_type self.schedule_id = schedule_id self.report_name = report_name self.environment = environment @@ -280,13 +286,16 @@ def get_reporting_exports_data(self): tmp = [ entity.get("id"), entity.get("downloadUrl"), - entity.get("filter").get("queueIds")[0], - entity.get("filter").get("mediaTypes")[0], + entity.get("filter").get("queueIds", [-1])[0], + entity.get("filter").get("mediaTypes", [-1])[0], + entity.get("status"), ] self.report_data.append(tmp) assert len(self.report_data) > 0 self.logger.info("Generated list of reports entities.") + print(self.report_data) + def download_report( self, report_url: str, @@ -343,9 +352,27 @@ def download_all_reporting_exports( self.logger.info("IDS_MAPPING loaded from local credential.") for single_report in self.report_data: - file_name = ( - temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] - ).upper() + if single_report[-1] == "RUNNING": + self.logger.warning( + "The request is still in progress, consider add more seconds to `view_type_time_sleep` parameter." + ) + continue + elif single_report[-1] == "FAILED": + self.logger.warning( + "This message 'FAILED_GETTING_DATA_FROM_SERVICE' raised during script execution." + ) + continue + if self.view_type == "queue_performance_detail": + file_name = ( + temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] + ).upper() + elif self.view_type == "agent_performance_summery_view": + file_name = self.view_type.upper() + "_" + f"{single_report[0][:8]}" + else: + self.logger.error( + f"View type {self.view_type} not defined in viador, yet..." + ) + self.download_report( report_url=single_report[1], output_file_name=file_name, diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index 5bf0d2f4a..af4df0dba 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -1,5 +1,5 @@ import time -from typing import Any, Dict, List +from typing import Any, Dict, List, Literal import pandas as pd import numpy as np @@ -18,6 +18,9 @@ class GenesysToCSV(Task): def __init__( self, report_name: str = "genesys_to_csv", + view_type: Literal[ + "queue_performance_detail", "agent_performance_summery_view" + ] = "queue_performance_detail", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -36,6 +39,8 @@ def __init__( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. + view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -53,6 +58,7 @@ def __init__( self.logger = prefect.context.get("logger") self.schedule_id = schedule_id self.report_name = report_name + self.view_type = view_type self.environment = environment self.report_url = report_url self.report_columns = report_columns @@ -76,6 +82,7 @@ def __call__(self, *args, **kwargs): @defaults_from_attrs( "report_name", + "view_type", "environment", "schedule_id", "report_url", @@ -91,6 +98,10 @@ def __call__(self, *args, **kwargs): def run( self, report_name: str = None, + view_type: Literal[ + "queue_performance_detail", "agent_performance_summery_view" + ] = "queue_performance_detail", + view_type_time_sleep: int = 80, environment: str = None, schedule_id: str = None, report_url: str = None, @@ -108,6 +119,8 @@ def run( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. + view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -127,6 +140,7 @@ def run( genesys = Genesys( report_name=report_name, + view_type=view_type, media_type_list=media_type_list, queueIds_list=queueIds_list, data_to_post_str=data_to_post_str, @@ -142,29 +156,46 @@ def run( genesys.genesys_generate_body() genesys.genesys_generate_exports() - logger.info(f"Waiting for caching data in Genesys database.") - - # in order to wait for API POST request add it - timeout_start = time.time() - # 30 seconds timeout is minimal but for safety added 60. - timeout = timeout_start + 60 - # while loop with timeout - while time.time() < timeout: - - try: - genesys.get_reporting_exports_data() - urls = [col for col in np.array(genesys.report_data).T][1] - if None in urls: - logger.warning("Found None object in list of urls.") - else: - break - except TypeError: - pass - - # There is a need to clear a list before repeating try statement. - genesys.report_data.clear() + + if view_type == "queue_performance_detail": + logger.info(f"Waiting for caching data in Genesys database.") + # in order to wait for API POST request add it + timeout_start = time.time() + # 30 seconds timeout is minimal but for safety added 60. + timeout = timeout_start + 60 + # while loop with timeout + while time.time() < timeout: + + try: + genesys.get_reporting_exports_data() + urls = [col for col in np.array(genesys.report_data).T][1] + if None in urls: + logger.warning("Found None object in list of urls.") + else: + break + except TypeError: + pass + + # There is a need to clear a list before repeating try statement. + genesys.report_data.clear() + + elif view_type == "agent_performance_summery_view": + logger.info( + f"Waiting for caching data in Genesys database ({view_type_time_sleep} seconds)." + ) + time.sleep(view_type_time_sleep) + + genesys.get_reporting_exports_data() + failed = [col for col in np.array(genesys.report_data).T][-1] + print(failed) + if "FAILED" in failed and "COMPLETED" in failed: + logger.warning("Some reports failed.") + elif np.unique(failed)[0] == "FAILED": + genesys.report_data.clear() if len(genesys.report_data) == 0: + genesys.delete_all_reporting_exports() + logger.warning(f"All existing reports were delted.") raise APIError("No exporting reports were generated.") elif not None in [col for col in np.array(genesys.report_data).T][1]: logger.info("Downloaded the data from the Genesys into the CSV.") From 8b52cc5821c2786dff2bb33e8229c6bd91352c15 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Mon, 7 Nov 2022 16:44:00 +0100 Subject: [PATCH 02/27] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20improved=20performan?= =?UTF-8?q?ce=20of=20Genesys.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 18 +++++++++++------- viadot/sources/genesys.py | 5 ++--- viadot/tasks/genesys.py | 9 ++++----- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 25472dd03..7e81714dd 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -66,6 +66,7 @@ def __init__( view_type: Literal[ "queue_performance_detail", "agent_performance_summery_view" ] = "queue_performance_detail", + view_type_time_sleep: int = 80, media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -92,6 +93,7 @@ def __init__( name (str): The name of the Flow. view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail". + view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -114,6 +116,7 @@ def __init__( # GenesysToCSV self.flow_name = name self.view_type = view_type + self.view_type_time_sleep = view_type_time_sleep self.media_type_list = media_type_list self.queueIds_list = queueIds_list self.data_to_post = data_to_post_str @@ -156,6 +159,7 @@ def gen_flow(self) -> Flow: elif self.view_type == "agent_performance_summery_view": file_names = to_csv.bind( view_type=self.view_type, + view_type_time_sleep=self.view_type_time_sleep, media_type_list=self.media_type_list, queueIds_list=[""], data_to_post_str=self.data_to_post, @@ -167,15 +171,15 @@ def gen_flow(self) -> Flow: add_timestamp.bind(file_names, sep=self.sep, flow=self) - # uploader = adls_bulk_upload( - # file_names=file_names, - # adls_file_path=self.adls_file_path, - # adls_sp_credentials_secret=self.adls_sp_credentials_secret, - # flow=self, - # ) + uploader = adls_bulk_upload( + file_names=file_names, + adls_file_path=self.adls_file_path, + adls_sp_credentials_secret=self.adls_sp_credentials_secret, + flow=self, + ) add_timestamp.set_upstream(file_names, flow=self) - # uploader.set_upstream(add_timestamp, flow=self) + uploader.set_upstream(add_timestamp, flow=self) class GenesysReportToADLS(Flow): diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 810cc0aa8..5ddf48180 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -294,8 +294,6 @@ def get_reporting_exports_data(self): assert len(self.report_data) > 0 self.logger.info("Generated list of reports entities.") - print(self.report_data) - def download_report( self, report_url: str, @@ -367,7 +365,8 @@ def download_all_reporting_exports( temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] ).upper() elif self.view_type == "agent_performance_summery_view": - file_name = self.view_type.upper() + "_" + f"{single_report[0][:8]}" + date = self.start_date.replace("-", "") + file_name = self.view_type.upper() + "_" + f"{date}" else: self.logger.error( f"View type {self.view_type} not defined in viador, yet..." diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index af4df0dba..069cbcf1d 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -121,6 +121,7 @@ def run( report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail". + view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -181,19 +182,17 @@ def run( elif view_type == "agent_performance_summery_view": logger.info( - f"Waiting for caching data in Genesys database ({view_type_time_sleep} seconds)." + f"Waiting for getting data in Genesys database ({view_type_time_sleep} seconds)." ) time.sleep(view_type_time_sleep) genesys.get_reporting_exports_data() failed = [col for col in np.array(genesys.report_data).T][-1] - print(failed) + if "FAILED" in failed and "COMPLETED" in failed: logger.warning("Some reports failed.") - elif np.unique(failed)[0] == "FAILED": - genesys.report_data.clear() - if len(genesys.report_data) == 0: + if len(genesys.report_data) == 0 or np.unique(failed)[0] == "FAILED": genesys.delete_all_reporting_exports() logger.warning(f"All existing reports were delted.") raise APIError("No exporting reports were generated.") From 94a18546dbd04fb04901a0cb1cb201eac16d6a34 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 8 Nov 2022 11:17:54 +0100 Subject: [PATCH 03/27] =?UTF-8?q?=F0=9F=8E=A8=20iproved=20structure=20of?= =?UTF-8?q?=20Genesys=20(multiple=20reports).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 24 ++++++++++++------------ viadot/sources/genesys.py | 16 +++++++++------- viadot/tasks/genesys.py | 25 +++++++++++++------------ 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 7e81714dd..216365b25 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -64,8 +64,8 @@ def __init__( self, name: str, view_type: Literal[ - "queue_performance_detail", "agent_performance_summery_view" - ] = "queue_performance_detail", + "queue_performance_detail_view", "agent_performance_summery_view" + ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, media_type_list: List[str] = None, queueIds_list: List[str] = None, @@ -91,8 +91,8 @@ def __init__( Args: name (str): The name of the Flow. - view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail". + view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. @@ -143,7 +143,7 @@ def gen_flow(self) -> Flow: to_csv = GenesysToCSV() - if self.view_type == "queue_performance_detail": + if self.view_type == "queue_performance_detail_view": file_names = to_csv.bind( view_type=self.view_type, media_type_list=self.media_type_list, @@ -171,15 +171,15 @@ def gen_flow(self) -> Flow: add_timestamp.bind(file_names, sep=self.sep, flow=self) - uploader = adls_bulk_upload( - file_names=file_names, - adls_file_path=self.adls_file_path, - adls_sp_credentials_secret=self.adls_sp_credentials_secret, - flow=self, - ) + # uploader = adls_bulk_upload( + # file_names=file_names, + # adls_file_path=self.adls_file_path, + # adls_sp_credentials_secret=self.adls_sp_credentials_secret, + # flow=self, + # ) add_timestamp.set_upstream(file_names, flow=self) - uploader.set_upstream(add_timestamp, flow=self) + # uploader.set_upstream(add_timestamp, flow=self) class GenesysReportToADLS(Flow): diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 5ddf48180..6e92a21df 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -24,8 +24,8 @@ class Genesys(Source): def __init__( self, view_type: Literal[ - "queue_performance_detail", "agent_performance_summery_view" - ] = "queue_performance_detail", + "queue_performance_detail_view", "agent_performance_summery_view" + ] = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -47,8 +47,8 @@ def __init__( Genesys connector which allows for reports scheduling, listing and downloading into Data Frame or specified format output. Args: - view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail". + view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -277,7 +277,7 @@ def get_reporting_exports_data(self): Function that generate list of reports metadata for further processing steps. """ request_json = self.load_reporting_exports() - + print(request_json) if request_json is not None: entities = request_json.get("entities") assert type(entities) == list @@ -288,6 +288,7 @@ def get_reporting_exports_data(self): entity.get("downloadUrl"), entity.get("filter").get("queueIds", [-1])[0], entity.get("filter").get("mediaTypes", [-1])[0], + entity.get("viewType"), entity.get("status"), ] self.report_data.append(tmp) @@ -350,6 +351,7 @@ def download_all_reporting_exports( self.logger.info("IDS_MAPPING loaded from local credential.") for single_report in self.report_data: + print(single_report) if single_report[-1] == "RUNNING": self.logger.warning( "The request is still in progress, consider add more seconds to `view_type_time_sleep` parameter." @@ -360,11 +362,11 @@ def download_all_reporting_exports( "This message 'FAILED_GETTING_DATA_FROM_SERVICE' raised during script execution." ) continue - if self.view_type == "queue_performance_detail": + if single_report[4].lower() == "queue_performance_detail_view": file_name = ( temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] ).upper() - elif self.view_type == "agent_performance_summery_view": + elif single_report[4].lower() == "agent_performance_summery_view": date = self.start_date.replace("-", "") file_name = self.view_type.upper() + "_" + f"{date}" else: diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index 069cbcf1d..dba7d42d5 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -19,8 +19,8 @@ def __init__( self, report_name: str = "genesys_to_csv", view_type: Literal[ - "queue_performance_detail", "agent_performance_summery_view" - ] = "queue_performance_detail", + "queue_performance_detail_view", "agent_performance_summery_view" + ] = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -39,8 +39,8 @@ def __init__( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail". + view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -99,8 +99,8 @@ def run( self, report_name: str = None, view_type: Literal[ - "queue_performance_detail", "agent_performance_summery_view" - ] = "queue_performance_detail", + "queue_performance_detail_view", "agent_performance_summery_view" + ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, environment: str = None, schedule_id: str = None, @@ -119,8 +119,8 @@ def run( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail". + view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. @@ -158,7 +158,7 @@ def run( genesys.genesys_generate_body() genesys.genesys_generate_exports() - if view_type == "queue_performance_detail": + if view_type == "queue_performance_detail_view": logger.info(f"Waiting for caching data in Genesys database.") # in order to wait for API POST request add it timeout_start = time.time() @@ -187,10 +187,11 @@ def run( time.sleep(view_type_time_sleep) genesys.get_reporting_exports_data() - failed = [col for col in np.array(genesys.report_data).T][-1] - if "FAILED" in failed and "COMPLETED" in failed: - logger.warning("Some reports failed.") + failed = [col for col in np.array(genesys.report_data).T][-1] + + if "FAILED" in failed and "COMPLETED" in failed: + logger.warning("Some reports failed.") if len(genesys.report_data) == 0 or np.unique(failed)[0] == "FAILED": genesys.delete_all_reporting_exports() From 8128f89713533bc6b97a26c6aed8ab4dea1fa9d8 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 8 Nov 2022 12:17:30 +0100 Subject: [PATCH 04/27] =?UTF-8?q?=F0=9F=8E=A8=20improved=20Genesys,=20filt?= =?UTF-8?q?er=20files=20by=20day.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/sources/genesys.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 6e92a21df..0790c20c8 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -277,7 +277,7 @@ def get_reporting_exports_data(self): Function that generate list of reports metadata for further processing steps. """ request_json = self.load_reporting_exports() - print(request_json) + if request_json is not None: entities = request_json.get("entities") assert type(entities) == list @@ -289,6 +289,7 @@ def get_reporting_exports_data(self): entity.get("filter").get("queueIds", [-1])[0], entity.get("filter").get("mediaTypes", [-1])[0], entity.get("viewType"), + entity.get("interval"), entity.get("status"), ] self.report_data.append(tmp) @@ -351,7 +352,9 @@ def download_all_reporting_exports( self.logger.info("IDS_MAPPING loaded from local credential.") for single_report in self.report_data: - print(single_report) + self.logger.info(single_report) + print(self.start_date, type(self.start_date)) + print(self.end_date, type(self.end_date)) if single_report[-1] == "RUNNING": self.logger.warning( "The request is still in progress, consider add more seconds to `view_type_time_sleep` parameter." @@ -362,6 +365,16 @@ def download_all_reporting_exports( "This message 'FAILED_GETTING_DATA_FROM_SERVICE' raised during script execution." ) continue + elif ( + self.start_date not in single_report[5] + or self.end_date not in single_report[5] + ): + self.logger.warning( + f"The report with ID {single_report[0]} doesn't match with the interval date that you have already defined. \ + The report won't be downloaded but will be deleted." + ) + continue + if single_report[4].lower() == "queue_performance_detail_view": file_name = ( temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] From 9513b633ff45b4b97d4e8eedd7052eedb6f9b629 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 8 Nov 2022 13:02:04 +0100 Subject: [PATCH 05/27] =?UTF-8?q?=F0=9F=93=9D=20updated=20Genesys=20docs.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 14 +++++++------- viadot/sources/genesys.py | 13 ++++--------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 216365b25..40dc1a0d9 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -171,15 +171,15 @@ def gen_flow(self) -> Flow: add_timestamp.bind(file_names, sep=self.sep, flow=self) - # uploader = adls_bulk_upload( - # file_names=file_names, - # adls_file_path=self.adls_file_path, - # adls_sp_credentials_secret=self.adls_sp_credentials_secret, - # flow=self, - # ) + uploader = adls_bulk_upload( + file_names=file_names, + adls_file_path=self.adls_file_path, + adls_sp_credentials_secret=self.adls_sp_credentials_secret, + flow=self, + ) add_timestamp.set_upstream(file_names, flow=self) - # uploader.set_upstream(add_timestamp, flow=self) + uploader.set_upstream(add_timestamp, flow=self) class GenesysReportToADLS(Flow): diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 0790c20c8..bcd83a95b 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -353,11 +353,9 @@ def download_all_reporting_exports( for single_report in self.report_data: self.logger.info(single_report) - print(self.start_date, type(self.start_date)) - print(self.end_date, type(self.end_date)) if single_report[-1] == "RUNNING": self.logger.warning( - "The request is still in progress, consider add more seconds to `view_type_time_sleep` parameter." + "The request is still in progress and will be deleted, consider add more seconds in `view_type_time_sleep` parameter." ) continue elif single_report[-1] == "FAILED": @@ -365,10 +363,7 @@ def download_all_reporting_exports( "This message 'FAILED_GETTING_DATA_FROM_SERVICE' raised during script execution." ) continue - elif ( - self.start_date not in single_report[5] - or self.end_date not in single_report[5] - ): + elif self.start_date not in single_report[5]: self.logger.warning( f"The report with ID {single_report[0]} doesn't match with the interval date that you have already defined. \ The report won't be downloaded but will be deleted." @@ -379,12 +374,12 @@ def download_all_reporting_exports( file_name = ( temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] ).upper() - elif single_report[4].lower() == "agent_performance_summery_view": + elif single_report[4].lower() == "agent_performance_summary_view": date = self.start_date.replace("-", "") file_name = self.view_type.upper() + "_" + f"{date}" else: self.logger.error( - f"View type {self.view_type} not defined in viador, yet..." + f"View type {self.view_type} not defined in viadot, yet..." ) self.download_report( From 387d1dcc7f3d0a08b0cddf68e5389e7bb8f42c36 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Wed, 9 Nov 2022 11:36:04 +0100 Subject: [PATCH 06/27] =?UTF-8?q?=F0=9F=93=9D=20updated=20docs=20in=20Gene?= =?UTF-8?q?sys.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 6 +++--- viadot/sources/genesys.py | 6 +++--- viadot/tasks/genesys.py | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 40dc1a0d9..9eb80dc7d 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -64,7 +64,7 @@ def __init__( self, name: str, view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summery_view" + "queue_performance_detail_view", "agent_performance_summary_view" ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, media_type_list: List[str] = None, @@ -91,7 +91,7 @@ def __init__( Args: name (str): The name of the Flow. - view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. @@ -156,7 +156,7 @@ def gen_flow(self) -> Flow: credentials_genesys=self.credentials_genesys, flow=self, ) - elif self.view_type == "agent_performance_summery_view": + elif self.view_type == "agent_performance_summary_view": file_names = to_csv.bind( view_type=self.view_type, view_type_time_sleep=self.view_type_time_sleep, diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index bcd83a95b..38632ba28 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -24,7 +24,7 @@ class Genesys(Source): def __init__( self, view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summery_view" + "queue_performance_detail_view", "agent_performance_summary_view" ] = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, @@ -47,7 +47,7 @@ def __init__( Genesys connector which allows for reports scheduling, listing and downloading into Data Frame or specified format output. Args: - view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. @@ -372,7 +372,7 @@ def download_all_reporting_exports( if single_report[4].lower() == "queue_performance_detail_view": file_name = ( - temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] + temp_ids_mapping.get(single_report[2]) + "_" + single_report[3] ).upper() elif single_report[4].lower() == "agent_performance_summary_view": date = self.start_date.replace("-", "") diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index dba7d42d5..022d56f48 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -19,7 +19,7 @@ def __init__( self, report_name: str = "genesys_to_csv", view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summery_view" + "queue_performance_detail_view", "agent_performance_summary_view" ] = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, @@ -39,7 +39,7 @@ def __init__( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. @@ -99,7 +99,7 @@ def run( self, report_name: str = None, view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summery_view" + "queue_performance_detail_view", "agent_performance_summary_view" ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, environment: str = None, @@ -119,7 +119,7 @@ def run( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. @@ -180,7 +180,7 @@ def run( # There is a need to clear a list before repeating try statement. genesys.report_data.clear() - elif view_type == "agent_performance_summery_view": + elif view_type == "agent_performance_summary_view": logger.info( f"Waiting for getting data in Genesys database ({view_type_time_sleep} seconds)." ) From 3f459fcf5320155f4b80bece99408d2652f4b388 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Fri, 11 Nov 2022 10:50:49 +0100 Subject: [PATCH 07/27] =?UTF-8?q?=E2=9C=85=20updated=20test=5Fgenesys.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/integration/test_genesys.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/test_genesys.py b/tests/integration/test_genesys.py index 649635585..d8a5b2a3c 100644 --- a/tests/integration/test_genesys.py +++ b/tests/integration/test_genesys.py @@ -8,6 +8,7 @@ @pytest.fixture def var_dictionary(): variables = { + "start_date": "2022-08-12", "media_type_list": ["callback", "chat"], "queueIds_list": [ "1234567890", @@ -38,24 +39,36 @@ def var_dictionary(): "https://apps.mypurecloud.de/example/url/test", "1234567890qwertyuiopasdfghjklazxcvbn", "chat", + "QUEUE_PERFORMANCE_DETAIL_VIEW", + "2022-08-12T23:00:00.000Z/2022-08-13T23:00:00.000Z", + "COMPLETED", ], [ "1234567890qwertyuiopasdfghjklazxcvbn", "https://apps.mypurecloud.de/example/url/test", "1234567890qwertyuiopasdfghjklazxcvbn", "chat", + "QUEUE_PERFORMANCE_DETAIL_VIEW", + "2022-08-12T23:00:00.000Z/2022-08-13T23:00:00.000Z", + "COMPLETED", ], [ "1234567890qwertyuiopasdfghjklazxcvbn", "https://apps.mypurecloud.de/example/url/test", "1234567890qwertyuiopasdfghjklazxcvbn", "callback", + "QUEUE_PERFORMANCE_DETAIL_VIEW", + "2022-08-12T23:00:00.000Z/2022-08-13T23:00:00.000Z", + "COMPLETED", ], [ "1234567890qwertyuiopasdfghjklazxcvbn", "https://apps.mypurecloud.de/example/url/test", "1234567890qwertyuiopasdfghjklazxcvbn", "callback", + "QUEUE_PERFORMANCE_DETAIL_VIEW", + "2022-08-12T23:00:00.000Z/2022-08-13T23:00:00.000Z", + "COMPLETED", ], ], "entities": { @@ -202,6 +215,7 @@ def test_download_reports(mock_download_files, var_dictionary): g = Genesys() g.ids_mapping = var_dictionary["ids_mapping"] g.report_data = var_dictionary["report_data"] + g.start_date = var_dictionary["start_date"] file_name_list = g.download_all_reporting_exports() assert type(file_name_list) == list and len(file_name_list) > 0 From 11d6dd1dfea5a3ee9316596bdd8a5d14a809b896 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 15 Nov 2022 10:21:18 +0100 Subject: [PATCH 08/27] =?UTF-8?q?=F0=9F=93=9D=20updated=20CHANGELOG.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d31ae1d17..8141a481a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Changed `duckdb` version to `0.5.1` - Added new column into Data Frames created with `Mindful`. - Added region parameter as an entry argument in `MindfulToADLS`. +- Added into `Genesys` a the new view type `AGENT`. ### Fixed - Fixed incorrect `if_exists="delete"` handling in `DuckDB.create_table_from_parquet()` From 28d3cc88004323fa87ed763faf11d24ce8ac6421 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 15 Nov 2022 10:22:56 +0100 Subject: [PATCH 09/27] =?UTF-8?q?=F0=9F=93=9D=20updated=20docstrings.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8141a481a..d22a57c7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Changed `duckdb` version to `0.5.1` - Added new column into Data Frames created with `Mindful`. - Added region parameter as an entry argument in `MindfulToADLS`. -- Added into `Genesys` a the new view type `AGENT`. +- Added into `Genesys` the new view type `AGENT`. ### Fixed - Fixed incorrect `if_exists="delete"` handling in `DuckDB.create_table_from_parquet()` From 17d2dd28720dc4faf64e72eba04482ad074f3063 Mon Sep 17 00:00:00 2001 From: Rafalz13 Date: Wed, 16 Nov 2022 12:59:37 +0100 Subject: [PATCH 10/27] =?UTF-8?q?=F0=9F=9A=80=20Bumped=20version=20after?= =?UTF-8?q?=20release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_viadot.py | 2 +- viadot/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_viadot.py b/tests/test_viadot.py index 7d0160ec2..60eb8acdf 100644 --- a/tests/test_viadot.py +++ b/tests/test_viadot.py @@ -2,4 +2,4 @@ def test_version(): - assert __version__ == "0.4.10" + assert __version__ == "0.4.11" diff --git a/viadot/__init__.py b/viadot/__init__.py index 805e7c470..58ce5cd17 100644 --- a/viadot/__init__.py +++ b/viadot/__init__.py @@ -1 +1 @@ -__version__ = "0.4.10" +__version__ = "0.4.11" From d9d5a9c42292665bc53eca4df050d3828e852998 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Mon, 7 Nov 2022 14:13:54 +0100 Subject: [PATCH 11/27] =?UTF-8?q?=E2=9C=A8=20added=20agent=20summary=20vie?= =?UTF-8?q?w=20in=20Genesys.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 55 ++++++++++++++++-------- viadot/sources/genesys.py | 39 ++++++++++++++--- viadot/tasks/genesys.py | 75 +++++++++++++++++++++++---------- 3 files changed, 123 insertions(+), 46 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 9e71131fb..25472dd03 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -63,6 +63,9 @@ class GenesysToADLS(Flow): def __init__( self, name: str, + view_type: Literal[ + "queue_performance_detail", "agent_performance_summery_view" + ] = "queue_performance_detail", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -87,6 +90,8 @@ def __init__( Args: name (str): The name of the Flow. + view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -108,6 +113,7 @@ def __init__( """ # GenesysToCSV self.flow_name = name + self.view_type = view_type self.media_type_list = media_type_list self.queueIds_list = queueIds_list self.data_to_post = data_to_post_str @@ -134,29 +140,42 @@ def gen_flow(self) -> Flow: to_csv = GenesysToCSV() - file_names = to_csv.bind( - media_type_list=self.media_type_list, - queueIds_list=self.queueIds_list, - data_to_post_str=self.data_to_post, - start_date=self.start_date, - end_date=self.end_date, - days_interval=self.days_interval, - environment=self.environment, - credentials_genesys=self.credentials_genesys, - flow=self, - ) + if self.view_type == "queue_performance_detail": + file_names = to_csv.bind( + view_type=self.view_type, + media_type_list=self.media_type_list, + queueIds_list=self.queueIds_list, + data_to_post_str=self.data_to_post, + start_date=self.start_date, + end_date=self.end_date, + days_interval=self.days_interval, + environment=self.environment, + credentials_genesys=self.credentials_genesys, + flow=self, + ) + elif self.view_type == "agent_performance_summery_view": + file_names = to_csv.bind( + view_type=self.view_type, + media_type_list=self.media_type_list, + queueIds_list=[""], + data_to_post_str=self.data_to_post, + start_date=self.start_date, + environment=self.environment, + credentials_genesys=self.credentials_genesys, + flow=self, + ) add_timestamp.bind(file_names, sep=self.sep, flow=self) - uploader = adls_bulk_upload( - file_names=file_names, - adls_file_path=self.adls_file_path, - adls_sp_credentials_secret=self.adls_sp_credentials_secret, - flow=self, - ) + # uploader = adls_bulk_upload( + # file_names=file_names, + # adls_file_path=self.adls_file_path, + # adls_sp_credentials_secret=self.adls_sp_credentials_secret, + # flow=self, + # ) add_timestamp.set_upstream(file_names, flow=self) - uploader.set_upstream(add_timestamp, flow=self) + # uploader.set_upstream(add_timestamp, flow=self) class GenesysReportToADLS(Flow): diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 27207a1c7..810cc0aa8 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -1,4 +1,4 @@ -import json +import json, sys import base64 import warnings import asyncio @@ -23,6 +23,9 @@ class Genesys(Source): def __init__( self, + view_type: Literal[ + "queue_performance_detail", "agent_performance_summery_view" + ] = "queue_performance_detail", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -44,6 +47,8 @@ def __init__( Genesys connector which allows for reports scheduling, listing and downloading into Data Frame or specified format output. Args: + view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -81,6 +86,7 @@ def __init__( super().__init__(*args, credentials=self.credentials_genesys, **kwargs) + self.view_type = view_type self.schedule_id = schedule_id self.report_name = report_name self.environment = environment @@ -280,13 +286,16 @@ def get_reporting_exports_data(self): tmp = [ entity.get("id"), entity.get("downloadUrl"), - entity.get("filter").get("queueIds")[0], - entity.get("filter").get("mediaTypes")[0], + entity.get("filter").get("queueIds", [-1])[0], + entity.get("filter").get("mediaTypes", [-1])[0], + entity.get("status"), ] self.report_data.append(tmp) assert len(self.report_data) > 0 self.logger.info("Generated list of reports entities.") + print(self.report_data) + def download_report( self, report_url: str, @@ -343,9 +352,27 @@ def download_all_reporting_exports( self.logger.info("IDS_MAPPING loaded from local credential.") for single_report in self.report_data: - file_name = ( - temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] - ).upper() + if single_report[-1] == "RUNNING": + self.logger.warning( + "The request is still in progress, consider add more seconds to `view_type_time_sleep` parameter." + ) + continue + elif single_report[-1] == "FAILED": + self.logger.warning( + "This message 'FAILED_GETTING_DATA_FROM_SERVICE' raised during script execution." + ) + continue + if self.view_type == "queue_performance_detail": + file_name = ( + temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] + ).upper() + elif self.view_type == "agent_performance_summery_view": + file_name = self.view_type.upper() + "_" + f"{single_report[0][:8]}" + else: + self.logger.error( + f"View type {self.view_type} not defined in viador, yet..." + ) + self.download_report( report_url=single_report[1], output_file_name=file_name, diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index 5bf0d2f4a..af4df0dba 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -1,5 +1,5 @@ import time -from typing import Any, Dict, List +from typing import Any, Dict, List, Literal import pandas as pd import numpy as np @@ -18,6 +18,9 @@ class GenesysToCSV(Task): def __init__( self, report_name: str = "genesys_to_csv", + view_type: Literal[ + "queue_performance_detail", "agent_performance_summery_view" + ] = "queue_performance_detail", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -36,6 +39,8 @@ def __init__( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. + view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -53,6 +58,7 @@ def __init__( self.logger = prefect.context.get("logger") self.schedule_id = schedule_id self.report_name = report_name + self.view_type = view_type self.environment = environment self.report_url = report_url self.report_columns = report_columns @@ -76,6 +82,7 @@ def __call__(self, *args, **kwargs): @defaults_from_attrs( "report_name", + "view_type", "environment", "schedule_id", "report_url", @@ -91,6 +98,10 @@ def __call__(self, *args, **kwargs): def run( self, report_name: str = None, + view_type: Literal[ + "queue_performance_detail", "agent_performance_summery_view" + ] = "queue_performance_detail", + view_type_time_sleep: int = 80, environment: str = None, schedule_id: str = None, report_url: str = None, @@ -108,6 +119,8 @@ def run( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. + view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -127,6 +140,7 @@ def run( genesys = Genesys( report_name=report_name, + view_type=view_type, media_type_list=media_type_list, queueIds_list=queueIds_list, data_to_post_str=data_to_post_str, @@ -142,29 +156,46 @@ def run( genesys.genesys_generate_body() genesys.genesys_generate_exports() - logger.info(f"Waiting for caching data in Genesys database.") - - # in order to wait for API POST request add it - timeout_start = time.time() - # 30 seconds timeout is minimal but for safety added 60. - timeout = timeout_start + 60 - # while loop with timeout - while time.time() < timeout: - - try: - genesys.get_reporting_exports_data() - urls = [col for col in np.array(genesys.report_data).T][1] - if None in urls: - logger.warning("Found None object in list of urls.") - else: - break - except TypeError: - pass - - # There is a need to clear a list before repeating try statement. - genesys.report_data.clear() + + if view_type == "queue_performance_detail": + logger.info(f"Waiting for caching data in Genesys database.") + # in order to wait for API POST request add it + timeout_start = time.time() + # 30 seconds timeout is minimal but for safety added 60. + timeout = timeout_start + 60 + # while loop with timeout + while time.time() < timeout: + + try: + genesys.get_reporting_exports_data() + urls = [col for col in np.array(genesys.report_data).T][1] + if None in urls: + logger.warning("Found None object in list of urls.") + else: + break + except TypeError: + pass + + # There is a need to clear a list before repeating try statement. + genesys.report_data.clear() + + elif view_type == "agent_performance_summery_view": + logger.info( + f"Waiting for caching data in Genesys database ({view_type_time_sleep} seconds)." + ) + time.sleep(view_type_time_sleep) + + genesys.get_reporting_exports_data() + failed = [col for col in np.array(genesys.report_data).T][-1] + print(failed) + if "FAILED" in failed and "COMPLETED" in failed: + logger.warning("Some reports failed.") + elif np.unique(failed)[0] == "FAILED": + genesys.report_data.clear() if len(genesys.report_data) == 0: + genesys.delete_all_reporting_exports() + logger.warning(f"All existing reports were delted.") raise APIError("No exporting reports were generated.") elif not None in [col for col in np.array(genesys.report_data).T][1]: logger.info("Downloaded the data from the Genesys into the CSV.") From ef1b69debf0349701c3b046edb77a7fa03d4e352 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Mon, 7 Nov 2022 16:44:00 +0100 Subject: [PATCH 12/27] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20improved=20performan?= =?UTF-8?q?ce=20of=20Genesys.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 18 +++++++++++------- viadot/sources/genesys.py | 5 ++--- viadot/tasks/genesys.py | 9 ++++----- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 25472dd03..7e81714dd 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -66,6 +66,7 @@ def __init__( view_type: Literal[ "queue_performance_detail", "agent_performance_summery_view" ] = "queue_performance_detail", + view_type_time_sleep: int = 80, media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -92,6 +93,7 @@ def __init__( name (str): The name of the Flow. view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail". + view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -114,6 +116,7 @@ def __init__( # GenesysToCSV self.flow_name = name self.view_type = view_type + self.view_type_time_sleep = view_type_time_sleep self.media_type_list = media_type_list self.queueIds_list = queueIds_list self.data_to_post = data_to_post_str @@ -156,6 +159,7 @@ def gen_flow(self) -> Flow: elif self.view_type == "agent_performance_summery_view": file_names = to_csv.bind( view_type=self.view_type, + view_type_time_sleep=self.view_type_time_sleep, media_type_list=self.media_type_list, queueIds_list=[""], data_to_post_str=self.data_to_post, @@ -167,15 +171,15 @@ def gen_flow(self) -> Flow: add_timestamp.bind(file_names, sep=self.sep, flow=self) - # uploader = adls_bulk_upload( - # file_names=file_names, - # adls_file_path=self.adls_file_path, - # adls_sp_credentials_secret=self.adls_sp_credentials_secret, - # flow=self, - # ) + uploader = adls_bulk_upload( + file_names=file_names, + adls_file_path=self.adls_file_path, + adls_sp_credentials_secret=self.adls_sp_credentials_secret, + flow=self, + ) add_timestamp.set_upstream(file_names, flow=self) - # uploader.set_upstream(add_timestamp, flow=self) + uploader.set_upstream(add_timestamp, flow=self) class GenesysReportToADLS(Flow): diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 810cc0aa8..5ddf48180 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -294,8 +294,6 @@ def get_reporting_exports_data(self): assert len(self.report_data) > 0 self.logger.info("Generated list of reports entities.") - print(self.report_data) - def download_report( self, report_url: str, @@ -367,7 +365,8 @@ def download_all_reporting_exports( temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] ).upper() elif self.view_type == "agent_performance_summery_view": - file_name = self.view_type.upper() + "_" + f"{single_report[0][:8]}" + date = self.start_date.replace("-", "") + file_name = self.view_type.upper() + "_" + f"{date}" else: self.logger.error( f"View type {self.view_type} not defined in viador, yet..." diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index af4df0dba..069cbcf1d 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -121,6 +121,7 @@ def run( report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail". + view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -181,19 +182,17 @@ def run( elif view_type == "agent_performance_summery_view": logger.info( - f"Waiting for caching data in Genesys database ({view_type_time_sleep} seconds)." + f"Waiting for getting data in Genesys database ({view_type_time_sleep} seconds)." ) time.sleep(view_type_time_sleep) genesys.get_reporting_exports_data() failed = [col for col in np.array(genesys.report_data).T][-1] - print(failed) + if "FAILED" in failed and "COMPLETED" in failed: logger.warning("Some reports failed.") - elif np.unique(failed)[0] == "FAILED": - genesys.report_data.clear() - if len(genesys.report_data) == 0: + if len(genesys.report_data) == 0 or np.unique(failed)[0] == "FAILED": genesys.delete_all_reporting_exports() logger.warning(f"All existing reports were delted.") raise APIError("No exporting reports were generated.") From 8effacf386e8b5ec2448710d7c02208138d4c15e Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 8 Nov 2022 11:17:54 +0100 Subject: [PATCH 13/27] =?UTF-8?q?=F0=9F=8E=A8=20iproved=20structure=20of?= =?UTF-8?q?=20Genesys=20(multiple=20reports).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 24 ++++++++++++------------ viadot/sources/genesys.py | 16 +++++++++------- viadot/tasks/genesys.py | 25 +++++++++++++------------ 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 7e81714dd..216365b25 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -64,8 +64,8 @@ def __init__( self, name: str, view_type: Literal[ - "queue_performance_detail", "agent_performance_summery_view" - ] = "queue_performance_detail", + "queue_performance_detail_view", "agent_performance_summery_view" + ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, media_type_list: List[str] = None, queueIds_list: List[str] = None, @@ -91,8 +91,8 @@ def __init__( Args: name (str): The name of the Flow. - view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail". + view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. @@ -143,7 +143,7 @@ def gen_flow(self) -> Flow: to_csv = GenesysToCSV() - if self.view_type == "queue_performance_detail": + if self.view_type == "queue_performance_detail_view": file_names = to_csv.bind( view_type=self.view_type, media_type_list=self.media_type_list, @@ -171,15 +171,15 @@ def gen_flow(self) -> Flow: add_timestamp.bind(file_names, sep=self.sep, flow=self) - uploader = adls_bulk_upload( - file_names=file_names, - adls_file_path=self.adls_file_path, - adls_sp_credentials_secret=self.adls_sp_credentials_secret, - flow=self, - ) + # uploader = adls_bulk_upload( + # file_names=file_names, + # adls_file_path=self.adls_file_path, + # adls_sp_credentials_secret=self.adls_sp_credentials_secret, + # flow=self, + # ) add_timestamp.set_upstream(file_names, flow=self) - uploader.set_upstream(add_timestamp, flow=self) + # uploader.set_upstream(add_timestamp, flow=self) class GenesysReportToADLS(Flow): diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 5ddf48180..6e92a21df 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -24,8 +24,8 @@ class Genesys(Source): def __init__( self, view_type: Literal[ - "queue_performance_detail", "agent_performance_summery_view" - ] = "queue_performance_detail", + "queue_performance_detail_view", "agent_performance_summery_view" + ] = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -47,8 +47,8 @@ def __init__( Genesys connector which allows for reports scheduling, listing and downloading into Data Frame or specified format output. Args: - view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail". + view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -277,7 +277,7 @@ def get_reporting_exports_data(self): Function that generate list of reports metadata for further processing steps. """ request_json = self.load_reporting_exports() - + print(request_json) if request_json is not None: entities = request_json.get("entities") assert type(entities) == list @@ -288,6 +288,7 @@ def get_reporting_exports_data(self): entity.get("downloadUrl"), entity.get("filter").get("queueIds", [-1])[0], entity.get("filter").get("mediaTypes", [-1])[0], + entity.get("viewType"), entity.get("status"), ] self.report_data.append(tmp) @@ -350,6 +351,7 @@ def download_all_reporting_exports( self.logger.info("IDS_MAPPING loaded from local credential.") for single_report in self.report_data: + print(single_report) if single_report[-1] == "RUNNING": self.logger.warning( "The request is still in progress, consider add more seconds to `view_type_time_sleep` parameter." @@ -360,11 +362,11 @@ def download_all_reporting_exports( "This message 'FAILED_GETTING_DATA_FROM_SERVICE' raised during script execution." ) continue - if self.view_type == "queue_performance_detail": + if single_report[4].lower() == "queue_performance_detail_view": file_name = ( temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] ).upper() - elif self.view_type == "agent_performance_summery_view": + elif single_report[4].lower() == "agent_performance_summery_view": date = self.start_date.replace("-", "") file_name = self.view_type.upper() + "_" + f"{date}" else: diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index 069cbcf1d..dba7d42d5 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -19,8 +19,8 @@ def __init__( self, report_name: str = "genesys_to_csv", view_type: Literal[ - "queue_performance_detail", "agent_performance_summery_view" - ] = "queue_performance_detail", + "queue_performance_detail_view", "agent_performance_summery_view" + ] = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -39,8 +39,8 @@ def __init__( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail". + view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -99,8 +99,8 @@ def run( self, report_name: str = None, view_type: Literal[ - "queue_performance_detail", "agent_performance_summery_view" - ] = "queue_performance_detail", + "queue_performance_detail_view", "agent_performance_summery_view" + ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, environment: str = None, schedule_id: str = None, @@ -119,8 +119,8 @@ def run( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail, agent_performance_summery_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail". + view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. @@ -158,7 +158,7 @@ def run( genesys.genesys_generate_body() genesys.genesys_generate_exports() - if view_type == "queue_performance_detail": + if view_type == "queue_performance_detail_view": logger.info(f"Waiting for caching data in Genesys database.") # in order to wait for API POST request add it timeout_start = time.time() @@ -187,10 +187,11 @@ def run( time.sleep(view_type_time_sleep) genesys.get_reporting_exports_data() - failed = [col for col in np.array(genesys.report_data).T][-1] - if "FAILED" in failed and "COMPLETED" in failed: - logger.warning("Some reports failed.") + failed = [col for col in np.array(genesys.report_data).T][-1] + + if "FAILED" in failed and "COMPLETED" in failed: + logger.warning("Some reports failed.") if len(genesys.report_data) == 0 or np.unique(failed)[0] == "FAILED": genesys.delete_all_reporting_exports() From b7d7354c5d4f8a77c06f3c7fbf9bea91e8a8bc4b Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 8 Nov 2022 12:17:30 +0100 Subject: [PATCH 14/27] =?UTF-8?q?=F0=9F=8E=A8=20improved=20Genesys,=20filt?= =?UTF-8?q?er=20files=20by=20day.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/sources/genesys.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 6e92a21df..0790c20c8 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -277,7 +277,7 @@ def get_reporting_exports_data(self): Function that generate list of reports metadata for further processing steps. """ request_json = self.load_reporting_exports() - print(request_json) + if request_json is not None: entities = request_json.get("entities") assert type(entities) == list @@ -289,6 +289,7 @@ def get_reporting_exports_data(self): entity.get("filter").get("queueIds", [-1])[0], entity.get("filter").get("mediaTypes", [-1])[0], entity.get("viewType"), + entity.get("interval"), entity.get("status"), ] self.report_data.append(tmp) @@ -351,7 +352,9 @@ def download_all_reporting_exports( self.logger.info("IDS_MAPPING loaded from local credential.") for single_report in self.report_data: - print(single_report) + self.logger.info(single_report) + print(self.start_date, type(self.start_date)) + print(self.end_date, type(self.end_date)) if single_report[-1] == "RUNNING": self.logger.warning( "The request is still in progress, consider add more seconds to `view_type_time_sleep` parameter." @@ -362,6 +365,16 @@ def download_all_reporting_exports( "This message 'FAILED_GETTING_DATA_FROM_SERVICE' raised during script execution." ) continue + elif ( + self.start_date not in single_report[5] + or self.end_date not in single_report[5] + ): + self.logger.warning( + f"The report with ID {single_report[0]} doesn't match with the interval date that you have already defined. \ + The report won't be downloaded but will be deleted." + ) + continue + if single_report[4].lower() == "queue_performance_detail_view": file_name = ( temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] From 08aa7994129adedeb8568d66f3c4b6bbad4b67ba Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 8 Nov 2022 13:02:04 +0100 Subject: [PATCH 15/27] =?UTF-8?q?=F0=9F=93=9D=20updated=20Genesys=20docs.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 14 +++++++------- viadot/sources/genesys.py | 13 ++++--------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 216365b25..40dc1a0d9 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -171,15 +171,15 @@ def gen_flow(self) -> Flow: add_timestamp.bind(file_names, sep=self.sep, flow=self) - # uploader = adls_bulk_upload( - # file_names=file_names, - # adls_file_path=self.adls_file_path, - # adls_sp_credentials_secret=self.adls_sp_credentials_secret, - # flow=self, - # ) + uploader = adls_bulk_upload( + file_names=file_names, + adls_file_path=self.adls_file_path, + adls_sp_credentials_secret=self.adls_sp_credentials_secret, + flow=self, + ) add_timestamp.set_upstream(file_names, flow=self) - # uploader.set_upstream(add_timestamp, flow=self) + uploader.set_upstream(add_timestamp, flow=self) class GenesysReportToADLS(Flow): diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 0790c20c8..bcd83a95b 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -353,11 +353,9 @@ def download_all_reporting_exports( for single_report in self.report_data: self.logger.info(single_report) - print(self.start_date, type(self.start_date)) - print(self.end_date, type(self.end_date)) if single_report[-1] == "RUNNING": self.logger.warning( - "The request is still in progress, consider add more seconds to `view_type_time_sleep` parameter." + "The request is still in progress and will be deleted, consider add more seconds in `view_type_time_sleep` parameter." ) continue elif single_report[-1] == "FAILED": @@ -365,10 +363,7 @@ def download_all_reporting_exports( "This message 'FAILED_GETTING_DATA_FROM_SERVICE' raised during script execution." ) continue - elif ( - self.start_date not in single_report[5] - or self.end_date not in single_report[5] - ): + elif self.start_date not in single_report[5]: self.logger.warning( f"The report with ID {single_report[0]} doesn't match with the interval date that you have already defined. \ The report won't be downloaded but will be deleted." @@ -379,12 +374,12 @@ def download_all_reporting_exports( file_name = ( temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] ).upper() - elif single_report[4].lower() == "agent_performance_summery_view": + elif single_report[4].lower() == "agent_performance_summary_view": date = self.start_date.replace("-", "") file_name = self.view_type.upper() + "_" + f"{date}" else: self.logger.error( - f"View type {self.view_type} not defined in viador, yet..." + f"View type {self.view_type} not defined in viadot, yet..." ) self.download_report( From 7e2782202e669869762804b88cc2f1ed9b20cd9d Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Wed, 9 Nov 2022 11:36:04 +0100 Subject: [PATCH 16/27] =?UTF-8?q?=F0=9F=93=9D=20updated=20docs=20in=20Gene?= =?UTF-8?q?sys.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 6 +++--- viadot/sources/genesys.py | 6 +++--- viadot/tasks/genesys.py | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 40dc1a0d9..9eb80dc7d 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -64,7 +64,7 @@ def __init__( self, name: str, view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summery_view" + "queue_performance_detail_view", "agent_performance_summary_view" ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, media_type_list: List[str] = None, @@ -91,7 +91,7 @@ def __init__( Args: name (str): The name of the Flow. - view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. @@ -156,7 +156,7 @@ def gen_flow(self) -> Flow: credentials_genesys=self.credentials_genesys, flow=self, ) - elif self.view_type == "agent_performance_summery_view": + elif self.view_type == "agent_performance_summary_view": file_names = to_csv.bind( view_type=self.view_type, view_type_time_sleep=self.view_type_time_sleep, diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index bcd83a95b..38632ba28 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -24,7 +24,7 @@ class Genesys(Source): def __init__( self, view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summery_view" + "queue_performance_detail_view", "agent_performance_summary_view" ] = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, @@ -47,7 +47,7 @@ def __init__( Genesys connector which allows for reports scheduling, listing and downloading into Data Frame or specified format output. Args: - view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. @@ -372,7 +372,7 @@ def download_all_reporting_exports( if single_report[4].lower() == "queue_performance_detail_view": file_name = ( - temp_ids_mapping.get(single_report[2]) + "_" + single_report[-1] + temp_ids_mapping.get(single_report[2]) + "_" + single_report[3] ).upper() elif single_report[4].lower() == "agent_performance_summary_view": date = self.start_date.replace("-", "") diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index dba7d42d5..022d56f48 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -19,7 +19,7 @@ def __init__( self, report_name: str = "genesys_to_csv", view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summery_view" + "queue_performance_detail_view", "agent_performance_summary_view" ] = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, @@ -39,7 +39,7 @@ def __init__( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. @@ -99,7 +99,7 @@ def run( self, report_name: str = None, view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summery_view" + "queue_performance_detail_view", "agent_performance_summary_view" ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, environment: str = None, @@ -119,7 +119,7 @@ def run( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail_view, agent_performance_summery_view], optional): + view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. @@ -180,7 +180,7 @@ def run( # There is a need to clear a list before repeating try statement. genesys.report_data.clear() - elif view_type == "agent_performance_summery_view": + elif view_type == "agent_performance_summary_view": logger.info( f"Waiting for getting data in Genesys database ({view_type_time_sleep} seconds)." ) From ac09362b3ade046ae36d58592354f7889aefe5b6 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Fri, 11 Nov 2022 10:50:49 +0100 Subject: [PATCH 17/27] =?UTF-8?q?=E2=9C=85=20updated=20test=5Fgenesys.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/integration/test_genesys.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/test_genesys.py b/tests/integration/test_genesys.py index 649635585..d8a5b2a3c 100644 --- a/tests/integration/test_genesys.py +++ b/tests/integration/test_genesys.py @@ -8,6 +8,7 @@ @pytest.fixture def var_dictionary(): variables = { + "start_date": "2022-08-12", "media_type_list": ["callback", "chat"], "queueIds_list": [ "1234567890", @@ -38,24 +39,36 @@ def var_dictionary(): "https://apps.mypurecloud.de/example/url/test", "1234567890qwertyuiopasdfghjklazxcvbn", "chat", + "QUEUE_PERFORMANCE_DETAIL_VIEW", + "2022-08-12T23:00:00.000Z/2022-08-13T23:00:00.000Z", + "COMPLETED", ], [ "1234567890qwertyuiopasdfghjklazxcvbn", "https://apps.mypurecloud.de/example/url/test", "1234567890qwertyuiopasdfghjklazxcvbn", "chat", + "QUEUE_PERFORMANCE_DETAIL_VIEW", + "2022-08-12T23:00:00.000Z/2022-08-13T23:00:00.000Z", + "COMPLETED", ], [ "1234567890qwertyuiopasdfghjklazxcvbn", "https://apps.mypurecloud.de/example/url/test", "1234567890qwertyuiopasdfghjklazxcvbn", "callback", + "QUEUE_PERFORMANCE_DETAIL_VIEW", + "2022-08-12T23:00:00.000Z/2022-08-13T23:00:00.000Z", + "COMPLETED", ], [ "1234567890qwertyuiopasdfghjklazxcvbn", "https://apps.mypurecloud.de/example/url/test", "1234567890qwertyuiopasdfghjklazxcvbn", "callback", + "QUEUE_PERFORMANCE_DETAIL_VIEW", + "2022-08-12T23:00:00.000Z/2022-08-13T23:00:00.000Z", + "COMPLETED", ], ], "entities": { @@ -202,6 +215,7 @@ def test_download_reports(mock_download_files, var_dictionary): g = Genesys() g.ids_mapping = var_dictionary["ids_mapping"] g.report_data = var_dictionary["report_data"] + g.start_date = var_dictionary["start_date"] file_name_list = g.download_all_reporting_exports() assert type(file_name_list) == list and len(file_name_list) > 0 From d4116e61ae9aac5f483f7c7634d8d831fd441d80 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 15 Nov 2022 10:21:18 +0100 Subject: [PATCH 18/27] =?UTF-8?q?=F0=9F=93=9D=20updated=20CHANGELOG.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ec633800..c3f4f28bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Changed `duckdb` version to `0.5.1` - Added new column into Data Frames created with `Mindful`. - Added region parameter as an entry argument in `MindfulToADLS`. +- Added into `Genesys` a the new view type `AGENT`. ### Fixed - Fixed incorrect `if_exists="delete"` handling in `DuckDB.create_table_from_parquet()` From 368dbdb58a5a1c5240abd1afb945d407842021ed Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Tue, 15 Nov 2022 10:22:56 +0100 Subject: [PATCH 19/27] =?UTF-8?q?=F0=9F=93=9D=20updated=20docstrings.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3f4f28bb..66a0b3908 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Changed `duckdb` version to `0.5.1` - Added new column into Data Frames created with `Mindful`. - Added region parameter as an entry argument in `MindfulToADLS`. -- Added into `Genesys` a the new view type `AGENT`. +- Added into `Genesys` the new view type `AGENT`. ### Fixed - Fixed incorrect `if_exists="delete"` handling in `DuckDB.create_table_from_parquet()` From 6fef7ad6fa03fa511cdc7621ff80770bba5b04d2 Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Thu, 24 Nov 2022 08:47:19 +0100 Subject: [PATCH 20/27] =?UTF-8?q?=E2=9C=A8=20added=20new=20view=20type=20t?= =?UTF-8?q?o=20Genesys=20connection.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 9 +++++++-- viadot/sources/genesys.py | 9 +++++++-- viadot/tasks/genesys.py | 11 ++++++++--- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 9eb80dc7d..0eae98316 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -64,7 +64,9 @@ def __init__( self, name: str, view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summary_view" + "queue_performance_detail_view", + "agent_performance_summary_view", + "agent_status_summary_view", ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, media_type_list: List[str] = None, @@ -156,7 +158,10 @@ def gen_flow(self) -> Flow: credentials_genesys=self.credentials_genesys, flow=self, ) - elif self.view_type == "agent_performance_summary_view": + elif self.view_type in [ + "agent_performance_summary_view", + "agent_status_summary_view", + ]: file_names = to_csv.bind( view_type=self.view_type, view_type_time_sleep=self.view_type_time_sleep, diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 38632ba28..275a5a92b 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -24,7 +24,9 @@ class Genesys(Source): def __init__( self, view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summary_view" + "queue_performance_detail_view", + "agent_performance_summary_view", + "agent_status_summary_view", ] = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, @@ -374,7 +376,10 @@ def download_all_reporting_exports( file_name = ( temp_ids_mapping.get(single_report[2]) + "_" + single_report[3] ).upper() - elif single_report[4].lower() == "agent_performance_summary_view": + elif single_report[4].lower() in [ + "agent_performance_summary_view", + "agent_status_summary_view", + ]: date = self.start_date.replace("-", "") file_name = self.view_type.upper() + "_" + f"{date}" else: diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index 022d56f48..9c37c5404 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -99,7 +99,9 @@ def run( self, report_name: str = None, view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summary_view" + "queue_performance_detail_view", + "agent_performance_summary_view", + "agent_status_summary_view", ] = "queue_performance_detail_view", view_type_time_sleep: int = 80, environment: str = None, @@ -180,14 +182,17 @@ def run( # There is a need to clear a list before repeating try statement. genesys.report_data.clear() - elif view_type == "agent_performance_summary_view": + elif view_type in [ + "agent_performance_summary_view", + "agent_status_summary_view", + ]: logger.info( f"Waiting for getting data in Genesys database ({view_type_time_sleep} seconds)." ) time.sleep(view_type_time_sleep) genesys.get_reporting_exports_data() - + # print(genesys.report_data) failed = [col for col in np.array(genesys.report_data).T][-1] if "FAILED" in failed and "COMPLETED" in failed: From ea139b5da83020e082b7493bee5415a908fc54dd Mon Sep 17 00:00:00 2001 From: Diego-H-S Date: Mon, 28 Nov 2022 08:16:31 +0100 Subject: [PATCH 21/27] =?UTF-8?q?=F0=9F=8E=A8=20added=20Michal=20W.=20feed?= =?UTF-8?q?back.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/genesys_to_adls.py | 9 ++------- viadot/sources/genesys.py | 18 +++++++++++------- viadot/tasks/genesys.py | 16 ++++------------ 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/viadot/flows/genesys_to_adls.py b/viadot/flows/genesys_to_adls.py index 0eae98316..c4c894a76 100644 --- a/viadot/flows/genesys_to_adls.py +++ b/viadot/flows/genesys_to_adls.py @@ -63,11 +63,7 @@ class GenesysToADLS(Flow): def __init__( self, name: str, - view_type: Literal[ - "queue_performance_detail_view", - "agent_performance_summary_view", - "agent_status_summary_view", - ] = "queue_performance_detail_view", + view_type: str = "queue_performance_detail_view", view_type_time_sleep: int = 80, media_type_list: List[str] = None, queueIds_list: List[str] = None, @@ -93,8 +89,7 @@ def __init__( Args: name (str): The name of the Flow. - view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail_view". + view_type (str, optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. diff --git a/viadot/sources/genesys.py b/viadot/sources/genesys.py index 275a5a92b..374372424 100644 --- a/viadot/sources/genesys.py +++ b/viadot/sources/genesys.py @@ -23,11 +23,7 @@ class Genesys(Source): def __init__( self, - view_type: Literal[ - "queue_performance_detail_view", - "agent_performance_summary_view", - "agent_status_summary_view", - ] = "queue_performance_detail_view", + view_type: str = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -49,8 +45,7 @@ def __init__( Genesys connector which allows for reports scheduling, listing and downloading into Data Frame or specified format output. Args: - view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail_view". + view_type (str, optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -89,6 +84,15 @@ def __init__( super().__init__(*args, credentials=self.credentials_genesys, **kwargs) self.view_type = view_type + if self.view_type not in [ + "queue_performance_detail_view", + "agent_performance_summary_view", + "agent_status_summary_view", + ]: + raise Exception( + f"View type {self.view_type} still is not implemented in viadot." + ) + self.schedule_id = schedule_id self.report_name = report_name self.environment = environment diff --git a/viadot/tasks/genesys.py b/viadot/tasks/genesys.py index 9c37c5404..aec674f17 100644 --- a/viadot/tasks/genesys.py +++ b/viadot/tasks/genesys.py @@ -18,9 +18,7 @@ class GenesysToCSV(Task): def __init__( self, report_name: str = "genesys_to_csv", - view_type: Literal[ - "queue_performance_detail_view", "agent_performance_summary_view" - ] = "queue_performance_detail_view", + view_type: str = "queue_performance_detail_view", media_type_list: List[str] = None, queueIds_list: List[str] = None, data_to_post_str: str = None, @@ -39,8 +37,7 @@ def __init__( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail_view". + view_type (str, optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. data_to_post_str (str, optional): String template to generate json body. Defaults to None. @@ -98,11 +95,7 @@ def __call__(self, *args, **kwargs): def run( self, report_name: str = None, - view_type: Literal[ - "queue_performance_detail_view", - "agent_performance_summary_view", - "agent_status_summary_view", - ] = "queue_performance_detail_view", + view_type: str = "queue_performance_detail_view", view_type_time_sleep: int = 80, environment: str = None, schedule_id: str = None, @@ -121,8 +114,7 @@ def run( Args: report_name (str, optional): The name of this task. Defaults to a general name 'genesys_to_csv'. - view_type (Literal[queue_performance_detail_view, agent_performance_summary_view], optional): - The type of view export job to be created. Defaults to "queue_performance_detail_view". + view_type (str, optional): The type of view export job to be created. Defaults to "queue_performance_detail_view". view_type_time_sleep (int, optional): Waiting time to retrieve data from Genesys API. Defaults to 80. media_type_list (List[str], optional): List of specific media types. Defaults to None. queueIds_list (List[str], optional): List of specific queues ids. Defaults to None. From 70a1650490190818d5757c6e35e1d49f64b1b7b7 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 28 Nov 2022 16:59:32 +0100 Subject: [PATCH 22/27] =?UTF-8?q?=F0=9F=90=9B=20outlook=20messages=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/sources/outlook.py | 136 +++++++++++++++++++++----------------- 1 file changed, 76 insertions(+), 60 deletions(-) diff --git a/viadot/sources/outlook.py b/viadot/sources/outlook.py index d67c9ea27..661f755b9 100644 --- a/viadot/sources/outlook.py +++ b/viadot/sources/outlook.py @@ -82,7 +82,7 @@ def __init__( print(f"{self.mailbox_name} NOT Authenticated!") self.mailbox_obj = self.account.mailbox() - self.mailbox_messages = self.mailbox_obj.get_messages(limit) + self.limit = limit super().__init__(*args, credentials=self.credentials, **kwargs) def to_df(self) -> pd.DataFrame: @@ -92,66 +92,82 @@ def to_df(self) -> pd.DataFrame: pd.DataFrame: the DataFrame with time range """ data = [] - - while True: - try: - message = next(self.mailbox_messages) - received_time = message.received - date_time_str = str(received_time) - date_time_string = date_time_str[0:19] - date_obj = datetime.datetime.strptime( - date_time_string, "%Y-%m-%d %H:%M:%S" - ) - if ( - date_obj < self.date_range_start_time - or date_obj > self.date_range_end_time - ): - continue - else: - fetched = message.to_api_data() - try: - sender_mail = fetched["from"]["emailAddress"]["address"] - reciver_list = fetched.get("toRecipients") - recivers = " " - if reciver_list is not None: - recivers = ", ".join( - reciver["emailAddress"]["address"] - for reciver in reciver_list - ) - - categories = " " - if message.categories is not None: - categories = ", ".join( - categories for categories in message.categories + sent_folder = self.mailbox_obj.sent_folder() + inbox_folder = self.mailbox_obj.inbox_folder() + junk_folder = self.mailbox_obj.junk_folder() + deleted_folder = self.mailbox_obj.deleted_folder() + drafts_folder = self.mailbox_obj.drafts_folder() + outbox_folder = self.mailbox_obj.outbox_folder() + archive_folder = self.mailbox_obj.archive_folder() + + for mailbox_generator in [ + sent_folder.get_messages(self.limit), + inbox_folder.get_messages(self.limit), + junk_folder.get_messages(self.limit), + deleted_folder.get_messages(self.limit), + drafts_folder.get_messages(self.limit), + outbox_folder.get_messages(self.limit), + archive_folder.get_messages(self.limit), + ]: + while True: + try: + message = next(mailbox_generator) + received_time = message.received + date_time_str = str(received_time) + date_time_string = date_time_str[0:19] + date_obj = datetime.datetime.strptime( + date_time_string, "%Y-%m-%d %H:%M:%S" + ) + if ( + date_obj < self.date_range_start_time + or date_obj > self.date_range_end_time + ): + continue + else: + fetched = message.to_api_data() + try: + sender_mail = fetched["from"]["emailAddress"]["address"] + reciver_list = fetched.get("toRecipients") + recivers = " " + if reciver_list is not None: + recivers = ", ".join( + reciver["emailAddress"]["address"] + for reciver in reciver_list + ) + + categories = " " + if message.categories is not None: + categories = ", ".join( + categories for categories in message.categories + ) + + conversation_index = " " + if message.conversation_index is not None: + conversation_index = message.conversation_index + row = { + "conversation ID": fetched.get("conversationId"), + "conversation index": conversation_index, + "categories": categories, + "sender": sender_mail, + "recivers": recivers, + "received_time": fetched.get("receivedDateTime"), + } + + row["mail_adress"] = ( + self.mailbox_name.split("@")[0] + .replace(".", "_") + .replace("-", "_") ) - - conversation_index = " " - if message.conversation_index is not None: - conversation_index = message.conversation_index - row = { - "conversation ID": fetched.get("conversationId"), - "conversation index": conversation_index, - "categories": categories, - "sender": sender_mail, - "recivers": recivers, - "received_time": fetched.get("receivedDateTime"), - } - - row["mail_adress"] = ( - self.mailbox_name.split("@")[0] - .replace(".", "_") - .replace("-", "_") - ) - if sender_mail == self.mailbox_name: - row["Inbox"] = False - else: - row["Inbox"] = True - - data.append(row) - except KeyError as e: - print("KeyError : " + str(e)) - except StopIteration: - break + if sender_mail == self.mailbox_name: + row["Inbox"] = False + else: + row["Inbox"] = True + + data.append(row) + except KeyError as e: + print("KeyError : " + str(e)) + except StopIteration: + break df = pd.DataFrame(data=data) return df From 5e5fbbbe7f31a8babd815364d52c736195e61014 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 29 Nov 2022 16:10:27 +0100 Subject: [PATCH 23/27] =?UTF-8?q?=F0=9F=8E=A8=20Added=20option=20to=20sele?= =?UTF-8?q?ct=20folders?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/sources/outlook.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/viadot/sources/outlook.py b/viadot/sources/outlook.py index 661f755b9..cc3062c06 100644 --- a/viadot/sources/outlook.py +++ b/viadot/sources/outlook.py @@ -1,5 +1,5 @@ import datetime -from typing import Any, Dict, List +from typing import Any, Dict, List, Literal import pandas as pd from O365 import Account @@ -17,6 +17,9 @@ def __init__( end_date: str = None, credentials: Dict[str, Any] = None, limit: int = 10000, + mailbox_folders: Literal[ + "sent", "inbox", "junk", "deleted", "drafts", "outbox", "archive" + ] = ["sent", "inbox", "junk", "deleted", "drafts", "outbox", "archive"], request_retries: int = 10, *args: List[Any], **kwargs: Dict[str, Any], @@ -34,6 +37,9 @@ def __init__( ACCOUNT_NAME and Service Principal credentials (TENANT_ID, CLIENT_ID, CLIENT_SECRET) for the Azure Application. Defaults to None. limit (int, optional): Number of fetched top messages. Defaults to 10000. + mailbox_folders (Literal["sent", "inbox", "junk", "deleted", "drafts", "outbox", "archive"]): + List of folders to select from the mailbox. Defaults to ["sent", "inbox", "junk", "deleted", "drafts", "outbox", "archive"] + request_retries (int): How many times retries to authorizate. Defaults to 10. """ try: DEFAULT_CREDENTIALS = local_config["OUTLOOK"] @@ -82,6 +88,7 @@ def __init__( print(f"{self.mailbox_name} NOT Authenticated!") self.mailbox_obj = self.account.mailbox() + self.mailbox_folders = mailbox_folders self.limit = limit super().__init__(*args, credentials=self.credentials, **kwargs) @@ -92,23 +99,12 @@ def to_df(self) -> pd.DataFrame: pd.DataFrame: the DataFrame with time range """ data = [] - sent_folder = self.mailbox_obj.sent_folder() - inbox_folder = self.mailbox_obj.inbox_folder() - junk_folder = self.mailbox_obj.junk_folder() - deleted_folder = self.mailbox_obj.deleted_folder() - drafts_folder = self.mailbox_obj.drafts_folder() - outbox_folder = self.mailbox_obj.outbox_folder() - archive_folder = self.mailbox_obj.archive_folder() - - for mailbox_generator in [ - sent_folder.get_messages(self.limit), - inbox_folder.get_messages(self.limit), - junk_folder.get_messages(self.limit), - deleted_folder.get_messages(self.limit), - drafts_folder.get_messages(self.limit), - outbox_folder.get_messages(self.limit), - archive_folder.get_messages(self.limit), - ]: + mailbox_generators_list = [] + for m in self.mailbox_folders: + base_str = f"self.mailbox_obj.{m}_folder().get_messages({self.limit})" + mailbox_generators_list.append(eval(base_str)) + + for mailbox_generator in mailbox_generators_list: while True: try: message = next(mailbox_generator) From 99cc9a258490861121f90b3f4e72fee61e940349 Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 1 Dec 2022 11:40:40 +0100 Subject: [PATCH 24/27] PR correction --- viadot/sources/outlook.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/viadot/sources/outlook.py b/viadot/sources/outlook.py index cc3062c06..1364d579f 100644 --- a/viadot/sources/outlook.py +++ b/viadot/sources/outlook.py @@ -88,15 +88,29 @@ def __init__( print(f"{self.mailbox_name} NOT Authenticated!") self.mailbox_obj = self.account.mailbox() - self.mailbox_folders = mailbox_folders + + allowed_mailbox_folders = [ + "sent", + "inbox", + "junk", + "deleted", + "drafts", + "outbox", + "archive", + ] + if all(x in allowed_mailbox_folders for x in mailbox_folders): + self.mailbox_folders = mailbox_folders + else: + raise Exception("Provided mailbox folders are not correct.") self.limit = limit + super().__init__(*args, credentials=self.credentials, **kwargs) def to_df(self) -> pd.DataFrame: """Download Outlook data into a pandas DataFrame. Returns: - pd.DataFrame: the DataFrame with time range + pd.DataFrame: the DataFrame with time range. """ data = [] mailbox_generators_list = [] @@ -110,9 +124,9 @@ def to_df(self) -> pd.DataFrame: message = next(mailbox_generator) received_time = message.received date_time_str = str(received_time) - date_time_string = date_time_str[0:19] + date_time_stripped = date_time_str[0:19] date_obj = datetime.datetime.strptime( - date_time_string, "%Y-%m-%d %H:%M:%S" + date_time_stripped, "%Y-%m-%d %H:%M:%S" ) if ( date_obj < self.date_range_start_time @@ -123,12 +137,12 @@ def to_df(self) -> pd.DataFrame: fetched = message.to_api_data() try: sender_mail = fetched["from"]["emailAddress"]["address"] - reciver_list = fetched.get("toRecipients") + recivers_list = fetched.get("toRecipients") recivers = " " - if reciver_list is not None: + if recivers_list is not None: recivers = ", ".join( reciver["emailAddress"]["address"] - for reciver in reciver_list + for reciver in recivers_list ) categories = " " From bcfc94d8111063af1f09e9a906dc672778f8eb1f Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 12 Dec 2022 15:20:34 +0100 Subject: [PATCH 25/27] =?UTF-8?q?=F0=9F=8E=A8=20added=20PR=20requested=20c?= =?UTF-8?q?hanges?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- viadot/flows/outlook_to_adls.py | 10 ++++++++-- viadot/sources/outlook.py | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/viadot/flows/outlook_to_adls.py b/viadot/flows/outlook_to_adls.py index bd9e3cf8d..c5e86346d 100644 --- a/viadot/flows/outlook_to_adls.py +++ b/viadot/flows/outlook_to_adls.py @@ -8,6 +8,7 @@ df_to_csv, df_to_parquet, union_dfs_task, + credentials_loader, ) from ..tasks import AzureDataLakeUpload, OutlookToDF @@ -29,6 +30,7 @@ def __init__( limit: int = 10000, timeout: int = 1200, if_exists: Literal["append", "replace", "skip"] = "append", + outlook_credentials_secret: str = "OUTLOOK", *args: List[Any], **kwargs: Dict[str, Any], ): @@ -46,6 +48,7 @@ def __init__( overwrite_adls (bool, optional): Whether to overwrite the file in ADLS. Defaults to True. adls_sp_credentials_secret (str, optional): The name of the Azure Key Vault secret containing a dictionary with ACCOUNT_NAME and Service Principal credentials (TENANT_ID, CLIENT_ID, CLIENT_SECRET) for the Azure Data Lake. Defaults to None. + outlook_credentials_secret (str, optional): The name of the Azure Key Vault secret containing a dictionary with outlook credentials. limit (int, optional): Number of fetched top messages. Defaults to 10000. timeout (int, optional): The amount of time (in seconds) to wait while running this task before a timeout occurs. Defaults to 1200. if_exists (Literal['append', 'replace', 'skip'], optional): What to do if the local file already exists. Defaults to "append". @@ -64,6 +67,7 @@ def __init__( self.output_file_extension = output_file_extension self.overwrite_adls = overwrite_adls self.adls_sp_credentials_secret = adls_sp_credentials_secret + self.outlook_credentials_secret = outlook_credentials_secret super().__init__(*args, name=name, **kwargs) @@ -72,8 +76,10 @@ def __init__( def gen_outlook_df( self, mailbox_list: Union[str, List[str]], flow: Flow = None ) -> Task: - - outlook_to_df = OutlookToDF(timeout=self.timeout) + credentials = credentials_loader.run( + credentials_secret=self.outlook_credentials_secret + ) + outlook_to_df = OutlookToDF(timeout=self.timeout, credentials=credentials) df = outlook_to_df.bind( mailbox_name=mailbox_list, diff --git a/viadot/sources/outlook.py b/viadot/sources/outlook.py index 1364d579f..260e98b8b 100644 --- a/viadot/sources/outlook.py +++ b/viadot/sources/outlook.py @@ -7,6 +7,7 @@ from ..config import local_config from ..exceptions import CredentialError from .base import Source +import prefect class Outlook(Source): @@ -39,13 +40,13 @@ def __init__( limit (int, optional): Number of fetched top messages. Defaults to 10000. mailbox_folders (Literal["sent", "inbox", "junk", "deleted", "drafts", "outbox", "archive"]): List of folders to select from the mailbox. Defaults to ["sent", "inbox", "junk", "deleted", "drafts", "outbox", "archive"] - request_retries (int): How many times retries to authorizate. Defaults to 10. + request_retries (int, optional): How many times retries to authorizate. Defaults to 10. """ - try: - DEFAULT_CREDENTIALS = local_config["OUTLOOK"] - except KeyError: - DEFAULT_CREDENTIALS = None - self.credentials = credentials or DEFAULT_CREDENTIALS + + self.logger = prefect.context.get("logger") + + self.credentials = credentials + if self.credentials is None: raise CredentialError("You do not provide credentials!") @@ -83,9 +84,9 @@ def __init__( request_retries=self.request_retries, ) if self.account.authenticate(): - print(f"{self.mailbox_name} Authenticated!") + self.logger.info(f"{self.mailbox_name} Authenticated!") else: - print(f"{self.mailbox_name} NOT Authenticated!") + self.logger.info(f"{self.mailbox_name} NOT Authenticated!") self.mailbox_obj = self.account.mailbox() @@ -175,7 +176,7 @@ def to_df(self) -> pd.DataFrame: data.append(row) except KeyError as e: - print("KeyError : " + str(e)) + self.logger.info("KeyError : " + str(e)) except StopIteration: break df = pd.DataFrame(data=data) From 1f9ae40ba4ba0a15bd4fe4bdd3c461c3c20b235f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Ziemianek?= <49795849+Rafalz13@users.noreply.github.com> Date: Wed, 14 Dec 2022 11:45:11 +0100 Subject: [PATCH 26/27] Moved information about changes to 'unreleased' --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66a0b3908..3c8d9cadb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Added into `Genesys` the new view type `AGENT`. + ## [0.4.10] - 2022-11-16 ### Added @@ -19,7 +22,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Changed `duckdb` version to `0.5.1` - Added new column into Data Frames created with `Mindful`. - Added region parameter as an entry argument in `MindfulToADLS`. -- Added into `Genesys` the new view type `AGENT`. ### Fixed - Fixed incorrect `if_exists="delete"` handling in `DuckDB.create_table_from_parquet()` From d2eae4278894759ca2fa8ba09cd697ab85544796 Mon Sep 17 00:00:00 2001 From: Rafalz13 Date: Thu, 15 Dec 2022 13:41:23 +0100 Subject: [PATCH 27/27] =?UTF-8?q?=F0=9F=93=9D=20Updated=20Changelog=20befo?= =?UTF-8?q?re=20release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c8d9cadb..178084905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + + +# [0.4.11] - 2022-12-15 ### Added - Added into `Genesys` the new view type `AGENT`. +### Changed +- Changed data extraction logic for `Outlook` data. + ## [0.4.10] - 2022-11-16 ### Added