From e9f1bf70eea0564e25a3580dd24be55b8f32b9a8 Mon Sep 17 00:00:00 2001 From: tamirmich Date: Wed, 23 Aug 2023 18:01:17 +0300 Subject: [PATCH] Fix graph startdate filter (#6) * graph api fix, change start date filter mechanics for auth api --- README.md | 4 ++++ src/api.py | 29 +++++++++++++++++++---------- src/auth_api.py | 12 ++++++++---- src/azure_graph.py | 10 ++-------- src/data/base_data/api_base_data.py | 4 +++- src/oauth_api.py | 5 ++--- 6 files changed, 38 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index daf455b..2371c74 100644 --- a/README.md +++ b/README.md @@ -240,5 +240,9 @@ If you stopped the container, you can continue from the exact place you stopped, ## Changelog: +- **0.0.5**: + - Bug fix for `azure_graph` task fails on second cycle. + - Changed start date filter mechanics for auth_api. + - **0.0.4**: - Bug fix for `azure_graph` task fails when trying to get start date from file. diff --git a/src/api.py b/src/api.py index 53769d8..ec7c075 100644 --- a/src/api.py +++ b/src/api.py @@ -14,7 +14,6 @@ from .data.base_data.api_custom_field import ApiCustomField from .data.general_type_data.api_general_type_data import ApiGeneralTypeData - logger = logging.getLogger(__name__) @@ -54,16 +53,12 @@ def _get_json_path_value_from_data(self, json_path: str, data: dict) -> Any: def update_start_date_filter(self) -> None: new_start_date = self._get_new_start_date() - filter_index = self._base_data.get_filter_index( self._general_type_data.start_date_name) - if filter_index == -1: - self._base_data.append_filters(ApiFilter( - self._general_type_data.start_date_name, new_start_date)) - return - - self._base_data.update_filter_value(filter_index, new_start_date) + # If date is a filter in the filters list, update the list value (cisco secure x). + if filter_index != -1: + self._base_data.update_filter_value(filter_index, new_start_date) def get_last_start_date(self) -> Optional[str]: for api_filter in self._base_data.filters: @@ -107,6 +102,17 @@ def _is_item_in_fetch_frame(self, item: dict, last_datetime_to_fetch: datetime) return True + def get_start_date_filter(self) -> str: + if self._current_data_last_date is not None: + start_date_str = self._get_new_start_date() + new_start_date = start_date_str.split('.')[0] + else: + start_date = datetime.utcnow() - timedelta(days=self.base_data.settings.days_back_to_fetch) + new_start_date = start_date.isoformat(' ', 'seconds') + new_start_date = new_start_date.replace(' ', 'T') + new_start_date += 'Z' + return new_start_date + def _get_new_start_date(self) -> str: new_start_date = str(parser.parse(self._current_data_last_date) + timedelta(seconds=1)) new_start_date = new_start_date.replace(' ', 'T') @@ -115,14 +121,17 @@ def _get_new_start_date(self) -> str: return new_start_date def _get_data_from_api(self, url: str) -> tuple[Optional[str], list]: + next_url = None try: response = self._get_response_from_api(url) except Exception: raise json_data = json.loads(response.content) - next_url = self._get_json_path_value_from_data( - self._general_type_data.json_paths.next_url, json_data) + + if self._general_type_data.json_paths.next_url: + next_url = self._get_json_path_value_from_data( + self._general_type_data.json_paths.next_url, json_data) data = self._get_json_path_value_from_data( self._general_type_data.json_paths.data, json_data) diff --git a/src/auth_api.py b/src/auth_api.py index 1a0fd26..dfbc503 100644 --- a/src/auth_api.py +++ b/src/auth_api.py @@ -11,7 +11,6 @@ from .data.base_data.auth_api_base_data import AuthApiBaseData from .data.general_type_data.auth_api_general_type_data import AuthApiGeneralTypeData - logger = logging.getLogger(__name__) @@ -65,9 +64,14 @@ def fetch_data(self) -> Generator[str, None, list[str]]: def _build_api_url(self) -> str: api_url = self._api_http_request.url api_filters_num = self._base_data.get_filters_size() - - if api_filters_num > 0: - api_url += '?' + date_filter_index = self._base_data.get_filter_index( + self._general_type_data.start_date_name) + api_url += '?' + if date_filter_index == -1: + new_start_date = self.get_start_date_filter() + api_url += self._general_type_data.start_date_name + '=' + new_start_date + if api_filters_num > 0: + api_url += '&' for api_filter in self._base_data.filters: api_url += api_filter.key + '=' + str(api_filter.value) diff --git a/src/azure_graph.py b/src/azure_graph.py index 78a9a57..cda0df7 100644 --- a/src/azure_graph.py +++ b/src/azure_graph.py @@ -25,14 +25,8 @@ def get_last_start_date(self) -> str: def _build_api_url(self) -> str: api_url = self._data_request.url api_filters_num = self._base_data.get_filters_size() - if self._current_data_last_date is not None: - start_date_str = self._get_new_start_date() - new_start_date = start_date_str.split('.')[0] - else: - start_date = datetime.utcnow() - timedelta(days=self.base_data.settings.days_back_to_fetch) - new_start_date = start_date.isoformat(' ', 'seconds') - new_start_date = new_start_date.replace(' ', 'T') - api_url += "?$filter=" + self._general_type_data.json_paths.data_date + ' gt ' + new_start_date + 'Z' + new_start_date = self.get_start_date_filter() + api_url += "?$filter=" + self._general_type_data.json_paths.data_date + ' gt ' + new_start_date if api_filters_num > 0: api_url += '&$' for api_filter in self._base_data.filters: diff --git a/src/data/base_data/api_base_data.py b/src/data/base_data/api_base_data.py index c198dda..d7c926e 100644 --- a/src/data/base_data/api_base_data.py +++ b/src/data/base_data/api_base_data.py @@ -8,12 +8,14 @@ class ApiBaseData: def __init__(self, api_type: str, api_name: str, api_credentials: ApiCredentials, api_settings: ApiSettings, - api_filters: list[ApiFilter], api_custom_fields: list[ApiCustomField]) -> None: + api_filters: list[ApiFilter], + api_custom_fields: list[ApiCustomField], date_filter: str = None) -> None: self._type = api_type self._name = api_name self._credentials = api_credentials self._settings = api_settings self._filters = api_filters + self._date_filter_value = date_filter self._custom_fields = api_custom_fields @property diff --git a/src/oauth_api.py b/src/oauth_api.py index 743d870..4692372 100644 --- a/src/oauth_api.py +++ b/src/oauth_api.py @@ -12,7 +12,6 @@ from .data.base_data.oauth_api_base_data import OAuthApiBaseData from .data.general_type_data.oauth_api_general_type_data import OAuthApiGeneralTypeData - logger = logging.getLogger(__name__) @@ -105,7 +104,7 @@ def _build_api_url(self) -> str: return api_url def _send_request(self, url: str) -> Response: - request_headers = {self.OAUTH_AUTHORIZATION_HEADER: self.token, + request_headers = {self.OAUTH_AUTHORIZATION_HEADER: f"Bearer {self.token}", self.OAUTH_TOKEN_REQUEST_CONTENT_TYPE: self.OAUTH_APPLICATION_JSON_CONTENT_TYPE} if self._data_request.headers is not None: request_headers.update(self._data_request.headers) @@ -116,7 +115,7 @@ def _send_request(self, url: str) -> Response: else: response = requests.post(url=url, headers=request_headers, - data=self._data_request.body) + data=json.dumps(self._data_request.body)) return response