Fix graph startdate filter (#6)

* graph api fix, change start date filter mechanics for auth api
logzio · Aug 23, 2023 · e9f1bf7 · e9f1bf7
1 parent db2db61
commit e9f1bf7
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -240,5 +240,9 @@ If you stopped the container, you can continue from the exact place you stopped,
 
 ## Changelog:
 
+- **0.0.5**:
+  - Bug fix for `azure_graph` task fails on second cycle.
+  - Changed start date filter mechanics for auth_api.
+
 - **0.0.4**:
   - Bug fix for `azure_graph` task fails when trying to get start date from file.
diff --git a/src/api.py b/src/api.py
@@ -14,7 +14,6 @@
 from .data.base_data.api_custom_field import ApiCustomField
 from .data.general_type_data.api_general_type_data import ApiGeneralTypeData
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -54,16 +53,12 @@ def _get_json_path_value_from_data(self, json_path: str, data: dict) -> Any:
 
     def update_start_date_filter(self) -> None:
         new_start_date = self._get_new_start_date()
-
         filter_index = self._base_data.get_filter_index(
             self._general_type_data.start_date_name)
 
-        if filter_index == -1:
-            self._base_data.append_filters(ApiFilter(
-                self._general_type_data.start_date_name, new_start_date))
-            return
-
-        self._base_data.update_filter_value(filter_index, new_start_date)
+        # If date is a filter in the filters list, update the list value (cisco secure x).
+        if filter_index != -1:
+            self._base_data.update_filter_value(filter_index, new_start_date)
 
     def get_last_start_date(self) -> Optional[str]:
         for api_filter in self._base_data.filters:
@@ -107,6 +102,17 @@ def _is_item_in_fetch_frame(self, item: dict, last_datetime_to_fetch: datetime)
 
         return True
 
+    def get_start_date_filter(self) -> str:
+        if self._current_data_last_date is not None:
+            start_date_str = self._get_new_start_date()
+            new_start_date = start_date_str.split('.')[0]
+        else:
+            start_date = datetime.utcnow() - timedelta(days=self.base_data.settings.days_back_to_fetch)
+            new_start_date = start_date.isoformat(' ', 'seconds')
+            new_start_date = new_start_date.replace(' ', 'T')
+            new_start_date += 'Z'
+        return new_start_date
+
     def _get_new_start_date(self) -> str:
         new_start_date = str(parser.parse(self._current_data_last_date) + timedelta(seconds=1))
         new_start_date = new_start_date.replace(' ', 'T')
@@ -115,14 +121,17 @@ def _get_new_start_date(self) -> str:
         return new_start_date
 
     def _get_data_from_api(self, url: str) -> tuple[Optional[str], list]:
+        next_url = None
         try:
             response = self._get_response_from_api(url)
         except Exception:
             raise
 
         json_data = json.loads(response.content)
-        next_url = self._get_json_path_value_from_data(
-            self._general_type_data.json_paths.next_url, json_data)
+
+        if self._general_type_data.json_paths.next_url:
+            next_url = self._get_json_path_value_from_data(
+                self._general_type_data.json_paths.next_url, json_data)
         data = self._get_json_path_value_from_data(
             self._general_type_data.json_paths.data, json_data)
 

diff --git a/src/auth_api.py b/src/auth_api.py
@@ -11,7 +11,6 @@
 from .data.base_data.auth_api_base_data import AuthApiBaseData
 from .data.general_type_data.auth_api_general_type_data import AuthApiGeneralTypeData
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -65,9 +64,14 @@ def fetch_data(self) -> Generator[str, None, list[str]]:
     def _build_api_url(self) -> str:
         api_url = self._api_http_request.url
         api_filters_num = self._base_data.get_filters_size()
-
-        if api_filters_num > 0:
-            api_url += '?'
+        date_filter_index = self._base_data.get_filter_index(
+            self._general_type_data.start_date_name)
+        api_url += '?'
+        if date_filter_index == -1:
+            new_start_date = self.get_start_date_filter()
+            api_url += self._general_type_data.start_date_name + '=' + new_start_date
+            if api_filters_num > 0:
+                api_url += '&'
 
         for api_filter in self._base_data.filters:
             api_url += api_filter.key + '=' + str(api_filter.value)

diff --git a/src/azure_graph.py b/src/azure_graph.py
@@ -25,14 +25,8 @@ def get_last_start_date(self) -> str:
     def _build_api_url(self) -> str:
         api_url = self._data_request.url
         api_filters_num = self._base_data.get_filters_size()
-        if self._current_data_last_date is not None:
-            start_date_str = self._get_new_start_date()
-            new_start_date = start_date_str.split('.')[0]
-        else:
-            start_date = datetime.utcnow() - timedelta(days=self.base_data.settings.days_back_to_fetch)
-            new_start_date = start_date.isoformat(' ', 'seconds')
-            new_start_date = new_start_date.replace(' ', 'T')
-        api_url += "?$filter=" + self._general_type_data.json_paths.data_date + ' gt ' + new_start_date + 'Z'
+        new_start_date = self.get_start_date_filter()
+        api_url += "?$filter=" + self._general_type_data.json_paths.data_date + ' gt ' + new_start_date
         if api_filters_num > 0:
             api_url += '&$'
         for api_filter in self._base_data.filters:

diff --git a/src/data/base_data/api_base_data.py b/src/data/base_data/api_base_data.py
@@ -8,12 +8,14 @@
 class ApiBaseData:
 
     def __init__(self, api_type: str, api_name: str, api_credentials: ApiCredentials, api_settings: ApiSettings,
-                 api_filters: list[ApiFilter], api_custom_fields: list[ApiCustomField]) -> None:
+                 api_filters: list[ApiFilter],
+                 api_custom_fields: list[ApiCustomField], date_filter: str = None) -> None:
         self._type = api_type
         self._name = api_name
         self._credentials = api_credentials
         self._settings = api_settings
         self._filters = api_filters
+        self._date_filter_value = date_filter
         self._custom_fields = api_custom_fields
 
     @property

diff --git a/src/oauth_api.py b/src/oauth_api.py
@@ -12,7 +12,6 @@
 from .data.base_data.oauth_api_base_data import OAuthApiBaseData
 from .data.general_type_data.oauth_api_general_type_data import OAuthApiGeneralTypeData
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -105,7 +104,7 @@ def _build_api_url(self) -> str:
         return api_url
 
     def _send_request(self, url: str) -> Response:
-        request_headers = {self.OAUTH_AUTHORIZATION_HEADER: self.token,
+        request_headers = {self.OAUTH_AUTHORIZATION_HEADER: f"Bearer {self.token}",
                            self.OAUTH_TOKEN_REQUEST_CONTENT_TYPE: self.OAUTH_APPLICATION_JSON_CONTENT_TYPE}
         if self._data_request.headers is not None:
             request_headers.update(self._data_request.headers)
@@ -116,7 +115,7 @@ def _send_request(self, url: str) -> Response:
         else:
             response = requests.post(url=url,
                                      headers=request_headers,
-                                     data=self._data_request.body)
+                                     data=json.dumps(self._data_request.body))
 
         return response