From 55043350a176702840c6e2b693728425670a687b Mon Sep 17 00:00:00 2001 From: tamirmich Date: Wed, 20 Dec 2023 08:58:58 -0500 Subject: [PATCH] add error handling, improved documentation (#7) --- README.md | 48 ++++++++++++++++++++++++-------------------- src/config_reader.py | 19 +++++++++++++----- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 2371c74..984a08d 100644 --- a/README.md +++ b/README.md @@ -78,28 +78,28 @@ Supported types: The following parameters are for every type: -| Parameter Name | Description | Required/Optional | Default | -| --- | --- | ---| ---| -| type | The type of the auth api. Currently we support the following types: cisco_secure_x, general. | Required | - | -| name | The name of the auth api. Please make names unique. | Required | - | -| credentials.id | The auth api credentials id. | Required | - | -| credentials.key | The auth api credentials key. | Required | - | -| http_request.method | The HTTP method. Can be GET or POST. | Required | - | -| http_request.url | The oauth api url. Make sure the url is without `?` at the end. | Required | - | -| http_request.headers | Pairs of key and value the represents the headers of the HTTP request. | Optional | - | -| http_request.body | The body of the HTTP request. Will be added to HTTP POST requests only. | Optional | - | -| token_http_request.method | The HTTP method. Can be GET or POST. | Required | - | -| token_http_request.url | The oauth api token request url. Make sure the url is without `?` at the end. | Required | - | -| token_http_request.headers | Pairs of key and value the represents the headers of the HTTP request. | Optional | - | -| token_http_request.body | The body of the HTTP request. Will be added to HTTP POST requests only. | Optional | - | -| json_paths.next_url | The json path to the next url value inside the response of the auth api. | Required/Optional for Azure | - | -| json_paths.data | The json path to the data value inside the response of the auth api. | Required/Optional for Azure | - | -| json_paths.data_date | The json path to the data's date value inside the response of the auth api. | Required | - | -| settings.time_interval | The auth api time interval between runs. | Required | - | -| settings.days_back_fetch | The max days back to fetch from the auth api. | Optional | 14 (days) | -| filters | Pairs of key and value of parameters that can be added to the auth api url. Make sure the keys and values are valid for the auth api. | Optional | - | -| custom_fields | Pairs of key and value that will be added to each data and be sent to Logz.io. Create **type** field to override the default type, to search your data easily in Logz.io. | Optional | type = api_fetcher | -| start_date_name| The start date parameter name of the oauth api url. (Same as json_paths.data_date in most cases)| Required | - | +| Parameter Name | Description | Required/Optional | Default | +|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ---| ---| +| type | The type of the auth api. Currently we support the following types: azure_graph, general. | Required | - | +| name | The name of the auth api. Please make names unique. | Required | - | +| credentials.id | The auth api credentials id. | Required | - | +| credentials.key | The auth api credentials key. | Required | - | +| data_http_request.method | The HTTP method. Can be GET or POST. | Required | - | +| data_http_request.url | The oauth api url. Make sure the url is without `?` at the end. | Required | - | +| data_http_request.headers | Pairs of key and value the represents the headers of the HTTP request. | Optional | - | +| data_http_request.body | The body of the HTTP request. Will be added to HTTP POST requests only. | Optional | - | +| token_http_request.method | The HTTP method. Can be GET or POST. | Required | - | +| token_http_request.url | The oauth api token request url. Make sure the url is without `?` at the end. | Required | - | +| token_http_request.headers | Pairs of key and value the represents the headers of the HTTP request. | Optional | - | +| token_http_request.body | The body of the HTTP request. Will be added to HTTP POST requests only. | Optional | - | +| json_paths.next_url | The json path to the next url value inside the response of the auth api. | Required/Optional for Azure | - | +| json_paths.data | The json path to the data value inside the response of the auth api. | Required/Optional for Azure | - | +| json_paths.data_date | The json path to the data's date value inside the response of the auth api. | Required | - | +| settings.time_interval | The auth api time interval between runs. | Required | - | +| settings.days_back_fetch | The max days back to fetch from the auth api. | Optional | 14 (days) | +| filters | Pairs of key and value of parameters that can be added to the auth api url. Make sure the keys and values are valid for the auth api. | Optional | - | +| custom_fields | Pairs of key and value that will be added to each data and be sent to Logz.io. Create **type** field to override the default type, to search your data easily in Logz.io. | Optional | type = api_fetcher | +| start_date_name | The start date parameter name of the oauth api url. (Same as json_paths.data_date in most cases) | Required | - | ### Example @@ -170,6 +170,7 @@ oauth_apis: method: POST data_http_request: url: https://graph.microsoft.com/v1.0/auditLogs/signIns + method: GET headers: json_paths: data_date: createdDateTime @@ -240,6 +241,9 @@ If you stopped the container, you can continue from the exact place you stopped, ## Changelog: +- **0.0.6**: + - Improved documentation. + - Added error log. - **0.0.5**: - Bug fix for `azure_graph` task fails on second cycle. - Changed start date filter mechanics for auth_api. diff --git a/src/config_reader.py b/src/config_reader.py index 5409a4e..c44ed8c 100644 --- a/src/config_reader.py +++ b/src/config_reader.py @@ -152,8 +152,11 @@ def _get_oauth_api_base_data(self, config_oauth_api_data: dict, oauth_api_num: i self._oauth_api_types)) return None - api_token_http_request, api_data_http_request = self._get_oauth_api_http_requests(config_oauth_api_data, - oauth_api_num) + try: + api_token_http_request, api_data_http_request = self._get_oauth_api_http_requests(config_oauth_api_data, + oauth_api_num) + except TypeError: + return None if api_token_http_request is None or api_data_http_request is None: return None @@ -289,6 +292,8 @@ def _get_api_general_type_data(self, config_api_data, api_group_type: str, api_json_paths = self._get_api_json_paths(config_api_data, api_group_type, api_num) if (api_start_date_name is None and api_group_type != self.OAUTH_API) or api_json_paths is None: + logger.error( + "Your configuration is not valid:\"json_paths\" must exist for all api types, \"start_date_name\" must exist for non oauth api types") return None return ApiGeneralTypeData(api_start_date_name, api_json_paths) @@ -297,8 +302,8 @@ def _get_api_start_date_name(self, config_api_data: dict, api_group_type: str, a try: api_start_date_name = config_api_data[ConfigReader.API_START_DATE_NAME_CONFIG_KEY] except KeyError: - logger.error( - "Your configuration is not valid: the general type {0} api #{1} must have start_date_name.".format( + logger.warning( + "Missing field in config: the general type {0} api #{1} must have start_date_name.".format( api_group_type, api_num)) return None @@ -349,11 +354,15 @@ def _get_oauth_api_http_requests(self, config_oauth_api_data: dict, try: api_token_http_request = config_oauth_api_data[ConfigReader.OAUTH_API_TOKEN_HTTP_REQUEST_CONFIG_KEY] api_data_http_request = config_oauth_api_data[ConfigReader.OAUTH_API_DATA_HTTP_REQUEST_CONFIG_KEY] - api_token_http_request_method = api_token_http_request[ConfigReader.API_HTTP_REQUEST_METHOD_CONFIG_KEY] api_token_url = api_token_http_request[ConfigReader.API_HTTP_REQUEST_URL_CONFIG_KEY] api_data_http_request_method = api_data_http_request[ConfigReader.API_HTTP_REQUEST_METHOD_CONFIG_KEY] api_data_url = api_data_http_request[ConfigReader.API_HTTP_REQUEST_URL_CONFIG_KEY] + except KeyError as e: + logger.error( + "Your configuration is not valid: missing parameter: \"{}\" from token_http_request or" + " data_http_request for oauth_api.".format(e.args[0])) + return None except TypeError: logger.error( "Your configuration is not valid: the general type oauth api #{} must have token_http_request and"