diff --git a/README.md b/README.md index 592bd0f..0900039 100644 --- a/README.md +++ b/README.md @@ -21,18 +21,19 @@ Configure your API inputs under `apis`. For every API, mention the input type un For structuring custom API calls use type `general` API with the parameters below. ## Configuration Options -| Parameter Name | Description | Required/Optional | Default | -|--------------------|-----------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------| -| name | Name of the API (custom name) | Optional | the defined `url` | -| url | The request URL | Required | - | -| headers | The request Headers | Optional | `{}` | -| body | The request body | Optional | - | -| method | The request method (`GET` or `POST`) | Optional | `GET` | -| pagination | Pagination settings if needed (see [options below](#pagination-configuration-options)) | Optional | - | -| next_url | If needed to update the URL in next requests based on the last response. Supports using variables ([see below](#using-variables)) | Optional | - | -| response_data_path | The path to the data inside the response | Optional | response root | -| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | Add `type` as `api-fetcher` | -| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| Parameter Name | Description | Required/Optional | Default | +|--------------------|---------------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| url | The request URL | Required | - | +| headers | The request Headers | Optional | `{}` | +| body | The request body | Optional | - | +| method | The request method (`GET` or `POST`) | Optional | `GET` | +| pagination | Pagination settings if needed (see [options below](#pagination-configuration-options)) | Optional | - | +| next_url | If needed to update the URL in the next request based on the last response. Supports using variables ([see below](#using-variables)) | Optional | - | +| next_body | If needed to update the body in the next request based on the last response. Supports using variables ([see below](#using-variables)) | Optional | - | +| response_data_path | The path to the data inside the response | Optional | response root | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | Add `type` as `api-fetcher` | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | ## Pagination Configuration Options If needed, you can configure pagination. @@ -207,6 +208,27 @@ By default `cloudflare` API type: | scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | | pagination_off | True if builtin pagination should be off, False otherwise | Optional | `False` | + +
+ + 1Password + + +By default `1password` API type has built in pagination settings and sets the `response_data_path` to `items` field. + +## Configuration Options +| Parameter Name | Description | Required/Optional | Default | +|--------------------------|--------------------------------------------------------------------------------------------------------------|-------------------|-------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| onepassword_bearer_token | The 1Password Bearer token | Required | - | +| url | The request URL | Required | - | +| method | The request method (`GET` or `POST`) | Optional | `GET` | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | +| days_back_fetch | The amount of days to fetch back in the first request. Applies a filter on 1password `start_time` parameter. | Optional | - | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| onepassword_limit | 1Password limit for number of events to return in a single request (allowed range: 100 to 1000) | Optional | 100 | +| pagination_off | True if builtin pagination should be off, False otherwise | Optional | `False` | +
@@ -262,6 +284,10 @@ docker stop -t 30 logzio-api-fetcher ``` ## Changelog: +- **0.2.1**: + - Add 1Password Support + - Add `next_body` support to allow more customization in general settings + - Support integers and boolean as values in pagination 'equals' stop condition - **0.2.0**: - **Breaking changes!!** - Deprecate configuration fields: diff --git a/src/apis/general/Api.py b/src/apis/general/Api.py index bcc1f70..591309a 100644 --- a/src/apis/general/Api.py +++ b/src/apis/general/Api.py @@ -3,7 +3,7 @@ import logging from pydantic import BaseModel, Field import requests -from typing import Union +from typing import Union, Optional from src.utils.processing_functions import extract_vars, substitute_vars from src.apis.general.PaginationSettings import PaginationSettings, PaginationType @@ -30,31 +30,37 @@ class ApiFetcher(BaseModel): :param method: Optional, the method to use for the request (default: GET) :param pagination_settings: Optional, PaginationSettings object that defines how to perform pagination :param next_url: Optional, If needed update a param in the url according to the response as we go + :param next_body: Optional, If needed update a param in the body according to the response as we go :param response_data_path: Optional, The path to find the data within the response. :param additional_fields: Optional, 'key: value' pairs that should be added to the API logs. :param scrape_interval_minutes: the interval between scraping jobs. :param url_vars: Not passed to the class, array of params that is generated based on next_url. + :param body_vars: Not passed to the class, array of params that is generated based on next_body. """ name: str = Field(default="") url: str headers: dict = Field(default={}) body: Union[str, dict, list] = Field(default=None) method: ReqMethod = Field(default=ReqMethod.GET, frozen=True) - pagination_settings: PaginationSettings = Field(default=None, frozen=True, alias="pagination") + pagination_settings: Optional[PaginationSettings] = Field(default=None, frozen=True, alias="pagination") next_url: str = Field(default=None) + next_body: Union[str, dict, list] = Field(default=None) response_data_path: str = Field(default=None, frozen=True) additional_fields: dict = Field(default={}) scrape_interval_minutes: int = Field(default=1, alias="scrape_interval", ge=1) url_vars: list = Field(default=[], init=False, init_var=True) + body_vars: list = Field(default=[], init=False, init_var=True) def __init__(self, **data): """ - Makes sure to format the body and generate the url_vars based on next_url. + Makes sure to format the body and generate the url_vars based on next_url and body_vars based on next_body. :param data: the fields for creation of the class. """ super().__init__(**data) self.body = self._format_body(self.body) + self.next_body = self._format_body(self.next_body) self.url_vars = extract_vars(self.next_url) + self.body_vars = extract_vars(self.next_body) if not self.name: self.name = self.url if not self.additional_fields.get("type"): @@ -208,6 +214,14 @@ def update_next_url(self, new_next_url): self.next_url = new_next_url self.url_vars = extract_vars(self.next_url) + def update_next_body(self, new_next_body): + """ + Supports updating the next request body format to make sure the 'self.body_vars' is updated accordingly. + :param new_next_body: new format for the next body. (if in future some customized APIs will need it supported) + """ + self.next_body = new_next_body + self.body_vars = extract_vars(self.next_body) + def send_request(self): """ Manages the request: @@ -236,4 +250,8 @@ def send_request(self): # Update the url if needed if self.next_url: self.url = substitute_vars(self.next_url, self.url_vars, r) + + # Update the body if needed + if self.next_body: + self.body = substitute_vars(self.next_body, self.body_vars, r) return responses diff --git a/src/apis/general/README.md b/src/apis/general/README.md index dd7259f..d86db9e 100644 --- a/src/apis/general/README.md +++ b/src/apis/general/README.md @@ -5,18 +5,19 @@ For structuring custom API calls use type `general` API with the parameters belo - [Example](#example) ## Configuration -| Parameter Name | Description | Required/Optional | Default | -|--------------------|-----------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------| -| name | Name of the API (custom name) | Optional | the defined `url` | -| url | The request URL | Required | - | -| headers | The request Headers | Optional | `{}` | -| body | The request body | Optional | - | -| method | The request method (`GET` or `POST`) | Optional | `GET` | -| pagination | Pagination settings if needed (see [options below](#pagination-configuration-options)) | Optional | - | -| next_url | If needed to update the URL in next requests based on the last response. Supports using variables ([see below](#using-variables)) | Optional | - | -| response_data_path | The path to the data inside the response | Optional | response root | -| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | Add `type` as `api-fetcher` | -| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| Parameter Name | Description | Required/Optional | Default | +|--------------------|---------------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| url | The request URL | Required | - | +| headers | The request Headers | Optional | `{}` | +| body | The request body | Optional | - | +| method | The request method (`GET` or `POST`) | Optional | `GET` | +| pagination | Pagination settings if needed (see [options below](#pagination-configuration-options)) | Optional | - | +| next_url | If needed to update the URL in the next request based on the last response. Supports using variables ([see below](#using-variables)) | Optional | - | +| next_body | If needed to update the body in the next request based on the last response. Supports using variables ([see below](#using-variables)) | Optional | - | +| response_data_path | The path to the data inside the response | Optional | response root | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | Add `type` as `api-fetcher` | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | ## Pagination Configuration Options If needed, you can configure pagination. diff --git a/src/apis/general/StopPaginationSettings.py b/src/apis/general/StopPaginationSettings.py index b38a4b2..5637e74 100644 --- a/src/apis/general/StopPaginationSettings.py +++ b/src/apis/general/StopPaginationSettings.py @@ -1,6 +1,7 @@ from enum import Enum import logging from pydantic import BaseModel, Field, model_validator +from typing import Union logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ class StopPaginationSettings(BaseModel): """ field: str condition: StopCondition - value: str = Field(default=None, frozen=True) + value: Union[str, int, bool] = Field(default=None, frozen=True) @model_validator(mode='after') def _check_conditional_fields(self): @@ -32,7 +33,7 @@ def _check_conditional_fields(self): if we got condition as 'contains' or 'equals' >> that we also got value :return: self """ - if self.condition in (StopCondition.EQUALS, StopCondition.CONTAINS) and not self.value: + if self.condition in (StopCondition.EQUALS, StopCondition.CONTAINS) and self.value is None: raise ValueError(f"Used stop condition {self.condition} but missing required 'value' field.") return self diff --git a/src/apis/onepassword/OnePassword.py b/src/apis/onepassword/OnePassword.py new file mode 100644 index 0000000..eb55cbb --- /dev/null +++ b/src/apis/onepassword/OnePassword.py @@ -0,0 +1,85 @@ +from datetime import datetime, timedelta, UTC +import json +import logging +from pydantic import Field + +from src.apis.general.Api import ApiFetcher +from src.apis.general.PaginationSettings import PaginationSettings +from src.apis.general.StopPaginationSettings import StopPaginationSettings + +logger = logging.getLogger(__name__) + + +class OnePassword(ApiFetcher): + """ + :param onepassword_bearer_token: The cloudflare Bearer token + :param pagination_off: True if pagination should be off, False otherwise + :param days_back_fetch: Amount of days to fetch back in the first request, Optional (adds a filter on 'start_time') + :param onepassword_limit: 1Password limit for number of events to return in a single request (for pagination) + """ + onepassword_bearer_token: str = Field(frozen=True) + pagination_off: bool = Field(default=False) + days_back_fetch: int = Field(default=-1, frozen=True) + onepassword_limit: int = Field(default=100, ge=1, le=1000) + + def __init__(self, **data): + # Initialize 1Password limit for number of events to return in a single request + limit = data.pop('onepassword_limit', 100) + + # Configure the request + res_data_path = "items" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {data.get('onepassword_bearer_token')}" + } + body = { + "limit": limit + } + next_body = { + "limit": limit, + "start_time": "{res.items.[0].timestamp}" + } + pagination = None + if not data.get("pagination_off"): + pagination = PaginationSettings(type="body", + body_format={"cursor": "{res.cursor}"}, + stop_indication=StopPaginationSettings(field="has_more", + condition="equals", + value=False)) + super().__init__(headers=headers, body=body, next_body=next_body, pagination=pagination, response_data_path=res_data_path, **data) + + # Initialize the date filter for the first request + if self.days_back_fetch > 0: + self._initialize_body_date() + + def _initialize_body_date(self): + """ + Initialize the first request's 'start_time' body argument. + """ + try: + start_time_field = {"start_time": (datetime.now(UTC) - timedelta(days=self.days_back_fetch)).strftime("%Y-%m-%dT%H:%M:%S.%fZ")} + new_body = json.loads(self.body) + new_body.update(start_time_field) + self.body = json.dumps(new_body) + except json.decoder.JSONDecodeError: + logger.error(f"Failed to update 'start_time' filter in the request body: {self.body}. Sending {self.name} " + f"request with no date filter.") + except TypeError: + logger.error(f"Got unexpected request body parameter. Please make sure the {self.name} API request body is " + f"a valid json.") + + def send_request(self): + """ + 1. Sends request using the super class + 2. In 1Password the latest timestamp is ordered last in the response items >> make sure to take it instead of + the first item. + :return: all the responses that were received + """ + data = super().send_request() + + if data: + latest_timestamp = data[-1].get("timestamp") + self.body = json.loads(self.body) + self.body["start_time"] = latest_timestamp + self.body = json.dumps(self.body) + return data diff --git a/src/apis/onepassword/README.md b/src/apis/onepassword/README.md new file mode 100644 index 0000000..c26e38c --- /dev/null +++ b/src/apis/onepassword/README.md @@ -0,0 +1,35 @@ +# 1Password API Configuration +By default `1password` API type has built in pagination settings and sets the `response_data_path` to `items` field. + +## Configuration +| Parameter Name | Description | Required/Optional | Default | +|--------------------------|--------------------------------------------------------------------------------------------------------------|-------------------|-------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| onepassword_bearer_token | The 1Password Bearer token | Required | - | +| url | The request URL | Required | - | +| method | The request method (`GET` or `POST`) | Optional | `GET` | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | +| days_back_fetch | The amount of days to fetch back in the first request. Applies a filter on 1password `start_time` parameter. | Optional | - | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| onepassword_limit | 1Password limit for number of events to return in a single request (allowed range: 100 to 1000) | Optional | 100 | +| pagination_off | True if builtin pagination should be off, False otherwise | Optional | `False` | + + +## Example +```Yaml +apis: + - name: 1Password example + type: 1password + onepassword_bearer_token: <<1PASSWORD_BEARER_TOKEN>> + url: https://events.1password.com/api/v1/auditevents + method: POST + days_back_fetch: 3 + scrape_interval: 5 + additional_fields: + type: 1password + onepassword_limit: 200 + +logzio: + url: https://listener-eu.logz.io:8071 # for us-east-1 region delete url param (default) + token: <> +``` diff --git a/src/apis/onepassword/__init__.py b/src/apis/onepassword/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/config/ConfigReader.py b/src/config/ConfigReader.py index c66132c..4696a6a 100644 --- a/src/config/ConfigReader.py +++ b/src/config/ConfigReader.py @@ -9,6 +9,7 @@ from src.apis.azure.AzureGraph import AzureGraph from src.apis.azure.AzureMailReports import AzureMailReports from src.apis.cloudflare.Cloudflare import Cloudflare +from src.apis.onepassword.OnePassword import OnePassword from src.output.LogzioShipper import LogzioShipper INPUT_API_FIELD = "apis" @@ -19,7 +20,8 @@ "azure_general": "AzureApi", "azure_graph": "AzureGraph", "azure_mail_reports": "AzureMailReports", - "cloudflare": "Cloudflare" + "cloudflare": "Cloudflare", + "1password": "OnePassword" } logger = logging.getLogger(__name__) diff --git a/tests/UnitTests/responsesExamples/onepassword_res_example.json b/tests/UnitTests/responsesExamples/onepassword_res_example.json new file mode 100644 index 0000000..9ed73d4 --- /dev/null +++ b/tests/UnitTests/responsesExamples/onepassword_res_example.json @@ -0,0 +1,33 @@ +{ + "cursor": "TYUDIKCVN_r45678_WDIJHFD", + "has_more": false, + "items": [ + { + "uuid": "AB5CDE7EXALDK7I2POLDS", + "timestamp": "2024-08-18T16:41:12.669943428Z", + "actor_uuid": "345PSODJKM68DNNDJ4", + "actor_details": { + "uuid": "23456UUID", + "name": "Peter Pan", + "email": "peter@pan.com" + }, + "action": "oiuytr", + "object_type": "asdfgh", + "object_uuid": "OISDUJ6789IOJ", + "aux_info": "{\"expiry\":\"2024-08-14T09:52:13Z\",\"deviceUuid\":\"456sifjlbf63bd\"}", + "session": { + "uuid": "ODIKSCJNFJFJFKDDS", + "login_time": "2024-08-14T09:40:11.253221891Z", + "device_uuid": "frtyuiklkdkdqw", + "ip": "100.100.200.100" + }, + "location": { + "country": "Wonder Land", + "region": "Dark Forest", + "city": "Boro Grove", + "latitude": 27.5195, + "longitude": -177.2484 + } + } + ] +} diff --git a/tests/UnitTests/test_onepassword_api.py b/tests/UnitTests/test_onepassword_api.py new file mode 100644 index 0000000..1d56775 --- /dev/null +++ b/tests/UnitTests/test_onepassword_api.py @@ -0,0 +1,73 @@ +from datetime import datetime, UTC, timedelta +import json +from os.path import abspath, dirname +from pydantic import ValidationError +import responses +import unittest + +from src.apis.general.Api import ReqMethod +from src.apis.onepassword.OnePassword import OnePassword + +curr_path = abspath(dirname(__file__)) + + +class TestOnePasswordApi(unittest.TestCase): + def test_invalid_setup(self): + with self.assertRaises(ValidationError): + OnePassword(url="https://events.1password.com/api/v1/auditevents") + OnePassword(url="https://events.1password.com/api/v1/auditevents", + onepassword_bearer_token="some-token", + onepassword_limit=1001) + OnePassword(url="https://events.1password.com/api/v1/auditevents", + onepassword_bearer_token="some-token", + onepassword_limit=0) + + def test_valid_setup(self): + # Test turning pagination off + op1 = OnePassword(onepassword_bearer_token="some-token", + url="https://events.1password.com/api/v1/someendpoint", + method="POST", + pagination_off=True) + # Test changing the limit + op2 = OnePassword(onepassword_bearer_token="some-token", + url="https://events.1password.com/api/v1/someendpoint", + method="POST", + onepassword_limit=1000) + # Test the days back + op3 = OnePassword(onepassword_bearer_token="some-token", + url="https://events.1password.com/api/v1/someendpoint", + days_back_fetch=5) + + self.assertEqual(op1.onepassword_limit, 100) + self.assertIsNone(op1.pagination_settings) + self.assertIsNotNone(op2.pagination_settings) + self.assertEqual(op3.method, ReqMethod.GET) + + op3_body_start_time = json.loads(op3.body).get("start_time") + self.assertLessEqual(datetime.strptime(op3_body_start_time, "%Y-%m-%dT%H:%M:%S.%fZ").timestamp(), + datetime.now(UTC).timestamp()) + + @responses.activate + def test_onepassword_send_request(self): + with open(f"{curr_path}/responsesExamples/onepassword_res_example.json", "r") as data_res_example_file: + data_res_body = json.loads(data_res_example_file.read()) + # Generate an updated start_time + data_res_body["items"][0]["timestamp"] = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + responses.add(responses.POST, + "https://events.1password.com/api/v1/auditevents", + json=data_res_body, + status=200) + + op = OnePassword(onepassword_bearer_token="some-token", + url="https://events.1password.com/api/v1/auditevents", + method="POST", + days_back_fetch=1) + + pre_req_start_date = json.loads(op.body).get("start_time") + result = op.send_request() + post_req_start_date = json.loads(op.body).get("start_time") + + self.assertLess(datetime.strptime(pre_req_start_date, "%Y-%m-%dT%H:%M:%S.%fZ").timestamp(), + datetime.strptime(post_req_start_date, "%Y-%m-%dT%H:%M:%S.%fZ").timestamp()) + self.assertEqual(result, data_res_body.get("items")) diff --git a/tests/UnitTests/test_utils.py b/tests/UnitTests/test_utils.py index 3a4cb28..0c13427 100644 --- a/tests/UnitTests/test_utils.py +++ b/tests/UnitTests/test_utils.py @@ -86,6 +86,7 @@ def test_substitute_vars(self): no_vars = "just a string with no vars" not_valid_vars = "{field}! just testing {res.obj_arr[2].f1}" empty_val_throw_error = "can also handle {res.none}!" + flattened_obj = "{\"limit\": 100, \"start_time\": \"{res.obj_arr.[1].f2}\"}" # Assert they behave as expected self.assertEqual(substitute_vars(valid_vars, extract_vars(valid_vars), test_dic), @@ -96,3 +97,5 @@ def test_substitute_vars(self): substitute_vars(empty_val_throw_error, extract_vars(empty_val_throw_error), test_dic) self.assertEqual(substitute_vars(no_vars, extract_vars(no_vars), test_dic), no_vars) + self.assertEqual(substitute_vars(flattened_obj, extract_vars(flattened_obj), test_dic), + "{\"limit\": 100, \"start_time\": \"abc\"}")