diff --git a/.github/workflows/integrationTest.yaml b/.github/workflows/integrationTest.yaml new file mode 100644 index 0000000..6cf7438 --- /dev/null +++ b/.github/workflows/integrationTest.yaml @@ -0,0 +1,39 @@ +name: Automatic integration e2e test +on: + workflow_run: + workflows: ["Automatic unit tests"] + types: [completed] +jobs: + e2e_test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install pytest + - name: Run api fetcher + env: + AZURE_AD_TENANT_ID: ${{ secrets.AZURE_AD_TENANT_ID }} + AZURE_AD_CLIENT_ID: ${{ secrets.AZURE_AD_CLIENT_ID }} + AZURE_AD_SECRET_VALUE: ${{ secrets.AZURE_AD_SECRET_VALUE }} + LOGZIO_SHIPPING_TOKEN: ${{ secrets.LOGZIO_SHIPPING_TOKEN }} + run: | + PYTHONPATH=. python3 ./tests/IntegrationTests/test_shipping.py & + echo $! > script_pid.txt + - name: Wait 30s + run: sleep 30 + - name: Kill api fetcher + run: | + kill -9 $(cat script_pid.txt) + rm script_pid.txt + - name: Wait 1m, to allow logs to get processed and indexed in logzio + run: sleep 60 + - name: Check Logzio for logs + env: + LOGZIO_API_TOKEN: ${{ secrets.LOGZIO_API_TOKEN }} + run: PYTHONPATH=. pytest ./tests/IntegrationTests/test_data_arrived.py diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..1828489 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,29 @@ +name: Build and Push to DockerHub +on: + workflow_dispatch: + release: + types: [published] +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + push: true + platforms: linux/amd64, linux/arm/v7, linux/arm/v8, linux/arm64 + tags: logzio/logzio-api-fetcher:latest, logzio/logzio-api-fetcher:${{ github.event.release.tag_name }} diff --git a/.github/workflows/unitTests.yaml b/.github/workflows/unitTests.yaml new file mode 100644 index 0000000..1547adb --- /dev/null +++ b/.github/workflows/unitTests.yaml @@ -0,0 +1,19 @@ +name: Automatic unit tests +on: [pull_request] +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install responses + pip install pytest + pip install pytest-cov + - name: Run tests + run: PYTHONPATH=. pytest --cov-report term-missing --cov=src tests/UnitTests/test_*.py diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml deleted file mode 100644 index 965d42a..0000000 --- a/.github/workflows/workflow.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: Automatic tests, code-coverage and -on: - workflow_dispatch: - push: - branches: - - main -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Add credentials to config files - run: | - cd tests - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_ID }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_KEY }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_ID }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_VALUE }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_CLIENT_ID }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_TENANT_ID }}/g' @ - - name: Run unit tests - run: | - pip install pytest - pip install httpretty - pip install requests - pip install pyyaml - pip install jsonpath-ng - pip install python-dateutil - pip install pytest-cov - pytest --cov-report xml:code_coverage.xml --cov=src tests/*_tests.py - - name: Code-coverage - run: | - # Get line-rate - line_rate=$(head -2 code_coverage.xml | tail -1 | egrep -o "line-rate=\"[0-1]\.?[0-9]*\"" | egrep -o "[0-1]\.?[0-9]*") - - # Print line-rate - echo | awk -v num=$line_rate '{ printf "line-rate: %d%\n", (num * 100) }' - - # Check code-coverage conditions - echo | awk -v num=$line_rate '{ if (num < 0.8) { printf "line-rate is less than 80%"; exit 1 } else { exit 0 }}' - exit_code=$? - if [ $exit_code -eq 1 ]; then - exit 1 - fi - - name: Log in to Docker Hub - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: my-docker-hub-namespace/my-docker-hub-repository - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 03af5fe..0000000 --- a/.gitignore +++ /dev/null @@ -1,132 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# macOS -.DS_Store diff --git a/README.md b/README.md index da88ab5..592bd0f 100644 --- a/README.md +++ b/README.md @@ -1,261 +1,259 @@ +# Fetcher to send API data to Logz.io +The API Fetcher offers way to configure fetching data from APIs every defined scrape interval. -# Ship Auth/OAuth Api's data to Logz.io - -Every time interval, fetches data of each api in the configuration and sends it to Logz.io. -Every api has set of settings that define it. - - -## Getting Started +## Setup ### Pull Docker Image - Download the logzio-api-fetcher image: ```shell docker pull logzio/logzio-api-fetcher ``` -### Mount a Host Directory as a Data Volume - -Create a local directory and move into it: - -```shell -mkdir logzio-api-fetcher -cd logzio-api-fetcher -``` - ### Configuration - -Create and edit the configuration file and name it `config.yaml`. There are 3 sections of the configuration: - -#### logzio - -| Parameter Name | Description | Required/Optional | Default | -| --- | --- | ---| ---| -| url | The Logz.io Listener URL for your region with port 8071. For example: https://listener.logz.io:8071 | Required | - | -| token | Your Logz.io log shipping token securely directs the data to your Logz.io account. | Required | - | - -#### auth_apis - -Supported types: - -- cisco_secure_x -- general - -The following parameters are for every type: - -| Parameter Name | Description | Required/Optional | Default | -| --- | --- | ---| ---| -| type | The type of the auth api. Currently we support the following types: cisco_secure_x, general. | Required | - | -| name | The name of the auth api. Please make names unique. | Required | - | -| credentials.id | The auth api credentials id. | Required | - | -| credentials.key | The auth api credentials key. | Required | - | -| settings.time_interval | The auth api time interval between runs. | Required | - | -| settings.days_back_fetch | The max days back to fetch from the auth api. | Optional | 14 (days) | -| filters | Pairs of key and value of parameters that can be added to the auth api url. Make sure the keys and values are valid for the auth api. | Optional | - | -| custom_fields | Pairs of key and value that will be added to each data and be sent to Logz.io. Create **type** field to override the default type, to search your data easily in Logz.io. | Optional | type = api_fetcher | - -The following parameters are for general type only: - -| Parameter Name | Description | Required/Optional | Default | -| --- | --- | ---| ---| -| start_date_name| The start date parameter name of the auth api url. | Required | - | -| http_request.method | The HTTP method. Can be GET or POST. | Required | - | -| http_request.url | The auth api url. Make sure the url is without `?` at the end. | Required | - | -| http_request.headers | Pairs of key and value the represents the headers of the HTTP request. | Optional | - | -| http_request.body | The body of the HTTP request. Will be added to HTTP POST requests only. | Optional | - | -| json_paths.next_url | The json path to the next url value inside the response of the auth api. | Required | - | -| json_paths.data | The json path to the data value inside the response of the auth api. | Required | - | -| json_paths.data_date | The json path to the data's date value inside the response of the auth api. | Required | - | - -#### oauth_apis -The following configuration uses OAuth 2.0 flow. - -Supported types: - -- azure_graph -- azure_mail_reports -- general - -The following parameters are for every type: - -| Parameter Name | Description | Required/Optional | Default | -|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ---| ---| -| type | The type of the auth api. Currently we support the following types: azure_graph, general. | Required | - | -| name | The name of the auth api. Please make names unique. | Required | - | -| credentials.id | The auth api credentials id. | Required | - | -| credentials.key | The auth api credentials key. | Required | - | -| data_http_request.method | The HTTP method. Can be GET or POST. | Required | - | -| data_http_request.url | The oauth api url. Make sure the url is without `?` at the end. | Required | - | -| data_http_request.headers | Pairs of key and value the represents the headers of the HTTP request. | Optional | - | -| data_http_request.body | The body of the HTTP request. Will be added to HTTP POST requests only. | Optional | - | -| token_http_request.method | The HTTP method. Can be GET or POST. | Required | - | -| token_http_request.url | The oauth api token request url. Make sure the url is without `?` at the end. | Required | - | -| token_http_request.headers | Pairs of key and value the represents the headers of the HTTP request. | Optional | - | -| token_http_request.body | The body of the HTTP request. Will be added to HTTP POST requests only. | Optional | - | -| json_paths.next_url | The json path to the next url value inside the response of the auth api. | Required/Optional for Azure | - | -| json_paths.data | The json path to the data value inside the response of the auth api. | Required/Optional for Azure | - | -| json_paths.data_date | The json path to the data's date value inside the response of the auth api. | Required | - | -| settings.time_interval | The auth api time interval between runs. | Required | - | -| settings.days_back_fetch | The max days back to fetch from the auth api. | Optional | 14 (days) | -| filters | Pairs of key and value of parameters that can be added to the auth api url. Make sure the keys and values are valid for the auth api. | Optional | - | -| custom_fields | Pairs of key and value that will be added to each data and be sent to Logz.io. Create **type** field to override the default type, to search your data easily in Logz.io. | Optional | type = api_fetcher | -| start_date_name | The start date parameter name of the oauth api url. (Same as json_paths.data_date in most cases) | Required | - | - -### Example - -Auth apis and Oauth apis can be combined in the same config file. Separated for readability. - -#### Auth api config: - -```yaml -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 5 - days_back_fetch: 7 - filters: - event_type%5B%5D: '1090519054' - start_date: 2021-10-05T10%3A10%3A10%2B00%3A00 - custom_fields: - type: cisco_amp - level: high - - type: general - name: cisco general - credentials: - id: <> - key: <> - settings: - time_interval: 2 - days_back_fetch: 5 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date - filters: - event_type%5B%5D: '1090519054' +Create a local config file `config.yaml`. +Configure your API inputs under `apis`. For every API, mention the input type under `type` field: +
+ + General API + + +For structuring custom API calls use type `general` API with the parameters below. + +## Configuration Options +| Parameter Name | Description | Required/Optional | Default | +|--------------------|-----------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| url | The request URL | Required | - | +| headers | The request Headers | Optional | `{}` | +| body | The request body | Optional | - | +| method | The request method (`GET` or `POST`) | Optional | `GET` | +| pagination | Pagination settings if needed (see [options below](#pagination-configuration-options)) | Optional | - | +| next_url | If needed to update the URL in next requests based on the last response. Supports using variables ([see below](#using-variables)) | Optional | - | +| response_data_path | The path to the data inside the response | Optional | response root | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | Add `type` as `api-fetcher` | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | + +## Pagination Configuration Options +If needed, you can configure pagination. + +| Parameter Name | Description | Required/Optional | Default | +|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------|---------| +| type | The pagination type (`url`, `body` or `headers`) | Required | - | +| url_format | If pagination type is `url`, configure the URL format used for the pagination. Supports using variables ([see below](#using-variables)). | Required if pagination type is `url` | - | +| update_first_url | `True` or `False`; If pagination type is `url`, and it's required to append new params to the first request URL and not reset it completely. | Optional if pagination type is `url` | False | +| headers_format | If pagination type is `headers`, configure the headers format used for the pagination. Supports using variables ([see below](#using-variables)). | Required if pagination type is `headers` | - | +| body_format | If pagination type is `body`, configure the body format used for the pagination. Supports using variables ([see below](#using-variables)). | Required if pagination type is `body` | - | +| stop_indication | When should the pagination end based on the response. (see [options below](#pagination-stop-indication-configuration)). | Optional (if not defined will stop on `max_calls`) | - | +| max_calls | Max calls that the pagination can make. (Supports up to 1000) | Optional | 1000 | + +## Pagination Stop Indication Configuration +| Parameter Name | Description | Required/Optional | Default | +|----------------|-----------------------------------------------------------------------------------------|-------------------------------------------------|---------| +| field | The name of the field in the response body, to search the stop indication at | Required | - | +| condition | The stop condition (`empty`, `equals` or `contains`) | Required | - | +| value | If condition is `equals` or `contains`, the value of the `field` that we should stop at | Required if condition is `equals` or `contains` | - | + +## Using Variables +Using variables allows taking values from the response of the first request, to structure the request after it. +Mathematical operations `+` and `-` are supported, in order to add or reduce a number from the variable value. + +Use case examples for variable usage: +1. Update a date filter at every call +2. Update a page number in pagination + +To use variables: +- Wrap the variable name in curly brackets +- Provide the full path to that variable in the response +- Add `res.` prefix to the path. + +Example: Say this is my response: +```json +{ + "field": "value", + "another_field": { + "nested": 123 + }, + "num_arr": [1, 2, 3], + "obj_arr": [ + { + "field2": 345 + }, + { + "field2": 567 + } + ] +} ``` - -#### OAuth Api config: - -```yaml +Paths to fields values are structured like so: +- `{res.field}` = `"value"` +- `{res.another_field.nested}` = `123` +- `{res.num_arr.[2]}` = `3` +- `{res.obj_arr.[0].field2}` = `345` + +Using the fields values in the `next_url` for example like so: +```Yaml +next_url: https://logz.io/{res.field}/{res.obj_arr[0].field2} +``` +Would update the URL at every call to have the value of the given fields from the response, in our example the url for the next call would be: +``` +https://logz.io/value/345 +``` +And in the call after it, it would update again according to the response and the `next_url` structure, and so on. + + +
+
+ + OAuth API + + +For structuring custom OAuth calls use type `oauth` API with the parameters below. + +## Configuration Options +| Parameter Name | Description | Required/Optional | Default | +|-------------------|---------------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| token_request | Nest here any detail relevant to the request to get the bearer access token. (Options in [General API](./src/apis/general/README.md)) | Required | - | +| data_request | Nest here any detail relevant to the data request. (Options in [General API](./src/apis/general/README.md)) | Required | - | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | Add `type` as `api-fetcher` | + +
+
+ + Azure Graph + + +For Azure Graph, use type `azure_graph` with the below parameters. + +## Configuration Options +| Parameter Name | Description | Required/Optional | Default | +|--------------------------------|----------------------------------------------------------------------|-------------------|-------------------| +| name | Name of the API (custom name) | Optional | `azure api` | +| azure_ad_tenant_id | The Azure AD Tenant id | Required | - | +| azure_ad_client_id | The Azure AD Client id | Required | - | +| azure_ad_secret_value | The Azure AD Secret value | Required | - | +| date_filter_key | The name of key to use for the date filter in the request URL params | Optional | `createdDateTime` | +| data_request.url | The request URL | Required | - | +| data_request.additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | +| days_back_fetch | The amount of days to fetch back in the first request | Optional | 1 (day) | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | + +
+ +
+ + Azure Mail Reports + + +For Azure Mail Reports, use type `azure_mail_reports` with the below parameters. + +## Configuration Options +| Parameter Name | Description | Required/Optional | Default | +|--------------------------------|-----------------------------------------------------------------------------|-------------------|-------------| +| name | Name of the API (custom name) | Optional | `azure api` | +| azure_ad_tenant_id | The Azure AD Tenant id | Required | - | +| azure_ad_client_id | The Azure AD Client id | Required | - | +| azure_ad_secret_value | The Azure AD Secret value | Required | - | +| start_date_filter_key | The name of key to use for the start date filter in the request URL params. | Optional | `startDate` | +| end_date_filter_key | The name of key to use for the end date filter in the request URL params. | Optional | `EndDate` | +| data_request.url | The request URL | Required | - | +| data_request.additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | +| days_back_fetch | The amount of days to fetch back in the first request | Optional | 1 (day) | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | + + +
+
+ + Azure General API + + +For structuring custom general Azure API calls use type `azure_general` API with the parameters below. + +## Configuration Options +| Parameter Name | Description | Required/Optional | Default | +|-----------------------|-------------------------------------------------------------------------------------------------------------|-------------------|-------------| +| name | Name of the API (custom name) | Optional | `azure api` | +| azure_ad_tenant_id | The Azure AD Tenant id | Required | - | +| azure_ad_client_id | The Azure AD Client id | Required | - | +| azure_ad_secret_value | The Azure AD Secret value | Required | - | +| data_request | Nest here any detail relevant to the data request. (Options in [General API](./src/apis/general/README.md)) | Required | - | +| days_back_fetch | The amount of days to fetch back in the first request | Optional | 1 (day) | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | + +
+
+ + Cloudflare + + +For Cloudflare API, use type as `cloudflare`. +By default `cloudflare` API type: + +- has built in pagination settings +- sets the `response_data_path` to `result` field. + +## Configuration Options +| Parameter Name | Description | Required/Optional | Default | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| cloudflare_account_id | The CloudFlare Account ID | Required | - | +| cloudflare_bearer_token | The Cloudflare Bearer token | Required | - | +| url | The request URL | Required | - | +| next_url | If needed to update the URL in next requests based on the last response. Supports using variables (see [General API](./general/README.md)) | Optional | - | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| pagination_off | True if builtin pagination should be off, False otherwise | Optional | `False` | + +
+ + +And your logzio output under `logzio`: + +| Parameter Name | Description | Required/Optional | Default | +|----------------|-----------------------------|-------------------|---------------------------------| +| url | The logzio Listener address | Optional | `https://listener.logz.io:8071` | +| token | The logzio shipping token | Required | - | + +#### Example +```Yaml +apis: + - name: random + type: general + additional_fields: + type: random + ... + - name: another api + type: oauth + additional_fields: + type: oauth-api + ... logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - headers: - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - headers: - json_paths: - data_date: createdDateTime - next_url: - data: - settings: - time_interval: 1 - days_back_fetch: 30 - - type: general - name: general_test - credentials: - id: aaaa-bbbb-cccc - key: abcabcabc - token_http_request: - url: https://login.microsoftonline.com/abcd-efgh-abcd-efgh/oauth2/v2.0/token - body: client_id=aaaa-bbbb-cccc - &scope=https://graph.microsoft.com/.default - &client_secret=abcabcabc - &grant_type=client_credentials - headers: - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/directoryAudits - headers: - json_paths: - data_date: activityDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 - start_date_name: activityDateTime - - type: azure_mail_reports - name: mail_reports - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/abcd-efgh-abcd-efgh/oauth2/v2.0/token - body: client_id=<> - &scope=https://outlook.office365.com/.default - &client_secret=<> - &grant_type=client_credentials - headers: - method: POST - data_http_request: - url: https://reports.office365.com/ecp/reportingwebservice/reporting.svc/MessageTrace - method: GET - headers: - json_paths: - data_date: EndDate - next_url: - data: - filters: - format: Json - settings: - time_interval: 60 # for mail reports we suggest no less than 60 minutes - days_back_fetch: 8 # for mail reports we suggest up to 8 days - start_date_name: StartDate - end_date_name: EndDate - + url: https://listener-ca.logz.io:8071 # for us-east-1 region delete url param (default) + token: <> ``` -### Azure mail reports type important notes and limitations -* We recommend setting the `days_back_fetch` parameter to no more than `8d` (~192 hours) as this might cause unexpected errors with the API. -* We recommend setting the `time_interval` parameter to no less than `60`, to avoid short time frames in which messages trace will be missed. -* Microsoft may delay trace events for up to 24 hours, and events are not guaranteed to be sequential during this delay. For more information, see the Data granularity, persistence, and availability section of the MessageTrace report topic in the Microsoft documentation: [MessageTrace report API](https://learn.microsoft.com/en-us/previous-versions/office/developer/o365-enterprise-developers/jj984335(v=office.15)#data-granularity-persistence-and-availability) - -### Create Last Start Dates Text File - -Create an empty text file named last_start_dates.txt in the same directory as the config file: +### Run The Docker Container +In the path where you saved your `config.yaml`, run: ```shell -$ touch last_start_dates.txt +docker run --name logzio-api-fetcher \ +-v "$(pwd)":/app/src/shared \ +logzio/logzio-api-fetcher ``` -### Run The Docker Container - +#### Change logging level +The default logging level is `INFO`. To change it, add `--level` flag to the command: ```shell docker run --name logzio-api-fetcher \ -v "$(pwd)":/app/src/shared \ -logzio/logzio-api-fetcher +logzio/logzio-api-fetcher \ +--level DEBUG ``` +Available Options: `INFO`, `WARN`, `ERROR`, `DEBUG` -## Stop Docker Container - +## Stopping the container When you stop the container, the code will run until completion of the iteration. To make sure it will finish the iteration on time, please give it a grace period of 30 seconds when you run the docker stop command: @@ -263,17 +261,29 @@ please give it a grace period of 30 seconds when you run the docker stop command docker stop -t 30 logzio-api-fetcher ``` -## Last Start Dates Text File - -After every successful iteration of each api, the last start date of the next iteration will be written to a file named `last_start_dates.txt`. -Each line starts with the api name and ends with the last start date. - -You can find the file inside your mounted host directory that you created. - -If you stopped the container, you can continue from the exact place you stopped, by adding the date to the api filters in the configuration. - ## Changelog: - +- **0.2.0**: + - **Breaking changes!!** + - Deprecate configuration fields: + - `credentials`, `start_date_name`, `end_date_name`, `json_paths`, `settings.days_back_fetch` + - `settings.time_interval` >> `scrape_interval` + - OAuth Changes: + - `token_http_request` >> `token_request` + - `data_http_request` >> `data_request` + - All APIs now nested in the config under `apis` and supported types are: + - `general`, `oauth`, `azure_general`, `azure_graph`, `azure_mail_reports`, `cloudflare`. + - Upgrade General API settings to allow more customization + - Adding Authorization Header is required, if needed, due to `credentials` field deprecation. + - Adding `Content-Type` may be required based on the API + - Deprecate Cisco SecureX + - **New Features!!** + - Add Pagination support + - Add variables support in `next_url` and pagination settings + - Supports `+` and `-` mathematical operations on values + - Supports arrays + - Upgrade python version `3.9` >> `3.12` + - Add Cloudflare Support + - Add option to control logging level with `--level` flag. - **0.1.3**: - Support another date format for `azure_graph`. - **0.1.2**: @@ -288,6 +298,5 @@ If you stopped the container, you can continue from the exact place you stopped, - **0.0.5**: - Bug fix for `azure_graph` task fails on second cycle. - Changed start date filter mechanics for auth_api. - - **0.0.4**: - Bug fix for `azure_graph` task fails when trying to get start date from file. diff --git a/dockerfile b/dockerfile index 761df2f..cb89b87 100644 --- a/dockerfile +++ b/dockerfile @@ -1,6 +1,20 @@ -FROM python:3.9-slim +FROM python:3.12-slim WORKDIR /app -COPY /src ./src + +# python settings +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +# Install dependencies COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +# Set user as non root +RUN chown nobody:nogroup /app +USER nobody + +# Copy the program to /src +COPY --chown=nobody:nogroup /src ./src + +# Run the program ENTRYPOINT ["python", "-m", "src.main"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 94b0ab9..a0fef54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -pyyaml -requests -urllib3 -python-dateutil -jsonpath-ng \ No newline at end of file +pydantic~=2.7.1 +pyyaml~=6.0.1 +requests~=2.32.2 diff --git a/src/__init__.py b/src/__init__.py index 0980070..e69de29 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1 +0,0 @@ -import os, sys; sys.path.append(os.path.dirname(os.path.realpath(__file__))) diff --git a/src/api.py b/src/api.py deleted file mode 100644 index a170211..0000000 --- a/src/api.py +++ /dev/null @@ -1,181 +0,0 @@ -import json -import logging -import urllib.parse -import requests - -from abc import ABC, abstractmethod -from datetime import datetime, timedelta -from typing import Generator, Any, Optional -from jsonpath_ng import parse -from dateutil import parser -from requests import Response -from .data.base_data.api_base_data import ApiBaseData -from .data.base_data.api_custom_field import ApiCustomField -from .data.general_type_data.api_general_type_data import ApiGeneralTypeData - -logger = logging.getLogger(__name__) - - -class Api(ABC): - - def __init__(self, api_base_data: ApiBaseData, api_general_type_data: ApiGeneralTypeData): - self._base_data = api_base_data - self._general_type_data = api_general_type_data - self._current_data_last_date: Optional[str] = None - - @property - def base_data(self) -> ApiBaseData: - return self._base_data - - @property - def general_type_data(self) -> ApiGeneralTypeData: - return self._general_type_data - - class ApiError(Exception): - pass - - @abstractmethod - def fetch_data(self) -> Generator[str, None, list[str]]: - pass - - @abstractmethod - def _send_request(self, url) -> Response: - pass - - def _get_json_path_value_from_data(self, json_path: str, data: dict) -> Any: - match = parse(json_path).find(data) - - if not match: - return None - - return match[0].value - - def update_start_date_filter(self) -> None: - new_start_date = self._get_new_start_date() - filter_index = self._base_data.get_filter_index( - self._general_type_data.start_date_name) - - # If date is a filter in the filters list, update the list value (cisco secure x). - if filter_index != -1: - self._base_data.update_filter_value(filter_index, new_start_date) - - def get_last_start_date(self) -> Optional[str]: - for api_filter in self._base_data.filters: - if api_filter.key != self._general_type_data.start_date_name: - continue - - return api_filter.value - - return None - - def get_api_name(self): - return self._base_data.name - - def get_api_time_interval(self) -> int: - return self._base_data.settings.time_interval - - def get_api_custom_fields(self) -> Generator[ApiCustomField, None, None]: - for custom_field in self._base_data.custom_fields: - yield custom_field - - def _get_last_date(self, first_item: dict) -> str: - first_item_date = self._get_json_path_value_from_data( - self._general_type_data.json_paths.data_date, first_item) - - if first_item_date is None: - logger.error( - "The json path for api {}'s data date is wrong. Please change your configuration.".format( - self._base_data.name)) - raise Api.ApiError - - return first_item_date - - def _is_item_in_fetch_frame(self, item: dict, last_datetime_to_fetch: datetime) -> bool: - item_date = self._get_json_path_value_from_data( - self._general_type_data.json_paths.data_date, item) - item_datetime = parser.parse(item_date) - if item_datetime < last_datetime_to_fetch: - return False - - return True - - def get_start_date_filter(self) -> str: - if self._current_data_last_date: - raw_new_start_date = self._get_new_start_date() - formatted_new_start_date = raw_new_start_date.split('.')[0].split('+')[0] # cut out milliseconds and timezone - if formatted_new_start_date == raw_new_start_date: - formatted_new_start_date = raw_new_start_date.split('%2E')[0].split('%2B')[0] # Support HTML encoded dates - formatted_new_start_date += 'Z' if not formatted_new_start_date.endswith('Z') else '' - else: - raw_new_start_date = datetime.utcnow() - timedelta(days=self.base_data.settings.days_back_to_fetch) - formatted_new_start_date = raw_new_start_date.isoformat(' ', 'seconds') - formatted_new_start_date = formatted_new_start_date.replace(' ', 'T') - formatted_new_start_date += 'Z' - return formatted_new_start_date - - def _get_new_start_date(self) -> str: - new_start_date = str(parser.parse(self._current_data_last_date) + timedelta(seconds=1)) - new_start_date = new_start_date.replace(' ', 'T') - new_start_date = urllib.parse.quote(new_start_date) - - return new_start_date - - def _get_data_from_api(self, url: str) -> tuple[Optional[str], list]: - next_url = None - json_data = self._parse_response_to_json(url) - if self._general_type_data.json_paths.next_url: - next_url = self._get_json_path_value_from_data( - self._general_type_data.json_paths.next_url, json_data) - data = self._parse_and_verify_data_received(json_data) - return next_url, data - - def _parse_response_to_json(self, url): - try: - response = self._get_response_from_api(url) - except Exception: - raise - json_data = json.loads(response.content) - return json_data - - def _parse_and_verify_data_received(self, json_data): - data = self._get_json_path_value_from_data( - self._general_type_data.json_paths.data, json_data) - if data is None: - logger.error( - "The json path for api {}'s data is wrong. Please change your configuration.".format( - self._base_data.name)) - raise Api.ApiError - data_size = len(data) - if data: - logger.info("Successfully got {0} data from api {1}.".format(data_size, self._base_data.name)) - return data - - def _get_response_from_api(self, url: str) -> Response: - try: - response = self._send_request(url) - response.raise_for_status() - except requests.HTTPError as e: - logger.error( - "Something went wrong while trying to get the data from api {0}. response: {1}".format( - self._base_data.name, e)) - if e.response.status_code == 400 or e.response.status_code == 401: - raise Api.ApiError() - raise - except Exception as e: - logger.error("Something went wrong with api {0}. response: {1}".format(self._base_data.name, e)) - raise - return response - - def get_current_time_utc_string(self): - time = datetime.utcnow() - time = time.isoformat(' ', 'seconds') - time = time.replace(' ', 'T') - time += 'Z' - return time - - def _get_next_page_url(self, json_data: dict): - if self._general_type_data.json_paths.next_url: - next_url = self._get_json_path_value_from_data( - self._general_type_data.json_paths.next_url, json_data) - return next_url - return None diff --git a/src/data/__init__.py b/src/apis/__init__.py similarity index 100% rename from src/data/__init__.py rename to src/apis/__init__.py diff --git a/src/apis/azure/AzureApi.py b/src/apis/azure/AzureApi.py new file mode 100644 index 0000000..1ceab7c --- /dev/null +++ b/src/apis/azure/AzureApi.py @@ -0,0 +1,38 @@ +from datetime import datetime, timedelta, UTC +from pydantic import Field + +from src.apis.oauth.OAuth import OAuthApi +from src.apis.general.Api import ReqMethod, ApiFetcher + + +class AzureApi(OAuthApi): + """ + Initialize a general Azure API call to prevent duplication of 'token_request' in subclasses. + :param name: Optional custom name for the API. + :param azure_ad_tenant_id: The Azure AD Tenant id + :param azure_ad_client_id: The Azure AD Client id + :param azure_ad_secret_value: The Azure AD Secret value + :param days_back_fetch: The amount of days to fetch back in the first request + :param date_filter_key: The name of key to use for the date filter in the request URL params. + """ + name: str = Field(default="azure api") + azure_ad_tenant_id: str = Field(frozen=True) + azure_ad_client_id: str = Field(frozen=True) + azure_ad_secret_value: str = Field(frozen=True) + days_back_fetch: int = Field(default=1, frozen=True) + date_filter_key: str = Field(default="createdDateTime") + + def __init__(self, **data): + token_request = ApiFetcher( + url=f"https://login.microsoftonline.com/{data.get('azure_ad_tenant_id')}/oauth2/v2.0/token", + body=f"""client_id={data.get('azure_ad_client_id')} + &scope=https://graph.microsoft.com/.default + &client_secret={data.get('azure_ad_secret_value')} + &grant_type=client_credentials + """, + method=ReqMethod.POST) + + super().__init__(token_request=token_request, **data) + + def generate_start_fetch_date(self): + return (datetime.now(UTC) - timedelta(days=self.days_back_fetch)).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/src/apis/azure/AzureGraph.py b/src/apis/azure/AzureGraph.py new file mode 100644 index 0000000..9bacfcc --- /dev/null +++ b/src/apis/azure/AzureGraph.py @@ -0,0 +1,80 @@ +from datetime import datetime, timedelta +import logging +import re + +from src.apis.azure.AzureApi import AzureApi +from src.apis.general.Api import ApiFetcher +from src.apis.general.PaginationSettings import PaginationSettings +from src.apis.general.StopPaginationSettings import StopPaginationSettings + + +DATE_FROM_END_PATTERN = re.compile(r'\S+$') + + +logger = logging.getLogger(__name__) + + +class AzureGraph(AzureApi): + + def __init__(self, **data): + """ + Initialize request for Azure Graph API. + """ + # Initializing the data requests + data_request = ApiFetcher(**data.pop("data_request"), + pagination=PaginationSettings( + type="url", + url_format="{res.@odata\\.nextLink}", + stop_indication=StopPaginationSettings(field="value", + condition="empty")), + response_data_path="value") + super().__init__(data_request=data_request, **data) + + # Initialize date filter in the first data request + self._initialize_url_date() + + # Initialize data request next_url format + self._initialize_next_url() + + def _initialize_url_date(self): + """ + initializing the data request url to be in format: + https://url/from/input?$filter=createdDateTime gt 2024-05-28T13:08:54Z + """ + self.data_request.url += f"?$filter={self.date_filter_key} gt {self.generate_start_fetch_date()}" + + def _initialize_next_url(self): + """ + initializing the data request next url to be in format: + https://url/from/input?$filter=createdDateTime gt {res.value.[0].createdDateTime} + + done to allow updating the date filter with each call. + """ + self.data_request.update_next_url(self._replace_url_date(f"{{res.value.[0].{self.date_filter_key}}}")) + + def _replace_url_date(self, req_val): + """ + replaces the date at the end of the URL (found with regex pattern DATE_FROM_END_PATTERN) with the given req_val. + :param req_val: the new date value + :return: the original URL with the new req_val as the date. + """ + return re.sub(DATE_FROM_END_PATTERN, req_val, self.data_request.url) + + def send_request(self): + """ + 1. Sends request using the super class + 2. Add 1 second to the date from the end of the URL to avoid duplicates in the next call + :return: all the responses that were received + """ + data = super().send_request() + + # Add 1s to the time we took from the response to avoid duplicates + org_date = re.search(DATE_FROM_END_PATTERN, self.data_request.url).group(0) + try: + org_date = datetime.strptime(org_date, "%Y-%m-%dT%H:%M:%SZ") + org_date_plus_second = (org_date + timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + self.data_request.url = self._replace_url_date(org_date_plus_second) + except ValueError: + logger.error(f"Failed to parse API {self.name} date in URL: {self.data_request.url}") + + return data diff --git a/src/apis/azure/AzureMailReports.py b/src/apis/azure/AzureMailReports.py new file mode 100644 index 0000000..6fa85c0 --- /dev/null +++ b/src/apis/azure/AzureMailReports.py @@ -0,0 +1,73 @@ +import re +from datetime import datetime, UTC +from pydantic import Field + +from src.apis.azure.AzureApi import AzureApi +from src.apis.general.Api import ApiFetcher +from src.apis.general.PaginationSettings import PaginationSettings +from src.apis.general.StopPaginationSettings import StopPaginationSettings + + +class AzureMailReports(AzureApi): + """ + :param date_filter_key: The name of key to use for the start date filter in the request URL params. + :param end_date_filter_key: The name of key to use for the end date filter in the request URL params. + """ + date_filter_key: str = Field(default="StartDate", alias="start_date_filter_key") # Overwrite parent default value + end_date_filter_key: str = Field(default="EndDate") + + def __init__(self, **data): + """ + Initialize request for Azure Mail Reports API. + """ + # Initializing the data requests + data_request = ApiFetcher(**data.pop("data_request"), + pagination=PaginationSettings( + type="url", + url_format="{res.d.@odata\\.nextLink}", + stop_indication=StopPaginationSettings(field="d.results", + condition="empty")), + response_data_path="d.results") + super().__init__(data_request=data_request, **data) + + # Structure the next URL format, to automatically update start date. + self._initialize_next_url() + + # Initialize date filter in the first data request and update the url + fetch_start_date = self.generate_start_fetch_date() + fetch_end_date = self._get_end_date() + self._initialize_url_date(fetch_start_date, fetch_end_date) + + def _initialize_url_date(self, fetch_start_date, fetch_end_date): + """ + initializing the data request url to be in format: + https://url/from/input?$filter=StartDate eq datetime '2024-05-28T13:08:54Z' and EndDate eq datetime '2024-05-29T13:08:54Z' + """ + self.data_request.url = (self.data_request.next_url + .replace(f"{{res.value.[0].{self.end_date_filter_key}}}", fetch_start_date) + .replace("NOW_DATE", fetch_end_date)) + + def _initialize_next_url(self): + """ + initializing the data request next url to be in format: + https://url/from/input?$filter=StartDate eq datetime '{res.d.results.[0].EndDate}' and EndDate eq datetime 'NOW_DATE' + """ + new_next_url = self.data_request.url + f"?$filter={self.date_filter_key} eq datetime '{{res.d.results.[0].{self.end_date_filter_key}}}' and {self.end_date_filter_key} eq datetime 'NOW_DATE'" + self.data_request.update_next_url(new_next_url) + + @staticmethod + def _get_end_date(): + return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + def send_request(self): + """ + 1. Before sending a request, updates the end date filter in the new URL to the time now. (relevant to all + requests besides the first request) + 2. Sends request using the super class. + :return: all the responses that were received + """ + # Update the end date in the URL before sending a request + self.data_request.url = self.data_request.url.replace("NOW_DATE", self._get_end_date()) + + data = super().send_request() + return data diff --git a/src/apis/azure/README.MD b/src/apis/azure/README.MD new file mode 100644 index 0000000..4833c9b --- /dev/null +++ b/src/apis/azure/README.MD @@ -0,0 +1,86 @@ +# Azure API Configuration +Currently, the below API types are supported: +- [Azure General](#azure-general) (`azure_general`) +- [Azure Graph](#azure-graph) (`azure_graph`) +- [Azure Mail Reports](#azure-mail-reports) (`azure_mail_reports`) + +Configuration [example here](#example). + + +## Azure General +Below fields are relevant for **all Azure API types** + +| Parameter Name | Description | Required/Optional | Default | +|-----------------------|-----------------------------------------------------------------------------------------------------|-------------------|-------------| +| name | Name of the API (custom name) | Optional | `azure api` | +| azure_ad_tenant_id | The Azure AD Tenant id | Required | - | +| azure_ad_client_id | The Azure AD Client id | Required | - | +| azure_ad_secret_value | The Azure AD Secret value | Required | - | +| data_request | Nest here any detail relevant to the data request. (Options in [General API](../general/README.md)) | Required | - | +| days_back_fetch | The amount of days to fetch back in the first request | Optional | 1 (day) | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | + +## Azure Graph +By default `azure_graph` API type has built in pagination settings and sets the `response_data_path` to `value` field. +The below fields are relevant **in addition** to the required ones listed under Azure General. + +| Parameter Name | Description | Required/Optional | Default | +|--------------------------------|----------------------------------------------------------------------|-------------------|-------------------| +| date_filter_key | The name of key to use for the date filter in the request URL params | Optional | `createdDateTime` | +| data_request.url | The request URL | Required | - | +| data_request.additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | + +## Azure Mail Reports +By default `azure_mail_reports` API type has built in pagination settings and sets the `response_data_path` to `d.results` field. +The below fields are relevant **in addition** to the required ones listed under Azure General. + +| Parameter Name | Description | Required/Optional | Default | +|--------------------------------|-----------------------------------------------------------------------------|-------------------|-------------| +| start_date_filter_key | The name of key to use for the start date filter in the request URL params. | Optional | `startDate` | +| end_date_filter_key | The name of key to use for the end date filter in the request URL params. | Optional | `EndDate` | +| data_request.url | The request URL | Required | - | +| data_request.additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | + +## Example + +```Yaml +apis: + - name: azure graph example + type: azure_graph + azure_ad_tenant_id: <> + azure_ad_client_id: <> + azure_ad_secret_value: <> + data_request: + url: https://graph.microsoft.com/v1.0/auditLogs/signIns + additional_fields: + type: azure_graph + field_to_add_to_my_logs: 123 + scrape_interval: 1 + days_back_fetch: 30 + + - name: mail reports example + type: azure_mail_reports + azure_ad_tenant_id: <> + azure_ad_client_id: <> + azure_ad_secret_value: <> + data_request: + url: https://login.microsoftonline.com/<>/oauth2/v2.0/token + additional_fields: + type: azure_mail_reports + scrape_interval: 60 # for mail reports we suggest no less than 60 minutes + days_back_fetch: 8 # for mail reports we suggest up to 8 days + + - name: azure general example + type: azure_general + azure_ad_tenant_id: <> + azure_ad_client_id: <> + azure_ad_secret_value: <> + data_request: + url: ... + scrape_interval: 30 + days_back_fetch: 30 + +logzio: + url: https://listener-eu.logz.io:8071 # for us-east-1 region delete url param (default) + token: <> +``` diff --git a/src/data/base_data/__init__.py b/src/apis/azure/__init__.py similarity index 100% rename from src/data/base_data/__init__.py rename to src/apis/azure/__init__.py diff --git a/src/apis/cloudflare/Cloudflare.py b/src/apis/cloudflare/Cloudflare.py new file mode 100644 index 0000000..5a6274a --- /dev/null +++ b/src/apis/cloudflare/Cloudflare.py @@ -0,0 +1,80 @@ +from datetime import datetime, timedelta, UTC +import logging +from pydantic import Field +import re + +from src.apis.general.Api import ApiFetcher +from src.apis.general.PaginationSettings import PaginationSettings +from src.apis.general.StopPaginationSettings import StopPaginationSettings + + +DATE_FILTER_PARAMETER = "since=" +FIND_DATE_PATTERN = re.compile(r'since=(\S+?)(?:&|$)') + +logger = logging.getLogger(__name__) + + +class Cloudflare(ApiFetcher): + """ + :param cloudflare_account_id: The CloudFlare Account ID + :param cloudflare_bearer_token: The cloudflare Bearer token + :param pagination_off: True if pagination should be off, False otherwise + :param days_back_fetch: Amount of days to fetch back in the first request, Optional (adds a filter on 'since') + """ + cloudflare_account_id: str = Field(frozen=True) + cloudflare_bearer_token: str = Field(frozen=True) + pagination_off: bool = Field(default=False) + days_back_fetch: int = Field(default=-1, frozen=True) + + def __init__(self, **data): + res_data_path = "result" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {data.get('cloudflare_bearer_token')}" + } + pagination = None + if not data.get("pagination_off"): + if "?" in data.get("url"): + url_format = "&page={res.result_info.page+1}" + else: + url_format = "?page={res.result_info.page+1}" + pagination = PaginationSettings(type="url", + url_format=url_format, + update_first_url=True, + stop_indication=StopPaginationSettings(field=res_data_path, + condition="empty")) + + super().__init__(headers=headers, pagination=pagination, response_data_path=res_data_path, **data) + + # Update the cloudflare account id in both the url and next url + self.url = self.url.replace("{account_id}", self.cloudflare_account_id) + self.next_url = self.next_url.replace("{account_id}", self.cloudflare_account_id) + + if self.days_back_fetch > 0: + self._initialize_url_date() + + def _initialize_url_date(self): + if "?" in self.url: + self.url += f"&since={self._generate_start_fetch_date()}" + else: + self.url += f"?since={self._generate_start_fetch_date()}" + + def _generate_start_fetch_date(self): + return (datetime.now(UTC) - timedelta(days=self.days_back_fetch)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + def send_request(self): + data = super().send_request() + + # Add 1 second to a known date filter to avoid duplicates in the logs + if DATE_FILTER_PARAMETER in self.url: + try: + org_date = re.search(FIND_DATE_PATTERN, self.url).group(1) + org_date_date = datetime.strptime(org_date, "%Y-%m-%dT%H:%M:%S.%fZ") + org_date_plus_second = (org_date_date + timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + self.url = self.url.replace(org_date, org_date_plus_second) + except IndexError: + logger.error(f"Failed to add 1s to the {self.name} api 'since' filter value, on url {self.url}") + except ValueError: + logger.error(f"Failed to parse API {self.name} date in URL: {self.url}") + + return data diff --git a/src/apis/cloudflare/README.md b/src/apis/cloudflare/README.md new file mode 100644 index 0000000..1fb771e --- /dev/null +++ b/src/apis/cloudflare/README.md @@ -0,0 +1,34 @@ +# Cloudflare API Configuration +By default `cloudflare` API type has built in pagination settings and sets the `response_data_path` to `result` field. + +## Configuration +| Parameter Name | Description | Required/Optional | Default | +|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------|-------------------|-------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| cloudflare_account_id | The CloudFlare Account ID | Required | - | +| cloudflare_bearer_token | The Cloudflare Bearer token | Required | - | +| url | The request URL | Required | - | +| next_url | If needed to update the URL in next requests based on the last response. Supports using variables (see [General API](../general/README.md)) | Optional | - | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | - | +| days_back_fetch | The amount of days to fetch back in the first request. Applies a filter on `since` parameter. | Optional | - | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| pagination_off | True if builtin pagination should be off, False otherwise | Optional | `False` | + +## Example +```Yaml +apis: + - name: cloudflare test + type: cloudflare + cloudflare_account_id: <> + cloudflare_bearer_token: <> + url: https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history + next_url: https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history?since={res.result.[0].sent} + days_back_fetch: 7 + scrape_interval: 5 + additional_fields: + type: cloudflare + +logzio: + url: https://listener-eu.logz.io:8071 # for us-east-1 region delete url param (default) + token: <> +``` diff --git a/src/data/general_type_data/__init__.py b/src/apis/cloudflare/__init__.py similarity index 100% rename from src/data/general_type_data/__init__.py rename to src/apis/cloudflare/__init__.py diff --git a/src/apis/general/Api.py b/src/apis/general/Api.py new file mode 100644 index 0000000..bcc1f70 --- /dev/null +++ b/src/apis/general/Api.py @@ -0,0 +1,239 @@ +from enum import Enum +import json +import logging +from pydantic import BaseModel, Field +import requests +from typing import Union + +from src.utils.processing_functions import extract_vars, substitute_vars +from src.apis.general.PaginationSettings import PaginationSettings, PaginationType +from src.utils.processing_functions import break_key_name, get_nested_value + +SUCCESS_CODES = [200, 204] +logger = logging.getLogger(__name__) + + +class ReqMethod(Enum): + """ + Supported methods for the API request + """ + GET = "GET" + POST = "POST" + + +class ApiFetcher(BaseModel): + """ + Class to initialize an API request + :param url: Required, the URL to send the request to + :param headers: Optional, the headers to use for the request + :param body: Optional, the body of the request + :param method: Optional, the method to use for the request (default: GET) + :param pagination_settings: Optional, PaginationSettings object that defines how to perform pagination + :param next_url: Optional, If needed update a param in the url according to the response as we go + :param response_data_path: Optional, The path to find the data within the response. + :param additional_fields: Optional, 'key: value' pairs that should be added to the API logs. + :param scrape_interval_minutes: the interval between scraping jobs. + :param url_vars: Not passed to the class, array of params that is generated based on next_url. + """ + name: str = Field(default="") + url: str + headers: dict = Field(default={}) + body: Union[str, dict, list] = Field(default=None) + method: ReqMethod = Field(default=ReqMethod.GET, frozen=True) + pagination_settings: PaginationSettings = Field(default=None, frozen=True, alias="pagination") + next_url: str = Field(default=None) + response_data_path: str = Field(default=None, frozen=True) + additional_fields: dict = Field(default={}) + scrape_interval_minutes: int = Field(default=1, alias="scrape_interval", ge=1) + url_vars: list = Field(default=[], init=False, init_var=True) + + def __init__(self, **data): + """ + Makes sure to format the body and generate the url_vars based on next_url. + :param data: the fields for creation of the class. + """ + super().__init__(**data) + self.body = self._format_body(self.body) + self.url_vars = extract_vars(self.next_url) + if not self.name: + self.name = self.url + if not self.additional_fields.get("type"): + self.additional_fields["type"] = "api-fetcher" + + @staticmethod + def _format_body(body): + """ + Makes sure to 'json.dumps' dictionaries to allow valid request to be made. + :param body: the request body. + :return: body in format for a request. + """ + if isinstance(body, dict) or isinstance(body, list): + return json.dumps(body) + return body + + def _extract_data_from_path(self, response): + """ + Extract the value from the given 'self.response_data_path' if it exists. + Returns array with the logs to support sending each log in a ''self.response_data_path' array as it's own log. + :param response: the requests' response. + :return: array with the logs to send from the response. + """ + if self.response_data_path: + data_path_value = get_nested_value(response, break_key_name(self.response_data_path)) + if data_path_value and isinstance(data_path_value, list): + return data_path_value + if data_path_value: + return [data_path_value] + else: + # This is set to debug and not error since it could be expected (no new data) + logger.debug(f"Did not find data in given '{self.response_data_path}' path. Response: {response}") + return [] + return [response] + + def _make_call(self): + """ + Sends the request and returns the response, or None if there was an issue. + :return: the response of the request. + """ + logger.debug(f"Sending API call with details:\nURL: {self.url}\nHeaders: {self.headers}\nBody: {self.body}") + + try: + r = requests.request(method=self.method.value, url=self.url, headers=self.headers, data=self.body) + r.raise_for_status() + except requests.ConnectionError: + logger.error(f"Failed to establish connection to the {self.name} API.") + return None + except requests.HTTPError as e: + logger.error(f"Failed to get data from {self.name} API due to error {e}") + return None + except Exception as e: + logger.error(f"Failed to send request to {self.name} API due to error {e}") + return None + + if r.status_code in SUCCESS_CODES: + try: + return json.loads(r.text) + except json.decoder.JSONDecodeError: + return r.text + else: + logger.warning(f"Issue with fetching data from {self.name} API: %s", r.text) + return None + + def _prepare_pagination_next_call(self, res, first_url): + """ + Updates the next pagination call according to the response from the last call. + :param res: the response from the last call + :param first_url: needed to support in URL pagination option to append params to the same base URL + :return: True if succeeded to update the pagination request, False if failed. + """ + # URL Pagination + if self.pagination_settings.pagination_type == PaginationType.URL: + new_url = self.pagination_settings.get_next_url(res, first_url) + if new_url: + self.url = new_url + else: + logger.debug(f"Stopping pagination due to issue with replacing the URL.") + return False + + # Body Pagination + elif self.pagination_settings.pagination_type == PaginationType.BODY: + new_body = self._format_body(self.pagination_settings.get_next_body(res)) + if new_body: + self.body = new_body + else: + logger.debug(f"Stopping pagination due to issue with replacing the Body.") + return False + + # Headers Pagination + else: + new_headers = self.pagination_settings.get_next_headers(res) + if new_headers: + self.headers = new_headers + else: + logger.debug(f"Stopping pagination due to issue with replacing the Headers.") + return False + return True + + def _revert_pagination_changes(self, org_url, org_headers, org_body): + """ + The pagination changes the original request information. + After it's done, we want to make sure we go reset the info to the original needed request. + Specifically for the URL, we will only reset it if there is no next_url defined (because otherwise it will be + overwritten anyway after the end of the pagination). + :param org_url: the original request URL + :param org_headers: the original request headers + :param org_body: the original request body + """ + if self.pagination_settings.pagination_type == PaginationType.URL and not self.next_url: + self.url = org_url + elif self.pagination_settings.pagination_type == PaginationType.BODY: + self.body = org_body + else: + self.headers = org_headers + + def _perform_pagination(self, res): + """ + Performs pagination calls until reaches stop condition or the max allowed calls. + :param res: the response of the first call + """ + logger.debug(f"Starting pagination for {self.name}") + call_count = 0 + first_url = self.url + org_headers = self.headers + org_body = self.body + + while not self.pagination_settings.did_pagination_end(res, call_count): + + # Prepare the next call, if fails >> stop pagination + if not self._prepare_pagination_next_call(res, first_url): + break + + logger.debug(f"Sending pagination call {call_count + 1} for api {self.name} in path '{self.url}'") + res = self._make_call() + call_count += 1 + + if not res: + # Had issue with sending request to the API, stopping the pagination + break + + yield self._extract_data_from_path(res) + + self._revert_pagination_changes(first_url, org_headers, org_body) + + def update_next_url(self, new_next_url): + """ + Supports updating the next URL format to make sure the 'self.url_vars' is updated accordingly. + :param new_next_url: new format for the next URL (relevant for customized APIs support) + """ + self.next_url = new_next_url + self.url_vars = extract_vars(self.next_url) + + def send_request(self): + """ + Manages the request: + - Calls _make_call() function to send request + - If Pagination is configured, calls _perform_pagination + - Updates the URL for the next request per 'next_url' if defined + :return: all the responses that were received + """ + responses = [] + r = self._make_call() + if r: + r_data_path = self._extract_data_from_path(r) + + if not r_data_path: + logger.info(f"No new data available from api {self.name}.") + return responses + + # New data found >> add to responses + responses.extend(r_data_path) + + # Perform pagination + if self.pagination_settings: + for data in self._perform_pagination(r): + responses.extend(data) + + # Update the url if needed + if self.next_url: + self.url = substitute_vars(self.next_url, self.url_vars, r) + return responses diff --git a/src/apis/general/PaginationSettings.py b/src/apis/general/PaginationSettings.py new file mode 100644 index 0000000..09b8af5 --- /dev/null +++ b/src/apis/general/PaginationSettings.py @@ -0,0 +1,147 @@ +from enum import Enum +import json +import logging +from pydantic import BaseModel, Field, model_validator +from typing import Union + +from src.utils.processing_functions import extract_vars, substitute_vars, get_nested_value, break_key_name +from src.apis.general.StopPaginationSettings import StopPaginationSettings + +logger = logging.getLogger(__name__) + + +class PaginationType(Enum): + """ + Supported Pagination types + """ + URL = "url" + BODY = "body" + HEADERS = "headers" + + +class PaginationSettings(BaseModel): + """ + Class that initialize API pagination settings. + :param pagination_type: Where is the pagination made, url, body or headers. + :param next_url: If the pagination is in the URL, the url to use and the params to update in it. + :param update_first_url: If the pagination is in the URL, supports using the original URL and adding new params + to it. Example: http://endpoint?date=XXX >> http://endpoint?date=XXX&newParam=YYY + :param next_headers: If the pagination is in the Headers, the headers to use and the params to update in it. + :param next_body: If the pagination is in the Body, the body to use and the params to update in it. + :param stop_indication: StopPaginationSettings object that defines when the pagination should stop + :param max_calls: The max calls the pagination can make. Max value is 1000. + :param url_vars, headers_vars, body_vars: Not passed to the class, array of params generated by matching next_XXX + """ + pagination_type: PaginationType = Field(alias="type") + next_url: str = Field(default=None, frozen=True, alias="url_format") + update_first_url: bool = Field(default=False, frozen=True) + next_headers: dict = Field(default={}, alias="headers_format") + next_body: Union[str, dict] = Field(default=None, alias="body_format") + stop_indication: StopPaginationSettings = Field(default=None, frozen=True) + max_calls: int = Field(default=100, le=1000, frozen=True) + url_vars: list = Field(default=[], init=False, init_var=True) + headers_vars: list = Field(default=[], init=False, init_var=True) + body_vars: list = Field(default=[], init=False, init_var=True) + + def __init__(self, **data): + """ + Generates the URL,Headers and Body parameters. + :param data: the fields for creation of the class. + """ + super().__init__(**data) + self.url_vars = extract_vars(self.next_url) + self.headers_vars = extract_vars(self.next_headers) + self.body_vars = extract_vars(self.next_body) + + @model_validator(mode='after') + def _check_conditional_fields(self): + """ + Validates that: + if type == url >> that we got next_url (or alias url_format) + if type == headers >> that we got next_headers (or alias headers_format) + if type == body >> that we got next_body (or alias body_format) + :return: self + """ + if ((self.pagination_type == PaginationType.URL and not self.next_url) or + (self.pagination_type == PaginationType.HEADERS and not self.next_headers) or + (self.pagination_type == PaginationType.BODY and not self.next_body)): + raise ValueError(f"Used pagination type {self.pagination_type.value.upper()} but missing required field " + f"{self.pagination_type.value}_format") + return self + + def get_next_url(self, values_dict, prev_url): + """ + Generates the next URL to use based on replacing the parameters in next_url with the values from given + values_dict. + :param values_dict: dictionary with values. + :param prev_url: URL of the previous request, if needed to only add new params in it and not replace all of it + :return: next_url with values instead of variables + """ + new_url = self.next_url + if self.update_first_url: + new_url = prev_url + self.next_url + try: + return substitute_vars(new_url, self.url_vars, values_dict) + except ValueError as e: + logger.warning(f"Failed to update the next URL for the pagination due to error: {e}") + return None + + def get_next_headers(self, values_dict): + """ + Generates the next Headers to use based on replacing the parameters in next_url with the values from given + values_dict. + :param values_dict: dictionary with values. + :return: next_headers with values instead of variables + """ + new_headers = self.next_headers + for header in self.next_headers: + try: + new_headers[header] = substitute_vars(self.next_headers.get(header), self.headers_vars, values_dict) + except ValueError as e: + logger.warning(f"Failed to update the next Headers for the pagination due to error: {e}") + return new_headers + + def get_next_body(self, values_dict): + """ + Generates the next Body to use based on replacing the parameters in next_url with the values from given + values_dict. + :param values_dict: dictionary with values + :return: next_body with values instead of variables + """ + new_body = self.next_body + if not isinstance(self.next_body, str): + try: + new_body = json.dumps(self.next_body) + except json.decoder.JSONDecodeError: + new_body = str(self.next_body) + try: + new_body = substitute_vars(new_body, self.body_vars, values_dict) + except ValueError as e: + logger.warning(f"Failed to update the next Body for the pagination due to error: {e}") + + # Revert flattening of object if needed + if isinstance(self.next_body, dict): + new_body = json.loads(new_body) + return new_body + + def did_pagination_end(self, res, call_count): + """ + Returns True if the pagination should end, False otherwise. + Decides that we should stop if either of the below: + 1. We reached the Stop indication + 2. We reached the max_calls. + :param res: The response we got from the last request. + :param call_count: the current amount of calls that were made. + :return: True if pagination should stop, False otherwise. + """ + should_stop = False + + # Check stop indication + if self.stop_indication: + should_stop = self.stop_indication.should_stop(get_nested_value(res, break_key_name(self.stop_indication.field))) + logger.debug(f"Pagination stop status: {should_stop}") + + # If stop indication says to not stop OR there is no stop indication >> stop if we reached the max call count + if not should_stop: + should_stop = self.max_calls <= call_count + return should_stop diff --git a/src/apis/general/README.md b/src/apis/general/README.md new file mode 100644 index 0000000..dd7259f --- /dev/null +++ b/src/apis/general/README.md @@ -0,0 +1,124 @@ +# General API Configuration +For structuring custom API calls use type `general` API with the parameters below. +- [Configuration](#configuration) +- [Pagination Configuration](#pagination-configuration-options) +- [Example](#example) + +## Configuration +| Parameter Name | Description | Required/Optional | Default | +|--------------------|-----------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| url | The request URL | Required | - | +| headers | The request Headers | Optional | `{}` | +| body | The request body | Optional | - | +| method | The request method (`GET` or `POST`) | Optional | `GET` | +| pagination | Pagination settings if needed (see [options below](#pagination-configuration-options)) | Optional | - | +| next_url | If needed to update the URL in next requests based on the last response. Supports using variables ([see below](#using-variables)) | Optional | - | +| response_data_path | The path to the data inside the response | Optional | response root | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | Add `type` as `api-fetcher` | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | + +## Pagination Configuration Options +If needed, you can configure pagination. + +| Parameter Name | Description | Required/Optional | Default | +|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------|---------| +| type | The pagination type (`url`, `body` or `headers`) | Required | - | +| url_format | If pagination type is `url`, configure the URL format used for the pagination. Supports using variables ([see below](#using-variables)). | Required if pagination type is `url` | - | +| update_first_url | `True` or `False`; If pagination type is `url`, and it's required to append new params to the first request URL and not reset it completely. | Optional if pagination type is `url` | False | +| headers_format | If pagination type is `headers`, configure the headers format used for the pagination. Supports using variables ([see below](#using-variables)). | Required if pagination type is `headers` | - | +| body_format | If pagination type is `body`, configure the body format used for the pagination. Supports using variables ([see below](#using-variables)). | Required if pagination type is `body` | - | +| stop_indication | When should the pagination end based on the response. (see [options below](#pagination-stop-indication-configuration)). | Optional (if not defined will stop on `max_calls`) | - | +| max_calls | Max calls that the pagination can make. (Supports up to 1000) | Optional | 1000 | + +## Pagination Stop Indication Configuration + +| Parameter Name | Description | Required/Optional | Default | +|----------------|-----------------------------------------------------------------------------------------|-------------------------------------------------|---------| +| field | The name of the field in the response body, to search the stop indication at | Required | - | +| condition | The stop condition (`empty`, `equals` or `contains`) | Required | - | +| value | If condition is `equals` or `contains`, the value of the `field` that we should stop at | Required if condition is `equals` or `contains` | - | + +## Using Variables +Using variables allows taking values from the response of the first request, to structure the request after it. +Mathematical operations `+` and `-` are supported, to add or reduce a number from the variable value. + +Use case examples for variable usage: +1. Update a date filter at every call +2. Update a page number in pagination + +To use variables: +- Wrap the variable name in curly brackets +- Provide the full path to that variable in the response +- Add `res.` prefix to the path. + +Example: Say this is my response: +```json +{ + "field": "value", + "another_field": { + "nested": 123 + }, + "num_arr": [1, 2, 3], + "obj_arr": [ + { + "field2": 345 + }, + { + "field2": 567 + } + ] +} +``` +Paths to fields values are structured like so: +- `{res.field}` = `"value"` +- `{res.another_field.nested}` = `123` +- `{res.num_arr.[2]}` = `3` +- `{res.obj_arr.[0].field2}` = `345` + +Using the fields values in the `next_url` for example like the below: +```Yaml +next_url: https://logz.io/{res.field}/{res.obj_arr[0].field2} +``` +Would update the URL at every call to have the value of the given fields from the response. In our example the url for the next call would be: +``` +https://logz.io/value/345 +``` +And in the call after it, it would update again according to the response and the `next_url` structure, and so on. + +## Example +```Yaml +apis: + - name: fetcher1 + type: general + url: https://first/request/url + headers: + CONTENT-TYPE: application/json + another-header: XXX + body: { + "size": 1000 + } + method: POST + additional_fields: + type: my_fetcher + another_field: 123 + pagination: + type: url + url_format: ?page={res.info.page+1} + update_first_url: True + stop_indication: + field: result + condition: empty + response_data_path: result + + - name: fetcher2 + type: general + url: https://first/request/url + additional_fields: + type: fetcher2 + next_url: https://url/for/any/request/after/first/?since={res.result.[0].sent} + +logzio: + url: https://listener-eu.logz.io:8071 # for us-east-1 region delete url param (default) + token: <> +``` diff --git a/src/apis/general/StopPaginationSettings.py b/src/apis/general/StopPaginationSettings.py new file mode 100644 index 0000000..b38a4b2 --- /dev/null +++ b/src/apis/general/StopPaginationSettings.py @@ -0,0 +1,53 @@ +from enum import Enum +import logging +from pydantic import BaseModel, Field, model_validator + +logger = logging.getLogger(__name__) + + +class StopCondition(Enum): + """ + Supported Stop conditions + """ + EMPTY = "empty" + EQUALS = "equals" + CONTAINS = "contains" + + +class StopPaginationSettings(BaseModel): + """ + Class that initialize API pagination settings stop condition. + :param field: the field in the response to check the stop condition on + :param condition: the condition to check on the field + :param value: the stop value of the field to check, required only if condition is 'equals' or 'contains' + """ + field: str + condition: StopCondition + value: str = Field(default=None, frozen=True) + + @model_validator(mode='after') + def _check_conditional_fields(self): + """ + Validates that: + if we got condition as 'contains' or 'equals' >> that we also got value + :return: self + """ + if self.condition in (StopCondition.EQUALS, StopCondition.CONTAINS) and not self.value: + raise ValueError(f"Used stop condition {self.condition} but missing required 'value' field.") + return self + + def should_stop(self, field_value): + """ + Returns True if the stop condition is met, False otherwise. + :param field_value: they value of 'self.field' in the response (None if it doesn't exist) + :return: True if the stop condition is met, False otherwise + """ + if self.condition == StopCondition.EMPTY: + return not field_value + elif self.condition == StopCondition.EQUALS: + return self.value == field_value + elif self.condition == StopCondition.CONTAINS: + return self.value in field_value + else: + logger.warning("Got invalid stop condition", self.condition) + return True diff --git a/tests/last_start_dates.txt b/src/apis/general/__init__.py similarity index 100% rename from tests/last_start_dates.txt rename to src/apis/general/__init__.py diff --git a/src/apis/oauth/OAuth.py b/src/apis/oauth/OAuth.py new file mode 100644 index 0000000..353fcab --- /dev/null +++ b/src/apis/oauth/OAuth.py @@ -0,0 +1,78 @@ +import logging +from pydantic import BaseModel, Field, model_validator +from time import time + + +from src.apis.general.Api import ApiFetcher + + +# Known keys to find token data +OAUTH_ACCESS_TOKEN_KEY = 'access_token' +OAUTH_TOKEN_EXPIRE_KEY = 'expires_in' + + +logger = logging.getLogger(__name__) + + +class OAuthApi(BaseModel): + """ + :param name: Optional custom name for the API. + :param token_request: ApiFetcher object that contains the request to get the token + :param data_request: ApiFetcher object that contains the request to get the data + :param scrape_interval_minutes: the interval between scraping jobs. + :param token: The access token, generated by the class after the first request call. + :param token_expire: The access token expiration time in UNIX, generated by the class after the first request call. + """ + name: str = Field(default="oauth") + token_request: ApiFetcher + data_request: ApiFetcher + scrape_interval_minutes: int = Field(default=1, alias="scrape_interval", ge=1) + additional_fields: dict = Field(default={}) + token: str = Field(default=None, init=False, init_var=True) + token_expire: float = Field(default=0, init=False, init_var=True) + + @model_validator(mode='after') + def _check_headers(self): + """ + Validates that: + - Data request has 'Content-Type' header as 'application/json' + :return: self + """ + # Make sure the content-type exists for the data request + if not self.data_request.headers.get("Content-Type"): + self.data_request.headers["Content-Type"] = "application/json" + + # Initialize the type + if not self.additional_fields.get("type"): + self.additional_fields["type"] = "api-fetcher" + + return self + + def _update_token(self): + """ + Checks if the token expiration passed and if so, gets a new one and updates the data request 'Authorization' + header accordingly. + """ + token_response = {} + + if time() > (self.token_expire - 60): + try: + logger.debug("Sending request to update the access token.") + token_response = self.token_request.send_request()[0] + self.token, self.token_expire = (token_response.get(OAUTH_ACCESS_TOKEN_KEY), + int(token_response.get(OAUTH_TOKEN_EXPIRE_KEY)) + time()) + self.data_request.headers["Authorization"] = f"Bearer {self.token}" + except IndexError: + logger.error("Failed to get token for OAuth API request.") + except ValueError: + logger.error(f"Failed to get token expiration time. Received value " + f"'{token_response.get(OAUTH_TOKEN_EXPIRE_KEY)}'.") + + def send_request(self): + """ + Makes sure the token expiration is not passed and sends a request to get data. + :return: all the responses that were received from the data request + """ + logger.debug("Checking if to update the access token and sending request to get data.") + self._update_token() + return self.data_request.send_request() diff --git a/src/apis/oauth/README.md b/src/apis/oauth/README.md new file mode 100644 index 0000000..6a2d5f5 --- /dev/null +++ b/src/apis/oauth/README.md @@ -0,0 +1,33 @@ +# OAuth API Configuration +For structuring custom OAuth calls use type `oauth` API with the parameters below. + +## Configuration +| Parameter Name | Description | Required/Optional | Default | +|-------------------|-------------------------------------------------------------------------------------------------------------------------------|-------------------|-----------------------------| +| name | Name of the API (custom name) | Optional | the defined `url` | +| token_request | Nest here any detail relevant to the request to get the bearer access token. (Options in [General API](../general/README.md)) | Required | - | +| data_request | Nest here any detail relevant to the data request. (Options in [General API](../general/README.md)) | Required | - | +| scrape_interval | Time interval to wait between runs (unit: `minutes`) | Optional | 1 (minute) | +| additional_fields | Additional custom fields to add to the logs before sending to logzio | Optional | Add `type` as `api-fetcher` | + +## Example +```Yaml +apis: + - name: oauth test + type: oauth + token_request: + url: https://the/token/url + headers: + my_client_id: my_client_password + data_request: + url: https://the/data/url + method: POST + scrape_interval: 8 + additional_fields: + type: oauth-test + some_field_to_add_to_logs: 1234 + +logzio: + url: https://listener-au.logz.io:8071 # for us-east-1 region delete url param (default) + token: <> +``` diff --git a/src/apis/oauth/__init__.py b/src/apis/oauth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/apis_manager.py b/src/apis_manager.py deleted file mode 100644 index 05470ed..0000000 --- a/src/apis_manager.py +++ /dev/null @@ -1,191 +0,0 @@ -import logging -import os -import signal -import threading -import requests - -from typing import Optional -from requests.sessions import InvalidSchema -from .azure_graph import AzureGraph -from .azure_mail_reports import AzureMailReports -from .config_reader import ConfigReader -from .data.logzio_connection import LogzioConnection -from .data.auth_api_data import AuthApiData -from .data.oauth_api_data import OAuthApiData -from .api import Api -from .cisco_secure_x import CiscoSecureX -from .general_auth_api import GeneralAuthApi -from .logzio_shipper import LogzioShipper -from .oauth_api import OAuthApi - -logger = logging.getLogger(__name__) - - -class ApisManager: - API_AZURE_MAIL_REPORTS_TYPE = "azure_mail_reports" - CONFIG_FILE = 'src/shared/config.yaml' - LAST_START_DATES_FILE = 'src/shared/last_start_dates.txt' - - API_GENERAL_TYPE = 'general' - API_CISCO_SECURE_X_TYPE = 'cisco_secure_x' - API_AZURE_GRAPH_TYPE = 'azure_graph' - - AUTH_API_TYPES = [API_GENERAL_TYPE, API_CISCO_SECURE_X_TYPE] - OAUTH_API_TYPES = [API_GENERAL_TYPE, API_AZURE_GRAPH_TYPE, API_AZURE_MAIL_REPORTS_TYPE] - - def __init__(self) -> None: - self._apis: list[Api] = [] - self._logzio_connection: Optional[LogzioConnection] = None - self._threads = [] - self._event = threading.Event() - self._lock = threading.Lock() - - def run(self) -> None: - if not self._read_data_from_config(): - return - - if len(self._apis) == 0: - return - - for api in self._apis: - self._threads.append(threading.Thread(target=self._run_api_scheduled_task, args=(api,))) - - for thread in self._threads: - thread.start() - - signal.sigwait([signal.SIGINT, signal.SIGTERM]) - self.__exit_gracefully() - - def _read_data_from_config(self) -> bool: - config_reader = ConfigReader(ApisManager.CONFIG_FILE, ApisManager.API_GENERAL_TYPE, ApisManager.AUTH_API_TYPES, - ApisManager.OAUTH_API_TYPES) - - logzio_connection = config_reader.get_logzio_connection() - - if logzio_connection is None: - return False - - self._logzio_connection = logzio_connection - - for auth_api_config_data in config_reader.get_auth_apis_data(): - if auth_api_config_data is None: - return False - - self._add_auth_api(auth_api_config_data) - - for oauth_api_config_data in config_reader.get_oauth_apis_data(): - if oauth_api_config_data is None: - return False - - self._add_oauth_api(oauth_api_config_data) - - return True - - def _add_auth_api(self, auth_api_data: AuthApiData) -> None: - if auth_api_data.base_data.base_data.type == ApisManager.API_GENERAL_TYPE: - self._apis.append(GeneralAuthApi(auth_api_data.base_data, auth_api_data.general_type_data)) - else: - self._apis.append(CiscoSecureX(auth_api_data.base_data)) - - def _add_oauth_api(self, oauth_api_data: OAuthApiData) -> None: - if oauth_api_data.base_data.base_data.type == ApisManager.API_GENERAL_TYPE: - self._apis.append(OAuthApi(oauth_api_data.base_data, oauth_api_data.general_type_data)) - elif oauth_api_data.base_data.base_data.type == ApisManager.API_AZURE_GRAPH_TYPE: - self._apis.append(AzureGraph(oauth_api_data)) - elif oauth_api_data.base_data.base_data.type == ApisManager.API_AZURE_MAIL_REPORTS_TYPE: - self._apis.append(AzureMailReports(oauth_api_data)) - - def _run_api_scheduled_task(self, api: Api) -> None: - logzio_shipper = LogzioShipper(self._logzio_connection.url, self._logzio_connection.token) - - for api_custom_field in api.get_api_custom_fields(): - logzio_shipper.add_custom_field_to_list(api_custom_field) - - while True: - thread = threading.Thread(target=self._send_data_to_logzio, args=(api, logzio_shipper,)) - - thread.start() - thread.join() - - if self._event.wait(timeout=api.get_api_time_interval()): - break - - def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None: - logger.info("Task is running for api {}...".format(api.get_api_name())) - - is_data_exist = False - is_data_sent_successfully = True - - try: - for data in api.fetch_data(): - is_data_exist = True - logzio_shipper.add_log_to_send(data) - - if is_data_exist: - logzio_shipper.send_to_logzio() - except requests.exceptions.InvalidURL as e: - logger.error(f"Failed to send data to Logz.io... Invalid url: {e}") - os.kill(os.getpid(), signal.SIGTERM) - return - except InvalidSchema as e: - logger.error(f"Failed to send data to Logz.io... Invalid schema: {e}") - os.kill(os.getpid(), signal.SIGTERM) - return - except requests.HTTPError as e: - logger.error(f"Failed to send data to Logz.io... HTTP error: {e}") - if e.response.status_code == 401: - os.kill(os.getpid(), signal.SIGTERM) - return - except Api.ApiError as e: - logger.error(f"Failed to send data to Logz.io... API error: {e}") - os.kill(os.getpid(), signal.SIGTERM) - return - except Exception as e: - logger.error(f"Failed to send data to Logz.io... exception: {e}") - is_data_sent_successfully = False - - if is_data_exist and is_data_sent_successfully: - api.update_start_date_filter() - self._write_last_start_date_to_file(api.get_api_name(), api.get_last_start_date()) - - logger.info( - "Task is over. A new Task for api {0} will run in {1} minute/s.".format(api.get_api_name(), - int(api.get_api_time_interval() / 60))) - - def _write_last_start_date_to_file(self, api_name: str, last_start_date: str) -> None: - self._lock.acquire() - - with open(ApisManager.LAST_START_DATES_FILE, 'r+') as file: - file_lines = file.readlines() - line_num = self._get_api_line_num_in_file(api_name, file_lines) - - if not file_lines: - file_lines.append("{0}: {1}\n".format(api_name, last_start_date)) - elif line_num == -1: - file_lines.append("{0}: {1}\n".format(api_name, last_start_date)) - else: - file_lines[line_num] = "{0}: {1}\n".format(api_name, last_start_date) - - file.seek(0) - file.writelines(file_lines) - - self._lock.release() - - def _get_api_line_num_in_file(self, api_name: str, file_lines: list[str]) -> int: - line_num = 0 - - for line in file_lines: - if line.split(':')[0] == api_name: - return line_num - - line_num += 1 - - return -1 - - def __exit_gracefully(self) -> None: - logger.info("Signal caught...") - - self._event.set() - - for thread in self._threads: - thread.join() diff --git a/src/auth_api.py b/src/auth_api.py deleted file mode 100644 index dfbc503..0000000 --- a/src/auth_api.py +++ /dev/null @@ -1,98 +0,0 @@ -import logging -import json -import requests - -from typing import Generator, Optional -from datetime import datetime, timedelta -from dateutil import parser -from requests import Response -from .api import Api -from .data.api_http_request import ApiHttpRequest -from .data.base_data.auth_api_base_data import AuthApiBaseData -from .data.general_type_data.auth_api_general_type_data import AuthApiGeneralTypeData - -logger = logging.getLogger(__name__) - - -class AuthApi(Api): - - def __init__(self, api_base_data: AuthApiBaseData, api_general_type_data: AuthApiGeneralTypeData) -> None: - self._api_http_request = api_general_type_data.http_request - super().__init__(api_base_data.base_data, api_general_type_data.general_type_data) - - def fetch_data(self) -> Generator[str, None, list[str]]: - total_data_num = 0 - api_url = self._build_api_url() - is_first_fetch = True - is_last_data_to_fetch = False - first_item_date: Optional[str] = None - last_datetime_to_fetch: Optional[datetime] = None - - while True: - try: - next_url, data = self._get_data_from_api(api_url) - except Exception: - raise - - if not data: - logger.info("No new data available from api {}.".format(self._base_data.name)) - return data - - if is_first_fetch: - first_item_date = self._get_last_date(data[0]) - last_datetime_to_fetch = parser.parse(first_item_date) - timedelta( - days=self._base_data.settings.days_back_to_fetch) - is_first_fetch = False - - for item in data: - if not self._is_item_in_fetch_frame(item, last_datetime_to_fetch): - is_last_data_to_fetch = True - break - - yield json.dumps(item) - total_data_num += 1 - - if next_url is None or is_last_data_to_fetch: - break - - api_url = next_url - - logger.info("Got {0} total data from api {1}".format(total_data_num, self._base_data.name)) - - self._current_data_last_date = first_item_date - - def _build_api_url(self) -> str: - api_url = self._api_http_request.url - api_filters_num = self._base_data.get_filters_size() - date_filter_index = self._base_data.get_filter_index( - self._general_type_data.start_date_name) - api_url += '?' - if date_filter_index == -1: - new_start_date = self.get_start_date_filter() - api_url += self._general_type_data.start_date_name + '=' + new_start_date - if api_filters_num > 0: - api_url += '&' - - for api_filter in self._base_data.filters: - api_url += api_filter.key + '=' + str(api_filter.value) - api_filters_num -= 1 - - if api_filters_num > 0: - api_url += '&' - - return api_url - - def _send_request(self, url) -> Response: - if self._api_http_request.method == ApiHttpRequest.GET_METHOD: - response = requests.get(url=url, - auth=(self._base_data.credentials.id, - self._base_data.credentials.key), - headers=self._api_http_request.headers) - else: - response = requests.post(url=url, - auth=(self._base_data.credentials.id, - self._base_data.credentials.key), - headers=self._api_http_request.headers, - data=self._api_http_request.body) - - return response diff --git a/src/azure_graph.py b/src/azure_graph.py deleted file mode 100644 index cda0df7..0000000 --- a/src/azure_graph.py +++ /dev/null @@ -1,38 +0,0 @@ -import logging -import urllib - -from datetime import datetime, timedelta -from src.data.oauth_api_data import OAuthApiData -from src.oauth_api import OAuthApi - -logger = logging.getLogger(__name__) - - -class AzureGraph(OAuthApi): - DEFAULT_GRAPH_DATA_LINK = 'value' - AZURE_GRAPH_TOKEN_SCOPE = 'scope' - AZURE_GRAPH_FILTER_CONCAT = '&$' - NEXT_LINK = '@odata.nextLink' - - def __init__(self, oauth_api_data: OAuthApiData) -> None: - oauth_api_data.general_type_data.general_type_data.json_paths.next_url = self.NEXT_LINK - oauth_api_data.general_type_data.general_type_data.json_paths.data = self.DEFAULT_GRAPH_DATA_LINK - super().__init__(oauth_api_data.base_data, oauth_api_data.general_type_data) - - def get_last_start_date(self) -> str: - return self._current_data_last_date - - def _build_api_url(self) -> str: - api_url = self._data_request.url - api_filters_num = self._base_data.get_filters_size() - new_start_date = self.get_start_date_filter() - api_url += "?$filter=" + self._general_type_data.json_paths.data_date + ' gt ' + new_start_date - if api_filters_num > 0: - api_url += '&$' - for api_filter in self._base_data.filters: - api_url += api_filter.key + '=' + str(api_filter.value) - api_filters_num -= 1 - - if api_filters_num > 0: - api_url += self.AZURE_GRAPH_FILTER_CONCAT - return api_url diff --git a/src/azure_mail_reports.py b/src/azure_mail_reports.py deleted file mode 100644 index 1e9e322..0000000 --- a/src/azure_mail_reports.py +++ /dev/null @@ -1,79 +0,0 @@ -import logging -from dateutil import parser -import re - -from datetime import datetime -from src.api import Api -from src.data.oauth_api_data import OAuthApiData -from src.oauth_api import OAuthApi - -logger = logging.getLogger(__name__) - - -class AzureMailReports(OAuthApi): - MAIL_REPORTS_DATA_LINK = 'd.results' - MAIL_REPORTS_FILTER_CONCAT = '&$' - MAIL_REPORTS_MAX_PAGE_SIZE = 1000 - DATE_REGEX_FILTER = '\d+' - - def __init__(self, oauth_api_data: OAuthApiData) -> None: - oauth_api_data.general_type_data.general_type_data.json_paths.data = self.MAIL_REPORTS_DATA_LINK - self._previous_end_date = None - super().__init__(oauth_api_data.base_data, oauth_api_data.general_type_data) - - def get_last_start_date(self) -> str: - return self._current_data_last_date - - def _build_api_url(self) -> str: - api_url = self._data_request.url - api_filters_num = self._base_data.get_filters_size() - new_end_date = self.get_new_end_date() - new_start_date = self.get_start_date_filter() - api_url += f"?$filter={self._general_type_data.start_date_name} eq datetime'{new_start_date}' and {self._general_type_data.end_date_name} eq datetime'{new_end_date}'" - self._previous_end_date = new_end_date - if api_filters_num > 0: - api_url += self.MAIL_REPORTS_FILTER_CONCAT - for api_filter in self._base_data.filters: - api_url += api_filter.key + '=' + str(api_filter.value) - api_filters_num -= 1 - if api_filters_num > 0: - api_url += self.MAIL_REPORTS_FILTER_CONCAT - return api_url - - def _get_last_date(self, first_item: dict) -> str: - first_item_date = self._get_json_path_value_from_data( - self._general_type_data.json_paths.data_date, first_item) - - if first_item_date is None: - logger.error( - "The json path for api {}'s data date is wrong. Please change your configuration.".format( - self._base_data.name)) - raise Api.ApiError - return self._get_formatted_date_from_date_path_value(first_item_date) - - def _is_item_in_fetch_frame(self, item: dict, last_datetime_to_fetch: datetime) -> bool: - item_date = self._get_json_path_value_from_data( - self._general_type_data.json_paths.data_date, item) - item_datetime = parser.parse(self._get_formatted_date_from_date_path_value(item_date)) - if item_datetime < last_datetime_to_fetch: - return False - - return True - - def _get_formatted_date_from_date_path_value(self, date_path_value: str) -> str: - epoch_milisec_date = re.findall(self.DATE_REGEX_FILTER, date_path_value) - date = datetime.fromtimestamp(int(int(epoch_milisec_date[0]) / 1000)) - formatted_date = date.isoformat(' ', 'seconds') - formatted_date = formatted_date.replace(' ', 'T') - formatted_date += 'Z' - return formatted_date - - def _set_current_data_last_date(self, date): - # This comparison might not work on other date formats - if (self._previous_end_date and date and self._previous_end_date > date) or not date: - self._set_current_data_last_date(self._previous_end_date) - else: - self._current_data_last_date = date - - def get_new_end_date(self): - return self.get_current_time_utc_string() diff --git a/src/cisco_secure_x.py b/src/cisco_secure_x.py deleted file mode 100644 index ebc9845..0000000 --- a/src/cisco_secure_x.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging - -from .auth_api import AuthApi -from .data.base_data.auth_api_base_data import AuthApiBaseData -from .data.general_type_data.auth_api_general_type_data import AuthApiGeneralTypeData -from .data.general_type_data.api_general_type_data import ApiGeneralTypeData -from .data.api_http_request import ApiHttpRequest -from .data.general_type_data.api_json_paths import ApiJsonPaths - - -logger = logging.getLogger(__name__) - - -class CiscoSecureX(AuthApi): - - HTTP_METHOD = 'GET' - URL = 'https://api.amp.cisco.com/v1/events' - START_DATE_NAME = 'start_date' - NEXT_URL_JSON_PATH = 'metadata.links.next' - DATA_JSON_PATH = 'data' - DATA_DATE_JSON_PATH = 'date' - - def __init__(self, api_base_data: AuthApiBaseData) -> None: - http_request = ApiHttpRequest(CiscoSecureX.HTTP_METHOD, CiscoSecureX.URL) - json_paths = ApiJsonPaths(api_next_url_json_path=CiscoSecureX.NEXT_URL_JSON_PATH, - api_data_json_path=CiscoSecureX.DATA_JSON_PATH, - api_data_date_json_path=CiscoSecureX.DATA_DATE_JSON_PATH) - general_type_data = ApiGeneralTypeData(CiscoSecureX.START_DATE_NAME, json_paths) - super().__init__(api_base_data, - AuthApiGeneralTypeData(general_type_data, http_request)) diff --git a/src/config/ConfigReader.py b/src/config/ConfigReader.py new file mode 100644 index 0000000..c66132c --- /dev/null +++ b/src/config/ConfigReader.py @@ -0,0 +1,98 @@ +import logging + +from pydantic import ValidationError +import yaml + +# Needed for creating the input and output instances +from src.apis.general.Api import ApiFetcher +from src.apis.oauth.OAuth import OAuthApi +from src.apis.azure.AzureGraph import AzureGraph +from src.apis.azure.AzureMailReports import AzureMailReports +from src.apis.cloudflare.Cloudflare import Cloudflare +from src.output.LogzioShipper import LogzioShipper + +INPUT_API_FIELD = "apis" +OUTPUT_LOGZIO_FIELD = "logzio" +API_TYPES_TO_CLASS_NAME_MAPPING = { + "general": "ApiFetcher", + "oauth": "OAuthApi", + "azure_general": "AzureApi", + "azure_graph": "AzureGraph", + "azure_mail_reports": "AzureMailReports", + "cloudflare": "Cloudflare" +} + +logger = logging.getLogger(__name__) + + +class ConfigReader: + """ + Class that reads a Yaml config and generates instances based on it + """ + def __init__(self, conf_file): + """ + Receives a path to a config file, reads it and generates other classes instances based on it. + :param conf_file: path to the config file + """ + self.config = self._read_config(conf_file) + self.api_instances, self.logzio_shipper = self.generate_instances() + + @staticmethod + def _read_config(conf_file): + """ + Opens the given Yaml configuration file. + :param conf_file: yaml config file path + :return: content of the given file + """ + try: + logger.debug(f"Reading config file {conf_file}") + with open(conf_file, "r") as conf: + return yaml.safe_load(conf) + except FileNotFoundError: + logger.error(f"Did not find file {conf_file}.") + except PermissionError: + logger.error(f"Missing read permission for file {conf_file}.") + except Exception as e: + logger.error(f"Failed to read config from path {conf_file} due to error {e}.") + return None + + def generate_instances(self): + """ + Uses 'pydantic' to validate the given APIs config and generates API fetcher per valid config. + :return: API fetcher (ApiFetcher) instances + """ + api_instances = [] + logzio_shipper_instance = None + shipper_cls = globals().get("LogzioShipper") + if not self.config: + return api_instances, logzio_shipper_instance + apis = self.config.get(INPUT_API_FIELD) + logzio_conf = self.config.get(OUTPUT_LOGZIO_FIELD) + + if not apis: + logger.error(f"No inputs defined. Please make sure your API input is configured under '{INPUT_API_FIELD}'") + return api_instances, logzio_shipper_instance + + # Generate API fetchers + for api_conf in apis: + try: + api_type_cls_name = API_TYPES_TO_CLASS_NAME_MAPPING.get(api_conf.get("type")) + api_cls = globals().get(api_type_cls_name) + api_instance = api_cls(**api_conf) + api_instances.append(api_instance) + logger.debug(f"Created {api_instance.name}.") + except (AttributeError, ValidationError, TypeError) as e: + logger.error(f"Failed to create API fetcher for config {api_conf} due to error: {e}") + + # Generate Logzio shipper + if not logzio_conf: + logger.warning(f"No Logzio shipper output defined. Please make sure your Logzio config is configured under " + f"{OUTPUT_LOGZIO_FIELD}") + else: + try: + logzio_shipper_instance = shipper_cls(**logzio_conf) + logger.debug("Created logzio shipper.") + except (ValidationError, TypeError) as e: + logger.error(f"Failed to create Logzio shipper for config {logzio_conf} due to error: {e}") + + return api_instances, logzio_shipper_instance diff --git a/src/config/__init__.py b/src/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/config_reader.py b/src/config_reader.py deleted file mode 100644 index 8f01fcf..0000000 --- a/src/config_reader.py +++ /dev/null @@ -1,413 +0,0 @@ -import logging -import yaml - -from typing import Optional, Generator -from .data.logzio_connection import LogzioConnection -from .data.auth_api_data import AuthApiData -from .data.oauth_api_data import OAuthApiData -from .data.base_data.api_base_data import ApiBaseData -from .data.base_data.auth_api_base_data import AuthApiBaseData -from .data.base_data.oauth_api_base_data import OAuthApiBaseData -from .data.base_data.api_credentials import ApiCredentials -from .data.base_data.api_settings import ApiSettings -from .data.base_data.api_filter import ApiFilter -from .data.base_data.api_custom_field import ApiCustomField -from .data.general_type_data.api_general_type_data import ApiGeneralTypeData -from .data.general_type_data.auth_api_general_type_data import AuthApiGeneralTypeData -from .data.general_type_data.oauth_api_general_type_data import OAuthApiGeneralTypeData -from .data.general_type_data.api_json_paths import ApiJsonPaths -from .data.api_http_request import ApiHttpRequest - -logger = logging.getLogger(__name__) - - -class ConfigReader: - AUTH_API = 'auth' - OAUTH_API = 'oauth' - - LOGZIO_CONFIG_KEY = 'logzio' - LOGZIO_URL_CONFIG_KEY = 'url' - LOGZIO_TOKEN_CONFIG_KEY = 'token' - - AUTH_APIS_CONFIG_KEY = 'auth_apis' - OAUTH_APIS_CONFIG_KEY = 'oauth_apis' - API_TYPE_CONFIG_KEY = 'type' - API_NAME_CONFIG_TYPE = 'name' - API_CREDENTIALS_CONFIG_KEY = 'credentials' - API_CREDENTIALS_ID_CONFIG_KEY = 'id' - API_CREDENTIALS_KEY_CONFIG_KEY = 'key' - API_SETTINGS_CONFIG_KEY = 'settings' - API_SETTINGS_TIME_INTERVAL_CONFIG_KEY = 'time_interval' - API_SETTINGS_DAYS_BACK_FETCH_CONFIG_KEY = 'days_back_fetch' - API_FILTERS_CONFIG_KEY = 'filters' - API_CUSTOM_FIELDS_CONFIG_KEY = 'custom_fields' - API_START_DATE_NAME_CONFIG_KEY = 'start_date_name' - API_END_DATE_NAME_CONFIG_KEY = "end_date_name" - GENERAL_AUTH_API_HTTP_REQUEST_CONFIG_KEY = 'http_request' - OAUTH_API_TOKEN_HTTP_REQUEST_CONFIG_KEY = 'token_http_request' - OAUTH_API_DATA_HTTP_REQUEST_CONFIG_KEY = 'data_http_request' - API_HTTP_REQUEST_METHOD_CONFIG_KEY = 'method' - API_HTTP_REQUEST_URL_CONFIG_KEY = 'url' - API_HTTP_REQUEST_HEADERS_CONFIG_KEY = 'headers' - API_HTTP_REQUEST_BODY_CONFIG_KEY = 'body' - API_HTTP_REQUEST_PAGE_SIZE = 'page_size' - GENERAL_API_JSON_PATHS_CONFIG_KEY = 'json_paths' - GENERAL_API_JSON_PATHS_NEXT_URL_CONFIG_KEY = 'next_url' - GENERAL_API_JSON_PATHS_DATA_CONFIG_KEY = 'data' - GENERAL_API_JSON_PATHS_DATA_DATE_CONFIG_KEY = 'data_date' - - def __init__(self, config_file: str, api_general_type: str, auth_api_types: list[str], - oauth_api_types: list[str]) -> None: - with open(config_file, 'r') as config: - self._config_data = yaml.safe_load(config) - - self._api_general_type = api_general_type - self._auth_api_types = auth_api_types - self._oauth_api_types = oauth_api_types - - def get_logzio_connection(self) -> Optional[LogzioConnection]: - try: - logzio = self._config_data[ConfigReader.LOGZIO_CONFIG_KEY] - - logzio_url = logzio[ConfigReader.LOGZIO_URL_CONFIG_KEY] - logzio_token = logzio[ConfigReader.LOGZIO_TOKEN_CONFIG_KEY] - except KeyError: - logger.error( - "Your configuration is not valid: logzio must have url and token.") - return None - - return LogzioConnection(logzio_url, logzio_token) - - def get_auth_apis_data(self) -> Generator[AuthApiData, None, None]: - auth_api_num = 1 - - if ConfigReader.AUTH_APIS_CONFIG_KEY in self._config_data: - for config_auth_api_data in self._config_data[ConfigReader.AUTH_APIS_CONFIG_KEY]: - yield self._get_auth_api_data(config_auth_api_data, auth_api_num) - auth_api_num += 1 - - def get_oauth_apis_data(self) -> Generator[OAuthApiData, None, None]: - oauth_api_num = 1 - - if ConfigReader.OAUTH_APIS_CONFIG_KEY in self._config_data: - for config_oauth_api_data in self._config_data[ConfigReader.OAUTH_APIS_CONFIG_KEY]: - yield self._get_oauth_api_data(config_oauth_api_data, oauth_api_num) - oauth_api_num += 1 - - def _get_auth_api_data(self, config_auth_api_data: dict, auth_api_num: int) -> Optional[AuthApiData]: - auth_api_base_data = self._get_auth_api_base_data(config_auth_api_data, auth_api_num) - - if auth_api_base_data is None: - return None - - if auth_api_base_data.base_data.type == self._api_general_type: - api_general_type_data = self._get_auth_api_general_type_data(config_auth_api_data, auth_api_num) - - if api_general_type_data is None: - return None - - return AuthApiData(auth_api_base_data, api_general_type_data) - - return AuthApiData(auth_api_base_data) - - def _get_oauth_api_data(self, config_oauth_api_data: dict, oauth_api_num: int) -> Optional[OAuthApiData]: - oauth_api_base_data = self._get_oauth_api_base_data(config_oauth_api_data, oauth_api_num) - - if oauth_api_base_data is None: - return None - - if oauth_api_base_data.base_data.type in self._oauth_api_types: - api_general_type_data = self._get_oauth_api_general_type_data(config_oauth_api_data, oauth_api_num) - - if api_general_type_data is None: - return None - - return OAuthApiData(oauth_api_base_data, api_general_type_data) - - return None - - def _get_auth_api_base_data(self, config_auth_api_data: dict, auth_api_num: int) -> Optional[AuthApiBaseData]: - api_base_data = self._get_api_basic_data(config_auth_api_data, ConfigReader.AUTH_API, auth_api_num) - - if api_base_data is None: - return None - - if api_base_data.type not in self._auth_api_types: - logger.error("Your configuration is not valid: the auth api #{0} has an unsupported type - {1}. " - "The supported types are: {2}".format(auth_api_num, - api_base_data.type, - self._auth_api_types)) - return None - - return AuthApiBaseData(api_base_data) - - def _get_oauth_api_base_data(self, config_oauth_api_data: dict, oauth_api_num: int) -> Optional[OAuthApiBaseData]: - api_base_data = self._get_api_basic_data(config_oauth_api_data, ConfigReader.OAUTH_API, oauth_api_num) - - if api_base_data is None: - return None - - if api_base_data.type not in self._oauth_api_types: - logger.error("Your configuration is not valid: the oauth api #{0} has an unsupported type - {1}. " - "The supported types are: {2}".format(oauth_api_num, - api_base_data.type, - self._oauth_api_types)) - return None - - try: - api_token_http_request, api_data_http_request = self._get_oauth_api_http_requests(config_oauth_api_data, - oauth_api_num) - except TypeError: - return None - - if api_token_http_request is None or api_data_http_request is None: - return None - - return OAuthApiBaseData(api_base_data, api_token_http_request, api_data_http_request) - - def _get_api_basic_data(self, config_api_data: dict, api_group_type: str, api_num: int) -> Optional[ApiBaseData]: - api_type = self._get_api_type(config_api_data, api_group_type, api_num) - api_name = self._get_api_name(config_api_data, api_group_type, api_num) - api_credentials = self._get_api_credentials(config_api_data, api_group_type, api_num) - api_settings = self._get_api_settings(config_api_data, api_group_type, api_num) - api_filters = self._get_api_filters(config_api_data) - api_custom_fields = self._get_api_custom_fields(config_api_data) - - if api_type is None or api_name is None or api_credentials is None or api_settings is None: - return None - - return ApiBaseData(api_type, api_name, api_credentials, api_settings, api_filters, api_custom_fields) - - def _get_api_type(self, config_api_data: dict, api_group_type: str, api_num: int) -> Optional[str]: - try: - api_type = config_api_data[ConfigReader.API_TYPE_CONFIG_KEY] - except KeyError: - logger.error( - "Your configuration is not valid: the {0} api #{1} must have type.".format(api_group_type, api_num)) - return None - - return api_type - - def _get_api_name(self, config_api_data: dict, api_group_type: str, api_num: int) -> Optional[str]: - try: - api_name = config_api_data[ConfigReader.API_NAME_CONFIG_TYPE] - except KeyError: - logger.error( - "Your configuration is not valid: the {0} api #{1} must have name.".format(api_group_type, api_num)) - return None - - return api_name - - def _get_api_credentials(self, config_api_data: dict, api_group_type: str, - api_num: int) -> Optional[ApiCredentials]: - try: - api_credentials = config_api_data[ConfigReader.API_CREDENTIALS_CONFIG_KEY] - - api_credentials_id = api_credentials[ConfigReader.API_CREDENTIALS_ID_CONFIG_KEY] - api_credentials_key = api_credentials[ConfigReader.API_CREDENTIALS_KEY_CONFIG_KEY] - except KeyError: - logger.error( - "Your configuration is not valid: the {0} api #{1} must have credentials with id and key.".format( - api_group_type, api_num)) - return None - - return ApiCredentials(api_credentials_id, api_credentials_key) - - def _get_api_settings(self, config_api_data: dict, api_group_type: str, api_num: int) -> Optional[ApiSettings]: - try: - settings = config_api_data[ConfigReader.API_SETTINGS_CONFIG_KEY] - - time_interval = settings[ConfigReader.API_SETTINGS_TIME_INTERVAL_CONFIG_KEY] * 60 - except KeyError: - logger.error( - "Your configuration is not valid: the {0} api #{1} must have settings with time_interval.".format( - api_group_type, api_num)) - return None - except TypeError: - logger.error("Your configuration is not valid: the time_interval (under settings) of {0} api #{1} must " - "be whole positive integer.".format(api_group_type, api_num)) - return None - - days_back_to_fetch = settings.get(ConfigReader.API_SETTINGS_DAYS_BACK_FETCH_CONFIG_KEY) - - if days_back_to_fetch is None: - return ApiSettings(time_interval) - - try: - int(days_back_to_fetch) - except ValueError: - logger.error("Your configuration is not valid: the days_back_fetch (under settings) of {0} api #{1} must " - "be whole positive integer.".format(api_group_type, api_num)) - return None - - return ApiSettings(time_interval, days_back_to_fetch) - - def _get_api_filters(self, config_api_data: dict) -> Optional[list[ApiFilter]]: - api_filters = [] - - if ConfigReader.API_FILTERS_CONFIG_KEY in config_api_data: - for filter_key, filter_value in config_api_data[ConfigReader.API_FILTERS_CONFIG_KEY].items(): - api_filters.append(ApiFilter(filter_key, filter_value)) - - return api_filters - - def _get_api_custom_fields(self, config_api_data: dict) -> Optional[list[ApiCustomField]]: - api_custom_fields = [] - - if ConfigReader.API_CUSTOM_FIELDS_CONFIG_KEY in config_api_data: - for api_custom_field_key, api_custom_field_value in config_api_data[ - ConfigReader.API_CUSTOM_FIELDS_CONFIG_KEY].items(): - api_custom_fields.append(ApiCustomField(api_custom_field_key, api_custom_field_value)) - - return api_custom_fields - - def _get_auth_api_general_type_data(self, config_auth_api_data: dict, - auth_api_num: int) -> Optional[AuthApiGeneralTypeData]: - api_general_type_data = self._get_api_general_type_data(config_auth_api_data, - ConfigReader.AUTH_API, - auth_api_num) - - if api_general_type_data is None: - return None - - api_http_request = self._get_auth_api_http_request(config_auth_api_data, auth_api_num) - - if api_http_request is None: - return None - - return AuthApiGeneralTypeData(api_general_type_data, api_http_request) - - def _get_oauth_api_general_type_data(self, config_oauth_api_data: dict, - oauth_api_num: int) -> Optional[OAuthApiGeneralTypeData]: - api_general_type_data = self._get_api_general_type_data(config_oauth_api_data, - ConfigReader.OAUTH_API, - oauth_api_num) - - if api_general_type_data is None: - return None - - return OAuthApiGeneralTypeData(api_general_type_data) - - def _get_api_general_type_data(self, config_api_data, api_group_type: str, - api_num: int) -> Optional[ApiGeneralTypeData]: - api_start_date_name = self._get_api_start_date_name(config_api_data, api_group_type, api_num) - api_end_date_name = self._get_api_end_date_name(config_api_data, api_group_type, api_num) - api_json_paths = self._get_api_json_paths(config_api_data, api_group_type, api_num) - - if (api_start_date_name is None and api_group_type != self.OAUTH_API) or api_json_paths is None: - logger.error( - "Your configuration is not valid:\"json_paths\" must exist for all api types, \"start_date_name\" must exist for non oauth api types") - return None - - return ApiGeneralTypeData(api_start_date_name, api_end_date_name, api_json_paths) - - def _get_api_start_date_name(self, config_api_data: dict, api_group_type: str, api_num: int) -> Optional[str]: - try: - api_start_date_name = config_api_data[ConfigReader.API_START_DATE_NAME_CONFIG_KEY] - except KeyError: - logger.warning( - "Missing field in config: the general type {0} api #{1} must have start_date_name.".format( - api_group_type, api_num)) - return None - - return api_start_date_name - - def _get_api_end_date_name(self, config_api_data: dict, api_group_type: str, api_num: int) -> Optional[str]: - try: - api_end_date_name = config_api_data[ConfigReader.API_END_DATE_NAME_CONFIG_KEY] - except KeyError: - logger.warning( - "Missing field in config: the general type {0} api #{1} must have end_date_name.".format( - api_group_type, api_num)) - return None - - return api_end_date_name - - def _get_api_json_paths(self, config_api_data: dict, api_group_type: str, api_num: int) -> Optional[ApiJsonPaths]: - api_json_paths = config_api_data[ConfigReader.GENERAL_API_JSON_PATHS_CONFIG_KEY] - api_data_date_json_path = api_json_paths.get(ConfigReader.GENERAL_API_JSON_PATHS_DATA_DATE_CONFIG_KEY) - api_next_url_json_path = api_json_paths.get(ConfigReader.GENERAL_API_JSON_PATHS_NEXT_URL_CONFIG_KEY) - api_data_json_path = api_json_paths.get(ConfigReader.GENERAL_API_JSON_PATHS_DATA_CONFIG_KEY) - - if api_data_date_json_path is not None: - return ApiJsonPaths(api_next_url_json_path, api_data_json_path, api_data_date_json_path) - else: - logger.error( - "Your configuration is not valid: the general type {0} api #{1} must have json_paths with next_url, " - "data and data_date.".format(api_group_type, api_num)) - if api_data_date_json_path is None: - return None - - def _get_auth_api_http_request(self, config_auth_api_data: dict, auth_api_num: int) -> Optional[ApiHttpRequest]: - try: - api_http_request = config_auth_api_data[ConfigReader.GENERAL_AUTH_API_HTTP_REQUEST_CONFIG_KEY] - - api_http_request_method = api_http_request[ConfigReader.API_HTTP_REQUEST_METHOD_CONFIG_KEY] - api_url = api_http_request[ConfigReader.API_HTTP_REQUEST_URL_CONFIG_KEY] - except TypeError: - logger.error( - "Your configuration is not valid: the general type auth api #{} must have http_request with " - "method and url.".format(auth_api_num)) - return None - - if api_http_request_method not in ApiHttpRequest.HTTP_METHODS: - logger.error( - "Your configuration is not valid: the general type auth api #{0} has an unsupported method (under " - "http_request) - {1}. The supported methods are: {2}".format(auth_api_num, - api_http_request_method, - ApiHttpRequest.HTTP_METHODS)) - return None - - return ApiHttpRequest(api_http_request_method, - api_url, - api_http_request.get(ConfigReader.API_HTTP_REQUEST_HEADERS_CONFIG_KEY), - api_http_request.get(ConfigReader.API_HTTP_REQUEST_BODY_CONFIG_KEY)) - - def _get_oauth_api_http_requests(self, config_oauth_api_data: dict, - oauth_api_num: int) -> Optional[tuple[ApiHttpRequest, ApiHttpRequest]]: - try: - api_token_http_request = config_oauth_api_data[ConfigReader.OAUTH_API_TOKEN_HTTP_REQUEST_CONFIG_KEY] - api_data_http_request = config_oauth_api_data[ConfigReader.OAUTH_API_DATA_HTTP_REQUEST_CONFIG_KEY] - api_token_http_request_method = api_token_http_request[ConfigReader.API_HTTP_REQUEST_METHOD_CONFIG_KEY] - api_token_url = api_token_http_request[ConfigReader.API_HTTP_REQUEST_URL_CONFIG_KEY] - api_data_http_request_method = api_data_http_request[ConfigReader.API_HTTP_REQUEST_METHOD_CONFIG_KEY] - api_data_url = api_data_http_request[ConfigReader.API_HTTP_REQUEST_URL_CONFIG_KEY] - except KeyError as e: - logger.error( - "Your configuration is not valid: missing parameter: \"{}\" from token_http_request or" - " data_http_request for oauth_api.".format(e.args[0])) - return None - except TypeError: - logger.error( - "Your configuration is not valid: the general type oauth api #{} must have token_http_request and" - "data_http_request both with method and url.".format(oauth_api_num)) - return None - - if api_token_http_request_method not in ApiHttpRequest.HTTP_METHODS: - logger.error( - "Your configuration is not valid: the general type oauth api #{0} has an unsupported method (under " - "token_http_request) - {1}. The supported methods are: {2}".format(oauth_api_num, - api_token_http_request_method, - ApiHttpRequest.HTTP_METHODS)) - return None - - if api_data_http_request_method not in ApiHttpRequest.HTTP_METHODS: - logger.error( - "Your configuration is not valid: the general type oauth api #{0} has an unsupported method (under " - "data_http_request) - {1}. The supported methods are: {2}".format(oauth_api_num, - api_data_http_request_method, - ApiHttpRequest.HTTP_METHODS)) - return None - - token_http_request = ApiHttpRequest(api_token_http_request_method, - api_token_url, - api_token_http_request.get( - ConfigReader.API_HTTP_REQUEST_HEADERS_CONFIG_KEY), - api_token_http_request.get(ConfigReader.API_HTTP_REQUEST_BODY_CONFIG_KEY)) - data_http_request = ApiHttpRequest(api_data_http_request_method, - api_data_url, - api_data_http_request.get(ConfigReader.API_HTTP_REQUEST_HEADERS_CONFIG_KEY), - api_data_http_request.get(ConfigReader.API_HTTP_REQUEST_BODY_CONFIG_KEY), - api_data_http_request.get(ConfigReader.API_HTTP_REQUEST_PAGE_SIZE)) - - return token_http_request, data_http_request diff --git a/src/data/api_http_request.py b/src/data/api_http_request.py deleted file mode 100644 index 9a6beec..0000000 --- a/src/data/api_http_request.py +++ /dev/null @@ -1,41 +0,0 @@ -class ApiHttpRequest: - GET_METHOD = 'GET' - POST_METHOD = 'POST' - HTTP_METHODS = [GET_METHOD, POST_METHOD] - - def __init__(self, api_http_request_method: str, api_url: str, - api_http_request_headers: dict = None, - api_http_request_body: str = None, page_size: int = None) -> None: - self._method = api_http_request_method - self._url = api_url - self._headers = api_http_request_headers - self._body = api_http_request_body - self._page_size = page_size - - @property - def page_size(self) -> int: - return self._page_size - - @page_size.setter - def page_size(self, page_size) -> None: - self._page_size = page_size - - @property - def method(self) -> str: - return self._method - - @property - def url(self) -> str: - return self._url - - @url.setter - def url(self, url) -> None: - self._url = url - - @property - def headers(self) -> dict: - return self._headers - - @property - def body(self) -> None: - return self._body diff --git a/src/data/auth_api_data.py b/src/data/auth_api_data.py deleted file mode 100644 index b6e5ac8..0000000 --- a/src/data/auth_api_data.py +++ /dev/null @@ -1,18 +0,0 @@ -from .base_data.auth_api_base_data import AuthApiBaseData -from .general_type_data.auth_api_general_type_data import AuthApiGeneralTypeData - - -class AuthApiData: - - def __init__(self, auth_api_base_data: AuthApiBaseData, - auth_api_general_type_data: AuthApiGeneralTypeData = None) -> None: - self._base_data = auth_api_base_data - self._general_type_data = auth_api_general_type_data - - @property - def base_data(self) -> AuthApiBaseData: - return self._base_data - - @property - def general_type_data(self) -> AuthApiGeneralTypeData: - return self._general_type_data diff --git a/src/data/base_data/api_base_data.py b/src/data/base_data/api_base_data.py deleted file mode 100644 index d7c926e..0000000 --- a/src/data/base_data/api_base_data.py +++ /dev/null @@ -1,65 +0,0 @@ -from typing import Generator -from .api_credentials import ApiCredentials -from .api_settings import ApiSettings -from .api_filter import ApiFilter -from .api_custom_field import ApiCustomField - - -class ApiBaseData: - - def __init__(self, api_type: str, api_name: str, api_credentials: ApiCredentials, api_settings: ApiSettings, - api_filters: list[ApiFilter], - api_custom_fields: list[ApiCustomField], date_filter: str = None) -> None: - self._type = api_type - self._name = api_name - self._credentials = api_credentials - self._settings = api_settings - self._filters = api_filters - self._date_filter_value = date_filter - self._custom_fields = api_custom_fields - - @property - def type(self) -> str: - return self._type - - @property - def name(self) -> str: - return self._name - - @property - def credentials(self) -> ApiCredentials: - return self._credentials - - @property - def settings(self) -> ApiSettings: - return self._settings - - @property - def filters(self) -> Generator[ApiFilter, None, None]: - for filter in self._filters: - yield filter - - @property - def custom_fields(self) -> Generator[ApiCustomField, None, None]: - for custom_field in self._custom_fields: - yield custom_field - - def get_filter_index(self, key: str) -> int: - index = 0 - - for filter in self._filters: - if filter.key == key: - return index - - index += 1 - - return -1 - - def get_filters_size(self) -> int: - return len(self._filters) - - def update_filter_value(self, index: int, value: str) -> None: - self._filters[index].value = value - - def append_filters(self, filter: ApiFilter) -> None: - self._filters.append(filter) diff --git a/src/data/base_data/api_credentials.py b/src/data/base_data/api_credentials.py deleted file mode 100644 index dc9e03d..0000000 --- a/src/data/base_data/api_credentials.py +++ /dev/null @@ -1,13 +0,0 @@ -class ApiCredentials: - - def __init__(self, api_credentials_id: str, api_credentials_key: str): - self._id = api_credentials_id - self._key = api_credentials_key - - @property - def id(self) -> str: - return self._id - - @property - def key(self) -> str: - return self._key diff --git a/src/data/base_data/api_custom_field.py b/src/data/base_data/api_custom_field.py deleted file mode 100644 index 33d9ba5..0000000 --- a/src/data/base_data/api_custom_field.py +++ /dev/null @@ -1,13 +0,0 @@ -class ApiCustomField: - - def __init__(self, field_key: str, field_value: str) -> None: - self._key = field_key - self._value = field_value - - @property - def key(self) -> str: - return self._key - - @property - def value(self) -> str: - return self._value diff --git a/src/data/base_data/api_filter.py b/src/data/base_data/api_filter.py deleted file mode 100644 index e72e9b6..0000000 --- a/src/data/base_data/api_filter.py +++ /dev/null @@ -1,17 +0,0 @@ -class ApiFilter: - - def __init__(self, api_filter_key: str, api_filter_value: str): - self._key = api_filter_key - self._value = api_filter_value - - @property - def key(self) -> str: - return self._key - - @property - def value(self) -> str: - return self._value - - @value.setter - def value(self, value: str) -> None: - self._value = value diff --git a/src/data/base_data/api_settings.py b/src/data/base_data/api_settings.py deleted file mode 100644 index 4dccfb7..0000000 --- a/src/data/base_data/api_settings.py +++ /dev/null @@ -1,13 +0,0 @@ -class ApiSettings: - - def __init__(self, time_interval: int, days_back_to_fetch: int = 14) -> None: - self._time_interval = time_interval - self._days_back_to_fetch = days_back_to_fetch - - @property - def time_interval(self) -> int: - return self._time_interval - - @property - def days_back_to_fetch(self) -> int: - return self._days_back_to_fetch diff --git a/src/data/base_data/auth_api_base_data.py b/src/data/base_data/auth_api_base_data.py deleted file mode 100644 index 525191d..0000000 --- a/src/data/base_data/auth_api_base_data.py +++ /dev/null @@ -1,11 +0,0 @@ -from .api_base_data import ApiBaseData - - -class AuthApiBaseData: - - def __init__(self, api_base_data: ApiBaseData) -> None: - self._base_data = api_base_data - - @property - def base_data(self) -> ApiBaseData: - return self._base_data diff --git a/src/data/base_data/oauth_api_base_data.py b/src/data/base_data/oauth_api_base_data.py deleted file mode 100644 index ed354be..0000000 --- a/src/data/base_data/oauth_api_base_data.py +++ /dev/null @@ -1,23 +0,0 @@ -from .api_base_data import ApiBaseData -from ..api_http_request import ApiHttpRequest - - -class OAuthApiBaseData: - - def __init__(self, api_base_data: ApiBaseData, api_token_http_request: ApiHttpRequest, - api_data_http_request: ApiHttpRequest) -> None: - self._base_data = api_base_data - self._token_http_request = api_token_http_request - self._data_http_request = api_data_http_request - - @property - def base_data(self) -> ApiBaseData: - return self._base_data - - @property - def token_http_request(self) -> ApiHttpRequest: - return self._token_http_request - - @property - def data_http_request(self) -> ApiHttpRequest: - return self._data_http_request diff --git a/src/data/general_type_data/api_general_type_data.py b/src/data/general_type_data/api_general_type_data.py deleted file mode 100644 index 17eec69..0000000 --- a/src/data/general_type_data/api_general_type_data.py +++ /dev/null @@ -1,22 +0,0 @@ -from .api_json_paths import ApiJsonPaths - - -class ApiGeneralTypeData: - - def __init__(self, api_start_date_name: str, api_end_date_name: str, - api_json_paths: ApiJsonPaths) -> None: - self._start_date_name = api_start_date_name - self._end_date_name = api_end_date_name - self._json_paths = api_json_paths - - @property - def end_date_name(self) -> str: - return self._end_date_name - - @property - def start_date_name(self) -> str: - return self._start_date_name - - @property - def json_paths(self) -> ApiJsonPaths: - return self._json_paths diff --git a/src/data/general_type_data/api_json_paths.py b/src/data/general_type_data/api_json_paths.py deleted file mode 100644 index 37230c1..0000000 --- a/src/data/general_type_data/api_json_paths.py +++ /dev/null @@ -1,26 +0,0 @@ -class ApiJsonPaths: - - def __init__(self, api_next_url_json_path: str, api_data_json_path: str, api_data_date_json_path: str) -> None: - self._next_url = api_next_url_json_path - self._data = api_data_json_path - self._data_date = api_data_date_json_path - - @property - def next_url(self) -> str: - return self._next_url - - @property - def data(self) -> str: - return self._data - - @property - def data_date(self) -> str: - return self._data_date - - @next_url.setter - def next_url(self, next_url) -> None: - self._next_url = next_url - - @data.setter - def data(self, data_link) -> None: - self._data = data_link diff --git a/src/data/general_type_data/auth_api_general_type_data.py b/src/data/general_type_data/auth_api_general_type_data.py deleted file mode 100644 index 334bf67..0000000 --- a/src/data/general_type_data/auth_api_general_type_data.py +++ /dev/null @@ -1,17 +0,0 @@ -from .api_general_type_data import ApiGeneralTypeData -from ..api_http_request import ApiHttpRequest - - -class AuthApiGeneralTypeData: - - def __init__(self, api_general_type_data: ApiGeneralTypeData, api_http_request: ApiHttpRequest) -> None: - self._general_type_data = api_general_type_data - self._http_request = api_http_request - - @property - def general_type_data(self) -> ApiGeneralTypeData: - return self._general_type_data - - @property - def http_request(self) -> ApiHttpRequest: - return self._http_request diff --git a/src/data/general_type_data/oauth_api_general_type_data.py b/src/data/general_type_data/oauth_api_general_type_data.py deleted file mode 100644 index e024095..0000000 --- a/src/data/general_type_data/oauth_api_general_type_data.py +++ /dev/null @@ -1,11 +0,0 @@ -from .api_general_type_data import ApiGeneralTypeData - - -class OAuthApiGeneralTypeData: - - def __init__(self, api_general_type_data: ApiGeneralTypeData) -> None: - self._general_type_data = api_general_type_data - - @property - def general_type_data(self) -> ApiGeneralTypeData: - return self._general_type_data diff --git a/src/data/logzio_connection.py b/src/data/logzio_connection.py deleted file mode 100644 index 548d72f..0000000 --- a/src/data/logzio_connection.py +++ /dev/null @@ -1,13 +0,0 @@ -class LogzioConnection: - - def __init__(self, logzio_url: str, logzio_token: str) -> None: - self._url = logzio_url - self._token = logzio_token - - @property - def url(self) -> str: - return self._url - - @property - def token(self) -> str: - return self._token diff --git a/src/data/oauth_api_data.py b/src/data/oauth_api_data.py deleted file mode 100644 index f352002..0000000 --- a/src/data/oauth_api_data.py +++ /dev/null @@ -1,18 +0,0 @@ -from .base_data.oauth_api_base_data import OAuthApiBaseData -from .general_type_data.oauth_api_general_type_data import OAuthApiGeneralTypeData - - -class OAuthApiData: - - def __init__(self, oauth_api_base_data: OAuthApiBaseData, - oauth_api_general_type_data: OAuthApiGeneralTypeData = None) -> None: - self._base_data = oauth_api_base_data - self._general_type_data = oauth_api_general_type_data - - @property - def base_data(self) -> OAuthApiBaseData: - return self._base_data - - @property - def general_type_data(self) -> OAuthApiGeneralTypeData: - return self._general_type_data diff --git a/src/general_auth_api.py b/src/general_auth_api.py deleted file mode 100644 index 36ef533..0000000 --- a/src/general_auth_api.py +++ /dev/null @@ -1,14 +0,0 @@ -import logging - -from .auth_api import AuthApi -from .data.base_data.auth_api_base_data import AuthApiBaseData -from .data.general_type_data.auth_api_general_type_data import AuthApiGeneralTypeData - - -logger = logging.getLogger(__name__) - - -class GeneralAuthApi(AuthApi): - - def __init__(self, api_base_data: AuthApiBaseData, api_general_type_data: AuthApiGeneralTypeData) -> None: - super().__init__(api_base_data, api_general_type_data) diff --git a/src/logging_config.ini b/src/logging_config.ini deleted file mode 100644 index e9954fb..0000000 --- a/src/logging_config.ini +++ /dev/null @@ -1,21 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=stream_handler - -[formatters] -keys=formatter - -[logger_root] -level=INFO -handlers=stream_handler - -[handler_stream_handler] -class=StreamHandler -level=INFO -formatter=formatter -args=(sys.stderr,) - -[formatter_formatter] -format=%(asctime)s %(name)-12s %(levelname)-8s %(message)s \ No newline at end of file diff --git a/src/logzio_shipper.py b/src/logzio_shipper.py deleted file mode 100644 index 22a5522..0000000 --- a/src/logzio_shipper.py +++ /dev/null @@ -1,152 +0,0 @@ -import logging -import requests -import gzip -import json - -from requests.adapters import HTTPAdapter, RetryError -from requests.sessions import InvalidSchema, Session -from urllib3.util.retry import Retry -from .data.base_data.api_custom_field import ApiCustomField - - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -VERSION = "1.0.1" - - -class LogzioShipper: - MAX_BODY_SIZE_BYTES = 10 * 1024 * 1024 # 10 MB - MAX_BULK_SIZE_BYTES = MAX_BODY_SIZE_BYTES / 10 # 1 MB - MAX_LOG_SIZE_BYTES = 500 * 1000 # 500 KB - - MAX_RETRIES = 3 - BACKOFF_FACTOR = 1 - STATUS_FORCELIST = [500, 502, 503, 504] - CONNECTION_TIMEOUT_SECONDS = 5 - - def __init__(self, logzio_url: str, token: str) -> None: - self._logzio_url = "{0}/?token={1}".format(logzio_url, token) - self._logs = [] - self._bulk_size = 0 - self._custom_fields = {'type': 'api_fetcher'} - - def add_log_to_send(self, log: str) -> None: - enriched_log = self._add_custom_fields_to_log(log) - enriched_log_size = len(enriched_log) - - if not self._is_log_valid_to_be_sent(enriched_log, enriched_log_size): - return - - if not self._bulk_size + enriched_log_size > LogzioShipper.MAX_BULK_SIZE_BYTES: - self._logs.append(enriched_log) - self._bulk_size += enriched_log_size - return - - try: - self.send_to_logzio() - except Exception: - raise - - self._logs.append(enriched_log) - self._bulk_size = enriched_log_size - - def send_to_logzio(self) -> None: - if self._logs is None: - return - - try: - headers = {"Content-Type": "application/json", - "Content-Encoding": "gzip", - "Logzio-Shipper": "logzio-azure-blob-trigger/v{0}/0/0.".format(VERSION)} - compressed_data = gzip.compress(str.encode('\n'.join(self._logs))) - response = self._get_request_retry_session().post(url=self._logzio_url, - data=compressed_data, - headers=headers, - timeout=LogzioShipper.CONNECTION_TIMEOUT_SECONDS) - response.raise_for_status() - logger.info("Successfully sent bulk of {} bytes to Logz.io.".format(self._bulk_size)) - self._reset_logs() - except requests.ConnectionError as e: - logger.error( - "Can't establish connection to {0} url. Please make sure your url is a Logz.io valid url. Max retries of {1} has reached. response: {2}".format( - self._logzio_url, LogzioShipper.MAX_RETRIES, e)) - raise - except RetryError as e: - logger.error( - "Something went wrong. Max retries of {0} has reached. response: {1}".format(LogzioShipper.MAX_RETRIES, - e)) - raise - except requests.exceptions.InvalidURL: - logger.error("Invalid url. Make sure your url is a valid url.") - raise - except InvalidSchema: - logger.error( - "No connection adapters were found for {}. Make sure your url starts with http:// or https://".format( - self._logzio_url)) - raise - except requests.HTTPError as e: - status_code = e.response.status_code - - if status_code == 400: - logger.error("The logs are bad formatted. response: {}".format(e)) - raise - - if status_code == 401: - logger.error("The token is missing or not valid. Make sure you’re using the right account token.") - raise - - logger.error("Somthing went wrong. response: {}".format(e)) - raise - except Exception as e: - logger.error("Something went wrong. response: {}".format(e)) - raise - - def add_custom_field_to_list(self, custom_field: ApiCustomField) -> None: - self._custom_fields[custom_field.key] = custom_field.value - - def _is_log_valid_to_be_sent(self, log: str, log_size: int) -> bool: - if log_size > LogzioShipper.MAX_LOG_SIZE_BYTES: - logger.error( - "The following log's size is greater than the max log size - {0} bytes, that can be sent to Logz.io: {1}".format( - LogzioShipper.MAX_LOG_SIZE_BYTES, log)) - - return False - - return True - - def _add_custom_fields_to_log(self, log: str) -> str: - json_log = json.loads(log) - - for key, value in self._custom_fields.items(): - json_log[key] = value - - return json.dumps(json_log) - - def _get_request_retry_session( - self, - retries=MAX_RETRIES, - backoff_factor=BACKOFF_FACTOR, - status_forcelist=STATUS_FORCELIST - ) -> Session: - session = requests.Session() - retry = Retry( - total=retries, - read=retries, - connect=retries, - status=retries, - backoff_factor=backoff_factor, - allowed_methods=frozenset(['GET', 'POST']), - status_forcelist=status_forcelist, - ) - adapter = HTTPAdapter(max_retries=retry) - - session.mount('http://', adapter) - session.mount('https://', adapter) - session.headers.update({"Content-Type": "application/json"}) - - return session - - def _reset_logs(self) -> None: - self._logs.clear() - self._bulk_size = 0 diff --git a/src/main.py b/src/main.py index b2dbc03..62be18a 100644 --- a/src/main.py +++ b/src/main.py @@ -1,10 +1,62 @@ +import argparse +import configparser +import logging from logging.config import fileConfig -from src.apis_manager import ApisManager +from src.config.ConfigReader import ConfigReader +from src.manager.TaskManager import TaskManager -def main() -> None: - fileConfig('src/logging_config.ini', disable_existing_loggers=False) - ApisManager().run() +LOGGER_CONFIG_PATH = "./src/utils/logging_config.ini" +FETCHER_CONFIG_PATH = "./src/shared/config.yaml" + + +def __get_args(): + """ + Gets the level arguments from the run command. + :return: arguments + """ + parser = argparse.ArgumentParser(description='Logzio API Fetcher') + parser.add_argument('--level', type=str, required=False, default='INFO', choices=['INFO', 'WARN', 'ERROR', 'DEBUG'], + help='Logging level (One of INFO, WARN, ERROR, DEBUG)') + return parser.parse_args() + + +def _setup_logger(level, test): + """ + Configures the logger and it's logging level. + :param level: the logger logging level + """ + # If test, always set to debug + if test: + level = "DEBUG" + + # Update the level in the config file + if level != "INFO": + logging_conf = configparser.RawConfigParser() + logging_conf.read(LOGGER_CONFIG_PATH) + + logging_conf['logger_root']['level'] = level + logging_conf['handler_stream_handler']['level'] = level + + with open(LOGGER_CONFIG_PATH, 'w') as configfile: + logging_conf.write(configfile) + + # Load the config to the logger + fileConfig(LOGGER_CONFIG_PATH, disable_existing_loggers=False) + logging.info(f'Starting Logzio API fetcher in {level} level.') + + +def main(conf_path=FETCHER_CONFIG_PATH, test=False): + """ + Get args >> Configure logger >> Read config file >> Start API fetching and shipping task based on config + """ + args = __get_args() + _setup_logger(args.level, test) + + conf = ConfigReader(conf_path) + + if conf.api_instances: + TaskManager(apis=conf.api_instances, logzio_shipper=conf.logzio_shipper).run() if __name__ == '__main__': diff --git a/src/manager/TaskManager.py b/src/manager/TaskManager.py new file mode 100644 index 0000000..7dbe206 --- /dev/null +++ b/src/manager/TaskManager.py @@ -0,0 +1,108 @@ +import logging +import os +import requests +from requests.sessions import InvalidSchema +import signal +import threading + + +logger = logging.getLogger(__name__) + + +class TaskManager: + """ + Class to run scheduled task that collects data from given APIs and sends them with the given logzio_shipper. + :param apis: List of ApiFetcher instances to fetch data from + :param logzio_shipper: LogzioShipper instance to send data to + """ + def __init__(self, apis=[], logzio_shipper=None): + self.apis = apis + self.logzio_shipper = logzio_shipper + self.threads = [] + self.event = threading.Event() + + @staticmethod + def _terminate_process(): + os.kill(os.getpid(), signal.SIGTERM) + + def _run_api_task(self, api): + """ + Collects data from the API and sends it to Logzio. + :param api: The API class instance + """ + logger.info(f"Starting task for api {api.name}.") + data_exists = False + + try: + for log in api.send_request(): + data_exists = True + self.logzio_shipper.add_log_to_send(log, api.additional_fields) + if data_exists: + self.logzio_shipper.send_to_logzio() + + except requests.exceptions.InvalidURL as e: + logger.error(f"Failed to send data to Logz.io... Invalid url: {e}") + self._terminate_process() + return + except InvalidSchema as e: + logger.error(f"Failed to send data to Logz.io... Invalid schema: {e}") + self._terminate_process() + return + except requests.HTTPError as e: + logger.error(f"Failed to send data to Logz.io... HTTP error: {e}") + if e.response.status_code == 401: + self._terminate_process() + return + except Exception as e: + logger.error(f"Failed to send data to Logz.io... exception: {e}") + logger.info(f"Task finished for api {api.name}. New task will run in {api.scrape_interval_minutes} minutes.") + + def _run_api_scheduled_task(self, api): + """ + Runs scheduled task, based on the API scrape interval, to collect and send data with _send_data_to_logzio + function. + :param api: The API class instance + """ + while True: + logger.debug(f"Starting thread to collect logs from {api.name}") + thread = threading.Thread(target=self._run_api_task, args=(api,)) + thread.start() + + # Enforce new task to run every scrape_interval + if self.event.wait(timeout=api.scrape_interval_minutes * 60): + break + + def __exit_gracefully(self, signum, frame): + """ + Clear up all threads before closing program + :param signum: the number of signal that called the function (required for 'signal.signal' usage) + :param frame: the frame number (required for 'signal.signal' usage) + """ + logger.info("Signal caught... Stopping") + self.event.set() + + for thread in self.threads: + thread.join() + + def run(self): + """ + Creates thread per API fetcher that runs a scheduled collection task based on the scrape interval. + """ + if not self.apis: + return + + for api in self.apis: + thread = threading.Thread(target=self._run_api_scheduled_task, args=(api,)) + self.threads.append(thread) + + logger.debug(f"Configured {len(self.threads)} API inputs.") + + for thread in self.threads: + thread.start() + + for thread in self.threads: + thread.join() + + signal.signal(signal.SIGINT, self.__exit_gracefully) + signal.signal(signal.SIGTERM, self.__exit_gracefully) + signal.sigwait([signal.SIGINT, signal.SIGTERM]) diff --git a/src/manager/__init__.py b/src/manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/oauth_api.py b/src/oauth_api.py deleted file mode 100644 index 8e2ea83..0000000 --- a/src/oauth_api.py +++ /dev/null @@ -1,134 +0,0 @@ -import json -import logging -import time -import requests - -from datetime import datetime, timedelta -from typing import Generator, Optional -from dateutil import parser -from requests import Response -from .api import Api -from .data.api_http_request import ApiHttpRequest -from .data.base_data.oauth_api_base_data import OAuthApiBaseData -from .data.general_type_data.oauth_api_general_type_data import OAuthApiGeneralTypeData - -logger = logging.getLogger(__name__) - - -class OAuthApi(Api): - OAUTH_GRANT_TYPE = 'grant_type' - OAUTH_CLIENT_CREDENTIALS = 'client_credentials' - OAUTH_CLIENT_ID = 'client_id' - OAUTH_CLIENT_SECRET = 'client_secret' - OAUTH_ACCESS_TOKEN = 'access_token' - OAUTH_TOKEN_EXPIRE = 'expires_in' - OAUTH_AUTHORIZATION_HEADER = 'Authorization' - OAUTH_TOKEN_REQUEST_CONTENT_TYPE = 'content-type' - OAUTH_APPLICATION_JSON_CONTENT_TYPE = 'application/json' - OAUTH_TOKEN_BEARER_PREFIX = 'Bearer ' - - def __init__(self, oauth_config: OAuthApiBaseData, general_config: OAuthApiGeneralTypeData): - self.token = None - self.token_expire = 0 - self._token_request = oauth_config.token_http_request - self._data_request = oauth_config.data_http_request - super().__init__(oauth_config.base_data, general_config.general_type_data) - - def get_token(self) -> [str, int]: - token_response = requests.post(self._token_request.url, - self._token_request.body) - return json.loads(token_response.content)[self.OAUTH_ACCESS_TOKEN], json.loads(token_response.content)[ - self.OAUTH_TOKEN_EXPIRE] - - def fetch_data(self) -> Generator: - if time.time() > (self.token_expire - 60): - self.token, token_expire = self.get_token() - self.token_expire = time.time() + int(token_expire) - return self._get_total_data_from_api() - - def _get_total_data_from_api(self) -> Generator: - total_data_num = 0 - api_url = self._build_api_url() - is_first_fetch = True - is_last_data_to_fetch = False - first_item_date: Optional[str] = None - last_datetime_to_fetch: Optional[datetime] = None - - while True: - try: - next_url, data = self._get_data_from_api(api_url) - except Exception: - raise - - if not data: - logger.info("No new data available from api {}.".format(self._base_data.name)) - self._set_current_data_last_date(first_item_date) - return data - - if is_first_fetch: - first_item_date = self._get_last_date(data[0]) - last_datetime_to_fetch = parser.parse(first_item_date) - timedelta( - days=self._base_data.settings.days_back_to_fetch) - is_first_fetch = False - - for item in data: - if not self._is_item_in_fetch_frame(item, last_datetime_to_fetch): - is_last_data_to_fetch = True - break - - yield json.dumps(item) - total_data_num += 1 - - if next_url is None or is_last_data_to_fetch: - break - - api_url = next_url - - logger.info("Got {0} total data from api {1}".format(total_data_num, self._base_data.name)) - - self._set_current_data_last_date(first_item_date) - - def _build_api_url(self) -> str: - api_url = self._data_request.url - api_filters_num = self._base_data.get_filters_size() - - if api_filters_num > 0: - api_url += '?' - - for api_filter in self._base_data.filters: - api_url += api_filter.key + '=' + str(api_filter.value) - api_filters_num -= 1 - - if api_filters_num > 0: - api_url += '&' - - return api_url - - def _send_request(self, url: str) -> Response: - request_headers = {self.OAUTH_AUTHORIZATION_HEADER: f"Bearer {self.token}", - self.OAUTH_TOKEN_REQUEST_CONTENT_TYPE: self.OAUTH_APPLICATION_JSON_CONTENT_TYPE} - if self._data_request.headers is not None: - request_headers.update(self._data_request.headers) - if self._data_request.method == ApiHttpRequest.GET_METHOD: - response = requests.get(url=url, - headers=request_headers, - data=self._data_request.body) - else: - response = requests.post(url=url, - headers=request_headers, - data=json.dumps(self._data_request.body)) - - return response - - @property - def get_data_request(self): - return self._data_request - - @property - def get_token_request(self): - return self._token_request - - def _set_current_data_last_date(self, date): - if date: - self._current_data_last_date = date - diff --git a/src/output/LogzioShipper.py b/src/output/LogzioShipper.py new file mode 100644 index 0000000..4990c59 --- /dev/null +++ b/src/output/LogzioShipper.py @@ -0,0 +1,201 @@ +import gzip +import json +import logging +from pydantic import BaseModel, Field +import requests +from requests.adapters import HTTPAdapter, RetryError +from requests.sessions import InvalidSchema +from urllib3.util.retry import Retry + +# Current integration version +INT_VERSION = "0.2.0" + +# Size limitations +MAX_BODY_SIZE_BYTES = 10 * 1024 * 1024 # 10 MB +MAX_BULK_SIZE_BYTES = MAX_BODY_SIZE_BYTES / 10 # 1 MB +MAX_LOG_SIZE_BYTES = 500 * 1000 # 500 KB + +# Retry Settings +MAX_RETRIES = 3 +BACKOFF_FACTOR = 1 +STATUS_FORCELIST = [500, 502, 503, 504] +CONNECTION_TIMEOUT_SECONDS = 5 + +logger = logging.getLogger(__name__) + + +class LogzioShipper(BaseModel): + """ + Class to send data to logzio + :param listener: The listener endpoint to send the logs to (Default: https://listener.logz.io:8071) + :param token: Required, the logzio shipping token + :param curr_logs: Not passed to the class, array of the logs that were yet to sent + :param curr_bulk_size: Not passed to the class, size of the current logs bulk (of data in 'self.curr_logs') + """ + listener: str = Field(default="https://listener.logz.io:8071", alias="url") + token: str = Field(frozen=True) + curr_logs: list = Field(default=[], init=False, init_var=True) + curr_bulk_size: int = Field(default=0, init=False, init_var=True) + + def __init__(self, **data): + super().__init__(**data) + self.listener = f"{self.listener}/?token={self.token}" + + @staticmethod + def _add_custom_fields_to_log(log, custom_fields): + """ + Makes sure the given log is in JSON format (if not, makes it) and adds the given custom fields to it. + :param log: the log + :param custom_fields: the fields to add to it + :return: the log in json format with the custom fields added to it + """ + + try: + json_log = json.loads(log) + except json.decoder.JSONDecodeError: + json_log = {"message": log} + except TypeError: + json_log = log + if custom_fields: + json_log.update(custom_fields) + return json.dumps(json_log) + + @staticmethod + def _is_valid_log(log_to_send, log_size): + """ + Validates that the given log size does not pass MAX_LOG_SIZE_BYTES. + :param log_to_send: the actual log + :param log_size: the log size + :return: True if log_size < MAX_LOG_SIZE_BYTES, false otherwise + """ + if log_size > MAX_LOG_SIZE_BYTES: + logger.error(f"The following log size of {log_size} bytes is passing the allowed " + f"{MAX_LOG_SIZE_BYTES} bytes logzio limit. Not sending the log: '{log_to_send}'") + return False + return True + + @staticmethod + def _get_request_retry_session(retries=MAX_RETRIES, backoff_factor=BACKOFF_FACTOR, + status_forcelist=STATUS_FORCELIST): + """ + Creates a retry session for the shipping request. + :param retries: amount of retries + :param backoff_factor: exponential backoff factor between attempts + :param status_forcelist: HTTP status codes to force retry on + :return: session object + """ + session = requests.Session() + retry = Retry( + total=retries, + read=retries, + connect=retries, + status=retries, + backoff_factor=backoff_factor, + allowed_methods=frozenset(['GET', 'POST']), + status_forcelist=status_forcelist, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + session.headers.update({"Content-Type": "application/json"}) + + return session + + def _reset_logs(self): + """ + Removes logs from the current queue and resets the bulk size. + """ + self.curr_logs.clear() + self.curr_bulk_size = 0 + + @staticmethod + def _handle_exception(exp, msg, *args): + """ + Logs the given error message and raises the exception. + :param exp: the exception + :param msg: error message to log regarding it + :param args: message arguments + """ + logger.error(msg.format(*args)) + raise exp + + def _handle_http_errors(self, status_code, exp): + """ + Handles HTTP error codes from an exception. + :param status_code: the HTTP status code + :param exp: the exception itself + """ + if status_code == 400: + self._handle_exception(exp, "The logs are bad formatted. Response: {}", exp) + elif status_code == 401: + logger.error("Logzio Shipping Token is missing or invalid. Make sure you’re using the right account " + "token.") + self._handle_exception(exp, "Logzio Shipping Token is missing or invalid. Make sure you’re using the right " + "account token.") + else: + self._handle_exception(exp, "Somthing went wrong. Response: {}", exp) + + def send_to_logzio(self): + """ + Sends logs from 'self.curr_logs' to logzio with retry mechanism. + """ + if not self.curr_logs: + return + + if self.curr_bulk_size == 0: + logger.info("bulk is 0 but logs are:", self.curr_logs) + + try: + headers = {"Content-Type": "application/json", + "Content-Encoding": "gzip", + "Logzio-Shipper": f"logzio-api-fetcher/{INT_VERSION}"} + compressed_data = gzip.compress(str.encode('\n'.join(self.curr_logs))) + response = self._get_request_retry_session().post(url=self.listener, + data=compressed_data, + headers=headers, + timeout=CONNECTION_TIMEOUT_SECONDS) + response.raise_for_status() + logger.info(f"Successfully sent bulk of {self.curr_bulk_size} bytes to Logz.io.") + self._reset_logs() + + except requests.ConnectionError as e: + self._handle_exception(e, "Failed to establish connection to the listener, max retries {} reached. " + "Please make sure '{}' is valid. Response: {}", MAX_RETRIES, self.listener, e) + except RetryError as e: + self._handle_exception(e, "Something went wrong, max retries {} reached. Response: {}", MAX_RETRIES, e) + except requests.exceptions.InvalidURL: + self._handle_exception(requests.exceptions.InvalidURL, "Invalid url. Make sure your url is a valid url.") + except InvalidSchema: + self._handle_exception(InvalidSchema, "No connection adapters were found for {}. Make sure your url " + "starts with http:// or https://") + except requests.HTTPError as e: + self._handle_http_errors(e.response.status_code, e) + except Exception as e: + self._handle_exception(e, "Something went wrong. response: {}", e) + + def add_log_to_send(self, log, custom_fields=None): + """ + Receives log to send, adds the given additional fields to it, validates it and adds it to a bulk. + If the bulk reaches the MAX_BULK_SIZE_BYTES >> send data. Otherwise, add the logs to the bulk. + :param log: log to add to the bulk + :param custom_fields: custom fields to add to the log + """ + enriched_log = self._add_custom_fields_to_log(log, custom_fields) + + if not self._is_valid_log(enriched_log, len(enriched_log)): + return + + # Bulk size was not reached yet + if not self.curr_bulk_size + len(enriched_log) > MAX_BULK_SIZE_BYTES: + self.curr_logs.append(enriched_log) + self.curr_bulk_size += len(enriched_log) + return + + # Bulk size was reached >> send current logs and append the new logs to the new bulk + try: + self.send_to_logzio() + except Exception: + raise + + self.curr_logs.append(enriched_log) + self.curr_bulk_size = len(enriched_log) diff --git a/src/output/__init__.py b/src/output/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utils/MaskInfoFormatter.py b/src/utils/MaskInfoFormatter.py new file mode 100644 index 0000000..7e85385 --- /dev/null +++ b/src/utils/MaskInfoFormatter.py @@ -0,0 +1,16 @@ +import logging +import re + + +class MaskInfoFormatter(logging.Formatter): + """ + Formatter that masks sensitive information such as tokens from the program logs. + """ + @staticmethod + def _filter(org_log): + return re.sub(r'(token=|grant_type=|client_secret=|Bearer |Authorization[\"|\']: [\"|\']Basic |TOKEN[\"|\']: [\"|\'])[^&\n]{0,26}', + r'\g<1>******', org_log) + + def format(self, record): + original = super().format(record) + return self._filter(original) diff --git a/src/utils/__init__.py b/src/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utils/logging_config.ini b/src/utils/logging_config.ini new file mode 100644 index 0000000..5cb5e41 --- /dev/null +++ b/src/utils/logging_config.ini @@ -0,0 +1,22 @@ +[loggers] +keys = root + +[handlers] +keys = stream_handler + +[formatters] +keys = formatter + +[logger_root] +level = INFO +handlers = stream_handler + +[handler_stream_handler] +class = StreamHandler +level = INFO +formatter = formatter +args = (sys.stderr,) + +[formatter_formatter] +class=src.utils.MaskInfoFormatter.MaskInfoFormatter +format = %(asctime)s [%(levelname)s]: %(message)s diff --git a/src/utils/processing_functions.py b/src/utils/processing_functions.py new file mode 100644 index 0000000..47c530a --- /dev/null +++ b/src/utils/processing_functions.py @@ -0,0 +1,169 @@ +import json +import logging +import re + +# EXPECTED_ +VARS_PATTERN = re.compile(r"\{res\.(.*?)\}") +EXPECTED_ARRAY_PREFIX = "[" +EXPECTED_ARRAY_SUFFIX = "]" + +logger = logging.getLogger(__name__) + + +def extract_vars(item): + """ + Receives String item, extract variables names from it and returns array with all the variable names. + Variable is recognized based on VARS_PATTERN. + :param item: String value + :return: array with all the variable names. + """ + if not item: + return [] + if not isinstance(item, str): + item = json.dumps(item) + return re.findall(VARS_PATTERN, item) + + +def _support_math_operations(last_nested_key): + """ + If the given key contains + or - operation, split operation from the key name, and return: + - the key name without the operation + - the integer value that needs to be added to the final value. + :param last_nested_key: key name that may contain a mathematical operation. + :return: the key name and mathematical operation to perform on the final value. + """ + if "+" in last_nested_key: + last_nested_key_without_op, math_operation = last_nested_key.split("+") + return last_nested_key_without_op, int(math_operation) + elif "-" in last_nested_key: + last_nested_key_without_op, math_operation = last_nested_key.split("-") + return last_nested_key_without_op, -int(math_operation) + return last_nested_key, None + + +def _get_key_from_nested_array(next_item, key): + """ + Expects to get array next_item and a key index. + Extracts the index from the given next item and returns it. + If encounters an error, returns None. + :param next_item: an array + :param key: the index to get from the array + :return: the value of item in given index of the next given item. + """ + key = key[len(EXPECTED_ARRAY_PREFIX):-len(EXPECTED_ARRAY_SUFFIX)] + try: + next_item = next_item[int(key)] + except (IndexError, ValueError): + logger.warning(f"Failed to find the next key: '{key}' nested in {next_item}") + logger.warning("Failed to find the next key: '%s' nested in %s", key, next_item) + next_item = None + return next_item + + +def _get_key_from_nested(next_item, key): + """ + Expects to get a dictionary next_item and a key name. + Extracts the key value from the dictionary and returns it. + If encounters an error, returns None. + :param next_item: dictionary with values + :param key: key in the dictionary + :return: the key value or None if it doesn't exist in the dict. + """ + _SENTINEL = object() # To allow differentiating between case of field not exists VS field value is None + original = next_item # Keeping it for logging purpose + + try: + next_item = next_item.get(key, _SENTINEL) + + # The key does not exist in the dictionary + if next_item is _SENTINEL: + logger.warning(f"Key '{key}' does not exists in: {original}") + next_item = None + + except AttributeError: + # Check if nested value is in a flattened object and extract it + try: + next_item = _get_key_from_nested(json.loads(next_item), key) + except json.decoder.JSONDecodeError: + logger.debug(f"Failed to find '{key}' in response due to error.") + next_item = None + return next_item + + +def get_nested_value(values_dic, nested_keys): + """ + Receives an array of keys who are nested, in their nesting order. + Examples: obj = {'key1': {'key2': 123}} >> nested_keys = ['key1', 'key2'] + obj = {'key1': [{'key2': 123}]} >> nested_keys = ['key1', '[0]', 'key2'] + And returns the final value from the values_dic. + If doesn't exist, returns None + :param values_dic: Dictionary with keys (such as from nested_keys) and their values + :param nested_keys: Array of key names that are nested, in order of their nesting + :return: value of the given nested keys + """ + next_item = values_dic + last_item_index = len(nested_keys) - 1 + + # Support + and - math operations + last_nested_key = nested_keys[last_item_index] + nested_keys[last_item_index], math_operation = _support_math_operations(last_nested_key) + + for key in nested_keys: + # Support array keys + if key.startswith(EXPECTED_ARRAY_PREFIX) and key.endswith(EXPECTED_ARRAY_SUFFIX): + next_item = _get_key_from_nested_array(next_item, key) + + # Not an array key + else: + key = key.replace("~~", ".") + next_item = _get_key_from_nested(next_item, key) + + # We either got an exception, the key does not exist or next_item == None >> break from the loop + if not next_item: + break + + if next_item and math_operation: + next_item += math_operation + return next_item + + +def replace_dots(string): + """ + Replaces all the dots of a given string with ~~ + :param string: some String value + :return: the given string value with ~~ instead of dots. + """ + return string.replace("\\.", "~~") + + +def break_key_name(key): + """ + Receives a key name in format: + - 'key', 'key.nested', 'key.[0].nested' ... + and returns an array with the nested keys in order. + :param key: nested key name + :return: array of the nested fields in the key in order. + """ + return replace_dots(key).split(".") + + +def substitute_vars(item, vars_arr, values_dic): + """ + Receives String item and replaces the variables from vars_arr in it with their values from the values_dic. + :param item: String item with variables in it + :param vars_arr: array with the variables to replace in the item + :param values_dic: dictionary with the keys and values. + :return: the item with values instead of variables. + """ + new_item = item + for var in vars_arr: + # Support dots in key names and nested objects + var_fields = break_key_name(var) + + # Find the key in the response and put value in next_item + value = get_nested_value(values_dic, var_fields) + if value: + new_item = new_item.replace("{res.%s}" % var, str(value)) + else: + raise ValueError(f"The response didn't contain {var} hence it won't be replaced in {item}.") + return new_item diff --git a/tests/IntegrationTests/__init__.py b/tests/IntegrationTests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/IntegrationTests/test_data_arrived.py b/tests/IntegrationTests/test_data_arrived.py new file mode 100644 index 0000000..db80487 --- /dev/null +++ b/tests/IntegrationTests/test_data_arrived.py @@ -0,0 +1,61 @@ +import glob +import json +import os +from os.path import abspath, dirname +import requests +import unittest + + +def _search_data(query): + """ + Send given search query to logzio and returns the result. + :param query: + :return: + """ + # Search logs in account + url = "https://api.logz.io/v1/search" + headers = { + "X-API-TOKEN": os.environ["LOGZIO_API_TOKEN"], + "CONTENT-TYPE": "application/json", + "ACCEPT": "application/json" + } + body = { + "query": { + "query_string": { + "query": query + } + } + } + + r = requests.post(url=url, headers=headers, json=body) + if r: + data = json.loads(r.text) + hits = data.get("hits").get("hits") + return hits + return [] + + +def delete_temp_files(): + """ + delete the temp config that generated for the test + """ + curr_path = abspath(dirname(dirname(__file__))) + test_configs_path = f"{curr_path}/testConfigs/*_temp.yaml" + + for file in glob.glob(test_configs_path): + os.remove(file) + + +class TestDataArrived(unittest.TestCase): + """ + Test data arrived to logzio + """ + + @classmethod + def tearDownClass(cls): + # Clean up temp files that the test created + delete_temp_files() + + def test_data_in_logz(self): + azure_logs_in_acc = _search_data("type:azure_graph_shipping_test") + self.assertTrue(azure_logs_in_acc) # make sure we have results diff --git a/tests/IntegrationTests/test_shipping.py b/tests/IntegrationTests/test_shipping.py new file mode 100644 index 0000000..114111b --- /dev/null +++ b/tests/IntegrationTests/test_shipping.py @@ -0,0 +1,55 @@ +import os +from os.path import abspath, dirname +import threading +import yaml + +from src.main import main + + +curr_path = abspath(dirname(dirname(__file__))) +temp_int_conf_path = f"{abspath(dirname(dirname(dirname(__file__))))}/src/shared/config.yaml" +test_config_paths = [f"{curr_path}/testConfigs/azure_api_conf.yaml"] + + +def _update_config_tokens(file_path): + """ + Updates the tokens in the given file. + """ + with open(file_path, "r") as conf: + content = yaml.safe_load(conf) + + content["apis"][0]["azure_ad_tenant_id"] = os.environ["AZURE_AD_TENANT_ID"] + content["apis"][0]["azure_ad_client_id"] = os.environ["AZURE_AD_CLIENT_ID"] + content["apis"][0]["azure_ad_secret_value"] = os.environ["AZURE_AD_SECRET_VALUE"] + content["logzio"]["token"] = os.environ["LOGZIO_SHIPPING_TOKEN"] + + path, ext = file_path.split(".") + temp_test_path = f"{path}_temp.{ext}" + + with open(temp_test_path, "w") as file: + yaml.dump(content, file) + + return temp_test_path + + +def integration_test(): + """ + Runs the integration to send real data + """ + threads = [] + + # in case we want to add more configs to the integration test in the future; + for file in test_config_paths: + temp_file = _update_config_tokens(file) + thread = threading.Thread(target=main, args=(temp_file, True,)) + threads.append(thread) + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + +if __name__ == '__main__': + integration_test() diff --git a/tests/UnitTests/__init_.py b/tests/UnitTests/__init_.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/api_body/azure_graph_body.json b/tests/UnitTests/responsesExamples/azure_graph_res_example.json similarity index 100% rename from tests/api_body/azure_graph_body.json rename to tests/UnitTests/responsesExamples/azure_graph_res_example.json diff --git a/tests/UnitTests/responsesExamples/azure_mail_res_example.json b/tests/UnitTests/responsesExamples/azure_mail_res_example.json new file mode 100644 index 0000000..cb915c1 --- /dev/null +++ b/tests/UnitTests/responsesExamples/azure_mail_res_example.json @@ -0,0 +1,25 @@ +{ + "d": { + "results": [ + { + "__metadata": { + "id": "https://reports.office365.com/ecp/ReportingWebService/Reporting.svc/MessageTrace(1)", + "uri": "https://reports.office365.com/ecp/ReportingWebService/Reporting.svc/MessageTrace(1)", + "type": "TenantReporting.MessageTrace" + }, + "Organization": "examplecorp.onmicrosoft.com", + "MessageId": "<3a273efc-cd65-4335-96ec-5f6934f0fb10@az.uksouth.production.microsoft.com>", + "Received": "2024-05-28T13:08:54Z", + "SenderAddress": "azure-noreply@microsoft.com", + "RecipientAddress": "foo.bar@example.corp", + "Subject": "PIM: MessageTrace API service account has the Privileged Role Administrator role", + "Status": "GettingStatus", + "ToIP": null, + "FromIP": "1.1.1.1", + "StartDate": "2024-05-28T13:08:54Z", + "EndDate": "2024-05-30T13:08:54Z" + } + ], + "@odata.nextLink": "https://reports.office365.com/ecp/ReportingWebService/Reporting.svc/MessageTrace?$skiptoken=abc123" + } +} \ No newline at end of file diff --git a/tests/UnitTests/responsesExamples/cloudflare_res_example.json b/tests/UnitTests/responsesExamples/cloudflare_res_example.json new file mode 100644 index 0000000..8cec049 --- /dev/null +++ b/tests/UnitTests/responsesExamples/cloudflare_res_example.json @@ -0,0 +1,36 @@ +{ + "result": [ + { + "id": "some-id", + "name": "Spike in Security Events - Blocked Requests", + "description": "Sent within two hours of the spike in events being detected ", + "alert_body": "{\"field\": 123}", + "alert_type": "clickhouse_alert_fw_anomaly", + "mechanism": "some-id@upstream.opsgenie.net", + "mechanism_type": "email", + "policy_id": "policy-id", + "sent": "2024-05-24T03:22:45.410294Z" + }, + { + "id": "some-id-2", + "name": "Spike in Security Events - Allowed Requests", + "description": "Sent within two hours of the spike in events being detected", + "alert_body": "{\"field2\":\"value\"}", + "alert_type": "clickhouse_alert_fw_anomaly", + "mechanism": "some-id@upstream.opsgenie.net", + "mechanism_type": "email", + "policy_id": "policy-id", + "sent": "2024-05-21T10:07:23.522451Z" + } + ], + "success": true, + "errors": [], + "messages": [], + "result_info": { + "page": 1, + "per_page": 25, + "count": 25, + "total_count": 2, + "total_pages": 1 + } +} \ No newline at end of file diff --git a/tests/UnitTests/test_api_fetcher.py b/tests/UnitTests/test_api_fetcher.py new file mode 100644 index 0000000..e620cf8 --- /dev/null +++ b/tests/UnitTests/test_api_fetcher.py @@ -0,0 +1,167 @@ +from pydantic import ValidationError +import responses +import unittest + +from src.apis.general.Api import ApiFetcher, ReqMethod +from src.apis.general.PaginationSettings import PaginationSettings, PaginationType +from src.apis.general.StopPaginationSettings import StopPaginationSettings, StopCondition + + +class TestApiFetcher(unittest.TestCase): + """ + Test cases for the API Fetcher + """ + + def test_log_type(self): + a = ApiFetcher(url="https://some/url", additional_fields={"type": "custom-type"}) + self.assertEqual(a.additional_fields.get("type"), "custom-type") + + a = ApiFetcher(url="https://some/url") + self.assertEqual(a.additional_fields.get("type"), "api-fetcher") + + def test_invalid_setup(self): + with self.assertRaises(ValidationError): + # Missing Required 'url' field + ApiFetcher(headers={"hi": "test"}) + + # Not supported method + ApiFetcher(url="https://random", method="DELETE") + + # scrape_interval too small + ApiFetcher(url="https://my-url", scrape_interval=0.5) + + # Invalid stop pagination condition + ApiFetcher(url="https://my-url", + pagination=PaginationSettings(type="headers", + headers_format={"header": "{res.field}"}, + stop_indication=StopPaginationSettings(field="results", + condition="not-existing"))) + + def test_invalid_pagination_setup(self): + with self.assertRaises(ValueError): + # Missing a required field + ApiFetcher(url="https://my-url", pagination=PaginationSettings(type="url")) + + # Missing field for stop condition + ApiFetcher(url="https://my-url", + pagination=PaginationSettings(type="headers", + headers_format={"header": "{res.field}"}, + stop_indication=StopPaginationSettings(field="results", + condition="contains"))) + + def test_change_next_url(self): + # Init no variables in the URL + a = ApiFetcher(url="https://random") + self.assertEqual(a.url_vars, []) + + # Vars should update + a.update_next_url("https://random?p={res.some_field}") + self.assertEqual(a.url_vars, ["some_field"]) + + # Vars should update to empty + a.update_next_url("https://random?") + self.assertEqual(a.url_vars, []) + + def test_extract_data_from_path(self): + # field does not exist in response + a = ApiFetcher(url="https://random", response_data_path="not_exist") + self.assertEqual(a._extract_data_from_path({"exists": "field value"}), []) + + # field exist in response + a = ApiFetcher(url="https://random", response_data_path="exists") + self.assertEqual(a._extract_data_from_path({"exists": "field value"}), ["field value"]) + + # invalid response + a = ApiFetcher(url="https://random", response_data_path="field") + self.assertEqual(a._extract_data_from_path("not proper res"), []) + + @responses.activate + def test_send_successful_request(self): + success_res_body = {"field": "abc", "arr": [1, 2], "objArr": [{"f2": "hi"}, {"f2": "hello"}]} + + # Mock response from some API + responses.add(responses.GET, "http://some/api", json=success_res_body, status=200) + + a = ApiFetcher(url="http://some/api", + next_url="http://some/api/{res.field}/{res.arr.[0]}/{res.objArr.[1].f2}") + result = a.send_request() + + # Validate we got the needed response + self.assertEqual([success_res_body], result) + + # Validate that next_url updates the url for next request as expected + self.assertEqual("http://some/api/abc/1/hello", a.url) + + @responses.activate + def test_send_bad_request(self): + # Mock response from some API + responses.add(responses.GET, "http://not/existing/api", status=404) + + a = ApiFetcher(name="test", url="http://not/existing/api", + next_url="http://some/api/{res.field}/{res.arr[0]}/{res.objArr[1].f2}") + + # Validate we get the needed error and no data + with self.assertLogs("src.apis.general.Api", level='INFO') as log: + result = a.send_request() + self.assertIn("ERROR:src.apis.general.Api:Failed to get data from test API due to error 404 Client Error: Not Found for url: http://not/existing/api", log.output) + self.assertEqual(result, []) + + @responses.activate + def test_pagination_stop_at_max_calls(self): + first_req_body = {"query": "some query that filters the data"} + pagination_req_body = {"page": "2"} + + first_res_body = {"data": [{"message": "log1"}, {"message": "log2"}], "info": {"page": 1}} + pagination_res_body = {"data": [{"message": "log3"}], "info": {"page": 2}} + + # Mock response from some API that had pagination + responses.add(responses.POST, "https://some/api", + match=[responses.matchers.json_params_matcher(first_req_body)], + json=first_res_body, + status=200) + responses.add(responses.POST, "https://some/api", + match=[responses.matchers.json_params_matcher(pagination_req_body)], + json=pagination_res_body, + status=200) + + a = ApiFetcher(url="https://some/api", + body=first_req_body, + method=ReqMethod.POST, + response_data_path="data", + pagination=PaginationSettings(type=PaginationType("body"), + body_format={"page": "{res.info.page+1}"}, + max_calls=1)) + result = a.send_request() + + # Ensure the final logs list contains only the necessary data in the correct format + self.assertEqual(result, [{"message": "log1"}, {"message": "log2"}, {"message": "log3"}]) + + @responses.activate + def test_pagination_stop_indication(self): + first_res_body = {"result": [{"msg": "random log1"}, {"msg": "random log2"}], "page": 1} + pagination_res_body = {"result": [{"msg": "random log3"}, {"msg": "random log4"}], "page": 2} + pagination_last_res_body = {"result": [], "page": 3} + + # Mock response from some API that had pagination + responses.add(responses.GET, "https://some/api", + json=first_res_body, + status=200) + responses.add(responses.GET, "https://some/api?page=2", + json=pagination_res_body, + status=200) + responses.add(responses.GET, "https://some/api?page=3", + json=pagination_last_res_body, + status=200) + + a = ApiFetcher(url="https://some/api", + response_data_path="result", + pagination=PaginationSettings(type=PaginationType("url"), + url_format="?page={res.page+1}", + update_first_url=True, + stop_indication=StopPaginationSettings(field="result", + condition=StopCondition.EMPTY))) + result = a.send_request() + + # Ensure the final logs list contains only the necessary data in the correct format + self.assertEqual(result, [{"msg": "random log1"}, {"msg": "random log2"}, {"msg": "random log3"}, + {"msg": "random log4"}]) diff --git a/tests/UnitTests/test_azure_api.py b/tests/UnitTests/test_azure_api.py new file mode 100644 index 0000000..d5944e4 --- /dev/null +++ b/tests/UnitTests/test_azure_api.py @@ -0,0 +1,162 @@ +from datetime import datetime, UTC, timedelta +import json +from os.path import abspath, dirname +from pydantic import ValidationError +import responses +import unittest + +from src.apis.azure.AzureApi import AzureApi +from src.apis.azure.AzureGraph import AzureGraph +from src.apis.azure.AzureMailReports import AzureMailReports + + +curr_path = abspath(dirname(__file__)) + + +class TestAzureApi(unittest.TestCase): + """ + Test cases for Azure API + """ + + def test_invalid_setup(self): + with self.assertRaises(ValidationError): + AzureApi(azure_ad_tenant_id="some-tenant", + azure_ad_client_id="some-client", + data_request={"url": "https://azure"}) + AzureApi(azure_ad_tenant_id="some-tenant", + azure_ad_secret_value="some-secret", + data_request={"url": "https://azure"}) + AzureApi(azure_ad_secret_value="some-secret", + azure_ad_client_id="some-client", + data_request={"url": "https://azure"}) + + def test_valid_setup(self): + ag = AzureGraph(azure_ad_tenant_id="some-tenant", + azure_ad_client_id="some-client", + azure_ad_secret_value="some-secret", + data_request={"url": "https://azure-graph"}) + + am = AzureMailReports(azure_ad_tenant_id="some-tenant", + azure_ad_client_id="some-client", + azure_ad_secret_value="some-secret", + data_request={"url": "https://azure-mail"}) + + # Important note: test for 'expected_data_body' is >>SPACE SENSITIVE !!! << + # For any change at AzureAPI token_request body, make sure to update the amount of tabs here if needed + expected_token_body = """client_id=some-client + &scope=https://graph.microsoft.com/.default + &client_secret=some-secret + &grant_type=client_credentials + """ + + # Validate the token request + self.assertEqual(ag.token_request.url, "https://login.microsoftonline.com/some-tenant/oauth2/v2.0/token") + self.assertEqual(am.token_request.url, "https://login.microsoftonline.com/some-tenant/oauth2/v2.0/token") + self.assertEqual(ag.token_request.body, expected_token_body) + self.assertEqual(am.token_request.body, expected_token_body) + + # Validate the data request URL was updated + self.assertIn("https://azure-graph?$filter=createdDateTime gt", ag.data_request.url) + self.assertIn("https://azure-mail?$filter=StartDate eq datetime", am.data_request.url) + self.assertIn("EndDate eq datetime", am.data_request.url) + + # Validate the data request next URL was updated + self.assertEqual("https://azure-graph?$filter=createdDateTime gt {res.value.[0].createdDateTime}", + ag.data_request.next_url) + self.assertEqual( + "https://azure-mail?$filter=StartDate eq datetime '{res.d.results.[0].EndDate}' and EndDate eq datetime 'NOW_DATE'", + am.data_request.next_url) + + def test_start_date_generator(self): + day_back = AzureApi(azure_ad_tenant_id="some-tenant", + azure_ad_client_id="some-client", + azure_ad_secret_value="some-secret", + data_request={"url": "https://some-api"}) + + five_days_back = AzureApi(azure_ad_tenant_id="some-tenant", + azure_ad_client_id="some-client", + azure_ad_secret_value="some-secret", + data_request={"url": "https://some-api"}, + days_back_fetch=5) + + # Make sure the current format and needed dates are generated + self.assertLessEqual(datetime.strptime(day_back.generate_start_fetch_date(), "%Y-%m-%dT%H:%M:%SZ").timestamp(), + datetime.now(UTC).timestamp()) + self.assertLessEqual( + datetime.strptime(five_days_back.generate_start_fetch_date(), "%Y-%m-%dT%H:%M:%SZ").timestamp(), + (datetime.now(UTC) - timedelta(days=5)).timestamp()) + + @responses.activate + def test_azure_graph_send_request(self): + # Mock response from Azure Graph API + token_res_body = {"token_type": "Bearer", "expires_in": 3599, + "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Ik1uQ19WWmNBVGZNNXBP"} + with open(f"{curr_path}/responsesExamples/azure_graph_res_example.json", "r") as data_res_example_file: + data_res_body = json.loads(data_res_example_file.read()) + + # token response + responses.add(responses.POST, + "https://login.microsoftonline.com/some-tenant/oauth2/v2.0/token", + json=token_res_body, + status=200) + + # data response + responses.add(responses.GET, + "https://graph.microsoft.com/v1.0/auditLogs/signIns", + json=data_res_body, + status=200) + + # Pagination data response + responses.add(responses.GET, + "https://graph.microsoft.com/v1.0/auditLogs/signIns?$top=1&$skiptoken=9177f2e3532fcd4c4d225f68f7b9bdf7_1", + json={"value": []}, + status=200) + + # Test sending request + a = AzureGraph(azure_ad_tenant_id="some-tenant", + azure_ad_client_id="some-client", + azure_ad_secret_value="some-secret", + data_request={"url": "https://graph.microsoft.com/v1.0/auditLogs/signIns"}) + result = a.send_request() + + # Make sure we get the needed data and the url for next request is updated properly + self.assertEqual(result, data_res_body.get("value")) + self.assertEqual(a.data_request.url, + "https://graph.microsoft.com/v1.0/auditLogs/signIns?$filter=createdDateTime gt 2020-03-13T19:15:41.6195833Z") + + @responses.activate + def test_azure_mail_send_request(self): + # Mock response from Azure Mail API + token_res_body = {"token_type": "Bearer", "expires_in": 3599, + "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Ik1uQ19WWmNBVGZNNXBP"} + with open(f"{curr_path}/responsesExamples/azure_mail_res_example.json", "r") as data_res_example_file: + data_res_body = json.loads(data_res_example_file.read()) + + # token response + responses.add(responses.POST, + "https://login.microsoftonline.com/some-tenant/oauth2/v2.0/token", + json=token_res_body, + status=200) + + # data response + responses.add(responses.GET, + "https://reports.office365.com/ecp/reportingwebservice/reporting.svc/MessageTrace", + json=data_res_body, + status=200) + + # Pagination data response + responses.add(responses.GET, + "https://reports.office365.com/ecp/ReportingWebService/Reporting.svc/MessageTrace?$skiptoken=abc123", + json={"d": {"results": []}}, + status=200) + + # Test sending request + a = AzureMailReports(azure_ad_tenant_id="some-tenant", + azure_ad_client_id="some-client", + azure_ad_secret_value="some-secret", + data_request={"url": "https://reports.office365.com/ecp/reportingwebservice/reporting.svc/MessageTrace"}) + result = a.send_request() + + self.assertEqual(result, data_res_body.get("d").get("results")) + self.assertEqual(a.data_request.url, + "https://reports.office365.com/ecp/reportingwebservice/reporting.svc/MessageTrace?$filter=StartDate eq datetime '2024-05-30T13:08:54Z' and EndDate eq datetime 'NOW_DATE'") diff --git a/tests/UnitTests/test_cloudflare_api.py b/tests/UnitTests/test_cloudflare_api.py new file mode 100644 index 0000000..8a5073b --- /dev/null +++ b/tests/UnitTests/test_cloudflare_api.py @@ -0,0 +1,51 @@ +import json +import os +from pydantic import ValidationError +import responses +import unittest + +from src.apis.cloudflare.Cloudflare import Cloudflare + + +curr_path = os.path.abspath(os.path.dirname(__file__)) + + +class TestCloudflareApi(unittest.TestCase): + """ + Test cases for Cloudflare API + """ + + def test_invalid_setup(self): + with self.assertRaises(ValidationError): + Cloudflare(cloudflare_account_id="abcd-efg", + url="https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history") + Cloudflare(cloudflare_bearer_token="mYbeReartOKen", + url="https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history") + Cloudflare(cloudflare_account_id="abcd-efg", cloudflare_bearer_token="mYbeReartOKen") + + @responses.activate + def test_valid_setup(self): + # Mock response from Cloudflare API + with open(f"{curr_path}/responsesExamples/cloudflare_res_example.json", "r") as res_example_file: + res = json.loads(res_example_file.read()) + + # First Data Request + responses.add(responses.GET, "https://api.cloudflare.com/client/v4/accounts/abcd-efg/alerting/v3/history", + json=res, + status=200) + + # Pagination Request + responses.add(responses.GET, "https://api.cloudflare.com/client/v4/accounts/abcd-efg/alerting/v3/history?page=2", + json={"result": [], "result_info": {"page": 2, "total_pages": 1}}, + status=200) + + a = Cloudflare(cloudflare_account_id="abcd-efg", + cloudflare_bearer_token="mYbeReartOKen", + url="https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history", + next_url="https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history?since={res.result.[0].sent}") + + # Test sending request + results = a.send_request() + + self.assertEqual(a.url, "https://api.cloudflare.com/client/v4/accounts/abcd-efg/alerting/v3/history?since=2024-05-24T03:22:46.410294Z") + self.assertEqual(results, res.get("result")) diff --git a/tests/UnitTests/test_config_reader.py b/tests/UnitTests/test_config_reader.py new file mode 100644 index 0000000..6a3eed4 --- /dev/null +++ b/tests/UnitTests/test_config_reader.py @@ -0,0 +1,37 @@ +from os.path import abspath, dirname +import unittest + +from src.config.ConfigReader import ConfigReader + + +curr_path = abspath(dirname(dirname(__file__))) + + +class TestConfigReader(unittest.TestCase): + """ + Test reading config from YAML + """ + + def test_invalid_config(self): + with self.assertLogs("src.config.ConfigReader", level='INFO') as log: + ConfigReader(f"{curr_path}/testConfigs/invalid_conf.yaml") + self.assertIn("ERROR:src.config.ConfigReader:No inputs defined. Please make sure your API input is configured under 'apis'", log.output) + + def test_multiple_apis(self): + conf = ConfigReader(f"{curr_path}/testConfigs/multiple_apis_conf.yaml") + + self.assertEqual(len(conf.api_instances), 2) + self.assertEqual(conf.api_instances[0].pagination_settings.max_calls, 20) + self.assertEqual(conf.api_instances[1].scrape_interval_minutes, 5) + + def test_missing_logzio_output(self): + with self.assertLogs("src.config.ConfigReader", level='INFO') as log: + conf = ConfigReader(f"{curr_path}/testConfigs/missing_logz_output_conf.yaml") + self.assertIn("WARNING:src.config.ConfigReader:No Logzio shipper output defined. Please make sure your Logzio config is configured under logzio", log.output) + self.assertEqual(conf.logzio_shipper, None) + + def test_open_config_file_fail(self): + with self.assertLogs("src.config.ConfigReader", level='INFO') as log: + conf = ConfigReader(f"./not/existing/conf.yaml") + self.assertEqual(conf.api_instances, []) + self.assertEqual(conf.logzio_shipper, None) diff --git a/tests/UnitTests/test_logzio_shipper.py b/tests/UnitTests/test_logzio_shipper.py new file mode 100644 index 0000000..4d877ca --- /dev/null +++ b/tests/UnitTests/test_logzio_shipper.py @@ -0,0 +1,49 @@ +import responses +import requests +import unittest + +from src.output.LogzioShipper import LogzioShipper + + +class TestLogzioShipper(unittest.TestCase): + """ + Test logzio shipper + """ + + def test_add_log_to_send(self): + s = LogzioShipper(token="myShippingToken") + + # Text log + s.add_log_to_send("random text log", {"type": "someType"}) + self.assertIn('{"message": "random text log", "type": "someType"}', s.curr_logs) + + # Json log + s.add_log_to_send('{"message": "json log", "field": 123}', + {"type": "api-fetcher", "field2": "value"}) + self.assertIn('{"message": "json log", "field": 123, "type": "api-fetcher", "field2": "value"}', + s.curr_logs) + + @responses.activate + def test_send_to_logzio(self): + s = LogzioShipper(token="myShippingToken") + + responses.add(responses.POST, "https://listener.logz.io:8071/?token=myShippingToken", + status=200) + + s.add_log_to_send("random text log", {"type": "someType"}) + with self.assertLogs("src.output.LogzioShipper", level='INFO') as log: + s.send_to_logzio() + self.assertIn("INFO:src.output.LogzioShipper:Successfully sent bulk of 50 bytes to Logz.io.", log.output) + + @responses.activate + def test_invalid_token(self): + s = LogzioShipper(token="notGoodShippingToken") + + responses.add(responses.POST, "https://listener.logz.io:8071/?token=notGoodShippingToken", + status=401) + + s.add_log_to_send("random text log", {"type": "someType"}) + with self.assertLogs("src.output.LogzioShipper", level='INFO') as log: + with self.assertRaises(requests.exceptions.HTTPError): + s.send_to_logzio() + self.assertIn("ERROR:src.output.LogzioShipper:Logzio Shipping Token is missing or invalid. Make sure you’re using the right account token.", log.output) diff --git a/tests/UnitTests/test_masking_tokens.py b/tests/UnitTests/test_masking_tokens.py new file mode 100644 index 0000000..c8ab555 --- /dev/null +++ b/tests/UnitTests/test_masking_tokens.py @@ -0,0 +1,26 @@ +import logging +import unittest + +from src.utils.MaskInfoFormatter import MaskInfoFormatter + + +class TestMaskingTokensLogging(unittest.TestCase): + """ + Test masking tokens in the logging + """ + + def test_masking_tokens_in_logs(self): + formatter = MaskInfoFormatter() + mock_log_record = logging.LogRecord( + name='test_logger', + level=logging.INFO, + pathname='test.py', + lineno=123, + msg='https://listener.logz.io:8071/?token=logz-io-super-secret-token-value', + args=(), + exc_info=None + ) + + formatted_log = formatter.format(mock_log_record) + + self.assertEqual("https://listener.logz.io:8071/?token=******-value", formatted_log) diff --git a/tests/UnitTests/test_oauth_class.py b/tests/UnitTests/test_oauth_class.py new file mode 100644 index 0000000..100991a --- /dev/null +++ b/tests/UnitTests/test_oauth_class.py @@ -0,0 +1,72 @@ +from pydantic import ValidationError +import responses +import unittest + +from src.apis.general.Api import ApiFetcher, ReqMethod +from src.apis.oauth.OAuth import OAuthApi + + +class TestOAuthApi(unittest.TestCase): + """ + Test cases for the OAuth API + """ + + def test_invalid_setup(self): + with self.assertRaises(ValidationError): + # missing token_request + OAuthApi(data_request=ApiFetcher(url="http://my-data-url")) + + # missing data_request + OAuthApi(token_request=ApiFetcher(url="http://my-token-url")) + + # scrape_interval too big + OAuthApi(token_request=ApiFetcher(url="http://my-token-url"), + data_request=ApiFetcher(url="http://my-data-url"), + scrape_interval=0) + + def test_headers_setup(self): + # No initializing to headers + a = OAuthApi(token_request=ApiFetcher(url="http://my-token-url"), + data_request=ApiFetcher(url="http://my-data-url")) + self.assertEqual(a.data_request.headers.get("Content-Type"), "application/json") + + # Initializing to headers + a = OAuthApi(token_request=ApiFetcher(url="http://my-token-url"), + data_request=ApiFetcher(url="http://my-data-url", + headers={"Content-Type": "application/gzip"})) + self.assertEqual(a.data_request.headers.get("Content-Type"), "application/gzip") + + def test_additional_fields_init(self): + a = OAuthApi(token_request=ApiFetcher(url="http://my-token-url"), + data_request=ApiFetcher(url="http://my-data-url"), + additional_fields={"type": "test"}) + self.assertEqual("test", a.additional_fields.get("type")) + + a = OAuthApi(token_request=ApiFetcher(url="http://my-token-url"), + data_request=ApiFetcher(url="http://my-data-url")) + self.assertEqual("api-fetcher", a.additional_fields.get("type")) + + @responses.activate + def test_send_request(self): + token_res = {"access_token": "epZliHdLuj", "expires_in": 1} + data_res = {"data": [{"msg": "hi"}, {"msg": "hello", "field": 567}]} + + # Mock response from some API + responses.add(responses.POST, "http://my-token-url", + json=token_res, + status=200) + responses.add(responses.GET, "http://my-data-url", + json=data_res, + status=200) + + a = OAuthApi(token_request=ApiFetcher(url="http://my-token-url", method=ReqMethod.POST), + data_request=ApiFetcher(url="http://my-data-url", response_data_path="data")) + result = a.send_request() + + # Ensure we got the needed results + self.assertEqual([{"msg": "hi"}, {"msg": "hello", "field": 567}], result) + + # Test needing to update the token (gave super short expiration in 'token_res') + with self.assertLogs("src.apis.oauth.OAuth", level='DEBUG') as log: + a.send_request() + self.assertIn("DEBUG:src.apis.oauth.OAuth:Sending request to update the access token.", log.output) diff --git a/tests/UnitTests/test_utils.py b/tests/UnitTests/test_utils.py new file mode 100644 index 0000000..3a4cb28 --- /dev/null +++ b/tests/UnitTests/test_utils.py @@ -0,0 +1,98 @@ +import unittest + +from src.utils.processing_functions import extract_vars, get_nested_value, replace_dots, break_key_name, substitute_vars + + +class TestUtilsFunctions(unittest.TestCase): + """ + Test utils functions + """ + + def test_extract_vars(self): + # Test cases + valid_var_use = "correct vars {res.field} usage {res.[0]}/{res.test\\.f2}" + invalid_var_use = "wrong vars {field} usage {[0]}/{test\\.f2}" + + # Assert they behave as expected + self.assertEqual(extract_vars(valid_var_use), ["field", "[0]", "test\\.f2"]) + self.assertEqual(extract_vars(invalid_var_use), []) + self.assertEqual(extract_vars(123), []) + self.assertEqual(extract_vars(["a", "b", "c"]), []) + self.assertEqual(extract_vars({"field": "{res.hello}"}), ["hello"]) + + def test_get_nested_value(self): + # Test case + test_dic = { + "field": "hello", + "arr": [1, 2, 3], + "obj_arr": [{"f1": 123}, {"f1": 456, "f2": "abc"}], + "obj": { + "nested": "random value" + }, + "none": None, + "dot.in.name": "the value" + } + + # Assert it behaves as expected + self.assertEqual(get_nested_value(test_dic, ["field"]), "hello") + self.assertEqual(get_nested_value(test_dic, ["arr", "[2]"]), 3) + self.assertEqual(get_nested_value(test_dic, ["obj_arr", "[0]", "f1"]), 123) + self.assertEqual(get_nested_value(test_dic, ["obj", "nested"]), "random value") + self.assertEqual(get_nested_value(test_dic, ["none"]), None) + self.assertEqual(get_nested_value(test_dic, ["dot~~in~~name"]), "the value") + + with self.assertLogs("src.utils.processing_functions", level='WARN') as log: + result = get_nested_value(test_dic, ["arr", "[3]"]) + result2 = get_nested_value(test_dic, ["not_existing_field"]) + self.assertIn("WARNING:src.utils.processing_functions:Failed to find the next key: '3' nested in [1, 2, 3]", log.output) + self.assertIn("WARNING:src.utils.processing_functions:Key 'not_existing_field' does not exists in: {'field': 'hello', 'arr': [1, 2, 3], 'obj_arr': [{'f1': 123}, {'f1': 456, 'f2': 'abc'}], 'obj': {'nested': 'random value'}, 'none': None, 'dot.in.name': 'the value'}", log.output) + self.assertEqual(result, None) + self.assertEqual(result2, None) + + def test_replace_dots(self): + # Test cases + should_replace = "field\\.name" + do_not_replace = "field.name" + joint_dots = "field.name\\.replace.test" + + # Assert they behave as expected + self.assertEqual(replace_dots(should_replace), "field~~name") + self.assertEqual(replace_dots(do_not_replace), "field.name") + self.assertEqual(replace_dots(joint_dots), "field.name~~replace.test") + + def test_break_key_name(self): + # Test cases + do_not_break = "field\\.name" + do_break = "field.name" + + # Assert they behave as expected + self.assertEqual(break_key_name(do_not_break), ["field~~name"]) + self.assertEqual(break_key_name(do_break), ["field", "name"]) + + def test_substitute_vars(self): + # Test cases + test_dic = { + "field": "hello", + "arr": [1, 2, 3], + "obj_arr": [{"f1": 123}, {"f1": 456, "f2": "abc"}], + "obj": { + "nested": "random value" + }, + "none": None, + "dot.in.name": "the value" + } + + valid_vars = "{res.field}! just a {res.obj.nested} {res.arr.[1]}" + no_vars = "just a string with no vars" + not_valid_vars = "{field}! just testing {res.obj_arr[2].f1}" + empty_val_throw_error = "can also handle {res.none}!" + + # Assert they behave as expected + self.assertEqual(substitute_vars(valid_vars, extract_vars(valid_vars), test_dic), + "hello! just a random value 2") + + with self.assertRaises(ValueError): + substitute_vars(not_valid_vars, extract_vars(not_valid_vars), test_dic) + substitute_vars(empty_val_throw_error, extract_vars(empty_val_throw_error), test_dic) + + self.assertEqual(substitute_vars(no_vars, extract_vars(no_vars), test_dic), no_vars) diff --git a/tests/api_body/azure_graph_token_body.json b/tests/api_body/azure_graph_token_body.json deleted file mode 100644 index d5f39f1..0000000 --- a/tests/api_body/azure_graph_token_body.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "token_type": "Bearer", - "expires_in": 3599, - "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6Ik1uQ19WWmNBVGZNNXBP" -} \ No newline at end of file diff --git a/tests/api_body/cisco_secure_x_body.json b/tests/api_body/cisco_secure_x_body.json deleted file mode 100644 index 0660973..0000000 --- a/tests/api_body/cisco_secure_x_body.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "version": "v1.2.0", - "metadata": { - "links": { - "self": "https://api.amp.cisco.com/v1/events" - }, - "results": { - "total": 2, - "current_item_count": 2, - "index": 0, - "items_per_page": 0 - } - }, - "data": [ - { - "data_num": 1, - "date": "2021-10-05T10:10:10+00:00" - }, - { - "data_num": 2, - "date": "2021-10-05T10:10:09+00:00" - } - ] -} \ No newline at end of file diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py deleted file mode 100644 index 784f67f..0000000 --- a/tests/azure_graph_api_tests.py +++ /dev/null @@ -1,195 +0,0 @@ -import unittest -import logging -import multiprocessing -import json -import math - -import httpretty - -from src.azure_graph import AzureGraph -from src.logzio_shipper import LogzioShipper -from .tests_utils import TestUtils - -logger = logging.getLogger(__name__) - - -class AzureGraphApiTests(unittest.TestCase): - BASE_CONFIG_FILE = 'tests/config/azure_graph/base_config.yaml' - DAYS_BACK_FETCH_CONFIG_FILE = 'tests/config/azure_graph/days_back_fetch_config.yaml' - FILTERS_CONFIG_FILE = 'tests/config/azure_graph/filters_config.yaml' - CUSTOM_FIELDS_CONFIG_FILE = 'tests/config/azure_graph/custom_fields_config.yaml' - MULTIPLE_CONFIG_FILE = 'tests/config/azure_graph/multiple_config.yaml' - TIME_INTERVAL_CONFIG_FILE = 'tests/config/azure_graph/time_interval_config.yaml' - BAD_CONFIG_FILE = 'tests/config/azure_graph/bad_config.yaml' - AZURE_GRAPH_BODY_JSON = 'tests/api_body/azure_graph_body.json' - AZURE_GRAPH_TOKEN_BODY_JSON = 'tests/api_body/azure_graph_token_body.json' - AZURE_GRAPH_TEST_URL = "https://graph.microsoft.com/v1.0/auditLogs/signIns" - AZURE_GRAPH_TOKEN_TEST_URL = 'https://login.microsoftonline.com/<>/oauth2/v2.0' \ - '/token' - AZURE_GRAPH_TEST_TOKEN = "1234-abcd-efgh-5678" - azure_graph_json_body: dict = None - azure_graph_token_json_body: dict = None - - @classmethod - def setUpClass(cls) -> None: - with open(cls.AZURE_GRAPH_BODY_JSON, 'r') as json_file: - cls.azure_graph_json_body = json.load(json_file) - with open(cls.AZURE_GRAPH_TOKEN_BODY_JSON, 'r') as json_file: - cls.azure_graph_token_json_body = json.load(json_file) - - def setUp(self) -> None: - self.tests_utils = TestUtils("GET", self.AZURE_GRAPH_TEST_URL, - AzureGraphApiTests.azure_graph_json_body, - "POST", self.AZURE_GRAPH_TOKEN_TEST_URL, - AzureGraphApiTests.azure_graph_token_json_body) - - with open(self.tests_utils.LAST_START_DATES_FILE, 'w'): - pass - - def test_fetch_data(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_azure_graph_api_total_data_bytes_and_num( - AzureGraphApiTests.BASE_CONFIG_FILE) - base_azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.BASE_CONFIG_FILE, is_auth_api=False) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - base_azure_graph) - - self.assertEqual(total_data_bytes, fetched_data_bytes) - self.assertEqual(total_data_num, fetched_data_num) - - def test_fetch_data_with_days_back_fetch(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_azure_graph_api_total_data_bytes_and_num( - AzureGraphApiTests.DAYS_BACK_FETCH_CONFIG_FILE) - azure_graph_days_back_fetch = self.tests_utils.get_first_api( - AzureGraphApiTests.DAYS_BACK_FETCH_CONFIG_FILE, is_auth_api=False) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - azure_graph_days_back_fetch) - - self.assertNotEqual(total_data_bytes, fetched_data_bytes) - self.assertNotEqual(total_data_num, fetched_data_num) - - def test_fetch_data_with_filters(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_azure_graph_api_total_data_bytes_and_num( - AzureGraphApiTests.FILTERS_CONFIG_FILE) - filters_azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.FILTERS_CONFIG_FILE, - is_auth_api=False) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - filters_azure_graph) - - self.assertNotEqual(total_data_bytes, fetched_data_bytes) - self.assertNotEqual(total_data_num, fetched_data_num) - - def test_sending_data(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - AzureGraphApiTests.BASE_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_sending_data_iterations(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - AzureGraphApiTests.DAYS_BACK_FETCH_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=70) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_sending_data_multiple_azure_graph(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - AzureGraphApiTests.MULTIPLE_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_sending_data_with_custom_fields(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - custom_fields_azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE, - is_auth_api=False) - data_bytes += data_num * self.tests_utils.get_api_custom_fields_bytes(custom_fields_azure_graph) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_time_interval(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - AzureGraphApiTests.TIME_INTERVAL_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=70) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - @httpretty.activate - def test_last_start_date(self) -> None: - httpretty.register_uri(httpretty.GET, self.AZURE_GRAPH_TEST_URL, - body=json.dumps(AzureGraphApiTests.azure_graph_json_body), status=200) - azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE, - is_auth_api=False) - httpretty.register_uri(httpretty.POST, azure_graph.get_token_request.url, - body=json.dumps(AzureGraphApiTests.azure_graph_token_json_body), status=200) - - for _ in azure_graph.fetch_data(): - continue - - azure_graph.update_start_date_filter() - - self.assertEqual('2020-03-13T19:15:41.6195833Z', azure_graph.get_last_start_date()) - - def test_bad_config(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - AzureGraphApiTests.BAD_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=1) - - requests_num, sent_logs_num, sent_bytes = queue.get() - - self.assertEqual(0, requests_num) - self.assertEqual(0, sent_logs_num) - self.assertEqual(0, sent_bytes) diff --git a/tests/cisco_secure_x_api_tests.py b/tests/cisco_secure_x_api_tests.py deleted file mode 100644 index 2310148..0000000 --- a/tests/cisco_secure_x_api_tests.py +++ /dev/null @@ -1,188 +0,0 @@ -import unittest -import logging -import multiprocessing -import json -import math - -import httpretty - -from src.cisco_secure_x import CiscoSecureX -from src.logzio_shipper import LogzioShipper -from .tests_utils import TestUtils - - -logger = logging.getLogger(__name__) - - -class CiscoSecureXApiTests(unittest.TestCase): - - BASE_CONFIG_FILE = 'tests/config/cisco_secure_x/base_config.yaml' - DAYS_BACK_FETCH_CONFIG_FILE = 'tests/config/cisco_secure_x/days_back_fetch_config.yaml' - FILTERS_CONFIG_FILE = 'tests/config/cisco_secure_x/filters_config.yaml' - CUSTOM_FIELDS_CONFIG_FILE = 'tests/config/cisco_secure_x/custom_fields_config.yaml' - MULTIPLE_CONFIG_FILE = 'tests/config/cisco_secure_x/multiple_config.yaml' - TIME_INTERVAL_CONFIG_FILE = 'tests/config/cisco_secure_x/time_interval_config.yaml' - BAD_CONFIG_FILE = 'tests/config/cisco_secure_x/bad_config.yaml' - - CISCO_SECURE_X_BODY_JSON = 'tests/api_body/cisco_secure_x_body.json' - - cisco_secure_x_json_body: dict = None - - @classmethod - def setUpClass(cls) -> None: - with open(cls.CISCO_SECURE_X_BODY_JSON, 'r') as json_file: - cls.cisco_secure_x_json_body = json.load(json_file) - - def setUp(self) -> None: - self.tests_utils = TestUtils(CiscoSecureX.HTTP_METHOD, CiscoSecureX.URL, - CiscoSecureXApiTests.cisco_secure_x_json_body) - - with open(self.tests_utils.LAST_START_DATES_FILE, 'w'): - pass - - def test_fetch_data(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_cisco_secure_x_api_total_data_bytes_and_num( - CiscoSecureXApiTests.BASE_CONFIG_FILE) - base_cisco_secure_x = self.tests_utils.get_first_api(CiscoSecureXApiTests.BASE_CONFIG_FILE, is_auth_api=True) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - base_cisco_secure_x) - - self.assertEqual(total_data_bytes, fetched_data_bytes) - self.assertEqual(total_data_num, fetched_data_num) - - def test_fetch_data_with_days_back_fetch(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_cisco_secure_x_api_total_data_bytes_and_num( - CiscoSecureXApiTests.DAYS_BACK_FETCH_CONFIG_FILE) - days_back_fetch_cisco_secure_x = self.tests_utils.get_first_api( - CiscoSecureXApiTests.DAYS_BACK_FETCH_CONFIG_FILE, is_auth_api=True) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - days_back_fetch_cisco_secure_x) - - self.assertNotEqual(total_data_bytes, fetched_data_bytes) - self.assertNotEqual(total_data_num, fetched_data_num) - - def test_fetch_data_with_filters(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_cisco_secure_x_api_total_data_bytes_and_num( - CiscoSecureXApiTests.FILTERS_CONFIG_FILE) - filters_cisco_secure_x = self.tests_utils.get_first_api(CiscoSecureXApiTests.FILTERS_CONFIG_FILE, - is_auth_api=True) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - filters_cisco_secure_x) - - self.assertNotEqual(total_data_bytes, fetched_data_bytes) - self.assertNotEqual(total_data_num, fetched_data_num) - - def test_sending_data(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - CiscoSecureXApiTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_sending_data_iterations(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - CiscoSecureXApiTests.DAYS_BACK_FETCH_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=70) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) - self.assertEqual(data_num * 2, sent_logs_num) - self.assertEqual(data_bytes * 2, sent_bytes) - - def test_sending_data_multiple_cisco_secure_x(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - CiscoSecureXApiTests.MULTIPLE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) - self.assertEqual(data_num * 2, sent_logs_num) - self.assertEqual(data_bytes * 2, sent_bytes) - - def test_sending_data_with_custom_fields(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - CiscoSecureXApiTests.CUSTOM_FIELDS_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - custom_fields_cisco_secure_x = self.tests_utils.get_first_api(CiscoSecureXApiTests.CUSTOM_FIELDS_CONFIG_FILE, - is_auth_api=True) - data_bytes += data_num * self.tests_utils.get_api_custom_fields_bytes(custom_fields_cisco_secure_x) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_time_interval(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - CiscoSecureXApiTests.TIME_INTERVAL_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=70) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - @httpretty.activate - def test_last_start_date(self) -> None: - httpretty.register_uri(httpretty.GET, CiscoSecureX.URL, - body=json.dumps(CiscoSecureXApiTests.cisco_secure_x_json_body), status=200) - - base_cisco_secure_x = self.tests_utils.get_first_api(CiscoSecureXApiTests.CUSTOM_FIELDS_CONFIG_FILE, - is_auth_api=True) - - for _ in base_cisco_secure_x.fetch_data(): - continue - - base_cisco_secure_x.update_start_date_filter() - - self.assertEqual('2021-10-05T10%3A10%3A11%2B00%3A00', base_cisco_secure_x.get_last_start_date()) - - def test_bad_config(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - CiscoSecureXApiTests.BAD_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=1) - - requests_num, sent_logs_num, sent_bytes = queue.get() - - self.assertEqual(0, requests_num) - self.assertEqual(0, sent_logs_num) - self.assertEqual(0, sent_bytes) diff --git a/tests/combined_api_tests.py b/tests/combined_api_tests.py deleted file mode 100644 index 35ee3ea..0000000 --- a/tests/combined_api_tests.py +++ /dev/null @@ -1,126 +0,0 @@ -import unittest -import logging -import multiprocessing -import json -import math - -from src.azure_graph import AzureGraph -from src.cisco_secure_x import CiscoSecureX -from src.logzio_shipper import LogzioShipper -from .azure_graph_api_tests import AzureGraphApiTests -from .cisco_secure_x_api_tests import CiscoSecureXApiTests -from .tests_utils import TestUtils - -logger = logging.getLogger(__name__) - - -class CombinedApiTests(unittest.TestCase): - BASE_CONFIG_FILE = 'tests/config/combined_config/base_config.yaml' - DAYS_BACK_FETCH_CONFIG_FILE = 'tests/config/combined_config/days_back_fetch_config.yaml' - FILTERS_CONFIG_FILE = 'tests/config/combined_config/filters_config.yaml' - azure_graph_json_body: dict = None - azure_graph_token_json_body: dict = None - cisco_secure_json_body: dict = None - - @classmethod - def setUpClass(cls) -> None: - with open(AzureGraphApiTests.AZURE_GRAPH_BODY_JSON, 'r') as json_file: - cls.azure_graph_json_body = json.load(json_file) - with open(AzureGraphApiTests.AZURE_GRAPH_TOKEN_BODY_JSON, 'r') as json_file: - cls.azure_graph_token_json_body = json.load(json_file) - with open(CiscoSecureXApiTests.CISCO_SECURE_X_BODY_JSON, 'r') as json_file: - cls.cisco_secure_json_body = json.load(json_file) - - def setUp(self) -> None: - self.tests_utils = TestUtils("GET", AzureGraphApiTests.AZURE_GRAPH_TEST_URL, - self.azure_graph_json_body, - "POST", AzureGraphApiTests.AZURE_GRAPH_TOKEN_TEST_URL, - self.azure_graph_token_json_body, CiscoSecureX.HTTP_METHOD, CiscoSecureX.URL, - self.cisco_secure_json_body) - - with open(self.tests_utils.LAST_START_DATES_FILE, 'w'): - pass - - def int_sum_data(self): - self.total_bytes = 0 - self.fetched_bytes = 0 - self.total_data_num = 0 - self.fetched_data_num = 0 - - def add_to_sum_data(self, bytes, fetched_bytes, data_num, fetched_data_num): - self.total_bytes += bytes - self.total_data_num += data_num - self.fetched_bytes += fetched_bytes - self.fetched_data_num += fetched_data_num - - def test_fetch_data(self) -> None: - self.int_sum_data() - data_bytes, data_num = self.tests_utils.get_azure_graph_api_total_data_bytes_and_num( - self.BASE_CONFIG_FILE) - base_azure_graph = self.tests_utils.get_first_api(self.BASE_CONFIG_FILE, is_auth_api=False) - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - base_azure_graph) - self.add_to_sum_data(data_bytes, fetched_data_bytes, data_num, fetched_data_num) - - data_bytes, data_num = self.tests_utils.get_cisco_secure_x_api_total_data_bytes_and_num( - CiscoSecureXApiTests.BASE_CONFIG_FILE) - base_cisco_secure_x = self.tests_utils.get_first_api(CiscoSecureXApiTests.BASE_CONFIG_FILE, is_auth_api=True) - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - base_cisco_secure_x) - self.add_to_sum_data(data_bytes, fetched_data_bytes, data_num, fetched_data_num) - - self.assertEqual(self.total_bytes, self.fetched_bytes) - self.assertEqual(self.total_data_num, self.fetched_data_num) - - def test_fetch_data_with_days_back_fetch(self) -> None: - self.int_sum_data() - data_bytes, data_num = self.tests_utils.get_azure_graph_api_total_data_bytes_and_num( - self.DAYS_BACK_FETCH_CONFIG_FILE) - azure_graph_days_back_fetch = self.tests_utils.get_first_api( - self.DAYS_BACK_FETCH_CONFIG_FILE, is_auth_api=False) - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - azure_graph_days_back_fetch) - self.add_to_sum_data(data_bytes, fetched_data_bytes, data_num, fetched_data_num) - - data_bytes, data_num = self.tests_utils.get_cisco_secure_x_api_total_data_bytes_and_num( - CiscoSecureXApiTests.DAYS_BACK_FETCH_CONFIG_FILE) - days_back_fetch_cisco_secure_x = self.tests_utils.get_first_api( - CiscoSecureXApiTests.DAYS_BACK_FETCH_CONFIG_FILE, is_auth_api=True) - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - days_back_fetch_cisco_secure_x) - self.add_to_sum_data(data_bytes, fetched_data_bytes, data_num, fetched_data_num) - - self.assertNotEqual(self.total_bytes, self.fetched_bytes) - self.assertNotEqual(self.total_data_num, self.fetched_data_num) - - def test_fetch_data_with_filters(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_azure_graph_api_total_data_bytes_and_num( - self.FILTERS_CONFIG_FILE) - filters_azure_graph = self.tests_utils.get_first_api(self.FILTERS_CONFIG_FILE, - is_auth_api=False) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - filters_azure_graph) - - self.assertNotEqual(total_data_bytes, fetched_data_bytes) - self.assertNotEqual(total_data_num, fetched_data_num) - - def test_sending_data(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - self.BASE_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10, - is_multi_test=True) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - data_bytes2, data_num2 = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) - self.assertEqual(data_num + data_num2, sent_logs_num) - self.assertEqual(data_bytes + data_bytes2, sent_bytes) diff --git a/tests/config/azure_graph/bad_config.yaml b/tests/config/azure_graph/bad_config.yaml deleted file mode 100644 index d4a5747..0000000 --- a/tests/config/azure_graph/bad_config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: - credentials: - id: aaaa-bbbb-cccc - key: abcabcabc - token_http_request: - url: https://login.microsoftonline.com/abcd-efgh-abcd-efgh/oauth2/v2.0/token - body: client_id=aaaa-bbbb-cccc - &scope=https://graph.microsoft.com/.default - &client_secret=abcabcabc - &grant_type=client_credentials - headers: - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - headers: - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - next_url: - data: - settings: - time_interval: 1 \ No newline at end of file diff --git a/tests/config/azure_graph/base_config.yaml b/tests/config/azure_graph/base_config.yaml deleted file mode 100644 index 3257506..0000000 --- a/tests/config/azure_graph/base_config.yaml +++ /dev/null @@ -1,26 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - json_paths: - data_date: createdDateTime - settings: - time_interval: 1 - days_back_fetch: 30 - start_date_name: createdDateTime diff --git a/tests/config/azure_graph/custom_fields_config.yaml b/tests/config/azure_graph/custom_fields_config.yaml deleted file mode 100644 index 2b56f57..0000000 --- a/tests/config/azure_graph/custom_fields_config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - headers: - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - settings: - time_interval: 1 - custom_fields: - field1: test1 - field2: test2 - diff --git a/tests/config/azure_graph/days_back_fetch_config.yaml b/tests/config/azure_graph/days_back_fetch_config.yaml deleted file mode 100644 index 0f2f243..0000000 --- a/tests/config/azure_graph/days_back_fetch_config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - headers: - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - additional_filters: - top: 100 - settings: - time_interval: 1 - days_back_fetch: 3 diff --git a/tests/config/azure_graph/filters_config.yaml b/tests/config/azure_graph/filters_config.yaml deleted file mode 100644 index ab0e731..0000000 --- a/tests/config/azure_graph/filters_config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - headers: - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - filters: - top: 100 - settings: - time_interval: 1 diff --git a/tests/config/azure_graph/multiple_config.yaml b/tests/config/azure_graph/multiple_config.yaml deleted file mode 100644 index 1b0970d..0000000 --- a/tests/config/azure_graph/multiple_config.yaml +++ /dev/null @@ -1,46 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - settings: - time_interval: 1 - - - type: azure_graph - name: azure_test2 - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: activityDateTime - json_paths: - data_date: activityDateTime - settings: - time_interval: 1 diff --git a/tests/config/azure_graph/time_interval_config.yaml b/tests/config/azure_graph/time_interval_config.yaml deleted file mode 100644 index 7a07d15..0000000 --- a/tests/config/azure_graph/time_interval_config.yaml +++ /dev/null @@ -1,26 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - headers: - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - settings: - time_interval: 2 diff --git a/tests/config/cisco_secure_x/bad_config.yaml b/tests/config/cisco_secure_x/bad_config.yaml deleted file mode 100644 index e0ecdad..0000000 --- a/tests/config/cisco_secure_x/bad_config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: cisco_secure_x - name: - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 diff --git a/tests/config/cisco_secure_x/base_config.yaml b/tests/config/cisco_secure_x/base_config.yaml deleted file mode 100644 index 2e815f3..0000000 --- a/tests/config/cisco_secure_x/base_config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 diff --git a/tests/config/cisco_secure_x/custom_fields_config.yaml b/tests/config/cisco_secure_x/custom_fields_config.yaml deleted file mode 100644 index d3862e8..0000000 --- a/tests/config/cisco_secure_x/custom_fields_config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - custom_fields: - field1: test1 - field2: test2 diff --git a/tests/config/cisco_secure_x/days_back_fetch_config.yaml b/tests/config/cisco_secure_x/days_back_fetch_config.yaml deleted file mode 100644 index 889ada5..0000000 --- a/tests/config/cisco_secure_x/days_back_fetch_config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 2 diff --git a/tests/config/cisco_secure_x/filters_config.yaml b/tests/config/cisco_secure_x/filters_config.yaml deleted file mode 100644 index b1d90d1..0000000 --- a/tests/config/cisco_secure_x/filters_config.yaml +++ /dev/null @@ -1,15 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - filters: - event_type%5B%5D: '1090519054' diff --git a/tests/config/cisco_secure_x/multiple_config.yaml b/tests/config/cisco_secure_x/multiple_config.yaml deleted file mode 100644 index 06400f5..0000000 --- a/tests/config/cisco_secure_x/multiple_config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: cisco_secure_x - name: cisco 1 - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - - type: cisco_secure_x - name: cisco 2 - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 diff --git a/tests/config/cisco_secure_x/time_interval_config.yaml b/tests/config/cisco_secure_x/time_interval_config.yaml deleted file mode 100644 index 87481d6..0000000 --- a/tests/config/cisco_secure_x/time_interval_config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 2 - days_back_fetch: 30 diff --git a/tests/config/combined_config/base_config.yaml b/tests/config/combined_config/base_config.yaml deleted file mode 100644 index 24b46bc..0000000 --- a/tests/config/combined_config/base_config.yaml +++ /dev/null @@ -1,37 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - next_url: - data: - settings: - time_interval: 1 - days_back_fetch: 30 - -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 diff --git a/tests/config/combined_config/custom_fields_config.yaml b/tests/config/combined_config/custom_fields_config.yaml deleted file mode 100644 index 30c3609..0000000 --- a/tests/config/combined_config/custom_fields_config.yaml +++ /dev/null @@ -1,40 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - settings: - time_interval: 1 - custom_fields: - field1: test1 - field2: test2 - -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - custom_fields: - field1: test1 - field2: test2 diff --git a/tests/config/combined_config/days_back_fetch_config.yaml b/tests/config/combined_config/days_back_fetch_config.yaml deleted file mode 100644 index 231861c..0000000 --- a/tests/config/combined_config/days_back_fetch_config.yaml +++ /dev/null @@ -1,37 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - additional_filters: - top: 100 - settings: - time_interval: 1 - days_back_fetch: 3 -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 3 diff --git a/tests/config/combined_config/filters_config.yaml b/tests/config/combined_config/filters_config.yaml deleted file mode 100644 index 26618fd..0000000 --- a/tests/config/combined_config/filters_config.yaml +++ /dev/null @@ -1,38 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - filters: - top: 100 - settings: - time_interval: 1 -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - filters: - event_type%5B%5D: '1090519054' diff --git a/tests/config/combined_config/time_interval_config.yaml b/tests/config/combined_config/time_interval_config.yaml deleted file mode 100644 index 91d4ce3..0000000 --- a/tests/config/combined_config/time_interval_config.yaml +++ /dev/null @@ -1,35 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - settings: - time_interval: 2 - -auth_apis: - - type: cisco_secure_x - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 2 - days_back_fetch: 30 diff --git a/tests/config/different_types_auth_api/different_types_config.yaml b/tests/config/different_types_auth_api/different_types_config.yaml deleted file mode 100644 index bcef8cb..0000000 --- a/tests/config/different_types_auth_api/different_types_config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: cisco_secure_x - name: cisco 1 - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - - type: general - name: cisco 2 - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date \ No newline at end of file diff --git a/tests/config/different_types_oauth_api/different_types_config.yaml b/tests/config/different_types_oauth_api/different_types_config.yaml deleted file mode 100644 index 24e14ca..0000000 --- a/tests/config/different_types_oauth_api/different_types_config.yaml +++ /dev/null @@ -1,49 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: azure_graph - name: azure_test - credentials: - id: aaaa-bbbb-cccc - key: abcabcabc - token_http_request: - url: https://login.microsoftonline.com/abcd-efgh-abcd-efgh/oauth2/v2.0/token - body: client_id=aaaa-bbbb-cccc - &scope=https://graph.microsoft.com/.default - &client_secret=abcabcabc - &grant_type=client_credentials - type: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - start_date_name: createdDateTime - json_paths: - data_date: createdDateTime - settings: - time_interval: 1 - - - type: general - name: general_test2 - credentials: - id: aaaa-bbbb-cccc - key: abcabcabc - token_http_request: - url: https://login.microsoftonline.com/abcd-efgh-abcd-efgh/oauth2/v2.0/token - body: client_id=aaaa-bbbb-cccc - &scope=https://graph.microsoft.com/.default - &client_secret=abcabcabc - &grant_type=client_credentials - headers: - type: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - json_paths: - data_date: createdDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 - start_date_name: createdDateTime \ No newline at end of file diff --git a/tests/config/general_type_auth_api/bad_config.yaml b/tests/config/general_type_auth_api/bad_config.yaml deleted file mode 100644 index d9f2ef4..0000000 --- a/tests/config/general_type_auth_api/bad_config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: general - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date \ No newline at end of file diff --git a/tests/config/general_type_auth_api/base_config.yaml b/tests/config/general_type_auth_api/base_config.yaml deleted file mode 100644 index 3e39766..0000000 --- a/tests/config/general_type_auth_api/base_config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: general - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date \ No newline at end of file diff --git a/tests/config/general_type_auth_api/custom_fields_config.yaml b/tests/config/general_type_auth_api/custom_fields_config.yaml deleted file mode 100644 index fe68d57..0000000 --- a/tests/config/general_type_auth_api/custom_fields_config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: general - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date - custom_fields: - field1: test1 - field2: test2 \ No newline at end of file diff --git a/tests/config/general_type_auth_api/days_back_fetch_config.yaml b/tests/config/general_type_auth_api/days_back_fetch_config.yaml deleted file mode 100644 index 9f9800f..0000000 --- a/tests/config/general_type_auth_api/days_back_fetch_config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: general - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 2 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date \ No newline at end of file diff --git a/tests/config/general_type_auth_api/filters_config.yaml b/tests/config/general_type_auth_api/filters_config.yaml deleted file mode 100644 index d621530..0000000 --- a/tests/config/general_type_auth_api/filters_config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: general - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date - filters: - event_type%5B%5D: '1090519054' diff --git a/tests/config/general_type_auth_api/http_request_headers_config.yaml b/tests/config/general_type_auth_api/http_request_headers_config.yaml deleted file mode 100644 index b25f4a8..0000000 --- a/tests/config/general_type_auth_api/http_request_headers_config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: general - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - headers: - content-type: application/json - json_paths: - next_url: metadata.links.next - data: data - data_date: date \ No newline at end of file diff --git a/tests/config/general_type_auth_api/multiple_config.yaml b/tests/config/general_type_auth_api/multiple_config.yaml deleted file mode 100644 index 6f9506b..0000000 --- a/tests/config/general_type_auth_api/multiple_config.yaml +++ /dev/null @@ -1,37 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: general - name: cisco 1 - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date - - type: general - name: cisco 2 - credentials: - id: <> - key: <> - settings: - time_interval: 1 - days_back_fetch: 30 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date \ No newline at end of file diff --git a/tests/config/general_type_auth_api/time_interval_config.yaml b/tests/config/general_type_auth_api/time_interval_config.yaml deleted file mode 100644 index 6fc99ac..0000000 --- a/tests/config/general_type_auth_api/time_interval_config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -auth_apis: - - type: general - name: cisco - credentials: - id: <> - key: <> - settings: - time_interval: 2 - days_back_fetch: 30 - start_date_name: start_date - http_request: - method: GET - url: https://api.amp.cisco.com/v1/events - json_paths: - next_url: metadata.links.next - data: data - data_date: date \ No newline at end of file diff --git a/tests/config/general_type_oauth_api/bad_config.yaml b/tests/config/general_type_oauth_api/bad_config.yaml deleted file mode 100644 index 752b141..0000000 --- a/tests/config/general_type_oauth_api/bad_config.yaml +++ /dev/null @@ -1,26 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: general - name: - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: GET - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - headers: - json_paths: - data_date: activityDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 diff --git a/tests/config/general_type_oauth_api/base_config.yaml b/tests/config/general_type_oauth_api/base_config.yaml deleted file mode 100644 index a98c1b2..0000000 --- a/tests/config/general_type_oauth_api/base_config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: general - name: general_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - json_paths: - data_date: createdDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 - days_back_fetch: 30 - start_date_name: createdDateTime diff --git a/tests/config/general_type_oauth_api/custom_fields_config.yaml b/tests/config/general_type_oauth_api/custom_fields_config.yaml deleted file mode 100644 index 0780e87..0000000 --- a/tests/config/general_type_oauth_api/custom_fields_config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: general - name: general_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - json_paths: - data_date: createdDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 - start_date_name: createdDateTime - custom_fields: - field1: test1 - field2: test2 diff --git a/tests/config/general_type_oauth_api/days_back_fetch_config.yaml b/tests/config/general_type_oauth_api/days_back_fetch_config.yaml deleted file mode 100644 index 74e9db8..0000000 --- a/tests/config/general_type_oauth_api/days_back_fetch_config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: general - name: general_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - json_paths: - data_date: createdDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 - days_back_fetch: 3 - start_date_name: createdDateTime diff --git a/tests/config/general_type_oauth_api/filters_config.yaml b/tests/config/general_type_oauth_api/filters_config.yaml deleted file mode 100644 index 1d8fb4d..0000000 --- a/tests/config/general_type_oauth_api/filters_config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: general - name: general_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - json_paths: - data_date: createdDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 - days_back_fetch: 1 - filters: - top: 100 - start_date_name: createdDateTime diff --git a/tests/config/general_type_oauth_api/http_request_headers_config.yaml b/tests/config/general_type_oauth_api/http_request_headers_config.yaml deleted file mode 100644 index 1b66f97..0000000 --- a/tests/config/general_type_oauth_api/http_request_headers_config.yaml +++ /dev/null @@ -1,32 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: general - name: general_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - headers: - content-type: application/json - json_paths: - data_date: createdDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 - days_back_fetch: 1 - filters: - top: 100 - start_date_name: createdDateTime diff --git a/tests/config/general_type_oauth_api/multiple_config.yaml b/tests/config/general_type_oauth_api/multiple_config.yaml deleted file mode 100644 index 91fa814..0000000 --- a/tests/config/general_type_oauth_api/multiple_config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: general - name: general_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - json_paths: - data_date: createdDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 - start_date_name: createdDateTime - - - type: general - name: general_test2 - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - json_paths: - data_date: createdDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 1 - start_date_name: createdDateTime diff --git a/tests/config/general_type_oauth_api/time_interval_config.yaml b/tests/config/general_type_oauth_api/time_interval_config.yaml deleted file mode 100644 index f596034..0000000 --- a/tests/config/general_type_oauth_api/time_interval_config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -logzio: - url: https://listener.logz.io:8071 - token: 123456789a - -oauth_apis: - - type: general - name: general_test - credentials: - id: <> - key: <> - token_http_request: - url: https://login.microsoftonline.com/<>/oauth2/v2.0/token - body: client_id=<> - &scope=https://graph.microsoft.com/.default - &client_secret=<> - &grant_type=client_credentials - method: POST - data_http_request: - url: https://graph.microsoft.com/v1.0/auditLogs/signIns - method: GET - json_paths: - data_date: createdDateTime - data: value - next_url: '@odata.nextLink' - settings: - time_interval: 5 - start_date_name: createdDateTime diff --git a/tests/different_types_auth_api_tests.py b/tests/different_types_auth_api_tests.py deleted file mode 100644 index ac30448..0000000 --- a/tests/different_types_auth_api_tests.py +++ /dev/null @@ -1,62 +0,0 @@ -import unittest -import logging -import multiprocessing -import json -import math - -from src.cisco_secure_x import CiscoSecureX -from src.logzio_shipper import LogzioShipper -from .tests_utils import TestUtils - - -logger = logging.getLogger(__name__) - - -class DifferentTypesAuthApiTests(unittest.TestCase): - - CONFIG_FILE = 'tests/config/different_types_auth_api/different_types_config.yaml' - - CISCO_SECURE_X_BODY_JSON = 'tests/api_body/cisco_secure_x_body.json' - - cisco_secure_x_json_body: dict = None - - @classmethod - def setUpClass(cls) -> None: - with open(cls.CISCO_SECURE_X_BODY_JSON, 'r') as json_file: - cls.cisco_secure_x_json_body = json.load(json_file) - - def setUp(self) -> None: - self.tests_utils = TestUtils(CiscoSecureX.HTTP_METHOD, CiscoSecureX.URL, - DifferentTypesAuthApiTests.cisco_secure_x_json_body) - - def test_sending_data(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - DifferentTypesAuthApiTests.CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) - self.assertEqual(data_num * 2, sent_logs_num) - self.assertEqual(data_bytes * 2, sent_bytes) - - def test_sending_data_iterations(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - DifferentTypesAuthApiTests.CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=70) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / 4 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 4, requests_num) - self.assertEqual(data_num * 4, sent_logs_num) - self.assertEqual(data_bytes * 4, sent_bytes) diff --git a/tests/general_tests.py b/tests/general_tests.py deleted file mode 100644 index 8980e11..0000000 --- a/tests/general_tests.py +++ /dev/null @@ -1,200 +0,0 @@ -import unittest -import logging -import multiprocessing -import json -import math -import httpretty -import requests - -from requests.sessions import InvalidSchema -from src.cisco_secure_x import CiscoSecureX -from src.logzio_shipper import LogzioShipper -from .tests_utils import TestUtils - - -logger = logging.getLogger(__name__) - - -class GeneralTests(unittest.TestCase): - - BASE_CONFIG_FILE = 'tests/config/cisco_secure_x/base_config.yaml' - CISCO_SECURE_X_BODY_JSON = 'tests/api_body/cisco_secure_x_body.json' - BAD_LOGZIO_URL = 'https://bad.endpoint:1234' - BAD_URI = 'https:/bad.uri:1234' - BAD_CONNECTION_ADAPTER_URL = 'bad://connection.adapter:1234' - - cisco_secure_x_json_body: dict = None - - @classmethod - def setUpClass(cls) -> None: - with open(cls.CISCO_SECURE_X_BODY_JSON, 'r') as json_file: - cls.cisco_secure_x_json_body = json.load(json_file) - - def setUp(self) -> None: - self.tests_utils = TestUtils(CiscoSecureX.HTTP_METHOD, CiscoSecureX.URL, - GeneralTests.cisco_secure_x_json_body) - - with open(self.tests_utils.LAST_START_DATES_FILE, 'w'): - pass - - def test_signal(self): - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - def test_task_scheduler(self): - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=70) - - requests_num, _, sent_bytes = queue.get() - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) - - def test_write_last_start_date_to_file(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - file_lines = self.tests_utils.get_last_start_dates_file_lines() - - if not file_lines[0] == 'cisco: 2021-10-05T10%3A10%3A11%2B00%3A00\n': - self.assertEqual(True, False) - - self.assertEqual(1, len(file_lines)) - - def test_append_last_start_date_to_file(self) -> None: - with open(TestUtils.LAST_START_DATES_FILE, 'w') as file: - file.writelines('cisco test: 2021-10-04T10%3A10%3A10%2B00%3A00\n') - - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - file_lines = self.tests_utils.get_last_start_dates_file_lines() - - if not file_lines[1] == 'cisco: 2021-10-05T10%3A10%3A11%2B00%3A00\n': - self.assertEqual(True, False) - - self.assertEqual(2, len(file_lines)) - - def test_override_last_start_date(self) -> None: - with open(TestUtils.LAST_START_DATES_FILE, 'w') as file: - file.writelines(['cisco test: 2021-10-04T10%3A10%3A10%2B00%3A00\n', - 'cisco: 2021-10-04T10%3A10%3A10%2B00%3A00\n', - 'cisco test: 2021-10-04T10%3A10%3A10%2B00%3A00\n']) - - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - file_lines = self.tests_utils.get_last_start_dates_file_lines() - - if not file_lines[1] == 'cisco: 2021-10-05T10%3A10%3A11%2B00%3A00\n': - self.assertEqual(True, False) - - self.assertEqual(3, len(file_lines)) - - def test_send_retry_status_500(self): - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=500, - sleep_time=10) - - requests_num, _, _ = queue.get() - - self.assertEqual(LogzioShipper.MAX_RETRIES + 1, requests_num) - - def test_send_retry_status_502(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=502, - sleep_time=10) - - requests_num, _, _ = queue.get() - - self.assertEqual(LogzioShipper.MAX_RETRIES + 1, requests_num) - - def test_send_retry_status_503(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=503, - sleep_time=10) - - requests_num, _, _ = queue.get() - - self.assertEqual(LogzioShipper.MAX_RETRIES + 1, requests_num) - - def test_send_retry_status_504(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=504, - sleep_time=10) - - requests_num, _, _ = queue.get() - - self.assertEqual(LogzioShipper.MAX_RETRIES + 1, requests_num) - - @httpretty.activate - def test_send_bad_format(self) -> None: - httpretty.register_uri(httpretty.POST, self.tests_utils.LOGZIO_URL, status=400) - - logzio_shipper = LogzioShipper(self.tests_utils.LOGZIO_URL, self.tests_utils.LOGZIO_TOKEN) - - logzio_shipper.add_log_to_send('{"test": "test"}') - - self.assertRaises(requests.HTTPError, logzio_shipper.send_to_logzio) - - def test_sending_bad_logzio_url(self) -> None: - logzio_shipper = LogzioShipper(GeneralTests.BAD_LOGZIO_URL, self.tests_utils.LOGZIO_TOKEN) - - logzio_shipper.add_log_to_send('{"test": "test"}') - - self.assertRaises(requests.ConnectionError, logzio_shipper.send_to_logzio) - - @httpretty.activate - def test_sending_bad_logzio_token(self) -> None: - httpretty.register_uri(httpretty.POST, self.tests_utils.LOGZIO_URL, status=401) - - logzio_shipper = LogzioShipper(self.tests_utils.LOGZIO_URL, self.tests_utils.LOGZIO_TOKEN) - - logzio_shipper.add_log_to_send('{"test": "test"}') - - self.assertRaises(requests.HTTPError, logzio_shipper.send_to_logzio) - - def test_sending_bad_uri(self) -> None: - logzio_shipper = LogzioShipper(GeneralTests.BAD_URI, self.tests_utils.LOGZIO_TOKEN) - - logzio_shipper.add_log_to_send('{"test": "test"}') - - self.assertRaises(requests.exceptions.InvalidURL, logzio_shipper.send_to_logzio) - - def test_sending_bad_connection_adapter(self) -> None: - logzio_shipper = LogzioShipper(GeneralTests.BAD_CONNECTION_ADAPTER_URL, self.tests_utils.LOGZIO_TOKEN) - - logzio_shipper.add_log_to_send('{"test": "test"}') - - self.assertRaises(InvalidSchema, logzio_shipper.send_to_logzio) diff --git a/tests/general_type_auth_api_tests.py b/tests/general_type_auth_api_tests.py deleted file mode 100644 index 7885a13..0000000 --- a/tests/general_type_auth_api_tests.py +++ /dev/null @@ -1,203 +0,0 @@ -import unittest -import logging -import multiprocessing -import json -import math -import httpretty - -from src.cisco_secure_x import CiscoSecureX -from src.logzio_shipper import LogzioShipper -from .tests_utils import TestUtils - - -logger = logging.getLogger(__name__) - - -class GeneralTypeAuthApiTests(unittest.TestCase): - - BASE_CONFIG_FILE = 'tests/config/general_type_auth_api/base_config.yaml' - DAYS_BACK_FETCH_CONFIG_FILE = 'tests/config/general_type_auth_api/days_back_fetch_config.yaml' - FILTERS_CONFIG_FILE = 'tests/config/general_type_auth_api/filters_config.yaml' - CUSTOM_FIELDS_CONFIG_FILE = 'tests/config/general_type_auth_api/custom_fields_config.yaml' - MULTIPLE_CONFIG_FILE = 'tests/config/general_type_auth_api/multiple_config.yaml' - TIME_INTERVAL_CONFIG_FILE = 'tests/config/general_type_auth_api/time_interval_config.yaml' - HTTP_REQUEST_HEADERS_CONFIG_FILE = 'tests/config/general_type_auth_api/http_request_headers_config.yaml' - BAD_CONFIG_FILE = 'tests/config/general_type_auth_api/bad_config.yaml' - CISCO_SECURE_X_BODY_JSON = 'tests/api_body/cisco_secure_x_body.json' - - cisco_secure_x_json_body: dict = None - - @classmethod - def setUpClass(cls) -> None: - with open(cls.CISCO_SECURE_X_BODY_JSON, 'r') as json_file: - cls.cisco_secure_x_json_body = json.load(json_file) - - def setUp(self) -> None: - self.tests_utils = TestUtils(CiscoSecureX.HTTP_METHOD, CiscoSecureX.URL, - GeneralTypeAuthApiTests.cisco_secure_x_json_body) - - with open(self.tests_utils.LAST_START_DATES_FILE, 'w'): - pass - - def test_fetch_data(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_cisco_secure_x_api_total_data_bytes_and_num( - GeneralTypeAuthApiTests.BASE_CONFIG_FILE) - base_cisco_secure_x = self.tests_utils.get_first_api(GeneralTypeAuthApiTests.BASE_CONFIG_FILE, is_auth_api=True) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - base_cisco_secure_x) - - self.assertEqual(total_data_bytes, fetched_data_bytes) - self.assertEqual(total_data_num, fetched_data_num) - - def test_fetch_data_with_days_back_fetch(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_cisco_secure_x_api_total_data_bytes_and_num( - GeneralTypeAuthApiTests.DAYS_BACK_FETCH_CONFIG_FILE) - days_back_fetch_cisco_secure_x = self.tests_utils.get_first_api( - GeneralTypeAuthApiTests.DAYS_BACK_FETCH_CONFIG_FILE, is_auth_api=True) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - days_back_fetch_cisco_secure_x) - - self.assertNotEqual(total_data_bytes, fetched_data_bytes) - self.assertNotEqual(total_data_num, fetched_data_num) - - def test_fetch_data_with_filters(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_cisco_secure_x_api_total_data_bytes_and_num( - GeneralTypeAuthApiTests.FILTERS_CONFIG_FILE) - filters_cisco_secure_x = self.tests_utils.get_first_api(GeneralTypeAuthApiTests.FILTERS_CONFIG_FILE, - is_auth_api=True) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - filters_cisco_secure_x) - - self.assertNotEqual(total_data_bytes, fetched_data_bytes) - self.assertNotEqual(total_data_num, fetched_data_num) - - def test_sending_data(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeAuthApiTests.BASE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_sending_data_with_http_request_headers(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeAuthApiTests.HTTP_REQUEST_HEADERS_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_sending_data_iterations(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeAuthApiTests.DAYS_BACK_FETCH_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=70) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) - self.assertEqual(data_num * 2, sent_logs_num) - self.assertEqual(data_bytes * 2, sent_bytes) - - def test_sending_data_multiple_general_type_apis(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeAuthApiTests.MULTIPLE_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) - self.assertEqual(data_num * 2, sent_logs_num) - self.assertEqual(data_bytes * 2, sent_bytes) - - def test_sending_data_with_custom_fields(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - custom_fields_cisco_secure_x = self.tests_utils.get_first_api(GeneralTypeAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, - is_auth_api=True) - data_bytes += data_num * self.tests_utils.get_api_custom_fields_bytes(custom_fields_cisco_secure_x) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_time_interval(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeAuthApiTests.TIME_INTERVAL_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=70) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.cisco_secure_x_json_body['data']) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - @httpretty.activate - def test_last_start_date(self) -> None: - httpretty.register_uri(httpretty.GET, CiscoSecureX.URL, - body=json.dumps(GeneralTypeAuthApiTests.cisco_secure_x_json_body), status=200) - - base_cisco_secure_x = self.tests_utils.get_first_api(GeneralTypeAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, - is_auth_api=True) - - for _ in base_cisco_secure_x.fetch_data(): - continue - - base_cisco_secure_x.update_start_date_filter() - - self.assertEqual('2021-10-05T10%3A10%3A11%2B00%3A00', base_cisco_secure_x.get_last_start_date()) - - def test_bad_config(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeAuthApiTests.BAD_CONFIG_FILE, - self.tests_utils.run_auth_api_process, - status=200, - sleep_time=1) - - requests_num, sent_logs_num, sent_bytes = queue.get() - - self.assertEqual(0, requests_num) - self.assertEqual(0, sent_logs_num) - self.assertEqual(0, sent_bytes) diff --git a/tests/general_type_oauth_api_tests.py b/tests/general_type_oauth_api_tests.py deleted file mode 100644 index 19a25b8..0000000 --- a/tests/general_type_oauth_api_tests.py +++ /dev/null @@ -1,210 +0,0 @@ -import unittest -import logging -import multiprocessing -import json -import math -from urllib.parse import unquote - -import httpretty - -from src.azure_graph import AzureGraph -from src.logzio_shipper import LogzioShipper -from .tests_utils import TestUtils - -logger = logging.getLogger(__name__) - - -class GeneralTypeOAuthApiTests(unittest.TestCase): - BASE_CONFIG_FILE = 'tests/config/general_type_oauth_api/base_config.yaml' - DAYS_BACK_FETCH_CONFIG_FILE = 'tests/config/general_type_oauth_api/days_back_fetch_config.yaml' - FILTERS_CONFIG_FILE = 'tests/config/general_type_oauth_api/filters_config.yaml' - CUSTOM_FIELDS_CONFIG_FILE = 'tests/config/general_type_oauth_api/custom_fields_config.yaml' - MULTIPLE_CONFIG_FILE = 'tests/config/general_type_oauth_api/multiple_config.yaml' - TIME_INTERVAL_CONFIG_FILE = 'tests/config/general_type_oauth_api/time_interval_config.yaml' - HTTP_REQUEST_HEADERS_CONFIG_FILE = 'tests/config/general_type_oauth_api/http_request_headers_config.yaml' - BAD_CONFIG_FILE = 'tests/config/general_type_oauth_api/bad_config.yaml' - AZURE_GRAPH_BODY_JSON = 'tests/api_body/azure_graph_body.json' - AZURE_GRAPH_TEST_URL = "https://graph.microsoft.com/v1.0/auditLogs/signIns" - AZURE_GRAPH_TEST_TOKEN = "1234-abcd-efgh-5678" - AZURE_GRAPH_TOKEN_TEST_URL = 'https://login.microsoftonline.com/<>/oauth2/v2.0/token' - AZURE_GRAPH_TOKEN_BODY_JSON = 'tests/api_body/azure_graph_token_body.json' - azure_graph_json_body: dict = None - - @classmethod - def setUpClass(cls) -> None: - with open(cls.AZURE_GRAPH_BODY_JSON, 'r') as json_file: - cls.azure_graph_json_body = json.load(json_file) - with open(cls.AZURE_GRAPH_TOKEN_BODY_JSON, 'r') as json_file: - cls.azure_graph_token_json_body = json.load(json_file) - - def setUp(self) -> None: - self.tests_utils = TestUtils("GET", self.AZURE_GRAPH_TEST_URL, - self.azure_graph_json_body, - "POST", self.AZURE_GRAPH_TOKEN_TEST_URL, - self.azure_graph_token_json_body) - - with open(self.tests_utils.LAST_START_DATES_FILE, 'w'): - pass - - def test_fetch_data(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_azure_graph_api_total_data_bytes_and_num( - GeneralTypeOAuthApiTests.BASE_CONFIG_FILE) - base_azure_graph = self.tests_utils.get_first_api(GeneralTypeOAuthApiTests.BASE_CONFIG_FILE, is_auth_api=False) - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - base_azure_graph) - - self.assertEqual(total_data_bytes, fetched_data_bytes) - self.assertEqual(total_data_num, fetched_data_num) - - def test_fetch_data_with_days_back_fetch(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_azure_graph_api_total_data_bytes_and_num( - GeneralTypeOAuthApiTests.DAYS_BACK_FETCH_CONFIG_FILE) - azure_graph_days_back_fetch = self.tests_utils.get_first_api( - GeneralTypeOAuthApiTests.DAYS_BACK_FETCH_CONFIG_FILE, is_auth_api=False) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - azure_graph_days_back_fetch) - - self.assertNotEqual(total_data_bytes, fetched_data_bytes) - self.assertNotEqual(total_data_num, fetched_data_num) - - def test_fetch_data_with_filters(self) -> None: - total_data_bytes, total_data_num = self.tests_utils.get_azure_graph_api_total_data_bytes_and_num( - GeneralTypeOAuthApiTests.FILTERS_CONFIG_FILE) - filters_cisco_secure_x = self.tests_utils.get_first_api(GeneralTypeOAuthApiTests.FILTERS_CONFIG_FILE, - is_auth_api=False) - - fetched_data_bytes, fetched_data_num = self.tests_utils.get_api_fetch_data_bytes_and_num( - filters_cisco_secure_x) - - self.assertNotEqual(total_data_bytes, fetched_data_bytes) - self.assertNotEqual(total_data_num, fetched_data_num) - - def test_sending_data(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - self.BASE_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_sending_data_with_http_request_headers(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeOAuthApiTests.HTTP_REQUEST_HEADERS_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_sending_data_iterations(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeOAuthApiTests.DAYS_BACK_FETCH_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=70) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) - self.assertEqual(data_num * 2, sent_logs_num) - self.assertEqual(data_bytes * 2, sent_bytes) - - def test_sending_data_multiple_general_type_apis(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeOAuthApiTests.MULTIPLE_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / 2 / LogzioShipper.MAX_BULK_SIZE_BYTES) * 2, requests_num) - self.assertEqual(data_num * 2, sent_logs_num) - self.assertEqual(data_bytes * 2, sent_bytes) - - def test_sending_data_with_custom_fields(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeOAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - custom_fields_azure_graph = self.tests_utils.get_first_api(GeneralTypeOAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, - is_auth_api=False) - data_bytes += data_num * self.tests_utils.get_api_custom_fields_bytes(custom_fields_azure_graph) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - def test_time_interval(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeOAuthApiTests.TIME_INTERVAL_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=70) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) - - @httpretty.activate - def test_last_start_date(self) -> None: - httpretty.register_uri(httpretty.GET, self.AZURE_GRAPH_TEST_URL, - body=json.dumps(GeneralTypeOAuthApiTests.azure_graph_json_body), status=200) - base_azure_graph = self.tests_utils.get_first_api(GeneralTypeOAuthApiTests.CUSTOM_FIELDS_CONFIG_FILE, - is_auth_api=False) - httpretty.register_uri(httpretty.POST, base_azure_graph.get_token_request.url, - body=json.dumps(self.azure_graph_token_json_body), status=200) - - for _ in base_azure_graph.fetch_data(): - continue - - base_azure_graph.update_start_date_filter() - - self.assertEqual('2020-03-13T19:15:42.619583+00:00', unquote(base_azure_graph.get_last_start_date())) - - def test_bad_config(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - GeneralTypeOAuthApiTests.BAD_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=1) - - requests_num, sent_logs_num, sent_bytes = queue.get() - - self.assertEqual(0, requests_num) - self.assertEqual(0, sent_logs_num) - self.assertEqual(0, sent_bytes) diff --git a/tests/logging_config.ini b/tests/logging_config.ini deleted file mode 100644 index cecd031..0000000 --- a/tests/logging_config.ini +++ /dev/null @@ -1,21 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=stream_handler - -[formatters] -keys=formatter - -[logger_root] -level=DEBUG -handlers=stream_handler - -[handler_stream_handler] -class=StreamHandler -level=DEBUG -formatter=formatter -args=(sys.stderr,) - -[formatter_formatter] -format=%(asctime)s %(name)-12s %(levelname)-8s %(message)s \ No newline at end of file diff --git a/tests/testConfigs/azure_api_conf.yaml b/tests/testConfigs/azure_api_conf.yaml new file mode 100644 index 0000000..1d44bf6 --- /dev/null +++ b/tests/testConfigs/azure_api_conf.yaml @@ -0,0 +1,16 @@ +apis: + - name: azure test + type: azure_graph + azure_ad_tenant_id: <> + azure_ad_client_id: <> + azure_ad_secret_value: <> + data_request: + url: https://graph.microsoft.com/v1.0/auditLogs/signIns + additional_fields: + type: azure_graph_shipping_test + days_back_fetch: 8 + scrape_interval: 1 + +logzio: + url: https://listener.logz.io:8071 + token: SHipPIngtoKen diff --git a/tests/testConfigs/invalid_conf.yaml b/tests/testConfigs/invalid_conf.yaml new file mode 100644 index 0000000..7796f9d --- /dev/null +++ b/tests/testConfigs/invalid_conf.yaml @@ -0,0 +1,40 @@ +api: + - name: logz-api + type: general + url: https://api.logz.io/v1/scroll + headers: + X-API-TOKEN: lOgzapiToKEn + CONTENT-TYPE: application/json + pagination: + type: body + body_format: {"scroll_id": "{res.scrollId}"} + stop_indication: + field: "hits" + condition: "contains" + value: "\"hits\":[]" + max_calls: 20 + body: { + "size": 1000, + "query": { + "bool": { + "must": [ + { + "query_string": { + "query": "type:lambda-extension-logs" + } + } + ] + } + } + } + method: POST + response_data_path: hits + additional_fields: + field1: value1 + field2: 123 + field3: another value + type: logz-api + +logzio: + url: https://listener.logz.io:8071 + token: SHipPIngtoKen diff --git a/tests/testConfigs/missing_logz_output_conf.yaml b/tests/testConfigs/missing_logz_output_conf.yaml new file mode 100644 index 0000000..2f91e58 --- /dev/null +++ b/tests/testConfigs/missing_logz_output_conf.yaml @@ -0,0 +1,43 @@ +apis: + - name: logz-api + type: general + url: https://api.logz.io/v1/scroll + headers: + X-API-TOKEN: lOgzapiToKEn + CONTENT-TYPE: application/json + pagination: + type: body + body_format: {"scroll_id": "{res.scrollId}"} + stop_indication: + field: "hits" + condition: "contains" + value: "\"hits\":[]" + max_calls: 20 + body: { + "size": 1000, + "query": { + "bool": { + "must": [ + { + "query_string": { + "query": "type:lambda-extension-logs" + } + } + ] + } + } + } + method: POST + response_data_path: hits + additional_fields: + field1: value1 + field2: 123 + field3: another value + type: logz-api + - name: cloudflare test + type: cloudflare + cloudflare_account_id: c10u5f1ar3acc0un7i6 + cloudflare_bearer_token: b3ar3r-t0k3n + url: https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history?since=2024-06-09T14:06:23.635421Z + next_url: https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history?since={res.result.[0].sent} + scrape_interval: 5 diff --git a/tests/testConfigs/multiple_apis_conf.yaml b/tests/testConfigs/multiple_apis_conf.yaml new file mode 100644 index 0000000..4b10307 --- /dev/null +++ b/tests/testConfigs/multiple_apis_conf.yaml @@ -0,0 +1,47 @@ +apis: + - name: logz-api + type: general + url: https://api.logz.io/v1/scroll + headers: + X-API-TOKEN: lOgzapiToKEn + CONTENT-TYPE: application/json + pagination: + type: body + body_format: {"scroll_id": "{res.scrollId}"} + stop_indication: + field: "hits" + condition: "contains" + value: "\"hits\":[]" + max_calls: 20 + body: { + "size": 1000, + "query": { + "bool": { + "must": [ + { + "query_string": { + "query": "type:lambda-extension-logs" + } + } + ] + } + } + } + method: POST + response_data_path: hits + additional_fields: + field1: value1 + field2: 123 + field3: another value + type: logz-api + - name: cloudflare test + type: cloudflare + cloudflare_account_id: c10u5f1ar3acc0un7i6 + cloudflare_bearer_token: b3ar3r-t0k3n + url: https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history?since=2024-06-09T14:06:23.635421Z + next_url: https://api.cloudflare.com/client/v4/accounts/{account_id}/alerting/v3/history?since={res.result.[0].sent} + scrape_interval: 5 + +logzio: + url: https://listener.logz.io:8071 + token: SHipPIngtoKen diff --git a/tests/tests_utils.py b/tests/tests_utils.py deleted file mode 100644 index 3ff75ec..0000000 --- a/tests/tests_utils.py +++ /dev/null @@ -1,229 +0,0 @@ -import logging -import httpretty -import multiprocessing -import requests -import json -import gzip -import time -import os -import signal - -from typing import Callable -from logging.config import fileConfig -from src.apis_manager import ApisManager -from src.api import Api -from src.azure_graph import AzureGraph -from src.config_reader import ConfigReader -from src.general_auth_api import GeneralAuthApi -from src.cisco_secure_x import CiscoSecureX -from src.oauth_api import OAuthApi - -fileConfig('tests/logging_config.ini', disable_existing_loggers=False) -logger = logging.getLogger(__name__) - - -class TestUtils: - LOGZIO_HTTPPRETTY_URL = 'https://listener.logz.io:8071/?token=123456789a&type=api_fetcher' - LOGZIO_URL = 'https://listener.logz.io:8071' - LOGZIO_TOKEN = '123456789a' - LAST_START_DATES_FILE = 'tests/last_start_dates.txt' - - def __init__(self, api_http_method: str, api_url: str, api_body: dict, token_http_method: str = None, - token_url: str = None, token_body: dict = None, second_http_method: str = None, - second_api_url: str = None, second_api_body: dict = None) -> None: - self.api_http_method = api_http_method - self.api_url = api_url - self.api_body = api_body - self.token_http_method = token_http_method - self.token_url = token_url - self.token_body = token_body - self.second_http_method = second_http_method - self.second_api_url = second_api_url - self.second_api_body = second_api_body - - def start_process_and_wait_until_finished(self, queue: multiprocessing.Queue, config_file: str, - delegate: Callable[[str], None], status: int, sleep_time: int, - is_multi_test: bool = False) -> None: - process = multiprocessing.Process(target=delegate, args=(config_file, status, queue, is_multi_test)) - process.start() - - time.sleep(sleep_time) - os.kill(process.pid, signal.SIGTERM) - process.join() - - @httpretty.activate - def run_auth_api_process(self, config_file: str, status: int, queue: multiprocessing.Queue, - is_multi_test: bool) -> None: - httpretty.register_uri(self.api_http_method, self.api_url, body=json.dumps(self.api_body), status=200) - httpretty.register_uri(httpretty.POST, TestUtils.LOGZIO_URL, status=status) - - ApisManager.CONFIG_FILE = config_file - ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE - logzio_requests = [] - - ApisManager().run() - - for request in httpretty.latest_requests(): - if request.url.startswith(self.api_url): - continue - - logzio_requests.append(request) - - queue.put(self._get_sending_data_results(logzio_requests)) - - @httpretty.activate - def run_oauth_api_process(self, config_file: str, status: int, queue: multiprocessing.Queue, - is_multi_test: bool) -> None: - from tests.azure_graph_api_tests import AzureGraphApiTests - httpretty.register_uri(httpretty.POST, TestUtils.LOGZIO_URL, status=status) - httpretty.register_uri(self.token_http_method, - self.token_url, - body=json.dumps(self.token_body)) - httpretty.register_uri(self.api_http_method, self.api_url, body=json.dumps(self.api_body), status=200, - headers={AzureGraph.OAUTH_AUTHORIZATION_HEADER: - AzureGraphApiTests.AZURE_GRAPH_TEST_TOKEN}) - if is_multi_test: - httpretty.register_uri(self.second_http_method, self.second_api_url, body=json.dumps(self.second_api_body), - status=200) - - ApisManager.CONFIG_FILE = config_file - ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE - logzio_requests = [] - - ApisManager().run() - - for request in httpretty.latest_requests(): - if request.url.startswith(self.api_url): - continue - - logzio_requests.append(request) - - queue.put(self._get_sending_data_results(logzio_requests)) - - def get_first_api(self, config_file: str, is_auth_api: bool) -> Api: - base_config_reader = ConfigReader(config_file, ApisManager.API_GENERAL_TYPE, - ApisManager.AUTH_API_TYPES, ApisManager.OAUTH_API_TYPES) - - if is_auth_api: - for auth_api_data in base_config_reader.get_auth_apis_data(): - if auth_api_data.base_data.base_data.type == ApisManager.API_GENERAL_TYPE: - return GeneralAuthApi(auth_api_data.base_data, auth_api_data.general_type_data) - - return CiscoSecureX(auth_api_data.base_data) - - for oauth_api_data in base_config_reader.get_oauth_apis_data(): - if oauth_api_data.base_data.base_data.type == ApisManager.API_GENERAL_TYPE: - return OAuthApi(oauth_api_data.base_data, oauth_api_data.general_type_data) - - return AzureGraph(oauth_api_data) - - def get_cisco_secure_x_api_total_data_bytes_and_num(self, config_file: str) -> tuple[int, int]: - url = CiscoSecureX.URL - config_reader = ConfigReader(config_file, ApisManager.API_GENERAL_TYPE, - ApisManager.AUTH_API_TYPES, ApisManager.OAUTH_API_TYPES) - cisco_secure_x = None - total_data_bytes = 0 - total_data_num = 0 - - for auth_api_data in config_reader.get_auth_apis_data(): - cisco_secure_x = CiscoSecureX(auth_api_data.base_data) - - while True: - response = requests.get(url=url, auth=(cisco_secure_x.base_data.credentials.id, - cisco_secure_x.base_data.credentials.key)) - json_data = json.loads(response.content) - - data_bytes, data_num = self.get_api_data_bytes_and_num_from_json_data(json_data['data']) - total_data_bytes += data_bytes - total_data_num += data_num - - next_url = json_data['metadata']['links'].get('next') - - if next_url is None: - break - - url = next_url - - return total_data_bytes, total_data_num - - def get_azure_graph_api_total_data_bytes_and_num(self, config_file: str) -> tuple[int, int]: - config_reader = ConfigReader(config_file, ApisManager.API_GENERAL_TYPE, - ApisManager.AUTH_API_TYPES, ApisManager.OAUTH_API_TYPES) - azure_graph = None - total_data_bytes = 0 - total_data_num = 0 - - for oauth_api_data in config_reader.get_oauth_apis_data(): - azure_graph = AzureGraph(oauth_api_data) - token, token_expire = azure_graph.get_token() - - url = self.api_url - while True: - response = requests.get(url=url, headers={OAuthApi.OAUTH_AUTHORIZATION_HEADER: - token, - AzureGraph.OAUTH_TOKEN_REQUEST_CONTENT_TYPE: AzureGraph.OAUTH_APPLICATION_JSON_CONTENT_TYPE}) - json_data = json.loads(response.content) - data_bytes, data_num = self.get_api_data_bytes_and_num_from_json_data( - json_data[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - total_data_bytes += data_bytes - total_data_num += data_num - next_url = json_data.get(AzureGraph.NEXT_LINK) - - if next_url is None: - break - - url = next_url - - return total_data_bytes, total_data_num - - def get_api_custom_fields_bytes(self, api: Api) -> int: - custom_fields: dict = {} - - for custom_field in api.base_data.custom_fields: - custom_fields[custom_field.key] = custom_field.value - - return len(json.dumps(custom_fields)) - - def get_last_start_dates_file_lines(self) -> list[str]: - with open(TestUtils.LAST_START_DATES_FILE, 'r') as file: - file_lines = file.readlines() - - return file_lines - - def get_api_fetch_data_bytes_and_num(self, api: Api) -> tuple[int, int]: - fetched_data_bytes = 0 - fetched_data_num = 0 - for data in api.fetch_data(): - fetched_data_bytes += len(data) - fetched_data_num += 1 - - return fetched_data_bytes, fetched_data_num - - def get_api_data_bytes_and_num_from_json_data(self, json_data: list) -> tuple[int, int]: - data_num = 0 - data_bytes = 0 - for data in json_data: - data_num += 1 - data_bytes += len(json.dumps(data)) - - return data_bytes, data_num - - def _get_sending_data_results(self, latest_requests: list) -> tuple[int, int, int]: - requests_num = 0 - sent_logs_num = 0 - sent_bytes = 0 - - for request in latest_requests: - if request.url == self.LOGZIO_HTTPPRETTY_URL: - requests_num += 1 - - try: - decompressed_gzip = gzip.decompress(request.parsed_body).splitlines() - except TypeError: - continue - - for log in decompressed_gzip: - sent_logs_num += 1 - sent_bytes += len(log) - - return int(requests_num / 2), int(sent_logs_num / 2), int(sent_bytes / 2)