diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..3105d1a --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,30 @@ +name: Build and Deploy to dockerhub +on: + workflow_dispatch: + release: + types: [published] +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: logzio/logzio-api-fetcher + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..0d3d92a --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,33 @@ +name: Automatic tests and code-coverage +on: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Add credentials to config files + run: | + cd tests + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_ID }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_KEY }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_ID }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_VALUE }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_CLIENT_ID }}/g' @ + grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_TENANT_ID }}/g' @ + - name: Install dependencies + run: | + pip install -r requirements.txt + pip install pytest-cov + pip install pytest + - name: Run unit tests + run: | + pytest --log-cli-level=debug tests/*_tests.py diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml deleted file mode 100644 index 965d42a..0000000 --- a/.github/workflows/workflow.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: Automatic tests, code-coverage and -on: - workflow_dispatch: - push: - branches: - - main -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Add credentials to config files - run: | - cd tests - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_ID }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.CISCO_SECURE_X_API_KEY }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_ID }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_VALUE }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_CLIENT_ID }}/g' @ - grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_TENANT_ID }}/g' @ - - name: Run unit tests - run: | - pip install pytest - pip install httpretty - pip install requests - pip install pyyaml - pip install jsonpath-ng - pip install python-dateutil - pip install pytest-cov - pytest --cov-report xml:code_coverage.xml --cov=src tests/*_tests.py - - name: Code-coverage - run: | - # Get line-rate - line_rate=$(head -2 code_coverage.xml | tail -1 | egrep -o "line-rate=\"[0-1]\.?[0-9]*\"" | egrep -o "[0-1]\.?[0-9]*") - - # Print line-rate - echo | awk -v num=$line_rate '{ printf "line-rate: %d%\n", (num * 100) }' - - # Check code-coverage conditions - echo | awk -v num=$line_rate '{ if (num < 0.8) { printf "line-rate is less than 80%"; exit 1 } else { exit 0 }}' - exit_code=$? - if [ $exit_code -eq 1 ]; then - exit 1 - fi - - name: Log in to Docker Hub - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: my-docker-hub-namespace/my-docker-hub-repository - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/requirements.txt b/requirements.txt index 94b0ab9..7519ddc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -pyyaml -requests -urllib3 -python-dateutil -jsonpath-ng \ No newline at end of file +pyyaml~=6.0.1 +requests~=2.31.0 +urllib3~=2.2.1 +python-dateutil~=2.9.0.post0 +jsonpath-ng +httpretty~=1.1.4 \ No newline at end of file diff --git a/src/apis_manager.py b/src/apis_manager.py index 05470ed..45b4b8e 100644 --- a/src/apis_manager.py +++ b/src/apis_manager.py @@ -2,6 +2,7 @@ import os import signal import threading + import requests from typing import Optional @@ -33,12 +34,13 @@ class ApisManager: AUTH_API_TYPES = [API_GENERAL_TYPE, API_CISCO_SECURE_X_TYPE] OAUTH_API_TYPES = [API_GENERAL_TYPE, API_AZURE_GRAPH_TYPE, API_AZURE_MAIL_REPORTS_TYPE] - def __init__(self) -> None: + def __init__(self, test=False) -> None: self._apis: list[Api] = [] self._logzio_connection: Optional[LogzioConnection] = None self._threads = [] self._event = threading.Event() self._lock = threading.Lock() + self.test = test def run(self) -> None: if not self._read_data_from_config(): @@ -107,7 +109,8 @@ def _run_api_scheduled_task(self, api: Api) -> None: thread.start() thread.join() - if self._event.wait(timeout=api.get_api_time_interval()): + if self._event.wait(timeout=api.get_api_time_interval()) or self.test: + logger.info("TEST: stopping shipping due to test or timeout") break def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None: diff --git a/src/oauth_api.py b/src/oauth_api.py index 8e2ea83..21ff475 100644 --- a/src/oauth_api.py +++ b/src/oauth_api.py @@ -131,4 +131,3 @@ def get_token_request(self): def _set_current_data_last_date(self, date): if date: self._current_data_last_date = date - diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index 784f67f..017a74b 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -88,8 +88,11 @@ def test_sending_data(self) -> None: self.tests_utils.run_oauth_api_process, status=200, sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() + if queue.empty(): + logger.info("TEST: queue empty") + else: + logger.info("TEST: queue not empty") + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -105,7 +108,7 @@ def test_sending_data_iterations(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -121,7 +124,7 @@ def test_sending_data_multiple_azure_graph(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -137,7 +140,7 @@ def test_sending_data_with_custom_fields(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) custom_fields_azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE, @@ -156,7 +159,7 @@ def test_time_interval(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -188,7 +191,7 @@ def test_bad_config(self) -> None: status=200, sleep_time=1) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) self.assertEqual(0, requests_num) self.assertEqual(0, sent_logs_num) diff --git a/tests/config/azure_graph/base_config.yaml b/tests/config/azure_graph/base_config.yaml index 3257506..9298a76 100644 --- a/tests/config/azure_graph/base_config.yaml +++ b/tests/config/azure_graph/base_config.yaml @@ -24,3 +24,4 @@ oauth_apis: time_interval: 1 days_back_fetch: 30 start_date_name: createdDateTime + end_date_name: EndDate diff --git a/tests/config/azure_graph/custom_fields_config.yaml b/tests/config/azure_graph/custom_fields_config.yaml index 2b56f57..ab5b0b2 100644 --- a/tests/config/azure_graph/custom_fields_config.yaml +++ b/tests/config/azure_graph/custom_fields_config.yaml @@ -20,6 +20,7 @@ oauth_apis: url: https://graph.microsoft.com/v1.0/auditLogs/signIns method: GET start_date_name: createdDateTime + end_date_name: EndDate json_paths: data_date: createdDateTime settings: diff --git a/tests/config/azure_graph/days_back_fetch_config.yaml b/tests/config/azure_graph/days_back_fetch_config.yaml index 0f2f243..6420cc9 100644 --- a/tests/config/azure_graph/days_back_fetch_config.yaml +++ b/tests/config/azure_graph/days_back_fetch_config.yaml @@ -20,6 +20,7 @@ oauth_apis: url: https://graph.microsoft.com/v1.0/auditLogs/signIns method: GET start_date_name: createdDateTime + end_date_name: EndDate json_paths: data_date: createdDateTime additional_filters: diff --git a/tests/config/azure_graph/filters_config.yaml b/tests/config/azure_graph/filters_config.yaml index ab0e731..182b0ea 100644 --- a/tests/config/azure_graph/filters_config.yaml +++ b/tests/config/azure_graph/filters_config.yaml @@ -20,6 +20,7 @@ oauth_apis: url: https://graph.microsoft.com/v1.0/auditLogs/signIns method: GET start_date_name: createdDateTime + end_date_name: EndDate json_paths: data_date: createdDateTime filters: diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 3ff75ec..800a34a 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -24,6 +24,7 @@ class TestUtils: LOGZIO_HTTPPRETTY_URL = 'https://listener.logz.io:8071/?token=123456789a&type=api_fetcher' + LOGZIO_HTTPPRETTY_URL2 = 'https://listener.logz.io:8071/?token=123456789a' LOGZIO_URL = 'https://listener.logz.io:8071' LOGZIO_TOKEN = '123456789a' LAST_START_DATES_FILE = 'tests/last_start_dates.txt' @@ -76,9 +77,7 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce is_multi_test: bool) -> None: from tests.azure_graph_api_tests import AzureGraphApiTests httpretty.register_uri(httpretty.POST, TestUtils.LOGZIO_URL, status=status) - httpretty.register_uri(self.token_http_method, - self.token_url, - body=json.dumps(self.token_body)) + httpretty.register_uri(self.token_http_method, self.token_url, body=json.dumps(self.token_body)) httpretty.register_uri(self.api_http_method, self.api_url, body=json.dumps(self.api_body), status=200, headers={AzureGraph.OAUTH_AUTHORIZATION_HEADER: AzureGraphApiTests.AZURE_GRAPH_TEST_TOKEN}) @@ -90,14 +89,22 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE logzio_requests = [] - ApisManager().run() + # logger.info("TEST: starting API Manager") + # p = multiprocessing.Process(target=ApisManager().run) + ApisManager(True).run() # test_sending_data is stuck here, never reaching after this line + # p.start() + # time.sleep(2) + # p.kill() + logger.info("TEST: Finished API Manager!") + # this now doesn't work, because the requests are made in a separate process for request in httpretty.latest_requests(): + logger.info("TEST: reading requests") if request.url.startswith(self.api_url): continue logzio_requests.append(request) - + logger.info("TEST: putting data in the queue") queue.put(self._get_sending_data_results(logzio_requests)) def get_first_api(self, config_file: str, is_auth_api: bool) -> Api: @@ -214,7 +221,8 @@ def _get_sending_data_results(self, latest_requests: list) -> tuple[int, int, in sent_bytes = 0 for request in latest_requests: - if request.url == self.LOGZIO_HTTPPRETTY_URL: + logger.info(f"TEST: checking request {request}") + if request.url in (self.LOGZIO_HTTPPRETTY_URL, self.LOGZIO_HTTPPRETTY_URL2): requests_num += 1 try: @@ -226,4 +234,5 @@ def _get_sending_data_results(self, latest_requests: list) -> tuple[int, int, in sent_logs_num += 1 sent_bytes += len(log) + logger.info(f"TEST: _get_sending_data_results params {requests_num}, {sent_logs_num}, {sent_bytes}") return int(requests_num / 2), int(sent_logs_num / 2), int(sent_bytes / 2)