Dev workflows

.github/workflows/deploy.yaml

@@ -0,0 +1,30 @@
+name: Build and Deploy to dockerhub
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: logzio/logzio-api-fetcher
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/tests.yaml

@@ -0,0 +1,33 @@
+name: Automatic tests and code-coverage
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Add credentials to config files
+        run: |
+          cd tests
+          grep -rli '<<CISCO_SECURE_X_API_ID>>' * | xargs -i@ sed -i 's/<<CISCO_SECURE_X_API_ID>>/${{ secrets.CISCO_SECURE_X_API_ID }}/g' @
+          grep -rli '<<CISCO_SECURE_X_API_KEY>>' * | xargs -i@ sed -i 's/<<CISCO_SECURE_X_API_KEY>>/${{ secrets.CISCO_SECURE_X_API_KEY }}/g' @
+          grep -rli '<<AZURE_AD_SECRET_ID>>' * | xargs -i@ sed -i 's/<<AZURE_AD_SECRET_ID>>/${{ secrets.AZURE_AD_SECRET_ID }}/g' @
+          grep -rli '<<AZURE_AD_SECRET_VALUE>>' * | xargs -i@ sed -i 's/<<AZURE_AD_SECRET_VALUE>>/${{ secrets.AZURE_AD_SECRET_VALUE }}/g' @
+          grep -rli '<<CLIENT_ID>>' * | xargs -i@ sed -i 's/<<CLIENT_ID>>/${{ secrets.AZURE_AD_CLIENT_ID }}/g' @
+          grep -rli '<<AZURE_AD_TENANT_ID>>' * | xargs -i@ sed -i 's/<<AZURE_AD_TENANT_ID>>/${{ secrets.AZURE_AD_TENANT_ID }}/g' @
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install pytest-cov
+          pip install pytest
+      - name: Run unit tests
+        run: |
+          pytest --log-cli-level=debug tests/*_tests.py

requirements.txt

@@ -1,5 +1,6 @@
-pyyaml
-requests
-urllib3
-python-dateutil
-jsonpath-ng
+pyyaml~=6.0.1
+requests~=2.31.0
+urllib3~=2.2.1
+python-dateutil~=2.9.0.post0
+jsonpath-ng
+httpretty~=1.1.4

src/apis_manager.py

@@ -2,6 +2,7 @@
 import os
 import signal
 import threading
+
 import requests
 
 from typing import Optional
@@ -33,12 +34,13 @@ class ApisManager:
     AUTH_API_TYPES = [API_GENERAL_TYPE, API_CISCO_SECURE_X_TYPE]
     OAUTH_API_TYPES = [API_GENERAL_TYPE, API_AZURE_GRAPH_TYPE, API_AZURE_MAIL_REPORTS_TYPE]
 
-    def __init__(self) -> None:
+    def __init__(self, test=False) -> None:
         self._apis: list[Api] = []
         self._logzio_connection: Optional[LogzioConnection] = None
         self._threads = []
         self._event = threading.Event()
         self._lock = threading.Lock()
+        self.test = test
 
     def run(self) -> None:
         if not self._read_data_from_config():
@@ -107,7 +109,8 @@ def _run_api_scheduled_task(self, api: Api) -> None:
             thread.start()
             thread.join()
 
-            if self._event.wait(timeout=api.get_api_time_interval()):
+            if self._event.wait(timeout=api.get_api_time_interval()) or self.test:
+                logger.info("TEST: stopping shipping due to test or timeout")
                 break
 
     def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None:

src/oauth_api.py

@@ -131,4 +131,3 @@ def get_token_request(self):
     def _set_current_data_last_date(self, date):
         if date:
             self._current_data_last_date = date
-

tests/azure_graph_api_tests.py

@@ -88,8 +88,11 @@ def test_sending_data(self) -> None:
                                                                self.tests_utils.run_oauth_api_process,
                                                                status=200,
                                                                sleep_time=10)
-
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        if queue.empty():
+            logger.info("TEST: queue empty")
+        else:
+            logger.info("TEST: queue not empty")
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
 
@@ -105,7 +108,7 @@ def test_sending_data_iterations(self) -> None:
                                                                status=200,
                                                                sleep_time=70)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
 
@@ -121,7 +124,7 @@ def test_sending_data_multiple_azure_graph(self) -> None:
                                                                status=200,
                                                                sleep_time=10)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
 
@@ -137,7 +140,7 @@ def test_sending_data_with_custom_fields(self) -> None:
                                                                status=200,
                                                                sleep_time=10)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
         custom_fields_azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE,
@@ -156,7 +159,7 @@ def test_time_interval(self) -> None:
                                                                status=200,
                                                                sleep_time=70)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
         data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data(
             self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK])
 
@@ -188,7 +191,7 @@ def test_bad_config(self) -> None:
                                                                status=200,
                                                                sleep_time=1)
 
-        requests_num, sent_logs_num, sent_bytes = queue.get()
+        requests_num, sent_logs_num, sent_bytes = queue.get(False)
 
         self.assertEqual(0, requests_num)
         self.assertEqual(0, sent_logs_num)

tests/config/azure_graph/base_config.yaml

@@ -24,3 +24,4 @@ oauth_apis:
       time_interval: 1
       days_back_fetch: 30
     start_date_name: createdDateTime
+    end_date_name: EndDate

tests/config/azure_graph/custom_fields_config.yaml

@@ -20,6 +20,7 @@ oauth_apis:
       url: https://graph.microsoft.com/v1.0/auditLogs/signIns
       method: GET
     start_date_name: createdDateTime
+    end_date_name: EndDate
     json_paths:
       data_date: createdDateTime
     settings:

tests/config/azure_graph/days_back_fetch_config.yaml

@@ -20,6 +20,7 @@ oauth_apis:
       url: https://graph.microsoft.com/v1.0/auditLogs/signIns
       method: GET
     start_date_name: createdDateTime
+    end_date_name: EndDate
     json_paths:
       data_date: createdDateTime
     additional_filters:

tests/config/azure_graph/filters_config.yaml

@@ -20,6 +20,7 @@ oauth_apis:
       url: https://graph.microsoft.com/v1.0/auditLogs/signIns
       method: GET
     start_date_name: createdDateTime
+    end_date_name: EndDate
     json_paths:
       data_date: createdDateTime
     filters:

tests/tests_utils.py

@@ -24,6 +24,7 @@
 
 class TestUtils:
     LOGZIO_HTTPPRETTY_URL = 'https://listener.logz.io:8071/?token=123456789a&type=api_fetcher'
+    LOGZIO_HTTPPRETTY_URL2 = 'https://listener.logz.io:8071/?token=123456789a'
     LOGZIO_URL = 'https://listener.logz.io:8071'
     LOGZIO_TOKEN = '123456789a'
     LAST_START_DATES_FILE = 'tests/last_start_dates.txt'
@@ -76,9 +77,7 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce
                               is_multi_test: bool) -> None:
         from tests.azure_graph_api_tests import AzureGraphApiTests
         httpretty.register_uri(httpretty.POST, TestUtils.LOGZIO_URL, status=status)
-        httpretty.register_uri(self.token_http_method,
-                               self.token_url,
-                               body=json.dumps(self.token_body))
+        httpretty.register_uri(self.token_http_method, self.token_url, body=json.dumps(self.token_body))
         httpretty.register_uri(self.api_http_method, self.api_url, body=json.dumps(self.api_body), status=200,
                                headers={AzureGraph.OAUTH_AUTHORIZATION_HEADER:
                                             AzureGraphApiTests.AZURE_GRAPH_TEST_TOKEN})
@@ -90,14 +89,22 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce
         ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE
         logzio_requests = []
 
-        ApisManager().run()
+        # logger.info("TEST: starting API Manager")
+        # p = multiprocessing.Process(target=ApisManager().run)
+        ApisManager(True).run()  # test_sending_data is stuck here, never reaching after this line
+        # p.start()
+        # time.sleep(2)
+        # p.kill()
+        logger.info("TEST: Finished API Manager!")
 
+        # this now doesn't work, because the requests are made in a separate process
         for request in httpretty.latest_requests():
+            logger.info("TEST: reading requests")
             if request.url.startswith(self.api_url):
                 continue
 
             logzio_requests.append(request)
-
+        logger.info("TEST: putting data in the queue")
         queue.put(self._get_sending_data_results(logzio_requests))
 
     def get_first_api(self, config_file: str, is_auth_api: bool) -> Api:
@@ -214,7 +221,8 @@ def _get_sending_data_results(self, latest_requests: list) -> tuple[int, int, in
         sent_bytes = 0
 
         for request in latest_requests:
-            if request.url == self.LOGZIO_HTTPPRETTY_URL:
+            logger.info(f"TEST: checking request {request}")
+            if request.url in (self.LOGZIO_HTTPPRETTY_URL, self.LOGZIO_HTTPPRETTY_URL2):
                 requests_num += 1
 
                 try:
@@ -226,4 +234,5 @@ def _get_sending_data_results(self, latest_requests: list) -> tuple[int, int, in
                     sent_logs_num += 1
                     sent_bytes += len(log)
 
+        logger.info(f"TEST: _get_sending_data_results params {requests_num}, {sent_logs_num}, {sent_bytes}")
         return int(requests_num / 2), int(sent_logs_num / 2), int(sent_bytes / 2)