From 23bb1a4fcd88420ccef0b9ae8b91d813a8475e5c Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Tue, 12 Mar 2024 12:51:30 +0700 Subject: [PATCH 01/29] Add `httpretty~=1.1.4` --- requirements.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 94b0ab9..7519ddc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -pyyaml -requests -urllib3 -python-dateutil -jsonpath-ng \ No newline at end of file +pyyaml~=6.0.1 +requests~=2.31.0 +urllib3~=2.2.1 +python-dateutil~=2.9.0.post0 +jsonpath-ng +httpretty~=1.1.4 \ No newline at end of file From cd629141c59c36b0d734525c945f0b142ec36ce8 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Tue, 12 Mar 2024 12:51:40 +0700 Subject: [PATCH 02/29] Update workflows --- .github/workflows/deploy.yaml | 30 +++++++++++++++++++ .../workflows/{workflow.yaml => tests.yaml} | 6 ++-- 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/deploy.yaml rename .github/workflows/{workflow.yaml => tests.yaml} (97%) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..f55e1e6 --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,30 @@ +name: Build and Deploy to dockerhub +on: + workflow_dispatch: + release: + types: [published] +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: logzio/logzio-api-fetcher + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/workflow.yaml b/.github/workflows/tests.yaml similarity index 97% rename from .github/workflows/workflow.yaml rename to .github/workflows/tests.yaml index 965d42a..e7f2831 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/tests.yaml @@ -1,9 +1,11 @@ -name: Automatic tests, code-coverage and +name: Automatic tests and code-coverage on: - workflow_dispatch: push: branches: - main + pull_request: + branches: + - main jobs: deploy: runs-on: ubuntu-latest From eb1e57d1b9097ac1188cb6bb3deb327ce55d8838 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Tue, 12 Mar 2024 12:53:56 +0700 Subject: [PATCH 03/29] Update workflows --- .github/workflows/tests.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e7f2831..99f385b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -47,20 +47,3 @@ jobs: if [ $exit_code -eq 1 ]; then exit 1 fi - - name: Log in to Docker Hub - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: my-docker-hub-namespace/my-docker-hub-repository - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} From 16cf53c56f435616f3806592b31fac1ed2773328 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Tue, 12 Mar 2024 13:00:17 +0700 Subject: [PATCH 04/29] Update `tests` workflow --- .github/workflows/tests.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 99f385b..ed8c545 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -7,13 +7,13 @@ on: branches: - main jobs: - deploy: + tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: '3.9' - name: Add credentials to config files run: | cd tests @@ -23,7 +23,7 @@ jobs: grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_SECRET_VALUE }}/g' @ grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_CLIENT_ID }}/g' @ grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_TENANT_ID }}/g' @ - - name: Run unit tests + - name: Install dependencies run: | pip install pytest pip install httpretty @@ -32,6 +32,8 @@ jobs: pip install jsonpath-ng pip install python-dateutil pip install pytest-cov + - name: Run unit tests + run: | pytest --cov-report xml:code_coverage.xml --cov=src tests/*_tests.py - name: Code-coverage run: | From 40be8ec2956b0b9ee5ffb0589321b12cf69f31d4 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Wed, 13 Mar 2024 12:40:53 +0700 Subject: [PATCH 05/29] Update workflows --- .github/workflows/deploy.yaml | 6 +++--- .github/workflows/tests.yaml | 23 ++--------------------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index f55e1e6..3105d1a 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -7,10 +7,10 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: '3.9' - name: Log in to Docker Hub uses: docker/login-action@v3 with: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ed8c545..cd68ea4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -25,27 +25,8 @@ jobs: grep -rli '<>' * | xargs -i@ sed -i 's/<>/${{ secrets.AZURE_AD_TENANT_ID }}/g' @ - name: Install dependencies run: | - pip install pytest - pip install httpretty - pip install requests - pip install pyyaml - pip install jsonpath-ng - pip install python-dateutil - pip install pytest-cov + pip install -r requirements.txt - name: Run unit tests run: | - pytest --cov-report xml:code_coverage.xml --cov=src tests/*_tests.py - - name: Code-coverage - run: | - # Get line-rate - line_rate=$(head -2 code_coverage.xml | tail -1 | egrep -o "line-rate=\"[0-1]\.?[0-9]*\"" | egrep -o "[0-1]\.?[0-9]*") - - # Print line-rate - echo | awk -v num=$line_rate '{ printf "line-rate: %d%\n", (num * 100) }' + pytest tests/*_tests.py - # Check code-coverage conditions - echo | awk -v num=$line_rate '{ if (num < 0.8) { printf "line-rate is less than 80%"; exit 1 } else { exit 0 }}' - exit_code=$? - if [ $exit_code -eq 1 ]; then - exit 1 - fi From 1950b66d68d6728329e10f0c87fb95008e423aa1 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Wed, 13 Mar 2024 12:46:42 +0700 Subject: [PATCH 06/29] Add dev deps --- .github/workflows/tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index cd68ea4..b900bbe 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -26,7 +26,8 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt + pip install pytest-cov + pip install pytest - name: Run unit tests run: | pytest tests/*_tests.py - From f283b5874e332f140abd776844f1ae06ee1e8d35 Mon Sep 17 00:00:00 2001 From: Yotam loewenbach Date: Wed, 13 Mar 2024 13:31:32 +0700 Subject: [PATCH 07/29] add pytest debug logs --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b900bbe..0d3d92a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -30,4 +30,4 @@ jobs: pip install pytest - name: Run unit tests run: | - pytest tests/*_tests.py + pytest --log-cli-level=debug tests/*_tests.py From 91f4e167e797a6c59248f7b5f8098eb04029f53e Mon Sep 17 00:00:00 2001 From: 8naama Date: Mon, 18 Mar 2024 17:45:37 +0200 Subject: [PATCH 08/29] fix test config files --- tests/config/azure_graph/base_config.yaml | 1 + tests/config/azure_graph/custom_fields_config.yaml | 1 + tests/config/azure_graph/days_back_fetch_config.yaml | 1 + tests/config/azure_graph/filters_config.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/tests/config/azure_graph/base_config.yaml b/tests/config/azure_graph/base_config.yaml index 3257506..9298a76 100644 --- a/tests/config/azure_graph/base_config.yaml +++ b/tests/config/azure_graph/base_config.yaml @@ -24,3 +24,4 @@ oauth_apis: time_interval: 1 days_back_fetch: 30 start_date_name: createdDateTime + end_date_name: EndDate diff --git a/tests/config/azure_graph/custom_fields_config.yaml b/tests/config/azure_graph/custom_fields_config.yaml index 2b56f57..ab5b0b2 100644 --- a/tests/config/azure_graph/custom_fields_config.yaml +++ b/tests/config/azure_graph/custom_fields_config.yaml @@ -20,6 +20,7 @@ oauth_apis: url: https://graph.microsoft.com/v1.0/auditLogs/signIns method: GET start_date_name: createdDateTime + end_date_name: EndDate json_paths: data_date: createdDateTime settings: diff --git a/tests/config/azure_graph/days_back_fetch_config.yaml b/tests/config/azure_graph/days_back_fetch_config.yaml index 0f2f243..6420cc9 100644 --- a/tests/config/azure_graph/days_back_fetch_config.yaml +++ b/tests/config/azure_graph/days_back_fetch_config.yaml @@ -20,6 +20,7 @@ oauth_apis: url: https://graph.microsoft.com/v1.0/auditLogs/signIns method: GET start_date_name: createdDateTime + end_date_name: EndDate json_paths: data_date: createdDateTime additional_filters: diff --git a/tests/config/azure_graph/filters_config.yaml b/tests/config/azure_graph/filters_config.yaml index ab0e731..182b0ea 100644 --- a/tests/config/azure_graph/filters_config.yaml +++ b/tests/config/azure_graph/filters_config.yaml @@ -20,6 +20,7 @@ oauth_apis: url: https://graph.microsoft.com/v1.0/auditLogs/signIns method: GET start_date_name: createdDateTime + end_date_name: EndDate json_paths: data_date: createdDateTime filters: From 525002d32296c689bd89dc42491c0d20c2b51892 Mon Sep 17 00:00:00 2001 From: 8naama Date: Thu, 21 Mar 2024 13:38:32 +0200 Subject: [PATCH 09/29] try fix deadlock in `test_sending_data` test --- tests/azure_graph_api_tests.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index 784f67f..edb929e 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -11,6 +11,7 @@ from .tests_utils import TestUtils logger = logging.getLogger(__name__) +multiprocessing.set_start_method("spawn") # to fix shipping tests deadlock class AzureGraphApiTests(unittest.TestCase): @@ -82,20 +83,21 @@ def test_fetch_data_with_filters(self) -> None: self.assertNotEqual(total_data_num, fetched_data_num) def test_sending_data(self) -> None: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - AzureGraphApiTests.BASE_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) + with multiprocessing.get_context("spawn").Pool() as pool: + queue = multiprocessing.Queue() + self.tests_utils.start_process_and_wait_until_finished(queue, + AzureGraphApiTests.BASE_CONFIG_FILE, + self.tests_utils.run_oauth_api_process, + status=200, + sleep_time=10) + + requests_num, sent_logs_num, sent_bytes = queue.get() + data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( + self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) + + self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) + self.assertEqual(data_num, sent_logs_num) + self.assertEqual(data_bytes, sent_bytes) def test_sending_data_iterations(self) -> None: queue = multiprocessing.Queue() From 55fd527794eabfc22701006d88d803e99e743cff Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 12:23:20 +0200 Subject: [PATCH 10/29] try another approach to fix tests --- tests/azure_graph_api_tests.py | 41 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index edb929e..ed396ee 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -5,13 +5,13 @@ import math import httpretty +from queue import Queue from src.azure_graph import AzureGraph from src.logzio_shipper import LogzioShipper from .tests_utils import TestUtils logger = logging.getLogger(__name__) -multiprocessing.set_start_method("spawn") # to fix shipping tests deadlock class AzureGraphApiTests(unittest.TestCase): @@ -83,24 +83,23 @@ def test_fetch_data_with_filters(self) -> None: self.assertNotEqual(total_data_num, fetched_data_num) def test_sending_data(self) -> None: - with multiprocessing.get_context("spawn").Pool() as pool: - queue = multiprocessing.Queue() - self.tests_utils.start_process_and_wait_until_finished(queue, - AzureGraphApiTests.BASE_CONFIG_FILE, - self.tests_utils.run_oauth_api_process, - status=200, - sleep_time=10) - - requests_num, sent_logs_num, sent_bytes = queue.get() - data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) + queue = Queue() + self.tests_utils.start_process_and_wait_until_finished(queue, + AzureGraphApiTests.BASE_CONFIG_FILE, + self.tests_utils.run_oauth_api_process, + status=200, + sleep_time=10) + + requests_num, sent_logs_num, sent_bytes = queue.get() + data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( + self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) + + self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) + self.assertEqual(data_num, sent_logs_num) + self.assertEqual(data_bytes, sent_bytes) def test_sending_data_iterations(self) -> None: - queue = multiprocessing.Queue() + queue = Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.DAYS_BACK_FETCH_CONFIG_FILE, self.tests_utils.run_oauth_api_process, @@ -116,7 +115,7 @@ def test_sending_data_iterations(self) -> None: self.assertEqual(data_bytes, sent_bytes) def test_sending_data_multiple_azure_graph(self) -> None: - queue = multiprocessing.Queue() + queue = Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.MULTIPLE_CONFIG_FILE, self.tests_utils.run_oauth_api_process, @@ -132,7 +131,7 @@ def test_sending_data_multiple_azure_graph(self) -> None: self.assertEqual(data_bytes, sent_bytes) def test_sending_data_with_custom_fields(self) -> None: - queue = multiprocessing.Queue() + queue = Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE, self.tests_utils.run_oauth_api_process, @@ -151,7 +150,7 @@ def test_sending_data_with_custom_fields(self) -> None: self.assertEqual(data_bytes, sent_bytes) def test_time_interval(self) -> None: - queue = multiprocessing.Queue() + queue = Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.TIME_INTERVAL_CONFIG_FILE, self.tests_utils.run_oauth_api_process, @@ -183,7 +182,7 @@ def test_last_start_date(self) -> None: self.assertEqual('2020-03-13T19:15:41.6195833Z', azure_graph.get_last_start_date()) def test_bad_config(self) -> None: - queue = multiprocessing.Queue() + queue = Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.BAD_CONFIG_FILE, self.tests_utils.run_oauth_api_process, From a2103caa87ca2aaa3e92145daed805299b2dba86 Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 13:37:18 +0200 Subject: [PATCH 11/29] revert attempts --- tests/azure_graph_api_tests.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index ed396ee..784f67f 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -5,7 +5,6 @@ import math import httpretty -from queue import Queue from src.azure_graph import AzureGraph from src.logzio_shipper import LogzioShipper @@ -83,7 +82,7 @@ def test_fetch_data_with_filters(self) -> None: self.assertNotEqual(total_data_num, fetched_data_num) def test_sending_data(self) -> None: - queue = Queue() + queue = multiprocessing.Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.BASE_CONFIG_FILE, self.tests_utils.run_oauth_api_process, @@ -99,7 +98,7 @@ def test_sending_data(self) -> None: self.assertEqual(data_bytes, sent_bytes) def test_sending_data_iterations(self) -> None: - queue = Queue() + queue = multiprocessing.Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.DAYS_BACK_FETCH_CONFIG_FILE, self.tests_utils.run_oauth_api_process, @@ -115,7 +114,7 @@ def test_sending_data_iterations(self) -> None: self.assertEqual(data_bytes, sent_bytes) def test_sending_data_multiple_azure_graph(self) -> None: - queue = Queue() + queue = multiprocessing.Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.MULTIPLE_CONFIG_FILE, self.tests_utils.run_oauth_api_process, @@ -131,7 +130,7 @@ def test_sending_data_multiple_azure_graph(self) -> None: self.assertEqual(data_bytes, sent_bytes) def test_sending_data_with_custom_fields(self) -> None: - queue = Queue() + queue = multiprocessing.Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE, self.tests_utils.run_oauth_api_process, @@ -150,7 +149,7 @@ def test_sending_data_with_custom_fields(self) -> None: self.assertEqual(data_bytes, sent_bytes) def test_time_interval(self) -> None: - queue = Queue() + queue = multiprocessing.Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.TIME_INTERVAL_CONFIG_FILE, self.tests_utils.run_oauth_api_process, @@ -182,7 +181,7 @@ def test_last_start_date(self) -> None: self.assertEqual('2020-03-13T19:15:41.6195833Z', azure_graph.get_last_start_date()) def test_bad_config(self) -> None: - queue = Queue() + queue = multiprocessing.Queue() self.tests_utils.start_process_and_wait_until_finished(queue, AzureGraphApiTests.BAD_CONFIG_FILE, self.tests_utils.run_oauth_api_process, From d0a6c19c1cb41516a999ff6296a028ec05a624a3 Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 13:53:03 +0200 Subject: [PATCH 12/29] try fix issue --- tests/tests_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 3ff75ec..d0eb71b 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -46,10 +46,11 @@ def start_process_and_wait_until_finished(self, queue: multiprocessing.Queue, co is_multi_test: bool = False) -> None: process = multiprocessing.Process(target=delegate, args=(config_file, status, queue, is_multi_test)) process.start() + process.join() time.sleep(sleep_time) os.kill(process.pid, signal.SIGTERM) - process.join() + @httpretty.activate def run_auth_api_process(self, config_file: str, status: int, queue: multiprocessing.Queue, From 535fd25acbaf2f54dcb978265df33b7303a12373 Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 13:58:25 +0200 Subject: [PATCH 13/29] try killing process different way --- tests/tests_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index d0eb71b..53ab92a 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -49,8 +49,9 @@ def start_process_and_wait_until_finished(self, queue: multiprocessing.Queue, co process.join() time.sleep(sleep_time) - os.kill(process.pid, signal.SIGTERM) - + process.kill() + # os.kill(process.pid, signal.SIGTERM) + @httpretty.activate def run_auth_api_process(self, config_file: str, status: int, queue: multiprocessing.Queue, From 65ae906e192d86207d52feb7baa48b2c359691d6 Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 14:03:00 +0200 Subject: [PATCH 14/29] try another approach --- tests/tests_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 53ab92a..1f2e522 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -46,11 +46,11 @@ def start_process_and_wait_until_finished(self, queue: multiprocessing.Queue, co is_multi_test: bool = False) -> None: process = multiprocessing.Process(target=delegate, args=(config_file, status, queue, is_multi_test)) process.start() - process.join() time.sleep(sleep_time) - process.kill() + process.terminate() # os.kill(process.pid, signal.SIGTERM) + process.join() @httpretty.activate From dd716ce3126c6163f64937f0c1bd23c7e2d0bfb8 Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 14:16:37 +0200 Subject: [PATCH 15/29] add print for process state --- tests/tests_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 1f2e522..e76f9ef 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -49,6 +49,7 @@ def start_process_and_wait_until_finished(self, queue: multiprocessing.Queue, co time.sleep(sleep_time) process.terminate() + print(f"TEST is alive: {process.is_alive()}") # os.kill(process.pid, signal.SIGTERM) process.join() From b3ee7b21288d713539738a5c0000f8aa4bc69b57 Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 14:23:17 +0200 Subject: [PATCH 16/29] add return at ApisManager --- src/apis_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/apis_manager.py b/src/apis_manager.py index 05470ed..32193f4 100644 --- a/src/apis_manager.py +++ b/src/apis_manager.py @@ -151,6 +151,8 @@ def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None: logger.info( "Task is over. A new Task for api {0} will run in {1} minute/s.".format(api.get_api_name(), int(api.get_api_time_interval() / 60))) + return + def _write_last_start_date_to_file(self, api_name: str, last_start_date: str) -> None: self._lock.acquire() From 6a7ccd3166a1ff40e353969090349f5e8b29cfaf Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 16:50:47 +0200 Subject: [PATCH 17/29] try new approach --- src/apis_manager.py | 2 -- tests/azure_graph_api_tests.py | 10 +++++----- tests/tests_utils.py | 11 ++++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/apis_manager.py b/src/apis_manager.py index 32193f4..05470ed 100644 --- a/src/apis_manager.py +++ b/src/apis_manager.py @@ -151,8 +151,6 @@ def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None: logger.info( "Task is over. A new Task for api {0} will run in {1} minute/s.".format(api.get_api_name(), int(api.get_api_time_interval() / 60))) - return - def _write_last_start_date_to_file(self, api_name: str, last_start_date: str) -> None: self._lock.acquire() diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index 784f67f..8b08983 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -89,13 +89,13 @@ def test_sending_data(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + # requests_num, sent_logs_num, sent_bytes = queue.get() data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) + self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - self.assertEqual(data_num, sent_logs_num) - self.assertEqual(data_bytes, sent_bytes) + # self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) + # self.assertEqual(data_num, sent_logs_num) + # self.assertEqual(data_bytes, sent_bytes) def test_sending_data_iterations(self) -> None: queue = multiprocessing.Queue() diff --git a/tests/tests_utils.py b/tests/tests_utils.py index e76f9ef..adfb8cb 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -43,16 +43,17 @@ def __init__(self, api_http_method: str, api_url: str, api_body: dict, token_htt def start_process_and_wait_until_finished(self, queue: multiprocessing.Queue, config_file: str, delegate: Callable[[str], None], status: int, sleep_time: int, - is_multi_test: bool = False) -> None: + is_multi_test: bool = False, should_get: bool = False) -> None: process = multiprocessing.Process(target=delegate, args=(config_file, status, queue, is_multi_test)) process.start() time.sleep(sleep_time) - process.terminate() - print(f"TEST is alive: {process.is_alive()}") - # os.kill(process.pid, signal.SIGTERM) - process.join() + os.kill(process.pid, signal.SIGTERM) + if should_get: + requests_num, sent_logs_num, sent_bytes = queue.get() + print("TEST should_get works?", requests_num, sent_logs_num, sent_bytes) + process.join() @httpretty.activate def run_auth_api_process(self, config_file: str, status: int, queue: multiprocessing.Queue, From 462202b9b2e623afc8bfa580c0166c96212b06fd Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 16:54:37 +0200 Subject: [PATCH 18/29] add missing param for new approach --- tests/azure_graph_api_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index 8b08983..a5b6159 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -87,7 +87,8 @@ def test_sending_data(self) -> None: AzureGraphApiTests.BASE_CONFIG_FILE, self.tests_utils.run_oauth_api_process, status=200, - sleep_time=10) + sleep_time=10, + should_get=True) # requests_num, sent_logs_num, sent_bytes = queue.get() data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( From 458e355fd0ba084def8ce15d2f66b755698c9f80 Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 17:13:53 +0200 Subject: [PATCH 19/29] try False on queue get block --- tests/azure_graph_api_tests.py | 11 +++++------ tests/tests_utils.py | 6 +----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index a5b6159..e96cd54 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -87,16 +87,15 @@ def test_sending_data(self) -> None: AzureGraphApiTests.BASE_CONFIG_FILE, self.tests_utils.run_oauth_api_process, status=200, - sleep_time=10, - should_get=True) + sleep_time=10) - # requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) - # self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) - # self.assertEqual(data_num, sent_logs_num) - # self.assertEqual(data_bytes, sent_bytes) + self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) + self.assertEqual(data_num, sent_logs_num) + self.assertEqual(data_bytes, sent_bytes) def test_sending_data_iterations(self) -> None: queue = multiprocessing.Queue() diff --git a/tests/tests_utils.py b/tests/tests_utils.py index adfb8cb..3ff75ec 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -43,16 +43,12 @@ def __init__(self, api_http_method: str, api_url: str, api_body: dict, token_htt def start_process_and_wait_until_finished(self, queue: multiprocessing.Queue, config_file: str, delegate: Callable[[str], None], status: int, sleep_time: int, - is_multi_test: bool = False, should_get: bool = False) -> None: + is_multi_test: bool = False) -> None: process = multiprocessing.Process(target=delegate, args=(config_file, status, queue, is_multi_test)) process.start() time.sleep(sleep_time) os.kill(process.pid, signal.SIGTERM) - - if should_get: - requests_num, sent_logs_num, sent_bytes = queue.get() - print("TEST should_get works?", requests_num, sent_logs_num, sent_bytes) process.join() @httpretty.activate From 1943bcb0363abd4652093ca8d0d52f7962da794f Mon Sep 17 00:00:00 2001 From: 8naama Date: Sun, 24 Mar 2024 17:26:08 +0200 Subject: [PATCH 20/29] verify deadlock in q.get --- tests/azure_graph_api_tests.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index e96cd54..66f00a2 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -88,10 +88,13 @@ def test_sending_data(self) -> None: self.tests_utils.run_oauth_api_process, status=200, sleep_time=10) - + if queue.empty(): + print("TEST: queue empty") + else: + print("TEST: queue not empty") requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( - self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) + self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) self.assertEqual(math.ceil(sent_bytes / LogzioShipper.MAX_BULK_SIZE_BYTES), requests_num) self.assertEqual(data_num, sent_logs_num) @@ -105,7 +108,7 @@ def test_sending_data_iterations(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -121,7 +124,7 @@ def test_sending_data_multiple_azure_graph(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -137,7 +140,7 @@ def test_sending_data_with_custom_fields(self) -> None: status=200, sleep_time=10) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) custom_fields_azure_graph = self.tests_utils.get_first_api(AzureGraphApiTests.CUSTOM_FIELDS_CONFIG_FILE, @@ -156,7 +159,7 @@ def test_time_interval(self) -> None: status=200, sleep_time=70) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) @@ -188,7 +191,7 @@ def test_bad_config(self) -> None: status=200, sleep_time=1) - requests_num, sent_logs_num, sent_bytes = queue.get() + requests_num, sent_logs_num, sent_bytes = queue.get(False) self.assertEqual(0, requests_num) self.assertEqual(0, sent_logs_num) From 8d4892f4a3fd64c559616ae368a2c6cdf184653a Mon Sep 17 00:00:00 2001 From: 8naama Date: Mon, 25 Mar 2024 11:15:46 +0200 Subject: [PATCH 21/29] update logger test to debug --- tests/azure_graph_api_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/azure_graph_api_tests.py b/tests/azure_graph_api_tests.py index 66f00a2..017a74b 100644 --- a/tests/azure_graph_api_tests.py +++ b/tests/azure_graph_api_tests.py @@ -89,9 +89,9 @@ def test_sending_data(self) -> None: status=200, sleep_time=10) if queue.empty(): - print("TEST: queue empty") + logger.info("TEST: queue empty") else: - print("TEST: queue not empty") + logger.info("TEST: queue not empty") requests_num, sent_logs_num, sent_bytes = queue.get(False) data_bytes, data_num = self.tests_utils.get_api_data_bytes_and_num_from_json_data( self.azure_graph_json_body[AzureGraph.DEFAULT_GRAPH_DATA_LINK]) From 553e9fc41757ff57cce95e6e01ca9454ae8fa964 Mon Sep 17 00:00:00 2001 From: 8naama Date: Mon, 25 Mar 2024 16:43:53 +0200 Subject: [PATCH 22/29] add checks to the functions that fill the queue --- tests/tests_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 3ff75ec..9d31581 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -97,7 +97,7 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce continue logzio_requests.append(request) - + logger.info("TEST: putting data in the queue") queue.put(self._get_sending_data_results(logzio_requests)) def get_first_api(self, config_file: str, is_auth_api: bool) -> Api: @@ -226,4 +226,5 @@ def _get_sending_data_results(self, latest_requests: list) -> tuple[int, int, in sent_logs_num += 1 sent_bytes += len(log) + logger.info(f"TEST: _get_sending_data_results params {requests_num}, {sent_logs_num}, {sent_bytes}") return int(requests_num / 2), int(sent_logs_num / 2), int(sent_bytes / 2) From e2a47ac86b0dd4f45a7a6c7fe63e0e9e72b494a5 Mon Sep 17 00:00:00 2001 From: 8naama Date: Tue, 26 Mar 2024 11:22:34 +0200 Subject: [PATCH 23/29] add test param to `run()`func in api manager to try stop the process for the test to work --- src/apis_manager.py | 8 +++++++- tests/tests_utils.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/apis_manager.py b/src/apis_manager.py index 05470ed..1f7cb24 100644 --- a/src/apis_manager.py +++ b/src/apis_manager.py @@ -2,6 +2,8 @@ import os import signal import threading +from time import sleep + import requests from typing import Optional @@ -40,7 +42,7 @@ def __init__(self) -> None: self._event = threading.Event() self._lock = threading.Lock() - def run(self) -> None: + def run(self, test=False) -> None: if not self._read_data_from_config(): return @@ -53,6 +55,10 @@ def run(self) -> None: for thread in self._threads: thread.start() + if test: + sleep(5) + self.__exit_gracefully() + signal.sigwait([signal.SIGINT, signal.SIGTERM]) self.__exit_gracefully() diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 9d31581..2e486a6 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -90,7 +90,7 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE logzio_requests = [] - ApisManager().run() + ApisManager().run() # test_sending_data is stuck here, never reaching after this line for request in httpretty.latest_requests(): if request.url.startswith(self.api_url): From d5555734ef211d1d9d8861ef0e2470ea78bd8041 Mon Sep 17 00:00:00 2001 From: 8naama Date: Tue, 26 Mar 2024 11:26:03 +0200 Subject: [PATCH 24/29] add test=True to the relevant tests --- tests/tests_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 2e486a6..c25c91c 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -61,7 +61,7 @@ def run_auth_api_process(self, config_file: str, status: int, queue: multiproces ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE logzio_requests = [] - ApisManager().run() + ApisManager().run(test=True) for request in httpretty.latest_requests(): if request.url.startswith(self.api_url): @@ -90,7 +90,7 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE logzio_requests = [] - ApisManager().run() # test_sending_data is stuck here, never reaching after this line + ApisManager().run(test=True) # test_sending_data is stuck here, never reaching after this line for request in httpretty.latest_requests(): if request.url.startswith(self.api_url): From bddecbd9a72f14ea6c015623a5a3cd0bd59a1687 Mon Sep 17 00:00:00 2001 From: 8naama Date: Tue, 26 Mar 2024 11:35:04 +0200 Subject: [PATCH 25/29] add extra test log and reduce sleep before killing process --- src/apis_manager.py | 4 +++- src/oauth_api.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/apis_manager.py b/src/apis_manager.py index 1f7cb24..807008c 100644 --- a/src/apis_manager.py +++ b/src/apis_manager.py @@ -56,8 +56,10 @@ def run(self, test=False) -> None: thread.start() if test: - sleep(5) + sleep(2) + os.kill(os.getpid(), signal.SIGTERM) self.__exit_gracefully() + return signal.sigwait([signal.SIGINT, signal.SIGTERM]) self.__exit_gracefully() diff --git a/src/oauth_api.py b/src/oauth_api.py index 8e2ea83..cf4fd2b 100644 --- a/src/oauth_api.py +++ b/src/oauth_api.py @@ -35,8 +35,10 @@ def __init__(self, oauth_config: OAuthApiBaseData, general_config: OAuthApiGener super().__init__(oauth_config.base_data, general_config.general_type_data) def get_token(self) -> [str, int]: + print(f"NAAMA TEST {self._token_request.url}\n{self._token_request.body}") token_response = requests.post(self._token_request.url, self._token_request.body) + print(token_response.content) return json.loads(token_response.content)[self.OAUTH_ACCESS_TOKEN], json.loads(token_response.content)[ self.OAUTH_TOKEN_EXPIRE] @@ -131,4 +133,3 @@ def get_token_request(self): def _set_current_data_last_date(self, date): if date: self._current_data_last_date = date - From 7f7f76fd0bc455062bb0f60f75eed1bf33f28fea Mon Sep 17 00:00:00 2001 From: 8naama Date: Tue, 26 Mar 2024 15:41:41 +0200 Subject: [PATCH 26/29] run ApisManager().run as process to kill it --- tests/tests_utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/tests_utils.py b/tests/tests_utils.py index c25c91c..50a3c66 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -90,9 +90,16 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE logzio_requests = [] - ApisManager().run(test=True) # test_sending_data is stuck here, never reaching after this line + logger.info("TEST: starting API Manager") + p = multiprocessing.Process(target=ApisManager().run, args=(True, )) + # ApisManager().run(test=True) # test_sending_data is stuck here, never reaching after this line + p.start() + time.sleep(2) + p.kill() + logger.info("TEST: Finished API Manager!") for request in httpretty.latest_requests(): + logger.info("TEST: reading requests") if request.url.startswith(self.api_url): continue From 5c11a7ea7c65f094d92a0c025bc445bc7778b076 Mon Sep 17 00:00:00 2001 From: 8naama Date: Tue, 26 Mar 2024 16:04:48 +0200 Subject: [PATCH 27/29] test another URL and restore prev api_managers --- src/apis_manager.py | 9 +-------- tests/tests_utils.py | 12 ++++++------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/apis_manager.py b/src/apis_manager.py index 807008c..937d0fc 100644 --- a/src/apis_manager.py +++ b/src/apis_manager.py @@ -2,7 +2,6 @@ import os import signal import threading -from time import sleep import requests @@ -42,7 +41,7 @@ def __init__(self) -> None: self._event = threading.Event() self._lock = threading.Lock() - def run(self, test=False) -> None: + def run(self) -> None: if not self._read_data_from_config(): return @@ -55,12 +54,6 @@ def run(self, test=False) -> None: for thread in self._threads: thread.start() - if test: - sleep(2) - os.kill(os.getpid(), signal.SIGTERM) - self.__exit_gracefully() - return - signal.sigwait([signal.SIGINT, signal.SIGTERM]) self.__exit_gracefully() diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 50a3c66..c87a5c7 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -24,6 +24,7 @@ class TestUtils: LOGZIO_HTTPPRETTY_URL = 'https://listener.logz.io:8071/?token=123456789a&type=api_fetcher' + LOGZIO_HTTPPRETTY_URL2 = 'https://listener.logz.io:8071/?token=123456789a' LOGZIO_URL = 'https://listener.logz.io:8071' LOGZIO_TOKEN = '123456789a' LAST_START_DATES_FILE = 'tests/last_start_dates.txt' @@ -61,7 +62,7 @@ def run_auth_api_process(self, config_file: str, status: int, queue: multiproces ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE logzio_requests = [] - ApisManager().run(test=True) + ApisManager().run() for request in httpretty.latest_requests(): if request.url.startswith(self.api_url): @@ -76,9 +77,7 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce is_multi_test: bool) -> None: from tests.azure_graph_api_tests import AzureGraphApiTests httpretty.register_uri(httpretty.POST, TestUtils.LOGZIO_URL, status=status) - httpretty.register_uri(self.token_http_method, - self.token_url, - body=json.dumps(self.token_body)) + httpretty.register_uri(self.token_http_method, self.token_url, body=json.dumps(self.token_body)) httpretty.register_uri(self.api_http_method, self.api_url, body=json.dumps(self.api_body), status=200, headers={AzureGraph.OAUTH_AUTHORIZATION_HEADER: AzureGraphApiTests.AZURE_GRAPH_TEST_TOKEN}) @@ -92,7 +91,7 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce logger.info("TEST: starting API Manager") p = multiprocessing.Process(target=ApisManager().run, args=(True, )) - # ApisManager().run(test=True) # test_sending_data is stuck here, never reaching after this line + # ApisManager().run() # test_sending_data is stuck here, never reaching after this line p.start() time.sleep(2) p.kill() @@ -221,7 +220,8 @@ def _get_sending_data_results(self, latest_requests: list) -> tuple[int, int, in sent_bytes = 0 for request in latest_requests: - if request.url == self.LOGZIO_HTTPPRETTY_URL: + logger.info(f"TEST: checking request {request}") + if request.url in (self.LOGZIO_HTTPPRETTY_URL, self.LOGZIO_HTTPPRETTY_URL2): requests_num += 1 try: From f4493768e4a8d7accc108884c895d00532470ea8 Mon Sep 17 00:00:00 2001 From: 8naama Date: Tue, 26 Mar 2024 16:23:06 +0200 Subject: [PATCH 28/29] revert oath_api --- src/oauth_api.py | 2 -- tests/tests_utils.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/oauth_api.py b/src/oauth_api.py index cf4fd2b..21ff475 100644 --- a/src/oauth_api.py +++ b/src/oauth_api.py @@ -35,10 +35,8 @@ def __init__(self, oauth_config: OAuthApiBaseData, general_config: OAuthApiGener super().__init__(oauth_config.base_data, general_config.general_type_data) def get_token(self) -> [str, int]: - print(f"NAAMA TEST {self._token_request.url}\n{self._token_request.body}") token_response = requests.post(self._token_request.url, self._token_request.body) - print(token_response.content) return json.loads(token_response.content)[self.OAUTH_ACCESS_TOKEN], json.loads(token_response.content)[ self.OAUTH_TOKEN_EXPIRE] diff --git a/tests/tests_utils.py b/tests/tests_utils.py index c87a5c7..9209a11 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -90,7 +90,7 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce logzio_requests = [] logger.info("TEST: starting API Manager") - p = multiprocessing.Process(target=ApisManager().run, args=(True, )) + p = multiprocessing.Process(target=ApisManager().run) # ApisManager().run() # test_sending_data is stuck here, never reaching after this line p.start() time.sleep(2) From 76ef30df3367fa10657be7484dcfb58938382013 Mon Sep 17 00:00:00 2001 From: 8naama Date: Wed, 27 Mar 2024 10:03:41 +0200 Subject: [PATCH 29/29] add test parameter to API manager to see if it will solve issue --- src/apis_manager.py | 6 ++++-- tests/tests_utils.py | 13 +++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/apis_manager.py b/src/apis_manager.py index 937d0fc..45b4b8e 100644 --- a/src/apis_manager.py +++ b/src/apis_manager.py @@ -34,12 +34,13 @@ class ApisManager: AUTH_API_TYPES = [API_GENERAL_TYPE, API_CISCO_SECURE_X_TYPE] OAUTH_API_TYPES = [API_GENERAL_TYPE, API_AZURE_GRAPH_TYPE, API_AZURE_MAIL_REPORTS_TYPE] - def __init__(self) -> None: + def __init__(self, test=False) -> None: self._apis: list[Api] = [] self._logzio_connection: Optional[LogzioConnection] = None self._threads = [] self._event = threading.Event() self._lock = threading.Lock() + self.test = test def run(self) -> None: if not self._read_data_from_config(): @@ -108,7 +109,8 @@ def _run_api_scheduled_task(self, api: Api) -> None: thread.start() thread.join() - if self._event.wait(timeout=api.get_api_time_interval()): + if self._event.wait(timeout=api.get_api_time_interval()) or self.test: + logger.info("TEST: stopping shipping due to test or timeout") break def _send_data_to_logzio(self, api: Api, logzio_shipper: LogzioShipper) -> None: diff --git a/tests/tests_utils.py b/tests/tests_utils.py index 9209a11..800a34a 100644 --- a/tests/tests_utils.py +++ b/tests/tests_utils.py @@ -89,14 +89,15 @@ def run_oauth_api_process(self, config_file: str, status: int, queue: multiproce ApisManager.LAST_START_DATES_FILE = TestUtils.LAST_START_DATES_FILE logzio_requests = [] - logger.info("TEST: starting API Manager") - p = multiprocessing.Process(target=ApisManager().run) - # ApisManager().run() # test_sending_data is stuck here, never reaching after this line - p.start() - time.sleep(2) - p.kill() + # logger.info("TEST: starting API Manager") + # p = multiprocessing.Process(target=ApisManager().run) + ApisManager(True).run() # test_sending_data is stuck here, never reaching after this line + # p.start() + # time.sleep(2) + # p.kill() logger.info("TEST: Finished API Manager!") + # this now doesn't work, because the requests are made in a separate process for request in httpretty.latest_requests(): logger.info("TEST: reading requests") if request.url.startswith(self.api_url):