feat: workflows refactoring

.github/workflows/test-pr-e2e.yml

@@ -4,9 +4,9 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - 'keep/**'
-      - 'keep-ui/**'
-      - 'tests/**'
+      - "keep/**"
+      - "keep-ui/**"
+      - "tests/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
@@ -123,7 +123,7 @@ jobs:
 
           # create the state directory
           # mkdir -p ./state && chown -R root:root ./state && chmod -R 777 ./state
-    
+
       - name: Run e2e tests and report coverage
         run: |
           poetry run coverage run --branch -m pytest -s tests/e2e_tests/
@@ -147,9 +147,9 @@ jobs:
 
       - name: Upload test artifacts on failure
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.4.3
         with:
-          name: test-artifacts
+          name: test-artifacts-my-artifacts-${{ matrix.db_type }}
           path: |
             playwright_dump_*.html
             playwright_dump_*.png

.github/workflows/test-pr.yml

@@ -91,9 +91,11 @@ jobs:
         run: poetry install --no-interaction --no-root --with dev
 
       - name: Run unit tests and report coverage
+        env:
+          LOG_LEVEL: DEBUG
+          SQLALCHEMY_WARN_20: 1
         run: |
-          poetry run coverage run --branch -m pytest -n auto --non-integration --ignore=tests/e2e_tests/
-
+          poetry run coverage run --branch -m pytest --timeout 10 -n auto --non-integration --ignore=tests/e2e_tests/
 
       - name: Run integration tests and report coverage
         run: |

docker/Dockerfile.api

@@ -42,4 +42,4 @@ USER keep
 
 ENTRYPOINT ["/venv/lib/python3.11/site-packages/keep/entrypoint.sh"]
 
-CMD ["gunicorn", "keep.api.api:get_app", "--bind" , "0.0.0.0:8080" , "--workers", "4" , "-k" , "uvicorn.workers.UvicornWorker", "-c", "/venv/lib/python3.11/site-packages/keep/api/config.py", "--preload"]
+CMD ["gunicorn", "keep.api.api:get_app", "--bind" , "0.0.0.0:8080" , "--workers", "4" , "-k" , "keep.api.custom_worker.CustomUvicornWorker", "-c", "/venv/lib/python3.11/site-packages/keep/api/config.py"]

keep/api/alert_deduplicator/alert_deduplicator.py

@@ -14,7 +14,7 @@
     get_alerts_fields,
     get_all_deduplication_rules,
     get_all_deduplication_stats,
-    get_custom_deduplication_rules,
+    get_custom_deduplication_rule,
     get_last_alert_hash_by_fingerprint,
     update_deduplication_rule,
 )
@@ -31,12 +31,16 @@
 
 class AlertDeduplicator:
 
+    DEDUPLICATION_DISTRIBUTION_ENABLED = config(
+        "KEEP_DEDUPLICATION_DISTRIBUTION_ENABLED", cast=bool, default=True
+    )
+    CUSTOM_DEDUPLICATION_DISTRIBUTION_ENABLED = config(
+        "KEEP_CUSTOM_DEDUPLICATION_ENABLED", cast=bool, default=True
+    )
+
     def __init__(self, tenant_id):
         self.logger = logging.getLogger(__name__)
         self.tenant_id = tenant_id
-        self.provider_distribution_enabled = config(
-            "PROVIDER_DISTRIBUTION_ENABLED", cast=bool, default=True
-        )
         self.search_engine = SearchEngine(self.tenant_id)
 
     def _apply_deduplication_rule(
@@ -91,13 +95,23 @@ def _apply_deduplication_rule(
                 },
             )
             alert.isPartialDuplicate = True
+        else:
+            self.logger.info(
+                "Alert is not deduplicated",
+                extra={
+                    "alert_id": alert.id,
+                    "fingerprint": alert.fingerprint,
+                    "tenant_id": self.tenant_id,
+                    "last_alert_hash_by_fingerprint": last_alert_hash_by_fingerprint,
+                },
+            )
 
         return alert
 
     def apply_deduplication(self, alert: AlertDto) -> bool:
         # IMPOTRANT NOTE TO SOMEONE WORKING ON THIS CODE:
         #   apply_deduplication runs AFTER _format_alert, so you can assume that alert fields are in the expected format.
-        #   you can also safe to assume that alert.fingerprint is set by the provider itself
+        #   you are also safe to assume that alert.fingerprint is set by the provider itself
 
         # get only relevant rules
         rules = self.get_deduplication_rules(
@@ -122,26 +136,30 @@ def apply_deduplication(self, alert: AlertDto) -> bool:
                     "is_partial_duplicate": alert.isPartialDuplicate,
                 },
             )
-            if alert.isFullDuplicate or alert.isPartialDuplicate:
-                # create deduplication event
-                create_deduplication_event(
-                    tenant_id=self.tenant_id,
-                    deduplication_rule_id=rule.id,
-                    deduplication_type="full" if alert.isFullDuplicate else "partial",
-                    provider_id=alert.providerId,
-                    provider_type=alert.providerType,
-                )
-                # we don't need to check the other rules
-                break
-            else:
-                # create none deduplication event, for statistics
-                create_deduplication_event(
-                    tenant_id=self.tenant_id,
-                    deduplication_rule_id=rule.id,
-                    deduplication_type="none",
-                    provider_id=alert.providerId,
-                    provider_type=alert.providerType,
-                )
+
+            if AlertDeduplicator.DEDUPLICATION_DISTRIBUTION_ENABLED:
+                if alert.isFullDuplicate or alert.isPartialDuplicate:
+                    # create deduplication event
+                    create_deduplication_event(
+                        tenant_id=self.tenant_id,
+                        deduplication_rule_id=rule.id,
+                        deduplication_type=(
+                            "full" if alert.isFullDuplicate else "partial"
+                        ),
+                        provider_id=alert.providerId,
+                        provider_type=alert.providerType,
+                    )
+                    # we don't need to check the other rules
+                    break
+                else:
+                    # create none deduplication event, for statistics
+                    create_deduplication_event(
+                        tenant_id=self.tenant_id,
+                        deduplication_rule_id=rule.id,
+                        deduplication_type="none",
+                        provider_id=alert.providerId,
+                        provider_type=alert.providerType,
+                    )
 
         return alert
 
@@ -166,11 +184,15 @@ def get_deduplication_rules(
         self, tenant_id, provider_id, provider_type
     ) -> DeduplicationRuleDto:
         # try to get the rule from the database
-        rules = get_custom_deduplication_rules(tenant_id, provider_id, provider_type)
+        rule = (
+            get_custom_deduplication_rule(tenant_id, provider_id, provider_type)
+            if AlertDeduplicator.CUSTOM_DEDUPLICATION_DISTRIBUTION_ENABLED
+            else None
+        )
 
-        if not rules:
+        if not rule:
             self.logger.debug(
-                "No custom deduplication rules found, using deafult full deduplication rule",
+                "No custom deduplication rule found, using deafult full deduplication rule",
                 extra={
                     "provider_id": provider_id,
                     "provider_type": provider_type,
@@ -189,12 +211,10 @@ def get_deduplication_rules(
                 "tenant_id": tenant_id,
             },
         )
-        #
-        # check that at least one of them is full deduplication rule
-        full_deduplication_rules = [rule for rule in rules if rule.full_deduplication]
+
         # if full deduplication rule found, return the rules
-        if full_deduplication_rules:
-            return rules
+        if rule.full_deduplication:
+            return [rule]
 
         # if not, assign them the default full deduplication rule ignore fields
         self.logger.info(
@@ -203,13 +223,8 @@ def get_deduplication_rules(
         default_full_dedup_rule = self._get_default_full_deduplication_rule(
             provider_id=provider_id, provider_type=provider_type
         )
-        for rule in rules:
-            if not rule.full_deduplication:
-                self.logger.debug(
-                    "Assigning default full deduplication rule ignore fields",
-                )
-                rule.ignore_fields = default_full_dedup_rule.ignore_fields
-        return rules
+        rule.ignore_fields = default_full_dedup_rule.ignore_fields
+        return [rule]
 
     def _generate_uuid(self, provider_id, provider_type):
         # this is a way to generate a unique uuid for the default deduplication rule per (provider_id, provider_type)
@@ -269,7 +284,11 @@ def get_deduplications(self) -> list[DeduplicationRuleDto]:
             provider_id, provider_type = dd.provider_id, dd.provider_type
             dd.id = self._generate_uuid(provider_id, provider_type)
         # get custom deduplication rules
-        custom_deduplications = get_all_deduplication_rules(self.tenant_id)
+        custom_deduplications = (
+            get_all_deduplication_rules(self.tenant_id)
+            if AlertDeduplicator.CUSTOM_DEDUPLICATION_DISTRIBUTION_ENABLED
+            else []
+        )
         # cast to dto
         custom_deduplications_dto = [
             DeduplicationRuleDto(
@@ -347,6 +366,14 @@ def get_deduplications(self) -> list[DeduplicationRuleDto]:
 
         result = []
         for dedup in final_deduplications:
+            self.logger.debug(
+                "Calculating deduplication stats",
+                extra={
+                    "deduplication_rule_id": dedup.id,
+                    "tenant_id": self.tenant_id,
+                    "deduplication_stats": deduplication_stats,
+                },
+            )
             key = dedup.id
             full_dedup = deduplication_stats.get(key, {"full_dedup_count": 0}).get(
                 "full_dedup_count", 0
@@ -377,7 +404,7 @@ def get_deduplications(self) -> list[DeduplicationRuleDto]:
                 )
             result.append(dedup)
 
-        if self.provider_distribution_enabled:
+        if AlertDeduplicator.DEDUPLICATION_DISTRIBUTION_ENABLED:
             for dedup in result:
                 for pd, stats in deduplication_stats.items():
                     if pd == f"{dedup.provider_id}_{dedup.provider_type}":