Unisender Go: Fix status tracking webhook and tests.

- Fix signature checking to avoid false validation errors
  on webhook payloads including `/` (including all "clicked"
  and most "opened" events). And in general, avoid depending
  on specific details of Unisender Go's JSON serialization.
  (Fixes #398.)
- Handle "use single event" webhook option (which has a different
  payload format).
- Verify basic auth when Anymail's WEBHOOK_SECRET is used.
  (This is optional for Unisender Go, since payloads are signed,
  but it needs to be checked when enabled.)
- Treat "soft_bounced" events as "deferred" rather than "bounced",
  since they will be retried later.
- Update validation error to reference Project ID if the webhook
  is configured for a specific project.
- Expose Unisender Go's delivery_status code and unsubscribe form
  comment as Anymail's normalized event.description.
- Update webhook tests based on actual payloads and add several
  missing tests.
- Update docs to clarify webhook use with Unisender Go projects.

.pre-commit-config.yaml

@@ -43,6 +43,7 @@ repos:
       - id: check-toml
       - id: check-yaml
       - id: end-of-file-fixer
+        exclude: "\\.(bin|raw)$"
       - id: fix-byte-order-marker
       - id: fix-encoding-pragma
         args: [--remove]

CHANGELOG.rst

@@ -36,6 +36,18 @@ Breaking changes
 
 * Require **Django 4.0 or later** and Python 3.8 or later.
 
+Fixes
+~~~~~
+
+* **Unisender Go:** Fix several problems in Anymail's Unisender Go status tracking
+  webhook. Rework signature checking to fix false validation errors (particularly
+  on "clicked" and "opened" events). Properly handle "use single event" webhook
+  option. Correctly verify WEBHOOK_SECRET when set. Provide Unisender Go's
+  ``delivery_status`` code and unsubscribe form ``comment`` in Anymail's
+  ``event.description``. Treat soft bounces as "deferred" rather than "bounced".
+  (Thanks to `@MikeVL`_ for fixing the signature validation problem.)
+
+
 Features
 ~~~~~~~~
 
@@ -1741,6 +1753,7 @@ Features
 .. _@mark-mishyn: https://github.com/mark-mishyn
 .. _@martinezleoml: https://github.com/martinezleoml
 .. _@mbk-ok: https://github.com/mbk-ok
+.. _@MikeVL: https://github.com/MikeVL
 .. _@mounirmesselmeni: https://github.com/mounirmesselmeni
 .. _@mwheels: https://github.com/mwheels
 .. _@nuschk: https://github.com/nuschk

anymail/webhooks/unisender_go.py

@@ -8,21 +8,23 @@
 from django.http import HttpRequest, HttpResponse
 from django.utils.crypto import constant_time_compare
 
-from anymail.exceptions import AnymailWebhookValidationFailure
-from anymail.signals import AnymailTrackingEvent, EventType, RejectReason, tracking
-from anymail.utils import get_anymail_setting
-from anymail.webhooks.base import AnymailCoreWebhookView
+from ..exceptions import AnymailWebhookValidationFailure
+from ..signals import AnymailTrackingEvent, EventType, RejectReason, tracking
+from ..utils import get_anymail_setting
+from .base import AnymailBaseWebhookView
 
 
-class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView):
-    """Handler for UniSender delivery and engagement tracking webhooks"""
+class UnisenderGoTrackingWebhookView(AnymailBaseWebhookView):
+    """Handler for Unisender Go delivery and engagement tracking webhooks"""
 
     # See https://godocs.unisender.ru/web-api-ref#callback-format for webhook payload
 
     esp_name = "Unisender Go"
     signal = tracking
     warn_if_no_basic_auth = False  # because we validate against signature
 
+    api_key: str | None = None  # allows kwargs override
+
     event_types = {
         "sent": EventType.SENT,
         "delivered": EventType.DELIVERED,
@@ -31,14 +33,14 @@ class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView):
         "unsubscribed": EventType.UNSUBSCRIBED,
         "subscribed": EventType.SUBSCRIBED,
         "spam": EventType.COMPLAINED,
-        "soft_bounced": EventType.BOUNCED,
+        "soft_bounced": EventType.DEFERRED,
         "hard_bounced": EventType.BOUNCED,
     }
 
     reject_reasons = {
         "err_user_unknown": RejectReason.BOUNCED,
         "err_user_inactive": RejectReason.BOUNCED,
-        "err_will_retry": RejectReason.BOUNCED,
+        "err_will_retry": None,  # not rejected
         "err_mailbox_discarded": RejectReason.BOUNCED,
         "err_mailbox_full": RejectReason.BOUNCED,
         "err_spam_rejected": RejectReason.SPAM,
@@ -56,49 +58,105 @@ class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView):
 
     http_method_names = ["post", "head", "options", "get"]
 
+    def __init__(self, **kwargs):
+        api_key = get_anymail_setting(
+            "api_key", esp_name=self.esp_name, allow_bare=True, kwargs=kwargs
+        )
+        self.api_key_bytes = api_key.encode("ascii")
+        super().__init__(**kwargs)
+
     def get(
         self, request: HttpRequest, *args: typing.Any, **kwargs: typing.Any
     ) -> HttpResponse:
         # Unisender Go verifies the webhook with a GET request at configuration time
         return HttpResponse()
 
+    def parse_json_body(self, request: HttpRequest) -> dict | list | None:
+        # Cache parsed JSON request.body on the request.
+        if hasattr(request, "_parsed_json"):
+            parsed = getattr(request, "_parsed_json")
+        else:
+            parsed = json.loads(request.body.decode())
+            setattr(request, "_parsed_json", parsed)
+        return parsed
+
     def validate_request(self, request: HttpRequest) -> None:
         """
-        How Unisender GO authenticate:
-        Hash the whole request body text and replace api key in "auth" field by this hash.
-
-        So it is both auth and encryption. Also, they hash JSON without spaces.
+        Validate Unisender Go webhook signature:
+        "MD5 hash of the string body of the message, with the auth value replaced
+        by the api_key of the user/project whose handler is being called."
+        https://godocs.unisender.ru/web-api-ref#callback-format
         """
-        request_json = json.loads(request.body.decode("utf-8"))
-        request_auth = request_json.get("auth", "")
-        request_json["auth"] = get_anymail_setting(
-            "api_key", esp_name=self.esp_name, allow_bare=True
-        )
-        json_with_key = json.dumps(request_json, separators=(",", ":"))
-
-        expected_auth = md5(json_with_key.encode("utf-8")).hexdigest()
+        # This must avoid any assumptions about how Unisender Go serializes JSON
+        # (key order, spaces, Unicode encoding vs. \u escapes, etc.). But we do
+        # assume the "auth" field MD5 hash is unique within the serialized JSON,
+        # so that we can use string replacement to calculate the expected hash.
+        body = request.body
+        try:
+            parsed = self.parse_json_body(request)
+            actual_auth = parsed["auth"]
+            actual_auth_bytes = actual_auth.encode()
+        except (AttributeError, KeyError, ValueError):
+            raise AnymailWebhookValidationFailure(
+                "Unisender Go webhook called with invalid payload"
+            )
 
-        if not constant_time_compare(request_auth, expected_auth):
+        body_to_sign = body.replace(actual_auth_bytes, self.api_key_bytes)
+        expected_auth = md5(body_to_sign).hexdigest()
+        if not constant_time_compare(actual_auth, expected_auth):
+            # If webhook has a selected project, include the project_id in the error.
+            try:
+                project_id = parsed["events_by_user"][0]["project_id"]
+            except (KeyError, IndexError):
+                project_id = parsed.get("project_id")  # try "single event" payload
+            is_for_project = f" is for Project ID {project_id}" if project_id else ""
             raise AnymailWebhookValidationFailure(
                 "Unisender Go webhook called with incorrect signature"
+                f" (check Anymail UNISENDER_GO_API_KEY setting{is_for_project})"
             )
 
     def parse_events(self, request: HttpRequest) -> list[AnymailTrackingEvent]:
-        request_json = json.loads(request.body.decode("utf-8"))
-        assert len(request_json["events_by_user"]) == 1  # per API docs
-        esp_events = request_json["events_by_user"][0]["events"]
-        return [
-            self.esp_to_anymail_event(esp_event)
-            for esp_event in esp_events
-            if esp_event["event_name"] == "transactional_email_status"
-        ]
-
-    def esp_to_anymail_event(self, esp_event: dict) -> AnymailTrackingEvent:
-        event_data = esp_event["event_data"]
+        parsed = self.parse_json_body(request)
+        # Unisender Go has two options for webhook payloads. We support both.
+        try:
+            events_by_user = parsed["events_by_user"]
+        except KeyError:
+            # "Use single event": one flat dict, combining "event_data" fields
+            # with "event_name", "user_id", "project_id", etc.
+            if parsed["event_name"] == "transactional_email_status":
+                esp_events = [parsed]
+            else:
+                esp_events = []
+        else:
+            # Not "use single event": we want the "event_data" from all events
+            # with event_name "transactional_email_status".
+            assert len(events_by_user) == 1  # "A single element array" per API docs
+            esp_events = [
+                event["event_data"]
+                for event in events_by_user[0]["events"]
+                if event["event_name"] == "transactional_email_status"
+            ]
+
+        return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events]
+
+    def esp_to_anymail_event(self, event_data: dict) -> AnymailTrackingEvent:
         event_type = self.event_types.get(event_data["status"], EventType.UNKNOWN)
-        timestamp = datetime.fromisoformat(event_data["event_time"])
-        timestamp_utc = timestamp.replace(tzinfo=timezone.utc)
-        metadata = event_data.get("metadata", {})
+
+        # Unisender Go does not provide any way to deduplicate webhook calls.
+        # (There is an "ID" HTTP header, but it has a unique value for every
+        # webhook call--including retransmissions of earlier failed calls.)
+        event_id = None
+
+        # event_time is ISO-like, without a stated time zone. (But it's UTC per docs.)
+        try:
+            timestamp = datetime.fromisoformat(event_data["event_time"]).replace(
+                tzinfo=timezone.utc
+            )
+        except KeyError:
+            timestamp = None
+
+        # Extract our message_id (see backend UNISENDER_GO_GENERATE_MESSAGE_ID).
+        metadata = event_data.get("metadata", {}).copy()
         message_id = metadata.pop("anymail_id", event_data.get("job_id"))
 
         delivery_info = event_data.get("delivery_info", {})
@@ -108,14 +166,18 @@ def esp_to_anymail_event(self, esp_event: dict) -> AnymailTrackingEvent:
         else:
             reject_reason = None
 
+        description = delivery_info.get("delivery_status") or event_data.get("comment")
+        mta_response = delivery_info.get("destination_response")
+
         return AnymailTrackingEvent(
             event_type=event_type,
-            timestamp=timestamp_utc,
+            timestamp=timestamp,
             message_id=message_id,
-            event_id=None,
+            event_id=event_id,
             recipient=event_data["email"],
             reject_reason=reject_reason,
-            mta_response=delivery_info.get("destination_response"),
+            description=description,
+            mta_response=mta_response,
             metadata=metadata,
             click_url=event_data.get("url"),
             user_agent=delivery_info.get("user_agent"),

docs/esps/unisender_go.rst

@@ -329,19 +329,8 @@ Status tracking webhooks
 ------------------------
 
 If you are using Anymail's normalized :ref:`status tracking <event-tracking>`, add
-the url in Unisender Go's dashboard. Where to set the webhook depends on where
-you got your :setting:`UNISENDER_GO_API_KEY <ANYMAIL_UNISENDER_GO_API_KEY>`:
-
-* If you are using an account-level API key, configure the webhook
-  under Settings > Webhooks (Настройки > Вебхуки).
-* If you are using a project-level API key, configure the webhook
-  under Settings > Projects (Настройки > Проекты).
-
-(If you try to mix account-level and project-level API keys and webhooks,
-webhook signature validation will fail, and you'll get
-:exc:`~anymail.exceptions.AnymailWebhookValidationFailure` errors.)
-
-Enter these settings for the webhook:
+the url in Unisender Go's dashboard under Settings > Webhooks (Настройки > Вебхуки).
+Create a webhook with these settings:
 
 *  **Notification Url:**
 
@@ -350,7 +339,8 @@ Enter these settings for the webhook:
    where *yoursite.example.com* is your Django site.
 
 *  **Status:** set to "Active" if you have already deployed your Django project
-   with Anymail installed. Otherwise set to "Inactive" and update after you deploy.
+   with Anymail installed. Otherwise set to "Inactive" and wait to activate it
+   until you deploy.
 
    (Unisender Go performs a GET request to verify the webhook URL
    when it is marked active.)
@@ -361,8 +351,9 @@ Enter these settings for the webhook:
    with a mod_deflate *input* filter---you could also use "json_post_compressed."
    Most web servers do not handle compressed input by default.)
 
-*  **Events:** your choice. Anymail supports any combination of ``sent, delivered,
-   soft_bounced, hard_bounced, opened, clicked, unsubscribed, subscribed, spam``.
+*  **Events:** your choice. Anymail supports any combination of ``sent``,
+   ``delivered``, ``soft_bounced``, ``hard_bounced``, ``opened``, ``clicked``,
+   ``unsubscribed``, ``subscribed``, and/or ``spam``.
 
    Anymail does not support Unisender Go's ``spam_block`` events (but will ignore
    them if you accidentally include it).
@@ -395,9 +386,17 @@ Enter these settings for the webhook:
    :attr:`~anymail.signals.AnymailTrackingEvent.user_agent` or
    :attr:`~anymail.signals.AnymailTrackingEvent.click_url`.)
 
+*  **Selected project:** Must match the project for your Anymail
+   :setting:`UNISENDER_GO_API_KEY <ANYMAIL_UNISENDER_GO_API_KEY>` setting,
+   if projects are enabled for your account and you are using a project level
+   API key. Leave blank if you are using your account level API key.
+
+   This affects webhook signing. If the selected project does not match your API key,
+   you'll get :exc:`~anymail.exceptions.AnymailWebhookValidationFailure` errors.
+
 Note that Unisender Go does not deliver tracking events for recipient
 addresses that are blocked at send time. You must check the message's
-:attr:`anymail_status.recipients[recipient_email].message_id <anymail.message.AnymailStatus.recipients>`
+:attr:`anymail_status.recipients[recipient_email].status <anymail.message.AnymailStatus.recipients>`
 immediately after sending to detect rejected recipients.
 
 Unisender Go implements webhook signing on the entire event payload,

tests/test_files/unisender-go-tracking-test-payload.json.raw

@@ -0,0 +1 @@
+{"auth":"b3cb4d6aef9d07095805c39e792e0542","events_by_user":[{"user_id":5960727,"project_name":"Testing","project_id":"6862471","events":[{"event_name":"transactional_email_status","event_data":{"job_id":"1sn15Z-0007Le-GtVN","email":"[email protected]","status":"clicked","event_time":"2024-09-08 19:56:41","url":"https:\/\/example.com","delivery_info":{"user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/128.0.0.0 Safari\/537.36","ip":"66.179.153.169"}}}]}]}