From e4331d224955a7df8b09063f639304d58163660a Mon Sep 17 00:00:00 2001 From: Mike Edmunds Date: Sun, 8 Sep 2024 15:27:47 -0700 Subject: [PATCH] Unisender Go: Fix status tracking webhook and tests. - Fix signature checking to avoid false validation errors on webhook payloads including `/` (including all "clicked" and most "opened" events). And in general, avoid depending on specific details of Unisender Go's JSON serialization. (Fixes #398.) - Handle "use single event" webhook option (which has a different payload format). - Verify basic auth when Anymail's WEBHOOK_SECRET is used. (This is optional for Unisender Go, since payloads are signed, but it needs to be checked when enabled.) - Treat "soft_bounced" events as "deferred" rather than "bounced", since they will be retried later. - Update validation error to reference Project ID if the webhook is configured for a specific project. - Expose Unisender Go's delivery_status code and unsubscribe form comment as Anymail's normalized event.description. - Update webhook tests based on actual payloads and add several missing tests. - Update docs to clarify webhook use with Unisender Go projects. --- .pre-commit-config.yaml | 1 + CHANGELOG.rst | 13 + anymail/webhooks/unisender_go.py | 138 ++-- docs/esps/unisender_go.rst | 33 +- ...nisender-go-tracking-test-payload.json.raw | 1 + tests/test_unisender_go_webhooks.py | 629 ++++++++++++++---- 6 files changed, 616 insertions(+), 199 deletions(-) create mode 100644 tests/test_files/unisender-go-tracking-test-payload.json.raw diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf1c03a8..ec0c4008 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,7 @@ repos: - id: check-toml - id: check-yaml - id: end-of-file-fixer + exclude: "\\.(bin|raw)$" - id: fix-byte-order-marker - id: fix-encoding-pragma args: [--remove] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8e449b75..f205062e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -36,6 +36,18 @@ Breaking changes * Require **Django 4.0 or later** and Python 3.8 or later. +Fixes +~~~~~ + +* **Unisender Go:** Fix several problems in Anymail's Unisender Go status tracking + webhook. Rework signature checking to fix false validation errors (particularly + on "clicked" and "opened" events). Properly handle "use single event" webhook + option. Correctly verify WEBHOOK_SECRET when set. Provide Unisender Go's + ``delivery_status`` code and unsubscribe form ``comment`` in Anymail's + ``event.description``. Treat soft bounces as "deferred" rather than "bounced". + (Thanks to `@MikeVL`_ for fixing the signature validation problem.) + + Features ~~~~~~~~ @@ -1741,6 +1753,7 @@ Features .. _@mark-mishyn: https://github.com/mark-mishyn .. _@martinezleoml: https://github.com/martinezleoml .. _@mbk-ok: https://github.com/mbk-ok +.. _@MikeVL: https://github.com/MikeVL .. _@mounirmesselmeni: https://github.com/mounirmesselmeni .. _@mwheels: https://github.com/mwheels .. _@nuschk: https://github.com/nuschk diff --git a/anymail/webhooks/unisender_go.py b/anymail/webhooks/unisender_go.py index 52d9e831..0842129c 100644 --- a/anymail/webhooks/unisender_go.py +++ b/anymail/webhooks/unisender_go.py @@ -8,14 +8,14 @@ from django.http import HttpRequest, HttpResponse from django.utils.crypto import constant_time_compare -from anymail.exceptions import AnymailWebhookValidationFailure -from anymail.signals import AnymailTrackingEvent, EventType, RejectReason, tracking -from anymail.utils import get_anymail_setting -from anymail.webhooks.base import AnymailCoreWebhookView +from ..exceptions import AnymailWebhookValidationFailure +from ..signals import AnymailTrackingEvent, EventType, RejectReason, tracking +from ..utils import get_anymail_setting +from .base import AnymailBaseWebhookView -class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView): - """Handler for UniSender delivery and engagement tracking webhooks""" +class UnisenderGoTrackingWebhookView(AnymailBaseWebhookView): + """Handler for Unisender Go delivery and engagement tracking webhooks""" # See https://godocs.unisender.ru/web-api-ref#callback-format for webhook payload @@ -23,6 +23,8 @@ class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView): signal = tracking warn_if_no_basic_auth = False # because we validate against signature + api_key: str | None = None # allows kwargs override + event_types = { "sent": EventType.SENT, "delivered": EventType.DELIVERED, @@ -31,14 +33,14 @@ class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView): "unsubscribed": EventType.UNSUBSCRIBED, "subscribed": EventType.SUBSCRIBED, "spam": EventType.COMPLAINED, - "soft_bounced": EventType.BOUNCED, + "soft_bounced": EventType.DEFERRED, "hard_bounced": EventType.BOUNCED, } reject_reasons = { "err_user_unknown": RejectReason.BOUNCED, "err_user_inactive": RejectReason.BOUNCED, - "err_will_retry": RejectReason.BOUNCED, + "err_will_retry": None, # not rejected "err_mailbox_discarded": RejectReason.BOUNCED, "err_mailbox_full": RejectReason.BOUNCED, "err_spam_rejected": RejectReason.SPAM, @@ -56,49 +58,105 @@ class UnisenderGoTrackingWebhookView(AnymailCoreWebhookView): http_method_names = ["post", "head", "options", "get"] + def __init__(self, **kwargs): + api_key = get_anymail_setting( + "api_key", esp_name=self.esp_name, allow_bare=True, kwargs=kwargs + ) + self.api_key_bytes = api_key.encode("ascii") + super().__init__(**kwargs) + def get( self, request: HttpRequest, *args: typing.Any, **kwargs: typing.Any ) -> HttpResponse: # Unisender Go verifies the webhook with a GET request at configuration time return HttpResponse() + def parse_json_body(self, request: HttpRequest) -> dict | list | None: + # Cache parsed JSON request.body on the request. + if hasattr(request, "_parsed_json"): + parsed = getattr(request, "_parsed_json") + else: + parsed = json.loads(request.body.decode()) + setattr(request, "_parsed_json", parsed) + return parsed + def validate_request(self, request: HttpRequest) -> None: """ - How Unisender GO authenticate: - Hash the whole request body text and replace api key in "auth" field by this hash. - - So it is both auth and encryption. Also, they hash JSON without spaces. + Validate Unisender Go webhook signature: + "MD5 hash of the string body of the message, with the auth value replaced + by the api_key of the user/project whose handler is being called." + https://godocs.unisender.ru/web-api-ref#callback-format """ - request_json = json.loads(request.body.decode("utf-8")) - request_auth = request_json.get("auth", "") - request_json["auth"] = get_anymail_setting( - "api_key", esp_name=self.esp_name, allow_bare=True - ) - json_with_key = json.dumps(request_json, separators=(",", ":")) - - expected_auth = md5(json_with_key.encode("utf-8")).hexdigest() + # This must avoid any assumptions about how Unisender Go serializes JSON + # (key order, spaces, Unicode encoding vs. \u escapes, etc.). But we do + # assume the "auth" field MD5 hash is unique within the serialized JSON, + # so that we can use string replacement to calculate the expected hash. + body = request.body + try: + parsed = self.parse_json_body(request) + actual_auth = parsed["auth"] + actual_auth_bytes = actual_auth.encode() + except (AttributeError, KeyError, ValueError): + raise AnymailWebhookValidationFailure( + "Unisender Go webhook called with invalid payload" + ) - if not constant_time_compare(request_auth, expected_auth): + body_to_sign = body.replace(actual_auth_bytes, self.api_key_bytes) + expected_auth = md5(body_to_sign).hexdigest() + if not constant_time_compare(actual_auth, expected_auth): + # If webhook has a selected project, include the project_id in the error. + try: + project_id = parsed["events_by_user"][0]["project_id"] + except (KeyError, IndexError): + project_id = parsed.get("project_id") # try "single event" payload + is_for_project = f" is for Project ID {project_id}" if project_id else "" raise AnymailWebhookValidationFailure( "Unisender Go webhook called with incorrect signature" + f" (check Anymail UNISENDER_GO_API_KEY setting{is_for_project})" ) def parse_events(self, request: HttpRequest) -> list[AnymailTrackingEvent]: - request_json = json.loads(request.body.decode("utf-8")) - assert len(request_json["events_by_user"]) == 1 # per API docs - esp_events = request_json["events_by_user"][0]["events"] - return [ - self.esp_to_anymail_event(esp_event) - for esp_event in esp_events - if esp_event["event_name"] == "transactional_email_status" - ] - - def esp_to_anymail_event(self, esp_event: dict) -> AnymailTrackingEvent: - event_data = esp_event["event_data"] + parsed = self.parse_json_body(request) + # Unisender Go has two options for webhook payloads. We support both. + try: + events_by_user = parsed["events_by_user"] + except KeyError: + # "Use single event": one flat dict, combining "event_data" fields + # with "event_name", "user_id", "project_id", etc. + if parsed["event_name"] == "transactional_email_status": + esp_events = [parsed] + else: + esp_events = [] + else: + # Not "use single event": we want the "event_data" from all events + # with event_name "transactional_email_status". + assert len(events_by_user) == 1 # "A single element array" per API docs + esp_events = [ + event["event_data"] + for event in events_by_user[0]["events"] + if event["event_name"] == "transactional_email_status" + ] + + return [self.esp_to_anymail_event(esp_event) for esp_event in esp_events] + + def esp_to_anymail_event(self, event_data: dict) -> AnymailTrackingEvent: event_type = self.event_types.get(event_data["status"], EventType.UNKNOWN) - timestamp = datetime.fromisoformat(event_data["event_time"]) - timestamp_utc = timestamp.replace(tzinfo=timezone.utc) - metadata = event_data.get("metadata", {}) + + # Unisender Go does not provide any way to deduplicate webhook calls. + # (There is an "ID" HTTP header, but it has a unique value for every + # webhook call--including retransmissions of earlier failed calls.) + event_id = None + + # event_time is ISO-like, without a stated time zone. (But it's UTC per docs.) + try: + timestamp = datetime.fromisoformat(event_data["event_time"]).replace( + tzinfo=timezone.utc + ) + except KeyError: + timestamp = None + + # Extract our message_id (see backend UNISENDER_GO_GENERATE_MESSAGE_ID). + metadata = event_data.get("metadata", {}).copy() message_id = metadata.pop("anymail_id", event_data.get("job_id")) delivery_info = event_data.get("delivery_info", {}) @@ -108,14 +166,18 @@ def esp_to_anymail_event(self, esp_event: dict) -> AnymailTrackingEvent: else: reject_reason = None + description = delivery_info.get("delivery_status") or event_data.get("comment") + mta_response = delivery_info.get("destination_response") + return AnymailTrackingEvent( event_type=event_type, - timestamp=timestamp_utc, + timestamp=timestamp, message_id=message_id, - event_id=None, + event_id=event_id, recipient=event_data["email"], reject_reason=reject_reason, - mta_response=delivery_info.get("destination_response"), + description=description, + mta_response=mta_response, metadata=metadata, click_url=event_data.get("url"), user_agent=delivery_info.get("user_agent"), diff --git a/docs/esps/unisender_go.rst b/docs/esps/unisender_go.rst index 8e73353a..be5f276b 100644 --- a/docs/esps/unisender_go.rst +++ b/docs/esps/unisender_go.rst @@ -329,19 +329,8 @@ Status tracking webhooks ------------------------ If you are using Anymail's normalized :ref:`status tracking `, add -the url in Unisender Go's dashboard. Where to set the webhook depends on where -you got your :setting:`UNISENDER_GO_API_KEY `: - -* If you are using an account-level API key, configure the webhook - under Settings > Webhooks (Настройки > Вебхуки). -* If you are using a project-level API key, configure the webhook - under Settings > Projects (Настройки > Проекты). - -(If you try to mix account-level and project-level API keys and webhooks, -webhook signature validation will fail, and you'll get -:exc:`~anymail.exceptions.AnymailWebhookValidationFailure` errors.) - -Enter these settings for the webhook: +the url in Unisender Go's dashboard under Settings > Webhooks (Настройки > Вебхуки). +Create a webhook with these settings: * **Notification Url:** @@ -350,7 +339,8 @@ Enter these settings for the webhook: where *yoursite.example.com* is your Django site. * **Status:** set to "Active" if you have already deployed your Django project - with Anymail installed. Otherwise set to "Inactive" and update after you deploy. + with Anymail installed. Otherwise set to "Inactive" and wait to activate it + until you deploy. (Unisender Go performs a GET request to verify the webhook URL when it is marked active.) @@ -361,8 +351,9 @@ Enter these settings for the webhook: with a mod_deflate *input* filter---you could also use "json_post_compressed." Most web servers do not handle compressed input by default.) -* **Events:** your choice. Anymail supports any combination of ``sent, delivered, - soft_bounced, hard_bounced, opened, clicked, unsubscribed, subscribed, spam``. +* **Events:** your choice. Anymail supports any combination of ``sent``, + ``delivered``, ``soft_bounced``, ``hard_bounced``, ``opened``, ``clicked``, + ``unsubscribed``, ``subscribed``, and/or ``spam``. Anymail does not support Unisender Go's ``spam_block`` events (but will ignore them if you accidentally include it). @@ -395,9 +386,17 @@ Enter these settings for the webhook: :attr:`~anymail.signals.AnymailTrackingEvent.user_agent` or :attr:`~anymail.signals.AnymailTrackingEvent.click_url`.) +* **Selected project:** Must match the project for your Anymail + :setting:`UNISENDER_GO_API_KEY ` setting, + if projects are enabled for your account and you are using a project level + API key. Leave blank if you are using your account level API key. + + This affects webhook signing. If the selected project does not match your API key, + you'll get :exc:`~anymail.exceptions.AnymailWebhookValidationFailure` errors. + Note that Unisender Go does not deliver tracking events for recipient addresses that are blocked at send time. You must check the message's -:attr:`anymail_status.recipients[recipient_email].message_id ` +:attr:`anymail_status.recipients[recipient_email].status ` immediately after sending to detect rejected recipients. Unisender Go implements webhook signing on the entire event payload, diff --git a/tests/test_files/unisender-go-tracking-test-payload.json.raw b/tests/test_files/unisender-go-tracking-test-payload.json.raw new file mode 100644 index 00000000..556fa07b --- /dev/null +++ b/tests/test_files/unisender-go-tracking-test-payload.json.raw @@ -0,0 +1 @@ +{"auth":"b3cb4d6aef9d07095805c39e792e0542","events_by_user":[{"user_id":5960727,"project_name":"Testing","project_id":"6862471","events":[{"event_name":"transactional_email_status","event_data":{"job_id":"1sn15Z-0007Le-GtVN","email":"unisendergo@anymail.dev","status":"clicked","event_time":"2024-09-08 19:56:41","url":"https:\/\/example.com","delivery_info":{"user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/128.0.0.0 Safari\/537.36","ip":"66.179.153.169"}}}]}]} \ No newline at end of file diff --git a/tests/test_unisender_go_webhooks.py b/tests/test_unisender_go_webhooks.py index 970800bd..8107cdbc 100644 --- a/tests/test_unisender_go_webhooks.py +++ b/tests/test_unisender_go_webhooks.py @@ -1,177 +1,518 @@ from __future__ import annotations -import datetime import hashlib -import uuid -from datetime import timezone +import json +from copy import deepcopy +from datetime import datetime, timezone +from unittest.mock import ANY -from django.test import RequestFactory, SimpleTestCase, override_settings, tag +from django.test import override_settings, tag -from anymail.exceptions import AnymailWebhookValidationFailure -from anymail.signals import EventType, RejectReason +from anymail.exceptions import AnymailConfigurationError +from anymail.signals import AnymailTrackingEvent from anymail.webhooks.unisender_go import UnisenderGoTrackingWebhookView -EVENT_TYPE = EventType.SENT -EVENT_TIME = "2015-11-30 15:09:42" -EVENT_DATETIME = datetime.datetime(2015, 11, 30, 15, 9, 42, tzinfo=timezone.utc) -JOB_ID = "1a3Q2V-0000OZ-S0" -DELIVERY_RESPONSE = "550 Spam rejected" -UNISENDER_TEST_EMAIL = "recipient.email@example.com" -TEST_API_KEY = "api_key" -TEST_EMAIL_ID = str(uuid.uuid4()) -UNISENDER_TEST_DEFAULT_EXAMPLE = { - "auth": TEST_API_KEY, - "events_by_user": [ - { - "user_id": 456, - "project_id": "6432890213745872", - "project_name": "MyProject", - "events": [ - { - "event_name": "transactional_email_status", - "event_data": { - "job_id": JOB_ID, - "metadata": {"key1": "val1", "anymail_id": TEST_EMAIL_ID}, - "email": UNISENDER_TEST_EMAIL, - "status": EVENT_TYPE, - "event_time": EVENT_TIME, - "url": "http://some.url.com", - "delivery_info": { - "delivery_status": "err_delivery_failed", - "destination_response": DELIVERY_RESPONSE, - "user_agent": ( - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" - ), - "ip": "111.111.111.111", - }, - }, - }, - { - "event_name": "transactional_spam_block", - "event_data": { - "block_time": "YYYY-MM-DD HH:MM:SS", - "block_type": "one_smtp", - "domain": "domain_name", - "SMTP_blocks_count": 8, - "domain_status": "blocked", - }, - }, - ], - } - ], -} -EXAMPLE_WITHOUT_DELIVERY_INFO = { - "auth": "", - "events_by_user": [ - { - "events": [ - { - "event_name": "transactional_email_status", - "event_data": { - "job_id": JOB_ID, - "metadata": {}, - "email": UNISENDER_TEST_EMAIL, - "status": EVENT_TYPE, - "event_time": EVENT_TIME, - }, - } - ] - } - ], -} -REQUEST_JSON = '{"auth":"api_key","key":"value"}' -REQUEST_JSON_MD5 = "8c64386327f53722434f44021a7a0d40" # md5 hash of REQUEST_JSON -REQUEST_DATA_AUTH = {"auth": REQUEST_JSON_MD5, "key": "value"} +from .utils import test_file_content +from .webhook_cases import WebhookBasicAuthTestCase, WebhookTestCase + +TEST_API_KEY = "TEST_API_KEY" -def _request_json_to_dict_with_hashed_key(request_json: bytes) -> dict[str, str]: - new_auth = hashlib.md5(request_json).hexdigest() - return {"auth": new_auth, "key": "value"} +def unisender_go_signed_payload( + data: dict, api_key: str, json_options: dict | None = None +) -> bytes: + """ + Return data serialized to JSON and signed with api_key, using Unisender Go's + webhook signature in the top-level "auth" field: + + "MD5 hash of the string body of the message, with the auth value replaced + by the api_key of the user/project whose handler is being called." + https://godocs.unisender.ru/web-api-ref#callback-format + + Any json_options are passed to json.dumps as kwargs. + + This modifies data to add the "auth" field. + """ + json_options = json_options or {"separators": (",", ":")} + placeholder = "__PLACEHOLDER_FOR_SIGNATURE__" + data["auth"] = placeholder + serialized_data = json.dumps(data, **json_options) + signature = hashlib.md5( + serialized_data.replace(placeholder, api_key).encode() + ).hexdigest() + signed_data = serialized_data.replace(placeholder, signature) + data["auth"] = signature # make available to the caller + return signed_data.encode() + + +class UnisenderGoWebhookTestCase(WebhookTestCase): + def client_post_signed( + self, + url: str, + data: dict, + api_key: str = TEST_API_KEY, + json_options: dict | None = None, + **kwargs, + ): + """ + Return self.client.post(url, serialized json_data) signed with api_key + using json_options. + + Additional kwargs are passed to self.client.post() + """ + signed_data = unisender_go_signed_payload(data, api_key, json_options) + return self.client.post( + url, content_type="application/json", data=signed_data, **kwargs + ) @tag("unisender_go") -class TestUnisenderGoWebhooks(SimpleTestCase): - def test_sent_event(self): - request = RequestFactory().post( - path="/", - data=UNISENDER_TEST_DEFAULT_EXAMPLE, +class UnisenderGoWebhookSettingsTestCase(UnisenderGoWebhookTestCase): + def test_requires_api_key(self): + with self.assertRaisesMessage( + AnymailConfigurationError, "UNISENDER_GO_API_KEY" + ): + self.client_post_signed("/anymail/unisender_go/tracking/", {}) + + @override_settings(ANYMAIL={"UNISENDER_GO_API_KEY": "SETTINGS_API_KEY"}) + def test_view_params_api_key_override(self): + """Webhook api_key can be provided as a view param""" + view = UnisenderGoTrackingWebhookView.as_view(api_key="VIEW_API_KEY") + view_instance = view.view_class(**view.view_initkwargs) + self.assertEqual(view_instance.api_key_bytes, b"VIEW_API_KEY") + + +@tag("unisender_go") +@override_settings( + # (Use expanded setting name because WebhookBasicAuthTestCase sets ANYMAIL={}.) + ANYMAIL_UNISENDER_GO_API_KEY=TEST_API_KEY, +) +class UnisenderGoWebhookSecurityTestCase( + UnisenderGoWebhookTestCase, WebhookBasicAuthTestCase +): + should_warn_if_no_auth = False # because we check webhook signature + + payload = { + # "auth" is added by self.client_post_signed() + "events_by_user": [ + { + "user_id": 123456, + "events": [ + { + "event_name": "transactional_email_status", + "event_data": { + "job_id": "1sn15Z-0007Le-GtVN", + "email": "test@example.com", + "status": "sent", + "metadata": {"can_be_unicode": "Метаданные"}, + "event_time": "2024-09-07 19:27:17", + }, + } + ], + } + ] + } + + def call_webhook(self): + return self.client_post_signed("/anymail/unisender_go/tracking/", self.payload) + + # Additional tests are in WebhookBasicAuthTestCase + + def test_verifies_correct_signature(self): + response = self.client_post_signed( + "/anymail/unisender_go/tracking/", self.payload + ) + self.assertEqual(response.status_code, 200) + + def test_rejects_bad_signature(self): + # This also verifies that the error log references the correct setting to check. + with self.assertLogs() as logs: + response = self.client_post_signed( + "/anymail/unisender_go/tracking/", + self.payload, + api_key="OTHER_API_KEY", + ) + # SuspiciousOperation causes 400 response (even in test client): + self.assertEqual(response.status_code, 400) + self.assertIn("check Anymail UNISENDER_GO_API_KEY", logs.output[0]) + self.assertNotIn("Project ID", logs.output[0]) + + def test_rejects_missing_signature(self): + payload = deepcopy(self.payload) + del payload["auth"] + # Post directly (without signing to add auth): + response = self.client.post( + "/anymail/unisender_go/tracking/", content_type="application/json", + data=json.dumps(payload), ) - view = UnisenderGoTrackingWebhookView() + self.assertEqual(response.status_code, 400) - events = view.parse_events(request) - event = events[0] + def test_rejects_problem_signatures(self): + # Make sure our `body.replace(auth, key)` approach can't be confused + # by invalid payloads. + for bad_auth in ["", " ", ":", "{", '"', 0, None, [], {}]: + with self.subTest(auth=bad_auth): + payload = deepcopy(self.payload) + payload["auth"] = bad_auth + response = self.client.post( + "/anymail/unisender_go/tracking/", + content_type="application/json", + data=json.dumps(payload), + ) + self.assertEqual(response.status_code, 400) - self.assertEqual(len(events), 1) - self.assertEqual(event.event_type, EVENT_TYPE) - self.assertEqual(event.timestamp, EVENT_DATETIME) - self.assertIsNone(event.event_id) - self.assertEqual(event.recipient, UNISENDER_TEST_EMAIL) - self.assertEqual(event.reject_reason, RejectReason.OTHER) - self.assertEqual(event.mta_response, DELIVERY_RESPONSE) - self.assertDictEqual(event.metadata, {"key1": "val1"}) - - def test_without_delivery_info(self): - request = RequestFactory().post( - path="/", - data=EXAMPLE_WITHOUT_DELIVERY_INFO, + def test_error_includes_project_id(self): + # If the webhook has a selected project, mention + # its id in the validation error to assist in debugging. + payload = deepcopy(self.payload) + payload["events_by_user"][0].update( + {"project_id": 999999, "project_name": "Test project"} + ) + with self.assertLogs() as logs: + response = self.client_post_signed( + "/anymail/unisender_go/tracking/", + payload, + api_key="OTHER_API_KEY", + ) + self.assertEqual(response.status_code, 400) + self.assertIn( + "check Anymail UNISENDER_GO_API_KEY setting is for Project ID 999999", + logs.output[0], + ) + + def test_error_includes_project_id_single_event(self): + # Selected project works with "single event" option. + payload = { + "user_id": 123456, + "project_id": 999999, + "project_name": "Test project", + "event_name": "transactional_email_status", + "job_id": "1sn15Z-0007Le-GtVN", + "email": "test@example.com", + "status": "sent", + "event_time": "2024-09-07 19:27:17", + } + with self.assertLogs() as logs: + response = self.client_post_signed( + "/anymail/unisender_go/tracking/", + payload, + api_key="OTHER_API_KEY", + ) + self.assertEqual(response.status_code, 400) + self.assertIn( + "check Anymail UNISENDER_GO_API_KEY setting is for Project ID 999999", + logs.output[0], + ) + + def test_insensitive_to_json_serialization_options(self): + # Our webhook signature verification must not depend on the exact + # details of how Unisender Go serializes the JSON payload. + for json_options in [ + {"separators": None}, + {"ensure_ascii": False}, + {"indent": 4}, + {"sort_keys": True}, + ]: + with self.subTest(options=json_options): + response = self.client_post_signed( + "/anymail/unisender_go/tracking/", + self.payload, + json_options=json_options, + ) + self.assertEqual(response.status_code, 200) + + @override_settings( + # noqa: secret-scanning: this API key has been disabled + ANYMAIL={"UNISENDER_GO_API_KEY": "6mjstx9gwi7qj8eni6m77hfiiw6aifmss154y4ze"} + ) + def test_actual_signed_payload(self): + # Test our signature verification using an actual payload and API key. + payload = test_file_content("unisender-go-tracking-test-payload.json.raw") + # (If an editor or pre-commit forces a trailing newline, the test breaks.) + assert payload[-1] != b"\n", "Test payload must not have end-of-file newline" + response = self.client.post( + "/anymail/unisender_go/tracking/", content_type="application/json", + data=payload, + ) + self.assertEqual(response.status_code, 200) + + +@tag("unisender_go") +@override_settings(ANYMAIL={"UNISENDER_GO_API_KEY": TEST_API_KEY}) +class UnisenderGoTestCase(UnisenderGoWebhookTestCase): + # Most of these tests use Unisender Go's "single event" option for brevity. + # Anymail also supports (and recommends) multiple event webhook option; + # tests for that are toward the end. + + def test_sent_event(self): + raw_event = { + "event_name": "transactional_email_status", + "user_id": 111111, + "project_id": 999999, + "project_name": "Testing", + "job_id": "1smi9f-00057m-86zr", + "metadata": { + "anymail_id": "00001111-2222-3333-4444-555566667777", + "cohort": "group a121", + }, + "email": "recipient@example.com", + "status": "sent", + "event_time": "2024-09-06 23:14:19", + } + response = self.client_post_signed("/anymail/unisender_go/tracking/", raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=UnisenderGoTrackingWebhookView, + event=ANY, + esp_name="Unisender Go", + ) + event = kwargs["event"] + self.assertIsInstance(event, AnymailTrackingEvent) + self.assertEqual(event.event_type, "sent") + self.assertEqual( + event.timestamp, + datetime(2024, 9, 6, 23, 14, 19, tzinfo=timezone.utc), ) - view = UnisenderGoTrackingWebhookView() + # event.message_id matches the message.anymail_status.message_id + # from when the message was sent. It comes from metadata.anymail_id + # if present (added by UNISENDER_GO_GENERATE_MESSAGE_ID). + self.assertEqual(event.message_id, "00001111-2222-3333-4444-555566667777") + # Unisender Go does not include a useful event_id + self.assertIsNone(event.event_id) + self.assertEqual(event.recipient, "recipient@example.com") + # Although Unisender Go's email-send docs claim tags are sent to webhooks, + # its webhook docs don't show tags (and they aren't actually sent 9/2024). + # self.assertEqual(event.tags, ["tag1", "Tag 2"]) + # Our added "anymail_id" should be removed from metadata. + self.assertEqual(event.metadata, {"cohort": "group a121"}) + self.assertEqual(event.esp_event, raw_event) - events = view.parse_events(request) + def test_delivered_event(self): + raw_event = { + "event_name": "transactional_email_status", + "user_id": 111111, + "job_id": "1smi9f-00057m-86zr", + "email": "recipient@example.com", + "status": "delivered", + "event_time": "2024-09-06 23:14:24", + } + response = self.client_post_signed("/anymail/unisender_go/tracking/", raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=UnisenderGoTrackingWebhookView, + event=ANY, + esp_name="Unisender Go", + ) + event = kwargs["event"] + self.assertEqual(event.event_type, "delivered") + # If UNISENDER_GO_GENERATE_MESSAGE_ID not enabled, Unisender Go's + # job_id becomes Anymail's message_id. + self.assertEqual(event.message_id, "1smi9f-00057m-86zr") + self.assertEqual(event.recipient, "recipient@example.com") - self.assertEqual(len(events), 1) - # Without metadata["anymail_id"], message_id uses the job_id. - # (This covers messages sent with "UNISENDER_GO_GENERATE_MESSAGE_ID": False.) - self.assertEqual(events[0].message_id, JOB_ID) + def test_hard_bounced_event(self): + raw_event = { + "event_name": "transactional_email_status", + "status": "hard_bounced", + "email": "bounce@example.com", + "delivery_info": { + "delivery_status": "err_user_unknown", + "destination_response": "555 5.7.1 User unknown 'bounce@example.com'.", + }, + "event_time": "2024-09-06 23:22:40", + } + response = self.client_post_signed("/anymail/unisender_go/tracking/", raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=UnisenderGoTrackingWebhookView, + event=ANY, + esp_name="Unisender Go", + ) + event = kwargs["event"] + self.assertEqual(event.event_type, "bounced") + self.assertEqual(event.recipient, "bounce@example.com") + self.assertEqual(event.reject_reason, "bounced") + self.assertEqual(event.description, "err_user_unknown") + self.assertEqual( + event.mta_response, "555 5.7.1 User unknown 'bounce@example.com'." + ) - @override_settings(ANYMAIL_UNISENDER_GO_API_KEY=TEST_API_KEY) - def test_check_authorization(self): - """Asserts that nothing is failing""" - request_data = _request_json_to_dict_with_hashed_key( - b'{"auth":"api_key","key":"value"}', + def test_soft_bounced_event(self): + raw_event = { + "event_name": "transactional_email_status", + "status": "soft_bounced", + "email": "full@example.com", + "delivery_info": { + "delivery_status": "err_mailbox_full", + "destination_response": "554 5.2.2 Mailbox full 'full@example.com'.", + }, + "event_time": "2024-09-06 23:22:40", + } + response = self.client_post_signed("/anymail/unisender_go/tracking/", raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=UnisenderGoTrackingWebhookView, + event=ANY, + esp_name="Unisender Go", ) - request = RequestFactory().post( - path="/", data=request_data, content_type="application/json" + event = kwargs["event"] + self.assertEqual(event.event_type, "deferred") + self.assertEqual(event.recipient, "full@example.com") + self.assertEqual(event.reject_reason, "bounced") + self.assertEqual(event.description, "err_mailbox_full") + self.assertEqual( + event.mta_response, "554 5.2.2 Mailbox full 'full@example.com'." ) - view = UnisenderGoTrackingWebhookView() - view.validate_request(request) + def test_spam_event(self): + raw_event = { + "event_name": "transactional_email_status", + "status": "spam", + "email": "to@example.com", + "delivery_info": { + "delivery_status": "err_spam_rejected", + "destination_response": "550 Spam rejected", + }, + } + response = self.client_post_signed("/anymail/unisender_go/tracking/", raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=UnisenderGoTrackingWebhookView, + event=ANY, + esp_name="Unisender Go", + ) + event = kwargs["event"] + self.assertEqual(event.event_type, "complained") + self.assertEqual(event.recipient, "to@example.com") + self.assertEqual(event.description, "err_spam_rejected") + self.assertEqual(event.mta_response, "550 Spam rejected") - @override_settings(ANYMAIL_UNISENDER_GO_API_KEY=TEST_API_KEY) - def test_check_authorization__fail__ordinar_quoters(self): - request_json = b"{'auth':'api_key','key':'value'}" - request_data = _request_json_to_dict_with_hashed_key(request_json) - request = RequestFactory().post( - path="/", data=request_data, content_type="application/json" + def test_unsubscribed_event(self): + raw_event = { + "event_name": "transactional_email_status", + "status": "unsubscribed", + "email": "to@example.com", + "comment": "From unsubscribe page 'comment' field", + } + response = self.client_post_signed("/anymail/unisender_go/tracking/", raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=UnisenderGoTrackingWebhookView, + event=ANY, + esp_name="Unisender Go", ) - view = UnisenderGoTrackingWebhookView() + event = kwargs["event"] + self.assertEqual(event.event_type, "unsubscribed") + self.assertEqual(event.recipient, "to@example.com") + self.assertEqual(event.description, "From unsubscribe page 'comment' field") - with self.assertRaises(AnymailWebhookValidationFailure): - view.validate_request(request) + def test_opened_event(self): + raw_event = { + "event_name": "transactional_email_status", + "status": "opened", + "email": "to@example.com", + "delivery_info": { + "user_agent": "... via ggpht.com GoogleImageProxy", + "ip": "10.10.1.333", + }, + } + response = self.client_post_signed("/anymail/unisender_go/tracking/", raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=UnisenderGoTrackingWebhookView, + event=ANY, + esp_name="Unisender Go", + ) + event = kwargs["event"] + self.assertEqual(event.event_type, "opened") + self.assertEqual(event.recipient, "to@example.com") + self.assertEqual(event.user_agent, "... via ggpht.com GoogleImageProxy") - @override_settings(ANYMAIL_UNISENDER_GO_API_KEY=TEST_API_KEY) - def test_check_authorization__fail__spaces_after_semicolon(self): - request_json = b'{"auth": "api_key","key": "value"}' - request_data = _request_json_to_dict_with_hashed_key(request_json) - request = RequestFactory().post( - path="/", data=request_data, content_type="application/json" + def test_clicked_event(self): + raw_event = { + "event_name": "transactional_email_status", + "status": "clicked", + "email": "to@example.com", + "url": "https://example.com", + "delivery_info": { + "user_agent": "Mozilla/5.0 AppleWebKit/537.36 ...", + "ip": "192.168.1.333", + }, + } + response = self.client_post_signed("/anymail/unisender_go/tracking/", raw_event) + self.assertEqual(response.status_code, 200) + kwargs = self.assert_handler_called_once_with( + self.tracking_handler, + sender=UnisenderGoTrackingWebhookView, + event=ANY, + esp_name="Unisender Go", ) - view = UnisenderGoTrackingWebhookView() + event = kwargs["event"] + self.assertEqual(event.event_type, "clicked") + self.assertEqual(event.recipient, "to@example.com") + self.assertEqual(event.click_url, "https://example.com") + self.assertEqual(event.user_agent, "Mozilla/5.0 AppleWebKit/537.36 ...") - with self.assertRaises(AnymailWebhookValidationFailure): - view.validate_request(request) + def test_multiple_event_option(self): + # Payload format is different when "Use single event" not checked. + raw_event = { + "events_by_user": [ + { + "user_id": 111111, + "events": [ + { + "event_name": "transactional_email_status", + "event_data": { + "job_id": "1sn15Z-0007Le-GtVN", + "email": "to@example.com", + "status": "sent", + "event_time": "2024-09-07 19:27:17", + }, + }, + { + "event_name": "transactional_email_status", + "event_data": { + "job_id": "1sn15Z-0007Le-GtVN", + "email": "cc@example.com", + "status": "delivered", + "event_time": "2024-09-07 19:27:17", + }, + }, + ], + } + ], + } + response = self.client_post_signed("/anymail/unisender_go/tracking/", raw_event) + self.assertEqual(response.status_code, 200) + self.assertEqual(self.tracking_handler.call_count, 2) + events = [ + kwargs["event"] for (args, kwargs) in self.tracking_handler.call_args_list + ] + self.assertEqual(events[0].event_type, "sent") + self.assertEqual(events[0].recipient, "to@example.com") + self.assertEqual(events[1].event_type, "delivered") + self.assertEqual(events[1].recipient, "cc@example.com") - @override_settings(ANYMAIL_UNISENDER_GO_API_KEY=TEST_API_KEY) - def test_check_authorization__fail__spaces_after_comma(self): - request_json = b'{"auth":"api_key", "key":"value"}' - request_data = _request_json_to_dict_with_hashed_key(request_json) - request = RequestFactory().post( - path="/", data=request_data, content_type="application/json" + # esp_event is event_data for each event + self.assertEqual( + events[0].esp_event, + raw_event["events_by_user"][0]["events"][0]["event_data"], + ) + self.assertEqual( + events[1].esp_event, + raw_event["events_by_user"][0]["events"][1]["event_data"], ) - view = UnisenderGoTrackingWebhookView() - with self.assertRaises(AnymailWebhookValidationFailure): - view.validate_request(request) + def test_webhook_setup_verification(self): + # Unisender Go verifies webhook at setup time by calling GET. + response = self.client.get("/anymail/unisender_go/tracking/") + self.assertEqual(response.status_code, 200)