diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..b52a55f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,142 @@ +name: jemail CI + +on: + push: + branches: + - main + tags: + - v* + pull_request: + +jobs: + tests: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + PYTHONPATH: "src" + strategy: + matrix: + # https://github.com/actions/python-versions/blob/main/versions-manifest.json + python-version: [3.8, 3.9, "3.10", "3.11", "3.12", "3.13.0-beta.3"] + django-version: + - "Django>=4.2,<5.0" + - "Django>=5.0,<5.1" + - "Django==5.1b1" + exclude: + - django-version: "Django>=5.0,<5.1" + python-version: 3.8 + - django-version: "Django>=5.0,<5.1" + python-version: 3.9 + - django-version: "Django==5.1b1" + python-version: 3.8 + - django-version: "Django==5.1b1" + python-version: 3.9 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + if: "!endsWith(matrix.python-version, '-dev')" + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: "pyproject.toml" + - name: Install deps + run: | + python -m pip install -e .[test] + python -m pip install "${{ matrix.django-version }}" ${{ matrix.drf }} + - run: pytest + + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + PYTHONPATH: "src" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + cache-dependency-path: "pyproject.toml" + - uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + - run: python -m pip install -e .[dev] + - run: pre-commit run --show-diff-on-failure --color=always --all-files + + package: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + cache-dependency-path: "pyproject.toml" + - name: Install dependencies + run: | + python -m pip install hatch + - name: Package + run: python -m hatch build + - name: Upload dist + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist + + publish: + runs-on: ubuntu-latest + needs: [package, tests, lint] + if: startsWith(github.ref, 'refs/tags/v') + timeout-minutes: 5 + environment: release + permissions: + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Set release env + id: release_output + run: | + VERSION="${GITHUB_REF:11}" + BODY=$(awk -v RS='### ' '/'$VERSION'.*/ {print $0}' CHANGELOG.md) + if [[ -z "$BODY" ]]; then + echo "No changelog record for version $VERSION." + fi + BODY="${BODY//'%'/'%25'}" + BODY="${BODY//$'\n'/'%0A'}" + BODY="${BODY//$'\r'/'%0D'}" + echo "::set-output name=VERSION::${VERSION}" + echo "::set-output name=BODY::${BODY}" + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Download dist + uses: actions/download-artifact@v4 + with: + name: dist + path: dist + - name: Install dependencies + run: | + python -m pip install twine + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + - name: Publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + twine upload dist/* + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ steps.release_output.outputs.VERSION }} + body: ${{ steps.release_output.outputs.BODY }} + draft: false + prerelease: ${{ contains(steps.release_output.outputs.VERSION, 'rc') || contains(steps.release_output.outputs.VERSION, 'b') || contains(steps.release_output.outputs.VERSION, 'a') }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..27093de --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.coverage +.envrc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..ca390e6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.1 + hooks: + - id: ruff + language: system + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + language: system + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: debug-statements + - id: check-added-large-files + - id: check-merge-conflict + - id: mixed-line-ending + args: ["--fix=lf"] + + - repo: local + hooks: + - id: mypy + name: mypy + language: system + entry: mypy + args: [src, tests] + pass_filenames: false diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4df0e12 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +## Change Log + +### Unreleased + +- Initial release diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..11c886a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2020 Konstantin Alekseev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..095c6a9 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# Jemail + +Django app to store emails in db + + +## Installation + +```sh +pip install jemail +pip install django-anymail[sendgrid] +``` + +Anymail used as email backend, for now jemail tested only with `sendgrid` backend. + +Update settings with: + +```python +EMAIL_BACKEND = "anymail.backends.sendgrid.EmailBackend" + +# defaults +JEMAIL = { + "METADATA_ID_KEY": "message_id", # tag name for EmailMessage.pk in message metadata + "HTML_MESSAGE_UPLOAD_TO": "emails/messages/", # path to save html messages to + "ATTACHMENT_UPLOAD_TO": "emails/attachments/", # path to save attachments to + # "IMPORT_HTML_MESSAGE_UPLOAD_TO": "myproject.utils.message_upload_to", # callable option + # "IMPORT_ATTACHMENT_UPLOAD_TO": "myproject.utils.attachment_upload_to", # callable option +} +``` + +## Usage + +```python +from jemail import EmailMessage, EmailAttachment + +# save email in db +message = EmailMessage.objects.create_with_objects( + from_email='no-reply@example.com', + to=['user@example.com'], + subject='Subject', + body='Hi User,...', + html_message='

Hi User...', + cc=['cc@example.com'], + reply_to='support@example.com', + attachments=[EmailAttachment.objects.create( + filename='doc.pdf', + mimetype='application/pdf', + file=ContentFile(b'...', name='doc.pdf') + )], + +# build EmailMultiAlternatives from db +msg = message.build_message() +# send email +msg.send() +) +``` + +## Development + +nix-direnv: + +```sh +echo "use flake" >> .envrc +direnv allow +app.install +pytest +``` + +nix: + +```sh +nix develop +app.install +pytest +``` + +uv: + +```sh +uv -q venv .venv +source .venv/bin/activate +uv pip install -e .[dev,test] +pre-commit install +pytest +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..bb7f2c7 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1716948383, + "narHash": "sha256-SzDKxseEcHR5KzPXLwsemyTR/kaM9whxeiJohbL04rs=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "ad57eef4ef0659193044870c731987a6df5cf56b", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..928a62f --- /dev/null +++ b/flake.nix @@ -0,0 +1,46 @@ +{ + description = "jemail"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = + { + self, + nixpkgs, + flake-utils, + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { inherit system; }; + app-test = pkgs.writeShellScriptBin "app.test" ''pytest $@''; + app-install = pkgs.writeShellScriptBin "app.install" ''uv pip install -e .[dev,test] && pre-commit install''; + app-lint = pkgs.writeShellScriptBin "app.lint" ''pre-commit run -a''; + in + { + devShells.default = pkgs.mkShell { + packages = [ + pkgs.python312 + pkgs.uv + pkgs.pyright + ]; + buildInputs = [ + app-test + app-install + app-lint + ]; + shellHook = '' + export PYTHONUNBUFFERED=1; + export PYTHONPATH=src; + export DJANGO_SETTINGS_MODULE=tests.settings; + export VIRTUAL_ENV="$(pwd)/.venv" + [[ -d $VIRTUAL_ENV ]] || uv -q venv $VIRTUAL_ENV + export PATH="$VIRTUAL_ENV/bin":$PATH + ''; + }; + } + ); +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..16e18ba --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,125 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "jemail" +dynamic = ["version"] +description = "Django app to store emails in db" +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Konstantin Alekseev", email = "mail@kalekseev.com" }, +] +keywords = [ + "email", + "django", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Web Environment", + "Framework :: Django :: 4.2", + "Framework :: Django :: 5.0", + "Framework :: Django :: 5.1", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Typing :: Typed", +] +dependencies = [ + "django>=4.2", + "django-anymail", + "html2text", +] + + +[project.optional-dependencies] +dev = [ + "django-stubs", + "mypy", + "pdbpp", + "pre-commit", + "ruff", +] +test = [ + "pytest", + "pytest-cov", + "pytest-django", +] + +[project.urls] +Repository = "https://github.com/kotify/jemail" +Changelog = "https://github.com/kotify/jemail/blob/main/CHANGELOG.md" + +[tool.hatch.version] +source = "vcs" +raw-options = { local_scheme = "no-local-version" } + +[tool.hatch.build.targets.sdist] +include = ["/src"] +[tool.hatch.build.targets.wheel] +packages = ["src/jemail"] + +[tool.ruff] +src = ["src"] +target-version = "py38" +[tool.ruff.lint] +select = [ + 'B', + 'C', + 'E', + 'F', + 'N', + 'W', + 'UP', + 'RUF', + 'INP', + 'I', + 'TCH', +] +ignore = [ + 'E501', + 'B904', + 'B905', + 'RUF012', +] +extend-safe-fixes = ["TCH"] + +[tool.pytest.ini_options] +addopts = "-p no:doctest --cov=jemail --cov-branch --ds=tests.settings" +django_find_project = false +pythonpath = "." + +[tool.pyright] +include = ["src", "tests"] + +[tool.mypy] +plugins = ["mypy_django_plugin.main"] + +disallow_untyped_defs = true +check_untyped_defs = true +ignore_missing_imports = true +implicit_reexport = true +strict_equality = true +warn_unreachable = true +show_error_codes = true + +no_implicit_optional = true +strict_optional = true +warn_no_return = true +warn_redundant_casts = true +warn_unused_ignores = true + +[[tool.mypy.overrides]] +module = ["tests.*"] +disallow_untyped_defs = false + +[tool.django-stubs] +django_settings_module = "tests.settings" diff --git a/src/jemail/__init__.py b/src/jemail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/jemail/apps.py b/src/jemail/apps.py new file mode 100644 index 0000000..35b9b5d --- /dev/null +++ b/src/jemail/apps.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +from django.apps import AppConfig + + +class JemailConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "jemail" diff --git a/src/jemail/migrations/0001_initial.py b/src/jemail/migrations/0001_initial.py new file mode 100644 index 0000000..be3cd66 --- /dev/null +++ b/src/jemail/migrations/0001_initial.py @@ -0,0 +1,250 @@ +# Generated by Django 5.0.7 on 2024-07-10 05:03 + +import django.db.models.deletion +import django.db.models.functions.text +from django.conf import settings +from django.db import migrations, models + +import jemail.settings + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="EmailAttachment", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField(auto_now=True, verbose_name="created at"), + ), + ("filename", models.CharField(max_length=256, verbose_name="name")), + ( + "file", + models.FileField( + max_length=512, + upload_to=jemail.settings.ATTACHMENT_UPLOAD_TO, + verbose_name="file", + ), + ), + ( + "mimetype", + models.CharField(max_length=128, verbose_name="MIME type"), + ), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + ], + ), + migrations.CreateModel( + name="EmailMessage", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField(auto_now=True, verbose_name="created at"), + ), + ( + "from_email", + models.EmailField(max_length=254, verbose_name="from email"), + ), + ( + "from_email_name", + models.CharField(max_length=128, verbose_name="from email name"), + ), + ( + "reply_to", + models.EmailField( + blank=True, max_length=254, verbose_name="reply-to email" + ), + ), + ( + "reply_to_name", + models.CharField( + blank=True, max_length=128, verbose_name="reply-to name" + ), + ), + ("subject", models.TextField(verbose_name="email subject")), + ("body", models.TextField(blank=True, verbose_name="email text")), + ( + "html_message", + models.FileField( + blank=True, + upload_to=jemail.settings.HTML_MESSAGE_UPLOAD_TO, + verbose_name="html message", + ), + ), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + ], + ), + migrations.CreateModel( + name="EmailMessageAttachment", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "attachment", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="jemail.emailattachment", + verbose_name="attachment", + ), + ), + ( + "message", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="jemail.emailmessage", + verbose_name="message", + ), + ), + ], + ), + migrations.AddField( + model_name="emailmessage", + name="attachments", + field=models.ManyToManyField( + related_name="messages", + through="jemail.EmailMessageAttachment", + to="jemail.emailattachment", + verbose_name="attachments", + ), + ), + migrations.CreateModel( + name="EmailRecipient", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "address", + models.EmailField(max_length=254, verbose_name="email address"), + ), + ( + "provider_id", + models.CharField( + max_length=128, + verbose_name="mail service provider recipient's id", + ), + ), + ( + "kind", + models.PositiveSmallIntegerField( + choices=[(0, "to"), (1, "cc"), (2, "bcc")], + verbose_name="recipient kind", + ), + ), + ( + "status", + models.CharField( + max_length=32, verbose_name="mail delivery status" + ), + ), + ( + "timestamp", + models.DateTimeField( + null=True, verbose_name="latest delivery event time" + ), + ), + ( + "clicks_count", + models.PositiveSmallIntegerField( + default=0, verbose_name="clicks count" + ), + ), + ( + "opens_count", + models.PositiveSmallIntegerField( + default=0, verbose_name="opens count" + ), + ), + ( + "message", + models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="recipients", + to="jemail.emailmessage", + verbose_name="message", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="emailmessageattachment", + constraint=models.UniqueConstraint( + fields=("message", "attachment"), + name="jemail_unique_message_attachment", + ), + ), + migrations.AddConstraint( + model_name="emailrecipient", + constraint=models.UniqueConstraint( + fields=("message", "address"), + name="jemail_message_has_unique_recipients", + ), + ), + migrations.AddConstraint( + model_name="emailrecipient", + constraint=models.CheckConstraint( + check=models.Q( + ("address", django.db.models.functions.text.Lower("address")) + ), + name="jemail_address_in_lowercase", + ), + ), + ] diff --git a/src/jemail/migrations/__init__.py b/src/jemail/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/jemail/models.py b/src/jemail/models.py new file mode 100644 index 0000000..fd65a4b --- /dev/null +++ b/src/jemail/models.py @@ -0,0 +1,432 @@ +from __future__ import annotations + +import email.utils +import logging +from typing import TYPE_CHECKING, Any, ClassVar, Protocol, cast + +import django.dispatch +import html2text +from anymail.message import AnymailMessageMixin, AnymailRecipientStatus +from anymail.signals import EventType +from django.conf import settings as django_settings +from django.core.files.base import ContentFile +from django.core.mail.message import EmailMultiAlternatives +from django.db import models, transaction +from django.db.models.functions import Lower +from django.utils.translation import gettext_lazy as _ + +from . import settings + +logger = logging.getLogger(__name__) +jemail_message_status_update = django.dispatch.Signal() + +if TYPE_CHECKING: + from collections.abc import Sequence + + +class EmailRecipientKind(models.IntegerChoices): + TO = 0, "to" + CC = 1, "cc" + BCC = 2, "bcc" + + +class TrackingEventProtocol(Protocol): + recipient: str + timestamp: Any + event_type: Any # EventType + metadata: dict[str, Any] + + +class EmailAttachment(models.Model): + created_at = models.DateTimeField(_("created at"), auto_now=True, editable=False) + created_by = models.ForeignKey( + django_settings.AUTH_USER_MODEL, + verbose_name=_("created by"), + related_name="+", + on_delete=models.SET_NULL, + null=True, + ) + filename = models.CharField(_("name"), max_length=256) + file = models.FileField( + verbose_name=_("file"), + upload_to=settings.ATTACHMENT_UPLOAD_TO, + max_length=512, + ) + mimetype = models.CharField(_("MIME type"), max_length=128) + + +def _normalize_email_list(addrs: Sequence[str], seen: Sequence[str]) -> list[str]: + result: list[str] = [] + for addr in addrs: + laddr = addr.lower() + if laddr not in result and laddr not in seen: + result.append(laddr) + return result + + +def _fix_email_recipient_duplication( + to: Sequence[str], + cc: Sequence[str] | None = None, + bcc: Sequence[str] | None = None, +) -> tuple[list[str], list[str], list[str]]: + to = _normalize_email_list(to, []) + cc = _normalize_email_list(cc or [], to) + bcc = _normalize_email_list(bcc or [], to + (cc or [])) + return to, cc, bcc + + +class EmailMessageQuerySet(models.QuerySet["EmailMessage"]): + def with_has_attachments(self) -> EmailMessageQuerySet: + return self.annotate( + has_attachments=models.Exists( + EmailMessageAttachment.objects.filter(message_id=models.OuterRef("pk")) + ) + ) + + def build_messages( + self, *, html_message: str | bytes | None = None + ) -> list[JemailMessage]: + qs = self.prefetch_related("recipients", "attachments") + return [obj.build_message(hint_html_message=html_message) for obj in qs] + + +class EmailMessageManager(models.Manager["EmailMessage"]): + def create_with_objects( + self, + from_email: str, + to: Sequence[str], + subject: str, + body: str, + html_message: str | None = None, + cc: Sequence[str] | None = None, + bcc: Sequence[str] | None = None, + attachments: Sequence[EmailAttachment] | None = None, + reply_to: str | None = None, + ) -> EmailMessage: + to, cc, bcc = _fix_email_recipient_duplication(to, cc, bcc) + from_email_name, from_email = email.utils.getaddresses([from_email])[0] + if reply_to: + reply_to_name, reply_to = email.utils.getaddresses([reply_to])[0] + else: + reply_to_name = reply_to = "" + # optimization to generate html_message path and pass it to create + html_message_path = "" + if html_message is not None: + _em = EmailMessage() + _em.html_message.save( + "body.html", ContentFile(html_message.encode("utf-8")), save=False + ) + html_message_path = _em.html_message.name + message = super().create( + from_email=from_email, + from_email_name=from_email_name, + subject=subject, + body=body, + reply_to=reply_to, + reply_to_name=reply_to_name, + html_message=html_message_path, + ) + # fmt: off + EmailRecipient.objects.bulk_create( + [ + *[EmailRecipient(message=message, address=address, kind=EmailRecipientKind.TO) for address in to], + *[EmailRecipient(message=message, address=address, kind=EmailRecipientKind.CC) for address in cc], + *[EmailRecipient(message=message, address=address, kind=EmailRecipientKind.BCC) for address in bcc], + ] + ) + # fmt: on + if attachments: + message.attachments.set(attachments) + return message + + +class JemailMessage(AnymailMessageMixin, EmailMultiAlternatives): + def __init__(self, dbmessage: EmailMessage, **kwargs: Any): + self.dbmessage = dbmessage + super().__init__(**kwargs) + + def send(self, fail_silently: bool = False) -> int: + result = super().send(fail_silently=fail_silently) + if result == 0 and fail_silently is True: + return result + recipients = {r.address: r for r in self.dbmessage.recipients.all()} + statuses = cast( + dict[str, AnymailRecipientStatus], + self.anymail_status.recipients, + ) + for address, status in statuses.items(): + recipients[address].status = status.status + recipients[address].provider_id = status.message_id + EmailRecipient.objects.filter(status="").bulk_update( + recipients.values(), fields=["status", "message_id"] + ) + return result + + +class EmailMessage(models.Model): + recipients: models.Manager[EmailRecipient] + + created_at = models.DateTimeField(_("created at"), auto_now=True, editable=False) + created_by = models.ForeignKey( + django_settings.AUTH_USER_MODEL, + verbose_name=_("created by"), + related_name="+", + on_delete=models.SET_NULL, + null=True, + ) + from_email = models.EmailField(_("from email")) + from_email_name = models.CharField(_("from email name"), max_length=128) + reply_to = models.EmailField(_("reply-to email"), blank=True) + reply_to_name = models.CharField(_("reply-to name"), blank=True, max_length=128) + subject = models.TextField(_("email subject")) + body = models.TextField(_("email text"), blank=True) + attachments: models.ManyToManyField[EmailAttachment, EmailMessageAttachment] = ( + models.ManyToManyField( + EmailAttachment, + through="EmailMessageAttachment", + verbose_name=_("attachments"), + related_name="messages", + ) + ) + html_message = models.FileField( + verbose_name=_("html message"), + upload_to=settings.HTML_MESSAGE_UPLOAD_TO, + blank=True, + ) + + objects: ClassVar[EmailMessageManager] = EmailMessageManager.from_queryset( # pyright: ignore + EmailMessageQuerySet + )() + + def build_message( + self, + hint_html_message: str | bytes | None = None, + hint_attachments: Sequence[EmailAttachment] | None = None, + ) -> JemailMessage: + _attachments = ( + self.attachments.all() if hint_attachments is None else hint_attachments + ) + attachments = [ + (a.filename, a.file.file.read(), a.mimetype) for a in _attachments + ] + if hint_html_message is None: + html_message = self.html_message.read().decode("utf-8") + elif isinstance(hint_html_message, str): + html_message = hint_html_message + elif isinstance(hint_html_message, bytes): + html_message = hint_html_message.decode("utf-8") + else: + raise ValueError( + f"Type {type(hint_html_message)} is not supported for `hint_html_message`." + ) + body = self.body or html2text.html2text(html_message) + to: list[str] = [] + cc: list[str] = [] + bcc: list[str] = [] + for r in self.recipients.all(): + if r.kind == EmailRecipientKind.TO: + to.append(r.address) + elif r.kind == EmailRecipientKind.CC: + cc.append(r.address) + elif r.kind == EmailRecipientKind.BCC: + bcc.append(r.address) + msg = JemailMessage( + dbmessage=self, + subject=self.subject, + body=body, + from_email=email.utils.formataddr((self.from_email_name, self.from_email)), + to=to, + cc=cc, + bcc=bcc, + attachments=attachments, + reply_to=[email.utils.formataddr((self.reply_to_name, self.reply_to))] + if self.reply_to + else None, + ) + if html_message: + msg.attach_alternative(html_message, "text/html") + msg.metadata = {settings.METADATA_ID_KEY: str(self.pk)} + return msg + + +class EmailRecipient(models.Model): + message_id: int + message = models.ForeignKey( + EmailMessage, + related_name="recipients", + on_delete=models.CASCADE, + verbose_name=_("message"), + db_index=False, + ) + address = models.EmailField(_("email address")) + provider_id = models.CharField( + _("mail service provider recipient's id"), max_length=128 + ) + kind = models.PositiveSmallIntegerField( + _("recipient kind"), choices=EmailRecipientKind.choices + ) + # anymail's EventType + status = models.CharField(max_length=32, verbose_name=_("mail delivery status")) + timestamp = models.DateTimeField( + verbose_name=_("latest delivery event time"), null=True + ) + clicks_count = models.PositiveSmallIntegerField( + verbose_name=_("clicks count"), default=0 + ) + opens_count = models.PositiveSmallIntegerField( + verbose_name=_("opens count"), default=0 + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["message", "address"], + name="jemail_message_has_unique_recipients", + ), + models.CheckConstraint( + check=models.Q(address=Lower("address")), + name="jemail_address_in_lowercase", + ), + ] + + @staticmethod + def is_delivery_event(status: str) -> bool: + return status in { + EventType.BOUNCED, + EventType.DEFERRED, + EventType.DELIVERED, + EventType.REJECTED, + EventType.QUEUED, + } + + def _fill_from_delivery_anymail_event( + self, anymail_event: TrackingEventProtocol + ) -> None: + if ( + # First event handled + self.timestamp is None + # Events may have equal timestamps, but different types. + or self.timestamp <= anymail_event.timestamp + ): + self.status = anymail_event.event_type + self.timestamp = anymail_event.timestamp + + def fill_from_anymail_event(self, anymail_event: TrackingEventProtocol) -> None: + if self.is_delivery_event(anymail_event.event_type): + self._fill_from_delivery_anymail_event(anymail_event) + else: + # Tracking events are the first when they are received + # in the single batch with other delivery events. + # So delivery events will be processed soon and the correct data will be set. + self.status = self.status or EventType.DELIVERED + if anymail_event.event_type == EventType.CLICKED: + self.clicks_count += 1 + if anymail_event.event_type == EventType.OPENED: + self.opens_count += 1 + + +def is_webhook_event_supported(anymail_event: TrackingEventProtocol) -> bool: + return ( + anymail_event.event_type + in [ + EventType.BOUNCED, + EventType.DEFERRED, + EventType.DELIVERED, + EventType.REJECTED, + EventType.QUEUED, + EventType.CLICKED, + EventType.OPENED, + ] + # Email belongs to MagiLoop. + and bool(anymail_event.metadata.get(settings.METADATA_ID_KEY)) + ) + + +# Delivery status allowed transitions. +# Key is the source status, value is allowed destination statuses. +DELIVERY_STATUS_TRANSITION_MAP = { + EventType.QUEUED: [ + EventType.DELIVERED, + EventType.BOUNCED, + EventType.REJECTED, + EventType.DEFERRED, + ], + EventType.DEFERRED: [EventType.DELIVERED, EventType.BOUNCED, EventType.REJECTED], + # It's rare case, but email can be bounced after delivery. + # E.g. "Recipient address rejected: Access denied", "Requested mail action aborted, mailbox not found". + EventType.DELIVERED: [EventType.BOUNCED], + # Final statuses + EventType.BOUNCED: [], + EventType.REJECTED: [], +} + + +def is_status_transition_allowed(current_status: str, new_status: str) -> bool: + """ + Check if tracking object can be transited from one status to another. + + `current_status` is always the delivery status. We store only delivery statuses in db. + `new_status` is any supported status. It's a webhook event status. + """ + if not current_status: + # New object, any status is allowed. + return True + if new_status in [EventType.CLICKED, EventType.OPENED]: + # Tracking events are always allowed. + # Object is transited to delivered state. + return True + return new_status in DELIVERY_STATUS_TRANSITION_MAP[current_status] + + +def process_mail_event(anymail_event: TrackingEventProtocol) -> None: + if not is_webhook_event_supported(anymail_event): + return + message_id = anymail_event.metadata[settings.METADATA_ID_KEY] + address = anymail_event.recipient + # Update `email_tracking.status` only if the `anymail_event.timestamp` is greater than `email_tracking.timestamp` + # or if `email_tracking.timestamp` is empty + # Locking `EmailDialogMessage` with `select_for_update` to prevent status update race condition + with transaction.atomic(): + try: + recipient = ( + EmailRecipient.objects.filter(message_id=message_id, address=address) + .select_for_update() + .get() + ) + except EmailRecipient.DoesNotExist: + logger.warning( + f"Email reciepient for tracking event not found. {settings.METADATA_ID_KEY}: {message_id}, email: {address}" + ) + return + if is_status_transition_allowed(recipient.status, anymail_event.event_type): + recipient.fill_from_anymail_event(anymail_event) + recipient.save() + else: + return + jemail_message_status_update.send_robust( + sender=process_mail_event, recipient=recipient + ) + + +class EmailMessageAttachment(models.Model): + message = models.ForeignKey( + EmailMessage, + verbose_name=_("message"), + related_name="+", + on_delete=models.CASCADE, + db_index=False, + ) + attachment = models.ForeignKey( + EmailAttachment, + verbose_name=_("attachment"), + related_name="+", + on_delete=models.CASCADE, + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=("message", "attachment"), + name="jemail_unique_message_attachment", + ) + ] diff --git a/src/jemail/py.typed b/src/jemail/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/jemail/settings.py b/src/jemail/settings.py new file mode 100644 index 0000000..962034e --- /dev/null +++ b/src/jemail/settings.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import importlib +from typing import TYPE_CHECKING + +from django.conf import settings + +if TYPE_CHECKING: + from .models import EmailAttachment, EmailMessage + +DEFAULTS = { + "METADATA_ID_KEY": "message_id", + "HTML_MESSAGE_UPLOAD_TO": "emails/messages/", + "ATTACHMENT_UPLOAD_TO": "emails/attachments/", +} + +_SETTINGS = { + **DEFAULTS, + **getattr(settings, "JEMAIL", {}), +} + +METADATA_ID_KEY = _SETTINGS["METADATA_ID_KEY"] + + +def HTML_MESSAGE_UPLOAD_TO(obj: EmailMessage, filename: str) -> str: # noqa [N802] + if _SETTINGS.get("IMPORT_HTML_MESSAGE_UPLOAD_TO"): + module, name = _SETTINGS["IMPORT_HTML_MESSAGE_UPLOAD_TO"].rsplit(".", 1) + return getattr(importlib.import_module(module), name)(obj, filename) + return _SETTINGS["HTML_MESSAGE_UPLOAD_TO"] + + +def ATTACHMENT_UPLOAD_TO(obj: EmailAttachment, filename: str) -> str: # noqa [N802] + if _SETTINGS.get("IMPORT_ATTACHMENT_UPLOAD_TO"): + module, name = _SETTINGS["IMPORT_ATTACHMENT_UPLOAD_TO"].rsplit(".", 1) + return getattr(importlib.import_module(module), name)(obj, filename) + return _SETTINGS["ATTACHMENT_UPLOAD_TO"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/example/__init__.py b/tests/example/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/example/apps.py b/tests/example/apps.py new file mode 100644 index 0000000..805c8f1 --- /dev/null +++ b/tests/example/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ExampleConfig(AppConfig): + name = "tests.example" diff --git a/tests/example/models.py b/tests/example/models.py new file mode 100644 index 0000000..3a0be84 --- /dev/null +++ b/tests/example/models.py @@ -0,0 +1,9 @@ +from anymail.signals import tracking as tracking_signal +from django.dispatch import receiver + +from jemail.models import process_mail_event + + +@receiver(tracking_signal) +def r(sender, event, **kwargs): + process_mail_event(event) diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 0000000..3821f0b --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,35 @@ +SECRET_KEY = "random" + +ALLOWED_HOSTS = ["*"] +USE_TZ = True + +INSTALLED_APPS = [ + "django.contrib.sites", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.messages", + "django.contrib.sessions", + "tests.example", + "jemail", +] + +DATABASES = {"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}} +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "OPTIONS": { + "context_processors": [ + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + } +] + +MIDDLEWARE = [ + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", +] + +ROOT_URLCONF = "tests.urls" diff --git a/tests/test_jemail.py b/tests/test_jemail.py new file mode 100644 index 0000000..658cd44 --- /dev/null +++ b/tests/test_jemail.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import datetime as dt + +import pytest +from anymail.signals import AnymailTrackingEvent, EventType +from anymail.signals import tracking as tracking_signal +from anymail.webhooks.sendgrid import SendGridTrackingWebhookView + +import jemail.settings +from jemail.models import ( + EmailMessage, + EmailRecipient, + EmailRecipientKind, + is_status_transition_allowed, + is_webhook_event_supported, +) + + +def from_utc_timestamp(timestamp): + return dt.datetime.fromtimestamp(timestamp, tz=dt.UTC) + + +@pytest.fixture(autouse=True) +def set_webhook_secret(settings): + # Just to disable warning. + settings.ANYMAIL_WEBHOOK_SECRET = "abc:123" + + +def test_is_webhook_event_supported(): + # No log ids. + anymail_event = AnymailTrackingEvent(event_type=EventType.CLICKED) + assert not is_webhook_event_supported(anymail_event) + # Log ids are empty. + anymail_event = AnymailTrackingEvent( + event_type=EventType.CLICKED, metadata={jemail.settings.METADATA_ID_KEY: ""} + ) + assert not is_webhook_event_supported(anymail_event) + # OK. + anymail_event = AnymailTrackingEvent( + event_type=EventType.CLICKED, metadata={jemail.settings.METADATA_ID_KEY: "abc"} + ) + assert is_webhook_event_supported(anymail_event) + # Event is not supported. + anymail_event = AnymailTrackingEvent( + event_type=EventType.COMPLAINED, + metadata={jemail.settings.METADATA_ID_KEY: "abc"}, + ) + assert not is_webhook_event_supported(anymail_event) + + +def test_is_status_transition_allowed(): + # Empty + assert is_status_transition_allowed("", "any") + # Tracking events + assert is_status_transition_allowed("any", EventType.CLICKED) + assert is_status_transition_allowed("any", EventType.OPENED) + # To deferred + assert is_status_transition_allowed(EventType.QUEUED, EventType.DEFERRED) + # To delivered + assert is_status_transition_allowed(EventType.QUEUED, EventType.DELIVERED) + assert is_status_transition_allowed(EventType.DEFERRED, EventType.DELIVERED) + # To bounced + assert is_status_transition_allowed(EventType.QUEUED, EventType.BOUNCED) + assert is_status_transition_allowed(EventType.DEFERRED, EventType.BOUNCED) + assert is_status_transition_allowed(EventType.DELIVERED, EventType.BOUNCED) + # To rejected + assert is_status_transition_allowed(EventType.QUEUED, EventType.REJECTED) + assert is_status_transition_allowed(EventType.DEFERRED, EventType.REJECTED) + # Impossible from deferred + assert not is_status_transition_allowed(EventType.DEFERRED, EventType.QUEUED) + # Impossible from delivered + assert not is_status_transition_allowed(EventType.DELIVERED, EventType.QUEUED) + assert not is_status_transition_allowed(EventType.DELIVERED, EventType.DEFERRED) + assert not is_status_transition_allowed(EventType.DELIVERED, EventType.REJECTED) + # Impossible from bounced + assert not is_status_transition_allowed(EventType.BOUNCED, EventType.QUEUED) + assert not is_status_transition_allowed(EventType.BOUNCED, EventType.DEFERRED) + assert not is_status_transition_allowed(EventType.BOUNCED, EventType.DELIVERED) + assert not is_status_transition_allowed(EventType.BOUNCED, EventType.REJECTED) + # Impossible from rejected + assert not is_status_transition_allowed(EventType.REJECTED, EventType.QUEUED) + assert not is_status_transition_allowed(EventType.REJECTED, EventType.DEFERRED) + assert not is_status_transition_allowed(EventType.REJECTED, EventType.BOUNCED) + assert not is_status_transition_allowed(EventType.REJECTED, EventType.DELIVERED) + # To the same status + assert not is_status_transition_allowed(EventType.QUEUED, EventType.QUEUED) + assert not is_status_transition_allowed(EventType.DEFERRED, EventType.DEFERRED) + assert not is_status_transition_allowed(EventType.BOUNCED, EventType.BOUNCED) + assert not is_status_transition_allowed(EventType.REJECTED, EventType.REJECTED) + assert not is_status_transition_allowed(EventType.DELIVERED, EventType.DELIVERED) + + +def test_fill_from_anymail_event_tracking_not_exist(): + anymail_event = AnymailTrackingEvent( + event_type=EventType.BOUNCED, + timestamp=from_utc_timestamp(1), + esp_event={"data": "abc"}, + recipient="test@example.com", + ) + recipient = EmailRecipient(timestamp=None) + recipient.fill_from_anymail_event(anymail_event) + assert recipient.status == EventType.BOUNCED + assert recipient.timestamp + assert recipient.timestamp.timestamp() == 1 + assert recipient.clicks_count == 0 + assert recipient.opens_count == 0 + + +def test_fill_from_anymail_event_old_event(): + anymail_event = AnymailTrackingEvent( + event_type=EventType.QUEUED, timestamp=from_utc_timestamp(1) + ) + recipient = EmailRecipient( + status=EventType.BOUNCED, timestamp=from_utc_timestamp(2) + ) + recipient.fill_from_anymail_event(anymail_event) + assert recipient.status == EventType.BOUNCED + assert recipient.timestamp + assert recipient.timestamp.timestamp() == 2 + + +def test_fill_from_anymail_event_new_event(): + anymail_event = AnymailTrackingEvent( + event_type=EventType.REJECTED, timestamp=from_utc_timestamp(2) + ) + recipient = EmailRecipient(status=EventType.QUEUED, timestamp=from_utc_timestamp(1)) + recipient.fill_from_anymail_event(anymail_event) + assert recipient.status == EventType.REJECTED + assert recipient.timestamp + assert recipient.timestamp.timestamp() == 2 + + +def test_fill_from_anymail_event_click_event(): + anymail_event = AnymailTrackingEvent( + event_type=EventType.CLICKED, timestamp=from_utc_timestamp(2) + ) + recipient = EmailRecipient(status=EventType.QUEUED, timestamp=from_utc_timestamp(1)) + recipient.fill_from_anymail_event(anymail_event) + assert recipient.status == EventType.QUEUED + assert recipient.timestamp + assert recipient.timestamp.timestamp() == 1 + assert recipient.clicks_count == 1 + assert recipient.opens_count == 0 + + +def test_fill_from_anymail_event_first_event_is_open(): + """Test that the first open event is handled.""" + anymail_event = AnymailTrackingEvent( + event_type=EventType.OPENED, + timestamp=from_utc_timestamp(1), + esp_event={"data": "abc"}, + recipient="test@example.com", + ) + recipient = EmailRecipient(timestamp=None) + recipient.fill_from_anymail_event(anymail_event) + assert recipient.status == EventType.DELIVERED + assert recipient.address == "" + assert recipient.timestamp is None + assert recipient.clicks_count == 0 + assert recipient.opens_count == 1 + + +def test_fill_from_anymail_event_old_open_event(): + anymail_event = AnymailTrackingEvent( + event_type=EventType.OPENED, timestamp=from_utc_timestamp(1) + ) + recipient = EmailRecipient(status=EventType.QUEUED, timestamp=from_utc_timestamp(2)) + recipient.fill_from_anymail_event(anymail_event) + assert recipient.status == EventType.QUEUED + assert recipient.timestamp + assert recipient.timestamp.timestamp() == 2 + assert recipient.clicks_count == 0 + assert recipient.opens_count == 1 + + +def test_fill_from_anymail_event_open_delivered_events(): + """Test events received in the same batch and include `open` and `delivered` events.""" + open_anymail_event = AnymailTrackingEvent( + event_type=EventType.OPENED, timestamp=from_utc_timestamp(2) + ) + delivered_anymail_event = AnymailTrackingEvent( + event_type=EventType.DELIVERED, timestamp=from_utc_timestamp(1) + ) + recipient = EmailRecipient() + # Most recent event goes first in batch. + recipient.fill_from_anymail_event(open_anymail_event) + recipient.fill_from_anymail_event(delivered_anymail_event) + assert recipient.status == EventType.DELIVERED + assert recipient.timestamp + assert recipient.timestamp.timestamp() == 1 + assert recipient.opens_count == 1 + + +def test_handle_sendgrid_events(db): + """ + Integration test for events handling. + Check sendgrid events are correctly handled. + Test that delivered event is handled before the queued. + """ + email = "test@example.com" + em = EmailMessage.objects.create() + EmailRecipient.objects.create(address=email, message=em, kind=EmailRecipientKind.TO) + data = [ + { + "email": email, + jemail.settings.METADATA_ID_KEY: str(em.pk), + "event": "delivered", + "timestamp": 1, + }, + { + "email": email, + jemail.settings.METADATA_ID_KEY: str(em.pk), + "event": "processed", + "timestamp": 1, + }, + ] + view = SendGridTrackingWebhookView() + qs = EmailRecipient.objects.all() + for item in data: + tracking_signal.send( + sender=view, event=view.esp_to_anymail_event(item), esp_name="SendGrid" + ) + assert qs.count() == 1 + recipient = qs.get() + assert recipient.status == EventType.DELIVERED + assert recipient.address == email + assert recipient.timestamp + assert recipient.timestamp.timestamp() == 1 + assert recipient.clicks_count == 0 + assert recipient.opens_count == 0 diff --git a/tests/urls.py b/tests/urls.py new file mode 100644 index 0000000..8ace68e --- /dev/null +++ b/tests/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from . import views + +urlpatterns = [ + path("", views.index), +] diff --git a/tests/views.py b/tests/views.py new file mode 100644 index 0000000..33e346c --- /dev/null +++ b/tests/views.py @@ -0,0 +1,5 @@ +from django.shortcuts import render + + +def index(request): + return render(request, "index.html")