diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml new file mode 100644 index 0000000..8591337 --- /dev/null +++ b/.github/workflows/run_tests.yaml @@ -0,0 +1,45 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Tests + +on: + push: + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12.0-rc.3"] + django-version: [">4.2,<5", "==5.0a1"] + exclude: + # exclude Django 5 on Python 3.9 + - python-version: "3.9" + django-version: "==5.0a1" + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Update Pip + run: | + python -m pip install --upgrade pip + - name: Install dependencies + run: | + python -m pip install "Django${{ matrix.django-version }}" + python -m pip install . + python -m pip install -r testing_requirements.txt + - name: Lint + run: | + # stop the build if there are Python syntax errors or undefined names + black . --check + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + ruff . + - name: Test with pytest + run: | + python run_tests.py diff --git a/README.md b/README.md index 3d39d9c..49ea091 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ # DC Signup Form -[![Build Status](https://travis-ci.org/DemocracyClub/dc_signup_form.svg?branch=master)](https://travis-ci.org/DemocracyClub/dc_signup_form) -[![Coverage Status](https://coveralls.io/repos/github/DemocracyClub/dc_signup_form/badge.svg?branch=master)](https://coveralls.io/github/DemocracyClub/dc_signup_form?branch=master) +![Tests](https://github.com/DemocracyClub/dc_signup_form/actions/workflows/run_tests.yaml/badge.svg) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) Django app with Email signup form for use on Democracy Club websites. Currently, this app is used on: * [Democracy Club](https://democracyclub.org.uk) * [Who Can I Vote For?](https://whocanivotefor.co.uk) -* [Where Do I Vote?](https://wheredoivote.co.uk) + ## Installation @@ -18,17 +17,18 @@ git+git://github.com/DemocracyClub/dc_signup_form.git ``` This project depends on [`dc_django_utils`](https://github.com/DemocracyClub/dc_django_utils) -and we assume this is already set up on the project +and we assume this is already set up on the project. + +For deployments in AWS you need to ensure that the role has +`events:put_events` permission on the EventBridge ARN. ## Configuration -For all backends, `dc_signup_form` needs to be in `INSTALLED_APPS` and +`dc_signup_form` needs to be in `INSTALLED_APPS` and `dc_signup_form.context_processors.signup_form` needs to be added as a context processor. -Using AWS EventBridge (recommended): - ```python EMAIL_SIGNUP_BACKEND = "event_bridge" @@ -40,45 +40,15 @@ EMAIL_SIGNUP_BACKEND_KWARGS = { ``` Note that `bus_arn` needs to change for dev, stage and prod accounts. It's -recomended to take this from the environment when running the app. - -Using the remote backend (deprecated): - -```python +recommended to take this from the environment when running the app. -EMAIL_SIGNUP_API_KEY = 'f00b42' -EMAIL_SIGNUP_ENDPOINT = 'https://foo.bar/baz/' -``` - -Using the local backend (deprecated): - -```python -INSTALLED_APPS = [ - ... - 'dc_signup_form.signup_server', -] - - -SENDGRID_API_KEY = 'f00b42' -``` ## Usage -Default routes: - -```python -url(r'^emails/', include('dc_signup_form.urls')), -``` - -Routes for local backend: -```python -url(r'^emails/api_signup/', include('dc_signup_form.signup_server.urls')), -``` - -Custom routes: ```python from dc_signup_form.views import SignupFormView +from dc_signup_form.forms import MailingListSignupForm email_urls = [ url(r'^$', @@ -90,7 +60,8 @@ email_urls = [ 'source': 'EveryElection', }, thanks_message="My custom thanks message", - backend='local_db' + backend='event_bridge', + backend_kwargs=settings.EMAIL_SIGNUP_BACKEND_KWARGS ), name='mailing_list_signup_view'), ] diff --git a/dc_signup_form/backends.py b/dc_signup_form/backends.py index be8ebc4..c3aebf3 100644 --- a/dc_signup_form/backends.py +++ b/dc_signup_form/backends.py @@ -1,8 +1,5 @@ -from dataclasses import dataclass -import requests import json -from django.conf import settings -from dc_signup_form.signup_server.models import SignupQueue +from dataclasses import dataclass class TestBackend: @@ -15,30 +12,6 @@ def submit(self, data, mailing_lists): ) -class LocalDbBackend: - def submit(self, data, mailing_lists): - record = SignupQueue( - email=data["email"], data=data, mailing_lists=mailing_lists - ) - record.save() - - -class RemoteDbBackend: - def submit(self, data, mailing_lists): - key = getattr(settings, "EMAIL_SIGNUP_API_KEY", "") - url = getattr(settings, "EMAIL_SIGNUP_ENDPOINT", "") - - headers = {"Authorization": key} - - payload = { - "data": data, - "mailing_lists": mailing_lists, - } - - r = requests.post(url, data=json.dumps(payload), headers=headers) - r.raise_for_status() - - @dataclass class EmailSubscriber: name: str @@ -56,9 +29,13 @@ def __post_init__(self): raise ValueError("Name cannot contain an email address") if not isinstance(self.list_uuids, list): raise ValueError("'list_uuids' must be a list") - self.list_uuids = list([int(x) for x in self.list_uuids]) + self.list_uuids = [int(x) for x in self.list_uuids] def as_listmonk_json(self): + if self.source: + if not self.extra_context: + self.extra_context = {} + self.extra_context["source_url"] = self.source return { "email": self.email, "name": self.name, @@ -71,7 +48,9 @@ def as_listmonk_json(self): class EventBridgeBackend: def __init__(self, source=None, bus_arn=None): if not source: - raise ValueError("'source' required. This should be the project name") + raise ValueError( + "'source' required. This should be the project name" + ) self.source = source @@ -111,10 +90,12 @@ def list_name_to_list_id(self, dev_mode): def submit(self, data, mailing_lists): list_id_map = self.list_name_to_list_id(self.dev_mode) - list_ids = list([list_id_map[list_name] for list_name in mailing_lists]) - + list_ids = [list_id_map[list_name] for list_name in mailing_lists] subscriber = EmailSubscriber( - email=data["email"], name=data["full_name"], list_uuids=list_ids + email=data["email"], + name=data["full_name"], + list_uuids=list_ids, + source=data.get("source_url"), ) self.client.put_events( diff --git a/dc_signup_form/context_processors.py b/dc_signup_form/context_processors.py index 3916f0a..6f506b2 100644 --- a/dc_signup_form/context_processors.py +++ b/dc_signup_form/context_processors.py @@ -1,5 +1,5 @@ -from .forms import ElectionRemindersSignupForm, MailingListSignupForm -from .constants import MAILING_LIST_FORM_PREFIX, ELECTION_REMINDERS_FORM_PREFIX +from .constants import MAILING_LIST_FORM_PREFIX +from .forms import MailingListSignupForm def signup_form(request): @@ -13,15 +13,6 @@ def signup_form(request): else: mailing_list_form = MailingListSignupForm(initial=initial) - if ELECTION_REMINDERS_FORM_PREFIX in request.POST: - election_reminders_form = ElectionRemindersSignupForm( - initial=initial, - data=request.POST, - ) - else: - election_reminders_form = ElectionRemindersSignupForm(initial=initial) - return { "mailing_list_form": mailing_list_form, - "election_reminders_form": election_reminders_form, } diff --git a/dc_signup_form/forms.py b/dc_signup_form/forms.py index 7c89fec..3fe9766 100644 --- a/dc_signup_form/forms.py +++ b/dc_signup_form/forms.py @@ -1,38 +1,34 @@ from django import forms -from django.core.validators import RegexValidator -from .constants import ELECTION_REMINDERS_FORM_PREFIX, MAILING_LIST_FORM_PREFIX +from .constants import MAILING_LIST_FORM_PREFIX def emails_not_accepted(value): - if '@' in value: - raise forms.ValidationError("Please enter your full name, not your email address.") + if "@" in value: + raise forms.ValidationError( + "Please enter your full name, not your email address." + ) + class EmailSignupForm(forms.Form): full_name = forms.CharField( - required=True, - max_length=1000, + required=True, + max_length=1000, label="Full Name", validators=[emails_not_accepted], - widget=forms.TextInput(attrs={'autocomplete': 'off', 'pattern': '[^@]+', 'title': 'Please enter your full name, not your email address.'}) + widget=forms.TextInput( + attrs={ + "autocomplete": "off", + "pattern": "[^@]+", + "title": "Please enter your full name, not your email address.", + } + ), ) email = forms.EmailField(required=True, max_length=255, label="Email") source_url = forms.CharField(widget=forms.HiddenInput()) -class ElectionRemindersSignupForm(EmailSignupForm): - - prefix = ELECTION_REMINDERS_FORM_PREFIX - - main_list = forms.BooleanField( - required=False, - initial=False, - label="Subscribe to the Democracy Club mailing list", - ) - election_reminders = forms.BooleanField(initial=True, widget=forms.HiddenInput()) - class MailingListSignupForm(EmailSignupForm): - prefix = MAILING_LIST_FORM_PREFIX main_list = forms.BooleanField(initial=True, widget=forms.HiddenInput()) diff --git a/dc_signup_form/signup_server/__init__.py b/dc_signup_form/signup_server/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dc_signup_form/signup_server/admin.py b/dc_signup_form/signup_server/admin.py deleted file mode 100644 index d8d283f..0000000 --- a/dc_signup_form/signup_server/admin.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.contrib import admin -from .models import Token - - -class TokenAdmin(admin.ModelAdmin): - readonly_fields = ("token",) - - -admin.site.register(Token, TokenAdmin) diff --git a/dc_signup_form/signup_server/management/__init__.py b/dc_signup_form/signup_server/management/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dc_signup_form/signup_server/management/commands/__init__.py b/dc_signup_form/signup_server/management/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dc_signup_form/signup_server/management/commands/sync_to_sendgrid.py b/dc_signup_form/signup_server/management/commands/sync_to_sendgrid.py deleted file mode 100644 index cae2ea7..0000000 --- a/dc_signup_form/signup_server/management/commands/sync_to_sendgrid.py +++ /dev/null @@ -1,90 +0,0 @@ -import sys -import time -from django.core.management.base import BaseCommand -from django.db.models import Q, Func, F -from dc_signup_form.signup_server.models import SignupQueue -from dc_signup_form.signup_server.wrappers import DCSendGridWrapper, SendGridAPIError - - -class Command(BaseCommand): - def get_unique_mailing_lists(self): - return ( - SignupQueue.objects.all() - .filter(added=False) - .values("mailing_lists") - .distinct("mailing_lists") - ) - - def get_new_users(self, mailing_lists): - return ( - SignupQueue.objects.all() - .filter( - ~Q(email="testy.mctest@democracyclub.org.uk"), - added=False, - mailing_lists=mailing_lists, - ) - .annotate(email_lower=Func(F("email"), function="lower")) - .distinct("email_lower") - ) - - def handle(self, *args, **kwargs): - # Assume we're going to finish sucessfully. - # If any errors happen, we'll set an error code - exit_code = 0 - - # The easiest way to add new accounts to SendGrid is to batch - # all of the users who will be added to the same combination - # of mailing lists in a single API call - mailing_lists = self.get_unique_mailing_lists() - - sendgrid = DCSendGridWrapper() - - for lsts in mailing_lists: - # new signups for this combination of mailing lists - new_users = self.get_new_users(lsts["mailing_lists"]) - - # add users to contacts db - try: - response = sendgrid.add_users( - sendgrid.get_users_payload([user.data for user in new_users]) - ) - except SendGridAPIError: - continue - - # It is possible some of the emails we've just POSTed worked - # and some may have failed - if response["error_count"] > 0: - # If there were any failures - # log them and exit with a non-zero status - # so we know to deal with it - exit_code = 1 - for error in response["errors"]: - self.stderr.write(error["message"]) - - # We still need to press on and add the ones that worked - # to the relevant mailing lists. - # We'll add any non-failed emails to each list in turn - for mailing_list in lsts["mailing_lists"]: - if mailing_list in sendgrid.SENDGRID_LISTS: - if response["persisted_recipients"]: - sendgrid.add_users_to_lists( - response["persisted_recipients"], - sendgrid.SENDGRID_LISTS[mailing_list], - ) - else: - exit_code = 1 - self.stderr.write("'%s' not found in SENDGRID_LISTS" % mailing_list) - - # have a little rest and make sure we don't hit the rate limits - time.sleep(1.6) - - # mark all the ones that worked as done - for i, user in enumerate(new_users): - if i not in response["error_indices"]: - user.added = True - user.save() - - # have a little rest and make sure we don't hit the rate limits - time.sleep(1.6) - - sys.exit(exit_code) diff --git a/dc_signup_form/signup_server/migrations/0001_initial.py b/dc_signup_form/signup_server/migrations/0001_initial.py deleted file mode 100644 index b1b21f3..0000000 --- a/dc_signup_form/signup_server/migrations/0001_initial.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.10.7 on 2017-08-14 16:49 -from __future__ import unicode_literals - -import dc_signup_form.signup_server.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="SignupQueue", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("email", models.EmailField(max_length=254)), - ( - "data", - dc_signup_form.signup_server.models.BackwardsCompatibleJSONField(), - ), - ( - "mailing_lists", - dc_signup_form.signup_server.models.BackwardsCompatibleJSONField(), - ), - ("added", models.BooleanField(default=False)), - ("created", models.DateTimeField(auto_now_add=True)), - ], - ), - migrations.CreateModel( - name="Token", - fields=[ - ( - "token", - models.CharField(max_length=40, primary_key=True, serialize=False), - ), - ("app_name", models.CharField(max_length=100)), - ], - ), - ] diff --git a/dc_signup_form/signup_server/migrations/0002_auto_20180225_1535.py b/dc_signup_form/signup_server/migrations/0002_auto_20180225_1535.py deleted file mode 100644 index 63f2dc0..0000000 --- a/dc_signup_form/signup_server/migrations/0002_auto_20180225_1535.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -import django.contrib.postgres.fields.jsonb -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("signup_server", "0001_initial"), - ] - - operations = [ - migrations.RunSQL( - """ - ALTER TABLE signup_server_signupqueue - ALTER COLUMN data - SET DATA TYPE jsonb - USING data::jsonb;""", - reverse_sql=""" - ALTER TABLE signup_server_signupqueue - ALTER COLUMN data - SET DATA TYPE json - USING data::json;""", - ), - migrations.RunSQL( - """ - ALTER TABLE signup_server_signupqueue - ALTER COLUMN mailing_lists - SET DATA TYPE jsonb - USING mailing_lists::jsonb;""", - reverse_sql=""" - ALTER TABLE signup_server_signupqueue - ALTER COLUMN mailing_lists - SET DATA TYPE json - USING mailing_lists::json;""", - ), - ] diff --git a/dc_signup_form/signup_server/migrations/__init__.py b/dc_signup_form/signup_server/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dc_signup_form/signup_server/models.py b/dc_signup_form/signup_server/models.py deleted file mode 100644 index d446efe..0000000 --- a/dc_signup_form/signup_server/models.py +++ /dev/null @@ -1,38 +0,0 @@ -import binascii -import os -from django.db import models - -try: - from django.db.models import JSONField -except ImportError: - from django.contrib.postgres.fields import JSONField - - -class BackwardsCompatibleJSONField(JSONField): - # retained for legacy reasons - # (this used to selectively support json/jsonb, but it doesn't any more) - def db_type(self, connection): - return "jsonb" - - -class Token(models.Model): - token = models.CharField(max_length=40, primary_key=True) - app_name = models.CharField(max_length=100) - - def save(self, *args, **kwargs): - self.token = self.generate_token() - return super(Token, self).save(*args, **kwargs) - - def generate_token(self): - return binascii.hexlify(os.urandom(20)).decode() - - def __str__(self): - return self.app_name - - -class SignupQueue(models.Model): - email = models.EmailField() - data = BackwardsCompatibleJSONField() - mailing_lists = BackwardsCompatibleJSONField() - added = models.BooleanField(default=False) - created = models.DateTimeField(auto_now_add=True) diff --git a/dc_signup_form/signup_server/tests/__init__.py b/dc_signup_form/signup_server/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/dc_signup_form/signup_server/tests/test_sync.py b/dc_signup_form/signup_server/tests/test_sync.py deleted file mode 100644 index 6e8ee23..0000000 --- a/dc_signup_form/signup_server/tests/test_sync.py +++ /dev/null @@ -1,110 +0,0 @@ -from django.test import TestCase -from dc_signup_form.signup_server.views import SignupQueue -from dc_signup_form.signup_server.management.commands.sync_to_sendgrid import Command - - -class TestSyncCommand(TestCase): - def test_get_new_users_no_data(self): - command = Command() - users = command.get_new_users(["main_list"]) - self.assertEqual(0, len(list(users))) - - def test_get_new_users_with_data(self): - command = Command() - records = [ - SignupQueue( - email="foo@example.com", - data={"foo": "bar"}, - mailing_lists=["main_list"], - ), - SignupQueue( - email="bar@example.com", - data={"foo": "bar"}, - mailing_lists=["main_list"], - ), - SignupQueue( - email="baz@example.com", - data={"foo": "bar"}, - mailing_lists=["some_other_list"], - ), - ] - for record in records: - record.save() - users = command.get_new_users(["main_list"]) - self.assertEqual(2, len(list(users))) - - def test_get_new_users_with_test_user(self): - command = Command() - records = [ - SignupQueue( - email="foo@example.com", - data={"foo": "bar"}, - mailing_lists=["main_list"], - ), - SignupQueue( - email="bar@example.com", - data={"foo": "bar"}, - mailing_lists=["main_list"], - ), - SignupQueue( - email="baz@example.com", - data={"foo": "bar"}, - mailing_lists=["some_other_list"], - ), - SignupQueue( - email="testy.mctest@democracyclub.org.uk", - data={"foo": "bar"}, - mailing_lists=["main_list"], - ), - ] - for record in records: - record.save() - users = command.get_new_users(["main_list"]) - self.assertEqual(2, len(list(users))) - - def test_unique_mailing_lists_no_data(self): - command = Command() - mailing_lists = command.get_unique_mailing_lists() - self.assertEqual(0, len(list(mailing_lists))) - - def test_unique_mailing_lists_with_data(self): - command = Command() - records = [ - SignupQueue( - email="foo@example.com", - data={"foo": "bar"}, - mailing_lists=["main_list"], - ), - SignupQueue( - email="bar@example.com", - data={"foo": "bar"}, - mailing_lists=["main_list"], - ), - SignupQueue( - email="baz@example.com", - data={"foo": "bar"}, - mailing_lists=["some_other_list", "main_list"], - ), - SignupQueue( - email="qux@example.com", - data={"foo": "bar"}, - mailing_lists=["main_list", "some_other_list"], - ), - SignupQueue( - email="norf@example.com", - data={"foo": "bar"}, - mailing_lists=["main_list", "some_other_list"], - ), - ] - for record in records: - record.save() - mailing_lists = command.get_unique_mailing_lists() - - self.assertEqual(3, len(list(mailing_lists))) - assert {"mailing_lists": ["main_list"]} in list(mailing_lists) - assert {"mailing_lists": ["main_list", "some_other_list"]} in list( - mailing_lists - ) - assert {"mailing_lists": ["some_other_list", "main_list"]} in list( - mailing_lists - ) diff --git a/dc_signup_form/signup_server/tests/test_views.py b/dc_signup_form/signup_server/tests/test_views.py deleted file mode 100644 index 7effcbf..0000000 --- a/dc_signup_form/signup_server/tests/test_views.py +++ /dev/null @@ -1,50 +0,0 @@ -import json -from django.test import TestCase, RequestFactory -from dc_signup_form.signup_server.views import SignupQueue, Token -from dc_signup_form.signup_server.views import email_signup - - -class TestSignupView(TestCase): - def setUp(self): - self.factory = RequestFactory() - - def test_no_auth_header(self): - req = self.factory.post( - "/foo/bar", - content_type="application/json", - data=json.dumps({"foo": "bar"}), - ) - response = email_signup(req) - self.assertEqual(403, response.status_code) - - def test_bad_auth_header(self): - req = self.factory.post( - "/foo/bar", - content_type="application/json", - data=json.dumps({"foo": "bar"}), - HTTP_AUTHORIZATION="foobar", - ) - response = email_signup(req) - self.assertEqual(403, response.status_code) - - def test_valid(self): - # ensure the queue is tempty before we start - self.assertEqual(0, len(SignupQueue.objects.all())) - - token = Token() - token.app_name = "myapp" - token.save() - - req = self.factory.post( - "/foo/bar", - content_type="application/json", - data=json.dumps( - {"data": {"email": "foo@bar.baz"}, "mailing_lists": ["main_list"]} - ), - HTTP_AUTHORIZATION=token.token, - ) - response = email_signup(req) - self.assertEqual(201, response.status_code) - - # now there should be a record in the queue - self.assertEqual(1, len(SignupQueue.objects.all())) diff --git a/dc_signup_form/signup_server/urls.py b/dc_signup_form/signup_server/urls.py deleted file mode 100644 index 9f6e3a1..0000000 --- a/dc_signup_form/signup_server/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -from django.urls import re_path - -from .views import email_signup - -urlpatterns = [ - re_path(r"^$", email_signup, name="email_signup_api_endpoint"), -] diff --git a/dc_signup_form/signup_server/views.py b/dc_signup_form/signup_server/views.py deleted file mode 100644 index f1f200c..0000000 --- a/dc_signup_form/signup_server/views.py +++ /dev/null @@ -1,27 +0,0 @@ -import json -from django.http import HttpResponse -from django.views.decorators.csrf import csrf_exempt -from .models import Token, SignupQueue - - -@csrf_exempt -def email_signup(request): - - # authorize - if "HTTP_AUTHORIZATION" not in request.META: - return HttpResponse(status=403) - try: - Token.objects.get(token=request.META["HTTP_AUTHORIZATION"]) - except Token.DoesNotExist: - return HttpResponse(status=403) - - # save - body = json.loads(request.body.decode("utf-8")) - record = SignupQueue( - email=body["data"]["email"], - data=body["data"], - mailing_lists=body["mailing_lists"], - ) - record.save() - - return HttpResponse(status=201) diff --git a/dc_signup_form/signup_server/wrappers.py b/dc_signup_form/signup_server/wrappers.py deleted file mode 100644 index 5e34739..0000000 --- a/dc_signup_form/signup_server/wrappers.py +++ /dev/null @@ -1,55 +0,0 @@ -import requests -import json -from django.conf import settings - - -class DCSendGridWrapper: - - BASE_URL = "https://api.sendgrid.com/v3" - SENDGRID_LISTS = { - "main_list": 1246575, - "election_reminders": 1342244, - } - - # list of properties we shouldn't pass to SendGrid - # because we don't have a custom field for it - FIELD_BLACKLIST = ["source_url"] - - def __init__(self): - key = getattr(settings, "SENDGRID_API_KEY", "") - self.headers = {"Authorization": "Bearer " + key} - - def get_users_payload(self, users): - payload = [] - for user in users: - data = {} - for key in user: - if key not in self.FIELD_BLACKLIST: - data[key] = user[key] - payload.append(data) - return payload - - def add_users(self, payload): - url = self.BASE_URL + "/contactdb/recipients" - r = requests.post(url, data=json.dumps(payload), headers=self.headers) - r.raise_for_status() - return r.json() - - def get_contact_lists(self): - url = self.BASE_URL + "/contactdb/lists" - r = requests.get(url, headers=self.headers) - # if we get a non-2xx status code, just raise it - r.raise_for_status() - return r.json() - - def add_users_to_lists(self, payload, mailing_list): - url = self.BASE_URL + "/contactdb/lists/{list_id}/recipients".format( - list_id=mailing_list - ) - r = requests.post(url, data=json.dumps(payload), headers=self.headers) - # if we get a non-2xx status code, just raise it - r.raise_for_status() - # if success response payload wil be empty so there is no return val - -class SendGridAPIError(ValueError): - pass \ No newline at end of file diff --git a/dc_signup_form/tests/test_backends.py b/dc_signup_form/tests/test_backends.py index b2805cc..b3f1ec2 100644 --- a/dc_signup_form/tests/test_backends.py +++ b/dc_signup_form/tests/test_backends.py @@ -1,35 +1,18 @@ -import boto3 from django.test import TestCase -from moto import mock_events -from dc_signup_form.backends import LocalDbBackend, EventBridgeBackend -from dc_signup_form.signup_server.views import SignupQueue - - -class TestLocalDbBackend(TestCase): - def test_valid(self): - backend = LocalDbBackend() - - # ensure the queue is tempty before we start - self.assertEqual(0, len(SignupQueue.objects.all())) - - backend.submit({"email": "foo@bar.baz"}, ["main_list"]) - - # now there should be a record in the queue - self.assertEqual(1, len(SignupQueue.objects.all())) - - def test_invalid(self): - backend = LocalDbBackend() - with self.assertRaises(KeyError): - backend.submit({"foo": "bar"}, ["main_list"]) +from dc_signup_form.backends import EventBridgeBackend +from dc_signup_form.tests.utils import mocked_eventbridge class TestEventBridgeBackend(TestCase): - @mock_events def test_valid(self): - bus_arn_name = "arn:testing" - client = boto3.client("events", region_name="eu-west-2") - client.create_event_bus(Name=bus_arn_name) - - backend = EventBridgeBackend(source="UnitTest", bus_arn="arn:testing") - backend.submit({"email": "foo@bar.baz", "full_name": "Test Face"}, ["main_list"]) + with mocked_eventbridge() as get_events: + backend = EventBridgeBackend( + source="UnitTest", bus_arn="arn:testing" + ) + backend.submit( + {"email": "foo@bar.baz", "full_name": "Test Face"}, + ["main_list"], + ) + events = get_events() + self.assertEqual(len(events), 1) diff --git a/dc_signup_form/tests/test_forms.py b/dc_signup_form/tests/test_forms.py index 34b4478..b059c14 100644 --- a/dc_signup_form/tests/test_forms.py +++ b/dc_signup_form/tests/test_forms.py @@ -1,9 +1,9 @@ from django.test import TestCase -from dc_signup_form.forms import ElectionRemindersSignupForm, MailingListSignupForm + from dc_signup_form.constants import ( MAILING_LIST_FORM_PREFIX, - ELECTION_REMINDERS_FORM_PREFIX, ) +from dc_signup_form.forms import MailingListSignupForm def add_data_prefix(prefix, data): @@ -47,7 +47,7 @@ def test_mailing_list_invalid(self): self.assertIn("This field is required.", form.errors["full_name"]) self.assertIn("email", form.errors) self.assertIn("Enter a valid email address.", form.errors["email"]) - + def test_mailing_list_invalid_name(self): data = add_data_prefix( MAILING_LIST_FORM_PREFIX, @@ -60,7 +60,10 @@ def test_mailing_list_invalid_name(self): ) form = MailingListSignupForm(data) self.assertFalse(form.is_valid()) - self.assertIn("Please enter your full name, not your email address.", form.errors["full_name"]) + self.assertIn( + "Please enter your full name, not your email address.", + form.errors["full_name"], + ) def test_mailing_list_required_fields(self): form = MailingListSignupForm(data={}) @@ -73,48 +76,3 @@ def test_mailing_list_required_fields(self): self.assertIn("This field is required.", form.errors["source_url"]) self.assertIn("main_list", form.errors) self.assertIn("This field is required.", form.errors["main_list"]) - - def test_election_reminders_form_valid(self): - data = add_data_prefix( - ELECTION_REMINDERS_FORM_PREFIX, - { - "full_name": "Chad Fernandez", - "email": "chad.fernandez@democracyclub.org.uk", - "source_url": "http://foo.bar/baz", - "main_list": False, - "election_reminders": True, - }, - ) - form = ElectionRemindersSignupForm(data) - if not form.is_valid(): - print(form.errors) - self.assertTrue(form.is_valid()) - - def test_election_reminders_invalid(self): - data = add_data_prefix( - ELECTION_REMINDERS_FORM_PREFIX, - { - "full_name": "", # empty field - "email": "foo", # not an email - "source_url": "http://foo.bar/baz", - "main_list": True, - }, - ) - form = ElectionRemindersSignupForm(data) - self.assertFalse(form.is_valid()) - self.assertIn("full_name", form.errors) - self.assertIn("This field is required.", form.errors["full_name"]) - self.assertIn("email", form.errors) - self.assertIn("Enter a valid email address.", form.errors["email"]) - - def test_election_reminders_required_fields(self): - form = ElectionRemindersSignupForm(data={}) - self.assertFalse(form.is_valid()) - self.assertIn("full_name", form.errors) - self.assertIn("This field is required.", form.errors["full_name"]) - self.assertIn("email", form.errors) - self.assertIn("This field is required.", form.errors["email"]) - self.assertIn("source_url", form.errors) - self.assertIn("This field is required.", form.errors["source_url"]) - self.assertIn("election_reminders", form.errors) - self.assertIn("This field is required.", form.errors["election_reminders"]) diff --git a/dc_signup_form/tests/test_views.py b/dc_signup_form/tests/test_views.py index b7c33ec..30351d8 100644 --- a/dc_signup_form/tests/test_views.py +++ b/dc_signup_form/tests/test_views.py @@ -1,12 +1,13 @@ +import json + from django.test import Client, TestCase -from dc_signup_form.signup_server.models import SignupQueue from dc_signup_form.constants import ( MAILING_LIST_FORM_PREFIX, - ELECTION_REMINDERS_FORM_PREFIX, ) from .test_forms import add_data_prefix +from .utils import mocked_eventbridge class TestView(TestCase): @@ -25,122 +26,73 @@ def test_get_mailing_list_form_view(self): self.assertContains(response, string, html=True) def test_post_mailing_list_form_view_valid(self): - self.assertEqual(0, len(SignupQueue.objects.all())) - c = Client() - response = c.post( - "/emails/mailing_list/", - add_data_prefix( - MAILING_LIST_FORM_PREFIX, - { - "source_url": "/emails/mailing_list/", - "main_list": True, - "full_name": "Chad Fernandez", - "email": "chad.fernandez@example.com", - "mailing_list_form": "", - }, - ), - ) - self.assertEqual(302, response.status_code) - self.assertEqual(1, len(SignupQueue.objects.all())) - - def test_source_url (self): - # add a test to make sure the source url is - # being passed through correctly - self.assertEqual(0, len(SignupQueue.objects.all())) - c = Client() - response = c.post( - "/emails/mailing_list/", - add_data_prefix( - MAILING_LIST_FORM_PREFIX, - { - "source_url": "/emails/mailing_list/", - "main_list": True, - "full_name": "Chad Fernandez", - "email": "chad@test.com", - "mailing_list_form": "", - }, - ), - ) - self.assertEqual(302, response.status_code) - self.assertEqual(1, len(SignupQueue.objects.all())) - self.assertEqual("/emails/mailing_list/", SignupQueue.objects.all()[0].data["source_url"]) - - + with mocked_eventbridge() as get_events: + self.assertEqual(0, len(get_events())) + c = Client() + response = c.post( + "/emails/mailing_list/", + add_data_prefix( + MAILING_LIST_FORM_PREFIX, + { + "source_url": "/emails/mailing_list/", + "main_list": True, + "full_name": "Chad Fernandez", + "email": "chad.fernandez@example.com", + "mailing_list_form": "", + }, + ), + ) + self.assertEqual(302, response.status_code) + self.assertEqual(1, len(get_events())) + def test_source_url(self): + with mocked_eventbridge() as get_events: + # add a test to make sure the source url is + # being passed through correctly + self.assertEqual(0, len(get_events())) + c = Client() + response = c.post( + "/emails/mailing_list/", + add_data_prefix( + MAILING_LIST_FORM_PREFIX, + { + "source_url": "/emails/mailing_list/", + "main_list": True, + "full_name": "Chad Fernandez", + "email": "chad@test.com", + "mailing_list_form": "", + }, + ), + ) + self.assertEqual(302, response.status_code) + events = get_events() + self.assertEqual(1, len(events)) + self.assertEqual( + "/emails/mailing_list/", + json.loads(events[0]["Body"])["detail"]["attribs"][ + "source_url" + ], + ) - def test_post_mailing_list_form_view_invalid(self): - self.assertEqual(0, len(SignupQueue.objects.all())) - c = Client() - - response = c.post( - "/emails/mailing_list/", - add_data_prefix( - MAILING_LIST_FORM_PREFIX, - { - "source_url": "/emails/mailing_list/", - "main_list": True, - "full_name": "Chad Fernandez", - "email": "", - "mailing_list_form": "", - }, - ), - ) - form = response.context[MAILING_LIST_FORM_PREFIX] - self.assertFalse(form.is_valid()) - self.assertEqual(200, response.status_code) - self.assertIn('
', str(response.content)) - self.assertEqual(0, len(SignupQueue.objects.all())) - - def test_get_election_reminders_form_view(self): - c = Client() - response = c.get("/emails/election_reminders/") - self.assertEqual(200, response.status_code) - expected_strings = [ - '', - '', - '', - '', - '', - ] - for string in expected_strings: - with self.subTest(string=string): - self.assertContains(response, string, html=True) - - def test_post_election_reminders_form_view_valid(self): - self.assertEqual(0, len(SignupQueue.objects.all())) - c = Client() - response = c.post( - "/emails/election_reminders/", - add_data_prefix( - ELECTION_REMINDERS_FORM_PREFIX, - { - "source_url": "/emails/election_reminders/", - "election_reminders": True, - "full_name": "Chad Fernandez", - "email": "chad.fernandez@example.com", - "main_list": False, - "election_reminders_form": "", - }, - ), - ) - self.assertEqual(302, response.status_code) - self.assertEqual(1, len(SignupQueue.objects.all())) + with mocked_eventbridge() as get_events: + c = Client() - def test_post_election_reminders_form_view_invalid(self): - self.assertEqual(0, len(SignupQueue.objects.all())) - c = Client() - response = c.post( - "/emails/election_reminders/", - { - "source_url": "/emails/election_reminders/", - "election_reminders": True, - "full_name": "Chad Fernandez", - "email": "", - "main_list": False, - "election_reminders_form": "", - }, - ) - self.assertEqual(200, response.status_code) - self.assertIn('
', str(response.content)) - self.assertEqual(0, len(SignupQueue.objects.all())) + response = c.post( + "/emails/mailing_list/", + add_data_prefix( + MAILING_LIST_FORM_PREFIX, + { + "source_url": "/emails/mailing_list/", + "main_list": True, + "full_name": "Chad Fernandez", + "email": "", + "mailing_list_form": "", + }, + ), + ) + form = response.context[MAILING_LIST_FORM_PREFIX] + self.assertFalse(form.is_valid()) + self.assertEqual(200, response.status_code) + self.assertIn('
', str(response.content)) + self.assertEqual(0, len(get_events())) diff --git a/dc_signup_form/tests/utils.py b/dc_signup_form/tests/utils.py new file mode 100644 index 0000000..9b2b47d --- /dev/null +++ b/dc_signup_form/tests/utils.py @@ -0,0 +1,61 @@ +import contextlib +import json + +import boto3 +from moto import mock_events, mock_sqs + + +@contextlib.contextmanager +def mocked_eventbridge(): + with mock_events(), mock_sqs(): + bus_arn_name = "arn:testing" + client_kwargs = { + "region_name": "eu-west-2", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing", + "aws_session_token": "testing", + } + client = boto3.client("events", **client_kwargs) + client.create_event_bus(Name=bus_arn_name) + + sqs_client = boto3.client("sqs", **client_kwargs) + statement = [ + { + "Effect": "Allow", + "Principal": {"Service": "events.amazonaws.com"}, + "Action": "sqs:SendMessage", + "Resource": "*", + } + ] + policy = json.dumps({"Version": "2012-10-17", "Statement": [statement]}) + QueueName = "TestQueue" + RuleName = "TestRule" + RuleTargetId = "TestTarget" + queue = sqs_client.create_queue( + QueueName=QueueName, Attributes={"Policy": policy} + ) + queueurl = queue["QueueUrl"] + queueattrs = sqs_client.get_queue_attributes( + QueueUrl=queueurl, AttributeNames=["QueueArn"] + ) + queuearn = queueattrs["Attributes"]["QueueArn"] + # eventpattern = {"source": [EventSource], "detail": {"hello": ["world"]}} + client.put_rule( + Name=RuleName, + State="ENABLED", + EventPattern='{"source": ["UnitTest"]}', + EventBusName=bus_arn_name, + ) + client.put_targets( + Rule=RuleName, + Targets=[{"Id": RuleTargetId, "Arn": queuearn}], + EventBusName=bus_arn_name, + ) + + def get_messages(): + resp = sqs_client.receive_message(QueueUrl=queueurl) + if "Messages" in resp: + return resp["Messages"] + return [] + + yield get_messages diff --git a/dc_signup_form/urls.py b/dc_signup_form/urls.py index 4aee7d6..3394cec 100644 --- a/dc_signup_form/urls.py +++ b/dc_signup_form/urls.py @@ -1,6 +1,6 @@ from django.urls import re_path -from .forms import ElectionRemindersSignupForm, MailingListSignupForm +from .forms import MailingListSignupForm from .views import SignupFormView app_name = "dc_signup_form" @@ -13,14 +13,4 @@ ), name="mailing_list_signup_view", ), - re_path( - r"^election_reminders/$", - SignupFormView.as_view( - template_name="email_form/election_reminders_form_view.html", - form_class=ElectionRemindersSignupForm, - get_vars=["postcode"], - thanks_message="Thanks for joining. We'll send you a reminder when there's an upcoming election in your area", - ), - name="election_reminders_signup_view", - ), ] diff --git a/dc_signup_form/views.py b/dc_signup_form/views.py index 2c63466..e36db88 100644 --- a/dc_signup_form/views.py +++ b/dc_signup_form/views.py @@ -1,11 +1,10 @@ from django.contrib import messages -from django.urls import reverse, NoReverseMatch +from django.urls import NoReverseMatch, reverse from django.views.generic import FormView + from .backends import ( - LocalDbBackend, - RemoteDbBackend, - TestBackend, EventBridgeBackend, + TestBackend, ) @@ -26,15 +25,13 @@ class SignupFormView(FormView): "election_reminders", ] # list of the mailing lists we support joining get_vars = [] # list of get vars we want to store with the user - extras = {} # dict of hard-coded key/value pairs we want to store with the user - thanks_message = ( - "Thanks for joining the Democracy Club mailing list. We will be in touch soon!" - ) + extras = ( + {} + ) # dict of hard-coded key/value pairs we want to store with the user + thanks_message = "Thanks for joining the Democracy Club mailing list. We will be in touch soon!" backends = { "test": TestBackend, - "local_db": LocalDbBackend, - "remote_db": RemoteDbBackend, "event_bridge": EventBridgeBackend, } backend = "remote_db" @@ -44,7 +41,7 @@ def get_success_url(self): messages.success(self.request, self.thanks_message) form_name = self.get_form_class().prefix - source_url = self.request.POST.get(f"{form_name}-source_url") + source_url = self.request.POST.get(f"{form_name}-source_url") try: mailing_list_signup_view = reverse( @@ -60,15 +57,14 @@ def get_success_url(self): election_reminders_signup_view = "" source_url_safe = get_http(source_url, host=self.request.get_host()) - + if ( source_url_safe and source_url != mailing_list_signup_view and source_url != election_reminders_signup_view ): return source_url - else: - return "/" + return "/" def form_invalid(self, form): """ @@ -89,7 +85,7 @@ def form_valid(self, form): # mailing lists for lst in self.mailing_lists: - key = "-".join([self.form_class.prefix, lst]) + "-".join([self.form_class.prefix, lst]) if data.pop(lst, False): mailing_lists.append(lst) @@ -103,6 +99,8 @@ def form_valid(self, form): # pass the payload off to a backend object # to deal with persisting the data to a db - self.backends[self.backend](**self.backend_kwargs).submit(data, mailing_lists) + self.backends[self.backend](**self.backend_kwargs).submit( + data, mailing_lists + ) return super(SignupFormView, self).form_valid(form) diff --git a/make_migrations.py b/make_migrations.py index 897d8f2..23cd00e 100644 --- a/make_migrations.py +++ b/make_migrations.py @@ -2,7 +2,6 @@ from django.conf import settings from django.core.management import call_command - if not settings.configured: settings.configure( DEBUG=True, diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..48295b3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[tool.black] +line-length = 80 + +[tool.ruff] +line-length = 80 +ignore = ["E501"] +extend-select = [ + "I", + "C4", + "SIM", + "Q003", + "RET", +] diff --git a/run_tests.py b/run_tests.py index 24e2808..a0a40d0 100755 --- a/run_tests.py +++ b/run_tests.py @@ -1,25 +1,25 @@ #!/usr/bin/env python import os import sys + import django from django.conf import settings from django.test.utils import get_runner - BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -root = lambda *x: os.path.join(BASE_DIR, *x) + + +def root(*x): + return os.path.join(BASE_DIR, *x) + if not settings.configured: settings.configure( DEBUG=True, DATABASES={ "default": { - "ENGINE": "django.db.backends.postgresql_psycopg2", - "NAME": "test", - "USER": "postgres", - "PASSWORD": "", - "HOST": "", - "PORT": "", + "ENGINE": "django.db.backends.sqlite3", + "NAME": "test_db", } }, INSTALLED_APPS=( @@ -28,7 +28,6 @@ "dc_utils", "test_project", "dc_signup_form", - "dc_signup_form.signup_server", ), ROOT_URLCONF="test_project.urls", TEMPLATES=[ @@ -51,7 +50,7 @@ "django.contrib.sessions.middleware.SessionMiddleware", "django.contrib.messages.middleware.MessageMiddleware", ], - SECRET_KEY="testing_key" + SECRET_KEY="testing_key", ) django.setup() diff --git a/setup.py b/setup.py index 54a3570..1f9a6ab 100644 --- a/setup.py +++ b/setup.py @@ -21,13 +21,13 @@ def get_version(): description="Email Signup form component for use on DC websites", url="https://github.com/DemocracyClub/dc_signup_form", install_requires=[ - 'requests', - 'Django >=3.2,<4.3', - 'psycopg2-binary', + "requests", + "Django >=4.2,<5.1", + "psycopg2-binary", ], setup_requires=["wheel"], # We don't want to force boto3 for all installs # For example AWS Lambda already has it by default. # Mark as optional - extras_require={'boto': ['boto3']} + extras_require={"boto": ["boto3"]}, ) diff --git a/test_project/urls.py b/test_project/urls.py index ffd690b..a5a1a36 100644 --- a/test_project/urls.py +++ b/test_project/urls.py @@ -1,30 +1,19 @@ from django.urls import include, re_path -from dc_signup_form.forms import ElectionRemindersSignupForm, MailingListSignupForm +from dc_signup_form.forms import MailingListSignupForm from dc_signup_form.views import SignupFormView - email_patterns = [ re_path( r"^mailing_list/$", SignupFormView.as_view( template_name="email_form/mailing_list_form_view.html", form_class=MailingListSignupForm, - backend="local_db", + backend="event_bridge", + backend_kwargs={"source": "UnitTest", "bus_arn": "arn:testing"}, ), name="mailing_list_signup_view", ), - re_path( - r"^election_reminders/$", - SignupFormView.as_view( - template_name="email_form/election_reminders_form_view.html", - form_class=ElectionRemindersSignupForm, - get_vars=["postcode"], - thanks_message="Thanks for joining. We'll send you a reminder when there's an upcoming election in your area", - backend="local_db", - ), - name="election_reminders_signup_view", - ), ] diff --git a/testing_requirements.txt b/testing_requirements.txt index 6134e92..ddbace7 100644 --- a/testing_requirements.txt +++ b/testing_requirements.txt @@ -3,3 +3,5 @@ https://github.com/DemocracyClub/dc_django_utils/archive/refs/tags/2.4.0.tar.gz django-localflavor==4.0 boto3 moto==4.2.2 +black==23.9.1 +ruff==0.0.291