implement encrypted national identities

gratipay/models/participant/__init__.py

@@ -38,6 +38,7 @@
 from gratipay.models.account_elsewhere import AccountElsewhere
 from gratipay.models.exchange_route import ExchangeRoute
 from gratipay.models.team import Team
+from gratipay.models.participant import mixins
 from gratipay.security.crypto import constant_time_compare
 from gratipay.utils import (
     i18n,
@@ -60,7 +61,7 @@
 
 USERNAME_MAX_SIZE = 32
 
-class Participant(Model):
+class Participant(Model, mixins.Identity):
     """Represent a Gratipay participant.
     """
 

gratipay/models/participant/mixins/__init__.py

@@ -0,0 +1,3 @@
+from .identity import IdentityMixin as Identity
+
+__all__ = ['Identity']

gratipay/models/participant/mixins/identity.py

@@ -0,0 +1,164 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from psycopg2 import IntegrityError
+from gratipay.models import add_event
+
+
+class ParticipantIdentityError(StandardError): pass
+class ParticipantIdentitySchemaUnknown(ParticipantIdentityError): pass
+class ParticipantIdentityInfoInvalid(ParticipantIdentityError): pass
+
+
+schema_validators = {'nothing-enforced': lambda info: None}
+
+
+def _validate_info(schema_name, info):
+    if schema_name not in schema_validators:
+        raise ParticipantIdentitySchemaUnknown("unknown schema '{}'".format(schema_name))
+    validate_schema = schema_validators[schema_name]
+    validate_schema(info)
+    return None
+
+
+class IdentityMixin(object):
+    """This mixin provides management of national identities for
+    :py:class:`~gratipay.models.participant.Participant` objects.
+
+    A participant may have zero or more national identities on file with
+    Gratipay, with at most one for any given country at any given time. When at
+    least one of a participant's national identities has been verified, then
+    they may join the payroll of one or more Teams.
+
+    Since national identity information is more sensitive than other
+    information in our database, we encrypt it in the application layer before
+    passing it to the database in :py:meth:`store_identity_info`. We then limit
+    access to the information to a single method,
+    :py:meth:`retrieve_identity_info`.
+
+    """
+
+    def store_identity_info(self, country_id, schema_name, info):
+        """Store the participant's national identity information for a given country.
+
+        :param int country_id: an ``id`` from the ``countries`` table
+        :param dict schema_name: the name of the schema of the identity information
+        :param dict info: a dictionary of identity information
+
+        :returns: the ``id`` of the identity info's record in the
+            ``participant_identities`` table
+
+        :raises ParticipantIdentitySchemaUnknown: if ``schema_name`` doesn't
+            name a known schema
+        :raises ParticipantIdentityInfoInvalid: if the ``info`` dictionary does
+            not conform to the schema named by ``schema_name``
+
+        The ``info`` dictionary will be serialized to JSON and then encrypted
+        with :py:class:`~gratipay.security.crypto.EncryptingPacker` before
+        being sent to the database. We anticipate multiple schemas evolving for
+        this dictionary, with enforcement in the application layer (since the
+        field is opaque in the database layer). For now there is only one
+        available schema: ``nothing-enforced``.
+
+        """
+        _validate_info(schema_name, info)
+        info = self.encrypting_packer.pack(info)
+
+        def _add_event(action):
+            payload = dict( id=self.id
+                          , country_id=country_id
+                          , identity_id=identity_id
+                          , action=action + ' identity'
+                           )
+            add_event(cursor, 'participant', payload)
+
+        params = dict( participant_id=self.id
+                     , country_id=country_id
+                     , info=info
+                     , schema_name=schema_name
+                      )
+
+        try:
+            with self.db.get_cursor() as cursor:
+                identity_id = cursor.one("""
+
+                    INSERT INTO participant_identities
+                                (participant_id, country_id, schema_name, info)
+                         VALUES (%(participant_id)s, %(country_id)s, %(schema_name)s, %(info)s)
+                      RETURNING id
+
+                """, params)
+                _add_event('insert')
+
+        except IntegrityError:
+            with self.db.get_cursor() as cursor:
+                identity_id, old_schema_name = cursor.one("""
+
+                    UPDATE participant_identities
+                       SET schema_name=%(schema_name)s, info=%(info)s
+                     WHERE participant_id=%(participant_id)s
+                       AND country_id=%(country_id)s
+                 RETURNING id, schema_name
+
+                """, params)
+                _add_event('update')
+
+        return identity_id
+
+
+    def retrieve_identity_info(self, country_id):
+        """Return the participant's national identity information for a given country.
+
+        :param int country_id: an ``id`` from the ``countries`` table
+
+        :returns: a dictionary of identity information, or ``None``
+
+        """
+        with self.db.get_cursor() as cursor:
+            identity_id, info = cursor.one("""
+
+                SELECT id, info
+                  FROM participant_identities
+                 WHERE participant_id=%s
+                   AND country_id=%s
+
+            """, (self.id, country_id), default=(None, None))
+
+            if info is not None:
+                info = bytes(info) # psycopg2 returns bytea as buffer; we want bytes
+                info = self.encrypting_packer.unpack(info)
+
+            payload = dict( id=self.id
+                          , identity_id=identity_id
+                          , country_id=country_id
+                          , action='retrieve identity'
+                           )
+
+            add_event(cursor, 'participant', payload)
+
+        return info
+
+
+    def list_identity_metadata(self):
+        """Return a list of identity metadata records, sorted by country name.
+
+        Identity metadata records have the following attributes:
+
+        :var int id: the record's primary key in the ``participant_identities`` table
+        :var Country country: the country this identity applies to
+        :var unicode schema_name: the name of the schema that the data itself conforms to
+
+        The national identity information itself is not included, only
+        metadata. Use :py:meth:`retrieve_identity_info` to get the actual data.
+
+        """
+        return self.db.all("""
+
+            SELECT pi.id
+                 , c.*::countries AS country
+                 , schema_name
+              FROM participant_identities pi
+              JOIN countries c ON pi.country_id=c.id
+             WHERE participant_id=%s
+          ORDER BY c.name
+
+        """, (self.id,))

gratipay/wireup.py

@@ -34,6 +34,7 @@
 from gratipay.models.country import Country
 from gratipay.models.exchange_route import ExchangeRoute
 from gratipay.models.participant import Participant
+from gratipay.models.participant.mixins import Identity
 from gratipay.models.team import Team
 from gratipay.models import GratipayDB
 from gratipay.security.crypto import EncryptingPacker
@@ -63,7 +64,7 @@ def db(env):
 
 def crypto(env):
     keys = [k.encode('ASCII') for k in env.crypto_keys.split()]
-    Participant.encrypting_packer = EncryptingPacker(*keys)
+    Identity.encrypting_packer = EncryptingPacker(*keys)
 
 def mail(env, project_root='.'):
     if env.aws_ses_access_key_id and env.aws_ses_secret_access_key and env.aws_ses_default_region:

sql/branch.sql

@@ -6,3 +6,12 @@ CREATE TABLE countries -- http://www.iso.org/iso/country_codes
  );
 
 \i sql/countries.sql
+
+CREATE TABLE participant_identities
+( id                bigserial   primary key
+, participant_id    bigint      NOT NULL REFERENCES participants(id)
+, country_id        bigint      NOT NULL REFERENCES countries(id)
+, schema_name       text        NOT NULL
+, info              bytea       NOT NULL
+, UNIQUE(participant_id, country_id)
+ );

tests/py/test_participant_identities.py

@@ -0,0 +1,145 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from gratipay.testing import Harness
+from gratipay.models.participant.mixins import identity, Identity
+from gratipay.models.participant.mixins.identity import _validate_info
+from gratipay.models.participant.mixins.identity import ParticipantIdentityInfoInvalid
+from gratipay.models.participant.mixins.identity import ParticipantIdentitySchemaUnknown
+from pytest import raises
+
+
+class Tests(Harness):
+
+    @classmethod
+    def setUpClass(cls):
+        Harness.setUpClass()
+        cls.TTO = cls.db.one("SELECT id FROM countries WHERE code3='TTO'")
+        cls.USA = cls.db.one("SELECT id FROM countries WHERE code3='USA'")
+
+        def _failer(info):
+            raise ParticipantIdentityInfoInvalid('You failed.')
+        identity.schema_validators['impossible'] = _failer
+
+    @classmethod
+    def tearDownClass(cls):
+        del identity.schema_validators['impossible']
+
+    def assert_events(self, crusher_id, identity_ids, country_ids, actions):
+        events = self.db.all("SELECT * FROM events ORDER BY ts ASC")
+        nevents = len(events)
+
+        assert [e.type for e in events] == ['participant'] * nevents
+        assert [e.payload['id'] for e in events] == [crusher_id] * nevents
+        assert [e.payload['identity_id'] for e in events] == identity_ids
+        assert [e.payload['country_id'] for e in events] == country_ids
+        assert [e.payload['action'] for e in events] == actions
+
+
+    # rii - retrieve_identity_info
+
+    def test_rii_retrieves_identity_info(self):
+        crusher = self.make_participant('crusher')
+        crusher.store_identity_info(self.USA, 'nothing-enforced', {'name': 'Crusher'})
+        assert crusher.retrieve_identity_info(self.USA)['name'] == 'Crusher'
+
+    def test_rii_retrieves_identity_when_there_are_multiple_identities(self):
+        crusher = self.make_participant('crusher')
+        crusher.store_identity_info(self.USA, 'nothing-enforced', {'name': 'Crusher'})
+        crusher.store_identity_info(self.TTO, 'nothing-enforced', {'name': 'Bruiser'})
+        assert crusher.retrieve_identity_info(self.USA)['name'] == 'Crusher'
+        assert crusher.retrieve_identity_info(self.TTO)['name'] == 'Bruiser'
+
+    def test_rii_returns_None_if_there_is_no_identity_info(self):
+        crusher = self.make_participant('crusher')
+        assert crusher.retrieve_identity_info(self.USA) is None
+
+    def test_rii_logs_event(self):
+        crusher = self.make_participant('crusher')
+        iid = crusher.store_identity_info(self.TTO, 'nothing-enforced', {'name': 'Crusher'})
+        crusher.retrieve_identity_info(self.TTO)
+        self.assert_events( crusher.id
+                          , [iid, iid]
+                          , [self.TTO, self.TTO]
+                          , ['insert identity', 'retrieve identity']
+                           )
+
+    def test_rii_still_logs_an_event_when_noop(self):
+        crusher = self.make_participant('crusher')
+        crusher.retrieve_identity_info(self.TTO)
+        self.assert_events( crusher.id
+                          , [None]
+                          , [self.TTO]
+                          , ['retrieve identity']
+                           )
+
+
+    # lim - list_identity_metadata
+
+    def test_lim_lists_identity_metadata(self):
+        crusher = self.make_participant('crusher')
+        crusher.store_identity_info(self.USA, 'nothing-enforced', {'name': 'Crusher'})
+        assert [x.country.code3 for x in crusher.list_identity_metadata()] == ['USA']
+
+    def test_lim_lists_metadata_for_multiple_identities(self):
+        crusher = self.make_participant('crusher')
+        for country in (self.USA, self.TTO):
+            crusher.store_identity_info(country, 'nothing-enforced', {'name': 'Crusher'})
+        assert [x.country.code3 for x in crusher.list_identity_metadata()] == ['TTO', 'USA']
+
+
+    # sii - store_identity_info
+
+    def test_sii_sets_identity_info(self):
+        crusher = self.make_participant('crusher')
+        crusher.store_identity_info(self.TTO, 'nothing-enforced', {'name': 'Crusher'})
+        assert [x.country.code3 for x in crusher.list_identity_metadata()] == ['TTO']
+
+    def test_sii_sets_a_second_identity(self):
+        crusher = self.make_participant('crusher')
+        crusher.store_identity_info(self.TTO, 'nothing-enforced', {'name': 'Crusher'})
+        crusher.store_identity_info(self.USA, 'nothing-enforced', {'name': 'Crusher'})
+        assert [x.country.code3 for x in crusher.list_identity_metadata()] == ['TTO', 'USA']
+
+    def test_sii_overwrites_first_identity(self):
+        crusher = self.make_participant('crusher')
+        crusher.store_identity_info(self.TTO, 'nothing-enforced', {'name': 'Crusher'})
+        crusher.store_identity_info(self.TTO, 'nothing-enforced', {'name': 'Bruiser'})
+        assert [x.country.code3 for x in crusher.list_identity_metadata()] == ['TTO']
+        assert crusher.retrieve_identity_info(self.TTO)['name'] == 'Bruiser'
+
+    def test_sii_validates_identity(self):
+        crusher = self.make_participant('crusher')
+        raises( ParticipantIdentityInfoInvalid
+              , crusher.store_identity_info
+              , self.TTO
+              , 'impossible'
+              , {'foo': 'bar'}
+               )
+
+    def test_sii_happily_overwrites_schema_name(self):
+        crusher = self.make_participant('crusher')
+        packed = Identity.encrypting_packer.pack({'name': 'Crusher'})
+        self.db.run( "INSERT INTO participant_identities "
+                     "(participant_id, country_id, schema_name, info) "
+                     "VALUES (%s, %s, %s, %s)"
+                   , (crusher.id, self.TTO, 'flah', packed)
+                    )
+        assert [x.schema_name for x in crusher.list_identity_metadata()] == ['flah']
+        crusher.store_identity_info(self.TTO, 'nothing-enforced', {'name': 'Crusher'})
+        assert [x.schema_name for x in crusher.list_identity_metadata()] == ['nothing-enforced']
+
+    def test_sii_logs_event(self):
+        crusher = self.make_participant('crusher')
+        iid = crusher.store_identity_info(self.TTO, 'nothing-enforced', {'name': 'Crusher'})
+        self.assert_events(crusher.id, [iid], [self.TTO], ['insert identity'])
+
+
+    # _vi - _validate_info
+
+    def test__vi_validates_info(self):
+        err = raises(ParticipantIdentityInfoInvalid, _validate_info, 'impossible', {'foo': 'bar'})
+        assert err.value.message == 'You failed.'
+
+    def test__vi_chokes_on_unknown_schema(self):
+        err = raises(ParticipantIdentitySchemaUnknown, _validate_info, 'floo-floo', {'foo': 'bar'})
+        assert err.value.message == "unknown schema 'floo-floo'"

tests/py/test_security.py

@@ -7,7 +7,7 @@
 from aspen.http.request import Request
 from base64 import urlsafe_b64decode
 from gratipay import security
-from gratipay.models.participant import Participant
+from gratipay.models.participant.mixins import Identity
 from gratipay.testing import Harness
 from pytest import raises
 
@@ -55,17 +55,17 @@ def test_ahtr_sets_x_xss_protection(self):
              b'5TdyoJsll5nMAicg=='
 
     def test_ep_packs_encryptingly(self):
-        packed = Participant.encrypting_packer.pack({"foo": "bar"})
+        packed = Identity.encrypting_packer.pack({"foo": "bar"})
         assert urlsafe_b64decode(packed)[0] == b'\x80'  # Fernet version
 
     def test_ep_unpacks_decryptingly(self):
-        assert Participant.encrypting_packer.unpack(self.packed) == {"foo": "bar"}
+        assert Identity.encrypting_packer.unpack(self.packed) == {"foo": "bar"}
 
     def test_ep_leaks_timestamp_derp(self):
         # https://github.com/pyca/cryptography/issues/2714
         timestamp, = struct.unpack(">Q", urlsafe_b64decode(self.packed)[1:9])  # unencrypted!
         assert datetime.datetime.fromtimestamp(timestamp).year == 2016
 
     def test_ep_demands_bytes(self):
-        raises(TypeError, Participant.encrypting_packer.unpack, buffer('buffer'))
-        raises(TypeError, Participant.encrypting_packer.unpack, 'unicode')
+        raises(TypeError, Identity.encrypting_packer.unpack, buffer('buffer'))
+        raises(TypeError, Identity.encrypting_packer.unpack, 'unicode')