Merge pull request #4028 from gratipay/encrypted-identities

Encrypted identities

bin/rekey.py

@@ -1,10 +1,14 @@
 #!/usr/bin/env python2
+"""See gratipay.models.participant.mixins.identity.rekey for documentation.
+"""
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from gratipay import wireup
+from gratipay.models.participant.mixins import identity as participant_identities
 
 env = wireup.env()
 db = wireup.db(env)
-wireup.crypto(env)
+packer = wireup.crypto(env)
 
-print("{} record(s) rekeyed.".format(0))  # stubbed until we have something to rekey
+n = participant_identities.rekey(db, packer)
+print("Rekeyed {} participant identity record(s).".format(n))

gratipay/models/country.py

@@ -7,12 +7,9 @@ class Country(Model):
     """Represent country records from our database (read-only).
 
     :var int id: the record's primary key in our ``countries`` table
-    :var unicode name: the name of the country
-    :var unicode code2: the country's `ISO 3166-1 alpha-2`_ code
-    :var unicode code3: the country's `ISO 3166-1 alpha-3`_ code
+    :var unicode code: the country's `ISO 3166-1 alpha-2`_ code
 
     .. _ISO 3166-1 alpha-2 : https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2
-    .. _ISO 3166-1 alpha-3 : https://en.wikipedia.org/wiki/ISO_3166-1_alpha-3
 
     """
     typname = 'countries'

gratipay/models/participant/__init__.py

@@ -38,6 +38,7 @@
 from gratipay.models.account_elsewhere import AccountElsewhere
 from gratipay.models.exchange_route import ExchangeRoute
 from gratipay.models.team import Team
+from gratipay.models.participant import mixins
 from gratipay.security.crypto import constant_time_compare
 from gratipay.utils import (
     i18n,
@@ -60,7 +61,7 @@
 
 USERNAME_MAX_SIZE = 32
 
-class Participant(Model):
+class Participant(Model, mixins.Identity):
     """Represent a Gratipay participant.
     """
 
@@ -355,6 +356,7 @@ def clear_personal_information(self, cursor):
 
             DELETE FROM emails WHERE participant_id = %(participant_id)s;
             DELETE FROM statements WHERE participant=%(participant_id)s;
+            DELETE FROM participant_identities WHERE participant_id=%(participant_id)s;
 
             UPDATE participants
                SET anonymous_giving=False
@@ -1437,6 +1439,15 @@ def take_over(self, account, have_confirmation=False):
                 return
 
 
+            # Hard fail if the other participant has an identity.
+            # ===================================================
+            # Our identity system is very young. Maybe some day we'll do
+            # something smarter here.
+
+            if other.list_identity_metadata():
+                raise WontTakeOverWithIdentities()
+
+
             # Make sure we have user confirmation if needed.
             # ==============================================
             # We need confirmation in whatever combination of the following
@@ -1651,3 +1662,5 @@ class LastElsewhere(Exception): pass
 class NonexistingElsewhere(Exception): pass
 
 class TeamCantBeOnlyAuth(Exception): pass
+
+class WontTakeOverWithIdentities(Exception): pass

gratipay/models/participant/mixins/__init__.py

@@ -0,0 +1,3 @@
+from .identity import IdentityMixin as Identity
+
+__all__ = ['Identity']

gratipay/models/participant/mixins/identity.py

@@ -0,0 +1,217 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from psycopg2 import IntegrityError
+from gratipay.models import add_event
+
+
+class ParticipantIdentityError(StandardError): pass
+class ParticipantIdentitySchemaUnknown(ParticipantIdentityError): pass
+class ParticipantIdentityInfoInvalid(ParticipantIdentityError): pass
+
+
+schema_validators = {'nothing-enforced': lambda info: None}
+
+
+def _validate_info(schema_name, info):
+    if schema_name not in schema_validators:
+        raise ParticipantIdentitySchemaUnknown("unknown schema '{}'".format(schema_name))
+    validate_schema = schema_validators[schema_name]
+    validate_schema(info)
+    return None
+
+
+class IdentityMixin(object):
+    """This mixin provides management of national identities for
+    :py:class:`~gratipay.models.participant.Participant` objects.
+
+    A participant may have zero or more national identities on file with
+    Gratipay, with at most one for any given country at any given time. When at
+    least one of a participant's national identities has been verified, then
+    they may join the payroll of one or more Teams.
+
+    Since national identity information is more sensitive than other
+    information in our database, we encrypt it in the application layer before
+    passing it to the database in :py:meth:`store_identity_info`. We then limit
+    access to the information to a single method,
+    :py:meth:`retrieve_identity_info`.
+
+    """
+
+    def store_identity_info(self, country_id, schema_name, info):
+        """Store the participant's national identity information for a given country.
+
+        :param int country_id: an ``id`` from the ``countries`` table
+        :param dict schema_name: the name of the schema of the identity information
+        :param dict info: a dictionary of identity information
+
+        :returns: the ``id`` of the identity info's record in the
+            ``participant_identities`` table
+
+        :raises ParticipantIdentitySchemaUnknown: if ``schema_name`` doesn't
+            name a known schema
+        :raises ParticipantIdentityInfoInvalid: if the ``info`` dictionary does
+            not conform to the schema named by ``schema_name``
+
+        The ``info`` dictionary will be serialized to JSON and then encrypted
+        with :py:class:`~gratipay.security.crypto.EncryptingPacker` before
+        being sent to the database. We anticipate multiple schemas evolving for
+        this dictionary, with enforcement in the application layer (since the
+        field is opaque in the database layer). For now there is only one
+        available schema: ``nothing-enforced``.
+
+        """
+        _validate_info(schema_name, info)
+        info = self.encrypting_packer.pack(info)
+
+        def _add_event(action):
+            payload = dict( id=self.id
+                          , country_id=country_id
+                          , identity_id=identity_id
+                          , action=action + ' identity'
+                           )
+            add_event(cursor, 'participant', payload)
+
+        params = dict( participant_id=self.id
+                     , country_id=country_id
+                     , info=info
+                     , schema_name=schema_name
+                      )
+
+        try:
+            with self.db.get_cursor() as cursor:
+                identity_id = cursor.one("""
+
+                    INSERT INTO participant_identities
+                                (participant_id, country_id, schema_name, info)
+                         VALUES (%(participant_id)s, %(country_id)s, %(schema_name)s, %(info)s)
+                      RETURNING id
+
+                """, params)
+                _add_event('insert')
+
+        except IntegrityError as exc:
+            if exc.pgcode != '23505':
+                raise
+            with self.db.get_cursor() as cursor:
+                identity_id, old_schema_name = cursor.one("""
+
+                    UPDATE participant_identities
+                       SET schema_name=%(schema_name)s, info=%(info)s
+                     WHERE participant_id=%(participant_id)s
+                       AND country_id=%(country_id)s
+                 RETURNING id, schema_name
+
+                """, params)
+                _add_event('update')
+
+        return identity_id
+
+
+    def retrieve_identity_info(self, country_id):
+        """Return the participant's national identity information for a given country.
+
+        :param int country_id: an ``id`` from the ``countries`` table
+
+        :returns: a dictionary of identity information, or ``None``
+
+        """
+        with self.db.get_cursor() as cursor:
+            identity_id, info = cursor.one("""
+
+                SELECT id, info
+                  FROM participant_identities
+                 WHERE participant_id=%s
+                   AND country_id=%s
+
+            """, (self.id, country_id), default=(None, None))
+
+            if info is not None:
+                info = bytes(info) # psycopg2 returns bytea as buffer; we want bytes
+                info = self.encrypting_packer.unpack(info)
+
+            payload = dict( id=self.id
+                          , identity_id=identity_id
+                          , country_id=country_id
+                          , action='retrieve identity'
+                           )
+
+            add_event(cursor, 'participant', payload)
+
+        return info
+
+
+    def list_identity_metadata(self):
+        """Return a list of identity metadata records, sorted by country name.
+
+        Identity metadata records have the following attributes:
+
+        :var int id: the record's primary key in the ``participant_identities`` table
+        :var Country country: the country this identity applies to
+        :var unicode schema_name: the name of the schema that the data itself conforms to
+
+        The national identity information itself is not included, only
+        metadata. Use :py:meth:`retrieve_identity_info` to get the actual data.
+
+        """
+        return self.db.all("""
+
+            SELECT pi.id
+                 , c.*::countries AS country
+                 , schema_name
+              FROM participant_identities pi
+              JOIN countries c ON pi.country_id=c.id
+             WHERE participant_id=%s
+          ORDER BY c.code
+
+        """, (self.id,))
+
+
+# Rekeying
+# ========
+
+def rekey(db, packer):
+    """Rekey the encrypted participant identity information in our database.
+
+    :param GratipayDB db: used to access the database
+    :param EncryptingPacker packer: used to decrypt and encrypt data
+
+    This function features prominently in our procedure for rekeying our
+    encrypted data, as documented in the "`Keep Secrets`_" howto. It operates
+    by loading records from `participant_identities` that haven't been updated
+    in the present month, in batches of 100. It updates a timestamp atomically
+    with each rekeyed `info`, so it can be safely rerun in the face of network
+    failure, etc.
+
+    .. _Keep Secrets: http://inside.gratipay.com/howto/keep-secrets
+
+    """
+    n = 0
+    while 1:
+        m = _rekey_one_batch(db, packer)
+        if m == 0:
+            break
+        n += m
+    return n
+
+
+def _rekey_one_batch(db, packer):
+    batch = db.all("""
+
+        SELECT id, info
+          FROM participant_identities
+         WHERE _info_last_keyed < date_trunc('month', now())
+      ORDER BY _info_last_keyed ASC
+         LIMIT 100
+
+    """)
+    if not batch:
+        return 0
+
+    for rec in batch:
+        plaintext = packer.unpack(bytes(rec.info))
+        new_token = packer.pack(plaintext)
+        db.run( "UPDATE participant_identities SET info=%s, _info_last_keyed=now() WHERE id=%s"
+              , (new_token, rec.id)
+               )
+
+    return len(batch)

gratipay/wireup.py

@@ -34,6 +34,7 @@
 from gratipay.models.country import Country
 from gratipay.models.exchange_route import ExchangeRoute
 from gratipay.models.participant import Participant
+from gratipay.models.participant.mixins import Identity
 from gratipay.models.team import Team
 from gratipay.models import GratipayDB
 from gratipay.security.crypto import EncryptingPacker
@@ -63,7 +64,8 @@ def db(env):
 
 def crypto(env):
     keys = [k.encode('ASCII') for k in env.crypto_keys.split()]
-    Participant.encrypting_packer = EncryptingPacker(*keys)
+    out = Identity.encrypting_packer = EncryptingPacker(*keys)
+    return out
 
 def mail(env, project_root='.'):
     if env.aws_ses_access_key_id and env.aws_ses_secret_access_key and env.aws_ses_default_region:

sql/branch.sql

@@ -0,0 +1,28 @@
+CREATE TABLE participant_identities
+( id                bigserial       primary key
+, participant_id    bigint          NOT NULL REFERENCES participants(id)
+, country_id        bigint          NOT NULL REFERENCES countries(id)
+, schema_name       text            NOT NULL
+, info              bytea           NOT NULL
+, _info_last_keyed  timestamptz     NOT NULL DEFAULT now()
+, UNIQUE(participant_id, country_id)
+ );
+
+
+-- fail_if_no_email
+
+CREATE FUNCTION fail_if_no_email() RETURNS trigger AS $$
+    BEGIN
+        IF (SELECT email_address FROM participants WHERE id=NEW.participant_id) IS NULL THEN
+            RAISE EXCEPTION
+            USING ERRCODE=23100
+                , MESSAGE='This operation requires a verified participant email address.';
+        END IF;
+        RETURN NEW;
+    END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER enforce_email_for_participant_identity
+    BEFORE INSERT ON participant_identities
+    FOR EACH ROW
+    EXECUTE PROCEDURE fail_if_no_email();

tests/py/test_close.py

@@ -174,6 +174,19 @@ def test_cpi_clears_communities(self):
 
         assert Community.from_slug('test').nmembers == 1
 
+    def test_cpi_clears_personal_identities(self):
+        alice = self.make_participant('alice', email_address='[email protected]')
+        US = self.db.one("SELECT id FROM countries WHERE code='US'")
+        alice.store_identity_info(US, 'nothing-enforced', {'name': 'Alice'})
+        assert len(alice.list_identity_metadata()) == 1
+        assert self.db.one('SELECT count(*) FROM participant_identities;') == 1
+
+        with self.db.get_cursor() as cursor:
+            alice.clear_personal_information(cursor)
+
+        assert len(alice.list_identity_metadata()) == 0
+        assert self.db.one('SELECT count(*) FROM participant_identities;') == 0
+
 
     # uic = update_is_closed