From aae4cad44db421b2ec6333d8de3833fee23740c8 Mon Sep 17 00:00:00 2001 From: Changaco Date: Thu, 12 Oct 2023 13:59:06 +0200 Subject: [PATCH 1/6] show account stats in `/admin/email-domains` --- www/admin/email-domains.spt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/www/admin/email-domains.spt b/www/admin/email-domains.spt index 63629bfe28..7bc3892f14 100644 --- a/www/admin/email-domains.spt +++ b/www/admin/email-domains.spt @@ -78,8 +78,13 @@ else: SELECT e.domain , count(DISTINCT lower(e.address)) AS n_addresses , sum(CASE WHEN e.verified IS TRUE THEN 1 ELSE 0 END) AS n_verified - FROM ( SELECT e.*, regexp_replace(lower(e.address), '.+@', '') AS domain + , count(*) FILTER (WHERE e.participant IS NOT NULL) AS n_accounts + , count(*) FILTER (WHERE e.participant_marked_as IN ('fraud', 'spam')) AS n_fake_accounts + FROM ( SELECT e.* + , regexp_replace(lower(e.address), '.+@', '') AS domain + , p.marked_as AS participant_marked_as FROM emails e + LEFT JOIN participants p ON p.id = e.participant ) e GROUP BY e.domain ) @@ -100,6 +105,8 @@ else: SELECT d.domain , coalesce(s.n_addresses, 0) AS n_addresses , coalesce(s.n_verified, 0) AS n_verified + , coalesce(s.n_accounts, 0) AS n_accounts + , coalesce(s.n_fake_accounts, 0) AS n_fake_accounts FROM ( {domains_query} ) d LEFT JOIN domain_stats s ON s.domain = d.domain @@ -202,6 +209,7 @@ title = "Email Domains" Connected addresses Verified addresses Blocked addresses + Fake accounts Banned? @@ -213,6 +221,9 @@ title = "Email Domains" {{ d.n_verified }} % set percent_blacklisted = d.n_blacklisted_addresses / d.n_addresses if d.n_addresses else 0 {{ d.n_blacklisted_addresses }} + % set percent_fake = d.n_fake_accounts / (d.n_accounts or 1) + {{ locale.format_percent(percent_fake) if d.n_accounts else "-" }} {{ 'Yes (' + d.ban_reason + ')' if d.ban_reason else 'No' }} % endfor From fa269f8ca2a63a665c69da18d3e5ae33fa1a97d5 Mon Sep 17 00:00:00 2001 From: Changaco Date: Mon, 25 Sep 2023 10:16:22 +0200 Subject: [PATCH 2/6] implement bulk (un)blocking of email domains --- js/10-base.js | 19 ++++- www/admin/email-domains.spt | 146 +++++++++++++++++++++++++++--------- 2 files changed, 127 insertions(+), 38 deletions(-) diff --git a/js/10-base.js b/js/10-base.js index b39fa9a81f..f71fd552d3 100644 --- a/js/10-base.js +++ b/js/10-base.js @@ -104,10 +104,21 @@ Liberapay.init = function() { $(this).children('input[type="radio"]').prop('checked', true).trigger('change'); }); - $('[data-toggle="enable"]').on('change', function() { - var $checkbox = $(this); - var $target = $($checkbox.data('target')); - $target.prop('disabled', !$checkbox.prop('checked')); + $('[data-toggle="enable"]').each(function() { + if (this.tagName == 'OPTION') { + var $option = $(this); + var $select = $option.parent(); + $select.on('change', function() { + var $target = $($option.data('target')); + $target.prop('disabled', !$option.prop('selected')); + }); + } else { + var $control = $(this); + $control.on('change', function() { + var $target = $($control.data('target')); + $target.prop('disabled', !$control.prop('checked')); + }); + } }); $('[data-email]').one('mouseover click', function () { diff --git a/www/admin/email-domains.spt b/www/admin/email-domains.spt index 7bc3892f14..5e453a7bf0 100644 --- a/www/admin/email-domains.spt +++ b/www/admin/email-domains.spt @@ -1,53 +1,97 @@ # coding: utf8 +import json + from liberapay.i18n.base import LOCALE_EN as locale -from liberapay.utils import partition +from liberapay.utils import form_post_success PAGE_SIZE = 50 REASONS = { - "bounce": "This domain bounces back all messages.", - "complaint": "The domain's admin asked us to blacklist it.", - "throwaway": "This domain provides throwaway addresses.", - "other": "Other", + "bounce": ( + "This domain bounces back all messages.", + "These domains bounce back all messages." + ), + "complaint": ( + "An administrator of this domain asked us to blacklist it.", + "An administrator of these domains asked us to blacklist them." + ), + "throwaway": ( + "This domain provides throwaway addresses.", + "These domains provide throwaway addresses.", + ), + "other": ("Other", "Other"), } [---] user.require_active_privilege('admin') -domain = request.qs.get('domain', '').lower() +domain = request.qs.get('domain', '').lower().strip() -if domain: - if request.method == 'POST': - if not domain: - raise response.invalid_input(domain, 'domain', 'querystring') - action = request.body['action'] - if action == 'remove_from_blacklist': - website.db.run(""" +if request.method == 'POST': + action = request.body['action'] + if domain: + if 'domains' in request.body: + raise response.error(400, ( + "ambiguous request: contains both `domain` in querystring and `domains` in body" + )) + addresses = ['@' + domain] + else: + addresses = [ + '@' + domain.lstrip('@') for domain in request.body['domains'].lower().split() + ] + if not addresses: + raise response.invalid_input(request.body['domains'], 'domains', 'body') + addresses = json.dumps(addresses) + if action == 'remove_from_blacklist': + n = website.db.one(""" + WITH updated AS ( UPDATE email_blacklist SET ignore_after = current_timestamp , ignored_by = %s - WHERE lower(address) = '@' || %s - """, (user.id, domain)) - elif action == 'add_to_blacklist': - reason = request.body['reason'] - if reason not in REASONS: - raise response.invalid_input(reason, 'reason', 'body') - details = request.body.get('details') - if details and len(details) > 1024: - raise response.invalid_input(details, 'details', 'body') - if reason == 'other' and len(details) < 10: - raise response.error(400, "Please specify the reason.") - website.db.run(""" + WHERE %s ? lower(address) + AND ( ignore_after IS NULL OR ignore_after > current_timestamp ) + RETURNING 1 + ) SELECT count(*) FROM updated + """, (user.id, addresses)) + form_post_success(state, msg=( + f"{n} domain{' has' if n == 1 else 's have'} been removed from the blacklist." + )) + elif action == 'add_to_blacklist': + reason = request.body['reason'] + if reason not in REASONS: + raise response.invalid_input(reason, 'reason', 'body') + details = request.body.get('details') + if details and len(details) > 1024: + raise response.invalid_input(details, 'details', 'body') + if reason == 'other' and len(details) < 10: + raise response.error(400, "Please specify the reason.") + n = website.db.one(""" + WITH inserted AS ( INSERT INTO email_blacklist (address, reason, details, added_by) - VALUES ('@' || %s, %s, %s, %s) - """, (domain, reason, details, user.id)) - else: - raise response.invalid_input(action, 'action', 'body') - raise response.redirect(request.line.uri.decoded) + SELECT address.value, %(reason)s, %(details)s, %(added_by)s + FROM json_array_elements_text(%(addresses)s) address + WHERE NOT EXISTS ( + SELECT 1 + FROM email_blacklist b + WHERE b.address = address.value + AND b.reason = %(reason)s + AND b.ignore_after IS NULL + ) + RETURNING 1 + ) SELECT count(*) FROM inserted + """, dict( + addresses=addresses, reason=reason, details=details, added_by=user.id, + )) + form_post_success(state, msg=( + f"{n} domain{' has' if n == 1 else 's have'} been added to the blacklist." + )) + else: + raise response.invalid_input(action, 'action', 'body') +if domain: blacklist_entries = website.db.all(""" SELECT bl.ts, bl.reason, bl.details, bl.ignore_after , added_by_p AS added_by, ignored_by_p AS ignored_by @@ -59,7 +103,6 @@ if domain: LIMIT 10 """, (domain,)) blacklisted = any(e.ignore_after is None for e in blacklist_entries) - else: show = request.qs.get_choice('show', {'all', 'blacklisted'}, default='all') offset = request.qs.get_int('offset', 0) @@ -140,7 +183,7 @@ title = "Email Domains" The domain {{ domain }} was added to the blacklist by {{ e.added_by.link() }} at {{ locale.format_time(e.ts.time()) }} on {{ locale.format_date(e.ts.date(), format='long') }}.
- Reason: {{ REASONS.get(e.reason, e.reason) }} + Reason: {{ REASONS[e.reason][0] }} % if e.details
Details: {{ e.details }} @@ -176,8 +219,8 @@ title = "Email Domains" Reason: @@ -237,6 +280,7 @@ title = "Email Domains" No email domains found. % endif +

Look up a domain

@@ -245,6 +289,40 @@ title = "Email Domains"
+
+
+

Bulk edit domains

+
+ + + +
+ + +
+ +
+ % endif % endblock From 851aa0052ca7849865c20ae6334e956ec7596c93 Mon Sep 17 00:00:00 2001 From: Changaco Date: Mon, 23 Oct 2023 18:26:52 +0200 Subject: [PATCH 3/6] enable admins to view newly-seen email domains --- www/admin/email-domains.spt | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/www/admin/email-domains.spt b/www/admin/email-domains.spt index 5e453a7bf0..e0ca73bffe 100644 --- a/www/admin/email-domains.spt +++ b/www/admin/email-domains.spt @@ -104,17 +104,25 @@ if domain: """, (domain,)) blacklisted = any(e.ignore_after is None for e in blacklist_entries) else: - show = request.qs.get_choice('show', {'all', 'blacklisted'}, default='all') + show = request.qs.get_choice('show', {'all', 'new', 'top', 'blacklisted'}, default='new') + if show == 'all': + show = 'top' + if show == 'new': + order = 'first_appearance DESC, domain' + elif show == 'top': + order = 'n_addresses DESC, domain' + else: + order = 'domain' offset = request.qs.get_int('offset', 0) domains_query = """ - SELECT substr(bl.address, 2) AS domain + SELECT substr(bl.address, 2) AS domain, bl.ts AS first_appearance FROM email_blacklist bl WHERE lower(bl.address) LIKE '@%%' AND (bl.ignore_after IS NULL OR bl.ignore_after > current_timestamp) """ - if show == 'all': + if show != 'blacklisted': domains_query += """ - UNION SELECT e.domain FROM domain_stats e + UNION SELECT e.domain, e.first_appearance FROM domain_stats e """ email_domains = website.db.all(""" WITH domain_stats AS ( @@ -123,6 +131,7 @@ else: , sum(CASE WHEN e.verified IS TRUE THEN 1 ELSE 0 END) AS n_verified , count(*) FILTER (WHERE e.participant IS NOT NULL) AS n_accounts , count(*) FILTER (WHERE e.participant_marked_as IN ('fraud', 'spam')) AS n_fake_accounts + , min(e.added_time) AS first_appearance FROM ( SELECT e.* , regexp_replace(lower(e.address), '.+@', '') AS domain , p.marked_as AS participant_marked_as @@ -146,6 +155,7 @@ else: ) AS ban_reason FROM ( SELECT d.domain + , d.first_appearance , coalesce(s.n_addresses, 0) AS n_addresses , coalesce(s.n_verified, 0) AS n_verified , coalesce(s.n_accounts, 0) AS n_accounts @@ -153,11 +163,11 @@ else: FROM ( {domains_query} ) d LEFT JOIN domain_stats s ON s.domain = d.domain - ORDER BY n_addresses DESC, domain + ORDER BY {order}, domain LIMIT %s OFFSET %s ) d - """.format(domains_query=domains_query), (PAGE_SIZE, offset)) + """.format(domains_query=domains_query, order=order), (PAGE_SIZE, offset)) title = "Email Domains" @@ -238,7 +248,8 @@ title = "Email Domains" % else
    {{ querystring_nav('show', [ - ('all', "All"), + ('new', "New"), + ('top', "Top"), ('blacklisted', "Blacklisted"), ], show) }}


@@ -254,6 +265,7 @@ title = "Email Domains" Blocked addresses Fake accounts Banned? + First appearance % for d in email_domains @@ -268,6 +280,7 @@ title = "Email Domains" {{ locale.format_percent(percent_fake) if d.n_accounts else "-" }} {{ 'Yes (' + d.ban_reason + ')' if d.ban_reason else 'No' }} + {{ locale.format_timedelta(to_age(d.first_appearance)) }} % endfor From 44e2195139dd5dbcee542d121112485c2ebe0541 Mon Sep 17 00:00:00 2001 From: Changaco Date: Wed, 25 Oct 2023 16:06:49 +0200 Subject: [PATCH 4/6] show stats for a looked up email domain --- www/admin/email-domains.spt | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/www/admin/email-domains.spt b/www/admin/email-domains.spt index e0ca73bffe..6b1445d581 100644 --- a/www/admin/email-domains.spt +++ b/www/admin/email-domains.spt @@ -103,6 +103,22 @@ if domain: LIMIT 10 """, (domain,)) blacklisted = any(e.ignore_after is None for e in blacklist_entries) + stats = website.db.one(""" + SELECT count(DISTINCT lower(e.address)) AS n_addresses + , sum(CASE WHEN e.verified IS TRUE THEN 1 ELSE 0 END) AS n_verified + , count(*) FILTER (WHERE e.participant IS NOT NULL) AS n_accounts + , count(*) FILTER (WHERE p.marked_as IN ('fraud', 'spam')) AS n_fake_accounts + , min(e.added_time) AS first_appearance + FROM emails e + LEFT JOIN participants p ON p.id = e.participant + WHERE e.address LIKE %s + """, ('%@' + domain,)) + n_blacklisted_addresses = website.db.one(""" + SELECT count(DISTINCT lower(bl.address)) + FROM email_blacklist bl + WHERE lower(bl.address) LIKE %s + AND (bl.ignore_after IS NULL OR bl.ignore_after > current_timestamp) + """, ('%@' + domain,)) else: show = request.qs.get_choice('show', {'all', 'new', 'top', 'blacklisted'}, default='new') if show == 'all': @@ -242,6 +258,13 @@ title = "Email Domains" % endif +

Stats

+ Connected addresses: {{ stats.n_addresses }}
+ Verified addresses: {{ stats.n_verified }}
+ Blocked addresses: {{ n_blacklisted_addresses }}
+ % set percent_fake = stats.n_fake_accounts / (stats.n_accounts or 1) + Fake accounts: {{ stats.n_fake_accounts }} ({{ locale.format_percent(percent_fake) }})
+

← Back to browsing email domains From 3eaba28c509638f98e0935934e5ae71c3972f2eb Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 28 Oct 2023 21:26:27 +0200 Subject: [PATCH 5/6] list accounts which use a looked up email domain --- liberapay/constants.py | 11 ++++++ www/admin/email-domains.spt | 68 +++++++++++++++++++++++++++++++++++++ www/admin/payments.spt | 12 +------ 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/liberapay/constants.py b/liberapay/constants.py index 960a67baa8..f2a7571927 100644 --- a/liberapay/constants.py +++ b/liberapay/constants.py @@ -24,6 +24,17 @@ def check_bits(bits): _ = lambda a: a +ACCOUNT_MARK_CLASSES = { + 'trusted': 'success', + 'okay': 'info', + 'unsettling': 'info', + 'controversial': 'warning', + 'irrelevant': 'warning', + 'misleading': 'warning', + 'fraud': 'danger', + 'spam': 'danger', +} + ASCII_ALLOWED_IN_USERNAME = set("0123456789" "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/www/admin/email-domains.spt b/www/admin/email-domains.spt index 6b1445d581..28647c6730 100644 --- a/www/admin/email-domains.spt +++ b/www/admin/email-domains.spt @@ -119,6 +119,34 @@ if domain: WHERE lower(bl.address) LIKE %s AND (bl.ignore_after IS NULL OR bl.ignore_after > current_timestamp) """, ('%@' + domain,)) + before = request.qs.get('before', default=None) + participants = website.db.all(""" + SELECT p.id, p.username, p.marked_as + , json_agg(json_build_object( + 'address', e.address, + 'verified', e.verified, + 'disavowed', e.disavowed, + 'blacklist_reason', ( + SELECT bl.reason + FROM email_blacklist bl + WHERE lower(bl.address) = lower(e.address) + AND (bl.ignore_after IS NULL OR bl.ignore_after > current_timestamp) + ) + ) ORDER BY e.added_time DESC) AS addresses + , max(e.added_time) AS max_added_time + FROM emails e + JOIN participants p ON p.id = e.participant + WHERE lower(e.address) LIKE %s + GROUP BY p.id + HAVING coalesce(max(e.added_time) < %s, true) + ORDER BY max(e.added_time) DESC + LIMIT %s + """, ('%@' + domain, before, PAGE_SIZE + 1)) + if len(participants) > PAGE_SIZE: + next_before = participants[-2].max_added_time.isoformat() + participants = participants[:PAGE_SIZE] + else: + next_before = None else: show = request.qs.get_choice('show', {'all', 'new', 'top', 'blacklisted'}, default='new') if show == 'all': @@ -265,6 +293,46 @@ title = "Email Domains" % set percent_fake = stats.n_fake_accounts / (stats.n_accounts or 1) Fake accounts: {{ stats.n_fake_accounts }} ({{ locale.format_percent(percent_fake) }})
+

Accounts

+ % if participants + + + + + + + + + % for p in participants + + + + + % endfor + +
AccountAddresses
{{ p.username }}{% if p.marked_as %}
+ [{{ p.marked_as }}]{% endif %}		 + % for ea in p.addresses + {{ ea.address }} + % if ea.verified + {{ glyphicon('ok-sign', "Verified") }} + % elif ea.disavowed + {{ glyphicon('exclamation-sign', "Disavowed") }} + % elif ea.blacklist_reason + {{ glyphicon('exclamation-sign', "Blacklisted (%s)" % ea.blacklist_reason) }} + % else + {{ glyphicon('warning-sign', "Unconfirmed") }} + % endif +
+ % endfor +
+ % if next_before + Next page →
+ % endif + % else + No accounts found. + % endif +

← Back to browsing email domains diff --git a/www/admin/payments.spt b/www/admin/payments.spt index 254e3da867..2f6f404d14 100644 --- a/www/admin/payments.spt +++ b/www/admin/payments.spt @@ -1,16 +1,6 @@ from liberapay.i18n.base import LOCALE_EN as locale from liberapay.utils import render_postal_address -MARKS_MAP = { - 'trusted': 'success', - 'okay': 'info', - 'unsettling': 'info', - 'controversial': 'warning', - 'irrelevant': 'warning', - 'misleading': 'warning', - 'fraud': 'danger', - 'spam': 'danger', -} PAGE_SIZE = 50 STATUS_MAP = { 'failed': 'danger', @@ -109,7 +99,7 @@ title = "Payments Admin" ) -}} {% endif %}{% if pi.payer_marked_as %}
- [{{ + [{{ pi.payer_marked_as }}]{% endif %} {{ From 4507ecbe0749d84fe9900667e3511c7a6ca852b0 Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 28 Oct 2023 21:27:29 +0200 Subject: [PATCH 6/6] replace an unnecessary form with links --- www/admin/email-domains.spt | 61 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/www/admin/email-domains.spt b/www/admin/email-domains.spt index 28647c6730..b0643629b1 100644 --- a/www/admin/email-domains.spt +++ b/www/admin/email-domains.spt @@ -346,37 +346,36 @@ title = "Email Domains"

% if email_domains -
- - - - - - - - - - - - - % for d in email_domains - - - - % set percent_verified = d.n_verified / d.n_addresses if d.n_addresses else 1 - - % set percent_blacklisted = d.n_blacklisted_addresses / d.n_addresses if d.n_addresses else 0 - - % set percent_fake = d.n_fake_accounts / (d.n_accounts or 1) - - - - - % endfor - -
DomainConnected addressesVerified addressesBlocked addressesFake accountsBanned?First appearance
{{ d.n_addresses }}{{ d.n_verified }}{{ d.n_blacklisted_addresses }}{{ locale.format_percent(percent_fake) if d.n_accounts else "-" }}{{ 'Yes (' + d.ban_reason + ')' if d.ban_reason else 'No' }}{{ locale.format_timedelta(to_age(d.first_appearance)) }}
-
+ + + + + + + + + + + + + + % for d in email_domains + + + + % set percent_verified = d.n_verified / d.n_addresses if d.n_addresses else 1 + + % set percent_blacklisted = d.n_blacklisted_addresses / d.n_addresses if d.n_addresses else 0 + + % set percent_fake = d.n_fake_accounts / (d.n_accounts or 1) + + + + + % endfor + +
DomainConnected addressesVerified addressesBlocked addressesFake accountsBanned?First appearance
{{ d.domain }}{{ d.n_addresses }}{{ d.n_verified }}{{ d.n_blacklisted_addresses }}{{ locale.format_percent(percent_fake) if d.n_accounts else "-" }}{{ 'Yes (' + d.ban_reason + ')' if d.ban_reason else 'No' }}{{ locale.format_timedelta(to_age(d.first_appearance)) }}
% if len(email_domains) == PAGE_SIZE Next page → % endif