Merge branch 'master' into feature/totp-support

.github/workflows/test.yml

@@ -11,17 +11,17 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.8'
       - name: Install Python 3.9
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
       - name: Cache the .tox directory to speed things up
-        uses: actions/cache@v2.1.6
+        uses: actions/cache@v3
         with:
           path: .tox
           key: test-${{ runner.os }}-${{ hashFiles('requirements_*.txt') }}
@@ -42,3 +42,5 @@ jobs:
         run: pip install tox
       - name: Run the tests
         run: tox
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3

Makefile

@@ -5,31 +5,31 @@ python := "$(shell { command -v python3.8 || command -v python3 || command -v py
 bin_dir := $(shell $(python) -c 'import sys; print("Scripts" if sys.platform == "win32" else "bin")')
 env := env
 env_bin := $(env)/$(bin_dir)
-env_py := $(env_bin)/python
-pip := pip --disable-pip-version-check
-with_local_env := $(env_py) cli/run.py -e defaults.env,local.env
-with_tests_env := $(env_py) cli/run.py -e defaults.env,tests/test.env,tests/local.env
-py_test := $(with_tests_env) $(env_bin)/python -m pytest -Wd $$PYTEST_ARGS
+env_py := $(env_bin)/$(shell basename $(python))
+pip := $(env_py) -m pip --disable-pip-version-check
+with_local_env := $(env_py) cli/run.py -e defaults.env,local.env -s PYTHONPATH=.
+with_tests_env := $(env_py) cli/run.py -e defaults.env,tests/test.env,tests/local.env -s PYTHONPATH=.
+py_test := $(with_tests_env) $(env_py) -m pytest -Wd $$PYTEST_ARGS
 
 echo:
 	@echo $($(var))
 
-$(env): requirements*.txt
+$(env): Makefile requirements*.txt
 	@if [ "$(python)" = "false" ]; then \
 		echo "Unable to find a 'python' executable. Please make sure that Python is installed."; \
 		exit 1; \
 	fi;
 	@$(python) cli/check-python-version.py
 	$(python) -m venv $(env)
-	$(env_bin)/$(pip) install wheel
-	$(env_bin)/$(pip) install --require-hashes $$(for f in requirements_*.txt; do echo "-r $$f"; done)
+	$(pip) install wheel
+	$(pip) install --require-hashes -r requirements_base.txt
+	$(pip) install -r requirements_tests.txt
 	@touch $(env)
 
 rehash-requirements:
-	$(env_bin)/$(pip) install hashin
-	for f in requirements*.txt; do \
-	    sed -E -e '/^ *#/d' -e '/^ +--hash/d' -e 's/(; .+)?\\$$//' $$f | xargs $(env_bin)/hashin -r $$f -p 3.8 -p 3.9; \
-	done
+	$(pip) install hashin
+	f=requirements_base.txt; \
+	sed -E -e '/^ *#/d' -e '/^ +--hash/d' -e 's/(; .+)?\\$$//' $$f | xargs $(env_bin)/hashin -r $$f;
 
 clean:
 	rm -rf $(env) *.egg *.egg-info
@@ -49,19 +49,19 @@ data: $(env)
 	$(with_local_env) $(env_py) -m liberapay.utils.fake_data
 
 db-migrations: sql/migrations.sql
-	PYTHONPATH=. $(with_local_env) $(env_py) liberapay/models/__init__.py
+	$(with_local_env) $(env_py) liberapay/models/__init__.py
 
 run: $(env)
 	@$(MAKE) --no-print-directory db-migrations || true
 	PATH=$(env_bin):$$PATH $(with_local_env) $(env_py) app.py
 
 py: $(env)
-	PYTHONPATH=. $(with_local_env) -s RUN_CRON_JOBS=no $(env_py) -i $${main-liberapay/main.py}
+	$(with_local_env) -s RUN_CRON_JOBS=no $(env_py) -i $${main-liberapay/main.py}
 
 shell: py
 
 test-shell: $(env)
-	PYTHONPATH=. $(with_tests_env) $(env_py) -i $${main-liberapay/main.py}
+	$(with_tests_env) $(env_py) -i $${main-liberapay/main.py}
 
 test-schema: $(env)
 	$(with_tests_env) ./recreate-schema.sh test
@@ -73,33 +73,33 @@ test: test-schema pytest
 tests: test
 
 pytest: $(env)
-	PYTHONPATH=. $(py_test) ./tests/py/test_$${PYTEST-*}.py
+	$(py_test) ./tests/py/test_$${PYTEST-*}.py
 	@$(MAKE) --no-print-directory pyflakes
 	$(py_test) --doctest-modules liberapay
 
 pytest-cov: $(env)
-	PYTHONPATH=. $(py_test) --cov-report html --cov liberapay ./tests/py/test_$${PYTEST-*}.py
+	$(py_test) --cov-report html --cov liberapay ./tests/py/test_$${PYTEST-*}.py
 	@$(MAKE) --no-print-directory pyflakes
 	$(py_test) --doctest-modules liberapay
 
 pytest-re: $(env)
-	PYTHONPATH=. $(py_test) --lf ./tests/py/
+	$(py_test) --lf ./tests/py/
 	@$(MAKE) --no-print-directory pyflakes
 
 pytest-i18n-browse: $(env)
 	@if [ -f sql/branch.sql ]; then $(MAKE) --no-print-directory test-schema; fi
-	PYTHONPATH=. LIBERAPAY_I18N_TEST=yes $(py_test) -k TestTranslations ./tests/py/
+	LIBERAPAY_I18N_TEST=yes $(py_test) -k TestTranslations ./tests/py/
 
 pytest-profiling: $(env)
-	PYTHONPATH=. LIBERAPAY_PROFILING=yes $(py_test) -k $${k-TestPerformance} --profile-svg ./tests/py/
+	LIBERAPAY_PROFILING=yes $(py_test) -k $${k-TestPerformance} --profile-svg ./tests/py/
 
 _i18n_extract: $(env)
 	@PYTHONPATH=. $(env_bin)/pybabel extract -F .babel_extract --no-wrap -o i18n/core.pot --sort-by-file $$(\
 		git ls-files | \
 		grep -E '^(liberapay/.+\.py|.+\.(spt|html))$$' | \
 		python -c "import sys; print(*sorted(sys.stdin, key=lambda l: l.rsplit('/', 1)))" \
 	)
-	@PYTHONPATH=. $(env_bin)/python cli/po-tools.py reflag i18n/core.pot
+	@$(env_bin)/python cli/po-tools.py reflag i18n/core.pot
 	@for f in i18n/*/*.po; do \
 		$(env_bin)/pybabel update -i i18n/core.pot -l $$(basename -s '.po' "$$f") -o "$$f" --ignore-obsolete --no-fuzzy-matching --no-wrap; \
 	done
@@ -120,7 +120,7 @@ _i18n_clean: $(env)
 	done
 
 _i18n_convert: $(env)
-	@PYTHONPATH=. $(env_bin)/python cli/convert-chinese.py
+	@$(env_bin)/python cli/convert-chinese.py
 
 i18n_update: _i18n_rebase _i18n_pull _i18n_extract _i18n_convert _i18n_clean
 	@if git commit --dry-run i18n &>/dev/null; then \
@@ -161,7 +161,7 @@ _i18n_merge:
 	git reset -q master -- i18n
 	@for f in i18n/*/*.po; do \
 		if test $$(sed -E -e '/\\n"$$/{d;d}' $$f | grep -c -E '^"' 2>/dev/null) -gt 10; then \
-			PYTHONPATH=. $(env_bin)/python cli/po-tools.py reformat $$f; \
+			$(env_bin)/python cli/po-tools.py reformat $$f; \
 		fi \
 	done
 	@$(MAKE) --no-print-directory _i18n_clean
@@ -190,4 +190,4 @@ bootstrap-upgrade:
 	rm -rf bootstrap-sass-$(version){,.tar.gz}
 
 stripe-bridge:
-	PYTHONPATH=. $(with_local_env) $(env_py) cli/stripe-bridge.py
+	$(with_local_env) $(env_py) cli/stripe-bridge.py

README.md

@@ -164,7 +164,7 @@ If you're testing an API which uses idempotency keys (for example Stripe's API),
 
 #### Speeding up the tests
 
-PostgreSQL is designed to prevent data loss, so it does a lot of synchronous disk writes by default. To reduce the number of those blocking writes, our `recreate-schema.sh` script automatically switches the `synchronous_commit` option to `off` for the test database, however this doesn't completely disable syncing. If your PostgreSQL instance only contains data that you can afford to lose, then you can speed things up further by setting `fsync` to `off` in the server's configuration file (`postgresql.conf`).
+PostgreSQL is designed to prevent data loss, so it does a lot of synchronous disk writes by default. To reduce the number of those blocking writes, our `recreate-schema.sh` script automatically switches the `synchronous_commit` option to `off` for the test database, however this doesn't completely disable syncing. If your PostgreSQL instance only contains data that you can afford to lose, then you can speed things up further by setting `fsync` to `off`, `wal_level` to `minimal` and `max_wal_senders` to `0` in the server's configuration file (`postgresql.conf`).
 
 ### Tinkering with payments
 
@@ -181,12 +181,13 @@ All new dependencies need to be audited to check that they don't contain malicio
 We use [pip's Hash-Checking Mode](https://pip.pypa.io/en/stable/topics/secure-installs/#hash-checking-mode) to protect ourselves from dependency tampering. Thus, when adding or upgrading a dependency the new hashes need to be computed and put in the requirements file. For that you can use [hashin](https://github.com/peterbe/hashin):
 
     pip install hashin
-    hashin package==x.y -r requirements_base.txt -p 3.8 -p 3.9
-    # note: we have several requirements files, use the right one
+    hashin package==x.y -r requirements_base.txt
 
 If for some reason you need to rehash all requirements, run `make rehash-requirements`.
 
-To upgrade all the dependencies in a requirements file, run `hashin -u -r requirements_XXX.txt -p 3.8 -p 3.9`. You may have to run extra `hashin` commands if new subdependencies are missing.
+To upgrade all the dependencies in the requirements file, run `hashin -u -r requirements_base.txt`. You may have to run extra `hashin` commands if new subdependencies are missing.
+
+The testing dependencies in `requirements_tests.txt` don't follow these rules because they're not installed in production. It's up to you to isolate your development environment from the rest of your system in order to protect it from possible vulnerabilities in the testing dependencies.
 
 ### Processing personal data
 

cli/paypal_payout_countries.py

@@ -0,0 +1,28 @@
+import re
+from time import sleep
+
+import requests
+
+
+sess = requests.Session()
+r = sess.get('https://www.paypal.com/webapps/mpp/country-worldwide')
+country_codes = set(re.findall(r"/([a-z]{2})/home", r.text))
+for cc in sorted(country_codes):
+    print(f"Requesting info for country code {cc.upper()}")
+    r = sess.get(f"https://www.paypal.com/{cc}/home")
+    if "Please wait while we perform security check" in r.text:
+        raise Exception("PayPal blocked the request")
+    if f"/{cc}/webapps/" not in r.text:
+        raise Exception("PayPal's response doesn't seem to contain the expected information")
+    is_supported = (
+        f"/{cc}/webapps/mpp/accept-payments-online" in r.text or
+        f"/{cc}/business/accept-payments" in r.text
+    )
+    if not is_supported:
+        country_codes.remove(cc)
+    sleep(1.5)
+
+country_codes.remove('uk')
+country_codes.add('gb')
+print(f"PayPal should be available to creators in the following {len(country_codes)} countries:")
+print(' '.join(map(str.upper, sorted(country_codes))))

defaults.env

@@ -12,6 +12,7 @@ CSP_EXTRA=
 OAUTHLIB_INSECURE_TRANSPORT=1
 OAUTHLIB_RELAX_TOKEN_SCOPE=1
 
+SENTRY_DEBUG=no
 SENTRY_DSN=
 SENTRY_RERAISE=no
 

emails/account_disabled.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your Liberapay account has been disabled") }}
 
 [---] text/html

emails/donate_reminder.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 % if len(donations) == 1
 {{ _("It's time to renew your donation to {username} on Liberapay", username=donations[0].tippee_username) }}
 % else

emails/donate_reminder~v2.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 % if len(donations) == 1
     % if overdue|default(False)
         {{ _("It's past time to renew your donation to {username} on Liberapay", username=donations[0].tippee_username) }}

emails/email_blacklisted.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your email address is temporarily blacklisted") if ignore_after|default(None) else
    _("Your email address has been blacklisted") }}
 

emails/income~v2.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 % if total.fuzzy
 {{ _("Your Liberapay income is approximately {money_amount} this week", money_amount=total) }}
 % else

emails/invoice_accepted.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your invoice to {0} has been accepted - Liberapay", addressee_name) }}
 
 [---] text/html

emails/invoice_paid.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your invoice to {0} has been accepted and paid - Liberapay", addressee_name) }}
 
 [---] text/html

emails/invoice_rejected.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your invoice to {0} has been rejected - Liberapay", addressee_name) }}
 
 [---] text/html

emails/login_link.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Log in to Liberapay") }}
 
 [---] text/html

emails/missing_route.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Liberapay donation renewal: no valid payment instrument") }}
 
 [---] text/html

emails/new_invoice.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Invoice from {0} on Liberapay", sender_name) }}
 
 [---] text/html

emails/newsletter.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ subject }}
 
 [---] text/html

emails/password_warning.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("The password of your Liberapay account is weak") }}
 
 [---] text/html

emails/payin_disputed.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your payment of {money_amount} has been disputed", money_amount=payin_amount) }}
 
 [---] text/html

emails/payin_failed.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your payment has failed") }}
 
 [---] text/html

emails/payin_refund_initiated.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("A refund of {money_amount} has been initiated", money_amount=refund_amount) }}
 
 [---] text/html

emails/payin_sdd_created.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your bank account is going to be debited") }}
 
 [---] text/html

emails/payin_succeeded.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your payment has succeeded") }}
 
 [---] text/html

emails/payment_account_required.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("You're missing out on donations through Liberapay") }}
 
 [---] text/html

emails/payment_schedule_modified.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Liberapay donation renewal: your upcoming payment has changed") }}
 
 [---] text/html

emails/pledgee_joined.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("{0} from {1} has joined Liberapay!", user_name, platform) }}
 
 [---] text/html

emails/pledgee_joined~v2.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("{user_name} from {platform} has joined Liberapay!", user_name=user_name, platform=platform) }}
 
 [---] text/html

emails/profile_description_missing.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your Liberapay profile is incomplete") }}
 
 [---] text/html

emails/renewal_aborted.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Liberapay donation renewal: payment aborted") }}
 
 [---] text/html

emails/renewal_actionable.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Liberapay donation renewal: manual action required") }}
 
 [---] text/html

emails/renewal_unauthorized.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Liberapay donation renewal: authentication required") }}
 
 [---] text/html

emails/stripe_account_missing.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("You can make it easier for your patrons to support you through Liberapay") }}
 
 [---] text/html

emails/team_currency_change.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("A team you're a member of has been modified") }}
 
 [---] text/html

emails/team_invite.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("You have been invited to join a team on Liberapay") }}
 
 [---] text/html

emails/team_rename.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("A team you're a member of has been renamed") }}
 
 [---] text/html

emails/upcoming_debit.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ ngettext(
     "Liberapay donation renewal: upcoming debit of {money_amount}",
     "Liberapay donation renewal: upcoming debits totaling {money_amount}",

emails/verification.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Email address verification - Liberapay") }}
 
 [---] text/html

emails/verification_notice.spt

@@ -1,3 +1,4 @@
+[---] -/subject
 {{ _("Your Liberapay account is being modified") }}
 
 [---] text/html