From da2d9129d3f64d5f8f082c6273e439a26e797223 Mon Sep 17 00:00:00 2001 From: Adalberto Vazquez Date: Wed, 19 Jun 2024 12:10:40 -0600 Subject: [PATCH] Feature/TP1-569: Add Invoke command to copy Staging DB to a specific Review App (#12493) * Feature: Add Invoke command to copy Staging DB to a specific Review App * Fix: Change Domain and Hostnames for Review Apps in cleanup.sql * Refactor: Avoid duplicate code for PLATFORM_ARG * Feature: Add the staging-db-to-review-app invoke command to the docs * Fix: Linting and formatting --- cleanup.sql | 59 ++++++++++++++++++ copy_staging_db_to_review_app.py | 103 +++++++++++++++++++++++++++++++ docs/local_development.md | 41 ++++++------ tasks.py | 10 +++ 4 files changed, 193 insertions(+), 20 deletions(-) create mode 100644 cleanup.sql create mode 100644 copy_staging_db_to_review_app.py diff --git a/cleanup.sql b/cleanup.sql new file mode 100644 index 00000000000..248cd9ef331 --- /dev/null +++ b/cleanup.sql @@ -0,0 +1,59 @@ +-- noinspection SqlNoDataSourceInspectionForFile + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE OR REPLACE FUNCTION clean_user_data() +RETURNS VOID AS $$ +DECLARE + user_row RECORD; + new_email varchar; + new_hash varchar; + new_username varchar; + counter integer := 1; +BEGIN +-- scrub the user table + TRUNCATE django_session; + +-- clean up non-staff social auth data + DELETE FROM social_auth_usersocialauth + WHERE uid NOT LIKE '%@mozillafoundation.org'; + +-- Update the site domain + UPDATE django_site + SET domain = '{DOMAIN}.mofostaging.net' + WHERE domain = 'foundation.mofostaging.net'; + + UPDATE wagtailcore_site + SET hostname = '{HOSTNAME}.mofostaging.net' + WHERE hostname = 'foundation.mofostaging.net'; + + UPDATE wagtailcore_site + SET hostname = 'mozfest-{HOSTNAME}.mofostaging.net' + WHERE hostname = 'mozillafestival.mofostaging.net'; + +-- Iterate over each non-staff user and remove any PII + FOR user_row IN + SELECT id + FROM auth_user + WHERE email NOT LIKE '%@mozillafoundation.org' + LOOP + new_email := concat(encode(gen_random_bytes(12), 'base64'), '@example.com'); + new_hash := crypt(encode(gen_random_bytes(32), 'base64'), gen_salt('bf', 6)); + new_username := concat('anonymouse', counter::varchar); + + UPDATE auth_user + SET + email = new_email, + password = new_hash, + username = new_username, + first_name = 'anony', + last_name = 'mouse' + Where id = user_row.id; + +-- Increase the counter + counter := counter + 1; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +SELECT clean_user_data(); diff --git a/copy_staging_db_to_review_app.py b/copy_staging_db_to_review_app.py new file mode 100644 index 00000000000..c56acad459b --- /dev/null +++ b/copy_staging_db_to_review_app.py @@ -0,0 +1,103 @@ +import tempfile +from time import sleep + +from tasks import PLATFORM_ARG + +STAGING_APP = "foundation-mofostaging-net" + + +def execute_command(ctx, command: str, custom_error: str = ""): + try: + result = ctx.run(command, hide=False, warn=True, **PLATFORM_ARG) + if result.failed: + raise Exception(f"{custom_error}: {result.stderr}") + return result.stdout.strip() + except Exception as e: + raise Exception(f"{custom_error}: {e}") from e + + +def log_step(message: str): + print(f"--> {message}\n", flush=True) + + +def log_step_completed(message: str): + print(f"✔️ {message} completed.\n", flush=True) + + +def replace_placeholders_in_sql(review_app_name: str, input_file: str) -> str: + with open(input_file, "r") as file: + sql_content = file.read() + + sql_content = sql_content.replace("{DOMAIN}", review_app_name) + sql_content = sql_content.replace("{HOSTNAME}", review_app_name) + + return sql_content + + +def main(ctx, review_app_name): + log_step(f"The review app name is: {review_app_name}, if not, please cancel now...") + sleep(5) + + log_step("Verifying if logged in Heroku") + heroku_user = execute_command(ctx, "heroku whoami", "Verify that you are logged in Heroku CLI") + print(f"Heroku user: {heroku_user}\n", flush=True) + log_step_completed("Heroku login verification") + + log_step("Verifying if psql is installed") + execute_command(ctx, "psql --version", "Verify that you have 'psql' installed") + log_step_completed("psql installation verification") + + try: + log_step("Enabling maintenance mode on the Review App") + execute_command(ctx, f"heroku maintenance:on -a {review_app_name}") + log_step_completed("Maintenance mode enabling") + + log_step("Scaling web dynos on Review App to 0") + execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=0") + log_step_completed("Web dynos scaling to 0") + + log_step("Backing up Staging DB") + execute_command(ctx, f"heroku pg:backups:capture -a {STAGING_APP}") + log_step_completed("Staging DB backup") + + log_step("Backing up Review App DB") + execute_command(ctx, f"heroku pg:backups:capture -a {review_app_name}") + log_step_completed("Review App DB backup") + + log_step("Restoring the latest Staging backup to Review App") + backup_staging_url = execute_command(ctx, f"heroku pg:backups:url -a {STAGING_APP}") + execute_command( + ctx, f"heroku pg:backups:restore --confirm {review_app_name} -a {review_app_name} '{backup_staging_url}'" + ) + log_step_completed("Staging backup restoration to Review App") + + log_step("Executing cleanup SQL script") + review_app_db_url = execute_command(ctx, f"heroku config:get -a {review_app_name} DATABASE_URL") + + # Replace placeholders and write to a temporary file + sql_content = replace_placeholders_in_sql(review_app_name, "./cleanup.sql") + with tempfile.NamedTemporaryFile(suffix=".sql", mode="w", delete=True) as temp_sql_file: + temp_sql_file.write(sql_content) + temp_sql_file.flush() + execute_command(ctx, f"psql {review_app_db_url} -f {temp_sql_file.name}") + + log_step_completed("Cleanup SQL script execution") + + log_step("Running migrations") + execute_command(ctx, f"heroku run -a {review_app_name} -- python network-api/manage.py migrate --no-input") + log_step_completed("Migrations running") + + except Exception as e: + log_step("Rolling back Review App") + execute_command(ctx, f"heroku pg:backups:restore -a {review_app_name} --confirm {review_app_name}") + print(e, flush=True) + log_step_completed("Review App rollback") + + finally: + log_step("Scaling web dynos on Review App to 1") + execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=1") + log_step_completed("Web dynos scaling to 1") + + log_step("Disabling maintenance mode on Review App") + execute_command(ctx, f"heroku maintenance:off -a {review_app_name}") + log_step_completed("Maintenance mode disabling") diff --git a/docs/local_development.md b/docs/local_development.md index 10796a30bc1..16eff3d76da 100644 --- a/docs/local_development.md +++ b/docs/local_development.md @@ -25,26 +25,27 @@ The general workflow is: To get a list of invoke commands available, run `invoke -l`: ``` - catch-up (catchup, docker-catchup) Rebuild images, install dependencies, and apply migrations - compilemessages (docker-compilemessages) Compile the latest translations - makemessages (docker-makemessages) Extract all template messages in .po files for localization - makemigrations (docker-makemigrations) Creates new migration(s) for apps - manage (docker-manage) Shorthand to manage.py. inv docker-manage "[COMMAND] [ARG]" - migrate (docker-migrate) Updates database schema - new-db (docker-new-db) Delete your database and create a new one with fake data - copy-stage-db Overwrite your local docker postgres DB with a copy of the staging database - copy-prod-db Overwrite your local docker postgres DB with a copy of the production database - new-env (docker-new-env) Get a new dev environment and a new database with fake data - npm (docker-npm) Shorthand to npm. inv docker-npm "[COMMAND] [ARG]" - npm-install (docker-npm-install) Install Node dependencies - pip-compile (docker-pip-compile) Shorthand to pip-tools. inv pip-compile "[filename]" "[COMMAND] [ARG]" - pip-compile-lock (docker-pip-compile-lock) Lock prod and dev dependencies - pip-sync (docker-pip-sync) Sync your python virtualenv - start-dev (docker-start, start) Start the dev server - start-lean-dev (docker-start-lean, start-lean) Start the dev server without rebuilding - test (docker-test) Run both Node and Python tests - test-node (docker-test-node) Run node tests - test-python (docker-test-python) Run python tests + catch-up (catchup, docker-catchup) Rebuild images, install dependencies, and apply migrations + compilemessages (docker-compilemessages) Compile the latest translations + makemessages (docker-makemessages) Extract all template messages in .po files for localization + makemigrations (docker-makemigrations) Creates new migration(s) for apps + manage (docker-manage) Shorthand to manage.py. inv docker-manage "[COMMAND] [ARG]" + migrate (docker-migrate) Updates database schema + new-db (docker-new-db) Delete your database and create a new one with fake data + copy-stage-db Overwrite your local docker postgres DB with a copy of the staging database + copy-prod-db Overwrite your local docker postgres DB with a copy of the production database + new-env (docker-new-env) Get a new dev environment and a new database with fake data + npm (docker-npm) Shorthand to npm. inv docker-npm "[COMMAND] [ARG]" + npm-install (docker-npm-install) Install Node dependencies + pip-compile (docker-pip-compile) Shorthand to pip-tools. inv pip-compile "[filename]" "[COMMAND] [ARG]" + pip-compile-lock (docker-pip-compile-lock) Lock prod and dev dependencies + pip-sync (docker-pip-sync) Sync your python virtualenv + staging-db-to-review-app (staging-to-review-app) Copy Staging DB to a specific Review App. inv staging-to-review-app "[REVIEW_APP_NAME]" + start-dev (docker-start, start) Start the dev server + start-lean-dev (docker-start-lean, start-lean) Start the dev server without rebuilding + test (docker-test) Run both Node and Python tests + test-node (docker-test-node) Run node tests + test-python (docker-test-python) Run python tests ``` Note the above commands carefully, as they should cover the majority of what you'd need for local development. diff --git a/tasks.py b/tasks.py index 1e8f9b42605..01be9217412 100644 --- a/tasks.py +++ b/tasks.py @@ -585,3 +585,13 @@ def compilemessages(ctx): "../dockerpythonvenv/bin/python manage.py compilemessages", **PLATFORM_ARG, ) + + +@task(aliases=["staging-to-review-app"]) +def staging_db_to_review_app(ctx, review_app_name): + """ + Copy Staging DB to a specific Review App. inv staging-to-review-app \"[REVIEW_APP_NAME]\" + """ + from copy_staging_db_to_review_app import main + + main(ctx, review_app_name)