From d28dec2e262a4a89d8b623a50c5c9b41aa367831 Mon Sep 17 00:00:00 2001 From: AdalbertoMoz Date: Mon, 17 Jun 2024 18:36:11 -0600 Subject: [PATCH] Feature: Add Invoke command to copy Staging DB to a specific Review App --- cleanup.sql | 59 ++++++++++++++++++ copy_staging_db_to_review_app.py | 103 +++++++++++++++++++++++++++++++ tasks.py | 7 +++ 3 files changed, 169 insertions(+) create mode 100644 cleanup.sql create mode 100644 copy_staging_db_to_review_app.py diff --git a/cleanup.sql b/cleanup.sql new file mode 100644 index 00000000000..19bc41e7a2d --- /dev/null +++ b/cleanup.sql @@ -0,0 +1,59 @@ +-- noinspection SqlNoDataSourceInspectionForFile + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE OR REPLACE FUNCTION clean_user_data() +RETURNS VOID AS $$ +DECLARE + user_row RECORD; + new_email varchar; + new_hash varchar; + new_username varchar; + counter integer := 1; +BEGIN +-- scrub the user table + TRUNCATE django_session; + +-- clean up non-staff social auth data + DELETE FROM social_auth_usersocialauth + WHERE uid NOT LIKE '%@mozillafoundation.org'; + +-- Update the site domain + UPDATE django_site + SET domain = '{DOMAIN}.herokuapp.com' + WHERE domain = 'foundation.mofostaging.net'; + + UPDATE wagtailcore_site + SET hostname = '{HOSTNAME}.herokuapp.com' + WHERE hostname = 'foundation.mofostaging.net'; + + UPDATE wagtailcore_site + SET hostname = 'mozillafestival.mofostaging.net' + WHERE hostname = 'mozillafestival.mofostaging.net'; + +-- Iterate over each non-staff user and remove any PII + FOR user_row IN + SELECT id + FROM auth_user + WHERE email NOT LIKE '%@mozillafoundation.org' + LOOP + new_email := concat(encode(gen_random_bytes(12), 'base64'), '@example.com'); + new_hash := crypt(encode(gen_random_bytes(32), 'base64'), gen_salt('bf', 6)); + new_username := concat('anonymouse', counter::varchar); + + UPDATE auth_user + SET + email = new_email, + password = new_hash, + username = new_username, + first_name = 'anony', + last_name = 'mouse' + Where id = user_row.id; + +-- Increase the counter + counter := counter + 1; + END LOOP; +END; +$$ LANGUAGE plpgsql; + +SELECT clean_user_data(); diff --git a/copy_staging_db_to_review_app.py b/copy_staging_db_to_review_app.py new file mode 100644 index 00000000000..e0f93f240c8 --- /dev/null +++ b/copy_staging_db_to_review_app.py @@ -0,0 +1,103 @@ +import tempfile +from sys import platform +from time import sleep + +PLATFORM_ARG = {"env": {"PYTHONUNBUFFERED": "True"}} if platform == "win32" else {"pty": True} +STAGING_APP = "foundation-mofostaging-net" + + +def execute_command(ctx, command: str, custom_error: str = ""): + try: + result = ctx.run(command, hide=False, warn=True, **PLATFORM_ARG) + if result.failed: + raise Exception(f"{custom_error}: {result.stderr}") + return result.stdout.strip() + except Exception as e: + raise Exception(f"{custom_error}: {e}") from e + + +def log_step(message: str): + print(f"--> {message}\n", flush=True) + + +def log_step_completed(message: str): + print(f"✔️ {message} completed.\n", flush=True) + + +def replace_placeholders_in_sql(review_app_name: str, input_file: str) -> str: + with open(input_file, "r") as file: + sql_content = file.read() + + sql_content = sql_content.replace("{DOMAIN}", review_app_name) + sql_content = sql_content.replace("{HOSTNAME}", review_app_name) + + return sql_content + + +def main(ctx, review_app_name): + log_step(f"The review app name is: {review_app_name}, if not, please cancel now...") + sleep(5) + + log_step("Verifying if logged in Heroku") + heroku_user = execute_command(ctx, "heroku whoami", "Verify that you are logged in Heroku CLI") + print(f"Heroku user: {heroku_user}\n", flush=True) + log_step_completed("Heroku login verification") + + log_step("Verifying if psql is installed") + execute_command(ctx, "psql --version", "Verify that you have 'psql' installed") + log_step_completed("psql installation verification") + + try: + log_step("Enabling maintenance mode on the Review App") + execute_command(ctx, f"heroku maintenance:on -a {review_app_name}") + log_step_completed("Maintenance mode enabling") + + log_step("Scaling web dynos on Review App to 0") + execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=0") + log_step_completed("Web dynos scaling to 0") + + log_step("Backing up Staging DB") + execute_command(ctx, f"heroku pg:backups:capture -a {STAGING_APP}") + log_step_completed("Staging DB backup") + + log_step("Backing up Review App DB") + execute_command(ctx, f"heroku pg:backups:capture -a {review_app_name}") + log_step_completed("Review App DB backup") + + log_step("Restoring the latest Staging backup to Review App") + backup_staging_url = execute_command(ctx, f"heroku pg:backups:url -a {STAGING_APP}") + execute_command( + ctx, f"heroku pg:backups:restore --confirm {review_app_name} -a {review_app_name} '{backup_staging_url}'" + ) + log_step_completed("Staging backup restoration to Review App") + + log_step("Executing cleanup SQL script") + review_app_db_url = execute_command(ctx, f"heroku config:get -a {review_app_name} DATABASE_URL") + + # Replace placeholders and write to a temporary file + sql_content = replace_placeholders_in_sql(review_app_name, "./cleanup.sql") + with tempfile.NamedTemporaryFile(suffix=".sql", mode="w", delete=True) as temp_sql_file: + temp_sql_file.write(sql_content) + temp_sql_file.flush() + execute_command(ctx, f"psql {review_app_db_url} -f {temp_sql_file.name}") + + log_step_completed("Cleanup SQL script execution") + + log_step("Running migrations") + execute_command(ctx, f"heroku run -a {review_app_name} -- python network-api/manage.py migrate --no-input") + log_step_completed("Migrations running") + + except Exception as e: + log_step("Rolling back Review App") + execute_command(ctx, f"heroku pg:backups:restore -a {review_app_name} --confirm {review_app_name}") + print(e, flush=True) + log_step_completed("Review App rollback") + + finally: + log_step("Scaling web dynos on Review App to 1") + execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=1") + log_step_completed("Web dynos scaling to 1") + + log_step("Disabling maintenance mode on Review App") + execute_command(ctx, f"heroku maintenance:off -a {review_app_name}") + log_step_completed("Maintenance mode disabling") diff --git a/tasks.py b/tasks.py index 1e8f9b42605..9774ce2bc0b 100644 --- a/tasks.py +++ b/tasks.py @@ -585,3 +585,10 @@ def compilemessages(ctx): "../dockerpythonvenv/bin/python manage.py compilemessages", **PLATFORM_ARG, ) + + +@task(aliases=["staging-to-review-app"]) +def staging_db_to_review_app(ctx, review_app_name): + from copy_staging_db_to_review_app import main + + main(ctx, review_app_name)