Skip to content

Commit

Permalink
Feature/TP1-569: Add Invoke command to copy Staging DB to a specific …
Browse files Browse the repository at this point in the history 
…Review App (#12493)

* Feature: Add Invoke command to copy Staging DB to a specific Review App

* Fix: Change Domain and Hostnames for Review Apps in cleanup.sql

* Refactor: Avoid duplicate code for PLATFORM_ARG

* Feature: Add the staging-db-to-review-app invoke command to the docs

* Fix: Linting and formatting
  • Loading branch information
@AdalbertoMoz
AdalbertoMoz authored Jun 19, 2024
1 parent d066a7c commit da2d912
Show file tree
Hide file tree
Showing 4 changed files with 193 additions and 20 deletions.
59 changes: 59 additions & 0 deletions cleanup.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
-- noinspection SqlNoDataSourceInspectionForFile

CREATE EXTENSION IF NOT EXISTS pgcrypto;

CREATE OR REPLACE FUNCTION clean_user_data()
RETURNS VOID AS $$
DECLARE
user_row RECORD;
new_email varchar;
new_hash varchar;
new_username varchar;
counter integer := 1;
BEGIN
-- scrub the user table
TRUNCATE django_session;

-- clean up non-staff social auth data
DELETE FROM social_auth_usersocialauth
WHERE uid NOT LIKE '%@mozillafoundation.org';

-- Update the site domain
UPDATE django_site
SET domain = '{DOMAIN}.mofostaging.net'
WHERE domain = 'foundation.mofostaging.net';

UPDATE wagtailcore_site
SET hostname = '{HOSTNAME}.mofostaging.net'
WHERE hostname = 'foundation.mofostaging.net';

UPDATE wagtailcore_site
SET hostname = 'mozfest-{HOSTNAME}.mofostaging.net'
WHERE hostname = 'mozillafestival.mofostaging.net';

-- Iterate over each non-staff user and remove any PII
FOR user_row IN
SELECT id
FROM auth_user
WHERE email NOT LIKE '%@mozillafoundation.org'
LOOP
new_email := concat(encode(gen_random_bytes(12), 'base64'), '@example.com');
new_hash := crypt(encode(gen_random_bytes(32), 'base64'), gen_salt('bf', 6));
new_username := concat('anonymouse', counter::varchar);

UPDATE auth_user
SET
email = new_email,
password = new_hash,
username = new_username,
first_name = 'anony',
last_name = 'mouse'
Where id = user_row.id;

-- Increase the counter
counter := counter + 1;
END LOOP;
END;
$$ LANGUAGE plpgsql;

SELECT clean_user_data();
103 changes: 103 additions & 0 deletions copy_staging_db_to_review_app.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
import tempfile
from time import sleep

from tasks import PLATFORM_ARG

STAGING_APP = "foundation-mofostaging-net"


def execute_command(ctx, command: str, custom_error: str = ""):
try:
result = ctx.run(command, hide=False, warn=True, **PLATFORM_ARG)
if result.failed:
raise Exception(f"{custom_error}: {result.stderr}")
return result.stdout.strip()
except Exception as e:
raise Exception(f"{custom_error}: {e}") from e


def log_step(message: str):
print(f"--> {message}\n", flush=True)


def log_step_completed(message: str):
print(f"✔️ {message} completed.\n", flush=True)


def replace_placeholders_in_sql(review_app_name: str, input_file: str) -> str:
with open(input_file, "r") as file:
sql_content = file.read()

sql_content = sql_content.replace("{DOMAIN}", review_app_name)
sql_content = sql_content.replace("{HOSTNAME}", review_app_name)

return sql_content


def main(ctx, review_app_name):
log_step(f"The review app name is: {review_app_name}, if not, please cancel now...")
sleep(5)

log_step("Verifying if logged in Heroku")
heroku_user = execute_command(ctx, "heroku whoami", "Verify that you are logged in Heroku CLI")
print(f"Heroku user: {heroku_user}\n", flush=True)
log_step_completed("Heroku login verification")

log_step("Verifying if psql is installed")
execute_command(ctx, "psql --version", "Verify that you have 'psql' installed")
log_step_completed("psql installation verification")

try:
log_step("Enabling maintenance mode on the Review App")
execute_command(ctx, f"heroku maintenance:on -a {review_app_name}")
log_step_completed("Maintenance mode enabling")

log_step("Scaling web dynos on Review App to 0")
execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=0")
log_step_completed("Web dynos scaling to 0")

log_step("Backing up Staging DB")
execute_command(ctx, f"heroku pg:backups:capture -a {STAGING_APP}")
log_step_completed("Staging DB backup")

log_step("Backing up Review App DB")
execute_command(ctx, f"heroku pg:backups:capture -a {review_app_name}")
log_step_completed("Review App DB backup")

log_step("Restoring the latest Staging backup to Review App")
backup_staging_url = execute_command(ctx, f"heroku pg:backups:url -a {STAGING_APP}")
execute_command(
ctx, f"heroku pg:backups:restore --confirm {review_app_name} -a {review_app_name} '{backup_staging_url}'"
)
log_step_completed("Staging backup restoration to Review App")

log_step("Executing cleanup SQL script")
review_app_db_url = execute_command(ctx, f"heroku config:get -a {review_app_name} DATABASE_URL")

# Replace placeholders and write to a temporary file
sql_content = replace_placeholders_in_sql(review_app_name, "./cleanup.sql")
with tempfile.NamedTemporaryFile(suffix=".sql", mode="w", delete=True) as temp_sql_file:
temp_sql_file.write(sql_content)
temp_sql_file.flush()
execute_command(ctx, f"psql {review_app_db_url} -f {temp_sql_file.name}")

log_step_completed("Cleanup SQL script execution")

log_step("Running migrations")
execute_command(ctx, f"heroku run -a {review_app_name} -- python network-api/manage.py migrate --no-input")
log_step_completed("Migrations running")

except Exception as e:
log_step("Rolling back Review App")
execute_command(ctx, f"heroku pg:backups:restore -a {review_app_name} --confirm {review_app_name}")
print(e, flush=True)
log_step_completed("Review App rollback")

finally:
log_step("Scaling web dynos on Review App to 1")
execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=1")
log_step_completed("Web dynos scaling to 1")

log_step("Disabling maintenance mode on Review App")
execute_command(ctx, f"heroku maintenance:off -a {review_app_name}")
log_step_completed("Maintenance mode disabling")
41 changes: 21 additions & 20 deletions docs/local_development.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,26 +25,27 @@ The general workflow is:
To get a list of invoke commands available, run `invoke -l`:

```
catch-up (catchup, docker-catchup) Rebuild images, install dependencies, and apply migrations
compilemessages (docker-compilemessages) Compile the latest translations
makemessages (docker-makemessages) Extract all template messages in .po files for localization
makemigrations (docker-makemigrations) Creates new migration(s) for apps
manage (docker-manage) Shorthand to manage.py. inv docker-manage "[COMMAND] [ARG]"
migrate (docker-migrate) Updates database schema
new-db (docker-new-db) Delete your database and create a new one with fake data
copy-stage-db Overwrite your local docker postgres DB with a copy of the staging database
copy-prod-db Overwrite your local docker postgres DB with a copy of the production database
new-env (docker-new-env) Get a new dev environment and a new database with fake data
npm (docker-npm) Shorthand to npm. inv docker-npm "[COMMAND] [ARG]"
npm-install (docker-npm-install) Install Node dependencies
pip-compile (docker-pip-compile) Shorthand to pip-tools. inv pip-compile "[filename]" "[COMMAND] [ARG]"
pip-compile-lock (docker-pip-compile-lock) Lock prod and dev dependencies
pip-sync (docker-pip-sync) Sync your python virtualenv
start-dev (docker-start, start) Start the dev server
start-lean-dev (docker-start-lean, start-lean) Start the dev server without rebuilding
test (docker-test) Run both Node and Python tests
test-node (docker-test-node) Run node tests
test-python (docker-test-python) Run python tests
catch-up (catchup, docker-catchup) Rebuild images, install dependencies, and apply migrations
compilemessages (docker-compilemessages) Compile the latest translations
makemessages (docker-makemessages) Extract all template messages in .po files for localization
makemigrations (docker-makemigrations) Creates new migration(s) for apps
manage (docker-manage) Shorthand to manage.py. inv docker-manage "[COMMAND] [ARG]"
migrate (docker-migrate) Updates database schema
new-db (docker-new-db) Delete your database and create a new one with fake data
copy-stage-db Overwrite your local docker postgres DB with a copy of the staging database
copy-prod-db Overwrite your local docker postgres DB with a copy of the production database
new-env (docker-new-env) Get a new dev environment and a new database with fake data
npm (docker-npm) Shorthand to npm. inv docker-npm "[COMMAND] [ARG]"
npm-install (docker-npm-install) Install Node dependencies
pip-compile (docker-pip-compile) Shorthand to pip-tools. inv pip-compile "[filename]" "[COMMAND] [ARG]"
pip-compile-lock (docker-pip-compile-lock) Lock prod and dev dependencies
pip-sync (docker-pip-sync) Sync your python virtualenv
staging-db-to-review-app (staging-to-review-app) Copy Staging DB to a specific Review App. inv staging-to-review-app "[REVIEW_APP_NAME]"
start-dev (docker-start, start) Start the dev server
start-lean-dev (docker-start-lean, start-lean) Start the dev server without rebuilding
test (docker-test) Run both Node and Python tests
test-node (docker-test-node) Run node tests
test-python (docker-test-python) Run python tests
```

Note the above commands carefully, as they should cover the majority of what you'd need for local development.
Expand Down
10 changes: 10 additions & 0 deletions tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -585,3 +585,13 @@ def compilemessages(ctx):
"../dockerpythonvenv/bin/python manage.py compilemessages",
**PLATFORM_ARG,
)


@task(aliases=["staging-to-review-app"])
def staging_db_to_review_app(ctx, review_app_name):
"""
Copy Staging DB to a specific Review App. inv staging-to-review-app \"[REVIEW_APP_NAME]\"
"""
from copy_staging_db_to_review_app import main

main(ctx, review_app_name)

0 comments on commit da2d912

Please sign in to comment.