diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 5724da4aeb..4966dc78a8 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -224,6 +224,129 @@ def upgrade(context: click.Context, from_release: t.Optional[str]) -> None: context.invoke(config_save_command) +def interactive_mysql_upgrade(context: click.Context) -> None: + click.echo(fmt.title("Interactive mysql charset upgrade")) + config = tutor_config.load(context.obj.root) + upgrade_all_tables = click.confirm( + fmt.question( + "Would you like to upgrade all tables?" + "Some tables can take a very long time and cause downtime!" + ), + ) + if upgrade_all_tables: + #TODO: upgrade_mysql(all_tables) + pass + else: + pass + +@click.command( + short_help="Upgrade mysql to a specific charset and collation", + help=( + "Upgrade mysql to a specific charset and collation. You can either upgrade all tables, specify only certain tables to upgrade or specify certain tables to exclude from the upgrade process" + ), + context_settings={"ignore_unknown_options": True}, +) +@click.option("--all-tables", help="Upgrade all tables in the openedx database", prompt="Are you sure you want to upgrade all the tables?") +@click.option("-s", "--status", type=click.Choice(['include', 'exclude'], case_sensitive=False), help="Upgrade all tables in the openedx database") +@click.option("--charset", required=True, type=str, help="The charset to updgrade the tables to") +@click.option("--collation", required=True, type=str, help="The collation to updgrade the tables to") +# @click.argument("apps", nargs=-1) +@click.argument("tables", nargs=-1) +@click.pass_context +def upgrade_mysql( + context: BaseComposeContext, + all_tables: bool, + status: str, + charset: str, + collation: str, + # apps: list[str], + tables: list[str], +) -> None: + config = tutor_config.load_full(context.obj.root) + if all_tables: + upgrade_mysql_charset_collation(context, config, charset, collation) + + + +def upgrade_mysql_charset_collation( + context: click.Context, + config: Config, + charset: str, + collation: str, +) -> None: + if not config["RUN_MYSQL"]: + fmt.echo_info( + f"You are not running MySQL (RUN_MYSQL=false). It is your " + f"responsibility to upgrade your MySQL instance to {charset} charset and {collation} collation." + ) + return + click.echo(fmt.title(f"Upgrading charset and collation of all tables in MySQL to {charset} and {collation} respectively.")) + context.invoke(start, detach=True, services=["mysql"]) + fmt.echo_info("Waiting for mysql to boot...") + # sleep(10) + context.invoke( + execute, + args=[ + "mysql", + "mysql", + f"--user={config['MYSQL_ROOT_USERNAME']}", + f"--password={config['MYSQL_ROOT_PASSWORD']}", + f"--host={config['MYSQL_HOST']}", + f"--port={config['MYSQL_PORT']}", + # f"--default-character-set=utf8mb4", + f"--database={config['OPENEDX_MYSQL_DATABASE']}", + # "--skip-column-names", + # "--silent", + "-e", + f"""DROP PROCEDURE IF EXISTS UpdateTable; + DELIMITER $$ + +CREATE PROCEDURE UpdateTable() +BEGIN + + DECLARE done INT DEFAULT FALSE; + DECLARE _table_name VARCHAR(255); + DECLARE cur CURSOR FOR + SELECT table_name FROM information_schema.tables + WHERE table_schema = '{config['OPENEDX_MYSQL_DATABASE']}' AND table_type = 'BASE TABLE'; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + + OPEN cur; + My_loop: LOOP + FETCH cur INTO _table_name; + SET @my_table_name = _table_name; + + IF done THEN + LEAVE My_loop; + END IF; + + SET FOREIGN_KEY_CHECKS = 0; + + SET @stmt = CONCAT('ALTER TABLE ', _table_name, ' CONVERT TO CHARACTER SET {charset} COLLATE {collation};'); + PREPARE stmt1 FROM @stmt; + EXECUTE stmt1; + DEALLOCATE PREPARE stmt1; + + SET FOREIGN_KEY_CHECKS = 1; + + END LOOP; + CLOSE cur; + +END$$ + +DELIMITER ; + CALL UpdateTable(); + """, + ], + ) + context.invoke(stop) + + + + + + + @click.command( short_help="Run all or a selection of services.", help="Run all or a selection of services. Docker images will be rebuilt where necessary.", @@ -442,6 +565,7 @@ def add_commands(command_group: click.Group) -> None: command_group.add_command(execute) command_group.add_command(logs) command_group.add_command(status) + command_group.add_command(upgrade_mysql) @hooks.Actions.PLUGINS_LOADED.add() def _add_do_commands() -> None: diff --git a/tutor/commands/upgrade/__init__.py b/tutor/commands/upgrade/__init__.py index b7a4e028de..4c10ad391f 100644 --- a/tutor/commands/upgrade/__init__.py +++ b/tutor/commands/upgrade/__init__.py @@ -9,4 +9,5 @@ "olive", "palm", "quince", + "redwood", ] diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index 5973da4d55..2c0eac524d 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -48,6 +48,9 @@ def upgrade_from(context: click.Context, from_release: str) -> None: if running_release == "quince": upgrade_from_quince(context, config) + + if running_release == "redwood": + upgrade_from_redwood(context, config) def upgrade_from_ironwood(context: click.Context, config: Config) -> None: @@ -191,7 +194,7 @@ def upgrade_mongodb( tutor_env.save(context.obj.root, config) context.invoke(compose.start, detach=True, services=["mongodb"]) fmt.echo_info("Waiting for mongodb to boot...") - sleep(10) + sleep(30) context.invoke( compose.execute, args=[ @@ -202,3 +205,168 @@ def upgrade_mongodb( ], ) context.invoke(compose.stop) + +# def upgrade_mysql_charset_collation( +# context: click.Context, +# config: Config, +# charset: str, +# collation: str, +# ) -> None: +# if not config["RUN_MYSQL"]: +# fmt.echo_info( +# f"You are not running MySQL (RUN_MYSQL=false). It is your " +# f"responsibility to upgrade your MySQL instance to {charset} charset and {collation} collation." +# ) +# return +# click.echo(fmt.title(f"Upgrading charset and collation of all tables in MySQL to {charset} and {collation} respectively.")) +# context.invoke(compose.start, detach=True, services=["mysql"]) +# fmt.echo_info("Waiting for mysql to boot...") +# # sleep(10) +# queries_to_run = context.invoke( +# compose.execute, +# args=[ +# "mysql", +# "mysql", +# f"--user={config['MYSQL_ROOT_USERNAME']}", +# f"--password={config['MYSQL_ROOT_PASSWORD']}", +# f"--host={config['MYSQL_HOST']}", +# f"--port={config['MYSQL_PORT']}", +# # f"--default-character-set=utf8mb4", +# f"--database={config['OPENEDX_MYSQL_DATABASE']}", +# # "--skip-column-names", +# # "--silent", +# "-e", +# f"""DROP PROCEDURE IF EXISTS UpdateTable; +# DELIMITER $$ + +# CREATE PROCEDURE UpdateTable +# ( +# database varchar(64) +# charset varchar (16) +# collation varchar(64) +# ) +# BEGIN + +# DECLARE done INT DEFAULT FALSE; +# DECLARE _table_name VARCHAR(255); +# DECLARE cur CURSOR FOR +# SELECT table_name FROM information_schema.tables +# WHERE table_schema = database AND table_type = "BASE TABLE"; +# DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + +# OPEN cur; +# My_loop: LOOP +# FETCH cur INTO _table_name; +# SET @my_table_name = _table_name; + +# IF done THEN +# LEAVE My_loop; +# END IF; + +# SET FOREIGN_KEY_CHECKS = 0; + +# SET @stmt = CONCAT('ALTER TABLE ', _table_name, ' CONVERT TO CHARACTER SET charset COLLATE collation;'); +# PREPARE stmt1 FROM @stmt; +# EXECUTE stmt1; +# DEALLOCATE PREPARE stmt1; + +# SET FOREIGN_KEY_CHECKS = 1; + +# END LOOP; +# CLOSE cur; + +# END$$ + +# DELIMITER ; +# CALL UpdateTable(); +# """, +# # f"SET @queryResults = '';SET @queries= 'SELECT CONCAT(\"ALTER TABLE \", TABLE_NAME,\" CONVERT TO CHARACTER SET {charset} COLLATE {collation};\") \ +# # FROM INFORMATION_SCHEMA.TABLES \ +# # WHERE TABLE_SCHEMA=\"{config['OPENEDX_MYSQL_DATABASE']}\" AND \ +# # TABLE_TYPE=\"BASE TABLE\" into @queryResults;'; \ +# # PREPARE stmnt from @queries; \ +# # EXECUTE stmnt; \ +# # DEALLOCATE PREPARE stmnt; \ +# # PREPARE stmnt from @queryResults; \ +# # ALTER DATABASE {config['OPENEDX_MYSQL_DATABASE']} CHARACTER SET {charset} COLLATE {collation}; \ +# # SET FOREIGN_KEY_CHECKS=0; \ +# # EXECUTE stmnt; \ +# # SET FOREIGN_KEY_CHECKS=1; \ +# # DEALLOCATE PREPARE stmnt;" + +# # f"tee change_charset_collation.sql; \ +# # SELECT CONCAT(\"ALTER TABLE \", TABLE_NAME,\" CONVERT TO CHARACTER SET {charset} COLLATE {collation};\") \ +# # FROM INFORMATION_SCHEMA.TABLES \ +# # WHERE TABLE_SCHEMA=\"{config['OPENEDX_MYSQL_DATABASE']}\" AND \ +# # TABLE_TYPE=\"BASE TABLE\"; \ +# # notee; \ +# # ALTER DATABASE {config['OPENEDX_MYSQL_DATABASE']} CHARACTER SET {charset} COLLATE {collation}; \ +# # SET FOREIGN_KEY_CHECKS=0; \ +# # source change_charset_collation.sql \ +# # SET FOREIGN_KEY_CHECKS=1;" +# ], +# ) +# # fmt.echo_info(f"The tables to be upgraded are:\n {queries_to_run}") +# # context.invoke( +# # compose.execute, +# # args=[ +# # "mysql", +# # "bash", +# # "mysql", +# # f"--user={config['MYSQL_ROOT_USERNAME']}", +# # f"--password={config['MYSQL_ROOT_PASSWORD']}", +# # f"--database={config['OPENEDX_MYSQL_DATABASE']}", +# # f"--execute=\"ALTER DATABASE {config['OPENEDX_MYSQL_DATABASE']} CHARACTER SET {charset} COLLATE {collation};SET FOREIGN_KEY_CHECKS=0;{queries_to_run}SET FOREIGN_KEY_CHECKS=1;\"" +# # ] +# # ) +# context.invoke(compose.stop) + +# f"""DROP PROCEDURE IF EXISTS UpdateTable; +# DELIMITER $$ + +# CREATE PROCEDURE UpdateTable() +# BEGIN + +# DECLARE done INT DEFAULT FALSE; +# DECLARE _table_name VARCHAR(255); +# DECLARE cur CURSOR FOR +# SELECT table_name FROM information_schema.tables +# WHERE table_schema = "{config['OPENEDX_MYSQL_DATABASE']}" AND table_type = "BASE TABLE"; +# DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; + +# OPEN cur; +# My_loop: LOOP +# FETCH cur INTO _table_name; +# SET @my_table_name = _table_name; + +# IF done THEN +# LEAVE My_loop; +# END IF; + +# SET FOREIGN_KEY_CHECKS = 0; + +# SET @stmt = CONCAT('ALTER TABLE ', _table_name, ' CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'); +# PREPARE stmt1 FROM @stmt; +# EXECUTE stmt1; +# DEALLOCATE PREPARE stmt1; + +# SET FOREIGN_KEY_CHECKS = 1; + +# END LOOP; +# CLOSE cur; + +# END$$ + +# DELIMITER ; +# CALL UpdateTable(); +# """ + +# upgrade_mysql_charset = ask_user() +# if upgrade_mysql_charset: +# upgrade_all_tables = ask_user() +# if upgrade_all_tables: +# upgrade_mysql_charset_collation(all_tables) +# else: +# tables_to_upgrade = ask_user() +# upgrade_mysql_charset_collation(tables_to_upgrade) + \ No newline at end of file diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index 19e81243cc..da7f950f0a 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -196,3 +196,4 @@ def upgrade_mongodb( tutor config save --unset DOCKER_IMAGE_MONGODB """ fmt.echo_info(message) + diff --git a/tutor/env.py b/tutor/env.py index 470f9dc5a3..ab912f99a7 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -468,6 +468,7 @@ def get_release(version: str) -> str: "15": "olive", "16": "palm", "17": "quince", + "18": "redwood", }[version.split(".", maxsplit=1)[0]] diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 04fe11301b..caaaca1764 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -45,6 +45,7 @@ services: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --binlog-expire-logs-seconds=259200 + --mysql-native-password=ON restart: unless-stopped user: "999:999" volumes: