feat: add command to change the charset of all tables in mysql

tutor/commands/compose.py

@@ -224,6 +224,129 @@ def upgrade(context: click.Context, from_release: t.Optional[str]) -> None:
     context.invoke(config_save_command)
 
 
+def interactive_mysql_upgrade(context: click.Context) -> None:
+    click.echo(fmt.title("Interactive mysql charset upgrade"))
+    config = tutor_config.load(context.obj.root)
+    upgrade_all_tables = click.confirm(
+        fmt.question(
+            "Would you like to upgrade all tables?"
+            "Some tables can take a very long time and cause downtime!"
+        ),
+    )
+    if upgrade_all_tables:
+        #TODO: upgrade_mysql(all_tables)
+        pass
+    else: 
+        pass
+
+@click.command(
+    short_help="Upgrade mysql to a specific charset and collation",
+    help=(
+        "Upgrade mysql to a specific charset and collation. You can either upgrade all tables, specify only certain tables to upgrade or specify certain tables to exclude from the upgrade process"
+    ),
+    context_settings={"ignore_unknown_options": True},
+)
+@click.option("--all-tables", help="Upgrade all tables in the openedx database", prompt="Are you sure you want to upgrade all the tables?")
+@click.option("-s", "--status", type=click.Choice(['include', 'exclude'], case_sensitive=False), help="Upgrade all tables in the openedx database")
+@click.option("--charset", required=True, type=str, help="The charset to updgrade the tables to")
+@click.option("--collation", required=True, type=str, help="The collation to updgrade the tables to")
+# @click.argument("apps", nargs=-1)
+@click.argument("tables", nargs=-1)
+@click.pass_context
+def upgrade_mysql(
+    context: BaseComposeContext,
+    all_tables: bool,
+    status: str,
+    charset: str,
+    collation: str,
+    # apps: list[str],
+    tables: list[str],
+) -> None:
+    config = tutor_config.load_full(context.obj.root)
+    if all_tables:
+        upgrade_mysql_charset_collation(context, config, charset, collation)
+
+
+
+def upgrade_mysql_charset_collation(
+    context: click.Context,
+    config: Config,
+    charset: str,
+    collation: str,
+) -> None:
+    if not config["RUN_MYSQL"]:
+        fmt.echo_info(
+            f"You are not running MySQL (RUN_MYSQL=false). It is your "
+            f"responsibility to upgrade your MySQL instance to {charset} charset and {collation} collation."
+        )
+        return
+    click.echo(fmt.title(f"Upgrading charset and collation of all tables in MySQL to {charset} and {collation} respectively."))
+    context.invoke(start, detach=True, services=["mysql"])
+    fmt.echo_info("Waiting for mysql to boot...")
+    # sleep(10)
+    context.invoke(
+        execute,
+        args=[
+            "mysql",
+            "mysql",
+            f"--user={config['MYSQL_ROOT_USERNAME']}",
+            f"--password={config['MYSQL_ROOT_PASSWORD']}",
+            f"--host={config['MYSQL_HOST']}",
+            f"--port={config['MYSQL_PORT']}",
+            # f"--default-character-set=utf8mb4",
+            f"--database={config['OPENEDX_MYSQL_DATABASE']}",
+            # "--skip-column-names",
+            # "--silent",
+            "-e",
+                f"""DROP PROCEDURE IF EXISTS UpdateTable;
+    DELIMITER $$
+
+CREATE PROCEDURE UpdateTable()
+BEGIN
+
+    DECLARE done INT DEFAULT FALSE;
+    DECLARE _table_name VARCHAR(255);
+    DECLARE cur CURSOR FOR
+            SELECT table_name FROM information_schema.tables
+            WHERE table_schema = '{config['OPENEDX_MYSQL_DATABASE']}' AND table_type = 'BASE TABLE';
+    DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+
+    OPEN cur;
+    My_loop: LOOP
+        FETCH cur INTO _table_name;
+        SET @my_table_name = _table_name;
+
+        IF done THEN
+          LEAVE My_loop;
+        END IF;
+
+        SET FOREIGN_KEY_CHECKS = 0;
+
+        SET @stmt = CONCAT('ALTER TABLE ', _table_name, ' CONVERT TO CHARACTER SET {charset} COLLATE {collation};'); 
+        PREPARE stmt1 FROM @stmt; 
+        EXECUTE stmt1; 
+        DEALLOCATE PREPARE stmt1;   
+        
+        SET FOREIGN_KEY_CHECKS = 1;
+            
+    END LOOP;
+    CLOSE cur;
+
+END$$
+
+DELIMITER ;
+    CALL UpdateTable();
+    """,    
+            ],
+    )
+    context.invoke(stop)
+
+
+
+
+
+
+
 @click.command(
     short_help="Run all or a selection of services.",
     help="Run all or a selection of services. Docker images will be rebuilt where necessary.",
@@ -442,6 +565,7 @@ def add_commands(command_group: click.Group) -> None:
     command_group.add_command(execute)
     command_group.add_command(logs)
     command_group.add_command(status)
+    command_group.add_command(upgrade_mysql)
 
     @hooks.Actions.PLUGINS_LOADED.add()
     def _add_do_commands() -> None:

tutor/commands/upgrade/__init__.py

@@ -9,4 +9,5 @@
     "olive",
     "palm",
     "quince",
+    "redwood",
 ]

tutor/commands/upgrade/compose.py

@@ -48,6 +48,9 @@ def upgrade_from(context: click.Context, from_release: str) -> None:
 
     if running_release == "quince":
         upgrade_from_quince(context, config)
+
+    if running_release == "redwood":
+        upgrade_from_redwood(context, config)
 
 
 def upgrade_from_ironwood(context: click.Context, config: Config) -> None:
@@ -191,7 +194,7 @@ def upgrade_mongodb(
     tutor_env.save(context.obj.root, config)
     context.invoke(compose.start, detach=True, services=["mongodb"])
     fmt.echo_info("Waiting for mongodb to boot...")
-    sleep(10)
+    sleep(30)
     context.invoke(
         compose.execute,
         args=[
@@ -202,3 +205,168 @@ def upgrade_mongodb(
         ],
     )
     context.invoke(compose.stop)
+
+# def upgrade_mysql_charset_collation(
+#     context: click.Context,
+#     config: Config,
+#     charset: str,
+#     collation: str,
+# ) -> None:
+#     if not config["RUN_MYSQL"]:
+#         fmt.echo_info(
+#             f"You are not running MySQL (RUN_MYSQL=false). It is your "
+#             f"responsibility to upgrade your MySQL instance to {charset} charset and {collation} collation."
+#         )
+#         return
+#     click.echo(fmt.title(f"Upgrading charset and collation of all tables in MySQL to {charset} and {collation} respectively."))
+#     context.invoke(compose.start, detach=True, services=["mysql"])
+#     fmt.echo_info("Waiting for mysql to boot...")
+#     # sleep(10)
+#     queries_to_run = context.invoke(
+#         compose.execute,
+#         args=[
+#             "mysql",
+#             "mysql",
+#             f"--user={config['MYSQL_ROOT_USERNAME']}",
+#             f"--password={config['MYSQL_ROOT_PASSWORD']}",
+#             f"--host={config['MYSQL_HOST']}",
+#             f"--port={config['MYSQL_PORT']}",
+#             # f"--default-character-set=utf8mb4",
+#             f"--database={config['OPENEDX_MYSQL_DATABASE']}",
+#             # "--skip-column-names",
+#             # "--silent",
+#             "-e",
+#                 f"""DROP PROCEDURE IF EXISTS UpdateTable;
+#     DELIMITER $$
+
+# CREATE PROCEDURE UpdateTable
+# (
+#     database varchar(64)
+#     charset varchar (16)
+#     collation varchar(64)
+# )
+# BEGIN
+
+#     DECLARE done INT DEFAULT FALSE;
+#     DECLARE _table_name VARCHAR(255);
+#     DECLARE cur CURSOR FOR
+#             SELECT table_name FROM information_schema.tables
+#             WHERE table_schema = database AND table_type = "BASE TABLE";
+#     DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+
+#     OPEN cur;
+#     My_loop: LOOP
+#         FETCH cur INTO _table_name;
+#         SET @my_table_name = _table_name;
+
+#         IF done THEN
+#           LEAVE My_loop;
+#         END IF;
+
+#         SET FOREIGN_KEY_CHECKS = 0;
+
+#         SET @stmt = CONCAT('ALTER TABLE ', _table_name, ' CONVERT TO CHARACTER SET charset COLLATE collation;'); 
+#         PREPARE stmt1 FROM @stmt; 
+#         EXECUTE stmt1; 
+#         DEALLOCATE PREPARE stmt1;   
+
+#         SET FOREIGN_KEY_CHECKS = 1;
+
+#     END LOOP;
+#     CLOSE cur;
+
+# END$$
+
+# DELIMITER ;
+#     CALL UpdateTable();
+#     """,
+#             # f"SET @queryResults = '';SET @queries= 'SELECT CONCAT(\"ALTER TABLE \", TABLE_NAME,\" CONVERT TO CHARACTER SET {charset} COLLATE {collation};\") \
+#             # FROM INFORMATION_SCHEMA.TABLES \
+#             # WHERE TABLE_SCHEMA=\"{config['OPENEDX_MYSQL_DATABASE']}\" AND \
+#             # TABLE_TYPE=\"BASE TABLE\" into @queryResults;'; \
+#             # PREPARE stmnt from @queries; \
+#             # EXECUTE stmnt; \
+#             # DEALLOCATE PREPARE stmnt; \
+#             # PREPARE stmnt from @queryResults; \
+#             # ALTER DATABASE {config['OPENEDX_MYSQL_DATABASE']} CHARACTER SET {charset} COLLATE {collation}; \
+#             # SET FOREIGN_KEY_CHECKS=0; \
+#             # EXECUTE stmnt; \
+#             # SET FOREIGN_KEY_CHECKS=1; \
+#             # DEALLOCATE PREPARE stmnt;"
+
+#             # f"tee change_charset_collation.sql; \
+#             # SELECT CONCAT(\"ALTER TABLE \", TABLE_NAME,\" CONVERT TO CHARACTER SET {charset} COLLATE {collation};\") \
+#             # FROM INFORMATION_SCHEMA.TABLES \
+#             # WHERE TABLE_SCHEMA=\"{config['OPENEDX_MYSQL_DATABASE']}\" AND \
+#             # TABLE_TYPE=\"BASE TABLE\"; \
+#             # notee; \
+#             # ALTER DATABASE {config['OPENEDX_MYSQL_DATABASE']} CHARACTER SET {charset} COLLATE {collation}; \
+#             # SET FOREIGN_KEY_CHECKS=0; \
+#             # source change_charset_collation.sql \
+#             # SET FOREIGN_KEY_CHECKS=1;"
+#         ],
+#     )
+#     # fmt.echo_info(f"The tables to be upgraded are:\n {queries_to_run}")
+#     # context.invoke(
+#     #     compose.execute,
+#     #     args=[
+#     #         "mysql",
+#     #         "bash",
+#     #         "mysql",
+#     #         f"--user={config['MYSQL_ROOT_USERNAME']}",
+#     #         f"--password={config['MYSQL_ROOT_PASSWORD']}",
+#     #         f"--database={config['OPENEDX_MYSQL_DATABASE']}",
+#     #         f"--execute=\"ALTER DATABASE {config['OPENEDX_MYSQL_DATABASE']} CHARACTER SET {charset} COLLATE {collation};SET FOREIGN_KEY_CHECKS=0;{queries_to_run}SET FOREIGN_KEY_CHECKS=1;\""
+#     #     ]
+#     # )
+#     context.invoke(compose.stop)
+
+#     f"""DROP PROCEDURE IF EXISTS UpdateTable;
+#     DELIMITER $$
+
+# CREATE PROCEDURE UpdateTable()
+# BEGIN
+
+#     DECLARE done INT DEFAULT FALSE;
+#     DECLARE _table_name VARCHAR(255);
+#     DECLARE cur CURSOR FOR
+#             SELECT table_name FROM information_schema.tables
+#             WHERE table_schema = "{config['OPENEDX_MYSQL_DATABASE']}" AND table_type = "BASE TABLE";
+#     DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+
+#     OPEN cur;
+#     My_loop: LOOP
+#         FETCH cur INTO _table_name;
+#         SET @my_table_name = _table_name;
+
+#         IF done THEN
+#           LEAVE My_loop;
+#         END IF;
+
+#         SET FOREIGN_KEY_CHECKS = 0;
+
+#         SET @stmt = CONCAT('ALTER TABLE ', _table_name, ' CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'); 
+#         PREPARE stmt1 FROM @stmt; 
+#         EXECUTE stmt1; 
+#         DEALLOCATE PREPARE stmt1;   
+
+#         SET FOREIGN_KEY_CHECKS = 1;
+
+#     END LOOP;
+#     CLOSE cur;
+
+# END$$
+
+# DELIMITER ;
+#     CALL UpdateTable();
+#     """
+
+# upgrade_mysql_charset = ask_user()
+# if upgrade_mysql_charset:
+#     upgrade_all_tables = ask_user()
+#     if upgrade_all_tables:
+#         upgrade_mysql_charset_collation(all_tables)
+#     else:
+#         tables_to_upgrade = ask_user()
+#         upgrade_mysql_charset_collation(tables_to_upgrade)
+

tutor/commands/upgrade/k8s.py

@@ -196,3 +196,4 @@ def upgrade_mongodb(
     tutor config save --unset DOCKER_IMAGE_MONGODB
     """
     fmt.echo_info(message)
+

tutor/env.py

@@ -468,6 +468,7 @@ def get_release(version: str) -> str:
         "15": "olive",
         "16": "palm",
         "17": "quince",
+        "18": "redwood",
     }[version.split(".", maxsplit=1)[0]]
 
 

tutor/templates/local/docker-compose.yml

@@ -45,6 +45,7 @@ services:
       --character-set-server=utf8mb4
       --collation-server=utf8mb4_unicode_ci
       --binlog-expire-logs-seconds=259200
+      --mysql-native-password=ON
     restart: unless-stopped
     user: "999:999"
     volumes: