fix: add option to only update the authentication plugin of select users

changelog.d/20240718_171945_danyal.faheem_mysql_authentication_plugin_change.md

@@ -1,4 +1 @@
-- [Bugfix] Manually update the authentication plugin of MySQL users from mysql_native_password to caching_sha2_password when upgrading from quince. (by @Danyal-Faheem)
-   This process should be automatic for most users. However, if you are running a third-party MySQL (i.e., RUN_MYSQL=false), you are expected to perform this process yourself. Please refer to the third-party provider's documentation for detailed instructions. Ensuring that your MySQL version is up-to-date is crucial for maintaining compatibility and security.
-- [Bugfix] Do not directly upgrade MySQL from v5.7 to v8.4 when upgrading from quince as MySQL does not allow that. First, upgrade to v8.1 and then to v8.4. (by @Danyal-Faheem)
-  This process should be automatic for most users. However, if you are running a third-party MySQL (i.e., RUN_MYSQL=false), you are expected to perform this process yourself. Please refer to the third-party provider's documentation for detailed instructions. Ensuring that your MySQL version is up-to-date is crucial for maintaining compatibility and security. 
+- [Improvement] Add a do command to update the authentication plugin of existing MySQL users from mysql_native_password to caching_sha2_password for compatibility with MySQL v8.4.0 and above. (by @Danyal-Faheem)

docs/local.rst

@@ -141,6 +141,26 @@ The default Open edX theme is rather bland, so Tutor makes it easy to switch to
 
 Out of the box, only the default "open-edx" theme is available. We also developed `Indigo, a beautiful, customizable theme <https://github.com/overhangio/indigo>`__ which is easy to install with Tutor.
 
+.. _update_mysql_authentication_plugin:
+
+Updating the authentication plugin of MySQL users
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As of MySQL v8.4.0, the ``mysql_native_password`` authentication plugin has been deprecated. Users created with this authentication plugin should ideally be updated to use the latest ``caching_sha2_password`` authentication plugin.
+
+Tutor makes it easy do so with this handy command::
+
+    tutor local do update_mysql_authentication_plugin
+
+If you only want to update the authentication plugin of specific users, you can use the ``--users`` option. This option takes comma seperated names of users to upgrade::
+
+    tutor local do update_mysql_authentication_plugin --users=discovery,ecommerce
+
+Do note that if you are updating a specific user, there should be corresponding entries in the configuration for the mysql username and password for that user. For example, if you are trying to update the user ``myuser``, the following case sensitive entries need to be present in the configuration::
+
+    MYUSER_MYSQL_USERNAME
+    MYUSER_MYSQL_PASSWORD
+
 Running arbitrary ``manage.py`` commands
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/troubleshooting.rst

@@ -216,3 +216,10 @@ NPM Dependency Conflict When overriding ``@edx/frontend-component-header`` or ``
 ----------------------------------------------------------------------------------------------------------------
 
 The detailed steps are mentioned in `tutor-mfe <https://github.com/overhangio/tutor-mfe?tab=readme-ov-file#npm-dependency-conflict-when-overriding-edxfrontend-component-header-or-edxfrontend-component-footer>`__ documentation.
+
+"Plugin 'mysql_native_password' is not loaded"
+----------------------------------------------
+
+This issue can occur when Tutor is upgraded from v15 (Olive) or earlier to v18 (Redwood) because the users created in Tutor v15 utilize the mysql_native_password authentication plugin by default. This plugin has been deprecated as of MySQL v8.4.0 which is the default MySQL server used in Tutor v18.
+
+The handy :ref:`update_mysql_authentication_plugin <update_mysql_authentication_plugin>` do command in tutor can be used to fix this issue.

tests/commands/test_jobs.py

@@ -90,3 +90,24 @@ def test_set_theme(self) -> None:
             self.assertIn("lms-job", dc_args)
             self.assertIn("assign_theme('beautiful', 'domain1')", dc_args[-1])
             self.assertIn("assign_theme('beautiful', 'domain2')", dc_args[-1])
+
+    def test_update_mysql_authentication_plugin(self) -> None:
+        with temporary_root() as root:
+            self.invoke_in_root(root, ["config", "save"])
+            with patch("tutor.utils.docker_compose") as mock_docker_compose:
+                result = self.invoke_in_root(
+                    root,
+                    [
+                        "local",
+                        "do",
+                        "update-mysql-authentication-plugin",
+                    ],
+                )
+                dc_args, _dc_kwargs = mock_docker_compose.call_args
+
+            self.assertIsNone(result.exception)
+            self.assertEqual(0, result.exit_code)
+            self.assertIn("lms-job", dc_args)
+            self.assertIn("caching_sha2_password", dc_args[-1])
+            self.assertIn("openedx", dc_args[-1])
+            self.assertIn("root", dc_args[-1])

tutor/commands/jobs.py

@@ -12,10 +12,11 @@
 from typing_extensions import ParamSpec
 
 from tutor import config as tutor_config
-from tutor import env, fmt, hooks
+from tutor import env, fmt, hooks, plugins
 from tutor.hooks import priorities
 from tutor.utils import get_mysql_change_authentication_plugin_query
 
+
 class DoGroup(click.Group):
     """
     A Click group that prints subcommands under 'Jobs' instead of 'Commands' when we run
@@ -314,24 +315,41 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]:
         command += " " + shlex.join(args)  # pylint: disable=protected-access
     yield ("lms", command)
 
+
 @click.command(context_settings={"ignore_unknown_options": True})
-def update_mysql_authentication_plugin(
-) -> t.Iterable[tuple[str, str]]:
+@click.option(
+    "--users",
+    is_flag=False,
+    nargs=1,
+    help="Specific users to upgrade the authentication plugin of. Requires comma-seperated values with no space in-between.",
+)
+def update_mysql_authentication_plugin(users: str) -> t.Iterable[tuple[str, str]]:
     """
     Update the authentication plugin of MySQL users from mysql_native_password to caching_sha2_password
     Handy command used when upgrading to v8.4 of MySQL which deprecates mysql_native_password
     """
-    
+
     context = click.get_current_context().obj
     config = tutor_config.load(context.root)
-
-    query = get_mysql_change_authentication_plugin_query(config)
-
+
+    if not config["RUN_MYSQL"]:
+        fmt.echo_info(
+            f"You are not running MySQL (RUN_MYSQL=False). It is your "
+            f"responsibility to update the authentication plugin of mysql users."
+        )
+        return
+
+    users_to_update = users.split(",") if users else list(plugins.iter_loaded())
+
+    query = get_mysql_change_authentication_plugin_query(
+        config, users_to_update, not users
+    )
+
     mysql_command = (
-        "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }}"
-        + shlex.join([f"--database={{ OPENEDX_MYSQL_DATABASE }}", "-e", query])
+        "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --database={{ OPENEDX_MYSQL_DATABASE }} "
+        + shlex.join(["-e", query])
     )
-    
+
     yield ("lms", mysql_command)
 
 

tutor/commands/upgrade/compose.py

@@ -174,44 +174,6 @@ def upgrade_from_quince(context: click.Context, config: Config) -> None:
         )
         return
 
-    # Revert the MySQL image first to build data dictionary on v8.1
-    old_mysql_docker_image = "docker.io/mysql:8.1.0"
-    new_mysql_docker_image = str(config["DOCKER_IMAGE_MYSQL"])
-    click.echo(fmt.title(f"Upgrading MySQL to v{new_mysql_docker_image.split(':')[1]}"))
-    config["DOCKER_IMAGE_MYSQL"] = old_mysql_docker_image
-    # Note that the DOCKER_IMAGE_MYSQL value is never saved, because we only save the
-    # environment, not the configuration.
-    tutor_env.save(context.obj.root, config)
-    context.invoke(compose.start, detach=True, services=["mysql"])
-    fmt.echo_info("Waiting for MySQL to boot...")
-    sleep(30)
-
-    query = common_upgrade.get_mysql_change_authentication_plugin_query(config)
-
-    # Update the authentication plugin in v8.1 as it is disabled in v8.4
-    context.invoke(
-        compose.execute,
-        args=[
-            "mysql",
-            "bash",
-            "-e",
-            "-c",
-            f"mysql --user={config['MYSQL_ROOT_USERNAME']} --password='{config['MYSQL_ROOT_PASSWORD']}' --host={config['MYSQL_HOST']} --port={config['MYSQL_PORT']} --skip-column-names --silent "
-            + shlex.join(
-                [f"--database={config['OPENEDX_MYSQL_DATABASE']}", "-e", query]
-            ),
-        ],
-    )
-
-    # Upgrade back to v8.4
-    config["DOCKER_IMAGE_MYSQL"] = new_mysql_docker_image
-    # Note that the DOCKER_IMAGE_MYSQL value is never saved, because we only save the
-    # environment, not the configuration.
-    tutor_env.save(context.obj.root, config)
-    context.invoke(compose.start, detach=True, services=["mysql"])
-    fmt.echo_info("Waiting for MySQL to boot...")
-    sleep(30)
-
 
 def upgrade_mongodb(
     context: click.Context,

tutor/templates/k8s/deployments.yml

@@ -397,6 +397,7 @@ spec:
             - "--character-set-server=utf8mb4"
             - "--collation-server=utf8mb4_unicode_ci"
             - "--binlog-expire-logs-seconds=259200"
+            {% if DOCKER_IMAGE_MYSQL >= "docker.io/mysql:8.4.0" -%}- "--mysql-native-password=ON"{%- endif %}
           env:
             - name: MYSQL_ROOT_PASSWORD
               value: "{{ MYSQL_ROOT_PASSWORD }}"

tutor/templates/local/docker-compose.yml

@@ -44,6 +44,7 @@ services:
       --character-set-server=utf8mb4
       --collation-server=utf8mb4_unicode_ci
       --binlog-expire-logs-seconds=259200
+      {% if DOCKER_IMAGE_MYSQL >= "docker.io/mysql:8.4.0" -%}--mysql-native-password=ON{%- endif %}
     restart: unless-stopped
     user: "999:999"
     volumes:

tutor/utils.py

@@ -19,7 +19,7 @@
 from Crypto.PublicKey import RSA
 from Crypto.PublicKey.RSA import RsaKey
 
-from . import exceptions, fmt, plugins
+from . import exceptions, fmt
 from tutor.types import Config, ConfigValue
 
 
@@ -369,7 +369,9 @@ def format_table(rows: List[Tuple[str, ...]], separator: str = "\t") -> str:
     return formatted
 
 
-def get_mysql_change_authentication_plugin_query(config: Config) -> str:
+def get_mysql_change_authentication_plugin_query(
+    config: Config, users: List[str], all_users: bool
+) -> str:
     """
     Helper function to generate queries to upgrade the authentication plugin of MySQL users
     By default, only the ROOT and OPENEDX users are upgraded
@@ -378,32 +380,43 @@ def get_mysql_change_authentication_plugin_query(config: Config) -> str:
         <plugin>_MYSQL_PASSWORD
     These users are also upgraded
     """
-    loaded_plugins = list(plugins.iter_loaded())
+
+    host = "%"
+    query = ""
 
     def generate_mysql_authentication_plugin_update_query(
         username: ConfigValue, password: ConfigValue, host: str
     ) -> str:
         return f"ALTER USER '{username}'@'{host}' IDENTIFIED with caching_sha2_password BY '{password}';"
 
-    host = "%"
-    query = ""
+    def generate_user_queries(users: List[str]) -> str:
+        query = ""
+        for user in users:
+            user_uppercase = user.upper()
+            if not (
+                f"{user_uppercase}_MYSQL_USERNAME" in config
+                and f"{user_uppercase}_MYSQL_PASSWORD" in config
+            ):
+                raise exceptions.TutorError(
+                    f"Username or Password for User {user} not found in config. "
+                    f"Please make sure that the following entries are present in the configuration:\n"
+                    f"{user_uppercase}_MYSQL_USERNAME\n{user_uppercase}_MYSQL_PASSWORD"
+                )
+            query += generate_mysql_authentication_plugin_update_query(
+                config[f"{user_uppercase}_MYSQL_USERNAME"],
+                config[f"{user_uppercase}_MYSQL_PASSWORD"],
+                host,
+            )
+        return query
+
+    if not all_users:
+        return generate_user_queries(users)
+
     query += generate_mysql_authentication_plugin_update_query(
         config["MYSQL_ROOT_USERNAME"], config["MYSQL_ROOT_PASSWORD"], host
     )
     query += generate_mysql_authentication_plugin_update_query(
         config["OPENEDX_MYSQL_USERNAME"], config["OPENEDX_MYSQL_PASSWORD"], host
     )
 
-    for plugin in loaded_plugins:
-        plugin_uppercase = plugin.upper()
-        if (
-            f"{plugin_uppercase}_MYSQL_USERNAME" in config
-            and f"{plugin_uppercase}_MYSQL_PASSWORD" in config
-        ):
-            query += generate_mysql_authentication_plugin_update_query(
-                config[f"{plugin_uppercase}_MYSQL_USERNAME"],
-                config[f"{plugin_uppercase}_MYSQL_PASSWORD"],
-                host,
-            )
-
-    return query
+    return query + generate_user_queries(users)