From 3882d9b65178d652cf8bef00a4bf4a12ddcc88d0 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Wed, 31 Jul 2024 00:07:45 -0500 Subject: [PATCH 01/25] feat: add picasso commands in simple way --- setup.py | 4 +- tutorpicasso/__about__.py | 2 +- tutorpicasso/commands/cli.py | 21 +++ .../commands/enable_private_packages.py | 64 +++++++++ tutorpicasso/commands/enable_themes.py | 39 +++++ tutorpicasso/commands/run_extra_commands.py | 136 ++++++++++++++++++ tutorpicasso/plugin.py | 12 +- 7 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 tutorpicasso/commands/cli.py create mode 100644 tutorpicasso/commands/enable_private_packages.py create mode 100644 tutorpicasso/commands/enable_themes.py create mode 100644 tutorpicasso/commands/run_extra_commands.py diff --git a/setup.py b/setup.py index 7cd3175..fc69309 100644 --- a/setup.py +++ b/setup.py @@ -41,10 +41,10 @@ def load_about(): packages=find_packages(exclude=["tests*"]), include_package_data=True, python_requires=">=3.8", - install_requires=["tutor>=18.0.0,<19.0.0"], + install_requires=["tutor"], extras_require={ "dev": [ - "tutor[dev]>=18.0.0,<19.0.0", + "tutor[dev]", ] }, entry_points={ diff --git a/tutorpicasso/__about__.py b/tutorpicasso/__about__.py index c6a8b8e..3dc1f76 100644 --- a/tutorpicasso/__about__.py +++ b/tutorpicasso/__about__.py @@ -1 +1 @@ -__version__ = "18.0.0" +__version__ = "0.1.0" diff --git a/tutorpicasso/commands/cli.py b/tutorpicasso/commands/cli.py new file mode 100644 index 0000000..1a31f7d --- /dev/null +++ b/tutorpicasso/commands/cli.py @@ -0,0 +1,21 @@ +""" +Picasso commands group. +""" + +import click +from tutorpicasso.commands.enable_private_packages import enable_private_packages +from tutorpicasso.commands.enable_themes import enable_themes +from tutorpicasso.commands.run_extra_commands import run_extra_commands + +@click.group(help="Run picasso commands") +def picasso() -> None: + """ + Main picasso command group. + + This command group provides functionality to run picasso commands. + """ + + +picasso.add_command(enable_private_packages) +picasso.add_command(run_extra_commands) +picasso.add_command(enable_themes) diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py new file mode 100644 index 0000000..92ec547 --- /dev/null +++ b/tutorpicasso/commands/enable_private_packages.py @@ -0,0 +1,64 @@ +import os +import click +import subprocess +from tutor import config as tutor_config + + + +@click.command(name="enable-private-packages", help="Enable picasso private packages") +def enable_private_packages(): + """ + Enable private packages command. + + This command enables picasso private packages by cloning the packages and + defining them as private. + + Raises: + Exception: If an error occurs during the cloning or defining process. + """ + tutor_root = subprocess.check_output("tutor config printroot", shell=True).\ + decode("utf-8").strip() + config = tutor_config.load(tutor_root) + packages = get_picasso_packages(config) + for package, info in packages.items(): + try: + if not {"name", "repo", "version"}.issubset(info): + raise KeyError(f"{package} is missing one of the required keys: 'name', 'repo', 'version'") + + if os.path.isdir(f'{tutor_root}/{info["name"]}'): + subprocess.call( + [ + "rm", "-rf", f'{tutor_root}/{info["name"]}' + ] + ) + + subprocess.call( + [ + "git", "clone", "-b", + info["version"], info["repo"] + ], + cwd=tutor_root + ) + subprocess.call( + [ + "tutor", "mounts", "add", f'{tutor_root}/{info["name"]}' + ] + ) + + except KeyError as e: + raise click.ClickException(str(e)) + +def get_picasso_packages(settings) -> dict: + """ + Get the distribution packages from the provided settings. + + Args: + settings (dict): The tutor configuration settings. + + Returns: + dict: A dictionary of distribution packages, where the keys are package names + and the values are package details. + """ + picasso_packages = {key: val for key, + val in settings.items() if key.endswith("_DPKG") and val != 'None'} + return picasso_packages diff --git a/tutorpicasso/commands/enable_themes.py b/tutorpicasso/commands/enable_themes.py new file mode 100644 index 0000000..02a360a --- /dev/null +++ b/tutorpicasso/commands/enable_themes.py @@ -0,0 +1,39 @@ + +import click +import subprocess +import os +from tutor import config as tutor_config + +@click.command(name="enable-themes", help="Enable picasso themes") +def enable_themes() -> None: + """ + Enable picasso themes. + + This function enables the themes specified in the `PICASSO_THEMES` configuration + and applies them using the ThemeEnabler and ThemeGitRepository classes. + """ + tutor_root = subprocess.check_output("tutor config printroot", shell=True)\ + .decode("utf-8").strip() + config = tutor_config.load(tutor_root) + + if config.get("PICASSO_THEMES"): + for theme in config["PICASSO_THEMES"]: + try: + if not {"name", "repo", "version"}.issubset(theme.keys()): + raise KeyError( + f"{theme} is missing one or more required keys: " + "'name', 'repo', 'version'" + ) + + theme_path = f'{tutor_root}/env/build/openedx/themes/{theme["name"]}' + if os.path.isdir(theme_path): + subprocess.call(["rm", "-rf", theme_path]) + + subprocess.call( + [ + "git", "clone", "-b", theme["version"], theme["repo"], + theme_path + ], + ) + except KeyError as e: + raise click.ClickException(f"Error: {str(e)}") diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py new file mode 100644 index 0000000..de415b1 --- /dev/null +++ b/tutorpicasso/commands/run_extra_commands.py @@ -0,0 +1,136 @@ + +import click +import subprocess +from tutor import config as tutor_config +import re +# Was necessary to use this for compatibility with Python 3.8 +from typing import List + +COMMAND_CHAINING_OPERATORS = ["&&", "&", "||", "|", ";"] + +@click.command(name="run-extra-commands", help="Run tutor commands") +def run_extra_commands(): + """ + This command runs tutor commands defined in PICASSO_EXTRA_COMMANDS + """ + tutor_root = subprocess.check_output("tutor config printroot", shell=True).\ + decode("utf-8").strip() + config = tutor_config.load(tutor_root) + picasso_extra_commands = config.get("PICASSO_EXTRA_COMMANDS", None) + if picasso_extra_commands is not None: + validate_commands(picasso_extra_commands) + for command in picasso_extra_commands: + run_command(command) + + +def validate_commands(commands: List[str]): + """ + Takes all the extra commands sent through config.yml and verifies that + all the commands are correct before executing them + + Args: + commands (list[str] | None): The commands sent through DISTRO_EXTRA_COMMANDS in config.yml + """ + splitted_commands = [ + split_string(command, COMMAND_CHAINING_OPERATORS) for command in commands + ] + flat_commands_array = sum(splitted_commands, []) + + invalid_commands = [] + misspelled_commands = [] + for command in flat_commands_array: + if "tutor" not in command.lower(): + if find_tutor_misspelled(command): + misspelled_commands.append(command) + else: + invalid_commands.append(command) + + if invalid_commands or misspelled_commands: + error_message = ( + f"Found some issues with the commands:\n\n" + f"{'=> Invalid commands: ' if invalid_commands else ''}" + f"{', '.join(invalid_commands) if invalid_commands else ''}\n" + f"{'=> Misspelled commands: ' if misspelled_commands else ''}" + f"{', '.join(misspelled_commands) if misspelled_commands else ''}\n" + f"Take a look at the official Tutor commands: " + f"https://docs.tutor.edly.io/reference/cli/index.html" + ) + raise click.ClickException(error_message) + +def run_command(command: str): + """ + Run an extra command. + + This method runs the extra command provided. + + Args: + command (str): Tutor command. + """ + try: + with subprocess.Popen( + command, + shell=True, + executable="/bin/bash", + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as process: + + # It is sent a 'y' to say 'yes' on overriding the existing folders + stdout, stderr = process.communicate(input="y") + + if process.returncode != 0 or "error" in stderr.lower(): + raise subprocess.CalledProcessError( + process.returncode, command, output=stdout, stderr=stderr + ) + + # This print is left on purpose to show the command output + click.echo(stdout) + + except subprocess.CalledProcessError as error: + raise click.ClickException(error) + + +def find_tutor_misspelled(command: str): + """ + This function takes a command and looks if it has the word 'tutor' misspelled + + Args: + command (str): Command to be reviewed + + Return: + If its found the word 'tutor' misspelled is returned True + """ + return re.match(r"[tT](?:[oru]{3}|[oru]{2}[rR]|[oru]u?)", command) + + +def create_regex_from_array(arr: List[str]): + """ + This functions compiles a new regex turning taking care of + escaping special characters + + Args: + arr (list[str]): String that would be used to create a new regex + + Return: + A new compiled regex pattern that can be used for comparisons + """ + escaped_arr = [re.escape(item) for item in arr] + regex_pattern = "|".join(escaped_arr) + return re.compile(regex_pattern) + + +def split_string(string: str, split_by: List[str]): + """ + Takes a string that is wanted to be split according to some + other strings received in a list + + Args: + string (str): String that will be split + split_by (list[str]): Array of strings which will be used to split the string + + Return: + The string split into an array + """ + return re.split(create_regex_from_array(split_by), string) diff --git a/tutorpicasso/plugin.py b/tutorpicasso/plugin.py index 60be7db..9cd2c13 100644 --- a/tutorpicasso/plugin.py +++ b/tutorpicasso/plugin.py @@ -6,13 +6,18 @@ import click import importlib_resources from tutor import hooks +from tutorpicasso.commands.cli import picasso from .__about__ import __version__ ######################################## # CONFIGURATION ######################################## - +hooks.Filters.MOUNTED_DIRECTORIES.add_items( + [ + ("openedx", r"eox-.*"), + ] +) hooks.Filters.CONFIG_DEFAULTS.add_items( [ # Add your new settings that have default values here. @@ -205,6 +210,11 @@ # To define a command group for your plugin, you would define a Click # group and then add it to CLI_COMMANDS: +hooks.Filters.CLI_COMMANDS.add_items( + [ + picasso, + ] +) ### @click.group() ### def picasso() -> None: From 77e973a8f93ab7b31dd4fd17676d3e2d09a67ff5 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Thu, 1 Aug 2024 21:55:16 -0500 Subject: [PATCH 02/25] test: add workflows and fix tests --- .github/workflows/commitlint.yml | 11 ++++ .github/workflows/test.yml | 2 +- Makefile | 2 +- tutorpicasso/commands/cli.py | 2 + .../commands/enable_private_packages.py | 48 ++++++++--------- tutorpicasso/commands/enable_themes.py | 17 +++--- tutorpicasso/commands/run_extra_commands.py | 54 ++++++++++--------- tutorpicasso/plugin.py | 1 + 8 files changed, 79 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/commitlint.yml diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml new file mode 100644 index 0000000..3d98fd6 --- /dev/null +++ b/.github/workflows/commitlint.yml @@ -0,0 +1,11 @@ +name: Lint Commit Messages +on: [pull_request] + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: wagoid/commitlint-github-action@v6 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d17724b..31a8e0e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: Run tests on: pull_request: - branches: [master] + branches: [main] jobs: tests: diff --git a/Makefile b/Makefile index 797134d..0784f44 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ test-lint: ## Run code linting tests pylint --errors-only --enable=unused-import,unused-argument --ignore=templates --ignore=docs/_ext ${SRC_DIRS} test-types: ## Run type checks. - mypy --exclude=templates --ignore-missing-imports --implicit-reexport --strict ${SRC_DIRS} + mypy --exclude=templates --ignore-missing-imports --implicit-reexport ${SRC_DIRS} format: ## Format code automatically black $(BLACK_OPTS) diff --git a/tutorpicasso/commands/cli.py b/tutorpicasso/commands/cli.py index 1a31f7d..439e03e 100644 --- a/tutorpicasso/commands/cli.py +++ b/tutorpicasso/commands/cli.py @@ -3,10 +3,12 @@ """ import click + from tutorpicasso.commands.enable_private_packages import enable_private_packages from tutorpicasso.commands.enable_themes import enable_themes from tutorpicasso.commands.run_extra_commands import run_extra_commands + @click.group(help="Run picasso commands") def picasso() -> None: """ diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index 92ec547..aa36238 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -1,12 +1,15 @@ import os -import click import subprocess -from tutor import config as tutor_config +# Was necessary to use this for compatibility with Python 3.8 +from typing import Any, Dict + +import click +from tutor import config as tutor_config @click.command(name="enable-private-packages", help="Enable picasso private packages") -def enable_private_packages(): +def enable_private_packages() -> None: """ Enable private packages command. @@ -16,39 +19,33 @@ def enable_private_packages(): Raises: Exception: If an error occurs during the cloning or defining process. """ - tutor_root = subprocess.check_output("tutor config printroot", shell=True).\ - decode("utf-8").strip() + tutor_root = ( + subprocess.check_output("tutor config printroot", shell=True) + .decode("utf-8") + .strip() + ) config = tutor_config.load(tutor_root) packages = get_picasso_packages(config) for package, info in packages.items(): try: if not {"name", "repo", "version"}.issubset(info): - raise KeyError(f"{package} is missing one of the required keys: 'name', 'repo', 'version'") + raise KeyError( + f"{package} is missing one of the required keys: 'name', 'repo', 'version'" + ) if os.path.isdir(f'{tutor_root}/{info["name"]}'): - subprocess.call( - [ - "rm", "-rf", f'{tutor_root}/{info["name"]}' - ] - ) + subprocess.call(["rm", "-rf", f'{tutor_root}/{info["name"]}']) subprocess.call( - [ - "git", "clone", "-b", - info["version"], info["repo"] - ], - cwd=tutor_root - ) - subprocess.call( - [ - "tutor", "mounts", "add", f'{tutor_root}/{info["name"]}' - ] + ["git", "clone", "-b", info["version"], info["repo"]], cwd=tutor_root ) + subprocess.call(["tutor", "mounts", "add", f'{tutor_root}/{info["name"]}']) except KeyError as e: raise click.ClickException(str(e)) -def get_picasso_packages(settings) -> dict: + +def get_picasso_packages(settings: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: """ Get the distribution packages from the provided settings. @@ -59,6 +56,9 @@ def get_picasso_packages(settings) -> dict: dict: A dictionary of distribution packages, where the keys are package names and the values are package details. """ - picasso_packages = {key: val for key, - val in settings.items() if key.endswith("_DPKG") and val != 'None'} + picasso_packages = { + key: val + for key, val in settings.items() + if key.endswith("_DPKG") and val != "None" + } return picasso_packages diff --git a/tutorpicasso/commands/enable_themes.py b/tutorpicasso/commands/enable_themes.py index 02a360a..c92f13c 100644 --- a/tutorpicasso/commands/enable_themes.py +++ b/tutorpicasso/commands/enable_themes.py @@ -1,9 +1,10 @@ +import os +import subprocess import click -import subprocess -import os from tutor import config as tutor_config + @click.command(name="enable-themes", help="Enable picasso themes") def enable_themes() -> None: """ @@ -12,8 +13,11 @@ def enable_themes() -> None: This function enables the themes specified in the `PICASSO_THEMES` configuration and applies them using the ThemeEnabler and ThemeGitRepository classes. """ - tutor_root = subprocess.check_output("tutor config printroot", shell=True)\ - .decode("utf-8").strip() + tutor_root = ( + subprocess.check_output("tutor config printroot", shell=True) + .decode("utf-8") + .strip() + ) config = tutor_config.load(tutor_root) if config.get("PICASSO_THEMES"): @@ -30,10 +34,7 @@ def enable_themes() -> None: subprocess.call(["rm", "-rf", theme_path]) subprocess.call( - [ - "git", "clone", "-b", theme["version"], theme["repo"], - theme_path - ], + ["git", "clone", "-b", theme["version"], theme["repo"], theme_path], ) except KeyError as e: raise click.ClickException(f"Error: {str(e)}") diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index de415b1..33386f0 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -1,29 +1,34 @@ - -import click -import subprocess -from tutor import config as tutor_config import re +import subprocess + # Was necessary to use this for compatibility with Python 3.8 from typing import List +import click +from tutor import config as tutor_config + COMMAND_CHAINING_OPERATORS = ["&&", "&", "||", "|", ";"] + @click.command(name="run-extra-commands", help="Run tutor commands") -def run_extra_commands(): +def run_extra_commands() -> None: """ This command runs tutor commands defined in PICASSO_EXTRA_COMMANDS """ - tutor_root = subprocess.check_output("tutor config printroot", shell=True).\ - decode("utf-8").strip() + tutor_root = ( + subprocess.check_output("tutor config printroot", shell=True) + .decode("utf-8") + .strip() + ) config = tutor_config.load(tutor_root) picasso_extra_commands = config.get("PICASSO_EXTRA_COMMANDS", None) if picasso_extra_commands is not None: validate_commands(picasso_extra_commands) for command in picasso_extra_commands: - run_command(command) + run_command(command) -def validate_commands(commands: List[str]): +def validate_commands(commands: List[str]) -> None: """ Takes all the extra commands sent through config.yml and verifies that all the commands are correct before executing them @@ -34,7 +39,7 @@ def validate_commands(commands: List[str]): splitted_commands = [ split_string(command, COMMAND_CHAINING_OPERATORS) for command in commands ] - flat_commands_array = sum(splitted_commands, []) + flat_commands_array: List[str] = sum(splitted_commands, []) invalid_commands = [] misspelled_commands = [] @@ -47,17 +52,18 @@ def validate_commands(commands: List[str]): if invalid_commands or misspelled_commands: error_message = ( - f"Found some issues with the commands:\n\n" - f"{'=> Invalid commands: ' if invalid_commands else ''}" - f"{', '.join(invalid_commands) if invalid_commands else ''}\n" - f"{'=> Misspelled commands: ' if misspelled_commands else ''}" - f"{', '.join(misspelled_commands) if misspelled_commands else ''}\n" - f"Take a look at the official Tutor commands: " - f"https://docs.tutor.edly.io/reference/cli/index.html" + f"Found some issues with the commands:\n\n" + f"{'=> Invalid commands: ' if invalid_commands else ''}" + f"{', '.join(invalid_commands) if invalid_commands else ''}\n" + f"{'=> Misspelled commands: ' if misspelled_commands else ''}" + f"{', '.join(misspelled_commands) if misspelled_commands else ''}\n" + f"Take a look at the official Tutor commands: " + f"https://docs.tutor.edly.io/reference/cli/index.html" ) raise click.ClickException(error_message) -def run_command(command: str): + +def run_command(command: str) -> None: """ Run an extra command. @@ -75,7 +81,7 @@ def run_command(command: str): stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - ) as process: + ) as process: # It is sent a 'y' to say 'yes' on overriding the existing folders stdout, stderr = process.communicate(input="y") @@ -89,10 +95,10 @@ def run_command(command: str): click.echo(stdout) except subprocess.CalledProcessError as error: - raise click.ClickException(error) + raise click.ClickException(str(error)) -def find_tutor_misspelled(command: str): +def find_tutor_misspelled(command: str) -> bool: """ This function takes a command and looks if it has the word 'tutor' misspelled @@ -102,10 +108,10 @@ def find_tutor_misspelled(command: str): Return: If its found the word 'tutor' misspelled is returned True """ - return re.match(r"[tT](?:[oru]{3}|[oru]{2}[rR]|[oru]u?)", command) + return bool(re.match(r"[tT](?:[oru]{3}|[oru]{2}[rR]|[oru]u?)", command)) -def create_regex_from_array(arr: List[str]): +def create_regex_from_array(arr: List[str]) -> re.Pattern[str]: """ This functions compiles a new regex turning taking care of escaping special characters @@ -121,7 +127,7 @@ def create_regex_from_array(arr: List[str]): return re.compile(regex_pattern) -def split_string(string: str, split_by: List[str]): +def split_string(string: str, split_by: List[str]) -> List[str]: """ Takes a string that is wanted to be split according to some other strings received in a list diff --git a/tutorpicasso/plugin.py b/tutorpicasso/plugin.py index 9cd2c13..f50889a 100644 --- a/tutorpicasso/plugin.py +++ b/tutorpicasso/plugin.py @@ -6,6 +6,7 @@ import click import importlib_resources from tutor import hooks + from tutorpicasso.commands.cli import picasso from .__about__ import __version__ From dea0046c9b0c44f4a2e00050f2003202c01a0fd5 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Thu, 1 Aug 2024 23:11:21 -0500 Subject: [PATCH 03/25] docs: add information in the readme --- README.rst | 115 +++++++++++++++++++- tutorpicasso/commands/run_extra_commands.py | 2 +- 2 files changed, 113 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index db57883..1332174 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,16 @@ -picasso plugin for `Tutor `__ -######################################################### +Picasso `Tutor`_ Plugin +######################### -picasso plugin for Tutor +|Maintainance Badge| |Test Badge| + +.. |Maintainance Badge| image:: https://img.shields.io/badge/Status-Maintained-brightgreen + :alt: Maintainance Status +.. |Test Badge| image:: https://img.shields.io/github/actions/workflow/status/edunext/tutor-contrib-picasso/.github%2Fworkflows%2Ftests.yml?label=Test + :alt: GitHub Actions Workflow Test Status + +Picasso is a `Tutor`_ plugin that simplifies pre-build processes, such as preparing Docker images with private requirements, executing commands before the build, and adding themes. Everything is managed through commands, making it easy to integrate into automated environments. + +This plugin is based on https://github.com/eduNEXT/tutor-contrib-edunext-distro Installation @@ -16,10 +25,110 @@ Usage .. code-block:: bash + # To enable the plugin tutor plugins enable picasso + # Show help + tutor picasso -h + + # Enable themes + tutor picasso enable-themes + + # Enable private packages + tutor picasso enable-private-packages + + # Run Tutor commands + tutor picasso run-extra-commands + +.. note:: + + Remember to run these commands before build your images if they are needed. + + +Compatibility notes +******************* + +This plugin was tested from Palm release. + +About the commands +******************* + +Enable Private Packages +^^^^^^^^^^^^^^^^^^^^^^^^ + +This command allows the installation of private Open edX Django apps. It clones the private repository and, through the ``tutor mounts`` command, adds it to the Dockerfile for inclusion in the build process. The input it takes is: + +.. code-block:: yaml + PICASSO__DPKG: + name: + repo: + version: + +.. note:: + + It is needed to use the SSH URL to clone private packages. + +.. warning:: + + For the mount to work correctly and include the package in the Dockerfile, it must be added to a tutor filter ``MOUNTED_DIRECTORIES``. By default, Picasso adds ``eox-*`` packages. If you need to add another private package, don't forget to include this configuration in a Tutor plugin. + + .. code-block:: python + + hooks.Filters.MOUNTED_DIRECTORIES.add_items( + [ + ("openedx", ""), + ] + ) + + +.. note:: + + If you want to use public packages, we recommend using the ``OPEN_EDX_EXTRA_PIP_REQUIREMENTS`` variable in the ``config.yml`` of your Tutor environment. + + +Enable Themes +^^^^^^^^^^^^^^ + +This command clones your theme repository into the folder that Tutor uses for themes. Documentation available at `Installing custom theme`_ tutorial. The input it takes is: + +.. code-block:: yaml + PICASSO_THEMES: + - name: + repo: + version: + - name: + repo: + version: + +.. note:: + + If your theme repository is public, you can also use the HTTPS URL in ``repo``. + +.. note:: + + Don't forget to add extra configurations in a Tutor plugin if your theme requires it. + + +Run Extra Commands +^^^^^^^^^^^^^^^^^^^ + +This command allows you to run a list of Tutor commands. These commands are executed in bash and, for security reasons, are restricted to running only Tutor commands. The input it takes is: + +.. code-block:: yaml + PICASSO_EXTRA_COMMANDS: + - + - + - + - + . + . + . License ******* This software is licensed under the terms of the AGPLv3. + + +.. _Tutor: https://docs.tutor.edly.io +.. _Installing custom theme: https://docs.tutor.edly.io/tutorials/theming.html#theming \ No newline at end of file diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index 33386f0..0d2e75a 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -34,7 +34,7 @@ def validate_commands(commands: List[str]) -> None: all the commands are correct before executing them Args: - commands (list[str] | None): The commands sent through DISTRO_EXTRA_COMMANDS in config.yml + commands (list[str] | None): The commands sent through PICASSO_EXTRA_COMMANDS in config.yml """ splitted_commands = [ split_string(command, COMMAND_CHAINING_OPERATORS) for command in commands From c740aa4f2e677c468fd15c775716be505b0fc1a6 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Fri, 2 Aug 2024 12:48:17 -0500 Subject: [PATCH 04/25] test: fix tests --- Makefile | 2 +- tutorpicasso/commands/enable_themes.py | 3 ++- tutorpicasso/commands/run_extra_commands.py | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 0784f44..797134d 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ test-lint: ## Run code linting tests pylint --errors-only --enable=unused-import,unused-argument --ignore=templates --ignore=docs/_ext ${SRC_DIRS} test-types: ## Run type checks. - mypy --exclude=templates --ignore-missing-imports --implicit-reexport ${SRC_DIRS} + mypy --exclude=templates --ignore-missing-imports --implicit-reexport --strict ${SRC_DIRS} format: ## Format code automatically black $(BLACK_OPTS) diff --git a/tutorpicasso/commands/enable_themes.py b/tutorpicasso/commands/enable_themes.py index c92f13c..80b4d93 100644 --- a/tutorpicasso/commands/enable_themes.py +++ b/tutorpicasso/commands/enable_themes.py @@ -1,5 +1,6 @@ import os import subprocess +from typing import Any import click from tutor import config as tutor_config @@ -18,7 +19,7 @@ def enable_themes() -> None: .decode("utf-8") .strip() ) - config = tutor_config.load(tutor_root) + config: Any = tutor_config.load(tutor_root) if config.get("PICASSO_THEMES"): for theme in config["PICASSO_THEMES"]: diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index 0d2e75a..f02cca6 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -2,7 +2,7 @@ import subprocess # Was necessary to use this for compatibility with Python 3.8 -from typing import List +from typing import Any, List import click from tutor import config as tutor_config @@ -20,15 +20,15 @@ def run_extra_commands() -> None: .decode("utf-8") .strip() ) - config = tutor_config.load(tutor_root) - picasso_extra_commands = config.get("PICASSO_EXTRA_COMMANDS", None) + config: Any = tutor_config.load(tutor_root) + picasso_extra_commands: Any = config.get("PICASSO_EXTRA_COMMANDS", None) if picasso_extra_commands is not None: validate_commands(picasso_extra_commands) for command in picasso_extra_commands: run_command(command) -def validate_commands(commands: List[str]) -> None: +def validate_commands(commands: Any) -> None: """ Takes all the extra commands sent through config.yml and verifies that all the commands are correct before executing them From a4c769b856ec3ad584e9c67908aba754a6ee2407 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 5 Aug 2024 10:15:21 -0500 Subject: [PATCH 05/25] fix: add importlib_resources for palm support --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fc69309..4841745 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def load_about(): packages=find_packages(exclude=["tests*"]), include_package_data=True, python_requires=">=3.8", - install_requires=["tutor"], + install_requires=["tutor", "importlib_resources"], extras_require={ "dev": [ "tutor[dev]", From 9e035ecaa8f6c61a3062bc3635cfb11c41068dfc Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 5 Aug 2024 10:44:34 -0500 Subject: [PATCH 06/25] fix: add commands in picasso installation --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index e938daf..b9d8645 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include tutorpicasso/patches * recursive-include tutorpicasso/templates * +recursive-include tutorpicasso/commands * From f977bd31542787950a503a1c90d25646a707ee57 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Fri, 16 Aug 2024 11:20:35 -0500 Subject: [PATCH 07/25] feat: add backward compatibility --- setup.py | 2 +- .../commands/enable_private_packages.py | 36 +++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 4841745..677969d 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def load_about(): packages=find_packages(exclude=["tests*"]), include_package_data=True, python_requires=">=3.8", - install_requires=["tutor", "importlib_resources"], + install_requires=["tutor", "importlib_resources", "packaging"], extras_require={ "dev": [ "tutor[dev]", diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index aa36238..c1b1ea9 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -1,8 +1,8 @@ import os import subprocess -# Was necessary to use this for compatibility with Python 3.8 from typing import Any, Dict +from packaging.version import Version import click from tutor import config as tutor_config @@ -21,11 +21,28 @@ def enable_private_packages() -> None: """ tutor_root = ( subprocess.check_output("tutor config printroot", shell=True) - .decode("utf-8") - .strip() + .decode("utf-8").strip() ) + tutor_version = ( + subprocess.check_output("tutor --version", shell=True) + .decode("utf-8").strip() + ).split()[-1] + tutor_version_obj = Version(tutor_version) + # Define Quince version as the method for installing private packages changes from this version + quince_version_obj = Version('v17.0.0') + # Use these specific paths as required by Tutor < Quince + private_requirements_root = f'{tutor_root}/env/build/openedx/requirements' + private_requirements_txt = f'{private_requirements_root}/private.txt' config = tutor_config.load(tutor_root) packages = get_picasso_packages(config) + + # Create necessary files and directories if they don't exist + if not os.path.exists(private_requirements_root): + os.makedirs(private_requirements_root) + if not os.path.exists(private_requirements_txt) and tutor_version_obj < quince_version_obj: + with open(private_requirements_txt, 'w') as file: + file.write('') + for package, info in packages.items(): try: if not {"name", "repo", "version"}.issubset(info): @@ -33,13 +50,18 @@ def enable_private_packages() -> None: f"{package} is missing one of the required keys: 'name', 'repo', 'version'" ) - if os.path.isdir(f'{tutor_root}/{info["name"]}'): - subprocess.call(["rm", "-rf", f'{tutor_root}/{info["name"]}']) + if os.path.isdir(f'{private_requirements_root}/{info["name"]}'): + subprocess.call(["rm", "-rf", f'{private_requirements_root}/{info["name"]}']) subprocess.call( - ["git", "clone", "-b", info["version"], info["repo"]], cwd=tutor_root + ["git", "clone", "-b", info["version"], info["repo"]], cwd=private_requirements_root ) - subprocess.call(["tutor", "mounts", "add", f'{tutor_root}/{info["name"]}']) + + if tutor_version_obj < quince_version_obj: + echo_command = f'echo "-e ./{info["name"]}/" >> {private_requirements_txt}' + subprocess.call(echo_command, shell=True) + else: + subprocess.call(["tutor", "mounts", "add", f'{private_requirements_root}/{info["name"]}']) except KeyError as e: raise click.ClickException(str(e)) From 1a5d0b534e1d2091baaacf982bec85d845c7af41 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Fri, 16 Aug 2024 12:42:17 -0500 Subject: [PATCH 08/25] fix: fix tests --- .../commands/enable_private_packages.py | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index c1b1ea9..30aedaa 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -21,27 +21,30 @@ def enable_private_packages() -> None: """ tutor_root = ( subprocess.check_output("tutor config printroot", shell=True) - .decode("utf-8").strip() + .decode("utf-8") + .strip() ) tutor_version = ( - subprocess.check_output("tutor --version", shell=True) - .decode("utf-8").strip() + subprocess.check_output("tutor --version", shell=True).decode("utf-8").strip() ).split()[-1] tutor_version_obj = Version(tutor_version) # Define Quince version as the method for installing private packages changes from this version - quince_version_obj = Version('v17.0.0') + quince_version_obj = Version("v17.0.0") # Use these specific paths as required by Tutor < Quince - private_requirements_root = f'{tutor_root}/env/build/openedx/requirements' - private_requirements_txt = f'{private_requirements_root}/private.txt' + private_requirements_root = f"{tutor_root}/env/build/openedx/requirements" + private_requirements_txt = f"{private_requirements_root}/private.txt" config = tutor_config.load(tutor_root) packages = get_picasso_packages(config) # Create necessary files and directories if they don't exist if not os.path.exists(private_requirements_root): os.makedirs(private_requirements_root) - if not os.path.exists(private_requirements_txt) and tutor_version_obj < quince_version_obj: - with open(private_requirements_txt, 'w') as file: - file.write('') + if ( + not os.path.exists(private_requirements_txt) + and tutor_version_obj < quince_version_obj + ): + with open(private_requirements_txt, "w") as file: + file.write("") for package, info in packages.items(): try: @@ -51,17 +54,29 @@ def enable_private_packages() -> None: ) if os.path.isdir(f'{private_requirements_root}/{info["name"]}'): - subprocess.call(["rm", "-rf", f'{private_requirements_root}/{info["name"]}']) + subprocess.call( + ["rm", "-rf", f'{private_requirements_root}/{info["name"]}'] + ) subprocess.call( - ["git", "clone", "-b", info["version"], info["repo"]], cwd=private_requirements_root + ["git", "clone", "-b", info["version"], info["repo"]], + cwd=private_requirements_root, ) if tutor_version_obj < quince_version_obj: - echo_command = f'echo "-e ./{info["name"]}/" >> {private_requirements_txt}' + echo_command = ( + f'echo "-e ./{info["name"]}/" >> {private_requirements_txt}' + ) subprocess.call(echo_command, shell=True) else: - subprocess.call(["tutor", "mounts", "add", f'{private_requirements_root}/{info["name"]}']) + subprocess.call( + [ + "tutor", + "mounts", + "add", + f'{private_requirements_root}/{info["name"]}', + ] + ) except KeyError as e: raise click.ClickException(str(e)) From 1cd0557fcb391c993458a464822bd724df628399 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Fri, 16 Aug 2024 13:30:36 -0500 Subject: [PATCH 09/25] fix: add condition before MOUNTED_DIRECTORIES --- .../commands/enable_private_packages.py | 7 ++----- tutorpicasso/plugin.py | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index 30aedaa..a70fd9d 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -1,11 +1,11 @@ import os import subprocess - from typing import Any, Dict -from packaging.version import Version import click +from packaging.version import Version from tutor import config as tutor_config +from tutor.__about__ import __version__ as tutor_version @click.command(name="enable-private-packages", help="Enable picasso private packages") @@ -24,9 +24,6 @@ def enable_private_packages() -> None: .decode("utf-8") .strip() ) - tutor_version = ( - subprocess.check_output("tutor --version", shell=True).decode("utf-8").strip() - ).split()[-1] tutor_version_obj = Version(tutor_version) # Define Quince version as the method for installing private packages changes from this version quince_version_obj = Version("v17.0.0") diff --git a/tutorpicasso/plugin.py b/tutorpicasso/plugin.py index f50889a..2c06e61 100644 --- a/tutorpicasso/plugin.py +++ b/tutorpicasso/plugin.py @@ -5,7 +5,9 @@ import click import importlib_resources +from packaging.version import Version from tutor import hooks +from tutor.__about__ import __version__ as tutor_version from tutorpicasso.commands.cli import picasso @@ -14,11 +16,16 @@ ######################################## # CONFIGURATION ######################################## -hooks.Filters.MOUNTED_DIRECTORIES.add_items( - [ - ("openedx", r"eox-.*"), - ] -) + +# Tutor introduces the MOUNTED_DIRECTORIES in the latest Palm version. +latest_palm_version = "16.1.8" +if Version(tutor_version) > Version(latest_palm_version): + hooks.Filters.MOUNTED_DIRECTORIES.add_items( + [ + ("openedx", r"eox-.*"), + ] + ) + hooks.Filters.CONFIG_DEFAULTS.add_items( [ # Add your new settings that have default values here. From e48b36d74cea085a64c2e93d709eb614c5a946c0 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Fri, 16 Aug 2024 13:59:31 -0500 Subject: [PATCH 10/25] test: add integration tests --- .github/workflows/integration_test.yml | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/integration_test.yml diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml new file mode 100644 index 0000000..d1b59d9 --- /dev/null +++ b/.github/workflows/integration_test.yml @@ -0,0 +1,71 @@ +name: Integration Tests +on: [pull_request] + +jobs: + tutor-integration-test: + name: Integration with Tutor + strategy: + matrix: + # tutor_version: ["Redwood", "Quince", "Palm", "Olive"] + tutor_version: ["<19.0.0", "<18.0.0", "<17.0.0", "<16.0.0"] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Prepare Tutor & launch + run: | + pip install "tutor${{ matrix.tutor_version }}" + pip install -e . + TUTOR_ROOT="$(pwd)" tutor --version + TUTOR_ROOT="$(pwd)" tutor config save + + - name: Set up SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan github.com >> ~/.ssh/known_hosts + + - name: Enable the tutor plugin + run: | + TUTOR_ROOT="$(pwd)" tutor plugins enable picasso + TUTOR_ROOT="$(pwd)" tutor picasso -h + + - name: Adding data to the config.yml + run: | + echo "PICASSO_MANAGE_DPKG:" | tee -a ./config.yml + echo " name: eox-manage" | tee -a ./config.yml + echo " repo: git@github.com:eduNEXT/eox-manage.git" | tee -a ./config.yml + echo " version: v5.2.0" | tee -a ./config.yml + echo "PICASSO_THEMES:" | tee -a ./config.yml + echo "- name: endx-saas-themes" | tee -a ./config.yml + echo " repo: git@github.com:eduNEXT/ednx-saas-themes.git" | tee -a ./config.yml + echo " version: master" | tee -a ./config.yml + echo "PICASSO_EXTRA_COMMANDS:" | tee -a ./config.yml + echo "- tutor plugins update" | tee -a ./config.yml + echo "- tutor plugins index add https://raw.githubusercontent.com/eduNEXT/tutor-plugin-indexes/picasso_test/" | tee -a ./config.yml + echo "- tutor plugins install mfe mfe_extensions aspects" | tee -a ./config.yml + echo "- tutor plugins enable mfe mfe_extensions aspects" | tee -a ./config.yml + echo "- tutor config save" | tee -a ./config.yml + + - name: Check run-extra-commands + run: | + TUTOR_ROOT="$(pwd)" tutor picasso run-extra-commands + + - name: Check enable-themes + run: | + TUTOR_ROOT="$(pwd)" tutor picasso enable-themes + + - name: Check enable-private-packages + run: | + TUTOR_ROOT="$(pwd)" tutor picasso enable-private-packages + + if grep -q 'eox-manage' env/build/openedx/requirements/private.txt; then + echo "'eox-manage' found in env/build/openedx/requirements/private.txt." + elif grep -q 'eox-manage' env/build/openedx/Dockerfile; then + echo "'eox-manage' found in env/build/openedx/Dockerfile." + else + echo "'eox-manage' not found for the building process." + exit 1 + fi From 9ca43a9f45752f5b124257fc88162926b52b353f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Fernanda=20Magallanes?= <35668326+MaferMazu@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:06:15 -0500 Subject: [PATCH 11/25] fix: implement feedback improvements Co-authored-by: Maria Grimaldi --- README.rst | 12 +++++++++--- tutorpicasso/commands/enable_private_packages.py | 7 ++----- tutorpicasso/commands/enable_themes.py | 7 ++----- tutorpicasso/commands/run_extra_commands.py | 16 +++++++--------- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index 1332174..12f683b 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,13 @@ Picasso `Tutor`_ Plugin .. |Test Badge| image:: https://img.shields.io/github/actions/workflow/status/edunext/tutor-contrib-picasso/.github%2Fworkflows%2Ftests.yml?label=Test :alt: GitHub Actions Workflow Test Status -Picasso is a `Tutor`_ plugin that simplifies pre-build processes, such as preparing Docker images with private requirements, executing commands before the build, and adding themes. Everything is managed through commands, making it easy to integrate into automated environments. +Picasso is a `Tutor`_ plugin that streamlines and automates complex pre-build tasks into a cohesive command. + +Current features include: + +- Adding private requirements: install private packages or dependencies in edx-platform. +- Executing a bundle of commands: run multiple commands in a specific order with a single command. +- Adding themes: manage custom themes to personalize Open edX. This plugin is based on https://github.com/eduNEXT/tutor-contrib-edunext-distro @@ -42,7 +48,7 @@ Usage .. note:: - Remember to run these commands before build your images if they are needed. + Please remember to run these commands before you build your images. Compatibility notes @@ -50,7 +56,7 @@ Compatibility notes This plugin was tested from Palm release. -About the commands +Usage ******************* Enable Private Packages diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index a70fd9d..012edf9 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -19,11 +19,8 @@ def enable_private_packages() -> None: Raises: Exception: If an error occurs during the cloning or defining process. """ - tutor_root = ( - subprocess.check_output("tutor config printroot", shell=True) - .decode("utf-8") - .strip() - ) + context = click.get_current_context().obj + tutor_conf = tutor_config.load(context.root) tutor_version_obj = Version(tutor_version) # Define Quince version as the method for installing private packages changes from this version quince_version_obj = Version("v17.0.0") diff --git a/tutorpicasso/commands/enable_themes.py b/tutorpicasso/commands/enable_themes.py index 80b4d93..f426240 100644 --- a/tutorpicasso/commands/enable_themes.py +++ b/tutorpicasso/commands/enable_themes.py @@ -14,11 +14,8 @@ def enable_themes() -> None: This function enables the themes specified in the `PICASSO_THEMES` configuration and applies them using the ThemeEnabler and ThemeGitRepository classes. """ - tutor_root = ( - subprocess.check_output("tutor config printroot", shell=True) - .decode("utf-8") - .strip() - ) + context = click.get_current_context().obj + tutor_conf = tutor_config.load(context.root) config: Any = tutor_config.load(tutor_root) if config.get("PICASSO_THEMES"): diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index f02cca6..3469cbe 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -24,8 +24,7 @@ def run_extra_commands() -> None: picasso_extra_commands: Any = config.get("PICASSO_EXTRA_COMMANDS", None) if picasso_extra_commands is not None: validate_commands(picasso_extra_commands) - for command in picasso_extra_commands: - run_command(command) + list(map(run_command, picasso_extra_commands)) def validate_commands(commands: Any) -> None: @@ -113,7 +112,7 @@ def find_tutor_misspelled(command: str) -> bool: def create_regex_from_array(arr: List[str]) -> re.Pattern[str]: """ - This functions compiles a new regex turning taking care of + Compile a new regex and escape special characters in the given string. escaping special characters Args: @@ -122,21 +121,20 @@ def create_regex_from_array(arr: List[str]) -> re.Pattern[str]: Return: A new compiled regex pattern that can be used for comparisons """ - escaped_arr = [re.escape(item) for item in arr] + escaped_arr = list(map(re.escape, arr)) regex_pattern = "|".join(escaped_arr) return re.compile(regex_pattern) def split_string(string: str, split_by: List[str]) -> List[str]: """ - Takes a string that is wanted to be split according to some - other strings received in a list + Split strings based on given patterns. Args: - string (str): String that will be split - split_by (list[str]): Array of strings which will be used to split the string + string (str): string to be split + split_by (list[str]): patterns to be used to split the string Return: - The string split into an array + The string split into a list """ return re.split(create_regex_from_array(split_by), string) From d1110cb9a97121bbac081113420d6a163c258ea4 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Tue, 20 Aug 2024 15:30:28 -0500 Subject: [PATCH 12/25] fix: correct test --- Makefile | 2 +- tutorpicasso/commands/enable_private_packages.py | 6 +++--- tutorpicasso/commands/enable_themes.py | 8 ++++---- tutorpicasso/commands/run_extra_commands.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 797134d..d4b0762 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ SRC_DIRS = ./tutorpicasso BLACK_OPTS = --exclude templates ${SRC_DIRS} # Warning: These checks are not necessarily run on every PR. -test: test-lint test-types test-format # Run some static checks. +test: test-lint test-format # Run some static checks. test-format: ## Run code formatting tests black --check --diff $(BLACK_OPTS) diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index 012edf9..8ffdc68 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -20,15 +20,15 @@ def enable_private_packages() -> None: Exception: If an error occurs during the cloning or defining process. """ context = click.get_current_context().obj - tutor_conf = tutor_config.load(context.root) + tutor_root = context.root + tutor_conf = tutor_config.load(tutor_root) tutor_version_obj = Version(tutor_version) # Define Quince version as the method for installing private packages changes from this version quince_version_obj = Version("v17.0.0") # Use these specific paths as required by Tutor < Quince private_requirements_root = f"{tutor_root}/env/build/openedx/requirements" private_requirements_txt = f"{private_requirements_root}/private.txt" - config = tutor_config.load(tutor_root) - packages = get_picasso_packages(config) + packages = get_picasso_packages(tutor_conf) # Create necessary files and directories if they don't exist if not os.path.exists(private_requirements_root): diff --git a/tutorpicasso/commands/enable_themes.py b/tutorpicasso/commands/enable_themes.py index f426240..176608b 100644 --- a/tutorpicasso/commands/enable_themes.py +++ b/tutorpicasso/commands/enable_themes.py @@ -15,11 +15,11 @@ def enable_themes() -> None: and applies them using the ThemeEnabler and ThemeGitRepository classes. """ context = click.get_current_context().obj - tutor_conf = tutor_config.load(context.root) - config: Any = tutor_config.load(tutor_root) + tutor_root = context.root + tutor_conf = tutor_config.load(tutor_root) - if config.get("PICASSO_THEMES"): - for theme in config["PICASSO_THEMES"]: + if tutor_conf.get("PICASSO_THEMES"): + for theme in tutor_conf["PICASSO_THEMES"]: try: if not {"name", "repo", "version"}.issubset(theme.keys()): raise KeyError( diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index 3469cbe..1d48862 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -131,7 +131,7 @@ def split_string(string: str, split_by: List[str]) -> List[str]: Split strings based on given patterns. Args: - string (str): string to be split + string (str): string to be split split_by (list[str]): patterns to be used to split the string Return: From e55f4b9b6e3bd57368fa9caec4a0abd6daf6cd02 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Tue, 20 Aug 2024 15:53:58 -0500 Subject: [PATCH 13/25] docs: improve usage description --- README.rst | 73 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 12f683b..fdf6fdb 100644 --- a/README.rst +++ b/README.rst @@ -26,25 +26,17 @@ Installation pip install git+https://github.com/eduNEXT/tutor-contrib-picasso -Usage -***** +Enable the plugin +****************** .. code-block:: bash # To enable the plugin tutor plugins enable picasso - # Show help + # Show the picasso commands tutor picasso -h - # Enable themes - tutor picasso enable-themes - - # Enable private packages - tutor picasso enable-private-packages - - # Run Tutor commands - tutor picasso run-extra-commands .. note:: @@ -54,15 +46,19 @@ Usage Compatibility notes ******************* -This plugin was tested from Palm release. +This plugin was tested from Olive release. Usage -******************* +******* Enable Private Packages ^^^^^^^^^^^^^^^^^^^^^^^^ -This command allows the installation of private Open edX Django apps. It clones the private repository and, through the ``tutor mounts`` command, adds it to the Dockerfile for inclusion in the build process. The input it takes is: +To enable the installation of private Open edX Django apps, follow these steps: + +1. Add configuration to the configuration file + +First, add the necessary configuration in your Tutor environment's ``config.yml`` file: .. code-block:: yaml PICASSO__DPKG: @@ -74,6 +70,20 @@ This command allows the installation of private Open edX Django apps. It clones It is needed to use the SSH URL to clone private packages. +2. Save the configuration with ``tutor config save`` + +3. Run the enable command + +Next, run the following command to enable private packages: + +.. code-block:: bash + + # Enable private packages + tutor picasso enable-private-packages + + +This command allows the installation of private Open edX Django apps. It clones the private repository and, through the ``tutor mounts`` command, adds it to the Dockerfile for inclusion in the build process. + .. warning:: For the mount to work correctly and include the package in the Dockerfile, it must be added to a tutor filter ``MOUNTED_DIRECTORIES``. By default, Picasso adds ``eox-*`` packages. If you need to add another private package, don't forget to include this configuration in a Tutor plugin. @@ -95,7 +105,11 @@ This command allows the installation of private Open edX Django apps. It clones Enable Themes ^^^^^^^^^^^^^^ -This command clones your theme repository into the folder that Tutor uses for themes. Documentation available at `Installing custom theme`_ tutorial. The input it takes is: +To enable themes in your Tutor environment, follow these steps: + +1. Add configuration to the configuration file + +First, add the necessary configuration in your Tutor environment's ``config.yml`` file: .. code-block:: yaml PICASSO_THEMES: @@ -110,6 +124,17 @@ This command clones your theme repository into the folder that Tutor uses for th If your theme repository is public, you can also use the HTTPS URL in ``repo``. +2. Save the configuration with ``tutor config save`` + +3. Run the enable command + +.. code-block:: bash + + # Enable themes + tutor picasso enable-themes + +This command clones your theme repository into the folder that Tutor uses for themes. Documentation available at `Installing custom theme`_ tutorial. + .. note:: Don't forget to add extra configurations in a Tutor plugin if your theme requires it. @@ -118,7 +143,11 @@ This command clones your theme repository into the folder that Tutor uses for th Run Extra Commands ^^^^^^^^^^^^^^^^^^^ -This command allows you to run a list of Tutor commands. These commands are executed in bash and, for security reasons, are restricted to running only Tutor commands. The input it takes is: +To execute a list of Tutor commands in your Tutor environment, follow these steps: + +1. Add configuration to the configuration file + +First, add the necessary configuration in your Tutor environment's ``config.yml`` file: .. code-block:: yaml PICASSO_EXTRA_COMMANDS: @@ -130,6 +159,18 @@ This command allows you to run a list of Tutor commands. These commands are exec . . +2. Save the configuration with ``tutor config save`` + +3. Run the following command + +.. code-block:: bash + + # Run Tutor commands + tutor picasso run-extra-commands + +This command allows you to run a list of Tutor commands. These commands are executed in bash and, for security reasons, are restricted to running only Tutor commands. + + License ******* From a078a8b2679210f47ba5ff1c57de0227a86eaf52 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Tue, 20 Aug 2024 16:02:41 -0500 Subject: [PATCH 14/25] fix: implement feedback --- .../commands/enable_private_packages.py | 56 ++++++-------- tutorpicasso/commands/enable_themes.py | 35 ++++----- tutorpicasso/commands/run_extra_commands.py | 77 +++++++++++-------- 3 files changed, 87 insertions(+), 81 deletions(-) diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index 8ffdc68..736eba7 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -17,7 +17,7 @@ def enable_private_packages() -> None: defining them as private. Raises: - Exception: If an error occurs during the cloning or defining process. + Exception: If there is not enough information to clone the repo. """ context = click.get_current_context().obj tutor_root = context.root @@ -41,39 +41,33 @@ def enable_private_packages() -> None: file.write("") for package, info in packages.items(): - try: - if not {"name", "repo", "version"}.issubset(info): - raise KeyError( - f"{package} is missing one of the required keys: 'name', 'repo', 'version'" - ) - - if os.path.isdir(f'{private_requirements_root}/{info["name"]}'): - subprocess.call( - ["rm", "-rf", f'{private_requirements_root}/{info["name"]}'] - ) + if not {"name", "repo", "version"}.issubset(info): + raise click.ClickException( + f"{package} is missing one of the required keys: 'name', 'repo', 'version'" + ) + if os.path.isdir(f'{private_requirements_root}/{info["name"]}'): subprocess.call( - ["git", "clone", "-b", info["version"], info["repo"]], - cwd=private_requirements_root, + ["rm", "-rf", f'{private_requirements_root}/{info["name"]}'] ) - if tutor_version_obj < quince_version_obj: - echo_command = ( - f'echo "-e ./{info["name"]}/" >> {private_requirements_txt}' - ) - subprocess.call(echo_command, shell=True) - else: - subprocess.call( - [ - "tutor", - "mounts", - "add", - f'{private_requirements_root}/{info["name"]}', - ] - ) + subprocess.call( + ["git", "clone", "-b", info["version"], info["repo"]], + cwd=private_requirements_root, + ) - except KeyError as e: - raise click.ClickException(str(e)) + if tutor_version_obj < quince_version_obj: + echo_command = f'echo "-e ./{info["name"]}/" >> {private_requirements_txt}' + subprocess.call(echo_command, shell=True) + else: + subprocess.call( + [ + "tutor", + "mounts", + "add", + f'{private_requirements_root}/{info["name"]}', + ] + ) def get_picasso_packages(settings: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: @@ -88,8 +82,6 @@ def get_picasso_packages(settings: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: and the values are package details. """ picasso_packages = { - key: val - for key, val in settings.items() - if key.endswith("_DPKG") and val != "None" + key: val for key, val in settings.items() if key.endswith("_DPKG") and val } return picasso_packages diff --git a/tutorpicasso/commands/enable_themes.py b/tutorpicasso/commands/enable_themes.py index 176608b..cc15dfe 100644 --- a/tutorpicasso/commands/enable_themes.py +++ b/tutorpicasso/commands/enable_themes.py @@ -18,21 +18,20 @@ def enable_themes() -> None: tutor_root = context.root tutor_conf = tutor_config.load(tutor_root) - if tutor_conf.get("PICASSO_THEMES"): - for theme in tutor_conf["PICASSO_THEMES"]: - try: - if not {"name", "repo", "version"}.issubset(theme.keys()): - raise KeyError( - f"{theme} is missing one or more required keys: " - "'name', 'repo', 'version'" - ) - - theme_path = f'{tutor_root}/env/build/openedx/themes/{theme["name"]}' - if os.path.isdir(theme_path): - subprocess.call(["rm", "-rf", theme_path]) - - subprocess.call( - ["git", "clone", "-b", theme["version"], theme["repo"], theme_path], - ) - except KeyError as e: - raise click.ClickException(f"Error: {str(e)}") + if not tutor_conf.get("PICASSO_THEMES"): + return + + for theme in tutor_conf["PICASSO_THEMES"]: + if not {"name", "repo", "version"}.issubset(theme.keys()): + raise click.ClickException( + f"{theme} is missing one or more required keys: " + "'name', 'repo', 'version'" + ) + + theme_path = f'{tutor_root}/env/build/openedx/themes/{theme["name"]}' + if os.path.isdir(theme_path): + subprocess.call(["rm", "-rf", theme_path]) + + subprocess.call( + ["git", "clone", "-b", theme["version"], theme["repo"], theme_path], + ) diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index 1d48862..9f8da9b 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -1,5 +1,6 @@ import re import subprocess +from itertools import chain # Was necessary to use this for compatibility with Python 3.8 from typing import Any, List @@ -15,16 +16,19 @@ def run_extra_commands() -> None: """ This command runs tutor commands defined in PICASSO_EXTRA_COMMANDS """ - tutor_root = ( - subprocess.check_output("tutor config printroot", shell=True) - .decode("utf-8") - .strip() - ) - config: Any = tutor_config.load(tutor_root) - picasso_extra_commands: Any = config.get("PICASSO_EXTRA_COMMANDS", None) - if picasso_extra_commands is not None: - validate_commands(picasso_extra_commands) - list(map(run_command, picasso_extra_commands)) + context = click.get_current_context().obj + tutor_conf = tutor_config.load(context.root) + + picasso_extra_commands: Any = tutor_conf.get("PICASSO_EXTRA_COMMANDS", None) + + if not picasso_extra_commands: + return + + error_message = validate_commands(picasso_extra_commands) + if error_message: + raise click.ClickException(error_message) + + list(map(run_command, picasso_extra_commands)) def validate_commands(commands: Any) -> None: @@ -38,28 +42,35 @@ def validate_commands(commands: Any) -> None: splitted_commands = [ split_string(command, COMMAND_CHAINING_OPERATORS) for command in commands ] - flat_commands_array: List[str] = sum(splitted_commands, []) + flat_commands_list: List[str] = chain.from_iterable(splitted_commands) invalid_commands = [] misspelled_commands = [] - for command in flat_commands_array: + for command in flat_commands_list: if "tutor" not in command.lower(): if find_tutor_misspelled(command): misspelled_commands.append(command) else: invalid_commands.append(command) - if invalid_commands or misspelled_commands: - error_message = ( + error_message = "" + + if invalid_commands: + error_message += ( f"Found some issues with the commands:\n\n" - f"{'=> Invalid commands: ' if invalid_commands else ''}" - f"{', '.join(invalid_commands) if invalid_commands else ''}\n" - f"{'=> Misspelled commands: ' if misspelled_commands else ''}" - f"{', '.join(misspelled_commands) if misspelled_commands else ''}\n" - f"Take a look at the official Tutor commands: " - f"https://docs.tutor.edly.io/reference/cli/index.html" + f"=> Invalid commands: {', '.join(invalid_commands)}\n" + ) + + if misspelled_commands: + error_message += ( + f"=> Misspelled commands: {', '.join(misspelled_commands)}\n" + ) + + if error_message: + error_message += ( + "Take a look at the official Tutor commands: " + "https://docs.tutor.edly.io/reference/cli/index.html" ) - raise click.ClickException(error_message) def run_command(command: str) -> None: @@ -69,7 +80,7 @@ def run_command(command: str) -> None: This method runs the extra command provided. Args: - command (str): Tutor command. + command (str): Tutor command. """ try: with subprocess.Popen( @@ -82,15 +93,13 @@ def run_command(command: str) -> None: text=True, ) as process: - # It is sent a 'y' to say 'yes' on overriding the existing folders - stdout, stderr = process.communicate(input="y") + stdout, stderr = process.communicate() if process.returncode != 0 or "error" in stderr.lower(): raise subprocess.CalledProcessError( process.returncode, command, output=stdout, stderr=stderr ) - # This print is left on purpose to show the command output click.echo(stdout) except subprocess.CalledProcessError as error: @@ -99,7 +108,13 @@ def run_command(command: str) -> None: def find_tutor_misspelled(command: str) -> bool: """ - This function takes a command and looks if it has the word 'tutor' misspelled + Look for misspelled occurrences of the word `tutor` in a given string. E.g. ... + + Args: + command (str): string to be reviewed. + + Return: + True if any misspelled occurrence is found, False otherwise. Args: command (str): Command to be reviewed @@ -110,19 +125,19 @@ def find_tutor_misspelled(command: str) -> bool: return bool(re.match(r"[tT](?:[oru]{3}|[oru]{2}[rR]|[oru]u?)", command)) -def create_regex_from_array(arr: List[str]) -> re.Pattern[str]: +def create_regex_from_list(special_chars: List[str]) -> re.Pattern[str]: """ Compile a new regex and escape special characters in the given string. escaping special characters Args: - arr (list[str]): String that would be used to create a new regex + special_chars (list[str]): String that would be used to create a new regex Return: A new compiled regex pattern that can be used for comparisons """ - escaped_arr = list(map(re.escape, arr)) - regex_pattern = "|".join(escaped_arr) + escaped_special_chars = list(map(re.escape, special_chars)) + regex_pattern = "|".join(escaped_special_chars) return re.compile(regex_pattern) @@ -137,4 +152,4 @@ def split_string(string: str, split_by: List[str]) -> List[str]: Return: The string split into a list """ - return re.split(create_regex_from_array(split_by), string) + return re.split(create_regex_from_list(split_by), string) From 3c8dc2ecaaa15d0aed2ace6ba1e09dc108412828 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Tue, 20 Aug 2024 18:38:22 -0500 Subject: [PATCH 15/25] test: add again the test-types --- Makefile | 2 +- tutorpicasso/commands/enable_themes.py | 8 ++++---- tutorpicasso/commands/run_extra_commands.py | 10 ++++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index d4b0762..797134d 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ SRC_DIRS = ./tutorpicasso BLACK_OPTS = --exclude templates ${SRC_DIRS} # Warning: These checks are not necessarily run on every PR. -test: test-lint test-format # Run some static checks. +test: test-lint test-types test-format # Run some static checks. test-format: ## Run code formatting tests black --check --diff $(BLACK_OPTS) diff --git a/tutorpicasso/commands/enable_themes.py b/tutorpicasso/commands/enable_themes.py index cc15dfe..5d01b58 100644 --- a/tutorpicasso/commands/enable_themes.py +++ b/tutorpicasso/commands/enable_themes.py @@ -21,17 +21,17 @@ def enable_themes() -> None: if not tutor_conf.get("PICASSO_THEMES"): return - for theme in tutor_conf["PICASSO_THEMES"]: - if not {"name", "repo", "version"}.issubset(theme.keys()): + for theme in tutor_conf["PICASSO_THEMES"]: # type: ignore + if not {"name", "repo", "version"}.issubset(theme.keys()): # type: ignore raise click.ClickException( f"{theme} is missing one or more required keys: " "'name', 'repo', 'version'" ) - theme_path = f'{tutor_root}/env/build/openedx/themes/{theme["name"]}' + theme_path = f'{tutor_root}/env/build/openedx/themes/{theme["name"]}' # type: ignore if os.path.isdir(theme_path): subprocess.call(["rm", "-rf", theme_path]) subprocess.call( - ["git", "clone", "-b", theme["version"], theme["repo"], theme_path], + ["git", "clone", "-b", theme["version"], theme["repo"], theme_path], # type: ignore ) diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index 9f8da9b..c464d9b 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -31,7 +31,7 @@ def run_extra_commands() -> None: list(map(run_command, picasso_extra_commands)) -def validate_commands(commands: Any) -> None: +def validate_commands(commands: Any) -> str: """ Takes all the extra commands sent through config.yml and verifies that all the commands are correct before executing them @@ -42,7 +42,7 @@ def validate_commands(commands: Any) -> None: splitted_commands = [ split_string(command, COMMAND_CHAINING_OPERATORS) for command in commands ] - flat_commands_list: List[str] = chain.from_iterable(splitted_commands) + flat_commands_list: chain[str] = chain.from_iterable(splitted_commands) invalid_commands = [] misspelled_commands = [] @@ -71,6 +71,8 @@ def validate_commands(commands: Any) -> None: "Take a look at the official Tutor commands: " "https://docs.tutor.edly.io/reference/cli/index.html" ) + return error_message + return "" def run_command(command: str) -> None: @@ -136,8 +138,8 @@ def create_regex_from_list(special_chars: List[str]) -> re.Pattern[str]: Return: A new compiled regex pattern that can be used for comparisons """ - escaped_special_chars = list(map(re.escape, special_chars)) - regex_pattern = "|".join(escaped_special_chars) + escaped_special_chars = list(map(re.escape, special_chars)) # type: ignore + regex_pattern = "|".join(escaped_special_chars) # type: ignore return re.compile(regex_pattern) From e1ad58497b8336df700cc23f2cb4f778dc27cdbf Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Wed, 21 Aug 2024 21:17:26 -0500 Subject: [PATCH 16/25] fix: apply feedback about quick checks and separate methods --- .github/workflows/integration_test.yml | 1 - .../commands/enable_private_packages.py | 64 +++++++++++++------ tutorpicasso/commands/run_extra_commands.py | 53 ++++++++------- 3 files changed, 71 insertions(+), 47 deletions(-) diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index d1b59d9..57fb98e 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -6,7 +6,6 @@ jobs: name: Integration with Tutor strategy: matrix: - # tutor_version: ["Redwood", "Quince", "Palm", "Olive"] tutor_version: ["<19.0.0", "<18.0.0", "<17.0.0", "<16.0.0"] runs-on: ubuntu-latest steps: diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index 736eba7..a27b686 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -22,23 +22,18 @@ def enable_private_packages() -> None: context = click.get_current_context().obj tutor_root = context.root tutor_conf = tutor_config.load(tutor_root) + tutor_version_obj = Version(tutor_version) # Define Quince version as the method for installing private packages changes from this version quince_version_obj = Version("v17.0.0") + # Use these specific paths as required by Tutor < Quince private_requirements_root = f"{tutor_root}/env/build/openedx/requirements" - private_requirements_txt = f"{private_requirements_root}/private.txt" packages = get_picasso_packages(tutor_conf) - # Create necessary files and directories if they don't exist + # Create necessary directory if it doesn't exist if not os.path.exists(private_requirements_root): os.makedirs(private_requirements_root) - if ( - not os.path.exists(private_requirements_txt) - and tutor_version_obj < quince_version_obj - ): - with open(private_requirements_txt, "w") as file: - file.write("") for package, info in packages.items(): if not {"name", "repo", "version"}.issubset(info): @@ -57,17 +52,50 @@ def enable_private_packages() -> None: ) if tutor_version_obj < quince_version_obj: - echo_command = f'echo "-e ./{info["name"]}/" >> {private_requirements_txt}' - subprocess.call(echo_command, shell=True) + private_requirements_txt = f"{private_requirements_root}/private.txt" + _enable_private_packages_before_quince(info, private_requirements_txt) else: - subprocess.call( - [ - "tutor", - "mounts", - "add", - f'{private_requirements_root}/{info["name"]}', - ] - ) + _enable_private_packages(info, private_requirements_root) + + +def _enable_private_packages_before_quince( + info: Dict[str, str], private_requirements_txt: str +) -> None: + """ + Copy the package name in the private.txt file to ensure that packages are added in the build process for Tutor versions < Quince. + + Args: + info (Dict[str, str]): A dictionary containing metadata about the package. Expected to have a "name" key. + private_requirements_txt (str): The file path to `private.txt`, which stores the list of private packages to be included in the build. + """ + + # Create necessary file if it doesn't exist + if not os.path.exists(private_requirements_txt): + with open(private_requirements_txt, "w") as file: + file.write("") + + echo_command = f'echo "-e ./{info["name"]}/" >> {private_requirements_txt}' + subprocess.call(echo_command, shell=True) + + +def _enable_private_packages( + info: Dict[str, str], private_requirements_root: str +) -> None: + """ + Use the tutor mounts method to ensure that packages are added in the build process. + + Args: + info (Dict[str, str]): A dictionary containing metadata about the package. Expected to have a "name" key. + private_requirements_root (str): The root directory where private packages are stored. + """ + subprocess.call( + [ + "tutor", + "mounts", + "add", + f'{private_requirements_root}/{info["name"]}', + ] + ) def get_picasso_packages(settings: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index c464d9b..4a5ecfc 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -2,7 +2,6 @@ import subprocess from itertools import chain -# Was necessary to use this for compatibility with Python 3.8 from typing import Any, List import click @@ -47,11 +46,13 @@ def validate_commands(commands: Any) -> str: invalid_commands = [] misspelled_commands = [] for command in flat_commands_list: - if "tutor" not in command.lower(): - if find_tutor_misspelled(command): - misspelled_commands.append(command) - else: - invalid_commands.append(command) + if "tutor" in command.lower(): + continue + + if find_tutor_misspelled(command): + misspelled_commands.append(command) + else: + invalid_commands.append(command) error_message = "" @@ -84,28 +85,24 @@ def run_command(command: str) -> None: Args: command (str): Tutor command. """ - try: - with subprocess.Popen( - command, - shell=True, - executable="/bin/bash", - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) as process: - - stdout, stderr = process.communicate() - - if process.returncode != 0 or "error" in stderr.lower(): - raise subprocess.CalledProcessError( - process.returncode, command, output=stdout, stderr=stderr - ) - - click.echo(stdout) - - except subprocess.CalledProcessError as error: - raise click.ClickException(str(error)) + with subprocess.Popen( + command, + shell=True, + executable="/bin/bash", + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as process: + + stdout, stderr = process.communicate() + + if process.returncode != 0 or "error" in stderr.lower(): + raise click.ClickException( + f"Command '{command}' failed with return code {process.returncode}. Output: {stdout}. Error: {stderr}" + ) + + click.echo(stdout) def find_tutor_misspelled(command: str) -> bool: From 0203763de751222338d37a1a624d4b1e96c33fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Fernanda=20Magallanes?= <35668326+MaferMazu@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:30:19 -0500 Subject: [PATCH 17/25] docs: update the readme format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Brayan Cerón <86393372+bra-i-am@users.noreply.github.com> --- README.rst | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index fdf6fdb..1202930 100644 --- a/README.rst +++ b/README.rst @@ -61,10 +61,11 @@ To enable the installation of private Open edX Django apps, follow these steps: First, add the necessary configuration in your Tutor environment's ``config.yml`` file: .. code-block:: yaml + PICASSO__DPKG: - name: - repo: - version: + name: + repo: + version: .. note:: @@ -72,9 +73,7 @@ First, add the necessary configuration in your Tutor environment's ``config.yml` 2. Save the configuration with ``tutor config save`` -3. Run the enable command - -Next, run the following command to enable private packages: +3. Run the following command to enable private packages: .. code-block:: bash @@ -107,11 +106,10 @@ Enable Themes To enable themes in your Tutor environment, follow these steps: -1. Add configuration to the configuration file - -First, add the necessary configuration in your Tutor environment's ``config.yml`` file: +1. Add the necessary configuration in your Tutor environment's ``config.yml`` file: .. code-block:: yaml + PICASSO_THEMES: - name: repo: @@ -145,11 +143,10 @@ Run Extra Commands To execute a list of Tutor commands in your Tutor environment, follow these steps: -1. Add configuration to the configuration file - -First, add the necessary configuration in your Tutor environment's ``config.yml`` file: +1. Add the necessary configuration in your Tutor environment's ``config.yml`` file: .. code-block:: yaml + PICASSO_EXTRA_COMMANDS: - - From 18f983e9bb410ee91bdd1afb324c95aa7cb59ffb Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Fri, 23 Aug 2024 16:36:37 -0500 Subject: [PATCH 18/25] chore: improve integration tests --- .github/workflows/integration_test.yml | 30 ++++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index 57fb98e..2032919 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -33,20 +33,22 @@ jobs: - name: Adding data to the config.yml run: | - echo "PICASSO_MANAGE_DPKG:" | tee -a ./config.yml - echo " name: eox-manage" | tee -a ./config.yml - echo " repo: git@github.com:eduNEXT/eox-manage.git" | tee -a ./config.yml - echo " version: v5.2.0" | tee -a ./config.yml - echo "PICASSO_THEMES:" | tee -a ./config.yml - echo "- name: endx-saas-themes" | tee -a ./config.yml - echo " repo: git@github.com:eduNEXT/ednx-saas-themes.git" | tee -a ./config.yml - echo " version: master" | tee -a ./config.yml - echo "PICASSO_EXTRA_COMMANDS:" | tee -a ./config.yml - echo "- tutor plugins update" | tee -a ./config.yml - echo "- tutor plugins index add https://raw.githubusercontent.com/eduNEXT/tutor-plugin-indexes/picasso_test/" | tee -a ./config.yml - echo "- tutor plugins install mfe mfe_extensions aspects" | tee -a ./config.yml - echo "- tutor plugins enable mfe mfe_extensions aspects" | tee -a ./config.yml - echo "- tutor config save" | tee -a ./config.yml + cat <> config.yml + PICASSO_MANAGE_DPKG: + name: eox-manage + repo: git@github.com:eduNEXT/eox-manage.git + version: v5.2.0 + PICASSO_THEMES: + - name: endx-saas-themes + repo: git@github.com:eduNEXT/ednx-saas-themes.git + version: master + PICASSO_EXTRA_COMMANDS: + - tutor plugins update + - tutor plugins index add https://raw.githubusercontent.com/eduNEXT/tutor-plugin-indexes/picasso_test/ + - tutor plugins install mfe mfe_extensions aspects + - tutor plugins enable mfe mfe_extensions aspects + - tutor config save + EOF - name: Check run-extra-commands run: | From c8eefa01da445f999aaa00d43d5da7dfc301bd71 Mon Sep 17 00:00:00 2001 From: Maria Grimaldi Date: Thu, 22 Aug 2024 10:57:35 -0400 Subject: [PATCH 19/25] refactor: use handler to manage requirement tasks depending on tutor version --- .../commands/enable_private_packages.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index a27b686..63ff042 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -23,15 +23,9 @@ def enable_private_packages() -> None: tutor_root = context.root tutor_conf = tutor_config.load(tutor_root) - tutor_version_obj = Version(tutor_version) - # Define Quince version as the method for installing private packages changes from this version - quince_version_obj = Version("v17.0.0") - - # Use these specific paths as required by Tutor < Quince - private_requirements_root = f"{tutor_root}/env/build/openedx/requirements" + private_requirements_root = f"{tutor_root}/env/build/openedx/private_requirements" packages = get_picasso_packages(tutor_conf) - # Create necessary directory if it doesn't exist if not os.path.exists(private_requirements_root): os.makedirs(private_requirements_root) @@ -51,11 +45,25 @@ def enable_private_packages() -> None: cwd=private_requirements_root, ) - if tutor_version_obj < quince_version_obj: - private_requirements_txt = f"{private_requirements_root}/private.txt" - _enable_private_packages_before_quince(info, private_requirements_txt) - else: - _enable_private_packages(info, private_requirements_root) + handle_private_requirements_by_tutor_version(info, private_requirements_root) + + +def handle_private_requirements_by_tutor_version(info: Dict[str, str], private_requirements_path: str) -> None: + """ + Handle the private requirements based on the Tutor version. + + Args: + info (Dict[str, str]): A dictionary containing metadata about the package. Expected to have a "name" key. + private_requirements_path (str): The directory path to store the private packages. + """ + tutor_version_obj = Version(tutor_version) + quince_version_obj = Version("v17.0.0") + + if tutor_version_obj < quince_version_obj: + private_txt_path = f"{private_requirements_path}/private.txt" + _enable_private_packages_before_quince(info, private_txt_path) + else: + _enable_private_packages_latest(info, private_requirements_path) def _enable_private_packages_before_quince( @@ -68,8 +76,6 @@ def _enable_private_packages_before_quince( info (Dict[str, str]): A dictionary containing metadata about the package. Expected to have a "name" key. private_requirements_txt (str): The file path to `private.txt`, which stores the list of private packages to be included in the build. """ - - # Create necessary file if it doesn't exist if not os.path.exists(private_requirements_txt): with open(private_requirements_txt, "w") as file: file.write("") @@ -78,7 +84,7 @@ def _enable_private_packages_before_quince( subprocess.call(echo_command, shell=True) -def _enable_private_packages( +def _enable_private_packages_latest( info: Dict[str, str], private_requirements_root: str ) -> None: """ From 5d8dc5bd9e1c3f96dc4ed531d5cdb2c71d6556cd Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 26 Aug 2024 16:06:57 -0500 Subject: [PATCH 20/25] refactor: improve the naming in enable_private_packages --- .../commands/enable_private_packages.py | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index 63ff042..dcc55bf 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -23,11 +23,11 @@ def enable_private_packages() -> None: tutor_root = context.root tutor_conf = tutor_config.load(tutor_root) - private_requirements_root = f"{tutor_root}/env/build/openedx/private_requirements" + private_requirements_path = f"{tutor_root}/env/build/openedx/requirements" packages = get_picasso_packages(tutor_conf) - if not os.path.exists(private_requirements_root): - os.makedirs(private_requirements_root) + if not os.path.exists(private_requirements_path): + os.makedirs(private_requirements_path) for package, info in packages.items(): if not {"name", "repo", "version"}.issubset(info): @@ -35,46 +35,48 @@ def enable_private_packages() -> None: f"{package} is missing one of the required keys: 'name', 'repo', 'version'" ) - if os.path.isdir(f'{private_requirements_root}/{info["name"]}'): + if os.path.isdir(f'{private_requirements_path}/{info["name"]}'): subprocess.call( - ["rm", "-rf", f'{private_requirements_root}/{info["name"]}'] + ["rm", "-rf", f'{private_requirements_path}/{info["name"]}'] ) subprocess.call( ["git", "clone", "-b", info["version"], info["repo"]], - cwd=private_requirements_root, + cwd=private_requirements_path, ) - handle_private_requirements_by_tutor_version(info, private_requirements_root) + handle_private_requirements_by_tutor_version(info, private_requirements_path) -def handle_private_requirements_by_tutor_version(info: Dict[str, str], private_requirements_path: str) -> None: +def handle_private_requirements_by_tutor_version( + info: Dict[str, str], private_requirements_path: str +) -> None: """ Handle the private requirements based on the Tutor version. Args: - info (Dict[str, str]): A dictionary containing metadata about the package. Expected to have a "name" key. - private_requirements_path (str): The directory path to store the private packages. + info (Dict[str, str]): A dictionary containing metadata about the requirement. Expected to have a "name" key. + private_requirements_path (str): The directory path to store the private requirements. """ tutor_version_obj = Version(tutor_version) quince_version_obj = Version("v17.0.0") if tutor_version_obj < quince_version_obj: private_txt_path = f"{private_requirements_path}/private.txt" - _enable_private_packages_before_quince(info, private_txt_path) + _enable_private_requirements_before_quince(info, private_txt_path) else: - _enable_private_packages_latest(info, private_requirements_path) + _enable_private_requirements_latest(info, private_requirements_path) -def _enable_private_packages_before_quince( +def _enable_private_requirements_before_quince( info: Dict[str, str], private_requirements_txt: str ) -> None: """ - Copy the package name in the private.txt file to ensure that packages are added in the build process for Tutor versions < Quince. + Copy the requirement name in the private.txt file to ensure that requirements are added in the build process for Tutor versions < Quince. Args: - info (Dict[str, str]): A dictionary containing metadata about the package. Expected to have a "name" key. - private_requirements_txt (str): The file path to `private.txt`, which stores the list of private packages to be included in the build. + info (Dict[str, str]): A dictionary containing metadata about the requirement. Expected to have a "name" key. + private_requirements_txt (str): The file path to `private.txt`, which stores the list of private requirements to be included in the build. """ if not os.path.exists(private_requirements_txt): with open(private_requirements_txt, "w") as file: @@ -84,22 +86,22 @@ def _enable_private_packages_before_quince( subprocess.call(echo_command, shell=True) -def _enable_private_packages_latest( - info: Dict[str, str], private_requirements_root: str +def _enable_private_requirements_latest( + info: Dict[str, str], private_requirements_path: str ) -> None: """ - Use the tutor mounts method to ensure that packages are added in the build process. + Use the tutor mounts method to ensure that requirements are added in the build process. Args: - info (Dict[str, str]): A dictionary containing metadata about the package. Expected to have a "name" key. - private_requirements_root (str): The root directory where private packages are stored. + info (Dict[str, str]): A dictionary containing metadata about the requirement. Expected to have a "name" key. + private_requirements_path (str): The root directory where private requirements are stored. """ subprocess.call( [ "tutor", "mounts", "add", - f'{private_requirements_root}/{info["name"]}', + f'{private_requirements_path}/{info["name"]}', ] ) From 821c5fc563c66fad6a259b9501851894b8f7dbdf Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 26 Aug 2024 16:35:51 -0500 Subject: [PATCH 21/25] docs: improve code docs --- tutorpicasso/commands/enable_private_packages.py | 2 +- tutorpicasso/commands/run_extra_commands.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py index dcc55bf..1036355 100644 --- a/tutorpicasso/commands/enable_private_packages.py +++ b/tutorpicasso/commands/enable_private_packages.py @@ -114,7 +114,7 @@ def get_picasso_packages(settings: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: settings (dict): The tutor configuration settings. Returns: - dict: A dictionary of distribution packages, where the keys are package names + A dictionary of distribution packages, where the keys are package names and the values are package details. """ picasso_packages = { diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index 4a5ecfc..afced20 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -114,12 +114,6 @@ def find_tutor_misspelled(command: str) -> bool: Return: True if any misspelled occurrence is found, False otherwise. - - Args: - command (str): Command to be reviewed - - Return: - If its found the word 'tutor' misspelled is returned True """ return bool(re.match(r"[tT](?:[oru]{3}|[oru]{2}[rR]|[oru]u?)", command)) From 018b455eb85639106185b5a96b16e4346a02c046 Mon Sep 17 00:00:00 2001 From: Maria Grimaldi Date: Thu, 22 Aug 2024 11:31:55 -0400 Subject: [PATCH 22/25] refactor: return all command error messages instead --- tutorpicasso/commands/run_extra_commands.py | 46 ++++++++++----------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index afced20..50970c4 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -49,31 +49,29 @@ def validate_commands(commands: Any) -> str: if "tutor" in command.lower(): continue - if find_tutor_misspelled(command): - misspelled_commands.append(command) - else: - invalid_commands.append(command) + if find_tutor_misspelled(command): + misspelled_commands.append(command) + else: + invalid_commands.append(command) + + error_message = "" + if invalid_commands: + error_message += ( + f"Found some issues with the commands:\n\n" + f"=> Invalid commands: {', '.join(invalid_commands)}\n" + ) + + if misspelled_commands: + error_message += ( + f"=> Misspelled commands: {', '.join(misspelled_commands)}\n" + ) - error_message = "" - - if invalid_commands: - error_message += ( - f"Found some issues with the commands:\n\n" - f"=> Invalid commands: {', '.join(invalid_commands)}\n" - ) - - if misspelled_commands: - error_message += ( - f"=> Misspelled commands: {', '.join(misspelled_commands)}\n" - ) - - if error_message: - error_message += ( - "Take a look at the official Tutor commands: " - "https://docs.tutor.edly.io/reference/cli/index.html" - ) - return error_message - return "" + if error_message: + error_message += ( + "Take a look at the official Tutor commands: " + "https://docs.tutor.edly.io/reference/cli/index.html" + ) + return error_message def run_command(command: str) -> None: From f03da32c34dad17540c4226b7ac16ed5ad84a56b Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 26 Aug 2024 16:45:31 -0500 Subject: [PATCH 23/25] fix: check the commands inside the loop and format --- tutorpicasso/commands/run_extra_commands.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index 50970c4..076c699 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -48,11 +48,10 @@ def validate_commands(commands: Any) -> str: for command in flat_commands_list: if "tutor" in command.lower(): continue - - if find_tutor_misspelled(command): - misspelled_commands.append(command) - else: - invalid_commands.append(command) + if find_tutor_misspelled(command): + misspelled_commands.append(command) + else: + invalid_commands.append(command) error_message = "" if invalid_commands: @@ -62,9 +61,7 @@ def validate_commands(commands: Any) -> str: ) if misspelled_commands: - error_message += ( - f"=> Misspelled commands: {', '.join(misspelled_commands)}\n" - ) + error_message += f"=> Misspelled commands: {', '.join(misspelled_commands)}\n" if error_message: error_message += ( From 26fb79acb93e14fcda1a9c8d9cac52a15d55f275 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 26 Aug 2024 16:56:10 -0500 Subject: [PATCH 24/25] docs: add documentation about the type ignore --- tutorpicasso/commands/enable_themes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tutorpicasso/commands/enable_themes.py b/tutorpicasso/commands/enable_themes.py index 5d01b58..a97e24f 100644 --- a/tutorpicasso/commands/enable_themes.py +++ b/tutorpicasso/commands/enable_themes.py @@ -21,6 +21,9 @@ def enable_themes() -> None: if not tutor_conf.get("PICASSO_THEMES"): return + # We use `type: ignore` for the `tutor_conf` object + # because it comes from the Tutor framework. + # We are not handle type errors related to this object. for theme in tutor_conf["PICASSO_THEMES"]: # type: ignore if not {"name", "repo", "version"}.issubset(theme.keys()): # type: ignore raise click.ClickException( From 035f49d30efe162d60747595d3f643ce65e39084 Mon Sep 17 00:00:00 2001 From: Maria Fernanda Magallanes Zubillaga Date: Mon, 26 Aug 2024 18:05:48 -0500 Subject: [PATCH 25/25] fix: use typing Pattern instead of regex one --- tutorpicasso/commands/run_extra_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py index 076c699..0e2c9ca 100644 --- a/tutorpicasso/commands/run_extra_commands.py +++ b/tutorpicasso/commands/run_extra_commands.py @@ -2,7 +2,7 @@ import subprocess from itertools import chain -from typing import Any, List +from typing import Any, List, Pattern import click from tutor import config as tutor_config @@ -113,7 +113,7 @@ def find_tutor_misspelled(command: str) -> bool: return bool(re.match(r"[tT](?:[oru]{3}|[oru]{2}[rR]|[oru]u?)", command)) -def create_regex_from_list(special_chars: List[str]) -> re.Pattern[str]: +def create_regex_from_list(special_chars: List[str]) -> Pattern[str]: """ Compile a new regex and escape special characters in the given string. escaping special characters