From 3f3a222e13720c7a1988ea27bc802faa38b31f52 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Tue, 4 Jul 2023 17:07:18 +0800 Subject: [PATCH 01/36] Add flask framework extension --- rockcraft/extensions/__init__.py | 4 ++ rockcraft/extensions/extension_flask.py | 96 +++++++++++++++++++++++++ rockcraft/providers.py | 7 +- 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 rockcraft/extensions/extension_flask.py diff --git a/rockcraft/extensions/__init__.py b/rockcraft/extensions/__init__.py index 36f83ef1b..79e5d9198 100644 --- a/rockcraft/extensions/__init__.py +++ b/rockcraft/extensions/__init__.py @@ -26,3 +26,7 @@ "register", "unregister", ] + +from .extension_flask import FlaskExtension as _FlaskExtension + +register("flask-extension", _FlaskExtension) diff --git a/rockcraft/extensions/extension_flask.py b/rockcraft/extensions/extension_flask.py new file mode 100644 index 000000000..743b6016e --- /dev/null +++ b/rockcraft/extensions/extension_flask.py @@ -0,0 +1,96 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright 2023 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Experimental Rockcraft extension for the Juju PaaS Flask framework.""" +import os.path +from typing import Tuple, Dict, Any, Optional +from overrides import override + +from .extension import Extension + + +class FlaskExtension(Extension): + @staticmethod + @override + def get_supported_bases() -> Tuple[str, ...]: + return "bare", "ubuntu:22.04" + + @staticmethod + @override + def is_experimental(base: Optional[str]) -> bool: + return True + + @override + def get_root_snippet(self) -> Dict[str, Any]: + """Fill in some default root components for the flask platform. + + Default values: + - run_user: _daemon_ + - build-base: ubuntu:22.04 (only if user specify bare without a build-base) + - platform: amd64 + """ + snippet = {} + if "run_user" not in self.yaml_data: + snippet["run_user"] = "_daemon_" + if ( + "build-base" not in self.yaml_data and + self.yaml_data.get("base", "bare") == "bare" + ): + snippet["build-base"] = "ubuntu:22.04" + if "platforms" not in self.yaml_data: + snippet["platforms"] = {"amd64": {}} + return snippet + + @override + def get_part_snippet(self) -> Dict[str, Any]: + return {} + + @override + def get_parts_snippet(self) -> Dict[str, Any]: + """Create necessary parts to facilitate the flask platform. + + Parts added: + - flask-extension/dependencies: install Python dependencies + - flask-extension/app: copy the entire flask project into the OCI image + """ + python_requirements = [] + if (self.project_root / "requirements.txt").exists(): + python_requirements.append("requirements.txt") + ignores = [".git", "node_modules"] + source_files = [ + f for f in os.listdir(self.project_root) + if f not in ignores and not f.endswith(".rock") + ] + renaming_map = { + f: os.path.join("srv/flask/app", f) + for f in source_files + } + snippet = { + "flask-extension/dependencies": { + "plugin": "python", + "stage-packages": ["python3-venv"], + "source": ".", + "python-packages": ["gunicorn"], + "python-requirements": python_requirements + }, + "flask-extension/app": { + "plugin": "dump", + "source": ".", + "organize": renaming_map, + "stage": list(renaming_map.values()) + } + } + return snippet diff --git a/rockcraft/providers.py b/rockcraft/providers.py index 123639e16..9a80fe11e 100644 --- a/rockcraft/providers.py +++ b/rockcraft/providers.py @@ -46,7 +46,12 @@ def get_command_environment() -> Dict[str, Optional[str]]: env["ROCKCRAFT_MANAGED_MODE"] = "1" # Pass-through host environment that target may need. - for env_key in ["http_proxy", "https_proxy", "no_proxy"]: + for env_key in [ + "http_proxy", + "https_proxy", + "no_proxy", + "ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS" + ]: if env_key in os.environ: env[env_key] = os.environ[env_key] From caa7dbfe4331b5fbf1ca3d503114caacefb9c912 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 5 Jul 2023 12:43:31 +0800 Subject: [PATCH 02/36] Update documents in the flask extension --- rockcraft/extensions/extension_flask.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rockcraft/extensions/extension_flask.py b/rockcraft/extensions/extension_flask.py index 743b6016e..e731d2b54 100644 --- a/rockcraft/extensions/extension_flask.py +++ b/rockcraft/extensions/extension_flask.py @@ -14,8 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""Experimental Rockcraft extension for the Juju PaaS Flask framework.""" -import os.path +"""An experimental extension for the Juju PaaS Flask framework.""" + +import os + from typing import Tuple, Dict, Any, Optional from overrides import override @@ -23,6 +25,12 @@ class FlaskExtension(Extension): + """An extension to enable Flask support within the Juju PaaS ecosystem. + + This extension is a part of the Juju PaaS ecosystem and facilitates the deployment of + Flask-based applications. + """ + @staticmethod @override def get_supported_bases() -> Tuple[str, ...]: @@ -35,7 +43,7 @@ def is_experimental(base: Optional[str]) -> bool: @override def get_root_snippet(self) -> Dict[str, Any]: - """Fill in some default root components for the flask platform. + """Fill in some default root components for Flask. Default values: - run_user: _daemon_ @@ -60,11 +68,11 @@ def get_part_snippet(self) -> Dict[str, Any]: @override def get_parts_snippet(self) -> Dict[str, Any]: - """Create necessary parts to facilitate the flask platform. + """Create necessary parts to facilitate the flask application. Parts added: - flask-extension/dependencies: install Python dependencies - - flask-extension/app: copy the entire flask project into the OCI image + - flask-extension/install-app: copy the flask project into the OCI image """ python_requirements = [] if (self.project_root / "requirements.txt").exists(): @@ -86,7 +94,7 @@ def get_parts_snippet(self) -> Dict[str, Any]: "python-packages": ["gunicorn"], "python-requirements": python_requirements }, - "flask-extension/app": { + "flask-extension/install-app": { "plugin": "dump", "source": ".", "organize": renaming_map, From 128c58df639e8b682ae33505604995d6867ba56c Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 5 Jul 2023 13:15:45 +0800 Subject: [PATCH 03/36] Fix some linting issues --- rockcraft/extensions/extension_flask.py | 23 ++++++++++++----------- rockcraft/providers.py | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/rockcraft/extensions/extension_flask.py b/rockcraft/extensions/extension_flask.py index e731d2b54..698227971 100644 --- a/rockcraft/extensions/extension_flask.py +++ b/rockcraft/extensions/extension_flask.py @@ -34,11 +34,13 @@ class FlaskExtension(Extension): @staticmethod @override def get_supported_bases() -> Tuple[str, ...]: + """Return supported bases.""" return "bare", "ubuntu:22.04" @staticmethod @override def is_experimental(base: Optional[str]) -> bool: + """Check if the extension is in an experimental state.""" return True @override @@ -50,12 +52,12 @@ def get_root_snippet(self) -> Dict[str, Any]: - build-base: ubuntu:22.04 (only if user specify bare without a build-base) - platform: amd64 """ - snippet = {} + snippet: Dict[str, Any] = {} if "run_user" not in self.yaml_data: snippet["run_user"] = "_daemon_" if ( - "build-base" not in self.yaml_data and - self.yaml_data.get("base", "bare") == "bare" + "build-base" not in self.yaml_data + and self.yaml_data.get("base", "bare") == "bare" ): snippet["build-base"] = "ubuntu:22.04" if "platforms" not in self.yaml_data: @@ -64,6 +66,7 @@ def get_root_snippet(self) -> Dict[str, Any]: @override def get_part_snippet(self) -> Dict[str, Any]: + """Return the part snippet to apply to existing parts.""" return {} @override @@ -79,26 +82,24 @@ def get_parts_snippet(self) -> Dict[str, Any]: python_requirements.append("requirements.txt") ignores = [".git", "node_modules"] source_files = [ - f for f in os.listdir(self.project_root) + f + for f in os.listdir(self.project_root) if f not in ignores and not f.endswith(".rock") ] - renaming_map = { - f: os.path.join("srv/flask/app", f) - for f in source_files - } + renaming_map = {f: os.path.join("srv/flask/app", f) for f in source_files} snippet = { "flask-extension/dependencies": { "plugin": "python", "stage-packages": ["python3-venv"], "source": ".", "python-packages": ["gunicorn"], - "python-requirements": python_requirements + "python-requirements": python_requirements, }, "flask-extension/install-app": { "plugin": "dump", "source": ".", "organize": renaming_map, - "stage": list(renaming_map.values()) - } + "stage": list(renaming_map.values()), + }, } return snippet diff --git a/rockcraft/providers.py b/rockcraft/providers.py index 9a80fe11e..e50fd9e43 100644 --- a/rockcraft/providers.py +++ b/rockcraft/providers.py @@ -50,7 +50,7 @@ def get_command_environment() -> Dict[str, Optional[str]]: "http_proxy", "https_proxy", "no_proxy", - "ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS" + "ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", ]: if env_key in os.environ: env[env_key] = os.environ[env_key] From 016b2b1d68f90d291cc41c697c872a5cb541fc92 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 5 Jul 2023 13:19:45 +0800 Subject: [PATCH 04/36] Fix some linting issues --- rockcraft/extensions/extension_flask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rockcraft/extensions/extension_flask.py b/rockcraft/extensions/extension_flask.py index 698227971..da2a67d98 100644 --- a/rockcraft/extensions/extension_flask.py +++ b/rockcraft/extensions/extension_flask.py @@ -17,8 +17,8 @@ """An experimental extension for the Juju PaaS Flask framework.""" import os +from typing import Any, Dict, Optional, Tuple -from typing import Tuple, Dict, Any, Optional from overrides import override from .extension import Extension From 10ec7d4bae98b7e1380934e6fde3557f00e868e1 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 5 Jul 2023 16:36:45 +0800 Subject: [PATCH 05/36] Update the name of the extension --- rockcraft/extensions/__init__.py | 4 ++-- rockcraft/extensions/{extension_flask.py => paas_flask.py} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename rockcraft/extensions/{extension_flask.py => paas_flask.py} (96%) diff --git a/rockcraft/extensions/__init__.py b/rockcraft/extensions/__init__.py index 79e5d9198..27e89bcf3 100644 --- a/rockcraft/extensions/__init__.py +++ b/rockcraft/extensions/__init__.py @@ -27,6 +27,6 @@ "unregister", ] -from .extension_flask import FlaskExtension as _FlaskExtension +from .paas_flask import PaasFlask as _PaasFlask -register("flask-extension", _FlaskExtension) +register("paas-flask", _PaasFlask) diff --git a/rockcraft/extensions/extension_flask.py b/rockcraft/extensions/paas_flask.py similarity index 96% rename from rockcraft/extensions/extension_flask.py rename to rockcraft/extensions/paas_flask.py index da2a67d98..cb77f09b9 100644 --- a/rockcraft/extensions/extension_flask.py +++ b/rockcraft/extensions/paas_flask.py @@ -24,7 +24,7 @@ from .extension import Extension -class FlaskExtension(Extension): +class PaasFlask(Extension): """An extension to enable Flask support within the Juju PaaS ecosystem. This extension is a part of the Juju PaaS ecosystem and facilitates the deployment of @@ -88,14 +88,14 @@ def get_parts_snippet(self) -> Dict[str, Any]: ] renaming_map = {f: os.path.join("srv/flask/app", f) for f in source_files} snippet = { - "flask-extension/dependencies": { + "paas-flask/dependencies": { "plugin": "python", "stage-packages": ["python3-venv"], "source": ".", "python-packages": ["gunicorn"], "python-requirements": python_requirements, }, - "flask-extension/install-app": { + "paas-flask/install-app": { "plugin": "dump", "source": ".", "organize": renaming_map, From 8f622fa1a5cabb71639f45f61cd4cd63e5374570 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Tue, 11 Jul 2023 00:43:11 +0800 Subject: [PATCH 06/36] Some document and naming changes --- rockcraft/extensions/__init__.py | 4 ++-- .../extensions/{paas_flask.py => flask.py} | 20 +++++++++---------- 2 files changed, 11 insertions(+), 13 deletions(-) rename rockcraft/extensions/{paas_flask.py => flask.py} (84%) diff --git a/rockcraft/extensions/__init__.py b/rockcraft/extensions/__init__.py index 27e89bcf3..3964cd471 100644 --- a/rockcraft/extensions/__init__.py +++ b/rockcraft/extensions/__init__.py @@ -27,6 +27,6 @@ "unregister", ] -from .paas_flask import PaasFlask as _PaasFlask +from .flask import Flask as _Flask -register("paas-flask", _PaasFlask) +register("flask", _Flask) diff --git a/rockcraft/extensions/paas_flask.py b/rockcraft/extensions/flask.py similarity index 84% rename from rockcraft/extensions/paas_flask.py rename to rockcraft/extensions/flask.py index cb77f09b9..eca823373 100644 --- a/rockcraft/extensions/paas_flask.py +++ b/rockcraft/extensions/flask.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""An experimental extension for the Juju PaaS Flask framework.""" +"""An experimental extension for the Flask framework.""" import os from typing import Any, Dict, Optional, Tuple @@ -24,12 +24,8 @@ from .extension import Extension -class PaasFlask(Extension): - """An extension to enable Flask support within the Juju PaaS ecosystem. - - This extension is a part of the Juju PaaS ecosystem and facilitates the deployment of - Flask-based applications. - """ +class Flask(Extension): + """An extension for constructing Python applications based on the Flask framework.""" @staticmethod @override @@ -74,8 +70,8 @@ def get_parts_snippet(self) -> Dict[str, Any]: """Create necessary parts to facilitate the flask application. Parts added: - - flask-extension/dependencies: install Python dependencies - - flask-extension/install-app: copy the flask project into the OCI image + - flask/dependencies: install Python dependencies + - flask/install-app: copy the flask project into the OCI image """ python_requirements = [] if (self.project_root / "requirements.txt").exists(): @@ -88,14 +84,16 @@ def get_parts_snippet(self) -> Dict[str, Any]: ] renaming_map = {f: os.path.join("srv/flask/app", f) for f in source_files} snippet = { - "paas-flask/dependencies": { + "flask/dependencies": { "plugin": "python", "stage-packages": ["python3-venv"], "source": ".", "python-packages": ["gunicorn"], "python-requirements": python_requirements, }, - "paas-flask/install-app": { + # Users are required to compile any static assets prior to executing the + # rockcraft pack command, so assets can be included in the final OCI image. + "flask/install-app": { "plugin": "dump", "source": ".", "organize": renaming_map, From d5a048dd9a70336d4c5fef9dab889a400de4b712 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 17 Jul 2023 13:42:21 +0800 Subject: [PATCH 07/36] Add a spread test for the flask extension --- tests/spread/general/extension-flask/app.py | 8 ++++++ .../general/extension-flask/requirements.txt | 1 + .../general/extension-flask/rockcraft.yaml | 9 +++++++ .../spread/general/extension-flask/task.yaml | 26 +++++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 tests/spread/general/extension-flask/app.py create mode 100644 tests/spread/general/extension-flask/requirements.txt create mode 100644 tests/spread/general/extension-flask/rockcraft.yaml create mode 100644 tests/spread/general/extension-flask/task.yaml diff --git a/tests/spread/general/extension-flask/app.py b/tests/spread/general/extension-flask/app.py new file mode 100644 index 000000000..536075b59 --- /dev/null +++ b/tests/spread/general/extension-flask/app.py @@ -0,0 +1,8 @@ +from flask import Flask + +app = Flask(__name__) + + +@app.route("/") +def hello_world(): + return "Hello, World!" diff --git a/tests/spread/general/extension-flask/requirements.txt b/tests/spread/general/extension-flask/requirements.txt new file mode 100644 index 000000000..8ab6294c6 --- /dev/null +++ b/tests/spread/general/extension-flask/requirements.txt @@ -0,0 +1 @@ +flask \ No newline at end of file diff --git a/tests/spread/general/extension-flask/rockcraft.yaml b/tests/spread/general/extension-flask/rockcraft.yaml new file mode 100644 index 000000000..b38f0581e --- /dev/null +++ b/tests/spread/general/extension-flask/rockcraft.yaml @@ -0,0 +1,9 @@ +name: flask-extension +summary: OCI image for a flask project. +description: OCI image for a flask project. +version: "0.1" +base: bare +license: Apache-2.0 + +extensions: + - flask \ No newline at end of file diff --git a/tests/spread/general/extension-flask/task.yaml b/tests/spread/general/extension-flask/task.yaml new file mode 100644 index 000000000..9b34910a2 --- /dev/null +++ b/tests/spread/general/extension-flask/task.yaml @@ -0,0 +1,26 @@ +summary: flask extension test + +execute: | + export ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS=true + + run_rockcraft pack + + test -f flask-extension_0.1_amd64.rock + test ! -d work + + # Ensure docker does not have this container image + docker rmi --force flask-extension + # Install container + sudo /snap/rockcraft/current/bin/skopeo --insecure-policy copy oci-archive:flask-extension_0.1_amd64.rock docker-daemon:flask-extension:latest + # Ensure container exists + docker images flask-extension | MATCH "flask-extension" + + # ensure container doesn't exist + docker rm -f flask-extension-container + + # test the flask project is ready to run inside the container + docker run --rm --entrypoint /bin/python3 flask-extension -m gunicorn --chdir /srv/flask/app --check-config app:app + +restore: | + rm -f flask-extension_0.1_amd64.rock + docker rmi -f flask-extension From dde4b336c65eeb6f708af65f8ce68b3029d41891 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 17 Jul 2023 15:09:43 +0800 Subject: [PATCH 08/36] raise an error when requirements.txt doesn't exist --- rockcraft/extensions/flask.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index eca823373..d656e27f4 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -73,9 +73,11 @@ def get_parts_snippet(self) -> Dict[str, Any]: - flask/dependencies: install Python dependencies - flask/install-app: copy the flask project into the OCI image """ - python_requirements = [] - if (self.project_root / "requirements.txt").exists(): - python_requirements.append("requirements.txt") + if not (self.project_root / "requirements.txt").exists(): + raise ValueError( + "missing requirements.txt file, " + "flask extension requires this file with flask specified as a dependency" + ) ignores = [".git", "node_modules"] source_files = [ f @@ -89,7 +91,7 @@ def get_parts_snippet(self) -> Dict[str, Any]: "stage-packages": ["python3-venv"], "source": ".", "python-packages": ["gunicorn"], - "python-requirements": python_requirements, + "python-requirements": ["requirements.txt"], }, # Users are required to compile any static assets prior to executing the # rockcraft pack command, so assets can be included in the final OCI image. From f40b7ddd443aa23e4bf9bedb05db01af2b30d8ef Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 17 Jul 2023 16:02:51 +0800 Subject: [PATCH 09/36] Fix the linting issue --- tests/spread/general/extension-flask/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/spread/general/extension-flask/app.py b/tests/spread/general/extension-flask/app.py index 536075b59..af5ffaabe 100644 --- a/tests/spread/general/extension-flask/app.py +++ b/tests/spread/general/extension-flask/app.py @@ -1,4 +1,4 @@ -from flask import Flask +from flask import Flask # pyright: ignore[reportMissingImports] app = Flask(__name__) From 2f4e560d71ec4af4c162337b37fca413c79bc083 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 24 Jul 2023 23:35:54 +0800 Subject: [PATCH 10/36] Add more items to the flask extension spread test --- tests/spread/general/extension-flask/node_modules/.gitkeep | 0 tests/spread/general/extension-flask/static/js/test.js | 1 + tests/spread/general/extension-flask/task.yaml | 2 ++ 3 files changed, 3 insertions(+) create mode 100644 tests/spread/general/extension-flask/node_modules/.gitkeep create mode 100644 tests/spread/general/extension-flask/static/js/test.js diff --git a/tests/spread/general/extension-flask/node_modules/.gitkeep b/tests/spread/general/extension-flask/node_modules/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/spread/general/extension-flask/static/js/test.js b/tests/spread/general/extension-flask/static/js/test.js new file mode 100644 index 000000000..5be36ccb7 --- /dev/null +++ b/tests/spread/general/extension-flask/static/js/test.js @@ -0,0 +1 @@ +console.log("hello") \ No newline at end of file diff --git a/tests/spread/general/extension-flask/task.yaml b/tests/spread/general/extension-flask/task.yaml index 9b34910a2..0b67b66e2 100644 --- a/tests/spread/general/extension-flask/task.yaml +++ b/tests/spread/general/extension-flask/task.yaml @@ -20,6 +20,8 @@ execute: | # test the flask project is ready to run inside the container docker run --rm --entrypoint /bin/python3 flask-extension -m gunicorn --chdir /srv/flask/app --check-config app:app + docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert pathlib.Path('/srv/flask/app/static/js/test.js').is_file()" + docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert not pathlib.Path('/srv/flask/app/node_modules').exists()" restore: | rm -f flask-extension_0.1_amd64.rock From 21d7dfb5cacb11bed312570bef03979cc4641246 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 26 Jul 2023 12:46:59 +0800 Subject: [PATCH 11/36] Add support for 20.04 in flask extension --- rockcraft/extensions/flask.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index d656e27f4..aef2025be 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -31,7 +31,7 @@ class Flask(Extension): @override def get_supported_bases() -> Tuple[str, ...]: """Return supported bases.""" - return "bare", "ubuntu:22.04" + return "bare", "ubuntu:20.04", "ubuntu:22.04" @staticmethod @override @@ -78,7 +78,7 @@ def get_parts_snippet(self) -> Dict[str, Any]: "missing requirements.txt file, " "flask extension requires this file with flask specified as a dependency" ) - ignores = [".git", "node_modules"] + ignores = [".git", "node_modules", ".yarn"] source_files = [ f for f in os.listdir(self.project_root) From 310408fe6b4c384fb7d1d956a372d7e605681e08 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 27 Jul 2023 10:58:21 +0800 Subject: [PATCH 12/36] Add extensions section in the rockcraft.yaml doc --- docs/reference/rockcraft.yaml.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/reference/rockcraft.yaml.rst b/docs/reference/rockcraft.yaml.rst index 8a5fbc8e3..56b499c9f 100644 --- a/docs/reference/rockcraft.yaml.rst +++ b/docs/reference/rockcraft.yaml.rst @@ -198,6 +198,18 @@ The set of parts that compose the ROCK's contents Rockcraft. All ROCKs have Pebble as their entrypoint, and thus you must use ``services`` to define your container application. +``extensions`` +--------- + +**Type**: list[string] + +**Required**: No + +Extensions to apply to this rock image. + +Currently supported extensions: + +- ``flask`` (experimental) Example ======= From 6f1a6277a549d1ccdaef81d27d6d021e36313e94 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 10 Aug 2023 12:41:13 +0800 Subject: [PATCH 13/36] Update document and part merging --- docs/how-to/use-flask-extension.rst | 58 +++++++++++++++++++++++++++++ rockcraft/extensions/flask.py | 20 ++++++---- 2 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 docs/how-to/use-flask-extension.rst diff --git a/docs/how-to/use-flask-extension.rst b/docs/how-to/use-flask-extension.rst new file mode 100644 index 000000000..2aee44989 --- /dev/null +++ b/docs/how-to/use-flask-extension.rst @@ -0,0 +1,58 @@ +The Flask Extension +******************* + +The Flask extension streamlines the process of building Flask application rocks. + +It facilitates the installation of Flask application dependencies, including +Gunicorn, in the rock image. Additionally, it transfers your project files to +``/srv/flask/app`` within the rock image. + +Using the Flask Extension +------------------------- + +The Flask extension is compatible with ``bare``, ``ubuntu:20.04``, and +``ubuntu:22.04``. To employ it, include ``extensions: [flask]`` in your +``rockcraft.yaml`` file. + +Example: + +.. code-block:: yaml + + name: example + summary: Example. + description: Example. + version: "0.1" + base: bare + license: Apache-2.0 + + extensions: + - flask + +Managing Project Files with the Flask Extension +---------------------------------------------- + +By default, all files within the Flask project directory are copied, excluding +certain common files and directories, such as ``node_modules``. However, +this behavior can be tailored to either specifically include or exclude files +from the Flask project directory in the rock image. + +To include only select files from the project directory in the rock image, +append the following part to ``rockcraft.yaml``: + +.. code-block:: yaml + + flask/install-app: + prime: + - srv/flask/app/static + - srv/flask/app/.env + - srv/flask/app/webapp + - srv/flask/app/templates + +To exclude certain files from the project directory in the rock image, +add the following part to ``rockcraft.yaml``: + +.. code-block:: yaml + + flask/install-app: + prime: + - -srv/flask/app/charmcraft.auth diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index aef2025be..71d933ece 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -85,6 +85,17 @@ def get_parts_snippet(self) -> Dict[str, Any]: if f not in ignores and not f.endswith(".rock") ] renaming_map = {f: os.path.join("srv/flask/app", f) for f in source_files} + # Users are required to compile any static assets prior to executing the + # rockcraft pack command, so assets can be included in the final OCI image. + install_app_part = { + "plugin": "dump", + "source": ".", + "organize": renaming_map, + "stage": list(renaming_map.values()), + } + install_app_part_name = "flask/install-app" + existing_install_app_part = self.yaml_data.get("parts", {}).get(install_app_part_name, {}) + install_app_part.update(existing_install_app_part) snippet = { "flask/dependencies": { "plugin": "python", @@ -93,13 +104,6 @@ def get_parts_snippet(self) -> Dict[str, Any]: "python-packages": ["gunicorn"], "python-requirements": ["requirements.txt"], }, - # Users are required to compile any static assets prior to executing the - # rockcraft pack command, so assets can be included in the final OCI image. - "flask/install-app": { - "plugin": "dump", - "source": ".", - "organize": renaming_map, - "stage": list(renaming_map.values()), - }, + "flask/install-app": install_app_part, } return snippet From 8c2580721e0d03afb3596d728efa6a4f286caed6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 10 Aug 2023 14:39:48 +0800 Subject: [PATCH 14/36] Only allow to overwrite prime in flask/install-app --- rockcraft/extensions/flask.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 71d933ece..28d58ea1c 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -95,7 +95,9 @@ def get_parts_snippet(self) -> Dict[str, Any]: } install_app_part_name = "flask/install-app" existing_install_app_part = self.yaml_data.get("parts", {}).get(install_app_part_name, {}) - install_app_part.update(existing_install_app_part) + prime = existing_install_app_part.get("prime") + if prime: + install_app_part["prime"] = prime snippet = { "flask/dependencies": { "plugin": "python", From 1e8f860af2ab49c47653ebf9973f9e1b21679a8a Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Sun, 13 Aug 2023 14:33:00 +0800 Subject: [PATCH 15/36] Update part merging and documents --- docs/how-to/use-flask-extension.rst | 15 ++---- docs/reference/extensions.rst | 16 ++++++ rockcraft/extensions/_utils.py | 8 +-- rockcraft/extensions/flask.py | 49 ++++++++++++++----- tests/spread/general/extension-flask/README | 1 + .../general/extension-flask/rockcraft.yaml | 7 ++- .../spread/general/extension-flask/task.yaml | 3 ++ 7 files changed, 69 insertions(+), 30 deletions(-) create mode 100644 docs/reference/extensions.rst create mode 100644 tests/spread/general/extension-flask/README diff --git a/docs/how-to/use-flask-extension.rst b/docs/how-to/use-flask-extension.rst index 2aee44989..eab8dc1a1 100644 --- a/docs/how-to/use-flask-extension.rst +++ b/docs/how-to/use-flask-extension.rst @@ -1,13 +1,4 @@ -The Flask Extension -******************* - -The Flask extension streamlines the process of building Flask application rocks. - -It facilitates the installation of Flask application dependencies, including -Gunicorn, in the rock image. Additionally, it transfers your project files to -``/srv/flask/app`` within the rock image. - -Using the Flask Extension +Using the flask extension ------------------------- The Flask extension is compatible with ``bare``, ``ubuntu:20.04``, and @@ -28,8 +19,8 @@ Example: extensions: - flask -Managing Project Files with the Flask Extension ----------------------------------------------- +Managing project files with the flask extension +----------------------------------------------- By default, all files within the Flask project directory are copied, excluding certain common files and directories, such as ``node_modules``. However, diff --git a/docs/reference/extensions.rst b/docs/reference/extensions.rst new file mode 100644 index 000000000..c2cd9afd2 --- /dev/null +++ b/docs/reference/extensions.rst @@ -0,0 +1,16 @@ +Extensions +********** + +Just as the snapcraft extensions are designed to simplify snap creation, +Rockcraft extensions are crafted to expand and modify the user-provided +rockcraft manifest file, aiming to minimize the boilerplate code when +initiating a new rock. + +The flask extension +------------------- + +The Flask extension streamlines the process of building Flask application rocks. + +It facilitates the installation of Flask application dependencies, including +Gunicorn, in the rock image. Additionally, it transfers your project files to +``/srv/flask/app`` within the rock image. diff --git a/rockcraft/extensions/_utils.py b/rockcraft/extensions/_utils.py index ac9a28e7a..47cae40ef 100644 --- a/rockcraft/extensions/_utils.py +++ b/rockcraft/extensions/_utils.py @@ -56,7 +56,7 @@ def _apply_extension( # Apply the root components of the extension (if any) root_extension = extension.get_root_snippet() for property_name, property_value in root_extension.items(): - yaml_data[property_name] = _apply_extension_property( + yaml_data[property_name] = apply_extension_property( yaml_data.get(property_name), property_value ) @@ -68,7 +68,7 @@ def _apply_extension( parts = yaml_data["parts"] for _, part_definition in parts.items(): for property_name, property_value in part_extension.items(): - part_definition[property_name] = _apply_extension_property( + part_definition[property_name] = apply_extension_property( part_definition.get(property_name), property_value ) @@ -79,7 +79,7 @@ def _apply_extension( parts[part_name] = parts_snippet[part_name] -def _apply_extension_property(existing_property: Any, extension_property: Any) -> Any: +def apply_extension_property(existing_property: Any, extension_property: Any) -> Any: if existing_property: # If the property is not scalar, merge them if isinstance(existing_property, list) and isinstance(extension_property, list): @@ -93,7 +93,7 @@ def _apply_extension_property(existing_property: Any, extension_property: Any) - if isinstance(existing_property, dict) and isinstance(extension_property, dict): for key, value in extension_property.items(): - existing_property[key] = _apply_extension_property( + existing_property[key] = apply_extension_property( existing_property.get(key), value ) return existing_property diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 28d58ea1c..65332ad42 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -22,6 +22,7 @@ from overrides import override from .extension import Extension +from ._utils import apply_extension_property class Flask(Extension): @@ -65,6 +66,26 @@ def get_part_snippet(self) -> Dict[str, Any]: """Return the part snippet to apply to existing parts.""" return {} + def _merge_part(self, base_part: dict, new_part: dict) -> dict: + """Merge two part definitions by the extension part merging rule.""" + result = {} + properties = set(base_part.keys()).union(set(new_part.keys())) + for property_name in properties: + if property_name in base_part and property_name not in new_part: + result[property_name] = base_part[property_name] + elif property_name not in base_part and property_name in new_part: + result[property_name] = new_part[property_name] + else: + result[property_name] = apply_extension_property( + base_part[property_name], new_part[property_name] + ) + return result + + def _merge_existing_part(self, part_name: str, part_def: dict) -> dict: + """Merge the new part with the existing part in the current rockcraft.yaml.""" + existing_part = self.yaml_data.get("parts", {}).get(part_name, {}) + return self._merge_part(existing_part, part_def) + @override def get_parts_snippet(self) -> Dict[str, Any]: """Create necessary parts to facilitate the flask application. @@ -85,6 +106,8 @@ def get_parts_snippet(self) -> Dict[str, Any]: if f not in ignores and not f.endswith(".rock") ] renaming_map = {f: os.path.join("srv/flask/app", f) for f in source_files} + install_app_part_name = "flask/install-app" + dependencies_part_name = "flask/dependencies" # Users are required to compile any static assets prior to executing the # rockcraft pack command, so assets can be included in the final OCI image. install_app_part = { @@ -93,19 +116,19 @@ def get_parts_snippet(self) -> Dict[str, Any]: "organize": renaming_map, "stage": list(renaming_map.values()), } - install_app_part_name = "flask/install-app" - existing_install_app_part = self.yaml_data.get("parts", {}).get(install_app_part_name, {}) - prime = existing_install_app_part.get("prime") - if prime: - install_app_part["prime"] = prime + dependencies_part = { + "plugin": "python", + "stage-packages": ["python3-venv"], + "source": ".", + "python-packages": ["gunicorn"], + "python-requirements": ["requirements.txt"], + } snippet = { - "flask/dependencies": { - "plugin": "python", - "stage-packages": ["python3-venv"], - "source": ".", - "python-packages": ["gunicorn"], - "python-requirements": ["requirements.txt"], - }, - "flask/install-app": install_app_part, + dependencies_part_name: self._merge_existing_part( + dependencies_part_name, dependencies_part + ), + install_app_part_name: self._merge_existing_part( + install_app_part_name, install_app_part + ), } return snippet diff --git a/tests/spread/general/extension-flask/README b/tests/spread/general/extension-flask/README new file mode 100644 index 000000000..30d74d258 --- /dev/null +++ b/tests/spread/general/extension-flask/README @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/tests/spread/general/extension-flask/rockcraft.yaml b/tests/spread/general/extension-flask/rockcraft.yaml index b38f0581e..c0f78f0cb 100644 --- a/tests/spread/general/extension-flask/rockcraft.yaml +++ b/tests/spread/general/extension-flask/rockcraft.yaml @@ -6,4 +6,9 @@ base: bare license: Apache-2.0 extensions: - - flask \ No newline at end of file + - flask + +parts: + flask/install-app: + prime: + - -README diff --git a/tests/spread/general/extension-flask/task.yaml b/tests/spread/general/extension-flask/task.yaml index 0b67b66e2..8be2d8c37 100644 --- a/tests/spread/general/extension-flask/task.yaml +++ b/tests/spread/general/extension-flask/task.yaml @@ -22,6 +22,9 @@ execute: | docker run --rm --entrypoint /bin/python3 flask-extension -m gunicorn --chdir /srv/flask/app --check-config app:app docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert pathlib.Path('/srv/flask/app/static/js/test.js').is_file()" docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert not pathlib.Path('/srv/flask/app/node_modules').exists()" + + # test the part merging + docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert not pathlib.Path('/srv/flask/app/README').exists()" restore: | rm -f flask-extension_0.1_amd64.rock From 2830f97f8bc069b742b5dea4b2b55e6e47058c92 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 14 Aug 2023 12:31:28 +0800 Subject: [PATCH 16/36] Use root snippet to modify parts --- rockcraft/extensions/flask.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 65332ad42..8b8676d2b 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -16,13 +16,14 @@ """An experimental extension for the Flask framework.""" +import copy import os from typing import Any, Dict, Optional, Tuple from overrides import override -from .extension import Extension from ._utils import apply_extension_property +from .extension import Extension class Flask(Extension): @@ -59,6 +60,9 @@ def get_root_snippet(self) -> Dict[str, Any]: snippet["build-base"] = "ubuntu:22.04" if "platforms" not in self.yaml_data: snippet["platforms"] = {"amd64": {}} + current_parts = copy.deepcopy(self.yaml_data.get("parts", {})) + current_parts.update(self._gen_new_parts()) + snippet["parts"] = current_parts return snippet @override @@ -86,9 +90,8 @@ def _merge_existing_part(self, part_name: str, part_def: dict) -> dict: existing_part = self.yaml_data.get("parts", {}).get(part_name, {}) return self._merge_part(existing_part, part_def) - @override - def get_parts_snippet(self) -> Dict[str, Any]: - """Create necessary parts to facilitate the flask application. + def _gen_new_parts(self) -> Dict[str, Any]: + """New parts generated by the flask extension. Parts added: - flask/dependencies: install Python dependencies @@ -132,3 +135,8 @@ def get_parts_snippet(self) -> Dict[str, Any]: ), } return snippet + + @override + def get_parts_snippet(self) -> Dict[str, Any]: + """Create necessary parts to facilitate the flask application.""" + return {} From d5890cddc8985b9ff7e4d641858075f5cda730a8 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 14 Aug 2023 15:45:54 +0800 Subject: [PATCH 17/36] Fix the spread test for the flask extension --- rockcraft/extensions/flask.py | 2 +- tests/spread/general/extension-flask/rockcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 8b8676d2b..24edf4d98 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -91,7 +91,7 @@ def _merge_existing_part(self, part_name: str, part_def: dict) -> dict: return self._merge_part(existing_part, part_def) def _gen_new_parts(self) -> Dict[str, Any]: - """New parts generated by the flask extension. + """Generate new parts for the flask extension. Parts added: - flask/dependencies: install Python dependencies diff --git a/tests/spread/general/extension-flask/rockcraft.yaml b/tests/spread/general/extension-flask/rockcraft.yaml index c0f78f0cb..00d204fac 100644 --- a/tests/spread/general/extension-flask/rockcraft.yaml +++ b/tests/spread/general/extension-flask/rockcraft.yaml @@ -11,4 +11,4 @@ extensions: parts: flask/install-app: prime: - - -README + - -srv/flask/app/README From efbfa3bf90be799293961fd6a9aa3145d22de6d6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Tue, 22 Aug 2023 19:53:57 +0800 Subject: [PATCH 18/36] Add services in flask extension and fix documents --- docs/reference/rockcraft.yaml.rst | 2 +- rockcraft/extensions/flask.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/reference/rockcraft.yaml.rst b/docs/reference/rockcraft.yaml.rst index 56b499c9f..2cbc00f64 100644 --- a/docs/reference/rockcraft.yaml.rst +++ b/docs/reference/rockcraft.yaml.rst @@ -199,7 +199,7 @@ The set of parts that compose the ROCK's contents ``services`` to define your container application. ``extensions`` ---------- +-------------- **Type**: list[string] diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 24edf4d98..2ffe8a4ce 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -63,8 +63,28 @@ def get_root_snippet(self) -> Dict[str, Any]: current_parts = copy.deepcopy(self.yaml_data.get("parts", {})) current_parts.update(self._gen_new_parts()) snippet["parts"] = current_parts + snippet["services"] = self._gen_services() return snippet + def _gen_services(self): + """Return the services snipped to be applied to the rockcraft file.""" + services = { + "flask": { + "override": "replace", + "startup": "enabled", + "command": "/bin/python3 -m gunicorn app:app", + "user": "_daemon_", + "working-dir": "/srv/flask/app" + } + } + existing_services = self.yaml_data.get("services", {}) + for existing_service_name, existing_service in existing_services.items(): + if existing_service_name in services: + services[existing_service_name].update(existing_service) + else: + services[existing_service_name] = existing_service + return services + @override def get_part_snippet(self) -> Dict[str, Any]: """Return the part snippet to apply to existing parts.""" From 781eab6bb9f71921e7426810974f5bc213e3ab95 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 23 Aug 2023 00:36:32 +0800 Subject: [PATCH 19/36] Add tmp in bare containers and fix docs --- docs/how-to/index.rst | 1 + docs/reference/index.rst | 1 + rockcraft/extensions/flask.py | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst index 3d3c19649..830fb75e2 100644 --- a/docs/how-to/index.rst +++ b/docs/how-to/index.rst @@ -22,3 +22,4 @@ adapt the steps to fit your specific requirements. Convert an entrypoint to a Pebble layer contribute-docs use-chisel + Use flask extension diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 502cba186..99b218c47 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -14,3 +14,4 @@ Rockcraft's components, commands and keywords. parts commands part_properties + extensions diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 2ffe8a4ce..69773c4c9 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -72,9 +72,9 @@ def _gen_services(self): "flask": { "override": "replace", "startup": "enabled", - "command": "/bin/python3 -m gunicorn app:app", + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 app:app", "user": "_daemon_", - "working-dir": "/srv/flask/app" + "working-dir": "/srv/flask/app", } } existing_services = self.yaml_data.get("services", {}) @@ -154,6 +154,16 @@ def _gen_new_parts(self) -> Dict[str, Any]: install_app_part_name, install_app_part ), } + if self.yaml_data["base"] == "bare": + snippet["flask/container-processing"] = { + "plugin": "nil", + "source": ".", + "override-prime": ( + "craftctl default\n" + "mkdir -p tmp\n" + "chmod 777 tmp" + ), + } return snippet @override From 97aa7e281fd20ab2ba75b61e67be113ab3c56981 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 23 Aug 2023 00:45:52 +0800 Subject: [PATCH 20/36] Fix some linting problems --- rockcraft/extensions/_utils.py | 8 ++++---- rockcraft/extensions/flask.py | 10 +++------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/rockcraft/extensions/_utils.py b/rockcraft/extensions/_utils.py index 47cae40ef..ac9a28e7a 100644 --- a/rockcraft/extensions/_utils.py +++ b/rockcraft/extensions/_utils.py @@ -56,7 +56,7 @@ def _apply_extension( # Apply the root components of the extension (if any) root_extension = extension.get_root_snippet() for property_name, property_value in root_extension.items(): - yaml_data[property_name] = apply_extension_property( + yaml_data[property_name] = _apply_extension_property( yaml_data.get(property_name), property_value ) @@ -68,7 +68,7 @@ def _apply_extension( parts = yaml_data["parts"] for _, part_definition in parts.items(): for property_name, property_value in part_extension.items(): - part_definition[property_name] = apply_extension_property( + part_definition[property_name] = _apply_extension_property( part_definition.get(property_name), property_value ) @@ -79,7 +79,7 @@ def _apply_extension( parts[part_name] = parts_snippet[part_name] -def apply_extension_property(existing_property: Any, extension_property: Any) -> Any: +def _apply_extension_property(existing_property: Any, extension_property: Any) -> Any: if existing_property: # If the property is not scalar, merge them if isinstance(existing_property, list) and isinstance(extension_property, list): @@ -93,7 +93,7 @@ def apply_extension_property(existing_property: Any, extension_property: Any) -> if isinstance(existing_property, dict) and isinstance(extension_property, dict): for key, value in extension_property.items(): - existing_property[key] = apply_extension_property( + existing_property[key] = _apply_extension_property( existing_property.get(key), value ) return existing_property diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 69773c4c9..eacdc9679 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -22,7 +22,7 @@ from overrides import override -from ._utils import apply_extension_property +from ._utils import _apply_extension_property from .extension import Extension @@ -100,7 +100,7 @@ def _merge_part(self, base_part: dict, new_part: dict) -> dict: elif property_name not in base_part and property_name in new_part: result[property_name] = new_part[property_name] else: - result[property_name] = apply_extension_property( + result[property_name] = _apply_extension_property( base_part[property_name], new_part[property_name] ) return result @@ -158,11 +158,7 @@ def _gen_new_parts(self) -> Dict[str, Any]: snippet["flask/container-processing"] = { "plugin": "nil", "source": ".", - "override-prime": ( - "craftctl default\n" - "mkdir -p tmp\n" - "chmod 777 tmp" - ), + "override-prime": "craftctl default\nmkdir -p tmp\nchmod 777 tmp", } return snippet From 75049def06a7f86623b5b21147979b870981fcd6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 23 Aug 2023 01:12:45 +0800 Subject: [PATCH 21/36] Change to British English and update wordlist --- docs/.wordlist.txt | 1 + docs/how-to/use-flask-extension.rst | 2 +- docs/reference/extensions.rst | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/.wordlist.txt b/docs/.wordlist.txt index 2d054c0b6..5a4c57976 100644 --- a/docs/.wordlist.txt +++ b/docs/.wordlist.txt @@ -27,6 +27,7 @@ fs GID github GPG +Gunicorn https init interoperable diff --git a/docs/how-to/use-flask-extension.rst b/docs/how-to/use-flask-extension.rst index eab8dc1a1..82737176d 100644 --- a/docs/how-to/use-flask-extension.rst +++ b/docs/how-to/use-flask-extension.rst @@ -24,7 +24,7 @@ Managing project files with the flask extension By default, all files within the Flask project directory are copied, excluding certain common files and directories, such as ``node_modules``. However, -this behavior can be tailored to either specifically include or exclude files +this behaviour can be tailored to either specifically include or exclude files from the Flask project directory in the rock image. To include only select files from the project directory in the rock image, diff --git a/docs/reference/extensions.rst b/docs/reference/extensions.rst index c2cd9afd2..73e6c0e15 100644 --- a/docs/reference/extensions.rst +++ b/docs/reference/extensions.rst @@ -3,7 +3,7 @@ Extensions Just as the snapcraft extensions are designed to simplify snap creation, Rockcraft extensions are crafted to expand and modify the user-provided -rockcraft manifest file, aiming to minimize the boilerplate code when +rockcraft manifest file, aiming to minimise the boilerplate code when initiating a new rock. The flask extension From 10ec23d2a57d1586cb39ae8c51caeb509349be42 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Tue, 29 Aug 2023 00:37:32 +0800 Subject: [PATCH 22/36] Add some unit tests for flask extension --- tests/unit/extensions/test_extensions.py | 99 ++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tests/unit/extensions/test_extensions.py b/tests/unit/extensions/test_extensions.py index 4214ed0f0..99cb42947 100644 --- a/tests/unit/extensions/test_extensions.py +++ b/tests/unit/extensions/test_extensions.py @@ -134,3 +134,102 @@ def test_project_load_extensions(fake_extensions, tmp_path): # New part assert parts[f"{FullExtension.NAME}/new-part"] == {"plugin": "nil", "source": None} + + +def test_flask_extensions(fake_extensions, tmp_path, input_yaml, monkeypatch): + monkeypatch.setenv("ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", "1") + extensions.register("flask", extensions.flask.Flask) + + (tmp_path / "requirements.txt").write_text("flask") + (tmp_path / "app.py").touch() + (tmp_path / "static").mkdir() + (tmp_path / "node_modules").mkdir() + + input_yaml["extensions"] = ["flask"] + + applied = extensions.apply_extensions(tmp_path, input_yaml) + + assert applied["run_user"] == "_daemon_" + assert applied["platforms"] == {"amd64": {}} + + # Root snippet extends the project's + services = applied["services"] + assert services["flask"] == { + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 app:app", + "override": "replace", + "startup": "enabled", + "user": "_daemon_", + "working-dir": "/srv/flask/app", + } + + parts = applied["parts"] + assert sorted(parts["flask/install-app"]["stage"]) == [ + "srv/flask/app/app.py", + "srv/flask/app/requirements.txt", + "srv/flask/app/static", + ] + del parts["flask/install-app"]["stage"] + assert parts == { + "flask/dependencies": { + "source": ".", + "plugin": "python", + "python-requirements": ["requirements.txt"], + "stage-packages": ["python3-venv"], + "python-packages": ["gunicorn"], + }, + "flask/install-app": { + "source": ".", + "organize": { + "app.py": "srv/flask/app/app.py", + "static": "srv/flask/app/static", + "requirements.txt": "srv/flask/app/requirements.txt", + }, + "plugin": "dump", + }, + } + + +def test_flask_extensions_overwrite(fake_extensions, tmp_path, input_yaml, monkeypatch): + monkeypatch.setenv("ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", "1") + extensions.register("flask", extensions.flask.Flask) + + (tmp_path / "requirements.txt").write_text("flask") + (tmp_path / "foobar").touch() + (tmp_path / "webapp").mkdir() + (tmp_path / "webapp/app.py").touch() + (tmp_path / "static").mkdir() + (tmp_path / "node_modules").mkdir() + + input_yaml["extensions"] = ["flask"] + input_yaml["parts"] = {"flask/install-app": {"prime": ["-srv/flask/app/foobar"]}} + input_yaml["services"] = { + "flask": { + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 webapp.app:app" + } + } + applied = extensions.apply_extensions(tmp_path, input_yaml) + + assert applied["services"] == { + "flask": { + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 webapp.app:app", + "override": "replace", + "startup": "enabled", + "user": "_daemon_", + "working-dir": "/srv/flask/app", + } + } + assert applied["parts"]["flask/install-app"]["prime"] == ["-srv/flask/app/foobar"] + + +def test_flask_extensions_bare(fake_extensions, tmp_path, monkeypatch): + monkeypatch.setenv("ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", "1") + extensions.register("flask", extensions.flask.Flask) + + (tmp_path / "requirements.txt").write_text("flask") + input_yaml = {"extensions": ["flask"], "base": "bare"} + applied = extensions.apply_extensions(tmp_path, input_yaml) + assert applied["parts"]["flask/container-processing"] == { + "plugin": "nil", + "source": ".", + "override-prime": "craftctl default\nmkdir -p tmp\nchmod 777 tmp", + } From 3212eae213d21fbaad08611b01e03e74834db395 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Tue, 29 Aug 2023 00:43:24 +0800 Subject: [PATCH 23/36] Fix the flask extension on windows --- rockcraft/extensions/flask.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index eacdc9679..0560e81b5 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -18,6 +18,7 @@ import copy import os +import posixpath from typing import Any, Dict, Optional, Tuple from overrides import override @@ -128,7 +129,7 @@ def _gen_new_parts(self) -> Dict[str, Any]: for f in os.listdir(self.project_root) if f not in ignores and not f.endswith(".rock") ] - renaming_map = {f: os.path.join("srv/flask/app", f) for f in source_files} + renaming_map = {f: posixpath.join("srv/flask/app", f) for f in source_files} install_app_part_name = "flask/install-app" dependencies_part_name = "flask/dependencies" # Users are required to compile any static assets prior to executing the From ce2ce3e159153f211cf07997858d7a3ee552715a Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 31 Aug 2023 14:08:38 +0800 Subject: [PATCH 24/36] Apply suggestions from code review Co-authored-by: Tiago Nobrega --- docs/how-to/index.rst | 2 +- docs/how-to/use-flask-extension.rst | 10 +++++----- docs/reference/extensions.rst | 4 ++-- docs/reference/rockcraft.yaml.rst | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst index 830fb75e2..48592fae7 100644 --- a/docs/how-to/index.rst +++ b/docs/how-to/index.rst @@ -22,4 +22,4 @@ adapt the steps to fit your specific requirements. Convert an entrypoint to a Pebble layer contribute-docs use-chisel - Use flask extension + Use the flask extension diff --git a/docs/how-to/use-flask-extension.rst b/docs/how-to/use-flask-extension.rst index 82737176d..98524aeff 100644 --- a/docs/how-to/use-flask-extension.rst +++ b/docs/how-to/use-flask-extension.rst @@ -1,15 +1,15 @@ Using the flask extension ------------------------- -The Flask extension is compatible with ``bare``, ``ubuntu:20.04``, and -``ubuntu:22.04``. To employ it, include ``extensions: [flask]`` in your +The Flask extension is compatible with the ``bare``, ``ubuntu:20.04``, and +``ubuntu:22.04`` bases. To employ it, include ``extensions: [flask]`` in your ``rockcraft.yaml`` file. Example: .. code-block:: yaml - name: example + name: example-flask summary: Example. description: Example. version: "0.1" @@ -25,9 +25,9 @@ Managing project files with the flask extension By default, all files within the Flask project directory are copied, excluding certain common files and directories, such as ``node_modules``. However, this behaviour can be tailored to either specifically include or exclude files -from the Flask project directory in the rock image. +from the Flask project directory in the ROCK image. -To include only select files from the project directory in the rock image, +To include only select files from the project directory in the ROCK image, append the following part to ``rockcraft.yaml``: .. code-block:: yaml diff --git a/docs/reference/extensions.rst b/docs/reference/extensions.rst index 73e6c0e15..18d5d5e2f 100644 --- a/docs/reference/extensions.rst +++ b/docs/reference/extensions.rst @@ -1,12 +1,12 @@ Extensions ********** -Just as the snapcraft extensions are designed to simplify snap creation, +Just as the Snapcraft extensions are designed to simplify Snap creation, Rockcraft extensions are crafted to expand and modify the user-provided rockcraft manifest file, aiming to minimise the boilerplate code when initiating a new rock. -The flask extension +The ``flask`` extension ------------------- The Flask extension streamlines the process of building Flask application rocks. diff --git a/docs/reference/rockcraft.yaml.rst b/docs/reference/rockcraft.yaml.rst index c28ea241d..04fd0f4f6 100644 --- a/docs/reference/rockcraft.yaml.rst +++ b/docs/reference/rockcraft.yaml.rst @@ -218,7 +218,7 @@ The set of parts that compose the ROCK's contents **Required**: No -Extensions to apply to this rock image. +Extensions to enable when building the ROCK. Currently supported extensions: From 6e3f6820584deb88b323e011ebfd481245b49d22 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 31 Aug 2023 15:41:00 +0800 Subject: [PATCH 25/36] Apply suggestions from code review --- docs/how-to/use-flask-extension.rst | 5 +++-- docs/reference/extensions.rst | 2 +- rockcraft/extensions/__init__.py | 5 ++--- rockcraft/extensions/flask.py | 4 ++-- tests/unit/extensions/test_extensions.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/how-to/use-flask-extension.rst b/docs/how-to/use-flask-extension.rst index 98524aeff..9841f20f1 100644 --- a/docs/how-to/use-flask-extension.rst +++ b/docs/how-to/use-flask-extension.rst @@ -27,8 +27,9 @@ certain common files and directories, such as ``node_modules``. However, this behaviour can be tailored to either specifically include or exclude files from the Flask project directory in the ROCK image. -To include only select files from the project directory in the ROCK image, -append the following part to ``rockcraft.yaml``: +You can include and exclude files from the project directory in the ROCK image +by using the standard prime declaration on the specially-named +``flask/install-app`` part. For example, to include only select files: .. code-block:: yaml diff --git a/docs/reference/extensions.rst b/docs/reference/extensions.rst index 18d5d5e2f..20069767b 100644 --- a/docs/reference/extensions.rst +++ b/docs/reference/extensions.rst @@ -3,7 +3,7 @@ Extensions Just as the Snapcraft extensions are designed to simplify Snap creation, Rockcraft extensions are crafted to expand and modify the user-provided -rockcraft manifest file, aiming to minimise the boilerplate code when +rockcraft project file, aiming to minimise the boilerplate code when initiating a new rock. The ``flask`` extension diff --git a/rockcraft/extensions/__init__.py b/rockcraft/extensions/__init__.py index 3964cd471..be49e9451 100644 --- a/rockcraft/extensions/__init__.py +++ b/rockcraft/extensions/__init__.py @@ -17,6 +17,7 @@ """Extension processor and related utilities.""" from ._utils import apply_extensions +from .flask import Flask from .registry import get_extension_class, get_extension_names, register, unregister __all__ = [ @@ -27,6 +28,4 @@ "unregister", ] -from .flask import Flask as _Flask - -register("flask", _Flask) +register("flask", Flask) diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 0560e81b5..1803ad774 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -159,11 +159,11 @@ def _gen_new_parts(self) -> Dict[str, Any]: snippet["flask/container-processing"] = { "plugin": "nil", "source": ".", - "override-prime": "craftctl default\nmkdir -p tmp\nchmod 777 tmp", + "override-build": "mkdir -m 777 ${CRAFT_PART_INSTALL}/tmp", } return snippet @override def get_parts_snippet(self) -> Dict[str, Any]: - """Create necessary parts to facilitate the flask application.""" + """Return the parts to add to parts.""" return {} diff --git a/tests/unit/extensions/test_extensions.py b/tests/unit/extensions/test_extensions.py index 99cb42947..1494b3d13 100644 --- a/tests/unit/extensions/test_extensions.py +++ b/tests/unit/extensions/test_extensions.py @@ -231,5 +231,5 @@ def test_flask_extensions_bare(fake_extensions, tmp_path, monkeypatch): assert applied["parts"]["flask/container-processing"] == { "plugin": "nil", "source": ".", - "override-prime": "craftctl default\nmkdir -p tmp\nchmod 777 tmp", + "override-build": "mkdir -m 777 ${CRAFT_PART_INSTALL}/tmp", } From 463e29893224ba517bf870920890085a01f4453e Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 31 Aug 2023 15:45:23 +0800 Subject: [PATCH 26/36] Update documents --- docs/how-to/use-flask-extension.rst | 4 ++-- docs/reference/extensions.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/how-to/use-flask-extension.rst b/docs/how-to/use-flask-extension.rst index 9841f20f1..84716829a 100644 --- a/docs/how-to/use-flask-extension.rst +++ b/docs/how-to/use-flask-extension.rst @@ -10,8 +10,8 @@ Example: .. code-block:: yaml name: example-flask - summary: Example. - description: Example. + summary: A Flask application + description: A ROCK packing a Flask application via the flask extension version: "0.1" base: bare license: Apache-2.0 diff --git a/docs/reference/extensions.rst b/docs/reference/extensions.rst index 20069767b..01c814e7c 100644 --- a/docs/reference/extensions.rst +++ b/docs/reference/extensions.rst @@ -7,7 +7,7 @@ rockcraft project file, aiming to minimise the boilerplate code when initiating a new rock. The ``flask`` extension -------------------- +----------------------- The Flask extension streamlines the process of building Flask application rocks. From cab47e831e1861b3f0b6ab788ac16f65fad23b6b Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 1 Sep 2023 14:46:05 +0800 Subject: [PATCH 27/36] Mandate the prime in flask/install-app --- docs/how-to/use-flask-extension.rst | 23 ++- rockcraft/extensions/flask.py | 32 ++- tests/spread/general/extension-flask/app.py | 4 +- .../spread/general/extension-flask/task.yaml | 5 + tests/unit/extensions/test_extensions.py | 99 ---------- tests/unit/extensions/test_flask.py | 184 ++++++++++++++++++ 6 files changed, 229 insertions(+), 118 deletions(-) create mode 100644 tests/unit/extensions/test_flask.py diff --git a/docs/how-to/use-flask-extension.rst b/docs/how-to/use-flask-extension.rst index 84716829a..31c7331ae 100644 --- a/docs/how-to/use-flask-extension.rst +++ b/docs/how-to/use-flask-extension.rst @@ -19,17 +19,21 @@ Example: extensions: - flask + flask/install-app: + prime: + - -srv/flask/app/.git + - -srv/flask/app/.venv + - -srv/flask/app/.yarn + - -srv/flask/app/node_modules + Managing project files with the flask extension ----------------------------------------------- -By default, all files within the Flask project directory are copied, excluding -certain common files and directories, such as ``node_modules``. However, -this behaviour can be tailored to either specifically include or exclude files -from the Flask project directory in the ROCK image. +The prime declaration must be included in the specially-named +``flask/install-app`` section to instruct the flask extension on which files +to include or exclude from the project directory in the ROCK image. -You can include and exclude files from the project directory in the ROCK image -by using the standard prime declaration on the specially-named -``flask/install-app`` part. For example, to include only select files: +For example, to include only select files: .. code-block:: yaml @@ -47,4 +51,7 @@ add the following part to ``rockcraft.yaml``: flask/install-app: prime: - - -srv/flask/app/charmcraft.auth + - -srv/flask/app/.git + - -srv/flask/app/.venv + - -srv/flask/app/.yarn + - -srv/flask/app/node_modules diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 1803ad774..f1a2133c2 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -17,14 +17,15 @@ """An experimental extension for the Flask framework.""" import copy -import os import posixpath +import re from typing import Any, Dict, Optional, Tuple from overrides import override from ._utils import _apply_extension_property from .extension import Extension +from ..errors import ExtensionError class Flask(Extension): @@ -78,7 +79,7 @@ def _gen_services(self): "working-dir": "/srv/flask/app", } } - existing_services = self.yaml_data.get("services", {}) + existing_services = copy.deepcopy(self.yaml_data.get("services", {})) for existing_service_name, existing_service in existing_services.items(): if existing_service_name in services: services[existing_service_name].update(existing_service) @@ -119,19 +120,32 @@ def _gen_new_parts(self) -> Dict[str, Any]: - flask/install-app: copy the flask project into the OCI image """ if not (self.project_root / "requirements.txt").exists(): - raise ValueError( + raise ExtensionError( "missing requirements.txt file, " "flask extension requires this file with flask specified as a dependency" ) - ignores = [".git", "node_modules", ".yarn"] - source_files = [ - f - for f in os.listdir(self.project_root) - if f not in ignores and not f.endswith(".rock") - ] + source_files = [f.name for f in sorted(self.project_root.iterdir())] renaming_map = {f: posixpath.join("srv/flask/app", f) for f in source_files} install_app_part_name = "flask/install-app" dependencies_part_name = "flask/dependencies" + + if install_app_part_name not in self.yaml_data.get("parts", {}): + raise ExtensionError( + "flask extension required flask/install-app not found " + "in parts of the rockcraft file" + ) + install_prime = self.yaml_data.get("parts")[install_app_part_name].get("prime") + if not install_prime: + raise ExtensionError( + "flask extension required prime list not found or empty" + "in the flask/install-app part of the rockcraft file" + ) + if not all(re.match("-? *srv/flask/app", p) for p in install_prime): + raise ExtensionError( + "flask extension required prime entry in the flask/install-app part" + "to start with srv/flask/app" + ) + # Users are required to compile any static assets prior to executing the # rockcraft pack command, so assets can be included in the final OCI image. install_app_part = { diff --git a/tests/spread/general/extension-flask/app.py b/tests/spread/general/extension-flask/app.py index af5ffaabe..128855f5b 100644 --- a/tests/spread/general/extension-flask/app.py +++ b/tests/spread/general/extension-flask/app.py @@ -4,5 +4,5 @@ @app.route("/") -def hello_world(): - return "Hello, World!" +def ok(): + return "ok" diff --git a/tests/spread/general/extension-flask/task.yaml b/tests/spread/general/extension-flask/task.yaml index 8be2d8c37..f66a30dfd 100644 --- a/tests/spread/general/extension-flask/task.yaml +++ b/tests/spread/general/extension-flask/task.yaml @@ -25,6 +25,11 @@ execute: | # test the part merging docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert not pathlib.Path('/srv/flask/app/README').exists()" + + # test the default flask service + docker run --rm --name flask-extension-container -d -p 8137:8000 flask-extension + retry -n 5 --wait 2 curl localhost:8137 + [ $(curl -sSf localhost:8137) == "ok" ] restore: | rm -f flask-extension_0.1_amd64.rock diff --git a/tests/unit/extensions/test_extensions.py b/tests/unit/extensions/test_extensions.py index 1494b3d13..4214ed0f0 100644 --- a/tests/unit/extensions/test_extensions.py +++ b/tests/unit/extensions/test_extensions.py @@ -134,102 +134,3 @@ def test_project_load_extensions(fake_extensions, tmp_path): # New part assert parts[f"{FullExtension.NAME}/new-part"] == {"plugin": "nil", "source": None} - - -def test_flask_extensions(fake_extensions, tmp_path, input_yaml, monkeypatch): - monkeypatch.setenv("ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", "1") - extensions.register("flask", extensions.flask.Flask) - - (tmp_path / "requirements.txt").write_text("flask") - (tmp_path / "app.py").touch() - (tmp_path / "static").mkdir() - (tmp_path / "node_modules").mkdir() - - input_yaml["extensions"] = ["flask"] - - applied = extensions.apply_extensions(tmp_path, input_yaml) - - assert applied["run_user"] == "_daemon_" - assert applied["platforms"] == {"amd64": {}} - - # Root snippet extends the project's - services = applied["services"] - assert services["flask"] == { - "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 app:app", - "override": "replace", - "startup": "enabled", - "user": "_daemon_", - "working-dir": "/srv/flask/app", - } - - parts = applied["parts"] - assert sorted(parts["flask/install-app"]["stage"]) == [ - "srv/flask/app/app.py", - "srv/flask/app/requirements.txt", - "srv/flask/app/static", - ] - del parts["flask/install-app"]["stage"] - assert parts == { - "flask/dependencies": { - "source": ".", - "plugin": "python", - "python-requirements": ["requirements.txt"], - "stage-packages": ["python3-venv"], - "python-packages": ["gunicorn"], - }, - "flask/install-app": { - "source": ".", - "organize": { - "app.py": "srv/flask/app/app.py", - "static": "srv/flask/app/static", - "requirements.txt": "srv/flask/app/requirements.txt", - }, - "plugin": "dump", - }, - } - - -def test_flask_extensions_overwrite(fake_extensions, tmp_path, input_yaml, monkeypatch): - monkeypatch.setenv("ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", "1") - extensions.register("flask", extensions.flask.Flask) - - (tmp_path / "requirements.txt").write_text("flask") - (tmp_path / "foobar").touch() - (tmp_path / "webapp").mkdir() - (tmp_path / "webapp/app.py").touch() - (tmp_path / "static").mkdir() - (tmp_path / "node_modules").mkdir() - - input_yaml["extensions"] = ["flask"] - input_yaml["parts"] = {"flask/install-app": {"prime": ["-srv/flask/app/foobar"]}} - input_yaml["services"] = { - "flask": { - "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 webapp.app:app" - } - } - applied = extensions.apply_extensions(tmp_path, input_yaml) - - assert applied["services"] == { - "flask": { - "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 webapp.app:app", - "override": "replace", - "startup": "enabled", - "user": "_daemon_", - "working-dir": "/srv/flask/app", - } - } - assert applied["parts"]["flask/install-app"]["prime"] == ["-srv/flask/app/foobar"] - - -def test_flask_extensions_bare(fake_extensions, tmp_path, monkeypatch): - monkeypatch.setenv("ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", "1") - extensions.register("flask", extensions.flask.Flask) - - (tmp_path / "requirements.txt").write_text("flask") - input_yaml = {"extensions": ["flask"], "base": "bare"} - applied = extensions.apply_extensions(tmp_path, input_yaml) - assert applied["parts"]["flask/container-processing"] == { - "plugin": "nil", - "source": ".", - "override-build": "mkdir -m 777 ${CRAFT_PART_INSTALL}/tmp", - } diff --git a/tests/unit/extensions/test_flask.py b/tests/unit/extensions/test_flask.py new file mode 100644 index 000000000..e5a643abb --- /dev/null +++ b/tests/unit/extensions/test_flask.py @@ -0,0 +1,184 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# Copyright 2023 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import pytest + +from rockcraft import extensions +from rockcraft.errors import ExtensionError + + +@pytest.fixture +def flask_extension(mock_extensions, monkeypatch): + monkeypatch.setenv("ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", "1") + extensions.register("flask", extensions.flask.Flask) + + +@pytest.fixture(name="input_yaml") +def input_yaml_fixture(): + return {"base": "ubuntu:22.04", "extensions": ["flask"]} + + +@pytest.mark.usefixtures("flask_extension") +def test_flask_extension(tmp_path, input_yaml): + (tmp_path / "requirements.txt").write_text("flask") + (tmp_path / "app.py").touch() + (tmp_path / "static").mkdir() + (tmp_path / "node_modules").mkdir() + + input_yaml["parts"] = { + "flask/install-app": { + "prime": [ + "srv/flask/app/app.py", + "srv/flask/app/requirements.txt", + "srv/flask/app/static", + ] + } + } + applied = extensions.apply_extensions(tmp_path, input_yaml) + + assert applied["run_user"] == "_daemon_" + assert applied["platforms"] == {"amd64": {}} + + # Root snippet extends the project's + services = applied["services"] + assert services["flask"] == { + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 app:app", + "override": "replace", + "startup": "enabled", + "user": "_daemon_", + "working-dir": "/srv/flask/app", + } + + parts = applied["parts"] + assert parts == { + "flask/dependencies": { + "source": ".", + "plugin": "python", + "python-requirements": ["requirements.txt"], + "stage-packages": ["python3-venv"], + "python-packages": ["gunicorn"], + }, + "flask/install-app": { + "source": ".", + "stage": [ + "srv/flask/app/app.py", + "srv/flask/app/node_modules", + "srv/flask/app/requirements.txt", + "srv/flask/app/static", + ], + "organize": { + "app.py": "srv/flask/app/app.py", + "node_modules": "srv/flask/app/node_modules", + "static": "srv/flask/app/static", + "requirements.txt": "srv/flask/app/requirements.txt", + }, + "plugin": "dump", + "prime": [ + "srv/flask/app/app.py", + "srv/flask/app/requirements.txt", + "srv/flask/app/static", + ], + }, + } + + +@pytest.mark.usefixtures("flask_extension") +def test_flask_extension_overwrite(tmp_path, input_yaml): + (tmp_path / "requirements.txt").write_text("flask") + (tmp_path / "foobar").touch() + (tmp_path / "webapp").mkdir() + (tmp_path / "webapp/app.py").touch() + (tmp_path / "static").mkdir() + (tmp_path / "node_modules").mkdir() + + input_yaml["parts"] = { + "flask/install-app": {"prime": ["-srv/flask/app/foobar"]}, + "flask/dependencies": {"python-requirements": ["requirements-jammy.txt"]}, + } + input_yaml["services"] = { + "flask": { + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 webapp.app:app" + } + } + applied = extensions.apply_extensions(tmp_path, input_yaml) + + assert applied["services"] == { + "flask": { + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 webapp.app:app", + "override": "replace", + "startup": "enabled", + "user": "_daemon_", + "working-dir": "/srv/flask/app", + } + } + assert applied["parts"]["flask/install-app"]["prime"] == ["-srv/flask/app/foobar"] + + assert applied["parts"]["flask/dependencies"] == { + "plugin": "python", + "python-packages": ["gunicorn"], + "python-requirements": ["requirements.txt", "requirements-jammy.txt"], + "source": ".", + "stage-packages": ["python3-venv"], + } + + +@pytest.mark.usefixtures("flask_extension") +def test_flask_extension_bare(tmp_path): + (tmp_path / "requirements.txt").write_text("flask") + input_yaml = { + "extensions": ["flask"], + "base": "bare", + "parts": {"flask/install-app": {"prime": ["-srv/flask/app/.git"]}}, + } + applied = extensions.apply_extensions(tmp_path, input_yaml) + assert applied["parts"]["flask/container-processing"] == { + "plugin": "nil", + "source": ".", + "override-build": "mkdir -m 777 ${CRAFT_PART_INSTALL}/tmp", + } + assert applied["build-base"] == "ubuntu:22.04" + + +@pytest.mark.usefixtures("flask_extension") +def test_flask_extension_error(tmp_path): + input_yaml = {"extensions": ["flask"], "base": "bare"} + with pytest.raises(ExtensionError) as exc: + extensions.apply_extensions(tmp_path, input_yaml) + assert "requirements.txt" in str(exc) + + (tmp_path / "requirements.txt").write_text("flask") + + with pytest.raises(ExtensionError) as exc: + extensions.apply_extensions(tmp_path, input_yaml) + assert "flask/install-app" in str(exc) + + input_yaml["parts"] = {"flask/install-app": {}} + + with pytest.raises(ExtensionError) as exc: + extensions.apply_extensions(tmp_path, input_yaml) + assert "flask/install-app" in str(exc) + + input_yaml["parts"] = {"flask/install-app": {"prime": []}} + + with pytest.raises(ExtensionError) as exc: + extensions.apply_extensions(tmp_path, input_yaml) + assert "flask/install-app" in str(exc) + + input_yaml["parts"] = {"flask/install-app": {"prime": ["requirement.txt"]}} + + with pytest.raises(ExtensionError) as exc: + extensions.apply_extensions(tmp_path, input_yaml) + assert "srv/flask/app" in str(exc) From b89ceb7aadb7a09d1acf6b1ffdadbcda2a6e3ea6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 1 Sep 2023 14:51:12 +0800 Subject: [PATCH 28/36] Add flask extension service overwrite test --- tests/unit/extensions/test_flask.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/unit/extensions/test_flask.py b/tests/unit/extensions/test_flask.py index e5a643abb..c04f367f1 100644 --- a/tests/unit/extensions/test_flask.py +++ b/tests/unit/extensions/test_flask.py @@ -111,7 +111,11 @@ def test_flask_extension_overwrite(tmp_path, input_yaml): input_yaml["services"] = { "flask": { "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 webapp.app:app" - } + }, + "foobar": { + "command": "/bin/foobar", + "override": "replace", + }, } applied = extensions.apply_extensions(tmp_path, input_yaml) @@ -122,7 +126,11 @@ def test_flask_extension_overwrite(tmp_path, input_yaml): "startup": "enabled", "user": "_daemon_", "working-dir": "/srv/flask/app", - } + }, + "foobar": { + "command": "/bin/foobar", + "override": "replace", + }, } assert applied["parts"]["flask/install-app"]["prime"] == ["-srv/flask/app/foobar"] From bf684a3c4d8d309bfd74a7ee8548ce77a6073845 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 1 Sep 2023 15:59:59 +0800 Subject: [PATCH 29/36] Fix linting and spread --- rockcraft/extensions/flask.py | 4 ++-- tests/spread/general/extension-flask/rockcraft.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index f1a2133c2..9a9610c2b 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -23,9 +23,9 @@ from overrides import override +from ..errors import ExtensionError from ._utils import _apply_extension_property from .extension import Extension -from ..errors import ExtensionError class Flask(Extension): @@ -134,7 +134,7 @@ def _gen_new_parts(self) -> Dict[str, Any]: "flask extension required flask/install-app not found " "in parts of the rockcraft file" ) - install_prime = self.yaml_data.get("parts")[install_app_part_name].get("prime") + install_prime = self.yaml_data["parts"][install_app_part_name].get("prime") if not install_prime: raise ExtensionError( "flask extension required prime list not found or empty" diff --git a/tests/spread/general/extension-flask/rockcraft.yaml b/tests/spread/general/extension-flask/rockcraft.yaml index 00d204fac..cb21b1fa4 100644 --- a/tests/spread/general/extension-flask/rockcraft.yaml +++ b/tests/spread/general/extension-flask/rockcraft.yaml @@ -12,3 +12,4 @@ parts: flask/install-app: prime: - -srv/flask/app/README + - -srv/flask/app/node_modules From ce9e067a167368e2603725cdef68d53e17856b47 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 4 Sep 2023 12:05:17 +0800 Subject: [PATCH 30/36] Fix the shellcheck linting problem --- tests/spread/general/extension-flask/task.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/spread/general/extension-flask/task.yaml b/tests/spread/general/extension-flask/task.yaml index f66a30dfd..fda052d99 100644 --- a/tests/spread/general/extension-flask/task.yaml +++ b/tests/spread/general/extension-flask/task.yaml @@ -29,7 +29,7 @@ execute: | # test the default flask service docker run --rm --name flask-extension-container -d -p 8137:8000 flask-extension retry -n 5 --wait 2 curl localhost:8137 - [ $(curl -sSf localhost:8137) == "ok" ] + [ "$(curl -sSf localhost:8137)" == "ok" ] restore: | rm -f flask-extension_0.1_amd64.rock From 3841e78e1bb342c24f9a53b5a2ff520a75eaabe5 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 21 Sep 2023 15:00:37 +0800 Subject: [PATCH 31/36] Apply suggestions from reviews --- docs/how-to/use-flask-extension.rst | 3 ++ .../spread/general/extension-flask/task.yaml | 3 +- tests/unit/extensions/test_flask.py | 37 +++++++++++++++++-- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/how-to/use-flask-extension.rst b/docs/how-to/use-flask-extension.rst index 31c7331ae..18197f5f0 100644 --- a/docs/how-to/use-flask-extension.rst +++ b/docs/how-to/use-flask-extension.rst @@ -32,6 +32,9 @@ Managing project files with the flask extension The prime declaration must be included in the specially-named ``flask/install-app`` section to instruct the flask extension on which files to include or exclude from the project directory in the ROCK image. +And since the extension places the files from the project folder in the +`/srv/flask/app` directory in the final image. Therefore, all inclusions and +exclusions must be prefixed with `src/flask/app`. For example, to include only select files: diff --git a/tests/spread/general/extension-flask/task.yaml b/tests/spread/general/extension-flask/task.yaml index fda052d99..24527b11b 100644 --- a/tests/spread/general/extension-flask/task.yaml +++ b/tests/spread/general/extension-flask/task.yaml @@ -27,10 +27,11 @@ execute: | docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert not pathlib.Path('/srv/flask/app/README').exists()" # test the default flask service - docker run --rm --name flask-extension-container -d -p 8137:8000 flask-extension + docker run --name flask-extension-container -d -p 8137:8000 flask-extension retry -n 5 --wait 2 curl localhost:8137 [ "$(curl -sSf localhost:8137)" == "ok" ] restore: | rm -f flask-extension_0.1_amd64.rock docker rmi -f flask-extension + docker rm -f flask-extension-container diff --git a/tests/unit/extensions/test_flask.py b/tests/unit/extensions/test_flask.py index c04f367f1..e793d9069 100644 --- a/tests/unit/extensions/test_flask.py +++ b/tests/unit/extensions/test_flask.py @@ -161,31 +161,60 @@ def test_flask_extension_bare(tmp_path): @pytest.mark.usefixtures("flask_extension") -def test_flask_extension_error(tmp_path): +def test_flask_extension_no_requirements_txt_error(tmp_path): input_yaml = {"extensions": ["flask"], "base": "bare"} with pytest.raises(ExtensionError) as exc: extensions.apply_extensions(tmp_path, input_yaml) assert "requirements.txt" in str(exc) + +@pytest.mark.usefixtures("flask_extension") +def test_flask_extension_no_install_app_part_error(tmp_path): + input_yaml = {"extensions": ["flask"], "base": "bare"} + (tmp_path / "requirements.txt").write_text("flask") with pytest.raises(ExtensionError) as exc: extensions.apply_extensions(tmp_path, input_yaml) assert "flask/install-app" in str(exc) - input_yaml["parts"] = {"flask/install-app": {}} + +@pytest.mark.usefixtures("flask_extension") +def test_flask_extension_no_install_app_prime_error(tmp_path): + input_yaml = { + "extensions": ["flask"], + "base": "bare", + "parts": {"flask/install-app": {}}, + } + (tmp_path / "requirements.txt").write_text("flask") with pytest.raises(ExtensionError) as exc: extensions.apply_extensions(tmp_path, input_yaml) assert "flask/install-app" in str(exc) - input_yaml["parts"] = {"flask/install-app": {"prime": []}} + +@pytest.mark.usefixtures("flask_extension") +def test_flask_extension_empty_install_app_prime_error(tmp_path): + input_yaml = { + "extensions": ["flask"], + "base": "bare", + "parts": {"flask/install-app": {"prime": []}}, + } + (tmp_path / "requirements.txt").write_text("flask") with pytest.raises(ExtensionError) as exc: extensions.apply_extensions(tmp_path, input_yaml) assert "flask/install-app" in str(exc) - input_yaml["parts"] = {"flask/install-app": {"prime": ["requirement.txt"]}} + +@pytest.mark.usefixtures("flask_extension") +def test_flask_extension_incorrect_install_app_prime_prefix_error(tmp_path): + input_yaml = { + "extensions": ["flask"], + "base": "bare", + "parts": {"flask/install-app": {"prime": ["requirement.txt"]}}, + } + (tmp_path / "requirements.txt").write_text("flask") with pytest.raises(ExtensionError) as exc: extensions.apply_extensions(tmp_path, input_yaml) From 2340cb69b2158dbc40964f8acfc85af95ae724ee Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 25 Sep 2023 16:10:45 +0800 Subject: [PATCH 32/36] Update the flask extension design --- docs/how-to/use-flask-extension.rst | 30 +++---- docs/reference/extensions.rst | 2 +- rockcraft/extensions/__init__.py | 4 +- rockcraft/extensions/flask.py | 36 ++++++-- .../general/extension-flask/rockcraft.yaml | 6 +- .../spread/general/extension-flask/task.yaml | 8 +- ...{test_flask.py => test_flask_framework.py} | 87 +++++++++++-------- 7 files changed, 107 insertions(+), 66 deletions(-) rename tests/unit/extensions/{test_flask.py => test_flask_framework.py} (72%) diff --git a/docs/how-to/use-flask-extension.rst b/docs/how-to/use-flask-extension.rst index 18197f5f0..382948aa9 100644 --- a/docs/how-to/use-flask-extension.rst +++ b/docs/how-to/use-flask-extension.rst @@ -21,10 +21,10 @@ Example: flask/install-app: prime: - - -srv/flask/app/.git - - -srv/flask/app/.venv - - -srv/flask/app/.yarn - - -srv/flask/app/node_modules + - -flask/app/.git + - -flask/app/.venv + - -flask/app/.yarn + - -flask/app/node_modules Managing project files with the flask extension ----------------------------------------------- @@ -32,9 +32,9 @@ Managing project files with the flask extension The prime declaration must be included in the specially-named ``flask/install-app`` section to instruct the flask extension on which files to include or exclude from the project directory in the ROCK image. -And since the extension places the files from the project folder in the -`/srv/flask/app` directory in the final image. Therefore, all inclusions and -exclusions must be prefixed with `src/flask/app`. +The extension places the files from the project folder in the ``/flask/app`` +directory in the final image - therefore, all inclusions and exclusions must +be prefixed with ``flask/app``. For example, to include only select files: @@ -42,10 +42,10 @@ For example, to include only select files: flask/install-app: prime: - - srv/flask/app/static - - srv/flask/app/.env - - srv/flask/app/webapp - - srv/flask/app/templates + - flask/app/static + - flask/app/.env + - flask/app/webapp + - flask/app/templates To exclude certain files from the project directory in the rock image, add the following part to ``rockcraft.yaml``: @@ -54,7 +54,7 @@ add the following part to ``rockcraft.yaml``: flask/install-app: prime: - - -srv/flask/app/.git - - -srv/flask/app/.venv - - -srv/flask/app/.yarn - - -srv/flask/app/node_modules + - -flask/app/.git + - -flask/app/.venv + - -flask/app/.yarn + - -flask/app/node_modules diff --git a/docs/reference/extensions.rst b/docs/reference/extensions.rst index 01c814e7c..349bd1b62 100644 --- a/docs/reference/extensions.rst +++ b/docs/reference/extensions.rst @@ -13,4 +13,4 @@ The Flask extension streamlines the process of building Flask application rocks. It facilitates the installation of Flask application dependencies, including Gunicorn, in the rock image. Additionally, it transfers your project files to -``/srv/flask/app`` within the rock image. +``/flask/app`` within the rock image. diff --git a/rockcraft/extensions/__init__.py b/rockcraft/extensions/__init__.py index be49e9451..8824d9d10 100644 --- a/rockcraft/extensions/__init__.py +++ b/rockcraft/extensions/__init__.py @@ -17,7 +17,7 @@ """Extension processor and related utilities.""" from ._utils import apply_extensions -from .flask import Flask +from .flask import FlaskFramework from .registry import get_extension_class, get_extension_names, register, unregister __all__ = [ @@ -28,4 +28,4 @@ "unregister", ] -register("flask", Flask) +register("flask-framework", FlaskFramework) diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask.py index 9a9610c2b..ad1012352 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask.py @@ -15,7 +15,7 @@ # along with this program. If not, see . """An experimental extension for the Flask framework.""" - +import ast import copy import posixpath import re @@ -28,7 +28,7 @@ from .extension import Extension -class Flask(Extension): +class FlaskFramework(Extension): """An extension for constructing Python applications based on the Flask framework.""" @staticmethod @@ -68,15 +68,35 @@ def get_root_snippet(self) -> Dict[str, Any]: snippet["services"] = self._gen_services() return snippet + def _check_wsgi_path(self): + """Ensure the flask application can be run with the WSGI path app:app.""" + app_file = self.project_root / "app.py" + if not app_file.exists(): + raise ExtensionError( + "flask application can not be imported from app:app, " + "no app.py file found in the project root" + ) + tree = ast.parse(app_file.read_text(encoding="utf-8")) + for node in ast.iter_child_nodes(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "app": + return + raise ExtensionError( + "flask application can not be imported from app:app, " + "no variable named app in app.py" + ) + def _gen_services(self): """Return the services snipped to be applied to the rockcraft file.""" + self._check_wsgi_path() services = { "flask": { "override": "replace", "startup": "enabled", "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 app:app", "user": "_daemon_", - "working-dir": "/srv/flask/app", + "working-dir": "/flask/app", } } existing_services = copy.deepcopy(self.yaml_data.get("services", {})) @@ -125,7 +145,11 @@ def _gen_new_parts(self) -> Dict[str, Any]: "flask extension requires this file with flask specified as a dependency" ) source_files = [f.name for f in sorted(self.project_root.iterdir())] - renaming_map = {f: posixpath.join("srv/flask/app", f) for f in source_files} + renaming_map = { + f: posixpath.join("flask/app", f) + for f in source_files + if source_files not in ("node_modules", ".git", ".yarn") + } install_app_part_name = "flask/install-app" dependencies_part_name = "flask/dependencies" @@ -140,10 +164,10 @@ def _gen_new_parts(self) -> Dict[str, Any]: "flask extension required prime list not found or empty" "in the flask/install-app part of the rockcraft file" ) - if not all(re.match("-? *srv/flask/app", p) for p in install_prime): + if not all(re.match("-? *flask/app", p) for p in install_prime): raise ExtensionError( "flask extension required prime entry in the flask/install-app part" - "to start with srv/flask/app" + "to start with flask/app" ) # Users are required to compile any static assets prior to executing the diff --git a/tests/spread/general/extension-flask/rockcraft.yaml b/tests/spread/general/extension-flask/rockcraft.yaml index cb21b1fa4..83acb7883 100644 --- a/tests/spread/general/extension-flask/rockcraft.yaml +++ b/tests/spread/general/extension-flask/rockcraft.yaml @@ -6,10 +6,10 @@ base: bare license: Apache-2.0 extensions: - - flask + - flask-framework parts: flask/install-app: prime: - - -srv/flask/app/README - - -srv/flask/app/node_modules + - -flask/app/README + - -flask/app/node_modules diff --git a/tests/spread/general/extension-flask/task.yaml b/tests/spread/general/extension-flask/task.yaml index 24527b11b..f23e71046 100644 --- a/tests/spread/general/extension-flask/task.yaml +++ b/tests/spread/general/extension-flask/task.yaml @@ -19,12 +19,12 @@ execute: | docker rm -f flask-extension-container # test the flask project is ready to run inside the container - docker run --rm --entrypoint /bin/python3 flask-extension -m gunicorn --chdir /srv/flask/app --check-config app:app - docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert pathlib.Path('/srv/flask/app/static/js/test.js').is_file()" - docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert not pathlib.Path('/srv/flask/app/node_modules').exists()" + docker run --rm --entrypoint /bin/python3 flask-extension -m gunicorn --chdir /flask/app --check-config app:app + docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert pathlib.Path('/flask/app/static/js/test.js').is_file()" + docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert not pathlib.Path('/flask/app/node_modules').exists()" # test the part merging - docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert not pathlib.Path('/srv/flask/app/README').exists()" + docker run --rm --entrypoint /bin/python3 flask-extension -c "import pathlib;assert not pathlib.Path('/flask/app/README').exists()" # test the default flask service docker run --name flask-extension-container -d -p 8137:8000 flask-extension diff --git a/tests/unit/extensions/test_flask.py b/tests/unit/extensions/test_flask_framework.py similarity index 72% rename from tests/unit/extensions/test_flask.py rename to tests/unit/extensions/test_flask_framework.py index e793d9069..4a5394110 100644 --- a/tests/unit/extensions/test_flask.py +++ b/tests/unit/extensions/test_flask_framework.py @@ -23,27 +23,27 @@ @pytest.fixture def flask_extension(mock_extensions, monkeypatch): monkeypatch.setenv("ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", "1") - extensions.register("flask", extensions.flask.Flask) + extensions.register("flask-framework", extensions.flask.FlaskFramework) @pytest.fixture(name="input_yaml") def input_yaml_fixture(): - return {"base": "ubuntu:22.04", "extensions": ["flask"]} + return {"base": "ubuntu:22.04", "extensions": ["flask-framework"]} @pytest.mark.usefixtures("flask_extension") def test_flask_extension(tmp_path, input_yaml): (tmp_path / "requirements.txt").write_text("flask") - (tmp_path / "app.py").touch() + (tmp_path / "app.py").write_text("app = object()") (tmp_path / "static").mkdir() (tmp_path / "node_modules").mkdir() input_yaml["parts"] = { "flask/install-app": { "prime": [ - "srv/flask/app/app.py", - "srv/flask/app/requirements.txt", - "srv/flask/app/static", + "flask/app/app.py", + "flask/app/requirements.txt", + "flask/app/static", ] } } @@ -59,7 +59,7 @@ def test_flask_extension(tmp_path, input_yaml): "override": "replace", "startup": "enabled", "user": "_daemon_", - "working-dir": "/srv/flask/app", + "working-dir": "/flask/app", } parts = applied["parts"] @@ -74,22 +74,22 @@ def test_flask_extension(tmp_path, input_yaml): "flask/install-app": { "source": ".", "stage": [ - "srv/flask/app/app.py", - "srv/flask/app/node_modules", - "srv/flask/app/requirements.txt", - "srv/flask/app/static", + "flask/app/app.py", + "flask/app/node_modules", + "flask/app/requirements.txt", + "flask/app/static", ], "organize": { - "app.py": "srv/flask/app/app.py", - "node_modules": "srv/flask/app/node_modules", - "static": "srv/flask/app/static", - "requirements.txt": "srv/flask/app/requirements.txt", + "app.py": "flask/app/app.py", + "node_modules": "flask/app/node_modules", + "static": "flask/app/static", + "requirements.txt": "flask/app/requirements.txt", }, "plugin": "dump", "prime": [ - "srv/flask/app/app.py", - "srv/flask/app/requirements.txt", - "srv/flask/app/static", + "flask/app/app.py", + "flask/app/requirements.txt", + "flask/app/static", ], }, } @@ -99,19 +99,15 @@ def test_flask_extension(tmp_path, input_yaml): def test_flask_extension_overwrite(tmp_path, input_yaml): (tmp_path / "requirements.txt").write_text("flask") (tmp_path / "foobar").touch() - (tmp_path / "webapp").mkdir() - (tmp_path / "webapp/app.py").touch() + (tmp_path / "app.py").write_text("app = object()") (tmp_path / "static").mkdir() (tmp_path / "node_modules").mkdir() input_yaml["parts"] = { - "flask/install-app": {"prime": ["-srv/flask/app/foobar"]}, + "flask/install-app": {"prime": ["-flask/app/foobar"]}, "flask/dependencies": {"python-requirements": ["requirements-jammy.txt"]}, } input_yaml["services"] = { - "flask": { - "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 webapp.app:app" - }, "foobar": { "command": "/bin/foobar", "override": "replace", @@ -121,18 +117,18 @@ def test_flask_extension_overwrite(tmp_path, input_yaml): assert applied["services"] == { "flask": { - "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 webapp.app:app", + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 app:app", "override": "replace", "startup": "enabled", "user": "_daemon_", - "working-dir": "/srv/flask/app", + "working-dir": "/flask/app", }, "foobar": { "command": "/bin/foobar", "override": "replace", }, } - assert applied["parts"]["flask/install-app"]["prime"] == ["-srv/flask/app/foobar"] + assert applied["parts"]["flask/install-app"]["prime"] == ["-flask/app/foobar"] assert applied["parts"]["flask/dependencies"] == { "plugin": "python", @@ -146,10 +142,11 @@ def test_flask_extension_overwrite(tmp_path, input_yaml): @pytest.mark.usefixtures("flask_extension") def test_flask_extension_bare(tmp_path): (tmp_path / "requirements.txt").write_text("flask") + (tmp_path / "app.py").write_text("app = object()") input_yaml = { - "extensions": ["flask"], + "extensions": ["flask-framework"], "base": "bare", - "parts": {"flask/install-app": {"prime": ["-srv/flask/app/.git"]}}, + "parts": {"flask/install-app": {"prime": ["-flask/app/.git"]}}, } applied = extensions.apply_extensions(tmp_path, input_yaml) assert applied["parts"]["flask/container-processing"] == { @@ -162,7 +159,7 @@ def test_flask_extension_bare(tmp_path): @pytest.mark.usefixtures("flask_extension") def test_flask_extension_no_requirements_txt_error(tmp_path): - input_yaml = {"extensions": ["flask"], "base": "bare"} + input_yaml = {"extensions": ["flask-framework"], "base": "bare"} with pytest.raises(ExtensionError) as exc: extensions.apply_extensions(tmp_path, input_yaml) assert "requirements.txt" in str(exc) @@ -170,7 +167,7 @@ def test_flask_extension_no_requirements_txt_error(tmp_path): @pytest.mark.usefixtures("flask_extension") def test_flask_extension_no_install_app_part_error(tmp_path): - input_yaml = {"extensions": ["flask"], "base": "bare"} + input_yaml = {"extensions": ["flask-framework"], "base": "bare"} (tmp_path / "requirements.txt").write_text("flask") @@ -182,7 +179,7 @@ def test_flask_extension_no_install_app_part_error(tmp_path): @pytest.mark.usefixtures("flask_extension") def test_flask_extension_no_install_app_prime_error(tmp_path): input_yaml = { - "extensions": ["flask"], + "extensions": ["flask-framework"], "base": "bare", "parts": {"flask/install-app": {}}, } @@ -196,7 +193,7 @@ def test_flask_extension_no_install_app_prime_error(tmp_path): @pytest.mark.usefixtures("flask_extension") def test_flask_extension_empty_install_app_prime_error(tmp_path): input_yaml = { - "extensions": ["flask"], + "extensions": ["flask-framework"], "base": "bare", "parts": {"flask/install-app": {"prime": []}}, } @@ -210,7 +207,7 @@ def test_flask_extension_empty_install_app_prime_error(tmp_path): @pytest.mark.usefixtures("flask_extension") def test_flask_extension_incorrect_install_app_prime_prefix_error(tmp_path): input_yaml = { - "extensions": ["flask"], + "extensions": ["flask-framework"], "base": "bare", "parts": {"flask/install-app": {"prime": ["requirement.txt"]}}, } @@ -218,4 +215,24 @@ def test_flask_extension_incorrect_install_app_prime_prefix_error(tmp_path): with pytest.raises(ExtensionError) as exc: extensions.apply_extensions(tmp_path, input_yaml) - assert "srv/flask/app" in str(exc) + assert "flask/app" in str(exc) + + +@pytest.mark.usefixtures("flask_extension") +def test_flask_extension_incorrect_wsgi_path_error(tmp_path): + input_yaml = { + "extensions": ["flask-framework"], + "base": "bare", + "parts": {"flask/install-app": {"prime": ["flask/app/requirement.txt"]}}, + } + (tmp_path / "requirements.txt").write_text("flask") + + with pytest.raises(ExtensionError) as exc: + extensions.apply_extensions(tmp_path, input_yaml) + assert "app:app" in str(exc) + + (tmp_path / "app.py").write_text("flask") + + with pytest.raises(ExtensionError) as exc: + extensions.apply_extensions(tmp_path, input_yaml) + assert "app:app" in str(exc) From 5487cb547c1f299602cdad94043306ea299a6106 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 25 Sep 2023 20:00:14 +0800 Subject: [PATCH 33/36] Fix some linting issues --- rockcraft/extensions/__init__.py | 2 +- rockcraft/extensions/{flask.py => flask_framework.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename rockcraft/extensions/{flask.py => flask_framework.py} (99%) diff --git a/rockcraft/extensions/__init__.py b/rockcraft/extensions/__init__.py index 8824d9d10..a737db51d 100644 --- a/rockcraft/extensions/__init__.py +++ b/rockcraft/extensions/__init__.py @@ -17,7 +17,7 @@ """Extension processor and related utilities.""" from ._utils import apply_extensions -from .flask import FlaskFramework +from .flask_framework import FlaskFramework from .registry import get_extension_class, get_extension_names, register, unregister __all__ = [ diff --git a/rockcraft/extensions/flask.py b/rockcraft/extensions/flask_framework.py similarity index 99% rename from rockcraft/extensions/flask.py rename to rockcraft/extensions/flask_framework.py index ad1012352..0847d8a0d 100644 --- a/rockcraft/extensions/flask.py +++ b/rockcraft/extensions/flask_framework.py @@ -148,7 +148,7 @@ def _gen_new_parts(self) -> Dict[str, Any]: renaming_map = { f: posixpath.join("flask/app", f) for f in source_files - if source_files not in ("node_modules", ".git", ".yarn") + if f not in ("node_modules", ".git", ".yarn") } install_app_part_name = "flask/install-app" dependencies_part_name = "flask/dependencies" From 1df6a353ad002d3318541b3b6e60a16515865384 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Mon, 25 Sep 2023 22:25:37 +0800 Subject: [PATCH 34/36] Update test_flask_framework.py --- tests/unit/extensions/test_flask_framework.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/extensions/test_flask_framework.py b/tests/unit/extensions/test_flask_framework.py index 4a5394110..196537cfb 100644 --- a/tests/unit/extensions/test_flask_framework.py +++ b/tests/unit/extensions/test_flask_framework.py @@ -23,7 +23,7 @@ @pytest.fixture def flask_extension(mock_extensions, monkeypatch): monkeypatch.setenv("ROCKCRAFT_ENABLE_EXPERIMENTAL_EXTENSIONS", "1") - extensions.register("flask-framework", extensions.flask.FlaskFramework) + extensions.register("flask-framework", extensions.flask_framework.FlaskFramework) @pytest.fixture(name="input_yaml") @@ -75,13 +75,11 @@ def test_flask_extension(tmp_path, input_yaml): "source": ".", "stage": [ "flask/app/app.py", - "flask/app/node_modules", "flask/app/requirements.txt", "flask/app/static", ], "organize": { "app.py": "flask/app/app.py", - "node_modules": "flask/app/node_modules", "static": "flask/app/static", "requirements.txt": "flask/app/requirements.txt", }, From 5d3a0991bfcb7afd79cccbdf1f53f93579c9adb7 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 27 Sep 2023 19:09:07 +0800 Subject: [PATCH 35/36] Update flask_framework.py --- rockcraft/extensions/flask_framework.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rockcraft/extensions/flask_framework.py b/rockcraft/extensions/flask_framework.py index 0847d8a0d..93464be2f 100644 --- a/rockcraft/extensions/flask_framework.py +++ b/rockcraft/extensions/flask_framework.py @@ -17,6 +17,7 @@ """An experimental extension for the Flask framework.""" import ast import copy +import fnmatch import posixpath import re from typing import Any, Dict, Optional, Tuple @@ -82,6 +83,12 @@ def _check_wsgi_path(self): for target in node.targets: if isinstance(target, ast.Name) and target.id == "app": return + if isinstance(node, ast.ImportFrom): + for name in node.names: + if (name.asname is not None and name.asname == "app") or ( + name.asname is None and name.name == "app" + ): + return raise ExtensionError( "flask application can not be imported from app:app, " "no variable named app in app.py" @@ -148,7 +155,10 @@ def _gen_new_parts(self) -> Dict[str, Any]: renaming_map = { f: posixpath.join("flask/app", f) for f in source_files - if f not in ("node_modules", ".git", ".yarn") + if not any( + fnmatch.fnmatch(f, p) + for p in ("node_modules", ".git", ".yarn", "*.rock") + ) } install_app_part_name = "flask/install-app" dependencies_part_name = "flask/dependencies" From c51286e2c1fa9baaa8a92c0df65dfbb601a7bbd6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 28 Sep 2023 10:26:32 +0800 Subject: [PATCH 36/36] Remove working-dir from flask service --- rockcraft/extensions/flask_framework.py | 3 +-- tests/unit/extensions/test_flask_framework.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/rockcraft/extensions/flask_framework.py b/rockcraft/extensions/flask_framework.py index 93464be2f..bc721415f 100644 --- a/rockcraft/extensions/flask_framework.py +++ b/rockcraft/extensions/flask_framework.py @@ -101,9 +101,8 @@ def _gen_services(self): "flask": { "override": "replace", "startup": "enabled", - "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 app:app", + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 --chdir /flask/app app:app", "user": "_daemon_", - "working-dir": "/flask/app", } } existing_services = copy.deepcopy(self.yaml_data.get("services", {})) diff --git a/tests/unit/extensions/test_flask_framework.py b/tests/unit/extensions/test_flask_framework.py index 196537cfb..66806e50a 100644 --- a/tests/unit/extensions/test_flask_framework.py +++ b/tests/unit/extensions/test_flask_framework.py @@ -55,11 +55,10 @@ def test_flask_extension(tmp_path, input_yaml): # Root snippet extends the project's services = applied["services"] assert services["flask"] == { - "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 app:app", + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 --chdir /flask/app app:app", "override": "replace", "startup": "enabled", "user": "_daemon_", - "working-dir": "/flask/app", } parts = applied["parts"] @@ -115,11 +114,10 @@ def test_flask_extension_overwrite(tmp_path, input_yaml): assert applied["services"] == { "flask": { - "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 app:app", + "command": "/bin/python3 -m gunicorn --bind 0.0.0.0:8000 --chdir /flask/app app:app", "override": "replace", "startup": "enabled", "user": "_daemon_", - "working-dir": "/flask/app", }, "foobar": { "command": "/bin/foobar",