From c9d0849cd7f353214b5f6db345a15b5d517cb6c0 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Thu, 7 Nov 2024 14:31:39 +0300 Subject: [PATCH 01/27] Feat(): Enable async workers for flask and django --- src/paas_charm/_gunicorn/charm.py | 37 ++++++++++++++++++++++++++- src/paas_charm/_gunicorn/webserver.py | 14 +++++++--- tests/integration/flask/conftest.py | 1 + 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index 0273687..a2a9f1b 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -3,12 +3,14 @@ """The base charm class for all charms.""" import logging +import typing from paas_charm._gunicorn.webserver import GunicornWebserver, WebserverConfig from paas_charm._gunicorn.workload_config import create_workload_config from paas_charm._gunicorn.wsgi_app import WsgiApp from paas_charm.app import App, WorkloadConfig from paas_charm.charm import PaasCharm +from paas_charm.exceptions import CharmConfigInvalidError logger = logging.getLogger(__name__) @@ -23,6 +25,39 @@ def _workload_config(self) -> WorkloadConfig: framework_name=self._framework_name, unit_name=self.unit.name ) + def check_gevent_package(self) -> bool: + """Check that gevent is installed. + + Returns: + True if gevent is installed. + """ + ddddd = self._container.exec(["pip", "list"]) + list_output = ddddd.wait_output()[0] + return "gevent" in list_output + + def create_webserver_config(self) -> WebserverConfig: + """Create a WebserverConfig instance from the charm config. + + Returns: + A new WebserverConfig instance. + + Raises: + CharmConfigInvalidError: if the charm configuration is not valid. + """ + _webserver_config: WebserverConfig = WebserverConfig.from_charm_config(dict(self.config)) + + if _webserver_config.worker_class == "sync": + _webserver_config.worker_class = None + + if _webserver_config.worker_class: + if "gevent" != typing.cast(str, _webserver_config.worker_class): + raise CharmConfigInvalidError("Only 'gevent' and 'sync' are allowed.") + + if not self.check_gevent_package(): + raise CharmConfigInvalidError("gunicorn[gevent] must be installed in the rock.") + + return _webserver_config + def _create_app(self) -> App: """Build an App instance for the Gunicorn based charm. @@ -32,7 +67,7 @@ def _create_app(self) -> App: charm_state = self._create_charm_state() webserver = GunicornWebserver( - webserver_config=WebserverConfig.from_charm_config(dict(self.config)), + webserver_config=self.create_webserver_config(), workload_config=self._workload_config, container=self.unit.get_container(self._workload_config.container_name), ) diff --git a/src/paas_charm/_gunicorn/webserver.py b/src/paas_charm/_gunicorn/webserver.py index 854ed72..a58db76 100644 --- a/src/paas_charm/_gunicorn/webserver.py +++ b/src/paas_charm/_gunicorn/webserver.py @@ -32,6 +32,7 @@ class WebserverConfig: Attributes: workers: The number of workers to use for the web server, or None if not specified. + worker_class: The method of workers to use for the web server, or sync if not specified. threads: The number of threads per worker to use for the web server, or None if not specified. keepalive: The time to wait for requests on a Keep-Alive connection, @@ -40,11 +41,12 @@ class WebserverConfig: """ workers: int | None = None + worker_class: str | None = None threads: int | None = None keepalive: datetime.timedelta | None = None timeout: datetime.timedelta | None = None - def items(self) -> typing.Iterable[tuple[str, int | datetime.timedelta | None]]: + def items(self) -> typing.Iterable[tuple[str, str | int | datetime.timedelta | None]]: """Return the dataclass values as an iterable of the key-value pairs. Returns: @@ -52,6 +54,7 @@ def items(self) -> typing.Iterable[tuple[str, int | datetime.timedelta | None]]: """ return { "workers": self.workers, + "worker_class": self.worker_class, "threads": self.threads, "keepalive": self.keepalive, "timeout": self.timeout, @@ -70,9 +73,12 @@ def from_charm_config(cls, config: dict[str, int | float | str | bool]) -> "Webs keepalive = config.get("webserver-keepalive") timeout = config.get("webserver-timeout") workers = config.get("webserver-workers") + worker_class = config.get("webserver-worker-class") + threads = config.get("webserver-threads") return cls( workers=int(typing.cast(str, workers)) if workers is not None else None, + worker_class=str(typing.cast(str, worker_class)) if worker_class is not None else None, threads=int(typing.cast(str, threads)) if threads is not None else None, keepalive=( datetime.timedelta(seconds=int(keepalive)) if keepalive is not None else None @@ -111,14 +117,16 @@ def _config(self) -> str: """ config_entries = [] for setting, setting_value in self._webserver_config.items(): - setting_value = typing.cast(None | int | datetime.timedelta, setting_value) + setting_value = typing.cast(None | str | int | datetime.timedelta, setting_value) if setting_value is None: continue setting_value = ( setting_value - if isinstance(setting_value, int) + if isinstance(setting_value, (int, str)) else int(setting_value.total_seconds()) ) + if isinstance(setting_value, str): + setting_value = f"'{setting_value}'" config_entries.append(f"{setting} = {setting_value}") if enable_pebble_log_forwarding(): access_log = "'-'" diff --git a/tests/integration/flask/conftest.py b/tests/integration/flask/conftest.py index 6a0280c..ac5b75a 100644 --- a/tests/integration/flask/conftest.py +++ b/tests/integration/flask/conftest.py @@ -75,6 +75,7 @@ async def build_charm_fixture(charm_file: str, tmp_path_factory) -> str: "foo-bool": {"type": "boolean"}, "foo-dict": {"type": "string"}, "application-root": {"type": "string"}, + "webserver-worker-class": {"type": "string"}, }, tmp_path_factory.mktemp("flask"), ) From f1a015199521b4f67fee59b315a919f5803a509c Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 8 Nov 2024 11:36:12 +0300 Subject: [PATCH 02/27] Chore(): Log error when worker_class setting is wrong. --- src/paas_charm/_gunicorn/charm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index a2a9f1b..c082abe 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -51,10 +51,12 @@ def create_webserver_config(self) -> WebserverConfig: if _webserver_config.worker_class: if "gevent" != typing.cast(str, _webserver_config.worker_class): - raise CharmConfigInvalidError("Only 'gevent' and 'sync' are allowed.") + logger.error("Only 'gevent' and 'sync' are allowed. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/%s-framework/#parts-%s-framework-async-dependencies", self._framework_name, self._framework_name) + raise CharmConfigInvalidError(f"Only 'gevent' and 'sync' are allowed. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/{self._framework_name}-framework/#parts-{self._framework_name}-framework-async-dependencies") if not self.check_gevent_package(): - raise CharmConfigInvalidError("gunicorn[gevent] must be installed in the rock.") + logger.error("gunicorn[gevent] must be installed in the rock. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/%s-framework/#parts-%s-framework-async-dependencies", self._framework_name, self._framework_name) + raise CharmConfigInvalidError(f"gunicorn[gevent] must be installed in the rock. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/{self._framework_name}-framework/#parts-{self._framework_name}-framework-async-dependencies") return _webserver_config From 4bf80baabdeed737b672e86492831c2dd29196d3 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 8 Nov 2024 12:08:37 +0300 Subject: [PATCH 03/27] chore(Lint): Run linter --- src/paas_charm/_gunicorn/charm.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index c082abe..c3934b8 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -51,12 +51,24 @@ def create_webserver_config(self) -> WebserverConfig: if _webserver_config.worker_class: if "gevent" != typing.cast(str, _webserver_config.worker_class): - logger.error("Only 'gevent' and 'sync' are allowed. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/%s-framework/#parts-%s-framework-async-dependencies", self._framework_name, self._framework_name) - raise CharmConfigInvalidError(f"Only 'gevent' and 'sync' are allowed. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/{self._framework_name}-framework/#parts-{self._framework_name}-framework-async-dependencies") + logger.error( + "Only 'gevent' and 'sync' are allowed. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/%s-framework/#parts-%s-framework-async-dependencies", + self._framework_name, + self._framework_name, + ) + raise CharmConfigInvalidError( + f"Only 'gevent' and 'sync' are allowed. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/{self._framework_name}-framework/#parts-{self._framework_name}-framework-async-dependencies" + ) if not self.check_gevent_package(): - logger.error("gunicorn[gevent] must be installed in the rock. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/%s-framework/#parts-%s-framework-async-dependencies", self._framework_name, self._framework_name) - raise CharmConfigInvalidError(f"gunicorn[gevent] must be installed in the rock. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/{self._framework_name}-framework/#parts-{self._framework_name}-framework-async-dependencies") + logger.error( + "gunicorn[gevent] must be installed in the rock. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/%s-framework/#parts-%s-framework-async-dependencies", + self._framework_name, + self._framework_name, + ) + raise CharmConfigInvalidError( + f"gunicorn[gevent] must be installed in the rock. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/{self._framework_name}-framework/#parts-{self._framework_name}-framework-async-dependencies" + ) return _webserver_config From 79441821380ba616b898930de89d668842fbf529 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 8 Nov 2024 14:33:51 +0300 Subject: [PATCH 04/27] chore(): Formatted strings --- src/paas_charm/_gunicorn/charm.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index c3934b8..e405879 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -2,6 +2,7 @@ # See LICENSE file for licensing details. """The base charm class for all charms.""" + import logging import typing @@ -52,22 +53,36 @@ def create_webserver_config(self) -> WebserverConfig: if _webserver_config.worker_class: if "gevent" != typing.cast(str, _webserver_config.worker_class): logger.error( - "Only 'gevent' and 'sync' are allowed. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/%s-framework/#parts-%s-framework-async-dependencies", + "Only 'gevent' and 'sync' are allowed. " + "https://documentation.ubuntu.com/rockcraft" + "/en/latest/reference/extensions/%s-framework" + "/#parts-%s-framework-async-dependencies", self._framework_name, self._framework_name, ) raise CharmConfigInvalidError( - f"Only 'gevent' and 'sync' are allowed. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/{self._framework_name}-framework/#parts-{self._framework_name}-framework-async-dependencies" + "Only 'gevent' and 'sync' are allowed. " + "https://documentation.ubuntu.com/rockcraft/en/latest" + f"/reference/extensions/{self._framework_name}-" + f"framework/#parts-{self._framework_name}-" + "framework-async-dependencies" ) if not self.check_gevent_package(): logger.error( - "gunicorn[gevent] must be installed in the rock. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/%s-framework/#parts-%s-framework-async-dependencies", + "gunicorn[gevent] must be installed in the rock. " + "https://documentation.ubuntu.com/rockcraft" + "/en/latest/reference/extensions/%s-framework" + "/#parts-%s-framework-async-dependencies", self._framework_name, self._framework_name, ) raise CharmConfigInvalidError( - f"gunicorn[gevent] must be installed in the rock. https://documentation.ubuntu.com/rockcraft/en/latest/reference/extensions/{self._framework_name}-framework/#parts-{self._framework_name}-framework-async-dependencies" + "gunicorn[gevent] must be installed in the rock. " + "https://documentation.ubuntu.com/rockcraft/en/latest" + f"/reference/extensions/{self._framework_name}-" + f"framework/#parts-{self._framework_name}-" + "framework-async-dependencies" ) return _webserver_config From e173772e67f9d2ecfe3d0bd227ba0cec042f85cd Mon Sep 17 00:00:00 2001 From: ali ugur Date: Tue, 12 Nov 2024 11:10:08 +0300 Subject: [PATCH 05/27] fix(websockets): Pinned websockets version to <14.0 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 93a7747..e435030 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ cosl jsonschema >=4.19,<4.20 ops >= 2.6 pydantic==2.9.2 +websockets < 14.0 From 9d7ed4624ae46aa66d7011606c987c5febe5933a Mon Sep 17 00:00:00 2001 From: ali ugur Date: Wed, 13 Nov 2024 07:57:54 +0300 Subject: [PATCH 06/27] Chore(): Minor refactors. Apply comments --- requirements.txt | 1 - src/paas_charm/_gunicorn/charm.py | 16 ++++++++-------- tox.ini | 1 + 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index e435030..93a7747 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,3 @@ cosl jsonschema >=4.19,<4.20 ops >= 2.6 pydantic==2.9.2 -websockets < 14.0 diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index e405879..2275b5b 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -32,8 +32,8 @@ def check_gevent_package(self) -> bool: Returns: True if gevent is installed. """ - ddddd = self._container.exec(["pip", "list"]) - list_output = ddddd.wait_output()[0] + pip_list_command = self._container.exec(["pip", "list"]) + list_output = pip_list_command.wait_output()[0] return "gevent" in list_output def create_webserver_config(self) -> WebserverConfig: @@ -45,13 +45,13 @@ def create_webserver_config(self) -> WebserverConfig: Raises: CharmConfigInvalidError: if the charm configuration is not valid. """ - _webserver_config: WebserverConfig = WebserverConfig.from_charm_config(dict(self.config)) + webserver_config: WebserverConfig = WebserverConfig.from_charm_config(dict(self.config)) - if _webserver_config.worker_class == "sync": - _webserver_config.worker_class = None + if webserver_config.worker_class == "sync": + webserver_config.worker_class = None - if _webserver_config.worker_class: - if "gevent" != typing.cast(str, _webserver_config.worker_class): + if webserver_config.worker_class: + if "gevent" != typing.cast(str, webserver_config.worker_class): logger.error( "Only 'gevent' and 'sync' are allowed. " "https://documentation.ubuntu.com/rockcraft" @@ -85,7 +85,7 @@ def create_webserver_config(self) -> WebserverConfig: "framework-async-dependencies" ) - return _webserver_config + return webserver_config def _create_app(self) -> App: """Build an App instance for the Gunicorn based charm. diff --git a/tox.ini b/tox.ini index d7adc89..15fb3cb 100644 --- a/tox.ini +++ b/tox.ini @@ -110,6 +110,7 @@ deps = cosl boto3 juju==3.5.2.0 + websockets < 14.0 git+https://github.com/canonical/saml-test-idp.git -r{toxinidir}/requirements.txt -r{toxinidir}/tests/integration/flask/requirements.txt From efb2b601dcb377036d04067bdfac84bc3a3d6f97 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 15 Nov 2024 14:38:02 +0300 Subject: [PATCH 07/27] Chore(async): Use Enum --- src/paas_charm/_gunicorn/charm.py | 76 ++++++++++++--------------- src/paas_charm/_gunicorn/webserver.py | 41 ++++++++++++--- 2 files changed, 67 insertions(+), 50 deletions(-) diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index 2275b5b..a27a077 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -4,9 +4,8 @@ """The base charm class for all charms.""" import logging -import typing -from paas_charm._gunicorn.webserver import GunicornWebserver, WebserverConfig +from paas_charm._gunicorn.webserver import GunicornWebserver, WebserverConfig, WorkerClassEnum from paas_charm._gunicorn.workload_config import create_workload_config from paas_charm._gunicorn.wsgi_app import WsgiApp from paas_charm.app import App, WorkloadConfig @@ -37,53 +36,46 @@ def check_gevent_package(self) -> bool: return "gevent" in list_output def create_webserver_config(self) -> WebserverConfig: - """Create a WebserverConfig instance from the charm config. + """Validate worker_class and create a WebserverConfig instance from the charm config. Returns: - A new WebserverConfig instance. + A validated WebserverConfig instance. Raises: CharmConfigInvalidError: if the charm configuration is not valid. """ webserver_config: WebserverConfig = WebserverConfig.from_charm_config(dict(self.config)) - - if webserver_config.worker_class == "sync": - webserver_config.worker_class = None - - if webserver_config.worker_class: - if "gevent" != typing.cast(str, webserver_config.worker_class): - logger.error( - "Only 'gevent' and 'sync' are allowed. " - "https://documentation.ubuntu.com/rockcraft" - "/en/latest/reference/extensions/%s-framework" - "/#parts-%s-framework-async-dependencies", - self._framework_name, - self._framework_name, - ) - raise CharmConfigInvalidError( - "Only 'gevent' and 'sync' are allowed. " - "https://documentation.ubuntu.com/rockcraft/en/latest" - f"/reference/extensions/{self._framework_name}-" - f"framework/#parts-{self._framework_name}-" - "framework-async-dependencies" - ) - - if not self.check_gevent_package(): - logger.error( - "gunicorn[gevent] must be installed in the rock. " - "https://documentation.ubuntu.com/rockcraft" - "/en/latest/reference/extensions/%s-framework" - "/#parts-%s-framework-async-dependencies", - self._framework_name, - self._framework_name, - ) - raise CharmConfigInvalidError( - "gunicorn[gevent] must be installed in the rock. " - "https://documentation.ubuntu.com/rockcraft/en/latest" - f"/reference/extensions/{self._framework_name}-" - f"framework/#parts-{self._framework_name}-" - "framework-async-dependencies" - ) + if not webserver_config.worker_class: + return webserver_config + + doc_link = "https://documentation.ubuntu.com/rockcraft" + \ + f"/en/latest/reference/extensions/{self._framework_name}-framework" + \ + f"/#parts-{self._framework_name}-framework-async-dependencies" + + worker_class = None + try: + worker_class = WorkerClassEnum(webserver_config.worker_class) + except ValueError: + logger.error( + "Only 'gevent' and 'sync' are allowed. %s", + doc_link, + ) + raise CharmConfigInvalidError( + f"Only 'gevent' and 'sync' are allowed. {doc_link}" + ) + + # If the worker_class = sync is the default. + if worker_class is WorkerClassEnum.SYNC: + return webserver_config + + if not self.check_gevent_package(): + logger.error( + "gunicorn[gevent] must be installed in the rock. %s", + doc_link, + ) + raise CharmConfigInvalidError( + f"gunicorn[gevent] must be installed in the rock. {doc_link}" + ) return webserver_config diff --git a/src/paas_charm/_gunicorn/webserver.py b/src/paas_charm/_gunicorn/webserver.py index a58db76..40573d7 100644 --- a/src/paas_charm/_gunicorn/webserver.py +++ b/src/paas_charm/_gunicorn/webserver.py @@ -10,6 +10,7 @@ import signal import textwrap import typing +from enum import Enum import ops from ops.pebble import ExecError, PathError @@ -26,6 +27,22 @@ logger = logging.getLogger(__name__) +class WorkerClassEnum(str, Enum): + """Enumeration class defining async modes. + + Attributes: + SYNC (str): String representation of worker class. + GEVENT (Enum): Enumeration representation of worker class. + + Args: + str (str): String representation of worker class. + Enum (Enum): Enumeration representation of worker class. + """ + + SYNC = "sync" + GEVENT = "gevent" + + @dataclasses.dataclass class WebserverConfig: """Represent the configuration values for a web server. @@ -41,12 +58,14 @@ class WebserverConfig: """ workers: int | None = None - worker_class: str | None = None + worker_class: WorkerClassEnum | None = None threads: int | None = None keepalive: datetime.timedelta | None = None timeout: datetime.timedelta | None = None - def items(self) -> typing.Iterable[tuple[str, str | int | datetime.timedelta | None]]: + def items( + self, + ) -> typing.Iterable[tuple[str, str | WorkerClassEnum | int | datetime.timedelta | None]]: """Return the dataclass values as an iterable of the key-value pairs. Returns: @@ -61,7 +80,9 @@ def items(self) -> typing.Iterable[tuple[str, str | int | datetime.timedelta | N }.items() @classmethod - def from_charm_config(cls, config: dict[str, int | float | str | bool]) -> "WebserverConfig": + def from_charm_config( + cls, config: dict[str, WorkerClassEnum | int | float | str | bool] + ) -> "WebserverConfig": """Create a WebserverConfig object from a charm state object. Args: @@ -78,7 +99,9 @@ def from_charm_config(cls, config: dict[str, int | float | str | bool]) -> "Webs threads = config.get("webserver-threads") return cls( workers=int(typing.cast(str, workers)) if workers is not None else None, - worker_class=str(typing.cast(str, worker_class)) if worker_class is not None else None, + worker_class=( + typing.cast(WorkerClassEnum, worker_class) if worker_class is not None else None + ), threads=int(typing.cast(str, threads)) if threads is not None else None, keepalive=( datetime.timedelta(seconds=int(keepalive)) if keepalive is not None else None @@ -117,16 +140,18 @@ def _config(self) -> str: """ config_entries = [] for setting, setting_value in self._webserver_config.items(): - setting_value = typing.cast(None | str | int | datetime.timedelta, setting_value) + setting_value = typing.cast( + None | str | WorkerClassEnum | int | datetime.timedelta, setting_value + ) if setting_value is None: continue setting_value = ( setting_value - if isinstance(setting_value, (int, str)) + if isinstance(setting_value, (int, WorkerClassEnum, str)) else int(setting_value.total_seconds()) ) - if isinstance(setting_value, str): - setting_value = f"'{setting_value}'" + if isinstance(setting_value, (WorkerClassEnum, str)): + setting_value = f"'{str(setting_value)}'" config_entries.append(f"{setting} = {setting_value}") if enable_pebble_log_forwarding(): access_log = "'-'" From 1b1ab51dac63c66669ca350500460137126fa016 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 15 Nov 2024 14:38:22 +0300 Subject: [PATCH 08/27] Chore(Integration test): Write integration tests for async workers. --- examples/flask/test_async_rock/app.py | 359 ++++++++++++++++++ .../flask/test_async_rock/requirements.txt | 10 + examples/flask/test_async_rock/rockcraft.yaml | 32 ++ tests/conftest.py | 1 + tests/integration/flask/conftest.py | 24 +- tests/integration/flask/test_workers.py | 43 +++ 6 files changed, 468 insertions(+), 1 deletion(-) create mode 100644 examples/flask/test_async_rock/app.py create mode 100644 examples/flask/test_async_rock/requirements.txt create mode 100644 examples/flask/test_async_rock/rockcraft.yaml diff --git a/examples/flask/test_async_rock/app.py b/examples/flask/test_async_rock/app.py new file mode 100644 index 0000000..7d0b087 --- /dev/null +++ b/examples/flask/test_async_rock/app.py @@ -0,0 +1,359 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +import logging +import os +import socket +import time +import urllib.parse +from urllib.parse import urlparse + +import boto3 +import botocore.config +import pika +import psycopg +import pymongo +import pymongo.database +import pymongo.errors +import pymysql +import pymysql.cursors +import redis +from celery import Celery, Task +from flask import Flask, g, jsonify, request + + +def hostname(): + """Get the hostname of the current machine.""" + return socket.gethostbyname(socket.gethostname()) + + +def celery_init_app(app: Flask, broker_url: str) -> Celery: + """Initialise celery using the redis connection string. + + See https://flask.palletsprojects.com/en/3.0.x/patterns/celery/#integrate-celery-with-flask. + """ + + class FlaskTask(Task): + def __call__(self, *args: object, **kwargs: object) -> object: + with app.app_context(): + return self.run(*args, **kwargs) + + celery_app = Celery(app.name, task_cls=FlaskTask) + celery_app.set_default() + app.extensions["celery"] = celery_app + app.config.from_mapping( + CELERY=dict( + broker_url=broker_url, + result_backend=broker_url, + task_ignore_result=True, + ), + ) + celery_app.config_from_object(app.config["CELERY"]) + return celery_app + + +app = Flask(__name__) +app.config.from_prefixed_env() + +broker_url = os.environ.get("REDIS_DB_CONNECT_STRING") +# Configure Celery only if Redis is configured +celery_app = celery_init_app(app, broker_url) +redis_client = redis.Redis.from_url(broker_url) if broker_url else None + + +@celery_app.on_after_configure.connect +def setup_periodic_tasks(sender, **kwargs): + """Set up periodic tasks in the scheduler.""" + try: + # This will only have an effect in the beat scheduler. + sender.add_periodic_task(0.5, scheduled_task.s(hostname()), name="every 0.5s") + except NameError as e: + logging.exception("Failed to configure the periodic task") + + +@celery_app.task +def scheduled_task(scheduler_hostname): + """Function to run a schedule task in a worker. + + The worker that will run this task will add the scheduler hostname argument + to the "schedulers" set in Redis, and the worker's hostname to the "workers" + set in Redis. + """ + worker_hostname = hostname() + logging.info( + "scheduler host received %s in worker host %s", scheduler_hostname, worker_hostname + ) + redis_client.sadd("schedulers", scheduler_hostname) + redis_client.sadd("workers", worker_hostname) + logging.info("schedulers: %s", redis_client.smembers("schedulers")) + logging.info("workers: %s", redis_client.smembers("workers")) + # The goal is to have all workers busy in all processes. + # For that it maybe necessary to exhaust all workers, but not to get the pending tasks + # too big, so all schedulers can manage to run their scheduled tasks. + # Celery prefetches tasks, and if they cannot be run they are put in reserved. + # If all processes have tasks in reserved, this task will finish immediately to not make + # queues any longer. + inspect_obj = celery_app.control.inspect() + reserved_sizes = [len(tasks) for tasks in inspect_obj.reserved().values()] + logging.info("number of reserved tasks %s", reserved_sizes) + delay = 0 if min(reserved_sizes) > 0 else 5 + time.sleep(delay) + + +def get_mysql_database(): + """Get the mysql db connection.""" + if "mysql_db" not in g: + if "MYSQL_DB_CONNECT_STRING" in os.environ: + uri_parts = urlparse(os.environ["MYSQL_DB_CONNECT_STRING"]) + g.mysql_db = pymysql.connect( + host=uri_parts.hostname, + user=uri_parts.username, + password=uri_parts.password, + database=uri_parts.path[1:], + port=uri_parts.port, + ) + else: + return None + return g.mysql_db + + +def get_postgresql_database(): + """Get the postgresql db connection.""" + if "postgresql_db" not in g: + if "POSTGRESQL_DB_CONNECT_STRING" in os.environ: + g.postgresql_db = psycopg.connect( + conninfo=os.environ["POSTGRESQL_DB_CONNECT_STRING"], + ) + else: + return None + return g.postgresql_db + + +def get_mongodb_database() -> pymongo.database.Database | None: + """Get the mongodb db connection.""" + if "mongodb_db" not in g: + if "MONGODB_DB_CONNECT_STRING" in os.environ: + uri = os.environ["MONGODB_DB_CONNECT_STRING"] + client = pymongo.MongoClient(uri) + db = urllib.parse.urlparse(uri).path.removeprefix("/") + g.mongodb_db = client.get_database(db) + else: + return None + return g.mongodb_db + + +def get_redis_database() -> redis.Redis | None: + if "redis_db" not in g: + if "REDIS_DB_CONNECT_STRING" in os.environ: + uri = os.environ["REDIS_DB_CONNECT_STRING"] + g.redis_db = redis.Redis.from_url(uri) + else: + return None + return g.redis_db + + +def get_rabbitmq_connection() -> pika.BlockingConnection | None: + """Get rabbitmq connection.""" + if "rabbitmq" not in g: + if "RABBITMQ_HOSTNAME" in os.environ: + username = os.environ["RABBITMQ_USERNAME"] + password = os.environ["RABBITMQ_PASSWORD"] + hostname = os.environ["RABBITMQ_HOSTNAME"] + vhost = os.environ["RABBITMQ_VHOST"] + port = os.environ["RABBITMQ_PORT"] + credentials = pika.PlainCredentials(username, password) + parameters = pika.ConnectionParameters(hostname, port, vhost, credentials) + g.rabbitmq = pika.BlockingConnection(parameters) + else: + return None + return g.rabbitmq + + +def get_rabbitmq_connection_from_uri() -> pika.BlockingConnection | None: + """Get rabbitmq connection from uri.""" + if "rabbitmq_from_uri" not in g: + if "RABBITMQ_CONNECT_STRING" in os.environ: + uri = os.environ["RABBITMQ_CONNECT_STRING"] + parameters = pika.URLParameters(uri) + g.rabbitmq_from_uri = pika.BlockingConnection(parameters) + else: + return None + return g.rabbitmq_from_uri + + +def get_boto3_client(): + if "boto3_client" not in g: + if "S3_ACCESS_KEY" in os.environ: + s3_client_config = botocore.config.Config( + s3={ + "addressing_style": os.environ["S3_ADDRESSING_STYLE"], + }, + # no_proxy env variable is not read by boto3, so + # this is needed for the tests to avoid hitting the proxy. + proxies={}, + ) + g.boto3_client = boto3.client( + "s3", + os.environ["S3_REGION"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + endpoint_url=os.environ["S3_ENDPOINT"], + use_ssl=False, + config=s3_client_config, + ) + else: + return None + return g.boto3_client + + +@app.teardown_appcontext +def teardown_database(_): + """Tear down databases connections.""" + mysql_db = g.pop("mysql_db", None) + if mysql_db is not None: + mysql_db.close() + postgresql_db = g.pop("postgresql_db", None) + if postgresql_db is not None: + postgresql_db.close() + mongodb_db = g.pop("mongodb_db", None) + if mongodb_db is not None: + mongodb_db.client.close() + boto3_client = g.pop("boto3_client", None) + if boto3_client is not None: + boto3_client.close() + rabbitmq = g.pop("rabbitmq", None) + if rabbitmq is not None: + rabbitmq.close() + rabbitmq_from_uri = g.pop("rabbitmq_from_uri", None) + if rabbitmq_from_uri is not None: + rabbitmq_from_uri.close() + + +@app.route("/") +def hello_world(): + return "Hello, World!" + + +@app.route("/sleep") +def sleep(): + duration_seconds = int(request.args.get("duration")) + time.sleep(duration_seconds) + return "" + + +@app.route("/config/") +def config(config_name: str): + return jsonify(app.config.get(config_name)) + + +@app.route("/mysql/status") +def mysql_status(): + """Mysql status endpoint.""" + if database := get_mysql_database(): + with database.cursor() as cursor: + sql = "SELECT version()" + cursor.execute(sql) + cursor.fetchone() + return "SUCCESS" + return "FAIL" + + +@app.route("/s3/status") +def s3_status(): + """S3 status endpoint.""" + if client := get_boto3_client(): + bucket_name = os.environ["S3_BUCKET"] + objectsresponse = client.list_objects(Bucket=bucket_name) + return "SUCCESS" + return "FAIL" + + +@app.route("/postgresql/status") +def postgresql_status(): + """Postgresql status endpoint.""" + if database := get_postgresql_database(): + with database.cursor() as cursor: + sql = "SELECT version()" + cursor.execute(sql) + cursor.fetchone() + return "SUCCESS" + return "FAIL" + + +@app.route("/mongodb/status") +def mongodb_status(): + """Mongodb status endpoint.""" + if (database := get_mongodb_database()) is not None: + database.list_collection_names() + return "SUCCESS" + return "FAIL" + + +@app.route("/redis/status") +def redis_status(): + """Redis status endpoint.""" + if database := get_redis_database(): + try: + database.set("foo", "bar") + return "SUCCESS" + except redis.exceptions.RedisError: + logging.exception("Error querying redis") + return "FAIL" + + +@app.route("/redis/clear_celery_stats") +def redis_celery_clear_stats(): + """Reset Redis statistics about workers and schedulers.""" + if database := get_redis_database(): + try: + database.delete("workers") + database.delete("schedulers") + return "SUCCESS" + except redis.exceptions.RedisError: + logging.exception("Error querying redis") + return "FAIL", 500 + + +@app.route("/redis/celery_stats") +def redis_celery_stats(): + """Read Redis statistics about workers and schedulers.""" + if database := get_redis_database(): + try: + worker_set = [str(host) for host in database.smembers("workers")] + beat_set = [str(host) for host in database.smembers("schedulers")] + return jsonify({"workers": worker_set, "schedulers": beat_set}) + except redis.exceptions.RedisError: + logging.exception("Error querying redis") + return "FAIL", 500 + + +@app.route("/rabbitmq/send") +def rabbitmq_send(): + """Send a message to "charm" queue.""" + if connection := get_rabbitmq_connection(): + channel = connection.channel() + channel.queue_declare(queue="charm") + channel.basic_publish(exchange="", routing_key="charm", body="SUCCESS") + return "SUCCESS" + return "FAIL" + + +@app.route("/rabbitmq/receive") +def rabbitmq_receive(): + """Receive a message from "charm" queue in blocking form.""" + if connection := get_rabbitmq_connection_from_uri(): + channel = connection.channel() + method_frame, _header_frame, body = channel.basic_get("charm") + if method_frame: + channel.basic_ack(method_frame.delivery_tag) + if body == b"SUCCESS": + return "SUCCESS" + return "FAIL. INCORRECT MESSAGE." + return "FAIL. NO MESSAGE." + return "FAIL. NO CONNECTION." + + +@app.route("/env") +def get_env(): + """Return environment variables""" + return jsonify(dict(os.environ)) diff --git a/examples/flask/test_async_rock/requirements.txt b/examples/flask/test_async_rock/requirements.txt new file mode 100644 index 0000000..2ff69c0 --- /dev/null +++ b/examples/flask/test_async_rock/requirements.txt @@ -0,0 +1,10 @@ +Flask +PyMySQL +PyMySQL[rsa] +PyMySQL[ed25519] +psycopg[binary] +pymongo +redis[hiredis] +boto3 +pika +celery diff --git a/examples/flask/test_async_rock/rockcraft.yaml b/examples/flask/test_async_rock/rockcraft.yaml new file mode 100644 index 0000000..43669af --- /dev/null +++ b/examples/flask/test_async_rock/rockcraft.yaml @@ -0,0 +1,32 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +name: test-async-flask +summary: A flask async worker test app +description: OCI image for the async worker test flask app +version: "0.1" +base: ubuntu@22.04 +license: Apache-2.0 +platforms: + amd64: + +extensions: + - flask-framework + +parts: + flask-framework/async-dependencies: + plugin: python + source: . + +services: + celery-worker: + override: replace + command: celery -A app:celery_app worker -c 2 --loglevel DEBUG + startup: enabled + user: _daemon_ + working-dir: /flask/app + celery-beat-scheduler: + override: replace + command: celery -A app:celery_app beat --loglevel DEBUG -s /tmp/celerybeat-schedule + startup: enabled + user: _daemon_ + working-dir: /flask/app diff --git a/tests/conftest.py b/tests/conftest.py index c6002bb..de22def 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ def pytest_addoption(parser): """Define some command line options for integration and unit tests.""" parser.addoption("--charm-file", action="extend", nargs="+", default=[]) parser.addoption("--test-flask-image", action="store") + parser.addoption("--test-async-flask-image", action="store") parser.addoption("--test-db-flask-image", action="store") parser.addoption("--django-app-image", action="store") parser.addoption("--fastapi-app-image", action="store") diff --git a/tests/integration/flask/conftest.py b/tests/integration/flask/conftest.py index ac5b75a..40ea25b 100644 --- a/tests/integration/flask/conftest.py +++ b/tests/integration/flask/conftest.py @@ -26,6 +26,14 @@ def cwd(): return os.chdir(PROJECT_ROOT / "examples/flask") +@pytest.fixture(scope="module", name="test_async_flask_image") +def fixture_test_async_flask_image(pytestconfig: Config): + """Return the --test-async-flask-image test parameter.""" + test_flask_image = pytestconfig.getoption("--test-async-flask-image") + if not test_flask_image: + raise ValueError("the following arguments are required: --test-async-flask-image") + return test_flask_image + @pytest.fixture(scope="module", name="test_flask_image") def fixture_test_flask_image(pytestconfig: Config): """Return the --test-flask-image test parameter.""" @@ -34,7 +42,6 @@ def fixture_test_flask_image(pytestconfig: Config): raise ValueError("the following arguments are required: --test-flask-image") return test_flask_image - @pytest.fixture(scope="module", name="test_db_flask_image") def fixture_test_db_flask_image(pytestconfig: Config): """Return the --test-flask-image test parameter.""" @@ -111,6 +118,21 @@ async def flask_db_app_fixture(build_charm: str, model: Model, test_db_flask_ima return app +@pytest_asyncio.fixture(scope="module", name="flask_async_app") +async def flask_async_app_fixture(build_charm: str, model: Model, test_async_flask_image: str): + """Build and deploy the flask charm with test-async-flask image.""" + app_name = "flask-async-k8s" + + resources = { + "flask-app-image": test_async_flask_image, + } + app = await model.deploy( + build_charm, resources=resources, application_name=app_name, series="jammy" + ) + await model.wait_for_idle(raise_on_blocked=True) + return app + + @pytest_asyncio.fixture(scope="module", name="traefik_app") async def deploy_traefik_fixture( model: Model, diff --git a/tests/integration/flask/test_workers.py b/tests/integration/flask/test_workers.py index 9dea9fd..0d8a2a6 100644 --- a/tests/integration/flask/test_workers.py +++ b/tests/integration/flask/test_workers.py @@ -68,3 +68,46 @@ def check_correct_celery_stats(num_schedulers, num_workers): ) except asyncio.TimeoutError: assert False, "Failed to get 2 workers and 1 scheduler" + +@pytest.mark.parametrize( + "worker_class, expected_result", + [ + ("eventlet", 'blocked'), + ("gevent", 'active'), + ("sync", 'active'), + ], +) +@pytest.mark.usefixtures("flask_async_app") +async def test_async_workers( + ops_test: OpsTest, model: Model, flask_async_app: Application, get_unit_ips, worker_class: str, expected_result: bool +): + """ + arrange: Flask is deployed with async enabled rock. + act: Change gunicorn worker class. + assert: Charm should only let the class to be 'sync' or 'gevent'. + If it is something other than these, then the unit should be blocked. + """ + await flask_async_app.set_config({"webserver-worker-class": worker_class}) + await model.wait_for_idle(apps=[flask_async_app.name], status=expected_result, timeout=60) + + +@pytest.mark.parametrize( + "worker_class, expected_result", + [ + ("gevent", 'blocked'), + ("eventlet", 'blocked'), + ("sync", 'active'), + ], +) +@pytest.mark.usefixtures("flask_app") +async def test_async_workers_fail( + ops_test: OpsTest, model: Model, flask_app: Application, get_unit_ips, worker_class: str, expected_result: str +): + """ + arrange: Flask is deployed with async not enabled rock. + act: Change gunicorn worker class. + assert: Charm should only let the class to be 'sync'. + If it is 'gevent' or something else, then the unit should be blocked. + """ + await flask_app.set_config({"webserver-worker-class": worker_class}) + await model.wait_for_idle(apps=[flask_app.name], status=expected_result, timeout=60) From 19f367a1a2954c06f4e17a3b316da1841eab6416 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 15 Nov 2024 14:43:39 +0300 Subject: [PATCH 09/27] Chore(): Make linter happy --- src/paas_charm/_gunicorn/charm.py | 12 ++++++----- tests/integration/flask/conftest.py | 2 ++ tests/integration/flask/test_workers.py | 27 +++++++++++++++++-------- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index a27a077..2f2e9c0 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -48,21 +48,23 @@ def create_webserver_config(self) -> WebserverConfig: if not webserver_config.worker_class: return webserver_config - doc_link = "https://documentation.ubuntu.com/rockcraft" + \ - f"/en/latest/reference/extensions/{self._framework_name}-framework" + \ - f"/#parts-{self._framework_name}-framework-async-dependencies" + doc_link = ( + "https://documentation.ubuntu.com/rockcraft" + + f"/en/latest/reference/extensions/{self._framework_name}-framework" + + f"/#parts-{self._framework_name}-framework-async-dependencies" + ) worker_class = None try: worker_class = WorkerClassEnum(webserver_config.worker_class) - except ValueError: + except ValueError as exc: logger.error( "Only 'gevent' and 'sync' are allowed. %s", doc_link, ) raise CharmConfigInvalidError( f"Only 'gevent' and 'sync' are allowed. {doc_link}" - ) + ) from exc # If the worker_class = sync is the default. if worker_class is WorkerClassEnum.SYNC: diff --git a/tests/integration/flask/conftest.py b/tests/integration/flask/conftest.py index 40ea25b..1e40a63 100644 --- a/tests/integration/flask/conftest.py +++ b/tests/integration/flask/conftest.py @@ -34,6 +34,7 @@ def fixture_test_async_flask_image(pytestconfig: Config): raise ValueError("the following arguments are required: --test-async-flask-image") return test_flask_image + @pytest.fixture(scope="module", name="test_flask_image") def fixture_test_flask_image(pytestconfig: Config): """Return the --test-flask-image test parameter.""" @@ -42,6 +43,7 @@ def fixture_test_flask_image(pytestconfig: Config): raise ValueError("the following arguments are required: --test-flask-image") return test_flask_image + @pytest.fixture(scope="module", name="test_db_flask_image") def fixture_test_db_flask_image(pytestconfig: Config): """Return the --test-flask-image test parameter.""" diff --git a/tests/integration/flask/test_workers.py b/tests/integration/flask/test_workers.py index 0d8a2a6..3c284f1 100644 --- a/tests/integration/flask/test_workers.py +++ b/tests/integration/flask/test_workers.py @@ -69,17 +69,23 @@ def check_correct_celery_stats(num_schedulers, num_workers): except asyncio.TimeoutError: assert False, "Failed to get 2 workers and 1 scheduler" + @pytest.mark.parametrize( "worker_class, expected_result", [ - ("eventlet", 'blocked'), - ("gevent", 'active'), - ("sync", 'active'), + ("eventlet", "blocked"), + ("gevent", "active"), + ("sync", "active"), ], ) @pytest.mark.usefixtures("flask_async_app") async def test_async_workers( - ops_test: OpsTest, model: Model, flask_async_app: Application, get_unit_ips, worker_class: str, expected_result: bool + ops_test: OpsTest, + model: Model, + flask_async_app: Application, + get_unit_ips, + worker_class: str, + expected_result: bool, ): """ arrange: Flask is deployed with async enabled rock. @@ -94,14 +100,19 @@ async def test_async_workers( @pytest.mark.parametrize( "worker_class, expected_result", [ - ("gevent", 'blocked'), - ("eventlet", 'blocked'), - ("sync", 'active'), + ("gevent", "blocked"), + ("eventlet", "blocked"), + ("sync", "active"), ], ) @pytest.mark.usefixtures("flask_app") async def test_async_workers_fail( - ops_test: OpsTest, model: Model, flask_app: Application, get_unit_ips, worker_class: str, expected_result: str + ops_test: OpsTest, + model: Model, + flask_app: Application, + get_unit_ips, + worker_class: str, + expected_result: str, ): """ arrange: Flask is deployed with async not enabled rock. From bd15e347cc67809f90c8b26122c74c91f34a222a Mon Sep 17 00:00:00 2001 From: ali ugur Date: Mon, 18 Nov 2024 09:12:38 +0300 Subject: [PATCH 10/27] Chore(): Renamed new tests. --- tests/integration/flask/test_workers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/flask/test_workers.py b/tests/integration/flask/test_workers.py index 3c284f1..5944479 100644 --- a/tests/integration/flask/test_workers.py +++ b/tests/integration/flask/test_workers.py @@ -79,7 +79,7 @@ def check_correct_celery_stats(num_schedulers, num_workers): ], ) @pytest.mark.usefixtures("flask_async_app") -async def test_async_workers( +async def test_async_workers_config( ops_test: OpsTest, model: Model, flask_async_app: Application, @@ -106,7 +106,7 @@ async def test_async_workers( ], ) @pytest.mark.usefixtures("flask_app") -async def test_async_workers_fail( +async def test_async_workers_config_fail( ops_test: OpsTest, model: Model, flask_app: Application, From 5795a9b505941ea8432040cd80c651f7e8c59a53 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Mon, 18 Nov 2024 16:02:26 +0300 Subject: [PATCH 11/27] Chore(tests): Add async config tests for Django framework --- .../django_app/{ => django_app}/__init__.py | 0 .../django_app/{ => django_app}/asgi.py | 0 .../django_app/{ => django_app}/settings.py | 0 .../django_app/{ => django_app}/urls.py | 0 .../django_app/{ => django_app}/wsgi.py | 0 .../django_app/{ => django_app}/manage.py | 0 .../django_app/{ => django_app}/migrate.sh | 0 .../{ => django_app}/testing/__init__.py | 0 .../{ => django_app}/testing/admin.py | 0 .../{ => django_app}/testing/apps.py | 0 .../testing/migrations/__init__.py | 0 .../{ => django_app}/testing/models.py | 0 .../{ => django_app}/testing/tests.py | 0 .../{ => django_app}/testing/views.py | 0 .../django/{ => django_app}/requirements.txt | 0 .../django/{ => django_app}/rockcraft.yaml | 0 .../django_async_app/__init__.py | 2 + .../django_async_app/django_async_app/asgi.py | 19 +++ .../django_async_app/settings.py | 130 ++++++++++++++++++ .../django_async_app/django_async_app/urls.py | 32 +++++ .../django_async_app/django_async_app/wsgi.py | 19 +++ .../django_async_app/manage.py | 26 ++++ .../django_async_app/migrate.sh | 5 + .../django_async_app/testing/__init__.py | 2 + .../django_async_app/testing/admin.py | 6 + .../django_async_app/testing/apps.py | 9 ++ .../testing/migrations/__init__.py | 2 + .../django_async_app/testing/models.py | 6 + .../django_async_app/testing/tests.py | 6 + .../django_async_app/testing/views.py | 39 ++++++ .../django/django_async_app/requirements.txt | 3 + .../django/django_async_app/rockcraft.yaml | 19 +++ examples/flask/test_async_rock/rockcraft.yaml | 4 +- tests/conftest.py | 1 + tests/integration/django/conftest.py | 30 +++- tests/integration/django/test_workers.py | 72 ++++++++++ 36 files changed, 429 insertions(+), 3 deletions(-) rename examples/django/django_app/django_app/{ => django_app}/__init__.py (100%) rename examples/django/django_app/django_app/{ => django_app}/asgi.py (100%) rename examples/django/django_app/django_app/{ => django_app}/settings.py (100%) rename examples/django/django_app/django_app/{ => django_app}/urls.py (100%) rename examples/django/django_app/django_app/{ => django_app}/wsgi.py (100%) rename examples/django/django_app/{ => django_app}/manage.py (100%) rename examples/django/django_app/{ => django_app}/migrate.sh (100%) rename examples/django/django_app/{ => django_app}/testing/__init__.py (100%) rename examples/django/django_app/{ => django_app}/testing/admin.py (100%) rename examples/django/django_app/{ => django_app}/testing/apps.py (100%) rename examples/django/django_app/{ => django_app}/testing/migrations/__init__.py (100%) rename examples/django/django_app/{ => django_app}/testing/models.py (100%) rename examples/django/django_app/{ => django_app}/testing/tests.py (100%) rename examples/django/django_app/{ => django_app}/testing/views.py (100%) rename examples/django/{ => django_app}/requirements.txt (100%) rename examples/django/{ => django_app}/rockcraft.yaml (100%) create mode 100644 examples/django/django_async_app/django_async_app/django_async_app/__init__.py create mode 100644 examples/django/django_async_app/django_async_app/django_async_app/asgi.py create mode 100644 examples/django/django_async_app/django_async_app/django_async_app/settings.py create mode 100644 examples/django/django_async_app/django_async_app/django_async_app/urls.py create mode 100644 examples/django/django_async_app/django_async_app/django_async_app/wsgi.py create mode 100755 examples/django/django_async_app/django_async_app/manage.py create mode 100644 examples/django/django_async_app/django_async_app/migrate.sh create mode 100644 examples/django/django_async_app/django_async_app/testing/__init__.py create mode 100644 examples/django/django_async_app/django_async_app/testing/admin.py create mode 100644 examples/django/django_async_app/django_async_app/testing/apps.py create mode 100644 examples/django/django_async_app/django_async_app/testing/migrations/__init__.py create mode 100644 examples/django/django_async_app/django_async_app/testing/models.py create mode 100644 examples/django/django_async_app/django_async_app/testing/tests.py create mode 100644 examples/django/django_async_app/django_async_app/testing/views.py create mode 100644 examples/django/django_async_app/requirements.txt create mode 100644 examples/django/django_async_app/rockcraft.yaml create mode 100644 tests/integration/django/test_workers.py diff --git a/examples/django/django_app/django_app/__init__.py b/examples/django/django_app/django_app/django_app/__init__.py similarity index 100% rename from examples/django/django_app/django_app/__init__.py rename to examples/django/django_app/django_app/django_app/__init__.py diff --git a/examples/django/django_app/django_app/asgi.py b/examples/django/django_app/django_app/django_app/asgi.py similarity index 100% rename from examples/django/django_app/django_app/asgi.py rename to examples/django/django_app/django_app/django_app/asgi.py diff --git a/examples/django/django_app/django_app/settings.py b/examples/django/django_app/django_app/django_app/settings.py similarity index 100% rename from examples/django/django_app/django_app/settings.py rename to examples/django/django_app/django_app/django_app/settings.py diff --git a/examples/django/django_app/django_app/urls.py b/examples/django/django_app/django_app/django_app/urls.py similarity index 100% rename from examples/django/django_app/django_app/urls.py rename to examples/django/django_app/django_app/django_app/urls.py diff --git a/examples/django/django_app/django_app/wsgi.py b/examples/django/django_app/django_app/django_app/wsgi.py similarity index 100% rename from examples/django/django_app/django_app/wsgi.py rename to examples/django/django_app/django_app/django_app/wsgi.py diff --git a/examples/django/django_app/manage.py b/examples/django/django_app/django_app/manage.py similarity index 100% rename from examples/django/django_app/manage.py rename to examples/django/django_app/django_app/manage.py diff --git a/examples/django/django_app/migrate.sh b/examples/django/django_app/django_app/migrate.sh similarity index 100% rename from examples/django/django_app/migrate.sh rename to examples/django/django_app/django_app/migrate.sh diff --git a/examples/django/django_app/testing/__init__.py b/examples/django/django_app/django_app/testing/__init__.py similarity index 100% rename from examples/django/django_app/testing/__init__.py rename to examples/django/django_app/django_app/testing/__init__.py diff --git a/examples/django/django_app/testing/admin.py b/examples/django/django_app/django_app/testing/admin.py similarity index 100% rename from examples/django/django_app/testing/admin.py rename to examples/django/django_app/django_app/testing/admin.py diff --git a/examples/django/django_app/testing/apps.py b/examples/django/django_app/django_app/testing/apps.py similarity index 100% rename from examples/django/django_app/testing/apps.py rename to examples/django/django_app/django_app/testing/apps.py diff --git a/examples/django/django_app/testing/migrations/__init__.py b/examples/django/django_app/django_app/testing/migrations/__init__.py similarity index 100% rename from examples/django/django_app/testing/migrations/__init__.py rename to examples/django/django_app/django_app/testing/migrations/__init__.py diff --git a/examples/django/django_app/testing/models.py b/examples/django/django_app/django_app/testing/models.py similarity index 100% rename from examples/django/django_app/testing/models.py rename to examples/django/django_app/django_app/testing/models.py diff --git a/examples/django/django_app/testing/tests.py b/examples/django/django_app/django_app/testing/tests.py similarity index 100% rename from examples/django/django_app/testing/tests.py rename to examples/django/django_app/django_app/testing/tests.py diff --git a/examples/django/django_app/testing/views.py b/examples/django/django_app/django_app/testing/views.py similarity index 100% rename from examples/django/django_app/testing/views.py rename to examples/django/django_app/django_app/testing/views.py diff --git a/examples/django/requirements.txt b/examples/django/django_app/requirements.txt similarity index 100% rename from examples/django/requirements.txt rename to examples/django/django_app/requirements.txt diff --git a/examples/django/rockcraft.yaml b/examples/django/django_app/rockcraft.yaml similarity index 100% rename from examples/django/rockcraft.yaml rename to examples/django/django_app/rockcraft.yaml diff --git a/examples/django/django_async_app/django_async_app/django_async_app/__init__.py b/examples/django/django_async_app/django_async_app/django_async_app/__init__.py new file mode 100644 index 0000000..e3979c0 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/django_async_app/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/examples/django/django_async_app/django_async_app/django_async_app/asgi.py b/examples/django/django_async_app/django_async_app/django_async_app/asgi.py new file mode 100644 index 0000000..76f2304 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/django_async_app/asgi.py @@ -0,0 +1,19 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +""" +ASGI config for django_async_app project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_async_app.settings") + +application = get_asgi_application() diff --git a/examples/django/django_async_app/django_async_app/django_async_app/settings.py b/examples/django/django_async_app/django_async_app/django_async_app/settings.py new file mode 100644 index 0000000..24b5e10 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/django_async_app/settings.py @@ -0,0 +1,130 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +""" +Django settings for django_async_app project. + +Generated by 'django-admin startproject' using Django 5.0.2. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/5.0/ref/settings/ +""" + +import json +import os +import urllib.parse +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY", "secret") + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = os.environ.get("DJANGO_DEBUG", "true") == "true" + +ALLOWED_HOSTS = json.loads(os.environ.get("DJANGO_ALLOWED_HOSTS", '["*"]')) + + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "django_async_app.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "django_async_app.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/5.0/ref/settings/#databases +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.environ.get("POSTGRESQL_DB_NAME"), + "USER": os.environ.get("POSTGRESQL_DB_USERNAME"), + "PASSWORD": os.environ.get("POSTGRESQL_DB_PASSWORD"), + "HOST": os.environ.get("POSTGRESQL_DB_HOSTNAME"), + "PORT": os.environ.get("POSTGRESQL_DB_PORT", "5432"), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/5.0/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/5.0/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/examples/django/django_async_app/django_async_app/django_async_app/urls.py b/examples/django/django_async_app/django_async_app/django_async_app/urls.py new file mode 100644 index 0000000..deb38d8 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/django_async_app/urls.py @@ -0,0 +1,32 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +""" +URL configuration for django_async_app project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/5.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" + +from django.contrib import admin +from django.urls import path +from testing.views import environ, get_settings, login, sleep, user_count + +urlpatterns = [ + path("admin/", admin.site.urls), + path("settings/", get_settings, name="get_settings"), + path("len/users", user_count, name="user_count"), + path("environ", environ, name="environ"), + path("sleep", sleep, name="sleep"), + path("login", login, name="login"), +] diff --git a/examples/django/django_async_app/django_async_app/django_async_app/wsgi.py b/examples/django/django_async_app/django_async_app/django_async_app/wsgi.py new file mode 100644 index 0000000..968f6f5 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/django_async_app/wsgi.py @@ -0,0 +1,19 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +""" +WSGI config for django_async_app project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_async_app.settings") + +application = get_wsgi_application() diff --git a/examples/django/django_async_app/django_async_app/manage.py b/examples/django/django_async_app/django_async_app/manage.py new file mode 100755 index 0000000..6c134b1 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/manage.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_async_app.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/examples/django/django_async_app/django_async_app/migrate.sh b/examples/django/django_async_app/django_async_app/migrate.sh new file mode 100644 index 0000000..ce3a73c --- /dev/null +++ b/examples/django/django_async_app/django_async_app/migrate.sh @@ -0,0 +1,5 @@ +#!/usr/bin/bash +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +python3 manage.py migrate diff --git a/examples/django/django_async_app/django_async_app/testing/__init__.py b/examples/django/django_async_app/django_async_app/testing/__init__.py new file mode 100644 index 0000000..e3979c0 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/testing/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/examples/django/django_async_app/django_async_app/testing/admin.py b/examples/django/django_async_app/django_async_app/testing/admin.py new file mode 100644 index 0000000..b111777 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/testing/admin.py @@ -0,0 +1,6 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +from django.contrib import admin + +# Register your models here. diff --git a/examples/django/django_async_app/django_async_app/testing/apps.py b/examples/django/django_async_app/django_async_app/testing/apps.py new file mode 100644 index 0000000..f435e0c --- /dev/null +++ b/examples/django/django_async_app/django_async_app/testing/apps.py @@ -0,0 +1,9 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +from django.apps import AppConfig + + +class TestingConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "testing" diff --git a/examples/django/django_async_app/django_async_app/testing/migrations/__init__.py b/examples/django/django_async_app/django_async_app/testing/migrations/__init__.py new file mode 100644 index 0000000..e3979c0 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/testing/migrations/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/examples/django/django_async_app/django_async_app/testing/models.py b/examples/django/django_async_app/django_async_app/testing/models.py new file mode 100644 index 0000000..dde0a81 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/testing/models.py @@ -0,0 +1,6 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +from django.db import models + +# Create your models here. diff --git a/examples/django/django_async_app/django_async_app/testing/tests.py b/examples/django/django_async_app/django_async_app/testing/tests.py new file mode 100644 index 0000000..922bda5 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/testing/tests.py @@ -0,0 +1,6 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +from django.test import TestCase + +# Create your tests here. diff --git a/examples/django/django_async_app/django_async_app/testing/views.py b/examples/django/django_async_app/django_async_app/testing/views.py new file mode 100644 index 0000000..07b39f7 --- /dev/null +++ b/examples/django/django_async_app/django_async_app/testing/views.py @@ -0,0 +1,39 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +import os +import time + +from django.conf import settings +from django.contrib.auth import authenticate +from django.contrib.auth.models import User +from django.http import HttpResponse, JsonResponse + + +def environ(request): + return JsonResponse(dict(os.environ)) + + +def user_count(request): + return JsonResponse(User.objects.count(), safe=False) + + +def get_settings(request, name): + if hasattr(settings, name): + return JsonResponse(getattr(settings, name), safe=False) + else: + return JsonResponse({"error": f"settings {name!r} not found"}, status=404) + + +def sleep(request): + duration = request.GET.get("duration") + time.sleep(int(duration)) + return HttpResponse() + + +def login(request): + user = authenticate(username=request.GET.get("username"), password=request.GET.get("password")) + if user is not None: + return HttpResponse(status=200) + else: + return HttpResponse(status=403) diff --git a/examples/django/django_async_app/requirements.txt b/examples/django/django_async_app/requirements.txt new file mode 100644 index 0000000..2efd6d5 --- /dev/null +++ b/examples/django/django_async_app/requirements.txt @@ -0,0 +1,3 @@ +Django +tzdata +psycopg2-binary diff --git a/examples/django/django_async_app/rockcraft.yaml b/examples/django/django_async_app/rockcraft.yaml new file mode 100644 index 0000000..55d9ea0 --- /dev/null +++ b/examples/django/django_async_app/rockcraft.yaml @@ -0,0 +1,19 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +name: django-async-app +summary: Example Async Django application image. +description: Example Async Django application image. +version: "0.1" +base: ubuntu@22.04 +license: Apache-2.0 +platforms: + amd64: + +extensions: + - django-framework + +parts: + django-framework/async-dependencies: + python-packages: + - gunicorn[gevent] diff --git a/examples/flask/test_async_rock/rockcraft.yaml b/examples/flask/test_async_rock/rockcraft.yaml index 43669af..06248ba 100644 --- a/examples/flask/test_async_rock/rockcraft.yaml +++ b/examples/flask/test_async_rock/rockcraft.yaml @@ -14,8 +14,8 @@ extensions: parts: flask-framework/async-dependencies: - plugin: python - source: . + python-packages: + - gunicorn[gevent] services: celery-worker: diff --git a/tests/conftest.py b/tests/conftest.py index de22def..974a0a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ def pytest_addoption(parser): parser.addoption("--test-async-flask-image", action="store") parser.addoption("--test-db-flask-image", action="store") parser.addoption("--django-app-image", action="store") + parser.addoption("--django-async-app-image", action="store") parser.addoption("--fastapi-app-image", action="store") parser.addoption("--go-app-image", action="store") parser.addoption("--localstack-address", action="store") diff --git a/tests/integration/django/conftest.py b/tests/integration/django/conftest.py index b93c3e8..76fdb1d 100644 --- a/tests/integration/django/conftest.py +++ b/tests/integration/django/conftest.py @@ -25,12 +25,20 @@ def cwd(): @pytest.fixture(scope="module", name="django_app_image") def fixture_django_app_image(pytestconfig: Config): - """Return the --flask-app-image test parameter.""" + """Return the --django-app-image test parameter.""" image = pytestconfig.getoption("--django-app-image") if not image: raise ValueError("the following arguments are required: --django-app-image") return image +@pytest.fixture(scope="module", name="django_async_app_image") +def fixture_django_async_app_image(pytestconfig: Config): + """Return the --django-async-app-image test parameter.""" + image = pytestconfig.getoption("--django-async-app-image") + if not image: + raise ValueError("the following arguments are required: --django-async-app-image") + return image + @pytest_asyncio.fixture(scope="module", name="charm_file") async def charm_file_fixture( @@ -49,6 +57,7 @@ async def charm_file_fixture( charm_file, { "allowed-hosts": {"type": "string"}, + "webserver-worker-class": {"type": "string"}, }, tmp_path_factory.mktemp("django"), ) @@ -74,6 +83,25 @@ async def django_app_fixture(charm_file: str, model: Model, django_app_image: st return app +@pytest_asyncio.fixture(scope="module", name="django_async_app") +async def django_async_app_fixture(charm_file: str, model: Model, django_async_app_image: str, postgresql_k8s): + """Build and deploy the async django charm.""" + app_name = "django-async-k8s" + + resources = { + "django-app-image": django_async_app_image, + } + app = await model.deploy( + charm_file, + application_name=app_name, + config={"django-allowed-hosts": "*"}, + resources=resources, + series="jammy", + ) + await model.integrate(app_name, "postgresql-k8s") + await model.wait_for_idle(status="active") + return app + @pytest_asyncio.fixture async def update_config(model: Model, request: FixtureRequest, django_app: Application): """Update the django application configuration. diff --git a/tests/integration/django/test_workers.py b/tests/integration/django/test_workers.py new file mode 100644 index 0000000..f65ce27 --- /dev/null +++ b/tests/integration/django/test_workers.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Integration tests for Django charm.""" +import asyncio +import logging +import time +import typing + +import pytest +import requests +from juju.application import Application +from juju.model import Model +from juju.utils import block_until +from pytest_operator.plugin import OpsTest + +logger = logging.getLogger(__name__) + + +@pytest.mark.parametrize( + "worker_class, expected_result", + [ + ("eventlet", "blocked"), + ("gevent", "active"), + ("sync", "active"), + ], +) +@pytest.mark.usefixtures("django_async_app") +async def test_async_workers_config( + ops_test: OpsTest, + model: Model, + django_async_app: Application, + get_unit_ips, + worker_class: str, + expected_result: bool, +): + """ + arrange: Django is deployed with async enabled rock. + act: Change gunicorn worker class. + assert: Charm should only let the class to be 'sync' or 'gevent'. + If it is something other than these, then the unit should be blocked. + """ + await django_async_app.set_config({"webserver-worker-class": worker_class}) + await model.wait_for_idle(apps=[django_async_app.name], status=expected_result, timeout=60) + + +@pytest.mark.parametrize( + "worker_class, expected_result", + [ + ("gevent", "blocked"), + ("eventlet", "blocked"), + ("sync", "active"), + ], +) +@pytest.mark.usefixtures("django_app") +async def test_async_workers_config_fail( + ops_test: OpsTest, + model: Model, + django_app: Application, + get_unit_ips, + worker_class: str, + expected_result: str, +): + """ + arrange: Django is deployed with async not enabled rock. + act: Change gunicorn worker class. + assert: Charm should only let the class to be 'sync'. + If it is 'gevent' or something else, then the unit should be blocked. + """ + await django_app.set_config({"webserver-worker-class": worker_class}) + await model.wait_for_idle(apps=[django_app.name], status=expected_result, timeout=60) From f2e22693974a9c8000b8645ae34903b3d890fd6f Mon Sep 17 00:00:00 2001 From: ali ugur Date: Mon, 18 Nov 2024 20:39:45 +0300 Subject: [PATCH 12/27] Chore(lint): Format the code --- tests/integration/django/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/django/conftest.py b/tests/integration/django/conftest.py index 76fdb1d..3f01742 100644 --- a/tests/integration/django/conftest.py +++ b/tests/integration/django/conftest.py @@ -31,6 +31,7 @@ def fixture_django_app_image(pytestconfig: Config): raise ValueError("the following arguments are required: --django-app-image") return image + @pytest.fixture(scope="module", name="django_async_app_image") def fixture_django_async_app_image(pytestconfig: Config): """Return the --django-async-app-image test parameter.""" @@ -84,7 +85,9 @@ async def django_app_fixture(charm_file: str, model: Model, django_app_image: st @pytest_asyncio.fixture(scope="module", name="django_async_app") -async def django_async_app_fixture(charm_file: str, model: Model, django_async_app_image: str, postgresql_k8s): +async def django_async_app_fixture( + charm_file: str, model: Model, django_async_app_image: str, postgresql_k8s +): """Build and deploy the async django charm.""" app_name = "django-async-k8s" @@ -102,6 +105,7 @@ async def django_async_app_fixture(charm_file: str, model: Model, django_async_a await model.wait_for_idle(status="active") return app + @pytest_asyncio.fixture async def update_config(model: Model, request: FixtureRequest, django_app: Application): """Update the django application configuration. From 4435b06e48d2985595f0f5124bae9cfa4154b971 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Wed, 20 Nov 2024 11:25:10 +0300 Subject: [PATCH 13/27] Chore(docs): Update async doc link --- src/paas_charm/_gunicorn/charm.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index 2f2e9c0..a6c589d 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -48,11 +48,7 @@ def create_webserver_config(self) -> WebserverConfig: if not webserver_config.worker_class: return webserver_config - doc_link = ( - "https://documentation.ubuntu.com/rockcraft" - + f"/en/latest/reference/extensions/{self._framework_name}-framework" - + f"/#parts-{self._framework_name}-framework-async-dependencies" - ) + doc_link = f"https://bit.ly/{self._framework_name}-async-doc" worker_class = None try: From 10a314bccae09628e0ee6c0b269fe634a49e5acc Mon Sep 17 00:00:00 2001 From: ali ugur Date: Wed, 27 Nov 2024 11:29:03 +0300 Subject: [PATCH 14/27] Chore(): Change config tests to uni, add integration test. Other comments --- examples/django/charm/charmcraft.yaml | 9 +- examples/flask/charmcraft.yaml | 9 +- examples/flask/test_async_rock/app.py | 337 ------------------ .../flask/test_async_rock/requirements.txt | 9 - examples/flask/test_async_rock/rockcraft.yaml | 13 - src/paas_charm/_gunicorn/charm.py | 6 +- tests/integration/flask/requirements.txt | 1 + tests/integration/flask/test_workers.py | 64 ++-- tests/unit/django/test_charm.py | 49 +++ tests/unit/django/test_workers.py | 96 +++++ tests/unit/flask/test_workers.py | 76 +++- 11 files changed, 258 insertions(+), 411 deletions(-) create mode 100644 tests/unit/django/test_workers.py diff --git a/examples/django/charm/charmcraft.yaml b/examples/django/charm/charmcraft.yaml index 05bf5df..4be002f 100644 --- a/examples/django/charm/charmcraft.yaml +++ b/examples/django/charm/charmcraft.yaml @@ -50,9 +50,9 @@ config: type: string django-secret-key-id: description: >- - This configuration is similar to `django-secret-key`, but instead accepts a Juju user secret ID. - The secret should contain a single key, "value", which maps to the actual Django secret key. - To create the secret, run the following command: + This configuration is similar to `django-secret-key`, but instead accepts a Juju user secret ID. + The secret should contain a single key, "value", which maps to the actual Django secret key. + To create the secret, run the following command: `juju add-secret my-django-secret-key value= && juju grant-secret my-django-secret-key django-k8s`, and use the outputted secret ID to configure this option. type: secret @@ -69,6 +69,9 @@ config: webserver-workers: description: The number of webserver worker processes for handling requests. type: int + webserver-worker-class: + description: The method of webserver worker processes for handling requests. Can be either 'gevent' or 'sync'. + type: string containers: django-app: resource: django-app-image diff --git a/examples/flask/charmcraft.yaml b/examples/flask/charmcraft.yaml index 292dba0..adb36bc 100644 --- a/examples/flask/charmcraft.yaml +++ b/examples/flask/charmcraft.yaml @@ -57,9 +57,9 @@ config: type: string flask-secret-key-id: description: >- - This configuration is similar to `flask-secret-key`, but instead accepts a Juju user secret ID. - The secret should contain a single key, "value", which maps to the actual Flask secret key. - To create the secret, run the following command: + This configuration is similar to `flask-secret-key`, but instead accepts a Juju user secret ID. + The secret should contain a single key, "value", which maps to the actual Flask secret key. + To create the secret, run the following command: `juju add-secret my-flask-secret-key value= && juju grant-secret my-flask-secret-key flask-k8s`, and use the outputted secret ID to configure this option. type: secret @@ -82,6 +82,9 @@ config: webserver-workers: description: The number of webserver worker processes for handling requests. type: int + webserver-worker-class: + description: The method of webserver worker processes for handling requests. Can be either 'gevent' or 'sync'. + type: string secret-test: description: A test configuration option for testing user provided Juju secrets. type: secret diff --git a/examples/flask/test_async_rock/app.py b/examples/flask/test_async_rock/app.py index 7d0b087..680963b 100644 --- a/examples/flask/test_async_rock/app.py +++ b/examples/flask/test_async_rock/app.py @@ -5,230 +5,10 @@ import os import socket import time -import urllib.parse -from urllib.parse import urlparse -import boto3 -import botocore.config -import pika -import psycopg -import pymongo -import pymongo.database -import pymongo.errors -import pymysql -import pymysql.cursors -import redis -from celery import Celery, Task from flask import Flask, g, jsonify, request - -def hostname(): - """Get the hostname of the current machine.""" - return socket.gethostbyname(socket.gethostname()) - - -def celery_init_app(app: Flask, broker_url: str) -> Celery: - """Initialise celery using the redis connection string. - - See https://flask.palletsprojects.com/en/3.0.x/patterns/celery/#integrate-celery-with-flask. - """ - - class FlaskTask(Task): - def __call__(self, *args: object, **kwargs: object) -> object: - with app.app_context(): - return self.run(*args, **kwargs) - - celery_app = Celery(app.name, task_cls=FlaskTask) - celery_app.set_default() - app.extensions["celery"] = celery_app - app.config.from_mapping( - CELERY=dict( - broker_url=broker_url, - result_backend=broker_url, - task_ignore_result=True, - ), - ) - celery_app.config_from_object(app.config["CELERY"]) - return celery_app - - app = Flask(__name__) -app.config.from_prefixed_env() - -broker_url = os.environ.get("REDIS_DB_CONNECT_STRING") -# Configure Celery only if Redis is configured -celery_app = celery_init_app(app, broker_url) -redis_client = redis.Redis.from_url(broker_url) if broker_url else None - - -@celery_app.on_after_configure.connect -def setup_periodic_tasks(sender, **kwargs): - """Set up periodic tasks in the scheduler.""" - try: - # This will only have an effect in the beat scheduler. - sender.add_periodic_task(0.5, scheduled_task.s(hostname()), name="every 0.5s") - except NameError as e: - logging.exception("Failed to configure the periodic task") - - -@celery_app.task -def scheduled_task(scheduler_hostname): - """Function to run a schedule task in a worker. - - The worker that will run this task will add the scheduler hostname argument - to the "schedulers" set in Redis, and the worker's hostname to the "workers" - set in Redis. - """ - worker_hostname = hostname() - logging.info( - "scheduler host received %s in worker host %s", scheduler_hostname, worker_hostname - ) - redis_client.sadd("schedulers", scheduler_hostname) - redis_client.sadd("workers", worker_hostname) - logging.info("schedulers: %s", redis_client.smembers("schedulers")) - logging.info("workers: %s", redis_client.smembers("workers")) - # The goal is to have all workers busy in all processes. - # For that it maybe necessary to exhaust all workers, but not to get the pending tasks - # too big, so all schedulers can manage to run their scheduled tasks. - # Celery prefetches tasks, and if they cannot be run they are put in reserved. - # If all processes have tasks in reserved, this task will finish immediately to not make - # queues any longer. - inspect_obj = celery_app.control.inspect() - reserved_sizes = [len(tasks) for tasks in inspect_obj.reserved().values()] - logging.info("number of reserved tasks %s", reserved_sizes) - delay = 0 if min(reserved_sizes) > 0 else 5 - time.sleep(delay) - - -def get_mysql_database(): - """Get the mysql db connection.""" - if "mysql_db" not in g: - if "MYSQL_DB_CONNECT_STRING" in os.environ: - uri_parts = urlparse(os.environ["MYSQL_DB_CONNECT_STRING"]) - g.mysql_db = pymysql.connect( - host=uri_parts.hostname, - user=uri_parts.username, - password=uri_parts.password, - database=uri_parts.path[1:], - port=uri_parts.port, - ) - else: - return None - return g.mysql_db - - -def get_postgresql_database(): - """Get the postgresql db connection.""" - if "postgresql_db" not in g: - if "POSTGRESQL_DB_CONNECT_STRING" in os.environ: - g.postgresql_db = psycopg.connect( - conninfo=os.environ["POSTGRESQL_DB_CONNECT_STRING"], - ) - else: - return None - return g.postgresql_db - - -def get_mongodb_database() -> pymongo.database.Database | None: - """Get the mongodb db connection.""" - if "mongodb_db" not in g: - if "MONGODB_DB_CONNECT_STRING" in os.environ: - uri = os.environ["MONGODB_DB_CONNECT_STRING"] - client = pymongo.MongoClient(uri) - db = urllib.parse.urlparse(uri).path.removeprefix("/") - g.mongodb_db = client.get_database(db) - else: - return None - return g.mongodb_db - - -def get_redis_database() -> redis.Redis | None: - if "redis_db" not in g: - if "REDIS_DB_CONNECT_STRING" in os.environ: - uri = os.environ["REDIS_DB_CONNECT_STRING"] - g.redis_db = redis.Redis.from_url(uri) - else: - return None - return g.redis_db - - -def get_rabbitmq_connection() -> pika.BlockingConnection | None: - """Get rabbitmq connection.""" - if "rabbitmq" not in g: - if "RABBITMQ_HOSTNAME" in os.environ: - username = os.environ["RABBITMQ_USERNAME"] - password = os.environ["RABBITMQ_PASSWORD"] - hostname = os.environ["RABBITMQ_HOSTNAME"] - vhost = os.environ["RABBITMQ_VHOST"] - port = os.environ["RABBITMQ_PORT"] - credentials = pika.PlainCredentials(username, password) - parameters = pika.ConnectionParameters(hostname, port, vhost, credentials) - g.rabbitmq = pika.BlockingConnection(parameters) - else: - return None - return g.rabbitmq - - -def get_rabbitmq_connection_from_uri() -> pika.BlockingConnection | None: - """Get rabbitmq connection from uri.""" - if "rabbitmq_from_uri" not in g: - if "RABBITMQ_CONNECT_STRING" in os.environ: - uri = os.environ["RABBITMQ_CONNECT_STRING"] - parameters = pika.URLParameters(uri) - g.rabbitmq_from_uri = pika.BlockingConnection(parameters) - else: - return None - return g.rabbitmq_from_uri - - -def get_boto3_client(): - if "boto3_client" not in g: - if "S3_ACCESS_KEY" in os.environ: - s3_client_config = botocore.config.Config( - s3={ - "addressing_style": os.environ["S3_ADDRESSING_STYLE"], - }, - # no_proxy env variable is not read by boto3, so - # this is needed for the tests to avoid hitting the proxy. - proxies={}, - ) - g.boto3_client = boto3.client( - "s3", - os.environ["S3_REGION"], - aws_access_key_id=os.environ["S3_ACCESS_KEY"], - aws_secret_access_key=os.environ["S3_SECRET_KEY"], - endpoint_url=os.environ["S3_ENDPOINT"], - use_ssl=False, - config=s3_client_config, - ) - else: - return None - return g.boto3_client - - -@app.teardown_appcontext -def teardown_database(_): - """Tear down databases connections.""" - mysql_db = g.pop("mysql_db", None) - if mysql_db is not None: - mysql_db.close() - postgresql_db = g.pop("postgresql_db", None) - if postgresql_db is not None: - postgresql_db.close() - mongodb_db = g.pop("mongodb_db", None) - if mongodb_db is not None: - mongodb_db.client.close() - boto3_client = g.pop("boto3_client", None) - if boto3_client is not None: - boto3_client.close() - rabbitmq = g.pop("rabbitmq", None) - if rabbitmq is not None: - rabbitmq.close() - rabbitmq_from_uri = g.pop("rabbitmq_from_uri", None) - if rabbitmq_from_uri is not None: - rabbitmq_from_uri.close() - - @app.route("/") def hello_world(): return "Hello, World!" @@ -240,120 +20,3 @@ def sleep(): time.sleep(duration_seconds) return "" - -@app.route("/config/") -def config(config_name: str): - return jsonify(app.config.get(config_name)) - - -@app.route("/mysql/status") -def mysql_status(): - """Mysql status endpoint.""" - if database := get_mysql_database(): - with database.cursor() as cursor: - sql = "SELECT version()" - cursor.execute(sql) - cursor.fetchone() - return "SUCCESS" - return "FAIL" - - -@app.route("/s3/status") -def s3_status(): - """S3 status endpoint.""" - if client := get_boto3_client(): - bucket_name = os.environ["S3_BUCKET"] - objectsresponse = client.list_objects(Bucket=bucket_name) - return "SUCCESS" - return "FAIL" - - -@app.route("/postgresql/status") -def postgresql_status(): - """Postgresql status endpoint.""" - if database := get_postgresql_database(): - with database.cursor() as cursor: - sql = "SELECT version()" - cursor.execute(sql) - cursor.fetchone() - return "SUCCESS" - return "FAIL" - - -@app.route("/mongodb/status") -def mongodb_status(): - """Mongodb status endpoint.""" - if (database := get_mongodb_database()) is not None: - database.list_collection_names() - return "SUCCESS" - return "FAIL" - - -@app.route("/redis/status") -def redis_status(): - """Redis status endpoint.""" - if database := get_redis_database(): - try: - database.set("foo", "bar") - return "SUCCESS" - except redis.exceptions.RedisError: - logging.exception("Error querying redis") - return "FAIL" - - -@app.route("/redis/clear_celery_stats") -def redis_celery_clear_stats(): - """Reset Redis statistics about workers and schedulers.""" - if database := get_redis_database(): - try: - database.delete("workers") - database.delete("schedulers") - return "SUCCESS" - except redis.exceptions.RedisError: - logging.exception("Error querying redis") - return "FAIL", 500 - - -@app.route("/redis/celery_stats") -def redis_celery_stats(): - """Read Redis statistics about workers and schedulers.""" - if database := get_redis_database(): - try: - worker_set = [str(host) for host in database.smembers("workers")] - beat_set = [str(host) for host in database.smembers("schedulers")] - return jsonify({"workers": worker_set, "schedulers": beat_set}) - except redis.exceptions.RedisError: - logging.exception("Error querying redis") - return "FAIL", 500 - - -@app.route("/rabbitmq/send") -def rabbitmq_send(): - """Send a message to "charm" queue.""" - if connection := get_rabbitmq_connection(): - channel = connection.channel() - channel.queue_declare(queue="charm") - channel.basic_publish(exchange="", routing_key="charm", body="SUCCESS") - return "SUCCESS" - return "FAIL" - - -@app.route("/rabbitmq/receive") -def rabbitmq_receive(): - """Receive a message from "charm" queue in blocking form.""" - if connection := get_rabbitmq_connection_from_uri(): - channel = connection.channel() - method_frame, _header_frame, body = channel.basic_get("charm") - if method_frame: - channel.basic_ack(method_frame.delivery_tag) - if body == b"SUCCESS": - return "SUCCESS" - return "FAIL. INCORRECT MESSAGE." - return "FAIL. NO MESSAGE." - return "FAIL. NO CONNECTION." - - -@app.route("/env") -def get_env(): - """Return environment variables""" - return jsonify(dict(os.environ)) diff --git a/examples/flask/test_async_rock/requirements.txt b/examples/flask/test_async_rock/requirements.txt index 2ff69c0..e3e9a71 100644 --- a/examples/flask/test_async_rock/requirements.txt +++ b/examples/flask/test_async_rock/requirements.txt @@ -1,10 +1 @@ Flask -PyMySQL -PyMySQL[rsa] -PyMySQL[ed25519] -psycopg[binary] -pymongo -redis[hiredis] -boto3 -pika -celery diff --git a/examples/flask/test_async_rock/rockcraft.yaml b/examples/flask/test_async_rock/rockcraft.yaml index 06248ba..443716a 100644 --- a/examples/flask/test_async_rock/rockcraft.yaml +++ b/examples/flask/test_async_rock/rockcraft.yaml @@ -17,16 +17,3 @@ parts: python-packages: - gunicorn[gevent] -services: - celery-worker: - override: replace - command: celery -A app:celery_app worker -c 2 --loglevel DEBUG - startup: enabled - user: _daemon_ - working-dir: /flask/app - celery-beat-scheduler: - override: replace - command: celery -A app:celery_app beat --loglevel DEBUG -s /tmp/celerybeat-schedule - startup: enabled - user: _daemon_ - working-dir: /flask/app diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index a6c589d..c1a94a9 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -31,9 +31,11 @@ def check_gevent_package(self) -> bool: Returns: True if gevent is installed. """ - pip_list_command = self._container.exec(["pip", "list"]) + pip_list_command = self._container.exec( + ["python3", "-c", "'import gevent;print(gevent.__version__)'"] + ) list_output = pip_list_command.wait_output()[0] - return "gevent" in list_output + return "ModuleNotFoundError" not in list_output def create_webserver_config(self) -> WebserverConfig: """Validate worker_class and create a WebserverConfig instance from the charm config. diff --git a/tests/integration/flask/requirements.txt b/tests/integration/flask/requirements.txt index 6ec779d..13ce280 100644 --- a/tests/integration/flask/requirements.txt +++ b/tests/integration/flask/requirements.txt @@ -1,2 +1,3 @@ ops >= 1.5.0 pytest-operator >= 0.32.0 +aiohttp == 3.11.7 diff --git a/tests/integration/flask/test_workers.py b/tests/integration/flask/test_workers.py index 5944479..1446953 100644 --- a/tests/integration/flask/test_workers.py +++ b/tests/integration/flask/test_workers.py @@ -6,7 +6,9 @@ import asyncio import logging import time +from datetime import datetime +import aiohttp import pytest import requests from juju.application import Application @@ -69,56 +71,32 @@ def check_correct_celery_stats(num_schedulers, num_workers): except asyncio.TimeoutError: assert False, "Failed to get 2 workers and 1 scheduler" - -@pytest.mark.parametrize( - "worker_class, expected_result", - [ - ("eventlet", "blocked"), - ("gevent", "active"), - ("sync", "active"), - ], -) @pytest.mark.usefixtures("flask_async_app") -async def test_async_workers_config( +async def test_async_workers( ops_test: OpsTest, model: Model, flask_async_app: Application, get_unit_ips, - worker_class: str, - expected_result: bool, ): """ - arrange: Flask is deployed with async enabled rock. - act: Change gunicorn worker class. - assert: Charm should only let the class to be 'sync' or 'gevent'. - If it is something other than these, then the unit should be blocked. + arrange: Flask is deployed with async enabled rock. Change gunicorn worker class. + act: Do 15 requests that would take 2 seconds each. + assert: All 15 requests should be served in under 3 seconds. """ - await flask_async_app.set_config({"webserver-worker-class": worker_class}) - await model.wait_for_idle(apps=[flask_async_app.name], status=expected_result, timeout=60) + await flask_async_app.set_config({"webserver-worker-class": "gevent"}) + await model.wait_for_idle(apps=[flask_async_app.name], status="active", timeout=60) + # the flask unit is not important. Take the first one + flask_unit_ip = (await get_unit_ips(flask_async_app.name))[0] -@pytest.mark.parametrize( - "worker_class, expected_result", - [ - ("gevent", "blocked"), - ("eventlet", "blocked"), - ("sync", "active"), - ], -) -@pytest.mark.usefixtures("flask_app") -async def test_async_workers_config_fail( - ops_test: OpsTest, - model: Model, - flask_app: Application, - get_unit_ips, - worker_class: str, - expected_result: str, -): - """ - arrange: Flask is deployed with async not enabled rock. - act: Change gunicorn worker class. - assert: Charm should only let the class to be 'sync'. - If it is 'gevent' or something else, then the unit should be blocked. - """ - await flask_app.set_config({"webserver-worker-class": worker_class}) - await model.wait_for_idle(apps=[flask_app.name], status=expected_result, timeout=60) + async def _fetch_page(session): + params = {"duration": 2} + async with session.get(f"http://{flask_unit_ip}:8000/sleep", params=params) as response: + return await response.text() + + start_time = datetime.now() + async with aiohttp.ClientSession() as session: + pages = [_fetch_page(session) for _ in range(15)] + await asyncio.gather(*pages) + print(f"TIMME: {(datetime.now() - start_time).seconds}") + assert (datetime.now() - start_time).seconds < 3, "The page took more than 2 seconds to load" diff --git a/tests/unit/django/test_charm.py b/tests/unit/django/test_charm.py index 8d66364..feb8637 100644 --- a/tests/unit/django/test_charm.py +++ b/tests/unit/django/test_charm.py @@ -155,6 +155,55 @@ def test_required_database_integration(harness_no_integrations: Harness): ) +@pytest.mark.parametrize("config, env", TEST_DJANGO_CONFIG_PARAMS) +def test_django_async_config(harness: Harness, config: dict, env: dict) -> None: + """ + arrange: None + act: Start the django charm and set django-app container to be ready. + assert: Django charm should submit the correct pebble layer to pebble. + """ + harness.begin() + container = harness.charm.unit.get_container("django-app") + # ops.testing framework apply layers by label in lexicographical order... + container.add_layer("a_layer", DEFAULT_LAYER) + secret_storage = unittest.mock.MagicMock() + secret_storage.is_secret_storage_ready = True + secret_storage.get_secret_key.return_value = "test" + harness.update_config(config) + charm_state = CharmState.from_charm( + config=harness.charm.config, + framework="django", + framework_config=harness.charm.get_framework_config(), + secret_storage=secret_storage, + database_requirers={}, + ) + webserver_config = WebserverConfig.from_charm_config(harness.charm.config) + workload_config = create_workload_config(framework_name="django", unit_name="django/0") + webserver = GunicornWebserver( + webserver_config=webserver_config, + workload_config=workload_config, + container=container, + ) + django_app = WsgiApp( + container=harness.charm.unit.get_container("django-app"), + charm_state=charm_state, + workload_config=workload_config, + webserver=webserver, + database_migration=harness.charm._database_migration, + ) + django_app.restart() + plan = container.get_plan() + django_layer = plan.to_dict()["services"]["django"] + assert django_layer == { + "environment": env, + "override": "replace", + "startup": "enabled", + "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application", + "after": ["statsd-exporter"], + "user": "_daemon_", + } + + def test_allowed_hosts_base_hostname_updates_correctly(harness: Harness): """ arrange: Deploy a Django charm without an ingress integration diff --git a/tests/unit/django/test_workers.py b/tests/unit/django/test_workers.py new file mode 100644 index 0000000..02cea0b --- /dev/null +++ b/tests/unit/django/test_workers.py @@ -0,0 +1,96 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for worker services.""" + +import ops +import pytest +from ops.testing import Harness + +from .constants import DEFAULT_LAYER + + +@pytest.mark.parametrize( + "worker_class, expected_status, expected_message", + [ + ( + "eventlet", + "blocked", + "Only 'gevent' and 'sync' are allowed. https://bit.ly/django-async-doc", + ), + ("gevent", "active", ""), + ("sync", "active", ""), + ], +) +def test_async_workers_config(harness: Harness, worker_class, expected_status, expected_message): + """ + arrange: Prepare a unit and run initial hooks. + act: Set the `webserver-worker-class` config. + assert: The charm should be blocked if the `webserver-worker-class` config is anything other + then `sync` or `gevent`. + """ + postgresql_relation_data = { + "database": "test-database", + "endpoints": "test-postgresql:5432,test-postgresql-2:5432", + "password": "test-password", + "username": "test-username", + } + harness.add_relation("postgresql", "postgresql-k8s", app_data=postgresql_relation_data) + container = harness.model.unit.get_container("django-app") + container.add_layer("a_layer", DEFAULT_LAYER) + harness.handle_exec( + container.name, + ["python3", "-c", "'import gevent;print(gevent.__version__)'"], + result="Gevent", + ) + harness.begin_with_initial_hooks() + harness.update_config({"webserver-worker-class": worker_class}) + assert harness.model.unit.status == ops.StatusBase.from_name( + name=expected_status, message=expected_message + ) + + +@pytest.mark.parametrize( + "worker_class, expected_status, expected_message", + [ + ( + "eventlet", + "blocked", + "Only 'gevent' and 'sync' are allowed. https://bit.ly/django-async-doc", + ), + ( + "gevent", + "blocked", + "gunicorn[gevent] must be installed in the rock. https://bit.ly/django-async-doc", + ), + ("sync", "active", ""), + ], +) +def test_async_workers_config_fail( + harness: Harness, worker_class, expected_status, expected_message +): + """ + arrange: Prepare a unit and run initial hooks. + act: Set the `webserver-worker-class` config. + assert: The charm should be blocked if the `webserver-worker-class` config is anything other + then `sync`. + """ + postgresql_relation_data = { + "database": "test-database", + "endpoints": "test-postgresql:5432,test-postgresql-2:5432", + "password": "test-password", + "username": "test-username", + } + harness.add_relation("postgresql", "postgresql-k8s", app_data=postgresql_relation_data) + container = harness.model.unit.get_container("django-app") + container.add_layer("a_layer", DEFAULT_LAYER) + harness.handle_exec( + container.name, + ["python3", "-c", "'import gevent;print(gevent.__version__)'"], + result="ModuleNotFoundError", + ) + harness.begin_with_initial_hooks() + harness.update_config({"webserver-worker-class": worker_class}) + assert harness.model.unit.status == ops.StatusBase.from_name( + name=expected_status, message=expected_message + ) diff --git a/tests/unit/flask/test_workers.py b/tests/unit/flask/test_workers.py index 144dc8f..51e62ec 100644 --- a/tests/unit/flask/test_workers.py +++ b/tests/unit/flask/test_workers.py @@ -11,7 +11,7 @@ import pytest from ops.testing import Harness -from .constants import FLASK_CONTAINER_NAME, LAYER_WITH_WORKER +from .constants import DEFAULT_LAYER, FLASK_CONTAINER_NAME, LAYER_WITH_WORKER def test_worker(harness: Harness): @@ -39,6 +39,80 @@ def test_worker(harness: Harness): assert "FLASK_SECRET_KEY" not in services["not-worker-service"].environment +@pytest.mark.parametrize( + "worker_class, expected_status, expected_message", + [ + ( + "eventlet", + "blocked", + "Only 'gevent' and 'sync' are allowed. https://bit.ly/flask-async-doc", + ), + ("gevent", "active", ""), + ("sync", "active", ""), + ], +) +def test_async_workers_config(harness: Harness, worker_class, expected_status, expected_message): + """ + arrange: Prepare a unit and run initial hooks. + act: Set the `webserver-worker-class` config. + assert: The charm should be blocked if the `webserver-worker-class` config is anything other + then `sync` or `gevent`. + """ + container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) + container.add_layer("a_layer", DEFAULT_LAYER) + + harness.handle_exec( + container.name, + ["python3", "-c", "'import gevent;print(gevent.__version__)'"], + result="Gevent", + ) + harness.begin_with_initial_hooks() + harness.update_config({"webserver-worker-class": worker_class}) + assert harness.model.unit.status == ops.StatusBase.from_name( + name=expected_status, message=expected_message + ) + + +@pytest.mark.parametrize( + "worker_class, expected_status, expected_message", + [ + ( + "eventlet", + "blocked", + "Only 'gevent' and 'sync' are allowed. https://bit.ly/flask-async-doc", + ), + ( + "gevent", + "blocked", + "gunicorn[gevent] must be installed in the rock. https://bit.ly/flask-async-doc", + ), + ("sync", "active", ""), + ], +) +def test_async_workers_config_fail( + harness: Harness, worker_class, expected_status, expected_message +): + """ + arrange: Prepare a unit and run initial hooks. + act: Set the `webserver-worker-class` config. + assert: The charm should be blocked if the `webserver-worker-class` config is anything other + then `sync`. + """ + container = harness.model.unit.get_container(FLASK_CONTAINER_NAME) + container.add_layer("a_layer", DEFAULT_LAYER) + + harness.handle_exec( + container.name, + ["python3", "-c", "'import gevent;print(gevent.__version__)'"], + result="ModuleNotFoundError", + ) + harness.begin_with_initial_hooks() + harness.update_config({"webserver-worker-class": worker_class}) + assert harness.model.unit.status == ops.StatusBase.from_name( + name=expected_status, message=expected_message + ) + + def test_worker_multiple_units(harness: Harness): """ arrange: Prepare a unit with workers that is not the first one (number 1) From 37333e214e9b92588350c4175d8ad63ddb6b50c4 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Wed, 27 Nov 2024 11:33:00 +0300 Subject: [PATCH 15/27] Chore(lint): Format code --- examples/flask/test_async_rock/app.py | 3 ++- tests/integration/flask/test_workers.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/flask/test_async_rock/app.py b/examples/flask/test_async_rock/app.py index 680963b..ec6131a 100644 --- a/examples/flask/test_async_rock/app.py +++ b/examples/flask/test_async_rock/app.py @@ -9,6 +9,8 @@ from flask import Flask, g, jsonify, request app = Flask(__name__) + + @app.route("/") def hello_world(): return "Hello, World!" @@ -19,4 +21,3 @@ def sleep(): duration_seconds = int(request.args.get("duration")) time.sleep(duration_seconds) return "" - diff --git a/tests/integration/flask/test_workers.py b/tests/integration/flask/test_workers.py index 1446953..a208db8 100644 --- a/tests/integration/flask/test_workers.py +++ b/tests/integration/flask/test_workers.py @@ -71,6 +71,7 @@ def check_correct_celery_stats(num_schedulers, num_workers): except asyncio.TimeoutError: assert False, "Failed to get 2 workers and 1 scheduler" + @pytest.mark.usefixtures("flask_async_app") async def test_async_workers( ops_test: OpsTest, @@ -99,4 +100,6 @@ async def _fetch_page(session): pages = [_fetch_page(session) for _ in range(15)] await asyncio.gather(*pages) print(f"TIMME: {(datetime.now() - start_time).seconds}") - assert (datetime.now() - start_time).seconds < 3, "The page took more than 2 seconds to load" + assert ( + datetime.now() - start_time + ).seconds < 3, "The page took more than 2 seconds to load" From b132e1ac89d0d685a29ac31ce42417f566a3b327 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Wed, 27 Nov 2024 13:27:52 +0300 Subject: [PATCH 16/27] Chore(): Add Django test, simplify Django async test app --- .../django_async_app/django_async_app/urls.py | 6 +- .../django_async_app/testing/views.py | 29 +-------- tests/integration/django/test_workers.py | 64 +++++++------------ tests/integration/flask/test_workers.py | 3 +- 4 files changed, 25 insertions(+), 77 deletions(-) diff --git a/examples/django/django_async_app/django_async_app/django_async_app/urls.py b/examples/django/django_async_app/django_async_app/django_async_app/urls.py index deb38d8..ba482f2 100644 --- a/examples/django/django_async_app/django_async_app/django_async_app/urls.py +++ b/examples/django/django_async_app/django_async_app/django_async_app/urls.py @@ -20,13 +20,9 @@ from django.contrib import admin from django.urls import path -from testing.views import environ, get_settings, login, sleep, user_count +from testing.views import sleep urlpatterns = [ path("admin/", admin.site.urls), - path("settings/", get_settings, name="get_settings"), - path("len/users", user_count, name="user_count"), - path("environ", environ, name="environ"), path("sleep", sleep, name="sleep"), - path("login", login, name="login"), ] diff --git a/examples/django/django_async_app/django_async_app/testing/views.py b/examples/django/django_async_app/django_async_app/testing/views.py index 07b39f7..149129c 100644 --- a/examples/django/django_async_app/django_async_app/testing/views.py +++ b/examples/django/django_async_app/django_async_app/testing/views.py @@ -1,39 +1,12 @@ # Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. -import os import time -from django.conf import settings -from django.contrib.auth import authenticate -from django.contrib.auth.models import User -from django.http import HttpResponse, JsonResponse - - -def environ(request): - return JsonResponse(dict(os.environ)) - - -def user_count(request): - return JsonResponse(User.objects.count(), safe=False) - - -def get_settings(request, name): - if hasattr(settings, name): - return JsonResponse(getattr(settings, name), safe=False) - else: - return JsonResponse({"error": f"settings {name!r} not found"}, status=404) +from django.http import HttpResponse def sleep(request): duration = request.GET.get("duration") time.sleep(int(duration)) return HttpResponse() - - -def login(request): - user = authenticate(username=request.GET.get("username"), password=request.GET.get("password")) - if user is not None: - return HttpResponse(status=200) - else: - return HttpResponse(status=403) diff --git a/tests/integration/django/test_workers.py b/tests/integration/django/test_workers.py index f65ce27..5f82bb3 100644 --- a/tests/integration/django/test_workers.py +++ b/tests/integration/django/test_workers.py @@ -7,7 +7,9 @@ import logging import time import typing +from datetime import datetime +import aiohttp import pytest import requests from juju.application import Application @@ -18,55 +20,33 @@ logger = logging.getLogger(__name__) -@pytest.mark.parametrize( - "worker_class, expected_result", - [ - ("eventlet", "blocked"), - ("gevent", "active"), - ("sync", "active"), - ], -) @pytest.mark.usefixtures("django_async_app") -async def test_async_workers_config( +async def test_async_workers( ops_test: OpsTest, model: Model, django_async_app: Application, get_unit_ips, - worker_class: str, - expected_result: bool, ): """ - arrange: Django is deployed with async enabled rock. - act: Change gunicorn worker class. - assert: Charm should only let the class to be 'sync' or 'gevent'. - If it is something other than these, then the unit should be blocked. + arrange: Django is deployed with async enabled rock. Change gunicorn worker class. + act: Do 15 requests that would take 2 seconds each. + assert: All 15 requests should be served in under 3 seconds. """ - await django_async_app.set_config({"webserver-worker-class": worker_class}) - await model.wait_for_idle(apps=[django_async_app.name], status=expected_result, timeout=60) + await django_async_app.set_config({"webserver-worker-class": "gevent"}) + await model.wait_for_idle(apps=[django_async_app.name], status="active", timeout=60) + # the django unit is not important. Take the first one + django_unit_ip = (await get_unit_ips(django_async_app.name))[0] -@pytest.mark.parametrize( - "worker_class, expected_result", - [ - ("gevent", "blocked"), - ("eventlet", "blocked"), - ("sync", "active"), - ], -) -@pytest.mark.usefixtures("django_app") -async def test_async_workers_config_fail( - ops_test: OpsTest, - model: Model, - django_app: Application, - get_unit_ips, - worker_class: str, - expected_result: str, -): - """ - arrange: Django is deployed with async not enabled rock. - act: Change gunicorn worker class. - assert: Charm should only let the class to be 'sync'. - If it is 'gevent' or something else, then the unit should be blocked. - """ - await django_app.set_config({"webserver-worker-class": worker_class}) - await model.wait_for_idle(apps=[django_app.name], status=expected_result, timeout=60) + async def _fetch_page(session): + params = {"duration": 2} + async with session.get(f"http://{django_unit_ip}:8000/sleep", params=params) as response: + return await response.text() + + start_time = datetime.now() + async with aiohttp.ClientSession() as session: + pages = [_fetch_page(session) for _ in range(15)] + await asyncio.gather(*pages) + assert ( + datetime.now() - start_time + ).seconds < 3, "Async workers for Django are not working!" diff --git a/tests/integration/flask/test_workers.py b/tests/integration/flask/test_workers.py index a208db8..323992c 100644 --- a/tests/integration/flask/test_workers.py +++ b/tests/integration/flask/test_workers.py @@ -99,7 +99,6 @@ async def _fetch_page(session): async with aiohttp.ClientSession() as session: pages = [_fetch_page(session) for _ in range(15)] await asyncio.gather(*pages) - print(f"TIMME: {(datetime.now() - start_time).seconds}") assert ( datetime.now() - start_time - ).seconds < 3, "The page took more than 2 seconds to load" + ).seconds < 3, "Async workers for Flask are not working!" From 128461ae095c136cc130931cafc44b99b3790afe Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 29 Nov 2024 14:33:33 +0300 Subject: [PATCH 17/27] Chore(test): Improve tests and gevent module check --- .../django_async_app/settings.py | 2 +- .../django_async_app/migrate.sh | 5 --- src/paas_charm/_gunicorn/charm.py | 32 +++++++++++-------- tests/unit/django/test_workers.py | 31 +++++++++++------- tests/unit/flask/test_workers.py | 30 ++++++++++------- 5 files changed, 57 insertions(+), 43 deletions(-) delete mode 100644 examples/django/django_async_app/django_async_app/migrate.sh diff --git a/examples/django/django_async_app/django_async_app/django_async_app/settings.py b/examples/django/django_async_app/django_async_app/django_async_app/settings.py index 24b5e10..614e67f 100644 --- a/examples/django/django_async_app/django_async_app/django_async_app/settings.py +++ b/examples/django/django_async_app/django_async_app/django_async_app/settings.py @@ -31,7 +31,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.environ.get("DJANGO_DEBUG", "true") == "true" -ALLOWED_HOSTS = json.loads(os.environ.get("DJANGO_ALLOWED_HOSTS", '["*"]')) +ALLOWED_HOSTS = json.loads(os.environ["DJANGO_ALLOWED_HOSTS"]) INSTALLED_APPS = [ diff --git a/examples/django/django_async_app/django_async_app/migrate.sh b/examples/django/django_async_app/django_async_app/migrate.sh deleted file mode 100644 index ce3a73c..0000000 --- a/examples/django/django_async_app/django_async_app/migrate.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/bash -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -python3 manage.py migrate diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index c1a94a9..a81187a 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -5,6 +5,8 @@ import logging +from ops.pebble import ExecError, ExecProcess + from paas_charm._gunicorn.webserver import GunicornWebserver, WebserverConfig, WorkerClassEnum from paas_charm._gunicorn.workload_config import create_workload_config from paas_charm._gunicorn.wsgi_app import WsgiApp @@ -25,18 +27,6 @@ def _workload_config(self) -> WorkloadConfig: framework_name=self._framework_name, unit_name=self.unit.name ) - def check_gevent_package(self) -> bool: - """Check that gevent is installed. - - Returns: - True if gevent is installed. - """ - pip_list_command = self._container.exec( - ["python3", "-c", "'import gevent;print(gevent.__version__)'"] - ) - list_output = pip_list_command.wait_output()[0] - return "ModuleNotFoundError" not in list_output - def create_webserver_config(self) -> WebserverConfig: """Validate worker_class and create a WebserverConfig instance from the charm config. @@ -68,7 +58,7 @@ def create_webserver_config(self) -> WebserverConfig: if worker_class is WorkerClassEnum.SYNC: return webserver_config - if not self.check_gevent_package(): + if not self._check_gevent_package(): logger.error( "gunicorn[gevent] must be installed in the rock. %s", doc_link, @@ -100,3 +90,19 @@ def _create_app(self) -> App: webserver=webserver, database_migration=self._database_migration, ) + + def _check_gevent_package(self) -> bool: + """Check that gevent is installed. + + Returns: + True if gevent is installed. + """ + try: + check_gevent_process: ExecProcess = self._container.exec( + ["python3", "-c", "import gevent"] + ) + check_gevent_process.wait_output() + return True + except ExecError as cmd_error: + logger.warning("gunicorn[gevent] install check failed: %s", cmd_error) + return False diff --git a/tests/unit/django/test_workers.py b/tests/unit/django/test_workers.py index 02cea0b..e4c605b 100644 --- a/tests/unit/django/test_workers.py +++ b/tests/unit/django/test_workers.py @@ -5,24 +5,27 @@ import ops import pytest -from ops.testing import Harness +from ops.testing import ExecResult, Harness from .constants import DEFAULT_LAYER @pytest.mark.parametrize( - "worker_class, expected_status, expected_message", + "worker_class, expected_status, expected_message, exec_res", [ ( "eventlet", "blocked", "Only 'gevent' and 'sync' are allowed. https://bit.ly/django-async-doc", + 1, ), - ("gevent", "active", ""), - ("sync", "active", ""), + ("gevent", "active", "", 0), + ("sync", "active", "", 0), ], ) -def test_async_workers_config(harness: Harness, worker_class, expected_status, expected_message): +def test_async_workers_config( + harness: Harness, worker_class, expected_status, expected_message, exec_res +): """ arrange: Prepare a unit and run initial hooks. act: Set the `webserver-worker-class` config. @@ -38,11 +41,13 @@ def test_async_workers_config(harness: Harness, worker_class, expected_status, e harness.add_relation("postgresql", "postgresql-k8s", app_data=postgresql_relation_data) container = harness.model.unit.get_container("django-app") container.add_layer("a_layer", DEFAULT_LAYER) + harness.handle_exec( container.name, - ["python3", "-c", "'import gevent;print(gevent.__version__)'"], - result="Gevent", + ["python3", "-c", "import gevent"], + result=ExecResult(exit_code=exec_res), ) + harness.begin_with_initial_hooks() harness.update_config({"webserver-worker-class": worker_class}) assert harness.model.unit.status == ops.StatusBase.from_name( @@ -51,23 +56,25 @@ def test_async_workers_config(harness: Harness, worker_class, expected_status, e @pytest.mark.parametrize( - "worker_class, expected_status, expected_message", + "worker_class, expected_status, expected_message, exec_res", [ ( "eventlet", "blocked", "Only 'gevent' and 'sync' are allowed. https://bit.ly/django-async-doc", + 1, ), ( "gevent", "blocked", "gunicorn[gevent] must be installed in the rock. https://bit.ly/django-async-doc", + 1, ), - ("sync", "active", ""), + ("sync", "active", "", 0), ], ) def test_async_workers_config_fail( - harness: Harness, worker_class, expected_status, expected_message + harness: Harness, worker_class, expected_status, expected_message, exec_res ): """ arrange: Prepare a unit and run initial hooks. @@ -86,8 +93,8 @@ def test_async_workers_config_fail( container.add_layer("a_layer", DEFAULT_LAYER) harness.handle_exec( container.name, - ["python3", "-c", "'import gevent;print(gevent.__version__)'"], - result="ModuleNotFoundError", + ["python3", "-c", "import gevent"], + result=ExecResult(exit_code=exec_res), ) harness.begin_with_initial_hooks() harness.update_config({"webserver-worker-class": worker_class}) diff --git a/tests/unit/flask/test_workers.py b/tests/unit/flask/test_workers.py index 51e62ec..9e0d077 100644 --- a/tests/unit/flask/test_workers.py +++ b/tests/unit/flask/test_workers.py @@ -9,7 +9,7 @@ import ops import pytest -from ops.testing import Harness +from ops.testing import ExecResult, Harness from .constants import DEFAULT_LAYER, FLASK_CONTAINER_NAME, LAYER_WITH_WORKER @@ -40,18 +40,21 @@ def test_worker(harness: Harness): @pytest.mark.parametrize( - "worker_class, expected_status, expected_message", + "worker_class, expected_status, expected_message, exec_res", [ ( "eventlet", "blocked", "Only 'gevent' and 'sync' are allowed. https://bit.ly/flask-async-doc", + 1, ), - ("gevent", "active", ""), - ("sync", "active", ""), + ("gevent", "active", "", 0), + ("sync", "active", "", 0), ], ) -def test_async_workers_config(harness: Harness, worker_class, expected_status, expected_message): +def test_async_workers_config( + harness: Harness, worker_class, expected_status, expected_message, exec_res +): """ arrange: Prepare a unit and run initial hooks. act: Set the `webserver-worker-class` config. @@ -63,9 +66,10 @@ def test_async_workers_config(harness: Harness, worker_class, expected_status, e harness.handle_exec( container.name, - ["python3", "-c", "'import gevent;print(gevent.__version__)'"], - result="Gevent", + ["python3", "-c", "import gevent"], + result=ExecResult(exit_code=exec_res), ) + harness.begin_with_initial_hooks() harness.update_config({"webserver-worker-class": worker_class}) assert harness.model.unit.status == ops.StatusBase.from_name( @@ -74,23 +78,25 @@ def test_async_workers_config(harness: Harness, worker_class, expected_status, e @pytest.mark.parametrize( - "worker_class, expected_status, expected_message", + "worker_class, expected_status, expected_message, exec_res", [ ( "eventlet", "blocked", "Only 'gevent' and 'sync' are allowed. https://bit.ly/flask-async-doc", + 1, ), ( "gevent", "blocked", "gunicorn[gevent] must be installed in the rock. https://bit.ly/flask-async-doc", + 1, ), - ("sync", "active", ""), + ("sync", "active", "", 0), ], ) def test_async_workers_config_fail( - harness: Harness, worker_class, expected_status, expected_message + harness: Harness, worker_class, expected_status, expected_message, exec_res ): """ arrange: Prepare a unit and run initial hooks. @@ -103,8 +109,8 @@ def test_async_workers_config_fail( harness.handle_exec( container.name, - ["python3", "-c", "'import gevent;print(gevent.__version__)'"], - result="ModuleNotFoundError", + ["python3", "-c", "import gevent"], + result=ExecResult(exit_code=exec_res), ) harness.begin_with_initial_hooks() harness.update_config({"webserver-worker-class": worker_class}) From 0af73251ca0acc2ae1b32c72bdd35c3a8bc1485a Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 6 Dec 2024 13:34:01 +0300 Subject: [PATCH 18/27] Chore(): Changed implementation --- .../django/django_async_app/requirements.txt | 1 + .../django/django_async_app/rockcraft.yaml | 5 - examples/flask/flask-async.rst | 437 ++++++++++++++++++ .../flask/test_async_rock/requirements.txt | 1 + examples/flask/test_async_rock/rockcraft.yaml | 6 - src/paas_charm/_gunicorn/charm.py | 2 +- src/paas_charm/_gunicorn/webserver.py | 11 +- src/paas_charm/_gunicorn/wsgi_app.py | 7 + src/paas_charm/app.py | 19 +- tests/unit/django/conftest.py | 12 +- tests/unit/django/constants.py | 2 +- tests/unit/django/test_charm.py | 5 +- tests/unit/flask/conftest.py | 12 +- tests/unit/flask/constants.py | 4 +- tests/unit/flask/test_charm.py | 2 +- tests/unit/flask/test_flask_app.py | 10 +- 16 files changed, 505 insertions(+), 31 deletions(-) create mode 100644 examples/flask/flask-async.rst diff --git a/examples/django/django_async_app/requirements.txt b/examples/django/django_async_app/requirements.txt index 2efd6d5..4567a43 100644 --- a/examples/django/django_async_app/requirements.txt +++ b/examples/django/django_async_app/requirements.txt @@ -1,3 +1,4 @@ Django tzdata psycopg2-binary +gevent diff --git a/examples/django/django_async_app/rockcraft.yaml b/examples/django/django_async_app/rockcraft.yaml index 55d9ea0..98b9e97 100644 --- a/examples/django/django_async_app/rockcraft.yaml +++ b/examples/django/django_async_app/rockcraft.yaml @@ -12,8 +12,3 @@ platforms: extensions: - django-framework - -parts: - django-framework/async-dependencies: - python-packages: - - gunicorn[gevent] diff --git a/examples/flask/flask-async.rst b/examples/flask/flask-async.rst new file mode 100644 index 0000000..f27f42d --- /dev/null +++ b/examples/flask/flask-async.rst @@ -0,0 +1,437 @@ +.. _build-a-rock-for-a-flask-application: + +Build a rock for an Async Flask application +------------------------------------ + +In this tutorial, we'll create a simple Async Flask application and learn how to +containerise it in a rock, using Rockcraft's ``flask-framework`` +:ref:`extension `. + +Setup +===== + +.. include:: /reuse/tutorial/setup.rst + +.. note:: + This tutorial requires version ``1.5.4`` or later for Rockcraft. Check + the version using ``rockcraft --version``. If there's an older version + of Rockcraft installed, use + ``sudo snap refresh rockcraft --channel latest/stable`` to get the latest + stable version. + +Finally, create a new directory for this tutorial and go inside it: + +.. code-block:: bash + + mkdir flask-hello-world + cd flask-hello-world + +Create the Flask application +============================ + +Start by creating the "Hello, world" Flask application that we'll use for +this tutorial. + +Create a ``requirements.txt`` file, copy the following text into it and then +save it: + +.. literalinclude:: code/flask-async/requirements.txt + +In order to test the Flask application locally (before packing it into a rock), +install ``python3-venv`` and create a virtual environment: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:create-venv] + :end-before: [docs:create-venv-end] + :dedent: 2 + +In the same directory, copy and save the following into a text file called +``app.py``: + +.. literalinclude:: code/flask-async/app.py + :language: python + + + Run the Flask application using ``flask run -p 8000`` to verify that it works. +Test the Flask application by using ``curl`` to send a request to the root +endpoint. We'll need a new terminal for this -- if we're using Multipass, run +``multipass shell rock-dev`` to get another terminal: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:curl-flask] + :end-before: [docs:curl-flask-end] + :dedent: 2 + +The Flask application should respond with ``Hello, world!``. + +Test the Flask applications io page using ``curl``. It should respond back after +2 seconds. +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:curl-flask-async] + :end-before: [docs:curl-flask-async-end] + :dedent: 2 + +The Flask application should respond with ``I/O task completed in 2 seconds``. + + +The Flask application looks good, so let's stop it for now by pressing +:kbd:`Ctrl` + :kbd:`C`. + +Pack the Flask application into a rock +====================================== + +First, we'll need a ``rockcraft.yaml`` file. Rockcraft will automate its +creation and tailoring for a Flask application by using the +``flask-framework`` profile: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:create-rockcraft-yaml] + :end-before: [docs:create-rockcraft-yaml-end] + :dedent: 2 + +The ``rockcraft.yaml`` file will automatically be created in the project's +working directory. Open it in a text editor and check that the ``name`` is +``flask-hello-world``. Ensure that ``platforms`` includes the host +architecture. For example, if the host uses the ARM +architecture, include ``arm64`` in ``platforms``. + +To enable async workers open the ``rockcraft.yaml`` file and uncomment the +``parts: `` line and the following lines: + +.. codeblock:: text + # parts: + # flask-framework/async-dependencies: + # python-packages: + # - gunicorn[gevent] + +.. note:: + For this tutorial, we'll use the ``name`` ``flask-hello-world`` and assume + we're running on the ``amd64`` platform. Check the architecture of the + system using ``dpkg --print-architecture``. + + The ``name``, ``version`` and ``platform`` all influence the name of the + generated ``.rock`` file. + +Pack the rock: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:pack] + :end-before: [docs:pack-end] + :dedent: 2 + +.. note:: + + Depending on the network, this step can take a couple of minutes to finish. + +Once Rockcraft has finished packing the Flask rock, we'll find a new file in +the project's working directory (an `OCI `_ archive) with the +``.rock`` extension: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:ls-rock] + :end-before: [docs:ls-rock-end] + :dedent: 2 + +The created rock is about 65MB in size. We will reduce its size later in this +tutorial. + +.. note:: + If we changed the ``name`` or ``version`` in ``rockcraft.yaml`` or are not + on an ``amd64`` platform, the name of the ``.rock`` file will be different. + + The size of the rock may vary depending on factors like the architecture + we're building on and the packages installed at the time of packing. + +Run the Flask rock with Docker +============================== + +We already have the rock as an `OCI `_ archive. Now we +need to load it into Docker: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:skopeo-copy] + :end-before: [docs:skopeo-copy-end] + :dedent: 2 + +Check that the image was successfully loaded into Docker: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:docker-images] + :end-before: [docs:docker-images-end] + :dedent: 2 + +The output should list the Flask container image, along with its tag, ID and +size: + +.. code-block:: text + :class: log-snippets + + REPOSITORY TAG IMAGE ID CREATED SIZE + flask-hello-world 0.1 c256056698ba 2 weeks ago 149MB + +.. note:: + The size of the image reported by Docker is the uncompressed size which is + larger than the size of the compressed ``.rock`` file. + +Now we're finally ready to run the rock and test the containerised Flask +application: + +.. literalinclude:: code/flask-async/task.yaml + :language: text + :start-after: [docs:docker-run] + :end-before: [docs:docker-run-end] + :dedent: 2 + +Use the same ``curl`` command as before to send a request to the Flask +application's root endpoint which is running inside the container: + +.. literalinclude:: code/flask-async/task.yaml + :language: text + :start-after: [docs:curl-flask-rock] + :end-before: [docs:curl-flask-rock-end] + :dedent: 2 + +The Flask application should again respond with ``Hello, world!``. + +View the application logs +~~~~~~~~~~~~~~~~~~~~~~~~~ + +When deploying the Flask rock, we can always get the application logs via +``pebble``: + +.. literalinclude:: code/flask-async/task.yaml + :language: text + :start-after: [docs:get-logs] + :end-before: [docs:get-logs-end] + :dedent: 2 + +As a result, :ref:`pebble_explanation_page` will give us the logs for the +``flask`` service running inside the container. +We expect to see something similar to this: + +.. code-block:: text + :class: log-snippets + + 2024-06-21T03:41:45.077Z [flask] [2024-06-21 03:41:45 +0000] [17] [INFO] Starting gunicorn 22.0.0 + 2024-06-21T03:41:45.077Z [flask] [2024-06-21 03:41:45 +0000] [17] [INFO] Listening at: http://0.0.0.0:8000 (17) + 2024-06-21T03:41:45.077Z [flask] [2024-06-21 03:41:45 +0000] [17] [INFO] Using worker: sync + 2024-06-21T03:41:45.078Z [flask] [2024-06-21 03:41:45 +0000] [18] [INFO] Booting worker with pid: 18 + +We can also choose to follow the logs by using the ``-f`` option with the +``pebble logs`` command above. To stop following the logs, press :kbd:`Ctrl` + +:kbd:`C`. + +Cleanup +~~~~~~~ + +Now we have a fully functional rock for a Flask application! This concludes +the first part of this tutorial, so we'll stop the container and remove the +respective image for now: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:stop-docker] + :end-before: [docs:stop-docker-end] + :dedent: 2 + +Chisel the rock +=============== + +This is an optional but recommended step, especially if we're looking to +deploy the rock into a production environment. With :ref:`chisel_explanation` +we can produce lean and production-ready rocks by getting rid of all the +contents that are not needed for the Flask application to run. This results +in a much smaller rock with a reduced attack surface. + +.. note:: + It is recommended to run chiselled images in production. For development, + we may prefer non-chiselled images as they will include additional + development tooling (such as for debugging). + +The first step towards chiselling the rock is to ensure we are using a +``bare`` :ref:`base `. +In ``rockcraft.yaml``, change the ``base`` to ``bare`` and add +``build-base: ubuntu@22.04``: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:change-base] + :end-before: [docs:change-base-end] + :dedent: 2 + +.. note:: + The ``sed`` command replaces the current ``base`` in ``rockcraft.yaml`` with + the ``bare`` base. The command also adds a ``build-base`` which is required + when using the ``bare`` base. + +So that we can compare the size after chiselling, open the ``rockcraft.yaml`` +file and change the ``version`` (e.g. to ``0.1-chiselled``). Pack the rock with +the new ``bare`` :ref:`base `: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:chisel-pack] + :end-before: [docs:chisel-pack-end] + :dedent: 2 + +As before, verify that the new rock was created: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:ls-bare-rock] + :end-before: [docs:ls-bare-rock-end] + :dedent: 2 + +We'll verify that the new Flask rock is now approximately **30% smaller** +in size! And that's just because of the simple change of ``base``. + +And the functionality is still the same. As before, we can confirm this by +running the rock with Docker + +.. literalinclude:: code/flask-async/task.yaml + :language: text + :start-after: [docs:docker-run-chisel] + :end-before: [docs:docker-run-chisel-end] + :dedent: 2 + +and then using the same ``curl`` request: + +.. literalinclude:: code/flask-async/task.yaml + :language: text + :start-after: [docs:curl-flask-bare-rock] + :end-before: [docs:curl-flask-bare-rock-end] + :dedent: 2 + +Unsurprisingly, the Flask application should still respond with +``Hello, world!``. + +Cleanup +~~~~~~~ + +And that's it. We can now stop the container and remove the corresponding +image: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:stop-docker-chisel] + :end-before: [docs:stop-docker-chisel-end] + :dedent: 2 + +.. _update-flask-application: + +Update the Flask application +============================ + +As a final step, let's update our application. For example, +we want to add a new ``/time`` endpoint which returns the current time. + +Start by opening the ``app.py`` file in a text editor and update the code to +look like the following: + +.. literalinclude:: code/flask-async/time_app.py + :language: python + +Since we are creating a new version of the application, open the +``rockcraft.yaml`` file and change the ``version`` (e.g. to ``0.2``). + +.. note:: + + ``rockcraft pack`` will create a new image with the updated code even if we + don't change the version. It is recommended to change the version whenever + we make changes to the application in the image. + +Pack and run the rock using similar commands as before: + +.. literalinclude:: code/flask-async/task.yaml + :language: text + :start-after: [docs:docker-run-update] + :end-before: [docs:docker-run-update-end] + :dedent: 2 + +.. note:: + + Note that the resulting ``.rock`` file will now be named differently, as + its new version will be part of the filename. + +Finally, use ``curl`` to send a request to the ``/time`` endpoint: + +.. literalinclude:: code/flask-async/task.yaml + :language: text + :start-after: [docs:curl-time] + :end-before: [docs:curl-time-end] + :dedent: 2 + +The updated application should respond with the current date and time (e.g. +``2024-06-21 09:47:56``). + +.. note:: + + If you are getting a ``404`` for the ``/time`` endpoint, check the + :ref:`troubleshooting-flask` steps below. + +Cleanup +~~~~~~~ + +We can now stop the container and remove the corresponding image: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:stop-docker-updated] + :end-before: [docs:stop-docker-updated-end] + :dedent: 2 + +Reset the environment +===================== + +We've reached the end of this tutorial. + +If we'd like to reset the working environment, we can simply run the +following: + +.. literalinclude:: code/flask-async/task.yaml + :language: bash + :start-after: [docs:cleanup] + :end-before: [docs:cleanup-end] + :dedent: 2 + +.. collapse:: If using Multipass... + + If we created an instance using Multipass, we can also clean it up. + Start by exiting it: + + .. code-block:: bash + + exit + + And then we can proceed with its deletion: + + .. code-block:: bash + + multipass delete rock-dev + multipass purge + +---- + +.. _troubleshooting-flask: + +Troubleshooting +=============== + +**Application updates not taking effect?** + +Upon changing the Flask application and re-packing the rock, if you believe +your changes are not taking effect (e.g. the ``/time`` +:ref:`endpoint ` is returning a +404), try running ``rockcraft clean`` and pack the rock again with +``rockcraft pack``. + +.. _`lxd-docker-connectivity-issue`: https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker +.. _`install-multipass`: https://multipass.run/docs/install-multipass diff --git a/examples/flask/test_async_rock/requirements.txt b/examples/flask/test_async_rock/requirements.txt index e3e9a71..ea9f5b2 100644 --- a/examples/flask/test_async_rock/requirements.txt +++ b/examples/flask/test_async_rock/requirements.txt @@ -1 +1,2 @@ Flask +gevent diff --git a/examples/flask/test_async_rock/rockcraft.yaml b/examples/flask/test_async_rock/rockcraft.yaml index 443716a..9c789eb 100644 --- a/examples/flask/test_async_rock/rockcraft.yaml +++ b/examples/flask/test_async_rock/rockcraft.yaml @@ -11,9 +11,3 @@ platforms: extensions: - flask-framework - -parts: - flask-framework/async-dependencies: - python-packages: - - gunicorn[gevent] - diff --git a/src/paas_charm/_gunicorn/charm.py b/src/paas_charm/_gunicorn/charm.py index a81187a..33c7e4e 100644 --- a/src/paas_charm/_gunicorn/charm.py +++ b/src/paas_charm/_gunicorn/charm.py @@ -42,7 +42,7 @@ def create_webserver_config(self) -> WebserverConfig: doc_link = f"https://bit.ly/{self._framework_name}-async-doc" - worker_class = None + worker_class = WorkerClassEnum.SYNC try: worker_class = WorkerClassEnum(webserver_config.worker_class) except ValueError as exc: diff --git a/src/paas_charm/_gunicorn/webserver.py b/src/paas_charm/_gunicorn/webserver.py index 40573d7..7dd4512 100644 --- a/src/paas_charm/_gunicorn/webserver.py +++ b/src/paas_charm/_gunicorn/webserver.py @@ -58,7 +58,7 @@ class WebserverConfig: """ workers: int | None = None - worker_class: WorkerClassEnum | None = None + worker_class: WorkerClassEnum | None = WorkerClassEnum.SYNC threads: int | None = None keepalive: datetime.timedelta | None = None timeout: datetime.timedelta | None = None @@ -95,7 +95,6 @@ def from_charm_config( timeout = config.get("webserver-timeout") workers = config.get("webserver-workers") worker_class = config.get("webserver-worker-class") - threads = config.get("webserver-threads") return cls( workers=int(typing.cast(str, workers)) if workers is not None else None, @@ -143,15 +142,15 @@ def _config(self) -> str: setting_value = typing.cast( None | str | WorkerClassEnum | int | datetime.timedelta, setting_value ) + if setting == "worker_class": + continue if setting_value is None: continue setting_value = ( setting_value - if isinstance(setting_value, (int, WorkerClassEnum, str)) + if isinstance(setting_value, (int, str)) else int(setting_value.total_seconds()) ) - if isinstance(setting_value, (WorkerClassEnum, str)): - setting_value = f"'{str(setting_value)}'" config_entries.append(f"{setting} = {setting_value}") if enable_pebble_log_forwarding(): access_log = "'-'" @@ -206,7 +205,7 @@ def update_config( self._container.push(webserver_config_path, self._config) if current_webserver_config == self._config: return - check_config_command = shlex.split(command) + check_config_command = shlex.split(command.split("-k")[0]) check_config_command.append("--check-config") exec_process = self._container.exec( check_config_command, diff --git a/src/paas_charm/_gunicorn/wsgi_app.py b/src/paas_charm/_gunicorn/wsgi_app.py index e5a5a20..9153a65 100644 --- a/src/paas_charm/_gunicorn/wsgi_app.py +++ b/src/paas_charm/_gunicorn/wsgi_app.py @@ -45,6 +45,13 @@ def __init__( # pylint: disable=too-many-arguments framework_config_prefix=f"{workload_config.framework.upper()}_", ) self._webserver = webserver + current_command = self._app_layer()["services"][self._workload_config.framework][ + "command" + ].split("-k")[0] + new_command = f"{current_command}-k sync" + if webserver._webserver_config.worker_class: + new_command = f"{current_command}-k {webserver._webserver_config.worker_class}" + self._alternate_service_command = new_command def _prepare_service_for_restart(self) -> None: """Specific framework operations before restarting the service.""" diff --git a/src/paas_charm/app.py b/src/paas_charm/app.py index 728abfc..df25dd5 100644 --- a/src/paas_charm/app.py +++ b/src/paas_charm/app.py @@ -67,7 +67,9 @@ def should_run_scheduler(self) -> bool: return unit_id == "0" -class App: +# too-many-instance-attributes is disabled because this class +# contains 1 more attributes than pylint allows +class App: # pylint: disable=too-many-instance-attributes """Base class for the application manager.""" def __init__( # pylint: disable=too-many-arguments @@ -92,6 +94,7 @@ def __init__( # pylint: disable=too-many-arguments configuration_prefix: prefix for environment variables related to configuration. integrations_prefix: prefix for environment variables related to integrations. """ + self.__alternate_service_command: str | None = None self._container = container self._charm_state = charm_state self._workload_config = workload_config @@ -172,6 +175,16 @@ def gen_environment(self) -> dict[str, str]: ) return env + @property + def _alternate_service_command(self) -> str | None: + """Specific framework operations before starting the service.""" + return self.__alternate_service_command + + @_alternate_service_command.setter + def _alternate_service_command(self, value: str | None) -> None: + """Specific framework operations before starting the service.""" + self.__alternate_service_command = value + def _prepare_service_for_restart(self) -> None: """Specific framework operations before restarting the service.""" @@ -213,6 +226,10 @@ def _app_layer(self) -> ops.pebble.LayerDict: services[self._workload_config.service_name]["override"] = "replace" services[self._workload_config.service_name]["environment"] = self.gen_environment() + if self._alternate_service_command: + services[self._workload_config.service_name][ + "command" + ] = self._alternate_service_command for service_name, service in services.items(): normalised_service_name = service_name.lower() diff --git a/tests/unit/django/conftest.py b/tests/unit/django/conftest.py index b5e1c3e..9fbb487 100644 --- a/tests/unit/django/conftest.py +++ b/tests/unit/django/conftest.py @@ -87,6 +87,16 @@ def database_migration_mock(): return mock +@pytest.fixture +def container_mock(): + """Create a mock instance for the Container.""" + mock = unittest.mock.MagicMock() + pull_result = unittest.mock.MagicMock() + pull_result.read.return_value = str(DEFAULT_LAYER["services"]).replace("'", '"') + mock.pull.return_value = pull_result + return mock + + def _build_harness(meta=None): """Create a harness instance with the specified metadata.""" harness = Harness(DjangoCharm, meta=meta) @@ -104,7 +114,7 @@ def check_config_handler(_): return ops.testing.ExecResult(1) check_config_command = [ - *shlex.split(DEFAULT_LAYER["services"]["django"]["command"]), + *shlex.split(DEFAULT_LAYER["services"]["django"]["command"].split("-k")[0]), "--check-config", ] harness.handle_exec( diff --git a/tests/unit/django/constants.py b/tests/unit/django/constants.py index 3145cf1..4829ade 100644 --- a/tests/unit/django/constants.py +++ b/tests/unit/django/constants.py @@ -6,7 +6,7 @@ "django": { "override": "replace", "startup": "enabled", - "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application", + "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application -k sync", "after": ["statsd-exporter"], "user": "_daemon_", }, diff --git a/tests/unit/django/test_charm.py b/tests/unit/django/test_charm.py index feb8637..f84f24c 100644 --- a/tests/unit/django/test_charm.py +++ b/tests/unit/django/test_charm.py @@ -94,7 +94,7 @@ def test_django_config(harness: Harness, config: dict, env: dict) -> None: "environment": env, "override": "replace", "startup": "enabled", - "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application", + "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application -k sync", "after": ["statsd-exporter"], "user": "_daemon_", } @@ -169,6 +169,7 @@ def test_django_async_config(harness: Harness, config: dict, env: dict) -> None: secret_storage = unittest.mock.MagicMock() secret_storage.is_secret_storage_ready = True secret_storage.get_secret_key.return_value = "test" + config["webserver-worker-class"] = "gevent" harness.update_config(config) charm_state = CharmState.from_charm( config=harness.charm.config, @@ -198,7 +199,7 @@ def test_django_async_config(harness: Harness, config: dict, env: dict) -> None: "environment": env, "override": "replace", "startup": "enabled", - "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application", + "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application -k gevent", "after": ["statsd-exporter"], "user": "_daemon_", } diff --git a/tests/unit/flask/conftest.py b/tests/unit/flask/conftest.py index 604d22f..cd4d210 100644 --- a/tests/unit/flask/conftest.py +++ b/tests/unit/flask/conftest.py @@ -42,7 +42,7 @@ def check_config_handler(_): return ops.testing.ExecResult(1) check_config_command = [ - *shlex.split(DEFAULT_LAYER["services"]["flask"]["command"]), + *shlex.split(DEFAULT_LAYER["services"]["flask"]["command"].split("-k")[0]), "--check-config", ] harness.handle_exec( @@ -62,3 +62,13 @@ def database_migration_mock(): mock.status = DatabaseMigrationStatus.PENDING mock.script = None return mock + + +@pytest.fixture +def container_mock(): + """Create a mock instance for the Container.""" + mock = unittest.mock.MagicMock() + pull_result = unittest.mock.MagicMock() + pull_result.read.return_value = str(DEFAULT_LAYER["services"]).replace("'", '"') + mock.pull.return_value = pull_result + return mock diff --git a/tests/unit/flask/constants.py b/tests/unit/flask/constants.py index d9936c3..68756a6 100644 --- a/tests/unit/flask/constants.py +++ b/tests/unit/flask/constants.py @@ -8,7 +8,7 @@ "flask": { "override": "replace", "startup": "enabled", - "command": f"/bin/python3 -m gunicorn -c /flask/gunicorn.conf.py app:app", + "command": f"/bin/python3 -m gunicorn -c /flask/gunicorn.conf.py app:app -k sync", "after": ["statsd-exporter"], "user": "_daemon_", }, @@ -31,7 +31,7 @@ "flask": { "override": "replace", "startup": "enabled", - "command": f"/bin/python3 -m gunicorn -c /flask/gunicorn.conf.py app:app", + "command": f"/bin/python3 -m gunicorn -c /flask/gunicorn.conf.py app:app -k sync", "after": ["statsd-exporter"], "user": "_daemon_", }, diff --git a/tests/unit/flask/test_charm.py b/tests/unit/flask/test_charm.py index 4e057e9..1b1bfeb 100644 --- a/tests/unit/flask/test_charm.py +++ b/tests/unit/flask/test_charm.py @@ -74,7 +74,7 @@ def test_flask_pebble_layer(harness: Harness) -> None: }, "override": "replace", "startup": "enabled", - "command": f"/bin/python3 -m gunicorn -c /flask/gunicorn.conf.py app:app", + "command": f"/bin/python3 -m gunicorn -c /flask/gunicorn.conf.py app:app -k sync", "after": ["statsd-exporter"], "user": "_daemon_", } diff --git a/tests/unit/flask/test_flask_app.py b/tests/unit/flask/test_flask_app.py index 596381c..a647a48 100644 --- a/tests/unit/flask/test_flask_app.py +++ b/tests/unit/flask/test_flask_app.py @@ -33,7 +33,7 @@ ), ], ) -def test_flask_env(flask_config: dict, app_config: dict, database_migration_mock): +def test_flask_env(flask_config: dict, app_config: dict, database_migration_mock, container_mock): """ arrange: create the Flask app object with a controlled charm state. act: none. @@ -48,7 +48,7 @@ def test_flask_env(flask_config: dict, app_config: dict, database_migration_mock ) workload_config = create_workload_config(framework_name="flask", unit_name="flask/0") flask_app = WsgiApp( - container=unittest.mock.MagicMock(), + container=container_mock, charm_state=charm_state, workload_config=workload_config, webserver=unittest.mock.MagicMock(), @@ -110,6 +110,7 @@ def test_http_proxy( expected: typing.Dict[str, str], monkeypatch, database_migration_mock, + container_mock, ): """ arrange: set juju charm http proxy related environment variables. @@ -125,7 +126,7 @@ def test_http_proxy( ) workload_config = create_workload_config(framework_name="flask", unit_name="flask/0") flask_app = WsgiApp( - container=unittest.mock.MagicMock(), + container=container_mock, charm_state=charm_state, workload_config=workload_config, webserver=unittest.mock.MagicMock(), @@ -162,6 +163,7 @@ def test_http_proxy( def test_integrations_env( monkeypatch, database_migration_mock, + container_mock, integrations, expected_vars, ): @@ -178,7 +180,7 @@ def test_integrations_env( ) workload_config = create_workload_config(framework_name="flask", unit_name="flask/0") flask_app = WsgiApp( - container=unittest.mock.MagicMock(), + container=container_mock, charm_state=charm_state, workload_config=workload_config, webserver=unittest.mock.MagicMock(), From 637542dc937f866b8e8487c8ac719852a4fe3e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20U=C4=9EUR?= <39213991+alithethird@users.noreply.github.com> Date: Fri, 6 Dec 2024 13:35:35 +0300 Subject: [PATCH 19/27] Update examples/flask/flask-async.rst Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- examples/flask/flask-async.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/flask/flask-async.rst b/examples/flask/flask-async.rst index f27f42d..14f39a3 100644 --- a/examples/flask/flask-async.rst +++ b/examples/flask/flask-async.rst @@ -1,3 +1,6 @@ +.. Copyright 2024 Canonical Ltd. +.. See LICENSE file for licensing details. + .. _build-a-rock-for-a-flask-application: Build a rock for an Async Flask application From 0b255a115918287632c9da089d73f52258e023ba Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 6 Dec 2024 13:37:10 +0300 Subject: [PATCH 20/27] Chore(): Remove wrongly placed doc --- examples/flask/flask-async.rst | 440 --------------------------------- 1 file changed, 440 deletions(-) delete mode 100644 examples/flask/flask-async.rst diff --git a/examples/flask/flask-async.rst b/examples/flask/flask-async.rst deleted file mode 100644 index 14f39a3..0000000 --- a/examples/flask/flask-async.rst +++ /dev/null @@ -1,440 +0,0 @@ -.. Copyright 2024 Canonical Ltd. -.. See LICENSE file for licensing details. - -.. _build-a-rock-for-a-flask-application: - -Build a rock for an Async Flask application ------------------------------------- - -In this tutorial, we'll create a simple Async Flask application and learn how to -containerise it in a rock, using Rockcraft's ``flask-framework`` -:ref:`extension `. - -Setup -===== - -.. include:: /reuse/tutorial/setup.rst - -.. note:: - This tutorial requires version ``1.5.4`` or later for Rockcraft. Check - the version using ``rockcraft --version``. If there's an older version - of Rockcraft installed, use - ``sudo snap refresh rockcraft --channel latest/stable`` to get the latest - stable version. - -Finally, create a new directory for this tutorial and go inside it: - -.. code-block:: bash - - mkdir flask-hello-world - cd flask-hello-world - -Create the Flask application -============================ - -Start by creating the "Hello, world" Flask application that we'll use for -this tutorial. - -Create a ``requirements.txt`` file, copy the following text into it and then -save it: - -.. literalinclude:: code/flask-async/requirements.txt - -In order to test the Flask application locally (before packing it into a rock), -install ``python3-venv`` and create a virtual environment: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:create-venv] - :end-before: [docs:create-venv-end] - :dedent: 2 - -In the same directory, copy and save the following into a text file called -``app.py``: - -.. literalinclude:: code/flask-async/app.py - :language: python - - - Run the Flask application using ``flask run -p 8000`` to verify that it works. -Test the Flask application by using ``curl`` to send a request to the root -endpoint. We'll need a new terminal for this -- if we're using Multipass, run -``multipass shell rock-dev`` to get another terminal: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:curl-flask] - :end-before: [docs:curl-flask-end] - :dedent: 2 - -The Flask application should respond with ``Hello, world!``. - -Test the Flask applications io page using ``curl``. It should respond back after -2 seconds. -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:curl-flask-async] - :end-before: [docs:curl-flask-async-end] - :dedent: 2 - -The Flask application should respond with ``I/O task completed in 2 seconds``. - - -The Flask application looks good, so let's stop it for now by pressing -:kbd:`Ctrl` + :kbd:`C`. - -Pack the Flask application into a rock -====================================== - -First, we'll need a ``rockcraft.yaml`` file. Rockcraft will automate its -creation and tailoring for a Flask application by using the -``flask-framework`` profile: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:create-rockcraft-yaml] - :end-before: [docs:create-rockcraft-yaml-end] - :dedent: 2 - -The ``rockcraft.yaml`` file will automatically be created in the project's -working directory. Open it in a text editor and check that the ``name`` is -``flask-hello-world``. Ensure that ``platforms`` includes the host -architecture. For example, if the host uses the ARM -architecture, include ``arm64`` in ``platforms``. - -To enable async workers open the ``rockcraft.yaml`` file and uncomment the -``parts: `` line and the following lines: - -.. codeblock:: text - # parts: - # flask-framework/async-dependencies: - # python-packages: - # - gunicorn[gevent] - -.. note:: - For this tutorial, we'll use the ``name`` ``flask-hello-world`` and assume - we're running on the ``amd64`` platform. Check the architecture of the - system using ``dpkg --print-architecture``. - - The ``name``, ``version`` and ``platform`` all influence the name of the - generated ``.rock`` file. - -Pack the rock: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:pack] - :end-before: [docs:pack-end] - :dedent: 2 - -.. note:: - - Depending on the network, this step can take a couple of minutes to finish. - -Once Rockcraft has finished packing the Flask rock, we'll find a new file in -the project's working directory (an `OCI `_ archive) with the -``.rock`` extension: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:ls-rock] - :end-before: [docs:ls-rock-end] - :dedent: 2 - -The created rock is about 65MB in size. We will reduce its size later in this -tutorial. - -.. note:: - If we changed the ``name`` or ``version`` in ``rockcraft.yaml`` or are not - on an ``amd64`` platform, the name of the ``.rock`` file will be different. - - The size of the rock may vary depending on factors like the architecture - we're building on and the packages installed at the time of packing. - -Run the Flask rock with Docker -============================== - -We already have the rock as an `OCI `_ archive. Now we -need to load it into Docker: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:skopeo-copy] - :end-before: [docs:skopeo-copy-end] - :dedent: 2 - -Check that the image was successfully loaded into Docker: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:docker-images] - :end-before: [docs:docker-images-end] - :dedent: 2 - -The output should list the Flask container image, along with its tag, ID and -size: - -.. code-block:: text - :class: log-snippets - - REPOSITORY TAG IMAGE ID CREATED SIZE - flask-hello-world 0.1 c256056698ba 2 weeks ago 149MB - -.. note:: - The size of the image reported by Docker is the uncompressed size which is - larger than the size of the compressed ``.rock`` file. - -Now we're finally ready to run the rock and test the containerised Flask -application: - -.. literalinclude:: code/flask-async/task.yaml - :language: text - :start-after: [docs:docker-run] - :end-before: [docs:docker-run-end] - :dedent: 2 - -Use the same ``curl`` command as before to send a request to the Flask -application's root endpoint which is running inside the container: - -.. literalinclude:: code/flask-async/task.yaml - :language: text - :start-after: [docs:curl-flask-rock] - :end-before: [docs:curl-flask-rock-end] - :dedent: 2 - -The Flask application should again respond with ``Hello, world!``. - -View the application logs -~~~~~~~~~~~~~~~~~~~~~~~~~ - -When deploying the Flask rock, we can always get the application logs via -``pebble``: - -.. literalinclude:: code/flask-async/task.yaml - :language: text - :start-after: [docs:get-logs] - :end-before: [docs:get-logs-end] - :dedent: 2 - -As a result, :ref:`pebble_explanation_page` will give us the logs for the -``flask`` service running inside the container. -We expect to see something similar to this: - -.. code-block:: text - :class: log-snippets - - 2024-06-21T03:41:45.077Z [flask] [2024-06-21 03:41:45 +0000] [17] [INFO] Starting gunicorn 22.0.0 - 2024-06-21T03:41:45.077Z [flask] [2024-06-21 03:41:45 +0000] [17] [INFO] Listening at: http://0.0.0.0:8000 (17) - 2024-06-21T03:41:45.077Z [flask] [2024-06-21 03:41:45 +0000] [17] [INFO] Using worker: sync - 2024-06-21T03:41:45.078Z [flask] [2024-06-21 03:41:45 +0000] [18] [INFO] Booting worker with pid: 18 - -We can also choose to follow the logs by using the ``-f`` option with the -``pebble logs`` command above. To stop following the logs, press :kbd:`Ctrl` + -:kbd:`C`. - -Cleanup -~~~~~~~ - -Now we have a fully functional rock for a Flask application! This concludes -the first part of this tutorial, so we'll stop the container and remove the -respective image for now: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:stop-docker] - :end-before: [docs:stop-docker-end] - :dedent: 2 - -Chisel the rock -=============== - -This is an optional but recommended step, especially if we're looking to -deploy the rock into a production environment. With :ref:`chisel_explanation` -we can produce lean and production-ready rocks by getting rid of all the -contents that are not needed for the Flask application to run. This results -in a much smaller rock with a reduced attack surface. - -.. note:: - It is recommended to run chiselled images in production. For development, - we may prefer non-chiselled images as they will include additional - development tooling (such as for debugging). - -The first step towards chiselling the rock is to ensure we are using a -``bare`` :ref:`base `. -In ``rockcraft.yaml``, change the ``base`` to ``bare`` and add -``build-base: ubuntu@22.04``: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:change-base] - :end-before: [docs:change-base-end] - :dedent: 2 - -.. note:: - The ``sed`` command replaces the current ``base`` in ``rockcraft.yaml`` with - the ``bare`` base. The command also adds a ``build-base`` which is required - when using the ``bare`` base. - -So that we can compare the size after chiselling, open the ``rockcraft.yaml`` -file and change the ``version`` (e.g. to ``0.1-chiselled``). Pack the rock with -the new ``bare`` :ref:`base `: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:chisel-pack] - :end-before: [docs:chisel-pack-end] - :dedent: 2 - -As before, verify that the new rock was created: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:ls-bare-rock] - :end-before: [docs:ls-bare-rock-end] - :dedent: 2 - -We'll verify that the new Flask rock is now approximately **30% smaller** -in size! And that's just because of the simple change of ``base``. - -And the functionality is still the same. As before, we can confirm this by -running the rock with Docker - -.. literalinclude:: code/flask-async/task.yaml - :language: text - :start-after: [docs:docker-run-chisel] - :end-before: [docs:docker-run-chisel-end] - :dedent: 2 - -and then using the same ``curl`` request: - -.. literalinclude:: code/flask-async/task.yaml - :language: text - :start-after: [docs:curl-flask-bare-rock] - :end-before: [docs:curl-flask-bare-rock-end] - :dedent: 2 - -Unsurprisingly, the Flask application should still respond with -``Hello, world!``. - -Cleanup -~~~~~~~ - -And that's it. We can now stop the container and remove the corresponding -image: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:stop-docker-chisel] - :end-before: [docs:stop-docker-chisel-end] - :dedent: 2 - -.. _update-flask-application: - -Update the Flask application -============================ - -As a final step, let's update our application. For example, -we want to add a new ``/time`` endpoint which returns the current time. - -Start by opening the ``app.py`` file in a text editor and update the code to -look like the following: - -.. literalinclude:: code/flask-async/time_app.py - :language: python - -Since we are creating a new version of the application, open the -``rockcraft.yaml`` file and change the ``version`` (e.g. to ``0.2``). - -.. note:: - - ``rockcraft pack`` will create a new image with the updated code even if we - don't change the version. It is recommended to change the version whenever - we make changes to the application in the image. - -Pack and run the rock using similar commands as before: - -.. literalinclude:: code/flask-async/task.yaml - :language: text - :start-after: [docs:docker-run-update] - :end-before: [docs:docker-run-update-end] - :dedent: 2 - -.. note:: - - Note that the resulting ``.rock`` file will now be named differently, as - its new version will be part of the filename. - -Finally, use ``curl`` to send a request to the ``/time`` endpoint: - -.. literalinclude:: code/flask-async/task.yaml - :language: text - :start-after: [docs:curl-time] - :end-before: [docs:curl-time-end] - :dedent: 2 - -The updated application should respond with the current date and time (e.g. -``2024-06-21 09:47:56``). - -.. note:: - - If you are getting a ``404`` for the ``/time`` endpoint, check the - :ref:`troubleshooting-flask` steps below. - -Cleanup -~~~~~~~ - -We can now stop the container and remove the corresponding image: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:stop-docker-updated] - :end-before: [docs:stop-docker-updated-end] - :dedent: 2 - -Reset the environment -===================== - -We've reached the end of this tutorial. - -If we'd like to reset the working environment, we can simply run the -following: - -.. literalinclude:: code/flask-async/task.yaml - :language: bash - :start-after: [docs:cleanup] - :end-before: [docs:cleanup-end] - :dedent: 2 - -.. collapse:: If using Multipass... - - If we created an instance using Multipass, we can also clean it up. - Start by exiting it: - - .. code-block:: bash - - exit - - And then we can proceed with its deletion: - - .. code-block:: bash - - multipass delete rock-dev - multipass purge - ----- - -.. _troubleshooting-flask: - -Troubleshooting -=============== - -**Application updates not taking effect?** - -Upon changing the Flask application and re-packing the rock, if you believe -your changes are not taking effect (e.g. the ``/time`` -:ref:`endpoint ` is returning a -404), try running ``rockcraft clean`` and pack the rock again with -``rockcraft pack``. - -.. _`lxd-docker-connectivity-issue`: https://documentation.ubuntu.com/lxd/en/latest/howto/network_bridge_firewalld/#prevent-connectivity-issues-with-lxd-and-docker -.. _`install-multipass`: https://multipass.run/docs/install-multipass From 6d16c78b405d677e33ab6d5fca80cd52440c1ef9 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Thu, 12 Dec 2024 14:25:44 +0300 Subject: [PATCH 21/27] chore(test): Use branch to source compile rockcraft & charmcraft for integration tests --- .github/workflows/integration_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 727e3f8..5f73e62 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,6 +12,8 @@ jobs: secrets: inherit with: extra-arguments: -x --localstack-address 172.17.0.1 + charmcraft-repository: https://github.com/alithethird/charmcraft + charmcraft-ref: flask-async-worker pre-run-script: localstack-installation.sh charmcraft-channel: latest/edge modules: '["test_charm.py", "test_cos.py", "test_database.py", "test_db_migration.py", "test_django.py", "test_django_integrations.py", "test_fastapi.py", "test_go.py", "test_integrations.py", "test_proxy.py", "test_workers.py"]' From 3bdc6aace7222905d113744276f7eb7e4a34806c Mon Sep 17 00:00:00 2001 From: ali ugur Date: Thu, 12 Dec 2024 14:27:31 +0300 Subject: [PATCH 22/27] chore(test): Fix repo links --- .github/workflows/integration_test.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 5f73e62..0dbf6be 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,11 +12,13 @@ jobs: secrets: inherit with: extra-arguments: -x --localstack-address 172.17.0.1 - charmcraft-repository: https://github.com/alithethird/charmcraft + charmcraft-repository: alithethird/charmcraft charmcraft-ref: flask-async-worker pre-run-script: localstack-installation.sh - charmcraft-channel: latest/edge + # charmcraft-channel: latest/edge modules: '["test_charm.py", "test_cos.py", "test_database.py", "test_db_migration.py", "test_django.py", "test_django_integrations.py", "test_fastapi.py", "test_go.py", "test_integrations.py", "test_proxy.py", "test_workers.py"]' - rockcraft-channel: latest/edge + rockcraft-repository: alithethird/rockcraft + rockcraft-ref: flask-django-extention-async-workers + # rockcraft-channel: latest/edge juju-channel: ${{ matrix.juju-version }} channel: 1.29-strict/stable From 88d898f105c40294c56e9b8d198bf4a9d4df108d Mon Sep 17 00:00:00 2001 From: ali ugur Date: Fri, 13 Dec 2024 06:41:37 +0300 Subject: [PATCH 23/27] chore(test): Increase timeout in integration tests --- .github/workflows/integration_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 0dbf6be..d9e9319 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -22,3 +22,4 @@ jobs: # rockcraft-channel: latest/edge juju-channel: ${{ matrix.juju-version }} channel: 1.29-strict/stable + test-timeout: 30 From 703aa75525b530e4e14a07540098a6be624c7298 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Mon, 16 Dec 2024 07:25:57 +0300 Subject: [PATCH 24/27] chore(): Fix compatibility issue --- src/paas_charm/_gunicorn/wsgi_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/paas_charm/_gunicorn/wsgi_app.py b/src/paas_charm/_gunicorn/wsgi_app.py index 9153a65..f6e3e20 100644 --- a/src/paas_charm/_gunicorn/wsgi_app.py +++ b/src/paas_charm/_gunicorn/wsgi_app.py @@ -48,9 +48,9 @@ def __init__( # pylint: disable=too-many-arguments current_command = self._app_layer()["services"][self._workload_config.framework][ "command" ].split("-k")[0] - new_command = f"{current_command}-k sync" + new_command = f"{current_command} -k sync" if webserver._webserver_config.worker_class: - new_command = f"{current_command}-k {webserver._webserver_config.worker_class}" + new_command = f"{current_command} -k {webserver._webserver_config.worker_class}" self._alternate_service_command = new_command def _prepare_service_for_restart(self) -> None: From 06beaf074d7f8944228a94bef76cf346d2cfc371 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Wed, 18 Dec 2024 06:46:08 +0300 Subject: [PATCH 25/27] chore(test): Fix unit tests. --- tests/unit/django/test_charm.py | 4 ++-- tests/unit/flask/test_charm.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/django/test_charm.py b/tests/unit/django/test_charm.py index f84f24c..88554ed 100644 --- a/tests/unit/django/test_charm.py +++ b/tests/unit/django/test_charm.py @@ -94,7 +94,7 @@ def test_django_config(harness: Harness, config: dict, env: dict) -> None: "environment": env, "override": "replace", "startup": "enabled", - "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application -k sync", + "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application -k sync", "after": ["statsd-exporter"], "user": "_daemon_", } @@ -199,7 +199,7 @@ def test_django_async_config(harness: Harness, config: dict, env: dict) -> None: "environment": env, "override": "replace", "startup": "enabled", - "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application -k gevent", + "command": "/bin/python3 -m gunicorn -c /django/gunicorn.conf.py django_app.wsgi:application -k gevent", "after": ["statsd-exporter"], "user": "_daemon_", } diff --git a/tests/unit/flask/test_charm.py b/tests/unit/flask/test_charm.py index 1b1bfeb..5acf5cf 100644 --- a/tests/unit/flask/test_charm.py +++ b/tests/unit/flask/test_charm.py @@ -74,7 +74,7 @@ def test_flask_pebble_layer(harness: Harness) -> None: }, "override": "replace", "startup": "enabled", - "command": f"/bin/python3 -m gunicorn -c /flask/gunicorn.conf.py app:app -k sync", + "command": f"/bin/python3 -m gunicorn -c /flask/gunicorn.conf.py app:app -k sync", "after": ["statsd-exporter"], "user": "_daemon_", } From bb2931117bb90e0cbb13dbebbf601869b7d4fcc4 Mon Sep 17 00:00:00 2001 From: ali ugur Date: Thu, 19 Dec 2024 06:03:30 +0300 Subject: [PATCH 26/27] chore(test): Use the latest/edge rockcraft instead of fork --- .github/workflows/integration_test.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index d9e9319..a35f9f2 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -17,9 +17,7 @@ jobs: pre-run-script: localstack-installation.sh # charmcraft-channel: latest/edge modules: '["test_charm.py", "test_cos.py", "test_database.py", "test_db_migration.py", "test_django.py", "test_django_integrations.py", "test_fastapi.py", "test_go.py", "test_integrations.py", "test_proxy.py", "test_workers.py"]' - rockcraft-repository: alithethird/rockcraft - rockcraft-ref: flask-django-extention-async-workers - # rockcraft-channel: latest/edge + rockcraft-channel: latest/edge juju-channel: ${{ matrix.juju-version }} channel: 1.29-strict/stable test-timeout: 30 From 2aac183d62e9207d8fa37be8c30c6171c10e52de Mon Sep 17 00:00:00 2001 From: ali ugur Date: Thu, 19 Dec 2024 07:31:00 +0300 Subject: [PATCH 27/27] chore(trivy): Add 2 ignore lines for go libraries --- .trivyignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.trivyignore b/.trivyignore index f39633c..6c2a020 100644 --- a/.trivyignore +++ b/.trivyignore @@ -4,3 +4,7 @@ CVE-2022-40897 CVE-2024-6345 # pebble: Calling Decoder.Decode on a message which contains deeply nested structures can cause a panic due to stack exhaustion CVE-2024-34156 +# pebble: Go stdlib +CVE-2024-45338 +# go-app: Go crypto lib +CVE-2024-45337