diff --git a/.gitignore b/.gitignore index e60bd8a..321f695 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,9 @@ target/ profile_default/ ipython_config.py +# aiotaskq test outputs +src/tests/apps/sample_app_django/out-*.json + # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: diff --git a/.vscode/launch.json b/.vscode/launch.json index 7a073b2..acccbd5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,7 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { "name": "Python: Attach", "type": "python", @@ -54,6 +55,37 @@ "tests.apps.simple_app" ], "console": "integratedTerminal", + }, + { + "name": "Celery Worker (Django Sample App)", + "type": "python", + "request": "launch", + "module": "celery", + "args": [ + "-A", + "sample_app_django", + "worker", + "--concurrency", + "4", + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/src/tests/apps/sample_app_django/", + "justMyCode": false + }, + { + "name": "AioTaskQ Worker (Django Sample App)", + "type": "python", + "request": "launch", + "module": "aiotaskq", + "args": [ + "worker", + "sample_app_django", + "--concurrency", + "4", + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/src/tests/apps/sample_app_django/", + "justMyCode": false } ] } diff --git a/README.md b/README.md index b8f127a..350577a 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,10 @@ Plus, it is also fully-typed for better productivity and correctness. Give it a try and let us know if you like it. For questions or feedback feel to file issues on this repository. +## Sampe codes + +1. [Simple Django App](/src/tests/apps/sample_app_django/README.md) + ## Example Usage Install aiotaskq ```bash diff --git a/docker-compose.yml b/docker-compose.yml index 5a3ffb6..ea85a13 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,3 +15,12 @@ services: command: redis-cli -h redis depends_on: - redis + + db: + image: "postgres" + ports: + - 5432:5432 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=sample_app_django diff --git a/pyproject.toml b/pyproject.toml index f6fbbc1..ae7031c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "typer >= 0.4.0, < 0.5.0", ] name = "aiotaskq" -version = "0.0.13" +version = "0.0.14" readme = "README.md" description = "A simple asynchronous task queue" authors = [ @@ -31,12 +31,13 @@ license = { file = "LICENSE" } dev = [ "black >= 22.2.0, < 23.0.0", "coverage >= 6.4.0, < 6.5.0", + "django-stubs >= 5.1.0, < 5.2.0", "mypy >= 0.931, < 1.0", "mypy-extensions >= 0.4.0, < 0.5.0", "pytest-asyncio >= 0.19.0, < 0.20.0", "pylint >= 2.14.0, < 2.15.0", "pytest >= 7.1.0, < 7.2.0", - "typing_extensions >= 4.1.1, < 4.2.0", + "typing_extensions >= 4.11.0, < 4.12.0", ] [project.urls] diff --git a/src/aiotaskq/__init__.py b/src/aiotaskq/__init__.py index 4d5fd35..66189ba 100644 --- a/src/aiotaskq/__init__.py +++ b/src/aiotaskq/__init__.py @@ -31,10 +31,9 @@ async def main(): import tomlkit +from .app import App from .task import task -with Path("pyproject.toml").open("r", encoding="utf-8") as f: - toml_dict = tomlkit.loads(f.read()) - __version__ = toml_dict["project"]["version"] -__all__ = ["__version__", "task"] +__version__ = "0.0.14" +__all__ = ["__version__", "task", "App"] diff --git a/src/aiotaskq/app.py b/src/aiotaskq/app.py new file mode 100644 index 0000000..09a4d6d --- /dev/null +++ b/src/aiotaskq/app.py @@ -0,0 +1,48 @@ +""" +Define the aiotaskq application instance. + +This app instance provides access to all tasks defined within the application. +""" + +from importlib import import_module +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from .task import Task + + +class App: + """Define the aiotaskq application instance.""" + + _task_registry: dict[str, "Task"] = {} + + def __getattribute__(self, name: str, /) -> Any: + """Get access to all task instances defined within the application.""" + + task_registry = object.__getattribute__(self, "_task_registry") + if name in task_registry: + return task_registry[name] + return object.__getattribute__(self, name) + + def autodiscover_tasks(self, tasks_module_name: str = "tasks"): + """ + Search for all tasks defined within the application and imported them. + + The tasks are expected to be defined in files named as "tasks.py". + """ + + import django # pylint: disable=import-outside-toplevel + from django.apps import apps # pylint: disable=import-outside-toplevel + + django.setup() + + module_names: list[str] = [config.name for config in apps.get_app_configs()] + for module_name in module_names: + _ = import_module(module_name) + try: + _ = import_module(f"{module_name}.{tasks_module_name}") + except ModuleNotFoundError: + pass + else: + pass diff --git a/src/aiotaskq/config.py b/src/aiotaskq/config.py index 1e5af18..cdebf54 100644 --- a/src/aiotaskq/config.py +++ b/src/aiotaskq/config.py @@ -33,7 +33,7 @@ def serialization_type() -> SerializationType: @staticmethod def log_level() -> int: """Return the log level as provided via env var LOG_LEVEL.""" - level: int = int(environ.get("AIOTASKQ_LOG_LEVEL", logging.DEBUG)) + level: int = getattr(logging, environ.get("AIOTASKQ_LOG_LEVEL", "DEBUG")) return level @staticmethod diff --git a/src/aiotaskq/utils.py b/src/aiotaskq/utils.py new file mode 100644 index 0000000..fc9c63d --- /dev/null +++ b/src/aiotaskq/utils.py @@ -0,0 +1,34 @@ +"""Define util functions for use within the whole library.""" + +import os +import sys +from contextlib import contextmanager +from importlib import import_module + + +def import_from_cwd(import_path): + """Import module as if the caller is located in current working directory (cwd).""" + with _cwd_in_path(): + return import_module(import_path) + + +# Private region + + +@contextmanager +def _cwd_in_path(): + """Context adding the current working directory to sys.path.""" + cwd = os.getcwd() + if cwd in sys.path: + yield + else: + sys.path.insert(0, cwd) + try: + yield cwd + finally: + try: + sys.path.remove(cwd) + except ValueError: # pragma: no cover + pass + +# Private region ends diff --git a/src/aiotaskq/worker.py b/src/aiotaskq/worker.py index 6dcc7ce..1aa7e4f 100755 --- a/src/aiotaskq/worker.py +++ b/src/aiotaskq/worker.py @@ -3,7 +3,6 @@ from abc import ABC, abstractmethod import asyncio from functools import cached_property -import importlib import inspect import logging import multiprocessing @@ -22,6 +21,7 @@ from .pubsub import PubSub from .serde import Serialization from .task import AsyncResult, Task +from .utils import import_from_cwd logger = logging.getLogger(__name__) @@ -40,7 +40,7 @@ class BaseWorker(ABC): concurrency_manager: IConcurrencyManager def __init__(self, app_import_path: str): - self.app = importlib.import_module(app_import_path) + self.app = import_from_cwd(app_import_path) def run_forever(self) -> None: """ @@ -308,7 +308,7 @@ async def _execute_task_and_publish( def validate_input(app_import_path: str) -> t.Optional[str]: """Validate all worker cli inputs and return an error string if any.""" try: - importlib.import_module(app_import_path) + import_from_cwd(app_import_path) except ModuleNotFoundError: return ( f"Error at argument `--app_import_path {app_import_path}`:" diff --git a/src/tests/apps/sample_app_django/api/__init__.py b/src/tests/apps/sample_app_django/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tests/apps/sample_app_django/api/apps.py b/src/tests/apps/sample_app_django/api/apps.py new file mode 100644 index 0000000..878e7d5 --- /dev/null +++ b/src/tests/apps/sample_app_django/api/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ApiConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "api" diff --git a/src/tests/apps/sample_app_django/api/management/commands/populate_db.py b/src/tests/apps/sample_app_django/api/management/commands/populate_db.py new file mode 100644 index 0000000..e3c0420 --- /dev/null +++ b/src/tests/apps/sample_app_django/api/management/commands/populate_db.py @@ -0,0 +1,90 @@ +import logging +import random +import string + +from django.core.management import BaseCommand + +from ...models import User, Order +from ...utils import chunker + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument("--n-users", type=int, default=1000) + parser.add_argument("--n-orders-per-user", type=int, default=1000) + parser.add_argument("--from-scratch", action="store_true") + + def handle(self, *args, **options): + if options["from_scratch"]: + _delete_all() + _populate_users(n=options["n_users"]) + _populate_orders(n_per_user=options["n_orders_per_user"]) + + +def _delete_all(): + r = Order.objects.all().delete() + logger.debug("Deleted %s: %s", Order.__name__, r) + r = User.objects.all().delete() + logger.debug("Deleted %s: %s", User.__name__, r) + + +def _populate_users(n: int) -> None: + users: list[User] = [] + username_length: int = User._meta.get_field("username").max_length + random_username_generator = RandomStringGenerator(length=username_length) + for _ in range(n): + username_random: str = random_username_generator.generate() + user = User(username=username_random) + users.append(user) + + for user_chunk in chunker(users, size=10_000): + User.objects.bulk_create(user_chunk) + print(f"Bulk-created {len(user_chunk)} random User(s)") + + +def _populate_orders(n_per_user: int) -> None: + user_ids: list[int] = list(User.objects.all().values_list("id", flat=True)) + order_name_length: int = Order._meta.get_field("name").max_length + random_order_name_id_generator = RandomStringGenerator(length=order_name_length) + + for user_ids_chunk in chunker(user_ids, size=1000): + orders: list[Order] = [] + + for user_id in user_ids_chunk: + for _ in range(n_per_user): + order = Order( + user_id=user_id, + name=random_order_name_id_generator.generate(), + price=random.uniform(0.0, 5_000.00), + ) + orders.append(order) + + Order.objects.bulk_create(orders) + print(f"Created {len(orders)} random Order(s) for {len(user_ids_chunk)} users") + + +class MaxRandomStringTryReached(Exception): + pass + + +class RandomStringGenerator: + max_tries = 10 + + def __init__(self, length: int): + self.length = length + self.existing: set[str] = set() + + def generate(self) -> str: + tries = 0 + while True: + s = ''.join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(self.length) + ) + if s not in self.existing: + self.existing.add(s) + return s + tries += 1 + if tries > self.max_tries: + raise MaxRandomStringTryReached diff --git a/src/tests/apps/sample_app_django/api/management/commands/show_total_spending_by_user.py b/src/tests/apps/sample_app_django/api/management/commands/show_total_spending_by_user.py new file mode 100644 index 0000000..75f9706 --- /dev/null +++ b/src/tests/apps/sample_app_django/api/management/commands/show_total_spending_by_user.py @@ -0,0 +1,72 @@ +import asyncio +import json +import logging +from pathlib import Path + +from django.conf import settings +from django.core.management.base import BaseCommand + +from ...models import User +from ...tasks import get_spending_by_user_celery, get_spending_by_user_aiotaskq +from ...utils import Timer, chunker + +logger = logging.getLogger(__name__) +timer = Timer(logger, logging.DEBUG) + + +SpendingByUser = dict[str, float] + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument("--celery", action="store_true") + parser.add_argument("--aiotaskq", action="store_true") + + def handle(self, *args, **options): + assert options["celery"] and not options["aiotaskq"] or not options["celery"] or options["aiotaskq"] + assert not (options["celery"] and options["aiotaskq"]) + + r: SpendingByUser + if options["celery"]: + with timer("CMD show_total_spending_by_user"): + r = get_total_spending_by_user_celery() + else: + with timer("CMD show_total_spending_by_user"): + loop = asyncio.get_event_loop() + r = loop.run_until_complete(get_total_spending_by_user_aiotaskq()) + + suffix = "celery" if options["celery"] else "aiotaskq" + with (Path(settings.BASE_DIR) / f"out-{suffix}.json").open(mode="w") as fo: + logger.debug("Writing output to %s ...", fo.name) + print(json.dumps(r, sort_keys=True, indent=2), file=fo) + + +def get_total_spending_by_user_celery() -> SpendingByUser: + logger.debug("get_total_spending_by_user_celery") + user_ids = list(User.objects.all().values_list("id", flat=True)) + + async_results = [] + for user_ids_chunk in chunker(user_ids): + async_result = get_spending_by_user_celery.si(user_ids=user_ids_chunk).apply_async() + async_results.append(async_result) + + ret: SpendingByUser = {} + for async_result in async_results: + ret.update(async_result.get()) + return ret + + +async def get_total_spending_by_user_aiotaskq() -> SpendingByUser: + logger.debug("get_total_spending_by_user_aiotaskq") + user_ids = [u async for u in User.objects.all().values_list("id", flat=True)] + + tasks = [] + for user_ids_chunk in chunker(user_ids): + task = get_spending_by_user_aiotaskq.apply_async(user_ids=user_ids_chunk) + tasks.append(task) + + ret: SpendingByUser = {} + results: list[SpendingByUser] = await asyncio.gather(*tasks) + for result in results: + ret.update(result) + return ret diff --git a/src/tests/apps/sample_app_django/api/migrations/0001_initial.py b/src/tests/apps/sample_app_django/api/migrations/0001_initial.py new file mode 100644 index 0000000..ba4412a --- /dev/null +++ b/src/tests/apps/sample_app_django/api/migrations/0001_initial.py @@ -0,0 +1,33 @@ +# Generated by Django 5.0.7 on 2024-08-25 18:35 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="User", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("username", models.CharField(db_index=True, max_length=100, unique=True)), + ], + ), + migrations.CreateModel( + name="Order", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("name", models.CharField(blank=True, max_length=30, null=True)), + ("price", models.DecimalField(decimal_places=2, max_digits=10)), + ( + "user", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="api.user"), + ), + ], + ), + ] diff --git a/src/tests/apps/sample_app_django/api/migrations/__init__.py b/src/tests/apps/sample_app_django/api/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tests/apps/sample_app_django/api/models.py b/src/tests/apps/sample_app_django/api/models.py new file mode 100644 index 0000000..a021501 --- /dev/null +++ b/src/tests/apps/sample_app_django/api/models.py @@ -0,0 +1,21 @@ +from decimal import Decimal +from enum import unique +from django.db import models + + +class User(models.Model): + id: str = models.AutoField(primary_key=True) + username: str = models.CharField( + null=False, + blank=False, + db_index=True, + unique=True, + max_length=100, + ) + + +class Order(models.Model): + id: str = models.BigAutoField(primary_key=True) + user: "User" = models.ForeignKey("api.User", null=False, on_delete=models.CASCADE) + name: str = models.CharField(null=True, blank=True, max_length=30) + price: Decimal = models.DecimalField(null=False, decimal_places=2, max_digits=10) diff --git a/src/tests/apps/sample_app_django/api/tasks.py b/src/tests/apps/sample_app_django/api/tasks.py new file mode 100644 index 0000000..5c7bae4 --- /dev/null +++ b/src/tests/apps/sample_app_django/api/tasks.py @@ -0,0 +1,43 @@ +import logging +from threading import get_native_id + +from aiotaskq.task import task +from celery import shared_task +from django.db.models import Sum + +from .models import Order +from .utils import Timer + +logger = logging.getLogger(__name__) +timer = Timer(logger) + + +SpendingByUser = dict[str, float] + + +@shared_task() +def get_spending_by_user_celery(user_ids: list[int]) -> dict[str, float]: + with timer(f"[{get_native_id()}][get_spending_by_user_celery] Calculate spending for {len(user_ids)} users"): + qs = ( + Order.objects + .filter(user_id__in=user_ids) + .values("user__username") + .order_by("user__username") + .annotate(total_spending=Sum("price")) + ) + ret = {v["user__username"]: float(v["total_spending"]) for v in qs} + return ret + + +@task() +async def get_spending_by_user_aiotaskq(user_ids: list[int]) -> dict[str, float]: + with timer(f"[{get_native_id()}][get_spending_by_user_aiotaskq] Calculate spending for {len(user_ids)} users"): + qs = ( + Order.objects + .filter(user_id__in=user_ids) + .values("user__username") + .order_by("user__username") + .annotate(total_spending=Sum("price")) + ) + ret = {v["user__username"]: float(v["total_spending"]) async for v in qs} + return ret diff --git a/src/tests/apps/sample_app_django/api/utils.py b/src/tests/apps/sample_app_django/api/utils.py new file mode 100644 index 0000000..5fbd86e --- /dev/null +++ b/src/tests/apps/sample_app_django/api/utils.py @@ -0,0 +1,38 @@ +import logging +from collections.abc import Iterator +from logging import Logger +from time import perf_counter +from typing import TypeVar + +T = TypeVar("T") + + +def chunker(seq: list[T], size: int = 100) -> Iterator[list[T]]: + chunk: list[T] = [] + for t in seq: + if len(chunk) == size: + yield chunk + chunk = [] + else: + chunk.append(t) + if chunk: + yield chunk + + +class Timer: + def __init__(self, logger: Logger, level: int = logging.DEBUG): + self._logger = logger + self._level = level + + def __call__(self, name: str) -> "Timer": + self._name = name + return self + + def __enter__(self): + self._logger.log(self._level, "%s ...", self._name) + self.t0 = perf_counter() + + def __exit__(self, *args): + self.t1 = perf_counter() + duration = self.t1 - self.t0 + self._logger.log(self._level, "%s took %s seconds", self._name, duration) diff --git a/src/tests/apps/sample_app_django/manage.py b/src/tests/apps/sample_app_django/manage.py new file mode 100755 index 0000000..9eb0528 --- /dev/null +++ b/src/tests/apps/sample_app_django/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sample_app_django.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/src/tests/apps/sample_app_django/sample_app_django/README.md b/src/tests/apps/sample_app_django/sample_app_django/README.md new file mode 100644 index 0000000..b89d161 --- /dev/null +++ b/src/tests/apps/sample_app_django/sample_app_django/README.md @@ -0,0 +1,18 @@ +# Simple Django App + +This simple Django App uses both Celery & AiotaskQ to run the same logic inside workers, so you can +compare side by side the usage of these two libraries in this sample code and you can see that they are very similar in terms of usage. + +You can also see that the difference in performance between these two. Run these two scripts one after the other: + +0. [populate_db.sh](/src/tests/apps/sample_app_django/scripts/populate_db.sh) +1. [test_show_total_spending_celery.sh](/src/tests/apps/sample_app_django/scripts/test_show_total_spending_celery.sh) +2. [test_show_total_spending_aiotaskq.sh](/src/tests/apps/sample_app_django/scripts/test_show_total_spending_aiotaskq.sh) + +As tested on my local machine, I got this results: +```bash + +``` + +Check out the sample code within this folder and see for yourself. Also, try to run it and see the performance +comparison for yourself. diff --git a/src/tests/apps/sample_app_django/sample_app_django/__init__.py b/src/tests/apps/sample_app_django/sample_app_django/__init__.py new file mode 100644 index 0000000..f90975a --- /dev/null +++ b/src/tests/apps/sample_app_django/sample_app_django/__init__.py @@ -0,0 +1,6 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app +from .aiotaskq import app as aiotaskq_app + +__all__ = ('celery_app', "aiotaskq_app") diff --git a/src/tests/apps/sample_app_django/sample_app_django/aiotaskq.py b/src/tests/apps/sample_app_django/sample_app_django/aiotaskq.py new file mode 100644 index 0000000..b3a96d3 --- /dev/null +++ b/src/tests/apps/sample_app_django/sample_app_django/aiotaskq.py @@ -0,0 +1,11 @@ +import os + +from aiotaskq import App + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sample_app_django.settings') + + +app = App() + +app.autodiscover_tasks() diff --git a/src/tests/apps/sample_app_django/sample_app_django/asgi.py b/src/tests/apps/sample_app_django/sample_app_django/asgi.py new file mode 100644 index 0000000..4a82435 --- /dev/null +++ b/src/tests/apps/sample_app_django/sample_app_django/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for sample_app_django project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sample_app_django.settings") + +application = get_asgi_application() diff --git a/src/tests/apps/sample_app_django/sample_app_django/celery.py b/src/tests/apps/sample_app_django/sample_app_django/celery.py new file mode 100644 index 0000000..2f2e6d5 --- /dev/null +++ b/src/tests/apps/sample_app_django/sample_app_django/celery.py @@ -0,0 +1,17 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sample_app_django.settings') + +app = Celery('sample_app_django') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() diff --git a/src/tests/apps/sample_app_django/sample_app_django/settings.py b/src/tests/apps/sample_app_django/sample_app_django/settings.py new file mode 100644 index 0000000..8ac096a --- /dev/null +++ b/src/tests/apps/sample_app_django/sample_app_django/settings.py @@ -0,0 +1,166 @@ +""" +Django settings for sample_app_django project. + +Generated by 'django-admin startproject' using Django 5.0.7. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/5.0/ref/settings/ +""" + +import sys + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-7l)c+qvih88naf^6v&5iihtu-h5qm!%40@_l4n24wx+hi#t##_" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "api", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "sample_app_django.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "sample_app_django.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/5.0/ref/settings/#databases + +DATABASES = { + "default": { + # "ENGINE": "django.db.backends.sqlite3", + # "NAME": BASE_DIR / "db.sqlite3", + "ENGINE": "django.db.backends.postgresql", + "NAME": "sample_app_django", + "USER": "postgres", + "PASSWORD": "postgres", + "HOST": "127.0.0.1", + "PORT": "5432", + } +} + + +# Password validation +# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/5.0/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/5.0/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +REDIS_URL = "redis://127.0.0.1:6379/0" +CELERY_BROKER_URL = REDIS_URL +CELERY_RESULT_BACKEND = REDIS_URL + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "%(asctime)s %(levelname)-8s %(name)-12s %(message)s", + }, + }, + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "stream": sys.stdout, + "formatter": "verbose", + }, + }, + "loggers": { + "api": { + "handlers": ["console"], + "level": "DEBUG", + "propagate": True, + }, + "aiotaskq": { + "handlers": ["console"], + "level": "INFO", + "propagate": True, + } + } +} diff --git a/src/tests/apps/sample_app_django/sample_app_django/urls.py b/src/tests/apps/sample_app_django/sample_app_django/urls.py new file mode 100644 index 0000000..7f964b3 --- /dev/null +++ b/src/tests/apps/sample_app_django/sample_app_django/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for sample_app_django project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/5.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path("admin/", admin.site.urls), +] diff --git a/src/tests/apps/sample_app_django/sample_app_django/wsgi.py b/src/tests/apps/sample_app_django/sample_app_django/wsgi.py new file mode 100644 index 0000000..abe1024 --- /dev/null +++ b/src/tests/apps/sample_app_django/sample_app_django/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for sample_app_django project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sample_app_django.settings") + +application = get_wsgi_application() diff --git a/src/tests/apps/sample_app_django/scripts/populate_db.sh b/src/tests/apps/sample_app_django/scripts/populate_db.sh new file mode 100755 index 0000000..f5dc49d --- /dev/null +++ b/src/tests/apps/sample_app_django/scripts/populate_db.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +cd "$(dirname "$0")"/../ + +echo "Populating the db ..." +coverage run --append ./manage.py populate_db \ + --n-users 1000 \ + --n-orders-per-user 100 \ + --from-scratch diff --git a/src/tests/apps/sample_app_django/scripts/test_show_total_spending_aiotaskq.sh b/src/tests/apps/sample_app_django/scripts/test_show_total_spending_aiotaskq.sh new file mode 100755 index 0000000..5221287 --- /dev/null +++ b/src/tests/apps/sample_app_django/scripts/test_show_total_spending_aiotaskq.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +cd "$(dirname "$0")"/../ + +AIOTASKQ_LOG_LEVEL=INFO coverage run --append -m aiotaskq \ + worker sample_app_django\ + --concurrency 4 & +WPID=$! + +trap "kill -TERM $WPID" SIGINT SIGTERM EXIT + +coverage run --append ./manage.py show_total_spending_by_user --aiotaskq diff --git a/src/tests/apps/sample_app_django/scripts/test_show_total_spending_celery.sh b/src/tests/apps/sample_app_django/scripts/test_show_total_spending_celery.sh new file mode 100755 index 0000000..08313b6 --- /dev/null +++ b/src/tests/apps/sample_app_django/scripts/test_show_total_spending_celery.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +cd "$(dirname "$0")"/../ + +coverage run --append -m celery -A sample_app_django worker -c 4 & +WPID=$! + +trap "kill -TERM $WPID" SIGINT SIGTERM EXIT + +coverage run --append ./manage.py show_total_spending_by_user --celery +echo "WPID=$WPID" diff --git a/test.sh b/test.sh index b703925..2287514 100755 --- a/test.sh +++ b/test.sh @@ -13,4 +13,14 @@ failed=$? coverage combine --quiet +echo "Running tests against sample codes ..." + +./src/tests/apps/sample_app_django/scripts/populate_db.sh +./src/tests/apps/sample_app_django/scripts/test_show_total_spending_celery.sh +./src/tests/apps/sample_app_django/scripts/test_show_total_spending_aiotaskq.sh +diff \ + ./src/tests/apps/sample_app_django/out-celery.json \ + ./src/tests/apps/sample_app_django/out-aiotaskq.json +failed=$? + exit $failed