From f09838196467b27c720b3f86c830e2e88b2db2a8 Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 13 Aug 2024 09:35:14 +0200 Subject: [PATCH] Improved testclient & databasez support >= 0.9 (#167) - improve testclient - fix tests - support databasez >= 0.9 * bump databasez requirement --- docs/test-client.md | 22 ++++++-- pyproject.toml | 2 +- saffier/core/db/models/model.py | 5 +- saffier/core/db/querysets/base.py | 4 +- saffier/core/utils/sync.py | 4 ++ saffier/testclient.py | 56 ++++++++++++++++++++- scripts/test | 1 + tests/cli/test_custom_template.py | 16 ++++++ tests/cli/test_custom_template_with_flag.py | 16 ++++++ tests/settings.py | 2 +- 10 files changed, 117 insertions(+), 11 deletions(-) diff --git a/docs/test-client.md b/docs/test-client.md index 52b6cbe..c2f1a3b 100644 --- a/docs/test-client.md +++ b/docs/test-client.md @@ -37,6 +37,11 @@ that rollbacks once the database is disconnected. Default: `False` +* **lazy_setup** - This sets up the db first up on connect not in init. + + Default: `True` + + * **use_existing** - Uses the existing `test_` database if previously created and not dropped. Default: `False` @@ -45,6 +50,18 @@ that rollbacks once the database is disconnected. Default: `False` +* **test_prefix** - Allow a custom test prefix or leave empty to use the url instead without changes. + + Default: `testclient_default_test_prefix` (defaults to `test_`) + +### Configuration via Environment + +Most parameters defaults can be changed via capitalized environment names with `SAFFIER_TESTCLIENT_`. + +E.g. `SAFFIER_TESTCLIENT_DEFAULT_PREFIX=foobar` or `SAFFIER_TESTCLIENT_FORCE_ROLLBACK=true`. + +This is used for the tests. + ### How to use it This is the easiest part because is already very familiar with the `Database` used by Saffier. In @@ -56,7 +73,7 @@ Let us assume you have a database url like this following: DATABASE_URL = "postgresql+asyncpg://postgres:postgres@localhost:5432/my_db" ``` -We know the database is called `my_db`, right? +We know the database is called `my_db`, right? When using the `DatabaseTestClient`, the client will ensure the tests will land on a `test_my_db`. @@ -74,8 +91,7 @@ Well, this is rather complex test and actually a real one from Saffier and what that is using the `DatabaseTestClient` which means the tests against models, fields or whatever database operation you want will be on a `test_` database. -But you can see a `drop_database=True`, so what is that? +But you can see a `drop_database=True`, so what is that? Well `drop_database=True` means that by the end of the tests finish running, drops the database into oblivion. - diff --git a/pyproject.toml b/pyproject.toml index 42b6760..c1d4d3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "click>=8.1.3,<9.0.0", "dymmond-settings>=1.0.4", "loguru>=0.6.0,<0.10.0", - "databasez>=0.8.5", + "databasez>=0.9.2", "orjson >=3.8.5,<4.0.0", "pydantic>=2.5.3,<3.0.0", "rich>=13.3.1,<14.0.0", diff --git a/saffier/core/db/models/model.py b/saffier/core/db/models/model.py index 65c18ab..71945f2 100644 --- a/saffier/core/db/models/model.py +++ b/saffier/core/db/models/model.py @@ -118,10 +118,9 @@ async def _save(self, **kwargs: typing.Any) -> "Model": autoincrement_value = await self.database.execute(expression) # sqlalchemy supports only one autoincrement column if autoincrement_value: - if isinstance(autoincrement_value, Row): - assert len(autoincrement_value) == 1 - autoincrement_value = autoincrement_value[0] column = self.table.autoincrement_column + if column is not None and isinstance(autoincrement_value, Row): + autoincrement_value = autoincrement_value._mapping[column.name] # can be explicit set, which causes an invalid value returned if column is not None and column.key not in kwargs: saffier_setattr(self, column.key, autoincrement_value) diff --git a/saffier/core/db/querysets/base.py b/saffier/core/db/querysets/base.py index 4827baa..98555b6 100644 --- a/saffier/core/db/querysets/base.py +++ b/saffier/core/db/querysets/base.py @@ -936,7 +936,7 @@ async def bulk_create(self, objs: List[Dict]) -> None: expression = queryset.table.insert().values(new_objs) queryset._set_query_expression(expression) - await queryset.database.execute(expression) + await queryset.database.execute_many(expression) async def bulk_update(self, objs: List[SaffierModel], fields: List[str]) -> None: """ @@ -987,7 +987,7 @@ async def bulk_update(self, objs: List[SaffierModel], fields: List[str]) -> None expression = expression.values(kwargs) queryset._set_query_expression(expression) - await queryset.database.execute(expression, query_list) + await queryset.database.execute_many(expression, query_list) async def delete(self) -> None: queryset: "QuerySet" = self._clone() diff --git a/saffier/core/utils/sync.py b/saffier/core/utils/sync.py index 1e09bd2..a6ed23b 100644 --- a/saffier/core/utils/sync.py +++ b/saffier/core/utils/sync.py @@ -3,6 +3,10 @@ from concurrent.futures import Future from typing import Any, Awaitable +import nest_asyncio + +nest_asyncio.apply() + def run_sync(async_function: Awaitable) -> Any: """ diff --git a/saffier/testclient.py b/saffier/testclient.py index f70df06..8d45a51 100644 --- a/saffier/testclient.py +++ b/saffier/testclient.py @@ -1 +1,55 @@ -from databasez.testclient import DatabaseTestClient as DatabaseTestClient # noqa +import os +import typing +from typing import TYPE_CHECKING, Any + +from databasez.testclient import DatabaseTestClient as _DatabaseTestClient + +if TYPE_CHECKING: + import sqlalchemy + from databasez import Database, DatabaseURL + +# TODO: move this to the settings +default_test_prefix: str = "test_" +# for allowing empty +if "SAFFIER_TESTCLIENT_TEST_PREFIX" in os.environ: + default_test_prefix = os.environ["SAFFIER_TESTCLIENT_TEST_PREFIX"] + +default_use_existing: bool = ( + os.environ.get("SAFFIER_TESTCLIENT_USE_EXISTING") or "" +).lower() == "true" +default_drop_database: bool = ( + os.environ.get("SAFFIER_TESTCLIENT_DROP_DATABASE") or "" +).lower() == "true" + + +class DatabaseTestClient(_DatabaseTestClient): + """ + Adaption of DatabaseTestClient for saffier. + + Note: the default of lazy_setup is True here. This enables the simple Registry syntax. + """ + + testclient_lazy_setup: bool = ( + os.environ.get("SAFFIER_TESTCLIENT_LAZY_SETUP", "true") or "" + ).lower() == "true" + testclient_force_rollback: bool = ( + os.environ.get("SAFFIER_TESTCLIENT_FORCE_ROLLBACK") or "" + ).lower() == "true" + + # TODO: replace by testclient default overwrites + def __init__( + self, + url: typing.Union[str, "DatabaseURL", "sqlalchemy.URL", "Database"], + *, + use_existing: bool = default_use_existing, + drop_database: bool = default_drop_database, + test_prefix: str = default_test_prefix, + **options: Any, + ): + super().__init__( + url, + use_existing=use_existing, + drop_database=drop_database, + test_prefix=test_prefix, + **options, + ) diff --git a/scripts/test b/scripts/test index 4868749..de6b057 100755 --- a/scripts/test +++ b/scripts/test @@ -6,6 +6,7 @@ if [ "$VIRTUAL_ENV" != '' ]; then elif [ -d 'venv' ] ; then export PREFIX="venv/bin/" fi +export SAFFIER_TESTCLIENT_TEST_PREFIX="" set -ex diff --git a/tests/cli/test_custom_template.py b/tests/cli/test_custom_template.py index e3eb176..7048ffc 100644 --- a/tests/cli/test_custom_template.py +++ b/tests/cli/test_custom_template.py @@ -1,10 +1,14 @@ +import asyncio import os import shutil import pytest +import sqlalchemy from esmerald import Esmerald +from sqlalchemy.ext.asyncio import create_async_engine from tests.cli.utils import run_cmd +from tests.settings import DATABASE_URL app = Esmerald(routes=[]) @@ -50,7 +54,19 @@ def test_alembic_version(): assert isinstance(v, int) +async def cleanup_prepare_db(): + engine = create_async_engine(DATABASE_URL, isolation_level="AUTOCOMMIT") + try: + async with engine.connect() as conn: + await conn.execute(sqlalchemy.text("DROP DATABASE test_saffier")) + except Exception: + pass + async with engine.connect() as conn: + await conn.execute(sqlalchemy.text("CREATE DATABASE test_saffier")) + + def test_migrate_upgrade(create_folders): + asyncio.run(cleanup_prepare_db()) (o, e, ss) = run_cmd("tests.cli.main:app", "saffier init -t ./custom") assert ss == 0 diff --git a/tests/cli/test_custom_template_with_flag.py b/tests/cli/test_custom_template_with_flag.py index 88841e5..c82c235 100644 --- a/tests/cli/test_custom_template_with_flag.py +++ b/tests/cli/test_custom_template_with_flag.py @@ -1,10 +1,14 @@ +import asyncio import os import shutil import pytest +import sqlalchemy from esmerald import Esmerald +from sqlalchemy.ext.asyncio import create_async_engine from tests.cli.utils import run_cmd +from tests.settings import DATABASE_URL app = Esmerald(routes=[]) @@ -50,7 +54,19 @@ def test_alembic_version(): assert isinstance(v, int) +async def cleanup_prepare_db(): + engine = create_async_engine(DATABASE_URL, isolation_level="AUTOCOMMIT") + try: + async with engine.connect() as conn: + await conn.execute(sqlalchemy.text("DROP DATABASE test_saffier")) + except Exception: + pass + async with engine.connect() as conn: + await conn.execute(sqlalchemy.text("CREATE DATABASE test_saffier")) + + def test_migrate_upgrade_with_app_flag(create_folders): + asyncio.run(cleanup_prepare_db()) (o, e, ss) = run_cmd( "tests.cli.main:app", "saffier --app tests.cli.main:app init -t ./custom", is_app=False ) diff --git a/tests/settings.py b/tests/settings.py index fb26647..8f3f7c3 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -8,7 +8,7 @@ ) DATABASE_ALTERNATIVE_URL = os.environ.get( "TEST_DATABASE_ALTERNATIVE_URL", - "postgresql+asyncpg://postgres:postgres@localhost:5433/edgy_alt", + "postgresql+asyncpg://postgres:postgres@localhost:5433/saffier_alt", ) TEST_DATABASE = "postgresql+asyncpg://postgres:postgres@localhost:5432/test_saffier"