From 62a3f4dbb49df1fd5f0f751d764fcfd5e5d2f651 Mon Sep 17 00:00:00 2001 From: Alireza Jafari Date: Sat, 28 Sep 2024 00:11:20 +0330 Subject: [PATCH] feat!: Original media url cache added BREAKING CHANGE: Redis dependency --- api/.env.example | 4 +- api/micro_media/main.py | 27 ++++++++--- api/micro_media/routers/v1/public/media.py | 9 +++- api/micro_media/settings/base.py | 3 ++ api/micro_media/utils/cache.py | 55 ++++++++++++++++++++++ api/poetry.lock | 42 ++++++++++++++++- api/prod.env.example | 2 + api/pyproject.toml | 2 + docker-compose.yml | 8 +++- 9 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 api/micro_media/utils/cache.py diff --git a/api/.env.example b/api/.env.example index 6217244..eecf0f9 100644 --- a/api/.env.example +++ b/api/.env.example @@ -4,7 +4,7 @@ APP_NAME=MicroMedia JWT_DECODE_ALGORITHMS=HS256,RS256 JWT_DECODE_KEY=SOME_KEY -#JWT_DECODE_KEY=file: jwt_pub.pem +# JWT_DECODE_KEY=file: jwt_pub.pem JWT_AUDIENCE= JWT_ISSUER= @@ -22,3 +22,5 @@ IMGPROXY_HOST=http://localhost:8080 IMGPROXY_KEY=SOME_HEX_KEY IMGPROXY_SALT=SOME_HEX_SALT IMGPROXY_RESIZE_ENLARGE=True + +REDIS_URL=redis://localhost:6379/0 diff --git a/api/micro_media/main.py b/api/micro_media/main.py index 72e178f..934eff3 100644 --- a/api/micro_media/main.py +++ b/api/micro_media/main.py @@ -1,7 +1,9 @@ +from contextlib import asynccontextmanager + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from starlette.middleware.authentication import AuthenticationMiddleware - +from redis.asyncio.client import Redis from auth_utils import APIKeyAuthBackend, AuthBackendsWrapper, JWTAuthBackend from fastapi_pagination import add_pagination @@ -9,6 +11,7 @@ from micro_media.routers import router as base_router from micro_media.auth import validate_api_keys, get_api_key_user from micro_media.exception_handlers import register_exception_handlers +from micro_media.utils.cache import AsyncRedisCache from micro_media.settings import ( DEBUG, APP_NAME, @@ -17,13 +20,30 @@ JWT_AUDIENCE, JWT_ISSUER, CORS_ALLOWED_ORIGINS, + REDIS_URL, + REDIS_PREFIX, ) + +@asynccontextmanager +async def lifespan(app: FastAPI): + validate_api_keys() + + redis = Redis.from_url(REDIS_URL, decode_responses=True) + await redis.ping() + AsyncRedisCache.init(redis=redis, prefix=REDIS_PREFIX + "caches:") + + yield + + await redis.aclose() + + app = FastAPI( title=APP_NAME, docs_url="/docs", redoc_url="/redoc", debug=DEBUG, + lifespan=lifespan, ) app.include_router(base_router) @@ -52,8 +72,3 @@ add_pagination(app) register_exception_handlers(app=app) - - -@app.on_event("startup") -async def startup(): - validate_api_keys() diff --git a/api/micro_media/routers/v1/public/media.py b/api/micro_media/routers/v1/public/media.py index 40f92e7..93d6960 100644 --- a/api/micro_media/routers/v1/public/media.py +++ b/api/micro_media/routers/v1/public/media.py @@ -8,6 +8,7 @@ from micro_media.utils import truthy_or_404 from micro_media.utils.sqlalchemy import get_one +from micro_media.utils.cache import AsyncRedisCache from micro_media.models import get_session, Media, MediaType from micro_media.storage import STORAGE_CONTEXT as SC from micro_media.media import MEDIA_CONTEXT as MC, IMGProxyThumbnailManager @@ -22,7 +23,13 @@ ) -async def _get_original_link(media: Media, expires_in: int): +@AsyncRedisCache.aredis_cache( + key_generator=lambda media, expires_in: f"original_link:{media.id}", + cache_deserializer=str, + cache_serializer=str, + ttl=60, +) +async def _get_original_link(media: Media, expires_in: int) -> str: storage_manager = SC.get_manager(storage_id=media.storage_id) file_link = await storage_manager.generate_file_link( diff --git a/api/micro_media/settings/base.py b/api/micro_media/settings/base.py index 1590f37..7feeef0 100644 --- a/api/micro_media/settings/base.py +++ b/api/micro_media/settings/base.py @@ -38,3 +38,6 @@ IMGPROXY_RESIZE_ENLARGE = cast( bool, config("IMGPROXY_RESIZE_ENLARGE", default=True) ) + +REDIS_URL = cast(str, config("REDIS_URL")) +REDIS_PREFIX = cast(str, config("REDIS_PREFIX", default="micro_media:")) diff --git a/api/micro_media/utils/cache.py b/api/micro_media/utils/cache.py new file mode 100644 index 0000000..e35f3f4 --- /dev/null +++ b/api/micro_media/utils/cache.py @@ -0,0 +1,55 @@ +import asyncio +from functools import wraps +from typing import Awaitable, Callable, TypeVar +from typing_extensions import ParamSpec + +from redis.asyncio.client import Redis + + +P = ParamSpec("P") +T = TypeVar("T") + + +class AsyncRedisCache: + prefix: str + redis: Redis + + @classmethod + def init(cls, redis: Redis, prefix: str) -> None: + cls.redis = redis + cls.prefix = prefix + + @classmethod + def aredis_cache( + cls, + key_generator: Callable[P, str], + cache_serializer: Callable[[T], str], + cache_deserializer: Callable[[str], T], + ttl: int, + ) -> Callable[[Callable[P, Awaitable[T]]], Callable[P, Awaitable[T]]]: + def decorator( + func: Callable[P, Awaitable[T]] + ) -> Callable[P, Awaitable[T]]: + lock = asyncio.Lock() + + @wraps(func) + async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + cache_key = cls.prefix + key_generator(*args, **kwargs) + + async with lock: + cache_value = await cls.redis.get(cache_key) + if cache_value is not None: + if isinstance(cache_value, bytes): + cache_value = cache_value.decode() + + return cache_deserializer(cache_value) + + res = await func(*args, **kwargs) + await cls.redis.setex( + cache_key, ttl, cache_serializer(res) + ) + return res + + return wrapper + + return decorator diff --git a/api/poetry.lock b/api/poetry.lock index 0f86507..5cc3f6c 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -257,6 +257,17 @@ doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.21.0b1)"] trio = ["trio (>=0.26.1)"] +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + [[package]] name = "asyncpg" version = "0.28.0" @@ -384,6 +395,17 @@ types-awscrt = "*" [package.extras] botocore = ["botocore"] +[[package]] +name = "cachetools" +version = "5.5.0" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, + {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, +] + [[package]] name = "cffi" version = "1.17.1" @@ -1385,6 +1407,24 @@ files = [ {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] +[[package]] +name = "redis" +version = "5.0.8" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.7" +files = [ + {file = "redis-5.0.8-py3-none-any.whl", hash = "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4"}, + {file = "redis-5.0.8.tar.gz", hash = "sha256:0c5b10d387568dfe0698c6fad6615750c24170e548ca2deac10c649d463e9870"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} + +[package.extras] +hiredis = ["hiredis (>1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + [[package]] name = "s3transfer" version = "0.6.2" @@ -2670,4 +2710,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "5dcc0c68dce26e387e95fef2e74df9aa8ae244094bd6f44e49c9052e915eb039" +content-hash = "e1f1050af91fd633c5b644d3e3037be3330daf145e9dc1ef37abdcf227d357ce" diff --git a/api/prod.env.example b/api/prod.env.example index cb13b2e..7226d6b 100644 --- a/api/prod.env.example +++ b/api/prod.env.example @@ -28,3 +28,5 @@ IMGPROXY_HOST=https://imgproxy.blah.blah IMGPROXY_KEY=SOME_HEX_KEY IMGPROXY_SALT=SOME_HEX_SALT IMGPROXY_RESIZE_ENLARGE=True + +REDIS_URL=redis://redis:6379/0 diff --git a/api/pyproject.toml b/api/pyproject.toml index 0538f7a..cc2364b 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -23,6 +23,8 @@ python-multipart = "^0.0.7" pyjwt = { extras = ["crypto"], version = "^2.8.0" } pillow = "^10.2.0" fastapi-auth-utils = "^1.1.2" +cachetools = "^5.5.0" +redis = "^5.0.8" [tool.poetry.group.dev.dependencies] diff --git a/docker-compose.yml b/docker-compose.yml index 6cf3538..26da315 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: api: restart: always @@ -41,6 +39,12 @@ services: env_file: - ./imgproxy/prod.env + redis: + image: redis:7.2-alpine + restart: always + networks: + - default + networks: default: driver: bridge