From ef6efff35e05964e622c39867af4247acf4d03d0 Mon Sep 17 00:00:00 2001 From: Andrii Andreiev Date: Thu, 23 Jan 2025 16:23:35 +0200 Subject: [PATCH 1/3] feat(python): FastAPI support --- Makefile | 5 ++ docker-compose.yml | 10 +++ packages/python/Makefile | 3 + packages/python/examples/fastapi/app.py | 48 ++++++++++ .../python/examples/fastapi/requirements.txt | 14 +++ .../python/readme_metrics/PayloadBuilder.py | 14 +-- packages/python/readme_metrics/fastapi.py | 90 +++++++++++++++++++ .../readme_metrics/tests/fastapi_test.py | 86 ++++++++++++++++++ packages/python/requirements.dev.txt | 3 +- packages/python/requirements.txt | 3 +- test/integrations/python/fastapi.Dockerfile | 13 +++ 11 files changed, 280 insertions(+), 9 deletions(-) create mode 100644 packages/python/examples/fastapi/app.py create mode 100644 packages/python/examples/fastapi/requirements.txt create mode 100644 packages/python/readme_metrics/fastapi.py create mode 100644 packages/python/readme_metrics/tests/fastapi_test.py create mode 100644 test/integrations/python/fastapi.Dockerfile diff --git a/Makefile b/Makefile index 1963896a7f..97289ce55d 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,11 @@ test-webhooks-python-flask: ## Run webhooks tests against the Python SDK + Flask SUPPORTS_HASHING=true npm run test:integration-webhooks || make cleanup-failure @make cleanup +test-metrics-python-fastapi: ## Run Metrics tests against the Python SDK + FastAPI + docker compose up --build --detach integration_python_fastapi_metrics + SUPPORTS_HASHING=true npm run test:integration-metrics || make cleanup-failure + @make cleanup + ## ## Ruby ## diff --git a/docker-compose.yml b/docker-compose.yml index 3c1063e48a..5e5b7b8667 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -126,6 +126,16 @@ services: environment: <<: *server-config + integration_python_fastapi_metrics: + build: + context: . + dockerfile: ./test/integrations/python/fastapi.Dockerfile + ports: + - 8000:8000 + extra_hosts: *default-extra_hosts + environment: + <<: *server-config + # # Ruby # diff --git a/packages/python/Makefile b/packages/python/Makefile index f2b0a92c0c..c3cb0fe310 100644 --- a/packages/python/Makefile +++ b/packages/python/Makefile @@ -24,6 +24,9 @@ serve-metrics-flask: ## Start the local Flask server to test Metrics serve-webhooks-flask: ## Start the local Flask server to test webhooks README_API_KEY=$(API_KEY) python3 examples/flask/webhooks.py +serve-metrics-fastapi: + cd examples/fastapi && README_API_KEY=$(API_KEY) uvicorn app:app --reload + test: ## Run unit tests pytest diff --git a/packages/python/examples/fastapi/app.py b/packages/python/examples/fastapi/app.py new file mode 100644 index 0000000000..9a68ccef2e --- /dev/null +++ b/packages/python/examples/fastapi/app.py @@ -0,0 +1,48 @@ +import os +import sys + +from fastapi import FastAPI +from readme_metrics import MetricsApiConfig +from readme_metrics.fastapi import ReadMeMetricsMiddleware + + +if os.getenv("README_API_KEY") is None: + sys.stderr.write("Missing `README_API_KEY` environment variable") + sys.stderr.flush() + sys.exit(1) + +app = FastAPI() + + +# pylint: disable=W0613 +def grouping_function(request): + return { + # User's API Key + "api_key": "owlbert-api-key", + # Username to show in the dashboard + "label": "Owlbert", + # User's email address + "email": "owlbert@example.com", + } + + +config = MetricsApiConfig( + api_key=os.getenv("README_API_KEY"), + grouping_function=grouping_function, + background_worker_mode=False, + buffer_length=1, + timeout=5, +) + +# Add middleware with configuration using a lambda +app.add_middleware(ReadMeMetricsMiddleware, config=config) + + +@app.get("/") +def read_root(): + return {"message": "hello world"} + + +@app.post("/") +def post_root(): + return "13414" diff --git a/packages/python/examples/fastapi/requirements.txt b/packages/python/examples/fastapi/requirements.txt new file mode 100644 index 0000000000..0850455fdb --- /dev/null +++ b/packages/python/examples/fastapi/requirements.txt @@ -0,0 +1,14 @@ +annotated-types==0.7.0 +anyio==4.8.0 +click==8.1.8 +exceptiongroup==1.2.2 +fastapi==0.115.6 +h11==0.14.0 +idna==3.10 +pydantic==2.10.4 +pydantic_core==2.27.2 +../../ +sniffio==1.3.1 +starlette==0.41.3 +typing_extensions==4.12.2 +uvicorn==0.34.0 diff --git a/packages/python/readme_metrics/PayloadBuilder.py b/packages/python/readme_metrics/PayloadBuilder.py index e9cbb2be5c..9612b96e70 100644 --- a/packages/python/readme_metrics/PayloadBuilder.py +++ b/packages/python/readme_metrics/PayloadBuilder.py @@ -178,7 +178,6 @@ def _build_request_payload(self, request) -> dict: dict: Wrapped request payload """ headers = self.redact_dict(request.headers) - queryString = parse.parse_qsl(self._get_query_string(request)) content_type = self._get_content_type(headers) @@ -187,9 +186,9 @@ def _build_request_payload(self, request) -> dict: request, "rm_content_length", None ): if content_type == "application/x-www-form-urlencoded": - # Flask creates `request.form` but Django puts that data in `request.body`, and - # then our `request.rm_body` store, instead. - if hasattr(request, "form"): + # Flask creates `request.form` and does not have a `body` property but Django + # puts that data in `request.body`, and then our `request.rm_body` store, instead. + if hasattr(request, "form") and not hasattr(request, "body"): params = [ # Reason this is not mixed in with the `rm_body` parsing if we don't have # `request.form` is that if we attempt to do `str(var, 'utf-8)` on data @@ -219,8 +218,9 @@ def _build_request_payload(self, request) -> dict: headers = dict(headers) - if "Authorization" in headers: - headers["Authorization"] = mask(headers["Authorization"]) + for key in ["Authorization", "authorization"]: + if key in headers: + headers[key] = mask(headers[key]) if hasattr(request, "environ"): http_version = request.environ["SERVER_PROTOCOL"] @@ -324,7 +324,7 @@ def _build_base_url(self, request): query_string = self._get_query_string(request) if hasattr(request, "base_url"): # Werkzeug request objects already have exactly what we need - base_url = request.base_url + base_url = str(request.base_url) if len(query_string) > 0: base_url += f"?{query_string}" return base_url diff --git a/packages/python/readme_metrics/fastapi.py b/packages/python/readme_metrics/fastapi.py new file mode 100644 index 0000000000..6490f6e01d --- /dev/null +++ b/packages/python/readme_metrics/fastapi.py @@ -0,0 +1,90 @@ +from datetime import datetime +import time + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +from readme_metrics import MetricsApiConfig +from readme_metrics.Metrics import Metrics +from readme_metrics.ResponseInfoWrapper import ResponseInfoWrapper + + +class ReadMeMetricsMiddleware(BaseHTTPMiddleware): + def __init__(self, app, config: MetricsApiConfig): + super().__init__(app) + self.config = config + self.metrics_core = Metrics(config) + + async def _safe_retrieve_body(self, request): + # Safely retrieve the request body. + try: + body = await request.body() + return body + except Exception as e: + self.config.LOGGER.exception(e) + return None + + async def _read_response_body(self, response): + # Reads and decodes the response body. + try: + body_chunks = [] + async for chunk in response.body_iterator: + body_chunks.append(chunk) + response.body_iterator = iter(body_chunks) + encoded_body = b"".join(body_chunks) + + try: + return encoded_body.decode("utf-8") + except UnicodeDecodeError: + return "[NOT VALID UTF-8]" + except Exception as e: + self.config.LOGGER.exception(e) + return "" + + async def preamble(self, request): + # Initialize metrics-related attributes on the request object. + try: + request.rm_start_dt = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + request.rm_start_ts = int(time.time() * 1000) + + content_length = request.headers.get("Content-Length") + body = await self._safe_retrieve_body(request) + + if content_length or body: + request.rm_content_length = content_length or "0" + request.rm_body = body or "" + except Exception as e: + self.config.LOGGER.exception(e) + + async def dispatch(self, request: Request, call_next): + if request.method == "OPTIONS": + return await call_next(request) + + await self.preamble(request) + + response = None + response_body = None + try: + response = await call_next(request) + response_body = await self._read_response_body(response) + + response_info = ResponseInfoWrapper( + headers=response.headers, + status=response.status_code, + content_type=response.headers.get("Content-Type"), + content_length=response.headers.get("Content-Length"), + body=response_body, + ) + + self.metrics_core.process(request, response_info) + + except Exception as e: + self.config.LOGGER.exception(e) + + return Response( + content=response_body, + status_code=response.status_code, + headers=response.headers, + media_type=response.media_type, + ) diff --git a/packages/python/readme_metrics/tests/fastapi_test.py b/packages/python/readme_metrics/tests/fastapi_test.py new file mode 100644 index 0000000000..3c12e094e7 --- /dev/null +++ b/packages/python/readme_metrics/tests/fastapi_test.py @@ -0,0 +1,86 @@ +from datetime import datetime, timedelta +import time +from unittest.mock import Mock, AsyncMock +import pytest +from fastapi import Response + +from readme_metrics import MetricsApiConfig +from readme_metrics.ResponseInfoWrapper import ResponseInfoWrapper +from readme_metrics.fastapi import ReadMeMetricsMiddleware + +mock_config = MetricsApiConfig( + "README_API_KEY", + lambda req: {"id": "123", "label": "testuser", "email": "user@email.com"}, + buffer_length=1000, +) + + +class TestFastAPIMiddleware: + def setup_middleware(self, is_async=False): + app = AsyncMock if is_async else Mock() + + middleware = ReadMeMetricsMiddleware(app, config=mock_config) + middleware.metrics_core = Mock() + return middleware + + def validate_metrics(self, middleware, request): + assert hasattr(request, "rm_start_dt") + req_start_dt = datetime.strptime(request.rm_start_dt, "%Y-%m-%dT%H:%M:%SZ") + current_dt = datetime.utcnow() + assert abs(current_dt - req_start_dt) < timedelta(seconds=1) + + assert hasattr(request, "rm_start_ts") + req_start_millis = request.rm_start_ts + current_millis = time.time() * 1000.0 + assert abs(current_millis - req_start_millis) < 1000.00 + + middleware.metrics_core.process.assert_called_once() + call_args = middleware.metrics_core.process.call_args + assert len(call_args[0]) == 2 + assert call_args[0][0] == request + assert isinstance(call_args[0][1], ResponseInfoWrapper) + assert call_args[0][1].headers.get("x-header") == "X Value!" + assert ( + getattr(request, "rm_content_length") == request.headers["Content-Length"] + ) + + @pytest.mark.asyncio + async def test(self): + middleware = self.setup_middleware() + + request = Mock() + request.headers = {"Content-Length": "123"} + + call_next = AsyncMock() + call_next.return_value = Response(content="", headers={"X-Header": "X Value!"}) + + await middleware.dispatch(request, call_next) + + self.validate_metrics(middleware, request) + + @pytest.mark.asyncio + async def test_missing_content_length(self): + middleware = self.setup_middleware() + + request = AsyncMock() + request.headers = {} + + call_next = AsyncMock() + call_next.return_value = Response(content="") + + await middleware.dispatch(request, call_next) + + assert getattr(request, "rm_content_length") == "0" + + @pytest.mark.asyncio + async def test_options_request(self): + middleware = self.setup_middleware() + + request = AsyncMock() + request.method = "OPTIONS" + + call_next = AsyncMock() + + await middleware.dispatch(request, call_next) + + assert not middleware.metrics_core.process.called diff --git a/packages/python/requirements.dev.txt b/packages/python/requirements.dev.txt index 7b0252c6be..f0ea15c79e 100644 --- a/packages/python/requirements.dev.txt +++ b/packages/python/requirements.dev.txt @@ -2,4 +2,5 @@ black==24.4.2 pylint==3.2.6 pytest-mock==3.14.0 pytest==8.3.2 -requests-mock==1.12.1 \ No newline at end of file +pytest-asyncio==0.25.2 +requests-mock==1.12.1 diff --git a/packages/python/requirements.txt b/packages/python/requirements.txt index ec3ed1c504..af04d0d850 100644 --- a/packages/python/requirements.txt +++ b/packages/python/requirements.txt @@ -1,5 +1,6 @@ Django==4.2.16 Flask==3.0.3 +fastapi==0.115.6 requests==2.32.3 Werkzeug==3.0.6 -uvicorn==0.34.0 \ No newline at end of file +uvicorn==0.34.0 diff --git a/test/integrations/python/fastapi.Dockerfile b/test/integrations/python/fastapi.Dockerfile new file mode 100644 index 0000000000..3c19a286a2 --- /dev/null +++ b/test/integrations/python/fastapi.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.10 + +COPY packages/python /src + +# Set up the Python SDK +WORKDIR /src +RUN pip3 install --no-cache-dir -r requirements.txt + +# Install Flask +WORKDIR /src/examples/fastapi +RUN pip3 install --no-cache-dir -r requirements.txt + +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] From c050db0f0d7dbcbf64bae006ea37cde1007aa8d6 Mon Sep 17 00:00:00 2001 From: Andrii Andreiev Date: Thu, 23 Jan 2025 16:56:55 +0200 Subject: [PATCH 2/3] feat(python/fastapi): FastAPI webhooks --- Makefile | 5 +++ docker-compose.yml | 10 +++++ packages/python/Makefile | 5 ++- packages/python/examples/fastapi/app.py | 2 +- packages/python/examples/fastapi/webhooks.py | 45 +++++++++++++++++++ .../python/fastapi-webhooks.Dockerfile | 13 ++++++ 6 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 packages/python/examples/fastapi/webhooks.py create mode 100644 test/integrations/python/fastapi-webhooks.Dockerfile diff --git a/Makefile b/Makefile index 97289ce55d..9938e9fea1 100644 --- a/Makefile +++ b/Makefile @@ -91,6 +91,11 @@ test-metrics-python-fastapi: ## Run Metrics tests against the Python SDK + FastA SUPPORTS_HASHING=true npm run test:integration-metrics || make cleanup-failure @make cleanup +test-webhooks-python-fastapi: ## Run webhooks tests against the Python SDK + FastAPI + docker compose up --build --detach integration_python_fastapi_webhooks + SUPPORTS_HASHING=true npm run test:integration-webhooks || make cleanup-failure + @make cleanup + ## ## Ruby ## diff --git a/docker-compose.yml b/docker-compose.yml index 5e5b7b8667..d2763f1bd9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -136,6 +136,16 @@ services: environment: <<: *server-config + integration_python_fastapi_webhooks: + build: + context: . + dockerfile: ./test/integrations/python/fastapi-webhooks.Dockerfile + ports: + - 8000:8000 + extra_hosts: *default-extra_hosts + environment: + <<: *server-config + # # Ruby # diff --git a/packages/python/Makefile b/packages/python/Makefile index c3cb0fe310..8349dee10d 100644 --- a/packages/python/Makefile +++ b/packages/python/Makefile @@ -24,9 +24,12 @@ serve-metrics-flask: ## Start the local Flask server to test Metrics serve-webhooks-flask: ## Start the local Flask server to test webhooks README_API_KEY=$(API_KEY) python3 examples/flask/webhooks.py -serve-metrics-fastapi: +serve-metrics-fastapi: ## Start the local FastAPI server to test Metrics cd examples/fastapi && README_API_KEY=$(API_KEY) uvicorn app:app --reload +serve-webhooks-fastapi: ## Start the local FastAPI server to test webhooks + cd examples/fastapi && README_API_KEY=$(API_KEY) uvicorn webhooks:app --reload + test: ## Run unit tests pytest diff --git a/packages/python/examples/fastapi/app.py b/packages/python/examples/fastapi/app.py index 9a68ccef2e..93139e155f 100644 --- a/packages/python/examples/fastapi/app.py +++ b/packages/python/examples/fastapi/app.py @@ -45,4 +45,4 @@ def read_root(): @app.post("/") def post_root(): - return "13414" + return "" diff --git a/packages/python/examples/fastapi/webhooks.py b/packages/python/examples/fastapi/webhooks.py new file mode 100644 index 0000000000..6a53698899 --- /dev/null +++ b/packages/python/examples/fastapi/webhooks.py @@ -0,0 +1,45 @@ +import os +import sys + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + +from readme_metrics.VerifyWebhook import VerifyWebhook + + +if os.getenv("README_API_KEY") is None: + sys.stderr.write("Missing `README_API_KEY` environment variable") + sys.stderr.flush() + sys.exit(1) + +app = FastAPI() + +# Your ReadMe secret +secret = os.getenv("README_API_KEY") + + +@app.post("/webhook") +async def webhook(request: Request): + # Verify the request is legitimate and came from ReadMe. + signature = request.headers.get("readme-signature", None) + + try: + body = await request.json() + VerifyWebhook(body, signature, secret) + except Exception as error: + return JSONResponse( + status_code=401, + headers={"Content-Type": "application/json; charset=utf-8"}, + content={"error": str(error)}, + ) + + # Fetch the user from the database and return their data for use with OpenAPI variables. + # user = User.objects.get(email__exact=request.values.get("email")) + return JSONResponse( + status_code=200, + headers={"Content-Type": "application/json; charset=utf-8"}, + content={ + "petstore_auth": "default-key", + "basic_auth": {"user": "user", "pass": "pass"}, + }, + ) diff --git a/test/integrations/python/fastapi-webhooks.Dockerfile b/test/integrations/python/fastapi-webhooks.Dockerfile new file mode 100644 index 0000000000..b5ceb7cb61 --- /dev/null +++ b/test/integrations/python/fastapi-webhooks.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.10 + +COPY packages/python /src + +# Set up the Python SDK +WORKDIR /src +RUN pip3 install --no-cache-dir -r requirements.txt + +# Install Flask +WORKDIR /src/examples/fastapi +RUN pip3 install --no-cache-dir -r requirements.txt + +CMD ["uvicorn", "webhooks:app", "--host", "0.0.0.0", "--port", "8000"] From f169f0a5871bd07c5fa221ee1fa51fd5f1a6ae55 Mon Sep 17 00:00:00 2001 From: Andrii Andreiev Date: Thu, 23 Jan 2025 16:57:26 +0200 Subject: [PATCH 3/3] feat(python/fastapi): add FastAPI tests to github workflow --- .github/workflows/python.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 31394db2a8..b791dc7a1a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -20,6 +20,8 @@ jobs: - run: make test-metrics-python-django-asgi - run: make test-metrics-python-flask - run: make test-webhooks-python-flask + - run: make test-metrics-python-fastapi + - run: make test-webhooks-python-fastapi - name: Cleanup if: always()