From e90026c63f59645fad0aac1a2a0c50f0f90980d1 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Mon, 11 Jul 2022 12:38:25 +0200 Subject: [PATCH 1/5] chore: prepare to implement redacting auth header --- tests/test_InfluxDBClient.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_InfluxDBClient.py b/tests/test_InfluxDBClient.py index 825decf3..9a8f1037 100644 --- a/tests/test_InfluxDBClient.py +++ b/tests/test_InfluxDBClient.py @@ -1,8 +1,10 @@ import http.server import json +import logging import os import threading import unittest +from io import StringIO import httpretty import pytest @@ -262,6 +264,25 @@ def test_init_without_token(self): self.assertIsNotNone(self.influxdb_client) self.influxdb_client.query_api().query("buckets()", "my-org") + def test_redacted_auth_header(self): + httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/query", status=200, body="") + self.influxdb_client = InfluxDBClient("http://localhost", "my-token", debug=True) + + log_stream = StringIO() + for _, logger in self.influxdb_client.conf.loggers.items(): + logger.setLevel(logging.DEBUG) + logger.addHandler(logging.StreamHandler(log_stream)) + + logger = logging.getLogger("urllib3") + logger.addHandler(logging.StreamHandler(log_stream)) + + self.influxdb_client.query_api().query("buckets()", "my-org") + requests = httpretty.httpretty.latest_requests + self.assertEqual(1, len(requests)) + self.assertEqual("Token my-token", requests[0].headers["Authorization"]) + + self.assertIn("Authorization: Token ***", log_stream.getvalue()) + class ServerWithSelfSingedSSL(http.server.SimpleHTTPRequestHandler): def _set_headers(self): From ee97da924f5855d67493a52ef991c79364958fe3 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 12 Jul 2022 08:47:30 +0200 Subject: [PATCH 2/5] feat: add custom logging for HTTP requests --- README.rst | 1 + influxdb_client/_async/rest.py | 15 ++++++++------- influxdb_client/_sync/rest.py | 16 +++++++++++++++- influxdb_client/client/_base.py | 3 ++- influxdb_client/configuration.py | 13 +++++++------ influxdb_client/rest.py | 26 ++++++++++++++++++++++++++ tests/test_InfluxDBClient.py | 8 ++------ tests/test_InfluxDBClientAsync.py | 15 +++++++++++++++ 8 files changed, 76 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index 94603bd6..c91855e5 100644 --- a/README.rst +++ b/README.rst @@ -1587,6 +1587,7 @@ The client uses uses Python's `logging >> Request: '{params.method} {params.url}'") - print(f">>> Headers: '{params.headers}'") + _BaseRESTClient.log_request(params.method, params.url) + _BaseRESTClient.log_headers(params.headers, '>>>') async def _on_request_chunk_sent(session, context, params): if params.chunk: - print(f">>> Body: {params.chunk}") + _BaseRESTClient.log_body(params.chunk, '>>>') async def _on_request_end(session, trace_config_ctx, params): - print(f"<<< Response: {params.response.status}") - print(f"<<< Headers: '{params.response.headers}") + _BaseRESTClient.log_response(params.response.status) + _BaseRESTClient.log_headers(params.headers, '<<<') response_content = params.response.content data = bytearray() @@ -42,7 +43,7 @@ async def _on_request_end(session, trace_config_ctx, params): break data += chunk if data: - print(f"<<< Body: {data.decode(_UTF_8_encoding)}") + _BaseRESTClient.log_body(data.decode(_UTF_8_encoding), '<<<') response_content.unread_data(data=data) @@ -69,7 +70,7 @@ def getheader(self, name, default=None): return self.aiohttp_response.headers.get(name, default) -class RESTClientObjectAsync(object): +class RESTClientObjectAsync(_BaseRESTClient): """NOTE: This class is auto generated by OpenAPI Generator. Ref: https://openapi-generator.tech diff --git a/influxdb_client/_sync/rest.py b/influxdb_client/_sync/rest.py index c469f1b5..f0a1836a 100644 --- a/influxdb_client/_sync/rest.py +++ b/influxdb_client/_sync/rest.py @@ -19,6 +19,7 @@ from urllib.parse import urlencode from influxdb_client.rest import ApiException +from influxdb_client.rest import _BaseRESTClient try: import urllib3 @@ -49,7 +50,7 @@ def getheader(self, name, default=None): return self.urllib3_response.getheader(name, default) -class RESTClientObject(object): +class RESTClientObject(_BaseRESTClient): """NOTE: This class is auto generated by OpenAPI Generator. Ref: https://openapi-generator.tech @@ -164,6 +165,11 @@ def request(self, method, url, query_params=None, headers=None, if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' + if self.configuration.debug: + _BaseRESTClient.log_request(method, f"{url}?{urlencode(query_params)}") + _BaseRESTClient.log_headers(headers, '>>>') + _BaseRESTClient.log_body(body, '>>>') + try: # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: @@ -239,6 +245,14 @@ def request(self, method, url, query_params=None, headers=None, # we need to decode it to string. r.data = r.data.decode('utf8') + if self.configuration.debug: + _BaseRESTClient.log_response(r.status) + if hasattr(r, 'headers'): + _BaseRESTClient.log_headers(r.headers, '<<<') + if hasattr(r, 'urllib3_response'): + _BaseRESTClient.log_headers(r.urllib3_response.headers, '<<<') + _BaseRESTClient.log_body(r.data, '<<<') + if not 200 <= r.status <= 299: raise ApiException(http_resp=r) diff --git a/influxdb_client/client/_base.py b/influxdb_client/client/_base.py index 5f1fff84..f2b5e995 100644 --- a/influxdb_client/client/_base.py +++ b/influxdb_client/client/_base.py @@ -38,7 +38,8 @@ 'influxdb_client.client.write.retry', 'influxdb_client.client.write.dataframe_serializer', 'influxdb_client.client.util.multiprocessing_helper', - 'influxdb_client.client.exceptions' + 'influxdb_client.client.http', + 'influxdb_client.client.exceptions', ] diff --git a/influxdb_client/configuration.py b/influxdb_client/configuration.py index 55caa8bb..e6ef2c61 100644 --- a/influxdb_client/configuration.py +++ b/influxdb_client/configuration.py @@ -13,7 +13,6 @@ from __future__ import absolute_import import copy -import http.client as httplib import logging import multiprocessing import sys @@ -164,17 +163,19 @@ def debug(self, value): self.__debug = value if self.__debug: # if debug status is True, turn on debug logging - for _, logger in self.loggers.items(): + for name, logger in self.loggers.items(): logger.setLevel(logging.DEBUG) - # turn on httplib debug - httplib.HTTPConnection.debuglevel = 1 + if name == 'influxdb_client.client.http': + logger.addHandler(logging.StreamHandler(sys.stdout)) + # we use 'influxdb_client.client.http' logger instead of this + # httplib.HTTPConnection.debuglevel = 1 else: # if debug status is False, turn off debug logging, # setting log level to default `logging.WARNING` for _, logger in self.loggers.items(): logger.setLevel(logging.WARNING) - # turn off httplib debug - httplib.HTTPConnection.debuglevel = 0 + # we use 'influxdb_client.client.http' logger instead of this + # httplib.HTTPConnection.debuglevel = 0 @property def logger_format(self): diff --git a/influxdb_client/rest.py b/influxdb_client/rest.py index 91e3d548..372d60f6 100644 --- a/influxdb_client/rest.py +++ b/influxdb_client/rest.py @@ -11,6 +11,8 @@ from __future__ import absolute_import +import logging + from influxdb_client.client.exceptions import InfluxDBError from influxdb_client.configuration import Configuration @@ -52,6 +54,30 @@ def __str__(self): return error_message +class _BaseRESTClient(object): + logger = logging.getLogger('influxdb_client.client.http') + + @staticmethod + def log_request(method: str, url: str): + _BaseRESTClient.logger.debug(f">>> Request: '{method} {url}'") + + @staticmethod + def log_response(status: str): + _BaseRESTClient.logger.debug(f"<<< Response: {status}") + + @staticmethod + def log_body(body: object, prefix: str): + _BaseRESTClient.logger.debug(f"{prefix} Body: {body}") + + @staticmethod + def log_headers(headers: dict[str, str], prefix: str): + for key, v in headers.items(): + value = v + if 'authorization' == key.lower(): + value = '***' + _BaseRESTClient.logger.debug(f"{prefix} {key}: {value}") + + def _requires_create_user_session(configuration: Configuration, cookie: str, resource_path: str): _unauthorized = ['/api/v2/signin', '/api/v2/signout'] return configuration.username and configuration.password and not cookie and resource_path not in _unauthorized diff --git a/tests/test_InfluxDBClient.py b/tests/test_InfluxDBClient.py index 9a8f1037..cd1e3910 100644 --- a/tests/test_InfluxDBClient.py +++ b/tests/test_InfluxDBClient.py @@ -269,11 +269,7 @@ def test_redacted_auth_header(self): self.influxdb_client = InfluxDBClient("http://localhost", "my-token", debug=True) log_stream = StringIO() - for _, logger in self.influxdb_client.conf.loggers.items(): - logger.setLevel(logging.DEBUG) - logger.addHandler(logging.StreamHandler(log_stream)) - - logger = logging.getLogger("urllib3") + logger = logging.getLogger("influxdb_client.client.http") logger.addHandler(logging.StreamHandler(log_stream)) self.influxdb_client.query_api().query("buckets()", "my-org") @@ -281,7 +277,7 @@ def test_redacted_auth_header(self): self.assertEqual(1, len(requests)) self.assertEqual("Token my-token", requests[0].headers["Authorization"]) - self.assertIn("Authorization: Token ***", log_stream.getvalue()) + self.assertIn("Authorization: ***", log_stream.getvalue()) class ServerWithSelfSingedSSL(http.server.SimpleHTTPRequestHandler): diff --git a/tests/test_InfluxDBClientAsync.py b/tests/test_InfluxDBClientAsync.py index fb2ddfce..a96f4285 100644 --- a/tests/test_InfluxDBClientAsync.py +++ b/tests/test_InfluxDBClientAsync.py @@ -1,7 +1,9 @@ import asyncio +import logging import unittest import os from datetime import datetime +from io import StringIO import pytest from aioresponses import aioresponses @@ -247,6 +249,19 @@ async def test_init_without_token(self, mocked): self.client = InfluxDBClientAsync("http://localhost") await self.client.query_api().query("buckets()", "my-org") + @async_test + async def test_redacted_auth_header(self): + await self.client.close() + self.client = InfluxDBClientAsync(url="http://localhost:8086", token="my-token", org="my-org", debug=True) + + log_stream = StringIO() + logger = logging.getLogger("influxdb_client.client.http") + logger.addHandler(logging.StreamHandler(log_stream)) + + await self.client.query_api().query("buckets()", "my-org") + + self.assertIn("Authorization: ***", log_stream.getvalue()) + async def _prepare_data(self, measurement: str): _point1 = Point(measurement).tag("location", "Prague").field("temperature", 25.3) _point2 = Point(measurement).tag("location", "New York").field("temperature", 24.3) From 2175ec81cad082326adab6677674884cf792f4e6 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 12 Jul 2022 08:57:37 +0200 Subject: [PATCH 3/5] fix: tests --- influxdb_client/rest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/influxdb_client/rest.py b/influxdb_client/rest.py index 372d60f6..8f50e51a 100644 --- a/influxdb_client/rest.py +++ b/influxdb_client/rest.py @@ -12,6 +12,7 @@ from __future__ import absolute_import import logging +from typing import Dict from influxdb_client.client.exceptions import InfluxDBError from influxdb_client.configuration import Configuration @@ -70,7 +71,7 @@ def log_body(body: object, prefix: str): _BaseRESTClient.logger.debug(f"{prefix} Body: {body}") @staticmethod - def log_headers(headers: dict[str, str], prefix: str): + def log_headers(headers: Dict[str, str], prefix: str): for key, v in headers.items(): value = v if 'authorization' == key.lower(): From 6884247907c58d927963b17997844e91004879b9 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 12 Jul 2022 09:04:18 +0200 Subject: [PATCH 4/5] docs: update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb862a2c..aa31a46c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## 1.31.0 [unreleased] +### Bug Fixes +1. [#462](https://github.com/influxdata/influxdb-client-python/pull/462): Redact the `Authorization` HTTP header from log + ## 1.30.0 [2022-06-24] ### Features From c5c5bb79e00a0b463a6cd2582b1ca61d5e35be1c Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 12 Jul 2022 09:36:44 +0200 Subject: [PATCH 5/5] feat: add custom logging for HTTP requests --- influxdb_client/_async/rest.py | 2 +- influxdb_client/_sync/rest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/influxdb_client/_async/rest.py b/influxdb_client/_async/rest.py index 76424493..dcf678d0 100644 --- a/influxdb_client/_async/rest.py +++ b/influxdb_client/_async/rest.py @@ -70,7 +70,7 @@ def getheader(self, name, default=None): return self.aiohttp_response.headers.get(name, default) -class RESTClientObjectAsync(_BaseRESTClient): +class RESTClientObjectAsync(object): """NOTE: This class is auto generated by OpenAPI Generator. Ref: https://openapi-generator.tech diff --git a/influxdb_client/_sync/rest.py b/influxdb_client/_sync/rest.py index f0a1836a..d3ac55c6 100644 --- a/influxdb_client/_sync/rest.py +++ b/influxdb_client/_sync/rest.py @@ -50,7 +50,7 @@ def getheader(self, name, default=None): return self.urllib3_response.getheader(name, default) -class RESTClientObject(_BaseRESTClient): +class RESTClientObject(object): """NOTE: This class is auto generated by OpenAPI Generator. Ref: https://openapi-generator.tech