From 3cb09707194dc81b041763eb1957b02d9b1a682b Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Sun, 3 Jan 2021 12:13:33 -0600 Subject: [PATCH 1/4] Create basic memory store for toolbar to reproduce existing functionality. --- debug_toolbar/panels/history/panel.py | 3 +- debug_toolbar/panels/history/views.py | 7 ++-- debug_toolbar/settings.py | 1 + debug_toolbar/store.py | 50 +++++++++++++++++++++++++++ debug_toolbar/toolbar.py | 13 ++----- debug_toolbar/views.py | 4 +-- tests/base.py | 5 +-- tests/panels/test_history.py | 35 +++++++++++-------- tests/test_integration.py | 10 ++++-- 9 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 debug_toolbar/store.py diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py index 541c59136..a5536cd93 100644 --- a/debug_toolbar/panels/history/panel.py +++ b/debug_toolbar/panels/history/panel.py @@ -12,6 +12,7 @@ from debug_toolbar.panels import Panel from debug_toolbar.panels.history import views from debug_toolbar.panels.history.forms import HistoryStoreForm +from debug_toolbar.store import store class HistoryPanel(Panel): @@ -81,7 +82,7 @@ def content(self): Fetch every store for the toolbar and include it in the template. """ stores = OrderedDict() - for id, toolbar in reversed(self.toolbar._store.items()): + for id, toolbar in reversed(store.all()): stores[id] = { "toolbar": toolbar, "form": SignedDataForm( diff --git a/debug_toolbar/panels/history/views.py b/debug_toolbar/panels/history/views.py index 10b4dcc1a..bcfc4a8c1 100644 --- a/debug_toolbar/panels/history/views.py +++ b/debug_toolbar/panels/history/views.py @@ -4,7 +4,7 @@ from debug_toolbar.decorators import require_show_toolbar, signed_data_view from debug_toolbar.forms import SignedDataForm from debug_toolbar.panels.history.forms import HistoryStoreForm -from debug_toolbar.toolbar import DebugToolbar +from debug_toolbar.store import store @require_show_toolbar @@ -15,7 +15,7 @@ def history_sidebar(request, verified_data): if form.is_valid(): store_id = form.cleaned_data["store_id"] - toolbar = DebugToolbar.fetch(store_id) + toolbar = store.get(store_id) context = {} if toolbar is None: # When the store_id has been popped already due to @@ -45,8 +45,7 @@ def history_refresh(request, verified_data): if form.is_valid(): requests = [] - # Convert to list to handle mutations happenening in parallel - for id, toolbar in list(DebugToolbar._store.items())[::-1]: + for id, toolbar in list(reversed(store.all())): requests.append( { "id": id, diff --git a/debug_toolbar/settings.py b/debug_toolbar/settings.py index d8e6868a3..0f21689f0 100644 --- a/debug_toolbar/settings.py +++ b/debug_toolbar/settings.py @@ -21,6 +21,7 @@ "ROOT_TAG_EXTRA_ATTRS": "", "SHOW_COLLAPSED": False, "SHOW_TOOLBAR_CALLBACK": "debug_toolbar.middleware.show_toolbar", + "TOOLBAR_STORE_CLASS": "debug_toolbar.store.MemoryStore", # Panel options "EXTRA_SIGNALS": [], "ENABLE_STACKTRACES": True, diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py new file mode 100644 index 000000000..15bbe2eaf --- /dev/null +++ b/debug_toolbar/store.py @@ -0,0 +1,50 @@ +from collections import OrderedDict + +from django.utils.module_loading import import_string + +from debug_toolbar import settings as dt_settings + + +class BaseStore: + config = dt_settings.get_config().copy() + + @classmethod + def get(cls, store_id): + raise NotImplementedError + + @classmethod + def all(cls): + raise NotImplementedError + + @classmethod + def set(cls, store_id, toolbar): + raise NotImplementedError + + @classmethod + def delete(cls, store_id): + raise NotImplementedError + + +class MemoryStore(BaseStore): + _store = OrderedDict() + + @classmethod + def get(cls, store_id): + return cls._store.get(store_id) + + @classmethod + def all(cls): + return cls._store.items() + + @classmethod + def set(cls, store_id, toolbar): + cls._store[store_id] = toolbar + for _ in range(cls.config["RESULTS_CACHE_SIZE"], len(cls._store)): + cls._store.popitem(last=False) + + @classmethod + def delete(cls, store_id): + del cls._store[store_id] + + +store = import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) diff --git a/debug_toolbar/toolbar.py b/debug_toolbar/toolbar.py index cb886c407..d15e62b65 100644 --- a/debug_toolbar/toolbar.py +++ b/debug_toolbar/toolbar.py @@ -14,6 +14,7 @@ from django.utils.module_loading import import_string from debug_toolbar import settings as dt_settings +from debug_toolbar.store import store class DebugToolbar: @@ -88,22 +89,12 @@ def should_render_panels(self): render_panels = self.request.META["wsgi.multiprocess"] return render_panels - # Handle storing toolbars in memory and fetching them later on - - _store = OrderedDict() - def store(self): # Store already exists. if self.store_id: return self.store_id = uuid.uuid4().hex - self._store[self.store_id] = self - for _ in range(self.config["RESULTS_CACHE_SIZE"], len(self._store)): - self._store.popitem(last=False) - - @classmethod - def fetch(cls, store_id): - return cls._store.get(store_id) + store.set(self.store_id, self) # Manually implement class-level caching of panel classes and url patterns # because it's more obvious than going through an abstraction. diff --git a/debug_toolbar/views.py b/debug_toolbar/views.py index 1d319027d..5624ca6f4 100644 --- a/debug_toolbar/views.py +++ b/debug_toolbar/views.py @@ -3,13 +3,13 @@ from django.utils.translation import gettext as _ from debug_toolbar.decorators import require_show_toolbar -from debug_toolbar.toolbar import DebugToolbar +from debug_toolbar.store import store @require_show_toolbar def render_panel(request): """Render the contents of a panel""" - toolbar = DebugToolbar.fetch(request.GET["store_id"]) + toolbar = store.get(request.GET["store_id"]) if toolbar is None: content = _( "Data for this panel isn't available anymore. " diff --git a/tests/base.py b/tests/base.py index c09828b4f..a35ae4bcf 100644 --- a/tests/base.py +++ b/tests/base.py @@ -2,6 +2,7 @@ from django.http import HttpResponse from django.test import RequestFactory, TestCase +from debug_toolbar.store import store from debug_toolbar.toolbar import DebugToolbar rf = RequestFactory() @@ -52,6 +53,6 @@ def setUp(self): # The HistoryPanel keeps track of previous stores in memory. # This bleeds into other tests and violates their idempotency. # Clear the store before each test. - for key in list(DebugToolbar._store.keys()): - del DebugToolbar._store[key] + for key, _ in list(store.all()): + store.delete(key) super().setUp() diff --git a/tests/panels/test_history.py b/tests/panels/test_history.py index 49e3bd0fa..02bd9e8d1 100644 --- a/tests/panels/test_history.py +++ b/tests/panels/test_history.py @@ -4,7 +4,7 @@ from django.urls import resolve, reverse from debug_toolbar.forms import SignedDataForm -from debug_toolbar.toolbar import DebugToolbar +from debug_toolbar.store import store from ..base import BaseTestCase, IntegrationTestCase @@ -83,14 +83,14 @@ class HistoryViewsTestCase(IntegrationTestCase): def test_history_panel_integration_content(self): """Verify the history panel's content renders properly..""" - self.assertEqual(len(DebugToolbar._store), 0) + self.assertEqual(len(store.all()), 0) data = {"foo": "bar"} self.client.get("/json_view/", data, content_type="application/json") # Check the history panel's stats to verify the toolbar rendered properly. - self.assertEqual(len(DebugToolbar._store), 1) - toolbar = list(DebugToolbar._store.values())[0] + self.assertEqual(len(store.all()), 1) + toolbar = list(store.all())[0][1] content = toolbar.get_panel_by_id("HistoryPanel").content self.assertIn("bar", content) @@ -98,14 +98,16 @@ def test_history_sidebar_invalid(self): response = self.client.get(reverse("djdt:history_sidebar")) self.assertEqual(response.status_code, 400) - data = {"signed": SignedDataForm.sign({"store_id": "foo"}) + "invalid"} + self.client.get("/json_view/") + store_id = list(store.all())[0][0] + data = {"signed": SignedDataForm.sign({"store_id": store_id}) + "invalid"} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 400) - def test_history_sidebar(self): - """Validate the history sidebar view.""" + def test_history_sidebar_hash(self): + """Validate the hashing mechanism.""" self.client.get("/json_view/") - store_id = list(DebugToolbar._store)[0] + store_id = list(store.all())[0][0] data = {"signed": SignedDataForm.sign({"store_id": store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) @@ -120,7 +122,7 @@ def test_history_sidebar(self): def test_history_sidebar_expired_store_id(self): """Validate the history sidebar view.""" self.client.get("/json_view/") - store_id = list(DebugToolbar._store)[0] + store_id = list(store.all())[0][0] data = {"signed": SignedDataForm.sign({"store_id": store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) @@ -130,14 +132,18 @@ def test_history_sidebar_expired_store_id(self): ) self.client.get("/json_view/") - # Querying old store_id should return in empty response + # Querying previous store_id should still work data = {"signed": SignedDataForm.sign({"store_id": store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) - self.assertEqual(response.json(), {}) + self.assertEqual( + set(response.json()), + self.PANEL_KEYS, + ) # Querying with latest store_id - latest_store_id = list(DebugToolbar._store)[0] + latest_store_id = list(store.all())[-1][0] + self.assertNotEqual(latest_store_id, store_id) data = {"signed": SignedDataForm.sign({"store_id": latest_store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) @@ -157,15 +163,14 @@ def test_history_refresh_invalid_signature(self): def test_history_refresh(self): """Verify refresh history response has request variables.""" - data = {"foo": "bar"} - self.client.get("/json_view/", data, content_type="application/json") + self.client.get("/json_view/", {"foo": "bar"}, content_type="application/json") data = {"signed": SignedDataForm.sign({"store_id": "foo"})} response = self.client.get(reverse("djdt:history_refresh"), data=data) self.assertEqual(response.status_code, 200) data = response.json() self.assertEqual(len(data["requests"]), 1) - store_id = list(DebugToolbar._store)[0] + store_id = list(store.all())[0][0] signature = SignedDataForm.sign({"store_id": store_id}) self.assertIn(html.escape(signature), data["requests"][0]["content"]) diff --git a/tests/test_integration.py b/tests/test_integration.py index 3be1ef589..bb8eae52c 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -16,6 +16,7 @@ from debug_toolbar.forms import SignedDataForm from debug_toolbar.middleware import DebugToolbarMiddleware, show_toolbar from debug_toolbar.panels import Panel +from debug_toolbar.store import store from debug_toolbar.toolbar import DebugToolbar from .base import BaseTestCase, IntegrationTestCase @@ -207,15 +208,15 @@ def get_response(request): def test_middleware_render_toolbar_json(self): """Verify the toolbar is rendered and data is stored for a json request.""" - self.assertEqual(len(DebugToolbar._store), 0) + self.assertEqual(len(store.all()), 0) data = {"foo": "bar"} response = self.client.get("/json_view/", data, content_type="application/json") self.assertEqual(response.status_code, 200) self.assertEqual(response.content.decode("utf-8"), '{"foo": "bar"}') # Check the history panel's stats to verify the toolbar rendered properly. - self.assertEqual(len(DebugToolbar._store), 1) - toolbar = list(DebugToolbar._store.values())[0] + self.assertEqual(len(store.all()), 1) + toolbar = list(store.all())[0][1] self.assertEqual( toolbar.get_panel_by_id("HistoryPanel").get_stats()["data"], {"foo": ["bar"]}, @@ -523,6 +524,8 @@ def test_rerender_on_history_switch(self): @override_settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 0}) def test_expired_store(self): + original_value = store.config["RESULTS_CACHE_SIZE"] + store.config["RESULTS_CACHE_SIZE"] = 0 self.get("/regular/basic/") version_panel = self.selenium.find_element_by_id("VersionsPanel") @@ -534,6 +537,7 @@ def test_expired_store(self): lambda selenium: version_panel.find_element_by_tag_name("p") ) self.assertIn("Data for this panel isn't available anymore.", error.text) + store.config["RESULTS_CACHE_SIZE"] = original_value @override_settings( TEMPLATES=[ From 9cf5bfd90a86d49de20cd4dbd5ea7307445b8237 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Sun, 3 Jan 2021 17:11:00 -0600 Subject: [PATCH 2/4] Support storing serialized panel stats directly into the store. In order to support the toolbar persisting stats, we need to move away from calling record_stats with non-primitives. --- debug_toolbar/panels/__init__.py | 22 +++++++++- debug_toolbar/panels/cache.py | 2 +- debug_toolbar/panels/history/panel.py | 4 +- debug_toolbar/panels/profiling.py | 18 +++++++- debug_toolbar/panels/request.py | 17 ++++---- debug_toolbar/panels/templates/panel.py | 24 ++++++----- debug_toolbar/store.py | 41 ++++++++++++++++++- .../templates/debug_toolbar/panels/cache.html | 3 +- .../debug_toolbar/panels/profiling.html | 3 +- .../templates/debug_toolbar/panels/sql.html | 6 ++- debug_toolbar/toolbar.py | 6 +-- tests/panels/test_history.py | 2 +- tests/test_integration.py | 10 ++--- 13 files changed, 117 insertions(+), 41 deletions(-) diff --git a/debug_toolbar/panels/__init__.py b/debug_toolbar/panels/__init__.py index 8fd433c63..f3d5c6c76 100644 --- a/debug_toolbar/panels/__init__.py +++ b/debug_toolbar/panels/__init__.py @@ -1,6 +1,7 @@ from django.template.loader import render_to_string from debug_toolbar import settings as dt_settings +from debug_toolbar.store import store from debug_toolbar.utils import get_name_from_obj @@ -150,6 +151,22 @@ def disable_instrumentation(self): # Store and retrieve stats (shared between panels for no good reason) + def deserialize_stats(self, data): + """ + Deserialize stats coming from the store. + + Provided to support future store mechanisms overriding a panel's content. + """ + return data + + def serialize_stats(self, stats): + """ + Serialize stats for the store. + + Provided to support future store mechanisms overriding a panel's content. + """ + return stats + def record_stats(self, stats): """ Store data gathered by the panel. ``stats`` is a :class:`dict`. @@ -157,12 +174,15 @@ def record_stats(self, stats): Each call to ``record_stats`` updates the statistics dictionary. """ self.toolbar.stats.setdefault(self.panel_id, {}).update(stats) + store.save_panel( + self.toolbar.store_id, self.panel_id, self.serialize_stats(stats) + ) def get_stats(self): """ Access data stored by the panel. Returns a :class:`dict`. """ - return self.toolbar.stats.get(self.panel_id, {}) + return self.deserialize_stats(store.panel(self.toolbar.store_id, self.panel_id)) def record_server_timing(self, key, title, value): """ diff --git a/debug_toolbar/panels/cache.py b/debug_toolbar/panels/cache.py index 41063c573..87e09c838 100644 --- a/debug_toolbar/panels/cache.py +++ b/debug_toolbar/panels/cache.py @@ -236,7 +236,7 @@ def _store_call_info( "kwargs": kwargs, "trace": render_stacktrace(trace), "template_info": template_info, - "backend": backend, + "backend": str(backend), } ) diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py index a5536cd93..f311f036c 100644 --- a/debug_toolbar/panels/history/panel.py +++ b/debug_toolbar/panels/history/panel.py @@ -47,9 +47,9 @@ def nav_subtitle(self): def generate_stats(self, request, response): try: if request.method == "GET": - data = request.GET.copy() + data = dict(request.GET.copy()) else: - data = request.POST.copy() + data = dict(request.POST.copy()) # GraphQL tends to not be populated in POST. If the request seems # empty, check if it's a JSON request. if ( diff --git a/debug_toolbar/panels/profiling.py b/debug_toolbar/panels/profiling.py index fdd5ed06e..21907effb 100644 --- a/debug_toolbar/panels/profiling.py +++ b/debug_toolbar/panels/profiling.py @@ -51,6 +51,7 @@ def __init__( self.id = id self.parent_ids = parent_ids self.hsv = hsv + self.has_subfuncs = False def parent_classes(self): return self.parent_classes @@ -141,6 +142,20 @@ def cumtime_per_call(self): def indent(self): return 16 * self.depth + def as_context(self): + return { + "id": self.id, + "parent_ids": self.parent_ids, + "func_std_string": self.func_std_string(), + "has_subfuncs": self.has_subfuncs, + "cumtime": self.cumtime(), + "cumtime_per_call": self.cumtime_per_call(), + "tottime": self.tottime(), + "tottime_per_call": self.tottime_per_call(), + "count": self.count(), + "indent": self.indent(), + } + class ProfilingPanel(Panel): """ @@ -157,7 +172,6 @@ def process_request(self, request): def add_node(self, func_list, func, max_depth, cum_time=0.1): func_list.append(func) - func.has_subfuncs = False if func.depth < max_depth: for subfunc in func.subfuncs(): if subfunc.stats[3] >= cum_time: @@ -183,4 +197,4 @@ def generate_stats(self, request, response): dt_settings.get_config()["PROFILER_MAX_DEPTH"], root.stats[3] / 8, ) - self.record_stats({"func_list": func_list}) + self.record_stats({"func_list": [func.as_context() for func in func_list]}) diff --git a/debug_toolbar/panels/request.py b/debug_toolbar/panels/request.py index 5255624b2..348f435ca 100644 --- a/debug_toolbar/panels/request.py +++ b/debug_toolbar/panels/request.py @@ -24,13 +24,11 @@ def nav_subtitle(self): return view_func.rsplit(".", 1)[-1] def generate_stats(self, request, response): - self.record_stats( - { - "get": get_sorted_request_variable(request.GET), - "post": get_sorted_request_variable(request.POST), - "cookies": get_sorted_request_variable(request.COOKIES), - } - ) + stats = { + "get": get_sorted_request_variable(request.GET), + "post": get_sorted_request_variable(request.POST), + "cookies": get_sorted_request_variable(request.COOKIES), + } view_info = { "view_func": _(""), @@ -56,10 +54,10 @@ def generate_stats(self, request, response): except Http404: pass - self.record_stats(view_info) + stats.update(view_info) if hasattr(request, "session"): - self.record_stats( + stats.update( { "session": [ (k, request.session.get(k)) @@ -67,3 +65,4 @@ def generate_stats(self, request, response): ] } ) + self.record_stats(stats) diff --git a/debug_toolbar/panels/templates/panel.py b/debug_toolbar/panels/templates/panel.py index 8ff06e27d..efd799678 100644 --- a/debug_toolbar/panels/templates/panel.py +++ b/debug_toolbar/panels/templates/panel.py @@ -171,20 +171,24 @@ def disable_instrumentation(self): def generate_stats(self, request, response): template_context = [] for template_data in self.templates: - info = {} # Clean up some info about templates template = template_data["template"] if hasattr(template, "origin") and template.origin and template.origin.name: - template.origin_name = template.origin.name - template.origin_hash = signing.dumps(template.origin.name) + origin_name = template.origin.name + origin_hash = signing.dumps(template.origin.name) else: - template.origin_name = _("No origin") - template.origin_hash = "" - info["template"] = template - # Clean up context for better readability - if self.toolbar.config["SHOW_TEMPLATE_CONTEXT"]: - context_list = template_data.get("context", []) - info["context"] = "\n".join(context_list) + origin_name = _("No origin") + origin_hash = "" + info = { + "template": { + "origin_name": origin_name, + "origin_hash": origin_hash, + "name": template.name, + }, + "context": "\n".join(template_data.get("context", [])) + if self.toolbar.config["SHOW_TEMPLATE_CONTEXT"] + else "", + } template_context.append(info) # Fetch context_processors/template_dirs from any template diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index 15bbe2eaf..849f6b47a 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -1,10 +1,32 @@ -from collections import OrderedDict +import json +from collections import OrderedDict, defaultdict +from django.core.serializers.json import DjangoJSONEncoder from django.utils.module_loading import import_string from debug_toolbar import settings as dt_settings +class DebugToolbarJSONEncoder(DjangoJSONEncoder): + def default(self, o): + try: + return super().default(o) + except TypeError: + return str(o) + + +def serialize(data): + return json.dumps(data, cls=DebugToolbarJSONEncoder) + + +def deserialize(data): + return json.loads(data) + + +# Record stats in serialized fashion. +# Remove use of fetching the toolbar as a whole from the store. + + class BaseStore: config = dt_settings.get_config().copy() @@ -24,9 +46,14 @@ def set(cls, store_id, toolbar): def delete(cls, store_id): raise NotImplementedError + @classmethod + def record_stats(cls, store_id, panel_id, stats): + raise NotImplementedError + class MemoryStore(BaseStore): _store = OrderedDict() + _stats = defaultdict(dict) @classmethod def get(cls, store_id): @@ -46,5 +73,17 @@ def set(cls, store_id, toolbar): def delete(cls, store_id): del cls._store[store_id] + @classmethod + def save_panel(cls, store_id, panel_id, stats=None): + cls._stats[store_id][panel_id] = serialize(stats) + + @classmethod + def panel(cls, store_id, panel_id): + try: + data = cls._stats[store_id][panel_id] + except KeyError: + data = None + return {} if data is None else deserialize(data) + store = import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) diff --git a/debug_toolbar/templates/debug_toolbar/panels/cache.html b/debug_toolbar/templates/debug_toolbar/panels/cache.html index 0e1ec2a4c..2390fff5a 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/cache.html +++ b/debug_toolbar/templates/debug_toolbar/panels/cache.html @@ -61,7 +61,8 @@

{% trans "Calls" %}

- 
{{ call.trace }}
+ {# The trace property is escaped when serialized into the store #} + 
{{ call.trace|safe }}
{% endfor %} diff --git a/debug_toolbar/templates/debug_toolbar/panels/profiling.html b/debug_toolbar/templates/debug_toolbar/panels/profiling.html index 837698889..86fe85770 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/profiling.html +++ b/debug_toolbar/templates/debug_toolbar/panels/profiling.html @@ -20,7 +20,8 @@ {% else %} {% endif %} - {{ call.func_std_string }} + {# The func_std_string property is escaped when serialized into the store #} + {{ call.func_std_string|safe }} {{ call.cumtime|floatformat:3 }} diff --git a/debug_toolbar/templates/debug_toolbar/panels/sql.html b/debug_toolbar/templates/debug_toolbar/panels/sql.html index 6080e9f19..13da1d08d 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/sql.html +++ b/debug_toolbar/templates/debug_toolbar/panels/sql.html @@ -77,7 +77,8 @@ {% if query.params %} {% if query.is_select %}
- {{ query.form }} + {# The form is rendered when serialized into storage #} + {{ query.form|safe }} {% if query.vendor == 'mysql' %} @@ -100,7 +101,8 @@

{% trans "Transaction status:" %} {{ query.trans_status }}

{% endif %} {% if query.stacktrace %} - 
{{ query.stacktrace }}
+ {# The stacktrace property is a rendered template. It is escaped when serialized into the store #} + 
{{ query.stacktrace|safe }}
{% endif %} {% if query.template_info %} diff --git a/debug_toolbar/toolbar.py b/debug_toolbar/toolbar.py index d15e62b65..217bebcf9 100644 --- a/debug_toolbar/toolbar.py +++ b/debug_toolbar/toolbar.py @@ -34,7 +34,7 @@ def __init__(self, request, get_response): self._panels[panel.panel_id] = panel self.stats = {} self.server_timing_stats = {} - self.store_id = None + self.store_id = uuid.uuid4().hex # Manage panels @@ -90,10 +90,6 @@ def should_render_panels(self): return render_panels def store(self): - # Store already exists. - if self.store_id: - return - self.store_id = uuid.uuid4().hex store.set(self.store_id, self) # Manually implement class-level caching of panel classes and url patterns diff --git a/tests/panels/test_history.py b/tests/panels/test_history.py index 02bd9e8d1..c02e2895f 100644 --- a/tests/panels/test_history.py +++ b/tests/panels/test_history.py @@ -25,7 +25,7 @@ def test_post(self): response = self.panel.process_request(self.request) self.panel.generate_stats(self.request, response) data = self.panel.get_stats()["data"] - self.assertEqual(data["foo"], "bar") + self.assertEqual(data["foo"], ["bar"]) def test_post_json(self): for data, expected_stats_data in ( diff --git a/tests/test_integration.py b/tests/test_integration.py index bb8eae52c..7e41faf1a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -70,17 +70,17 @@ def test_url_resolving_positional(self): stats = self._resolve_stats("/resolving1/a/b/") self.assertEqual(stats["view_urlname"], "positional-resolving") self.assertEqual(stats["view_func"], "tests.views.resolving_view") - self.assertEqual(stats["view_args"], ("a", "b")) + self.assertEqual(stats["view_args"], ["a", "b"]) self.assertEqual(stats["view_kwargs"], {}) def test_url_resolving_named(self): stats = self._resolve_stats("/resolving2/a/b/") - self.assertEqual(stats["view_args"], ()) + self.assertEqual(stats["view_args"], []) self.assertEqual(stats["view_kwargs"], {"arg1": "a", "arg2": "b"}) def test_url_resolving_mixed(self): stats = self._resolve_stats("/resolving3/a/") - self.assertEqual(stats["view_args"], ("a",)) + self.assertEqual(stats["view_args"], ["a"]) self.assertEqual(stats["view_kwargs"], {"arg2": "default"}) def test_url_resolving_bad(self): @@ -210,7 +210,7 @@ def test_middleware_render_toolbar_json(self): """Verify the toolbar is rendered and data is stored for a json request.""" self.assertEqual(len(store.all()), 0) - data = {"foo": "bar"} + data = {"foo": "bar", "spam[]": ["eggs", "ham"]} response = self.client.get("/json_view/", data, content_type="application/json") self.assertEqual(response.status_code, 200) self.assertEqual(response.content.decode("utf-8"), '{"foo": "bar"}') @@ -219,7 +219,7 @@ def test_middleware_render_toolbar_json(self): toolbar = list(store.all())[0][1] self.assertEqual( toolbar.get_panel_by_id("HistoryPanel").get_stats()["data"], - {"foo": ["bar"]}, + {"foo": ["bar"], "spam[]": ["eggs", "ham"]}, ) def test_template_source_checks_show_toolbar(self): From 54ff7305fe9a23142d93218552d37bb309e04210 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Sat, 9 Jan 2021 10:57:19 -0600 Subject: [PATCH 3/4] Support stats only version of toolbar. Removes storing the entire toolbar in the Store. --- debug_toolbar/panels/__init__.py | 14 ++++++-- debug_toolbar/panels/history/panel.py | 12 +++---- debug_toolbar/panels/history/views.py | 14 +++++--- debug_toolbar/store.py | 36 +++++++++---------- .../debug_toolbar/panels/history.html | 2 +- .../debug_toolbar/panels/history_tr.html | 10 +++--- debug_toolbar/toolbar.py | 13 ++++--- debug_toolbar/views.py | 6 ++-- tests/base.py | 2 +- tests/panels/test_history.py | 34 ++++++++++++------ tests/test_integration.py | 13 ++++--- 11 files changed, 94 insertions(+), 62 deletions(-) diff --git a/debug_toolbar/panels/__init__.py b/debug_toolbar/panels/__init__.py index f3d5c6c76..ee2467d1b 100644 --- a/debug_toolbar/panels/__init__.py +++ b/debug_toolbar/panels/__init__.py @@ -38,7 +38,13 @@ def enabled(self): else: default = "on" # The user's cookies should override the default value - return self.toolbar.request.COOKIES.get("djdt" + self.panel_id, default) == "on" + if self.toolbar.request is not None: + return ( + self.toolbar.request.COOKIES.get("djdt" + self.panel_id, default) + == "on" + ) + else: + return bool(store.panel(self.toolbar.store_id, self.panel_id)) # Titles and content @@ -151,7 +157,8 @@ def disable_instrumentation(self): # Store and retrieve stats (shared between panels for no good reason) - def deserialize_stats(self, data): + @classmethod + def deserialize_stats(cls, data): """ Deserialize stats coming from the store. @@ -159,7 +166,8 @@ def deserialize_stats(self, data): """ return data - def serialize_stats(self, stats): + @classmethod + def serialize_stats(cls, stats): """ Serialize stats for the store. diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py index f311f036c..e9e65942e 100644 --- a/debug_toolbar/panels/history/panel.py +++ b/debug_toolbar/panels/history/panel.py @@ -81,10 +81,10 @@ def content(self): Fetch every store for the toolbar and include it in the template. """ - stores = OrderedDict() - for id, toolbar in reversed(store.all()): - stores[id] = { - "toolbar": toolbar, + histories = OrderedDict() + for id in reversed(store.ids()): + histories[id] = { + "stats": self.deserialize_stats(store.panel(id, self.panel_id)), "form": SignedDataForm( initial=HistoryStoreForm(initial={"store_id": id}).initial ), @@ -94,10 +94,10 @@ def content(self): self.template, { "current_store_id": self.toolbar.store_id, - "stores": stores, + "histories": histories, "refresh_form": SignedDataForm( initial=HistoryStoreForm( - initial={"store_id": self.toolbar.store_id} + initial={"store_id": str(self.toolbar.store_id)} ).initial ), }, diff --git a/debug_toolbar/panels/history/views.py b/debug_toolbar/panels/history/views.py index bcfc4a8c1..e69fbc302 100644 --- a/debug_toolbar/panels/history/views.py +++ b/debug_toolbar/panels/history/views.py @@ -5,6 +5,7 @@ from debug_toolbar.forms import SignedDataForm from debug_toolbar.panels.history.forms import HistoryStoreForm from debug_toolbar.store import store +from debug_toolbar.toolbar import stats_only_toolbar @require_show_toolbar @@ -15,7 +16,8 @@ def history_sidebar(request, verified_data): if form.is_valid(): store_id = form.cleaned_data["store_id"] - toolbar = store.get(store_id) + toolbar = stats_only_toolbar(store_id) + context = {} if toolbar is None: # When the store_id has been popped already due to @@ -25,6 +27,7 @@ def history_sidebar(request, verified_data): if not panel.is_historical: continue panel_context = {"panel": panel} + context[panel.panel_id] = { "button": render_to_string( "debug_toolbar/includes/panel_button.html", panel_context @@ -45,7 +48,8 @@ def history_refresh(request, verified_data): if form.is_valid(): requests = [] - for id, toolbar in list(reversed(store.all())): + for id in reversed(store.ids()): + toolbar = stats_only_toolbar(id) requests.append( { "id": id, @@ -53,8 +57,10 @@ def history_refresh(request, verified_data): "debug_toolbar/panels/history_tr.html", { "id": id, - "store_context": { - "toolbar": toolbar, + "history": { + "stats": toolbar.get_panel_by_id( + "HistoryPanel" + ).get_stats(), "form": SignedDataForm( initial=HistoryStoreForm( initial={"store_id": id} diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index 849f6b47a..d873fd8ee 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -1,5 +1,5 @@ import json -from collections import OrderedDict, defaultdict +from collections import defaultdict from django.core.serializers.json import DjangoJSONEncoder from django.utils.module_loading import import_string @@ -31,47 +31,47 @@ class BaseStore: config = dt_settings.get_config().copy() @classmethod - def get(cls, store_id): + def ids(cls): raise NotImplementedError @classmethod - def all(cls): + def exists(cls, store_id): raise NotImplementedError @classmethod - def set(cls, store_id, toolbar): + def set(cls, store_id): raise NotImplementedError @classmethod def delete(cls, store_id): raise NotImplementedError - @classmethod - def record_stats(cls, store_id, panel_id, stats): - raise NotImplementedError - class MemoryStore(BaseStore): - _store = OrderedDict() + _ids = list() _stats = defaultdict(dict) @classmethod - def get(cls, store_id): - return cls._store.get(store_id) + def ids(cls): + return cls._ids @classmethod - def all(cls): - return cls._store.items() + def exists(cls, store_id): + return store_id in cls._ids @classmethod - def set(cls, store_id, toolbar): - cls._store[store_id] = toolbar - for _ in range(cls.config["RESULTS_CACHE_SIZE"], len(cls._store)): - cls._store.popitem(last=False) + def set(cls, store_id): + if store_id not in cls._ids: + cls._ids.append(store_id) + if len(cls._ids) > cls.config["RESULTS_CACHE_SIZE"]: + cls.delete(cls._ids[0]) @classmethod def delete(cls, store_id): - del cls._store[store_id] + if store_id in cls._stats: + del cls._stats[store_id] + if store_id in cls._ids: + cls._ids.remove(store_id) @classmethod def save_panel(cls, store_id, panel_id, stats=None): diff --git a/debug_toolbar/templates/debug_toolbar/panels/history.html b/debug_toolbar/templates/debug_toolbar/panels/history.html index 84c6cb5bd..af95123e8 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/history.html +++ b/debug_toolbar/templates/debug_toolbar/panels/history.html @@ -15,7 +15,7 @@ - {% for id, store_context in stores.items %} + {% for id, history in histories.items %} {% include "debug_toolbar/panels/history_tr.html" %} {% endfor %} diff --git a/debug_toolbar/templates/debug_toolbar/panels/history_tr.html b/debug_toolbar/templates/debug_toolbar/panels/history_tr.html index 31793472a..9029cf522 100644 --- a/debug_toolbar/templates/debug_toolbar/panels/history_tr.html +++ b/debug_toolbar/templates/debug_toolbar/panels/history_tr.html @@ -1,13 +1,13 @@ {% load i18n %} - {% for key, value in store_context.toolbar.stats.HistoryPanel.data.items %} + {% for key, value in history.stats.data.items %} @@ -43,7 +43,7 @@ diff --git a/debug_toolbar/toolbar.py b/debug_toolbar/toolbar.py index 217bebcf9..38c900f95 100644 --- a/debug_toolbar/toolbar.py +++ b/debug_toolbar/toolbar.py @@ -18,7 +18,8 @@ class DebugToolbar: - def __init__(self, request, get_response): + def __init__(self, request, get_response, store_id=None): + self.store_id = store_id or uuid.uuid4().hex self.request = request self.config = dt_settings.get_config().copy() panels = [] @@ -34,7 +35,6 @@ def __init__(self, request, get_response): self._panels[panel.panel_id] = panel self.stats = {} self.server_timing_stats = {} - self.store_id = uuid.uuid4().hex # Manage panels @@ -64,8 +64,7 @@ def render_toolbar(self): """ Renders the overall Toolbar with panels inside. """ - if not self.should_render_panels(): - self.store() + self.store() try: context = {"toolbar": self} return render_to_string("debug_toolbar/base.html", context) @@ -90,7 +89,7 @@ def should_render_panels(self): return render_panels def store(self): - store.set(self.store_id, self) + store.set(self.store_id) # Manually implement class-level caching of panel classes and url patterns # because it's more obvious than going through an abstraction. @@ -141,5 +140,9 @@ def is_toolbar_request(cls, request): return resolver_match.namespaces and resolver_match.namespaces[-1] == app_name +def stats_only_toolbar(store_id): + return DebugToolbar(request=None, get_response=lambda r: r, store_id=store_id) + + app_name = "djdt" urlpatterns = DebugToolbar.get_urls() diff --git a/debug_toolbar/views.py b/debug_toolbar/views.py index 5624ca6f4..12666f8c3 100644 --- a/debug_toolbar/views.py +++ b/debug_toolbar/views.py @@ -4,13 +4,14 @@ from debug_toolbar.decorators import require_show_toolbar from debug_toolbar.store import store +from debug_toolbar.toolbar import stats_only_toolbar @require_show_toolbar def render_panel(request): """Render the contents of a panel""" - toolbar = store.get(request.GET["store_id"]) - if toolbar is None: + store_id = request.GET["store_id"] + if not store.exists(store_id): content = _( "Data for this panel isn't available anymore. " "Please reload the page and retry." @@ -18,6 +19,7 @@ def render_panel(request): content = "

%s

" % escape(content) scripts = [] else: + toolbar = stats_only_toolbar(store_id) panel = toolbar.get_panel_by_id(request.GET["panel_id"]) content = panel.content scripts = panel.scripts diff --git a/tests/base.py b/tests/base.py index a35ae4bcf..a9fddcdb0 100644 --- a/tests/base.py +++ b/tests/base.py @@ -53,6 +53,6 @@ def setUp(self): # The HistoryPanel keeps track of previous stores in memory. # This bleeds into other tests and violates their idempotency. # Clear the store before each test. - for key, _ in list(store.all()): + for key in list(store.ids()): store.delete(key) super().setUp() diff --git a/tests/panels/test_history.py b/tests/panels/test_history.py index c02e2895f..ef323dd63 100644 --- a/tests/panels/test_history.py +++ b/tests/panels/test_history.py @@ -5,6 +5,7 @@ from debug_toolbar.forms import SignedDataForm from debug_toolbar.store import store +from debug_toolbar.toolbar import stats_only_toolbar from ..base import BaseTestCase, IntegrationTestCase @@ -83,14 +84,14 @@ class HistoryViewsTestCase(IntegrationTestCase): def test_history_panel_integration_content(self): """Verify the history panel's content renders properly..""" - self.assertEqual(len(store.all()), 0) + self.assertEqual(len(store.ids()), 0) data = {"foo": "bar"} self.client.get("/json_view/", data, content_type="application/json") # Check the history panel's stats to verify the toolbar rendered properly. - self.assertEqual(len(store.all()), 1) - toolbar = list(store.all())[0][1] + self.assertEqual(len(store.ids()), 1) + toolbar = stats_only_toolbar(store.ids()[0]) content = toolbar.get_panel_by_id("HistoryPanel").content self.assertIn("bar", content) @@ -99,7 +100,7 @@ def test_history_sidebar_invalid(self): self.assertEqual(response.status_code, 400) self.client.get("/json_view/") - store_id = list(store.all())[0][0] + store_id = store.ids()[0] data = {"signed": SignedDataForm.sign({"store_id": store_id}) + "invalid"} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 400) @@ -107,13 +108,26 @@ def test_history_sidebar_invalid(self): def test_history_sidebar_hash(self): """Validate the hashing mechanism.""" self.client.get("/json_view/") - store_id = list(store.all())[0][0] + store_id = store.ids()[0] data = {"signed": SignedDataForm.sign({"store_id": store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) self.assertEqual( - set(response.json()), - self.PANEL_KEYS, + list(response.json().keys()), + [ + "VersionsPanel", + "TimerPanel", + "SettingsPanel", + "HeadersPanel", + "RequestPanel", + "SQLPanel", + "StaticFilesPanel", + "TemplatesPanel", + "CachePanel", + "SignalsPanel", + "LoggingPanel", + "ProfilingPanel", + ], ) @override_settings( @@ -122,7 +136,7 @@ def test_history_sidebar_hash(self): def test_history_sidebar_expired_store_id(self): """Validate the history sidebar view.""" self.client.get("/json_view/") - store_id = list(store.all())[0][0] + store_id = list(store.ids())[0] data = {"signed": SignedDataForm.sign({"store_id": store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) self.assertEqual(response.status_code, 200) @@ -142,7 +156,7 @@ def test_history_sidebar_expired_store_id(self): ) # Querying with latest store_id - latest_store_id = list(store.all())[-1][0] + latest_store_id = store.ids()[-1] self.assertNotEqual(latest_store_id, store_id) data = {"signed": SignedDataForm.sign({"store_id": latest_store_id})} response = self.client.get(reverse("djdt:history_sidebar"), data=data) @@ -170,7 +184,7 @@ def test_history_refresh(self): data = response.json() self.assertEqual(len(data["requests"]), 1) - store_id = list(store.all())[0][0] + store_id = store.ids()[0] signature = SignedDataForm.sign({"store_id": store_id}) self.assertIn(html.escape(signature), data["requests"][0]["content"]) diff --git a/tests/test_integration.py b/tests/test_integration.py index 7e41faf1a..8c99176d8 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -17,7 +17,7 @@ from debug_toolbar.middleware import DebugToolbarMiddleware, show_toolbar from debug_toolbar.panels import Panel from debug_toolbar.store import store -from debug_toolbar.toolbar import DebugToolbar +from debug_toolbar.toolbar import DebugToolbar, stats_only_toolbar from .base import BaseTestCase, IntegrationTestCase from .views import regular_view @@ -190,7 +190,6 @@ def get_response(request): return HttpResponse() toolbar = DebugToolbar(rf.get("/"), get_response) - toolbar.store() url = "/__debug__/render_panel/" data = {"store_id": toolbar.store_id, "panel_id": "VersionsPanel"} @@ -208,15 +207,16 @@ def get_response(request): def test_middleware_render_toolbar_json(self): """Verify the toolbar is rendered and data is stored for a json request.""" - self.assertEqual(len(store.all()), 0) + self.assertEqual(len(store.ids()), 0) data = {"foo": "bar", "spam[]": ["eggs", "ham"]} response = self.client.get("/json_view/", data, content_type="application/json") self.assertEqual(response.status_code, 200) self.assertEqual(response.content.decode("utf-8"), '{"foo": "bar"}') # Check the history panel's stats to verify the toolbar rendered properly. - self.assertEqual(len(store.all()), 1) - toolbar = list(store.all())[0][1] + self.assertEqual(len(store.ids()), 1) + toolbar = stats_only_toolbar(store.ids()[0]) + self.assertEqual( toolbar.get_panel_by_id("HistoryPanel").get_stats()["data"], {"foo": ["bar"], "spam[]": ["eggs", "ham"]}, @@ -522,10 +522,9 @@ def test_rerender_on_history_switch(self): self.assertNotIn("1 query", current_button_panel) self.assertIn("1 query", previous_button_panel) - @override_settings(DEBUG_TOOLBAR_CONFIG={"RESULTS_CACHE_SIZE": 0}) def test_expired_store(self): original_value = store.config["RESULTS_CACHE_SIZE"] - store.config["RESULTS_CACHE_SIZE"] = 0 + store.config["RESULTS_CACHE_SIZE"] = 1 self.get("/regular/basic/") version_panel = self.selenium.find_element_by_id("VersionsPanel") From 3fb274563b5edf5239b9e044578778339f24e594 Mon Sep 17 00:00:00 2001 From: Tim Schilling Date: Fri, 15 Jan 2021 13:32:25 -0600 Subject: [PATCH 4/4] WIP --- Makefile | 2 +- debug_toolbar/db_store.py | 50 ++++++++++++++++++++ debug_toolbar/migrations/0001_initial.py | 58 ++++++++++++++++++++++++ debug_toolbar/migrations/__init__.py | 0 debug_toolbar/models.py | 13 ++++++ debug_toolbar/panels/__init__.py | 10 ++-- debug_toolbar/panels/cache.py | 5 +- debug_toolbar/panels/history/panel.py | 18 ++++---- debug_toolbar/panels/history/views.py | 4 +- debug_toolbar/panels/sql/panel.py | 14 ++++-- debug_toolbar/panels/staticfiles.py | 7 +-- debug_toolbar/panels/timer.py | 6 +-- debug_toolbar/store.py | 5 +- debug_toolbar/toolbar.py | 4 +- debug_toolbar/views.py | 4 +- tests/base.py | 9 ++-- tests/commands/test_debugsqlshell.py | 2 + tests/panels/test_history.py | 7 ++- tests/settings.py | 11 +++++ tests/test_integration.py | 23 +++++++--- 20 files changed, 208 insertions(+), 44 deletions(-) create mode 100644 debug_toolbar/db_store.py create mode 100644 debug_toolbar/migrations/0001_initial.py create mode 100644 debug_toolbar/migrations/__init__.py create mode 100644 debug_toolbar/models.py diff --git a/Makefile b/Makefile index 5b5ca4d76..2131ffbe8 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ package-lock.json: package.json touch $@ test: - DJANGO_SETTINGS_MODULE=tests.settings \ + DB_BACKEND=sqlite3 DB_NAME=":memory:" DJANGO_SETTINGS_MODULE=tests.settings \ python -m django test $${TEST_ARGS:-tests} test_selenium: diff --git a/debug_toolbar/db_store.py b/debug_toolbar/db_store.py new file mode 100644 index 000000000..3babb92f9 --- /dev/null +++ b/debug_toolbar/db_store.py @@ -0,0 +1,50 @@ +from debug_toolbar import store +from debug_toolbar.models import PanelStore, ToolbarStore + + +class DBStore(store.BaseStore): + @classmethod + def ids(cls): + return ( + ToolbarStore.objects.using("debug_toolbar") + .values_list("key", flat=True) + .order_by("created") + ) + + @classmethod + def exists(cls, store_id): + return ToolbarStore.objects.using("debug_toolbar").filter(key=store_id).exists() + + @classmethod + def set(cls, store_id): + _, created = ToolbarStore.objects.using("debug_toolbar").get_or_create( + key=store_id + ) + if ( + created + and ToolbarStore.objects.using("debug_toolbar").all().count() + > cls.config["RESULTS_CACHE_SIZE"] + ): + ToolbarStore.objects.using("debug_toolbar").earliest("created").delete() + + @classmethod + def delete(cls, store_id): + ToolbarStore.objects.using("debug_toolbar").filter(key=store_id).delete() + + @classmethod + def save_panel(cls, store_id, panel_id, stats=None): + toolbar, _ = ToolbarStore.objects.using("debug_toolbar").get_or_create( + key=store_id + ) + toolbar.panelstore_set.update_or_create( + panel=panel_id, defaults={"data": store.serialize(stats)} + ) + + @classmethod + def panel(cls, store_id, panel_id): + panel = ( + PanelStore.objects.using("debug_toolbar") + .filter(toolbar__key=store_id, panel=panel_id) + .first() + ) + return {} if not panel else store.deserialize(panel.data) diff --git a/debug_toolbar/migrations/0001_initial.py b/debug_toolbar/migrations/0001_initial.py new file mode 100644 index 000000000..ec6c8c770 --- /dev/null +++ b/debug_toolbar/migrations/0001_initial.py @@ -0,0 +1,58 @@ +# Generated by Django 3.1.5 on 2021-01-09 17:02 +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="ToolbarStore", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("key", models.CharField(max_length=64, unique=True)), + ], + ), + migrations.CreateModel( + name="PanelStore", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("panel", models.CharField(max_length=128)), + ("data", models.TextField()), + ( + "toolbar", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="debug_toolbar.toolbarstore", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="panelstore", + constraint=models.UniqueConstraint( + fields=("toolbar", "panel"), name="unique_toolbar_panel" + ), + ), + ] diff --git a/debug_toolbar/migrations/__init__.py b/debug_toolbar/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug_toolbar/models.py b/debug_toolbar/models.py new file mode 100644 index 000000000..1adf0f32e --- /dev/null +++ b/debug_toolbar/models.py @@ -0,0 +1,13 @@ +from django.db import models + + +class ToolbarStore(models.Model): + created = models.DateTimeField(auto_now_add=True) + key = models.CharField(max_length=64, unique=True) + + +class PanelStore(models.Model): + created = models.DateTimeField(auto_now_add=True) + toolbar = models.ForeignKey(ToolbarStore, on_delete=models.CASCADE) + panel = models.CharField(max_length=128) + data = models.TextField() diff --git a/debug_toolbar/panels/__init__.py b/debug_toolbar/panels/__init__.py index ee2467d1b..2f0c6dbe1 100644 --- a/debug_toolbar/panels/__init__.py +++ b/debug_toolbar/panels/__init__.py @@ -1,7 +1,7 @@ from django.template.loader import render_to_string from debug_toolbar import settings as dt_settings -from debug_toolbar.store import store +from debug_toolbar.store import get_store from debug_toolbar.utils import get_name_from_obj @@ -44,7 +44,7 @@ def enabled(self): == "on" ) else: - return bool(store.panel(self.toolbar.store_id, self.panel_id)) + return bool(get_store().panel(self.toolbar.store_id, self.panel_id)) # Titles and content @@ -182,7 +182,7 @@ def record_stats(self, stats): Each call to ``record_stats`` updates the statistics dictionary. """ self.toolbar.stats.setdefault(self.panel_id, {}).update(stats) - store.save_panel( + get_store().save_panel( self.toolbar.store_id, self.panel_id, self.serialize_stats(stats) ) @@ -190,7 +190,9 @@ def get_stats(self): """ Access data stored by the panel. Returns a :class:`dict`. """ - return self.deserialize_stats(store.panel(self.toolbar.store_id, self.panel_id)) + return self.deserialize_stats( + get_store().panel(self.toolbar.store_id, self.panel_id) + ) def record_server_timing(self, key, title, value): """ diff --git a/debug_toolbar/panels/cache.py b/debug_toolbar/panels/cache.py index 87e09c838..32dd0ba62 100644 --- a/debug_toolbar/panels/cache.py +++ b/debug_toolbar/panels/cache.py @@ -246,14 +246,15 @@ def _store_call_info( @property def nav_subtitle(self): - cache_calls = len(self.calls) + stats = self.get_stats() + cache_calls = len(stats["calls"]) return ( ngettext( "%(cache_calls)d call in %(time).2fms", "%(cache_calls)d calls in %(time).2fms", cache_calls, ) - % {"cache_calls": cache_calls, "time": self.total_time} + % {"cache_calls": cache_calls, "time": stats["total_time"]} ) @property diff --git a/debug_toolbar/panels/history/panel.py b/debug_toolbar/panels/history/panel.py index e9e65942e..3e328190d 100644 --- a/debug_toolbar/panels/history/panel.py +++ b/debug_toolbar/panels/history/panel.py @@ -12,7 +12,7 @@ from debug_toolbar.panels import Panel from debug_toolbar.panels.history import views from debug_toolbar.panels.history.forms import HistoryStoreForm -from debug_toolbar.store import store +from debug_toolbar.store import get_store class HistoryPanel(Panel): @@ -82,13 +82,15 @@ def content(self): Fetch every store for the toolbar and include it in the template. """ histories = OrderedDict() - for id in reversed(store.ids()): - histories[id] = { - "stats": self.deserialize_stats(store.panel(id, self.panel_id)), - "form": SignedDataForm( - initial=HistoryStoreForm(initial={"store_id": id}).initial - ), - } + for id in reversed(get_store().ids()): + stats = self.deserialize_stats(get_store().panel(id, self.panel_id)) + if stats: + histories[id] = { + "stats": stats, + "form": SignedDataForm( + initial=HistoryStoreForm(initial={"store_id": str(id)}).initial + ), + } return render_to_string( self.template, diff --git a/debug_toolbar/panels/history/views.py b/debug_toolbar/panels/history/views.py index e69fbc302..06b1a46b9 100644 --- a/debug_toolbar/panels/history/views.py +++ b/debug_toolbar/panels/history/views.py @@ -4,7 +4,7 @@ from debug_toolbar.decorators import require_show_toolbar, signed_data_view from debug_toolbar.forms import SignedDataForm from debug_toolbar.panels.history.forms import HistoryStoreForm -from debug_toolbar.store import store +from debug_toolbar.store import get_store from debug_toolbar.toolbar import stats_only_toolbar @@ -48,7 +48,7 @@ def history_refresh(request, verified_data): if form.is_valid(): requests = [] - for id in reversed(store.ids()): + for id in reversed(get_store().ids()): toolbar = stats_only_toolbar(id) requests.append( { diff --git a/debug_toolbar/panels/sql/panel.py b/debug_toolbar/panels/sql/panel.py index f8b92a5bd..37d0a85fd 100644 --- a/debug_toolbar/panels/sql/panel.py +++ b/debug_toolbar/panels/sql/panel.py @@ -110,18 +110,20 @@ def record(self, alias, **kwargs): @property def nav_subtitle(self): + stats = self.get_stats() + num_queries = len(stats["queries"]) return ngettext( "%(query_count)d query in %(sql_time).2fms", "%(query_count)d queries in %(sql_time).2fms", - self._num_queries, + num_queries, ) % { - "query_count": self._num_queries, - "sql_time": self._sql_time, + "query_count": num_queries, + "sql_time": stats["sql_time"], } @property def title(self): - count = len(self._databases) + count = len(self.get_stats()["databases"]) return ( ngettext( "SQL queries from %(count)d connection", @@ -144,10 +146,14 @@ def get_urls(cls): def enable_instrumentation(self): # This is thread-safe because database connections are thread-local. for connection in connections.all(): + if connection.alias == "debug_toolbar": + continue wrap_cursor(connection, self) def disable_instrumentation(self): for connection in connections.all(): + if connection.alias == "debug_toolbar": + continue unwrap_cursor(connection) def generate_stats(self, request, response): diff --git a/debug_toolbar/panels/staticfiles.py b/debug_toolbar/panels/staticfiles.py index d90b6501a..243569ece 100644 --- a/debug_toolbar/panels/staticfiles.py +++ b/debug_toolbar/panels/staticfiles.py @@ -82,9 +82,10 @@ class StaticFilesPanel(panels.Panel): @property def title(self): + stats = self.get_stats() return _("Static files (%(num_found)s found, %(num_used)s used)") % { - "num_found": self.num_found, - "num_used": self.num_used, + "num_found": stats["num_found"], + "num_used": stats["num_used"], } def __init__(self, *args, **kwargs): @@ -107,7 +108,7 @@ def num_used(self): @property def nav_subtitle(self): - num_used = self.num_used + num_used = self.get_stats()["num_used"] return ngettext( "%(num_used)s file used", "%(num_used)s files used", num_used ) % {"num_used": num_used} diff --git a/debug_toolbar/panels/timer.py b/debug_toolbar/panels/timer.py index 801c9c6fd..dff323c32 100644 --- a/debug_toolbar/panels/timer.py +++ b/debug_toolbar/panels/timer.py @@ -19,11 +19,9 @@ class TimerPanel(Panel): def nav_subtitle(self): stats = self.get_stats() - if hasattr(self, "_start_rusage"): - utime = self._end_rusage.ru_utime - self._start_rusage.ru_utime - stime = self._end_rusage.ru_stime - self._start_rusage.ru_stime + if stats.get("utime"): return _("CPU: %(cum)0.2fms (%(total)0.2fms)") % { - "cum": (utime + stime) * 1000.0, + "cum": stats["utime"], "total": stats["total_time"], } elif "total_time" in stats: diff --git a/debug_toolbar/store.py b/debug_toolbar/store.py index d873fd8ee..a14059c4e 100644 --- a/debug_toolbar/store.py +++ b/debug_toolbar/store.py @@ -63,7 +63,7 @@ def exists(cls, store_id): def set(cls, store_id): if store_id not in cls._ids: cls._ids.append(store_id) - if len(cls._ids) > cls.config["RESULTS_CACHE_SIZE"]: + for _ in range(len(cls._ids) - cls.config["RESULTS_CACHE_SIZE"]): cls.delete(cls._ids[0]) @classmethod @@ -86,4 +86,5 @@ def panel(cls, store_id, panel_id): return {} if data is None else deserialize(data) -store = import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) +def get_store(): + return import_string(dt_settings.get_config()["TOOLBAR_STORE_CLASS"]) diff --git a/debug_toolbar/toolbar.py b/debug_toolbar/toolbar.py index 38c900f95..56483674a 100644 --- a/debug_toolbar/toolbar.py +++ b/debug_toolbar/toolbar.py @@ -14,7 +14,7 @@ from django.utils.module_loading import import_string from debug_toolbar import settings as dt_settings -from debug_toolbar.store import store +from debug_toolbar.store import get_store class DebugToolbar: @@ -89,7 +89,7 @@ def should_render_panels(self): return render_panels def store(self): - store.set(self.store_id) + get_store().set(self.store_id) # Manually implement class-level caching of panel classes and url patterns # because it's more obvious than going through an abstraction. diff --git a/debug_toolbar/views.py b/debug_toolbar/views.py index 12666f8c3..c05b7305c 100644 --- a/debug_toolbar/views.py +++ b/debug_toolbar/views.py @@ -3,7 +3,7 @@ from django.utils.translation import gettext as _ from debug_toolbar.decorators import require_show_toolbar -from debug_toolbar.store import store +from debug_toolbar.store import get_store from debug_toolbar.toolbar import stats_only_toolbar @@ -11,7 +11,7 @@ def render_panel(request): """Render the contents of a panel""" store_id = request.GET["store_id"] - if not store.exists(store_id): + if not get_store().exists(store_id): content = _( "Data for this panel isn't available anymore. " "Please reload the page and retry." diff --git a/tests/base.py b/tests/base.py index a9fddcdb0..ca3e5f1d8 100644 --- a/tests/base.py +++ b/tests/base.py @@ -2,13 +2,14 @@ from django.http import HttpResponse from django.test import RequestFactory, TestCase -from debug_toolbar.store import store +from debug_toolbar.store import get_store from debug_toolbar.toolbar import DebugToolbar rf = RequestFactory() class BaseTestCase(TestCase): + databases = {"default", "debug_toolbar"} panel_id = None def setUp(self): @@ -49,10 +50,12 @@ def assertValidHTML(self, content, msg=None): class IntegrationTestCase(TestCase): """Base TestCase for tests involving clients making requests.""" + databases = {"default", "debug_toolbar"} + def setUp(self): # The HistoryPanel keeps track of previous stores in memory. # This bleeds into other tests and violates their idempotency. # Clear the store before each test. - for key in list(store.ids()): - store.delete(key) + for key in list(get_store().ids()): + get_store().delete(key) super().setUp() diff --git a/tests/commands/test_debugsqlshell.py b/tests/commands/test_debugsqlshell.py index 9520d0dd8..d5e8c03ae 100644 --- a/tests/commands/test_debugsqlshell.py +++ b/tests/commands/test_debugsqlshell.py @@ -16,6 +16,8 @@ @override_settings(DEBUG=True) class DebugSQLShellTestCase(TestCase): + databases = {"default", "debug_toolbar"} + def setUp(self): self.original_wrapper = base_module.CursorDebugWrapper # Since debugsqlshell monkey-patches django.db.backends.utils, we can diff --git a/tests/panels/test_history.py b/tests/panels/test_history.py index ef323dd63..165a3ec8a 100644 --- a/tests/panels/test_history.py +++ b/tests/panels/test_history.py @@ -4,7 +4,7 @@ from django.urls import resolve, reverse from debug_toolbar.forms import SignedDataForm -from debug_toolbar.store import store +from debug_toolbar.store import get_store from debug_toolbar.toolbar import stats_only_toolbar from ..base import BaseTestCase, IntegrationTestCase @@ -84,6 +84,7 @@ class HistoryViewsTestCase(IntegrationTestCase): def test_history_panel_integration_content(self): """Verify the history panel's content renders properly..""" + store = get_store() self.assertEqual(len(store.ids()), 0) data = {"foo": "bar"} @@ -96,6 +97,7 @@ def test_history_panel_integration_content(self): self.assertIn("bar", content) def test_history_sidebar_invalid(self): + store = get_store() response = self.client.get(reverse("djdt:history_sidebar")) self.assertEqual(response.status_code, 400) @@ -107,6 +109,7 @@ def test_history_sidebar_invalid(self): def test_history_sidebar_hash(self): """Validate the hashing mechanism.""" + store = get_store() self.client.get("/json_view/") store_id = store.ids()[0] data = {"signed": SignedDataForm.sign({"store_id": store_id})} @@ -135,6 +138,7 @@ def test_history_sidebar_hash(self): ) def test_history_sidebar_expired_store_id(self): """Validate the history sidebar view.""" + store = get_store() self.client.get("/json_view/") store_id = list(store.ids())[0] data = {"signed": SignedDataForm.sign({"store_id": store_id})} @@ -177,6 +181,7 @@ def test_history_refresh_invalid_signature(self): def test_history_refresh(self): """Verify refresh history response has request variables.""" + store = get_store() self.client.get("/json_view/", {"foo": "bar"}, content_type="application/json") data = {"signed": SignedDataForm.sign({"store_id": "foo"})} response = self.client.get(reverse("djdt:history_refresh"), data=data) diff --git a/tests/settings.py b/tests/settings.py index 2a4b5e68c..5681bf948 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -92,6 +92,17 @@ "USER": "default_test", }, }, + "debug_toolbar": { + "ENGINE": "django.db.backends.%s" % os.getenv("DB_BACKEND"), + "NAME": "debug_toolbar_store", + "USER": os.getenv("DB_USER"), + "PASSWORD": os.getenv("DB_PASSWORD"), + "HOST": os.getenv("DB_HOST", ""), + "PORT": os.getenv("DB_PORT", ""), + "TEST": { + "USER": "default_test", + }, + }, } DEFAULT_AUTO_FIELD = "django.db.models.AutoField" diff --git a/tests/test_integration.py b/tests/test_integration.py index 8c99176d8..3402a8952 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,6 +1,7 @@ import os import re import unittest +import uuid import django import html5lib @@ -16,7 +17,7 @@ from debug_toolbar.forms import SignedDataForm from debug_toolbar.middleware import DebugToolbarMiddleware, show_toolbar from debug_toolbar.panels import Panel -from debug_toolbar.store import store +from debug_toolbar.store import get_store from debug_toolbar.toolbar import DebugToolbar, stats_only_toolbar from .base import BaseTestCase, IntegrationTestCase @@ -207,6 +208,7 @@ def get_response(request): def test_middleware_render_toolbar_json(self): """Verify the toolbar is rendered and data is stored for a json request.""" + store = get_store() self.assertEqual(len(store.ids()), 0) data = {"foo": "bar", "spam[]": ["eggs", "ham"]} @@ -443,6 +445,8 @@ def test_auth_login_view_without_redirect(self): ) @override_settings(DEBUG=True) class DebugToolbarLiveTestCase(StaticLiveServerTestCase): + databases = {"default", "debug_toolbar"} + @classmethod def setUpClass(cls): super().setUpClass() @@ -503,9 +507,10 @@ def test_basic_jinja(self): ) def test_rerender_on_history_switch(self): self.get("/regular_jinja/basic") + self.selenium.find_element_by_id("HistoryPanel") # Make a new request so the history panel has more than one option. self.get("/execute_sql/") - template_panel = self.selenium.find_element_by_id("HistoryPanel") + history_panel = self.selenium.find_element_by_id("HistoryPanel") # Record the current side panel of buttons for later comparison. previous_button_panel = self.selenium.find_element_by_id( "djDebugPanelList" @@ -514,7 +519,12 @@ def test_rerender_on_history_switch(self): # Click to show the history panel self.selenium.find_element_by_class_name("HistoryPanel").click() # Click to switch back to the jinja page view snapshot - list(template_panel.find_elements_by_css_selector("button"))[-1].click() + list(history_panel.find_elements_by_css_selector("button"))[-1].click() + + self.wait.until( + lambda selenium: self.selenium.find_element_by_id("djDebugPanelList").text + != previous_button_panel + ) current_button_panel = self.selenium.find_element_by_id("djDebugPanelList").text # Verify the button side panels have updated. @@ -523,11 +533,13 @@ def test_rerender_on_history_switch(self): self.assertIn("1 query", previous_button_panel) def test_expired_store(self): - original_value = store.config["RESULTS_CACHE_SIZE"] - store.config["RESULTS_CACHE_SIZE"] = 1 + store = get_store() self.get("/regular/basic/") version_panel = self.selenium.find_element_by_id("VersionsPanel") + for i in range(store.config["RESULTS_CACHE_SIZE"]): + store.set(uuid.uuid4().hex) + # Click to show the version panel self.selenium.find_element_by_class_name("VersionsPanel").click() @@ -536,7 +548,6 @@ def test_expired_store(self): lambda selenium: version_panel.find_element_by_tag_name("p") ) self.assertIn("Data for this panel isn't available anymore.", error.text) - store.config["RESULTS_CACHE_SIZE"] = original_value @override_settings( TEMPLATES=[
- {{ store_context.toolbar.stats.HistoryPanel.time|escape }} + {{ history.stats.time|escape }} -

{{ store_context.toolbar.stats.HistoryPanel.request_method|escape }}

+

{{ history.stats.request_method|escape }}

 -

{{ store_context.toolbar.stats.HistoryPanel.request_url|truncatechars:100|escape }}

+

{{ history.stats.request_url|truncatechars:100|escape }}

 @@ -24,7 +24,7 @@
{{ key|pprint }} {{ value|pprint }} - {{ store_context.form }} + {{ history.form }}