From 74d7dcaa28cf3dc1e682808dfc007349e284a45a Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 01:12:41 +0100 Subject: [PATCH 01/11] stream_parameters utility looks at contents not params --- holoviews/core/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/core/util.py b/holoviews/core/util.py index 3266de497b..b2ecb27981 100644 --- a/holoviews/core/util.py +++ b/holoviews/core/util.py @@ -787,7 +787,7 @@ def stream_parameters(streams, no_duplicates=True, exclude=['name']): If no_duplicates is enabled, a KeyError will be raised if there are parameter name clashes across the streams. """ - param_groups = [s.params().keys() for s in streams] + param_groups = [s.contents.keys() for s in streams] names = [name for group in param_groups for name in group] if no_duplicates: From 7da2ea83c235e634a568fc9d5aff7472530df0ee Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 01:13:17 +0100 Subject: [PATCH 02/11] Added utility to access cache of DynamicMap with streams --- holoviews/core/spaces.py | 4 +++- holoviews/core/traversal.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/holoviews/core/spaces.py b/holoviews/core/spaces.py index 0ea1603961..1cbdbd622e 100644 --- a/holoviews/core/spaces.py +++ b/holoviews/core/spaces.py @@ -477,6 +477,7 @@ def __init__(self, callback, initial_items=None, **params): self.call_mode = self._validate_mode() self.mode = 'bounded' if self.call_mode == 'key' else 'open' + self._stream_cache_lookup = False def _initial_key(self): @@ -672,7 +673,8 @@ def __getitem__(self, key): # Cache lookup try: - if util.dimensionless_contents(self.streams, self.kdims): + if (util.dimensionless_contents(self.streams, self.kdims) and + not self._stream_cache_lookup): raise KeyError('Using dimensionless streams disables DynamicMap cache') cache = super(DynamicMap,self).__getitem__(key) # Return selected cache items in a new DynamicMap diff --git a/holoviews/core/traversal.py b/holoviews/core/traversal.py index 896e69f708..933917634d 100644 --- a/holoviews/core/traversal.py +++ b/holoviews/core/traversal.py @@ -128,3 +128,29 @@ def hierarchical(keys): store1[v2].append(v1) hierarchies.append(store2 if hierarchy else {}) return hierarchies + + +class enable_streams_cache(object): + """ + Context manager which temporarily enables lookup of frame in cache + on a DynamicMap with streams. Allows passing any Dimensioned + object which might contain a DynamicMap and whether to enable the + cache. This allows looking up an item without triggering the + callback. Useful when the object is looked up multiple times as + part of some processing pipeline. + """ + + def __init__(self, obj, enable): + self.obj = obj + self._enable = enable + + def __enter__(self): + self.set_cache_flag(self._enable) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.set_cache_flag(False) + + def set_cache_flag(self, value) + self.obj.traverse(lambda x: setattr(x, '_stream_cache_lookup', value), + ['DynamicMap']) + From 1b376198a99d0edfd881d50277e22ecdee7df9b1 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 01:14:19 +0100 Subject: [PATCH 03/11] Plot.refresh forces an update on all DynamicMaps --- holoviews/core/traversal.py | 2 +- holoviews/plotting/plot.py | 29 ++++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/holoviews/core/traversal.py b/holoviews/core/traversal.py index 933917634d..fe1c32865f 100644 --- a/holoviews/core/traversal.py +++ b/holoviews/core/traversal.py @@ -150,7 +150,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.set_cache_flag(False) - def set_cache_flag(self, value) + def set_cache_flag(self, value): self.obj.traverse(lambda x: setattr(x, '_stream_cache_lookup', value), ['DynamicMap']) diff --git a/holoviews/plotting/plot.py b/holoviews/plotting/plot.py index c8f4e6940f..2448974483 100644 --- a/holoviews/plotting/plot.py +++ b/holoviews/plotting/plot.py @@ -18,6 +18,7 @@ from ..core.options import Store, Compositor, SkipRendering from ..core.overlay import NdOverlay from ..core.spaces import HoloMap, DynamicMap +from ..core.traversal import enable_streams_cache from ..element import Table from .util import (get_dynamic_mode, initialize_sampled, dim_axis_label, attach_streams) @@ -198,6 +199,7 @@ def __init__(self, keys=None, dimensions=None, layout_dimensions=None, self.ranges = {} self.renderer = renderer if renderer else Store.renderers[self.backend].instance() self.comm = None + self._force = True params = {k: v for k, v in params.items() if k in self.params()} @@ -208,6 +210,7 @@ def __getitem__(self, frame): """ Get the state of the Plot for a given frame number. """ + self.force = True if not self.dynamic == 'open' and isinstance(frame, int) and frame > len(self): self.warning("Showing last frame available: %d" % len(self)) if not self.drawn: self.handles['fig'] = self.initialize_plot() @@ -258,6 +261,15 @@ def traverse(self, fn=None, specs=None, full_breadth=True): if not full_breadth: break return accumulator + @property + def force(self): + return self._force + + + @force.setter + def force(self, value): + self.traverse(lambda x: setattr(x, '_force', value)) + def _frame_title(self, key, group_size=2, separator='\n'): """ @@ -481,6 +493,7 @@ def refresh(self, **kwargs): Refreshes the plot by rerendering it and then pushing the updated data if the plot has an associated Comm. """ + self.force = True if self.current_key: self.update(self.current_key) else: @@ -600,7 +613,9 @@ def _get_frame(self, key): self.current_key = key return self.current_frame elif self.dynamic: - key, frame = util.get_dynamic_item(self.hmap, self.dimensions, key) + with enable_streams_cache(self.hmap, not self.force): + key, frame = util.get_dynamic_item(self.hmap, self.dimensions, key) + self.force = False if not isinstance(key, tuple): key = (key,) key_map = dict(zip([d.name for d in self.hmap.kdims], key)) key = tuple(key_map.get(d.name, None) for d in self.dimensions) @@ -898,10 +913,6 @@ def get_extents(self, overlay, ranges): class GenericCompositePlot(DimensionedPlot): def __init__(self, layout, keys=None, dimensions=None, **params): - dynamic, sampled = get_dynamic_mode(layout) - if sampled: - initialize_sampled(layout, dimensions, keys[0]) - if 'uniform' not in params: params['uniform'] = traversal.uniform(layout) @@ -909,6 +920,9 @@ def __init__(self, layout, keys=None, dimensions=None, **params): if top_level: dimensions, keys = traversal.unique_dimkeys(layout) + dynamic, sampled = get_dynamic_mode(layout) + if sampled: + initialize_sampled(layout, dimensions, keys[0]) self.layout = layout super(GenericCompositePlot, self).__init__(keys=keys, dynamic=dynamic, @@ -944,6 +958,11 @@ def _get_frame(self, key): dim_keys = zip([d.name for d in self.dimensions if d in item.dimensions('key')], key) self.current_key = tuple(k[1] for k in dim_keys) + elif item.traverse(lambda x: x, [DynamicMap]): + with enable_streams_cache(item, not self.force): + key, frame = util.get_dynamic_item(item, self.dimensions, key) + layout_frame[path] = frame + continue elif self.uniform: dim_keys = zip([d.name for d in self.dimensions if d in item.dimensions('key')], key) From ea797559c3140dbaf61b8207698c5564ee261281 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 01:33:43 +0100 Subject: [PATCH 04/11] Allowed using cache when drawing initial frame --- holoviews/plotting/plot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/holoviews/plotting/plot.py b/holoviews/plotting/plot.py index 2448974483..0924c20fe8 100644 --- a/holoviews/plotting/plot.py +++ b/holoviews/plotting/plot.py @@ -613,7 +613,7 @@ def _get_frame(self, key): self.current_key = key return self.current_frame elif self.dynamic: - with enable_streams_cache(self.hmap, not self.force): + with enable_streams_cache(self.hmap, not self.force or not self.drawn): key, frame = util.get_dynamic_item(self.hmap, self.dimensions, key) self.force = False if not isinstance(key, tuple): key = (key,) @@ -959,7 +959,7 @@ def _get_frame(self, key): if d in item.dimensions('key')], key) self.current_key = tuple(k[1] for k in dim_keys) elif item.traverse(lambda x: x, [DynamicMap]): - with enable_streams_cache(item, not self.force): + with enable_streams_cache(item, not self.force or not self.drawn): key, frame = util.get_dynamic_item(item, self.dimensions, key) layout_frame[path] = frame continue From 742116f5370b635593e180ed3779d8c8acf79e85 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 11:47:09 +0100 Subject: [PATCH 05/11] Only force DynamicMap lookup on refresh --- holoviews/plotting/plot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/holoviews/plotting/plot.py b/holoviews/plotting/plot.py index 0924c20fe8..bd509e857f 100644 --- a/holoviews/plotting/plot.py +++ b/holoviews/plotting/plot.py @@ -210,7 +210,6 @@ def __getitem__(self, frame): """ Get the state of the Plot for a given frame number. """ - self.force = True if not self.dynamic == 'open' and isinstance(frame, int) and frame > len(self): self.warning("Showing last frame available: %d" % len(self)) if not self.drawn: self.handles['fig'] = self.initialize_plot() From 1edb4092e9c295676a881a74d93067f6e00aadbd Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 13:08:53 +0100 Subject: [PATCH 06/11] Replaced force property with traverse_setter utility --- holoviews/plotting/plot.py | 20 ++++++-------------- holoviews/plotting/util.py | 8 ++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/holoviews/plotting/plot.py b/holoviews/plotting/plot.py index bd509e857f..5357f34380 100644 --- a/holoviews/plotting/plot.py +++ b/holoviews/plotting/plot.py @@ -21,7 +21,7 @@ from ..core.traversal import enable_streams_cache from ..element import Table from .util import (get_dynamic_mode, initialize_sampled, dim_axis_label, - attach_streams) + attach_streams, traverse_setter) class Plot(param.Parameterized): @@ -260,15 +260,6 @@ def traverse(self, fn=None, specs=None, full_breadth=True): if not full_breadth: break return accumulator - @property - def force(self): - return self._force - - - @force.setter - def force(self, value): - self.traverse(lambda x: setattr(x, '_force', value)) - def _frame_title(self, key, group_size=2, separator='\n'): """ @@ -492,7 +483,7 @@ def refresh(self, **kwargs): Refreshes the plot by rerendering it and then pushing the updated data if the plot has an associated Comm. """ - self.force = True + traverse_setter(self, '_force', True) if self.current_key: self.update(self.current_key) else: @@ -612,9 +603,9 @@ def _get_frame(self, key): self.current_key = key return self.current_frame elif self.dynamic: - with enable_streams_cache(self.hmap, not self.force or not self.drawn): + with enable_streams_cache(self.hmap, not self._force or not self.drawn): key, frame = util.get_dynamic_item(self.hmap, self.dimensions, key) - self.force = False + traverse_setter(self, '_force', False) if not isinstance(key, tuple): key = (key,) key_map = dict(zip([d.name for d in self.hmap.kdims], key)) key = tuple(key_map.get(d.name, None) for d in self.dimensions) @@ -958,7 +949,7 @@ def _get_frame(self, key): if d in item.dimensions('key')], key) self.current_key = tuple(k[1] for k in dim_keys) elif item.traverse(lambda x: x, [DynamicMap]): - with enable_streams_cache(item, not self.force or not self.drawn): + with enable_streams_cache(item, not self._force or not self.drawn): key, frame = util.get_dynamic_item(item, self.dimensions, key) layout_frame[path] = frame continue @@ -975,6 +966,7 @@ def _get_frame(self, key): layout_frame[path] = obj else: layout_frame[path] = item + traverse_setter(self, '_force', False) self.current_frame = layout_frame return layout_frame diff --git a/holoviews/plotting/util.py b/holoviews/plotting/util.py index 171f7362f2..77bc131d9f 100644 --- a/holoviews/plotting/util.py +++ b/holoviews/plotting/util.py @@ -285,3 +285,11 @@ def append_refresh(dmap): for stream in dmap.streams: stream._hidden_subscribers.append(plot.refresh) return obj.traverse(append_refresh, [DynamicMap]) + + +def traverse_setter(obj, attribute, value): + """ + Traverses the object and sets the supplied attribute on the + object. Supports Dimensioned and DimensionedPlot types. + """ + obj.traverse(lambda x: setattr(x, attribute, value)) From f323413789e36af6762af50444b3705e3ac03a52 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 13:14:51 +0100 Subject: [PATCH 07/11] Made enable_stream_cache allow_cache_lookup optional --- holoviews/core/traversal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/holoviews/core/traversal.py b/holoviews/core/traversal.py index fe1c32865f..3c2ee1676e 100644 --- a/holoviews/core/traversal.py +++ b/holoviews/core/traversal.py @@ -140,12 +140,12 @@ class enable_streams_cache(object): part of some processing pipeline. """ - def __init__(self, obj, enable): + def __init__(self, obj, allow_cache_lookup=True): self.obj = obj - self._enable = enable + self._allow_cache_lookup = enable def __enter__(self): - self.set_cache_flag(self._enable) + self.set_cache_flag(self._allow_cache_lookup) def __exit__(self, exc_type, exc_val, exc_tb): self.set_cache_flag(False) From 0fea9973dd9e5e4fe73ad4063dec15733967f25c Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 14:10:56 +0100 Subject: [PATCH 08/11] Fixed bug in renderer required for testing --- holoviews/plotting/mpl/renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/plotting/mpl/renderer.py b/holoviews/plotting/mpl/renderer.py index dd0e2063ff..16f29df0c5 100644 --- a/holoviews/plotting/mpl/renderer.py +++ b/holoviews/plotting/mpl/renderer.py @@ -150,7 +150,7 @@ def diff(self, plot): if self.mode == 'mpld3': figure_format = 'json' elif self.fig == 'auto': - figure_format = self.renderer.params('fig').objects[0] + figure_format = self.params('fig').objects[0] else: figure_format = self.fig data = self.html(plot, figure_format, comm=False) From bcdebc15a472d57e55cd4a57ffc59184a5def9a4 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 14:15:30 +0100 Subject: [PATCH 09/11] Renamed dimensionless cache context manager --- holoviews/core/spaces.py | 6 +++--- holoviews/core/traversal.py | 16 ++++++++-------- holoviews/plotting/plot.py | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/holoviews/core/spaces.py b/holoviews/core/spaces.py index 1cbdbd622e..4dbf0f7d16 100644 --- a/holoviews/core/spaces.py +++ b/holoviews/core/spaces.py @@ -477,7 +477,7 @@ def __init__(self, callback, initial_items=None, **params): self.call_mode = self._validate_mode() self.mode = 'bounded' if self.call_mode == 'key' else 'open' - self._stream_cache_lookup = False + self._dimensionless_cache = False def _initial_key(self): @@ -673,8 +673,8 @@ def __getitem__(self, key): # Cache lookup try: - if (util.dimensionless_contents(self.streams, self.kdims) and - not self._stream_cache_lookup): + dimensionless = util.dimensionless_contents(self.streams, self.kdims) + if (dimensionless and not self._dimensionless_cache): raise KeyError('Using dimensionless streams disables DynamicMap cache') cache = super(DynamicMap,self).__getitem__(key) # Return selected cache items in a new DynamicMap diff --git a/holoviews/core/traversal.py b/holoviews/core/traversal.py index 3c2ee1676e..c13793a374 100644 --- a/holoviews/core/traversal.py +++ b/holoviews/core/traversal.py @@ -130,19 +130,19 @@ def hierarchical(keys): return hierarchies -class enable_streams_cache(object): +class dimensionless_cache(object): """ - Context manager which temporarily enables lookup of frame in cache - on a DynamicMap with streams. Allows passing any Dimensioned - object which might contain a DynamicMap and whether to enable the - cache. This allows looking up an item without triggering the - callback. Useful when the object is looked up multiple times as - part of some processing pipeline. + Context manager which temporarily enables lookup of frame in the + cache on a DynamicMap with dimensionless streams. Allows passing + any Dimensioned object which might contain a DynamicMap and + whether to enable the cache. This allows looking up an item + without triggering the callback. Useful when the object is looked + up multiple times as part of some processing pipeline. """ def __init__(self, obj, allow_cache_lookup=True): self.obj = obj - self._allow_cache_lookup = enable + self._allow_cache_lookup = allow_cache_lookup def __enter__(self): self.set_cache_flag(self._allow_cache_lookup) diff --git a/holoviews/plotting/plot.py b/holoviews/plotting/plot.py index 5357f34380..2e4fd3c73e 100644 --- a/holoviews/plotting/plot.py +++ b/holoviews/plotting/plot.py @@ -18,7 +18,7 @@ from ..core.options import Store, Compositor, SkipRendering from ..core.overlay import NdOverlay from ..core.spaces import HoloMap, DynamicMap -from ..core.traversal import enable_streams_cache +from ..core.traversal import dimensionless_cache from ..element import Table from .util import (get_dynamic_mode, initialize_sampled, dim_axis_label, attach_streams, traverse_setter) @@ -603,7 +603,7 @@ def _get_frame(self, key): self.current_key = key return self.current_frame elif self.dynamic: - with enable_streams_cache(self.hmap, not self._force or not self.drawn): + with dimensionless_cache(self.hmap, not self._force or not self.drawn): key, frame = util.get_dynamic_item(self.hmap, self.dimensions, key) traverse_setter(self, '_force', False) if not isinstance(key, tuple): key = (key,) @@ -949,7 +949,7 @@ def _get_frame(self, key): if d in item.dimensions('key')], key) self.current_key = tuple(k[1] for k in dim_keys) elif item.traverse(lambda x: x, [DynamicMap]): - with enable_streams_cache(item, not self._force or not self.drawn): + with dimensionless_cache(item, not self._force or not self.drawn): key, frame = util.get_dynamic_item(item, self.dimensions, key) layout_frame[path] = frame continue From 7bae4037fde7325857ffa6a00f051ddf6e32f064 Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 14:16:17 +0100 Subject: [PATCH 10/11] Added unit test for refresh on plot with dimensionless stream --- tests/testplotinstantiation.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/testplotinstantiation.py b/tests/testplotinstantiation.py index af9059b595..339a45eb37 100644 --- a/tests/testplotinstantiation.py +++ b/tests/testplotinstantiation.py @@ -3,17 +3,20 @@ """ from unittest import SkipTest +from io import BytesIO + import numpy as np from holoviews import (Dimension, Curve, Scatter, Overlay, DynamicMap, Store, Image, VLine, NdOverlay, Points) from holoviews.element.comparison import ComparisonTestCase +from holoviews.streams import PositionXY # Standardize backend due to random inconsistencies try: from matplotlib import pyplot pyplot.switch_backend('agg') from holoviews.plotting.mpl import OverlayPlot - from holoviews.plotting.comms import JupyterPushComm + from holoviews.plotting.comms import Comm mpl_renderer = Store.renderers['matplotlib'] except: mpl_renderer = None @@ -31,7 +34,7 @@ def setUp(self): if mpl_renderer is None: raise SkipTest("Matplotlib required to test plot instantiation") self.default_comm, _ = mpl_renderer.comms['default'] - mpl_renderer.comms['default'] = (JupyterPushComm, '') + mpl_renderer.comms['default'] = (Comm, '') def teardown(self): mpl_renderer.comms['default'] = (self.default_comm, '') @@ -53,6 +56,18 @@ def test_dynamic_nonoverlap(self): mpl_renderer.get_widget(dmap1 + dmap2, 'selection') + def test_dynamic_streams_refresh(self): + stream = PositionXY() + dmap = DynamicMap(lambda x, y: Points([(x, y)]), + kdims=[], streams=[stream]) + plot = mpl_renderer.get_plot(dmap) + plot.initialize_plot() + pre = mpl_renderer(plot, fmt='png') + stream.update(x=1, y=1) + plot.refresh() + post = mpl_renderer(plot, fmt='png') + self.assertNotEqual(pre, post) + class TestBokehPlotInstantiation(ComparisonTestCase): From e97a530fbe3168cafe31c80f8a1e6844a9fdb06e Mon Sep 17 00:00:00 2001 From: Philipp Rudiger Date: Mon, 12 Sep 2016 14:46:33 +0100 Subject: [PATCH 11/11] Fixed python3 compatibility issue in DynamicMap --- holoviews/core/spaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holoviews/core/spaces.py b/holoviews/core/spaces.py index 4dbf0f7d16..fdcc4ed427 100644 --- a/holoviews/core/spaces.py +++ b/holoviews/core/spaces.py @@ -710,7 +710,7 @@ def _cache(self, key, val): if self.mode == 'open' and (self.counter % self.cache_interval)!=0: return if len(self) >= cache_size: - first_key = next(self.data.iterkeys()) + first_key = next(k for k in self.data) self.data.pop(first_key) self.data[key] = val