From f078a280537767b0ad756511cb6cc416db122e98 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Thu, 3 Mar 2016 16:41:26 -0600 Subject: [PATCH 01/22] Added support for pure counts (no corresponding field). Fixed outdated refs to taxi --- examples/dashboard/dashboard.py | 16 ++++++++-------- examples/dashboard/nyc_taxi.yml | 3 +++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 433803bfe..29b25196b 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -142,26 +142,26 @@ def load_config_file(self, config_path): # parse summary field self.fields = OrderedDict() for f in self.config['summary_fields']: - self.fields[f['name']] = f['field'] + self.fields[f['name']] = None if f['field'] == 'None' else f['field'] self.field = list(self.fields.values())[0] def load_datasets(self): print('Loading Data...') - taxi_path = self.config['file'] + data_path = self.config['file'] - if not path.isabs(taxi_path): + if not path.isabs(data_path): config_dir = path.split(self.config_path)[0] - taxi_path = path.join(config_dir, taxi_path) + data_path = path.join(config_dir, data_path) - if not path.exists(taxi_path): - raise IOError('Unable to find input dataset: "{}"'.format(taxi_path)) + if not path.exists(data_path): + raise IOError('Unable to find input dataset: "{}"'.format(data_path)) axes_fields = [] for f in self.axes.values(): axes_fields += [f[1], f[2]] - load_fields = list(self.fields.values()) + axes_fields - self.df = pd.read_csv(taxi_path, usecols=load_fields) + load_fields = [f for f in self.fields.values() if f is not None] + axes_fields + self.df = pd.read_csv(data_path, usecols=load_fields) class AppView(object): diff --git a/examples/dashboard/nyc_taxi.yml b/examples/dashboard/nyc_taxi.yml index 81819c2e3..0bd18aa0a 100644 --- a/examples/dashboard/nyc_taxi.yml +++ b/examples/dashboard/nyc_taxi.yml @@ -19,6 +19,9 @@ axes: yaxis: dropoff_y summary_fields: + - name: Trip Count + field: None + - name: Passenger Count field: passenger_count From 749d600b8f20d4f05dbbda2c970b2dd11db65f69 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Fri, 4 Mar 2016 08:09:10 -0600 Subject: [PATCH 02/22] Added support for multiple filetypes --- examples/dashboard/dashboard.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 29b25196b..391647f28 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -161,8 +161,16 @@ def load_datasets(self): axes_fields += [f[1], f[2]] load_fields = [f for f in self.fields.values() if f is not None] + axes_fields - self.df = pd.read_csv(data_path, usecols=load_fields) + if data_path.endswith(".csv") : + self.df = pd.read_csv(data_path, usecols=load_fields) + elif data_path.endswith(".castra"): + import dask.dataframe as dd + self.df = dd.from_castra(data_path) + else: + raise IOError("Unknown data file type; .csv and .castra currently supported") + + class AppView(object): def __init__(self, app_model): @@ -278,7 +286,7 @@ def on_labels_change(self, new): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--config', help='yaml config file (e.g. nyc_taxi.yml)', required=True) + parser.add_argument('-c', '--config', help='yaml config file (e.g. nyc_taxi.yml)', required=True) args = vars(parser.parse_args()) APP_PORT = 5000 From 5cea5979742f39c2623d796636aa36f3131b4dae Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Fri, 4 Mar 2016 10:41:39 -0600 Subject: [PATCH 03/22] Added data file path to startup message --- examples/dashboard/dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 391647f28..5d304deac 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -146,8 +146,8 @@ def load_config_file(self, config_path): self.field = list(self.fields.values())[0] def load_datasets(self): - print('Loading Data...') data_path = self.config['file'] + print('Loading Data from {}...'.format(data_path)) if not path.isabs(data_path): config_dir = path.split(self.config_path)[0] From 4a39b971a6e24e8c7e86f23e868fc33a0e52b6ca Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Fri, 4 Mar 2016 10:43:57 -0600 Subject: [PATCH 04/22] Initial version of census.yml; works for counts --- examples/dashboard/census.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 examples/dashboard/census.yml diff --git a/examples/dashboard/census.yml b/examples/dashboard/census.yml new file mode 100644 index 000000000..bd9cef885 --- /dev/null +++ b/examples/dashboard/census.yml @@ -0,0 +1,30 @@ +--- + +file: ../data/census.castra + +initial_extent: + is_geo: true + xmin: -13914936 + ymin: 2632019 + xmax: -7235767 + ymax: 6446276 + +axes: + - name: US Census Synthetic people + xaxis: meterswest + yaxis: metersnorth + +summary_fields: + - name: Counts + field: None + + - name: Race + field: race + cat_colors: race_colors + +race_colors: + w: blue + b: green + a: red + h: orange + o: saddlebrown From f24e96ebf574d482018d6a5c269473a707638be4 Mon Sep 17 00:00:00 2001 From: bcollins Date: Thu, 10 Mar 2016 15:16:50 -0600 Subject: [PATCH 05/22] moved location of colormap to nest under summary_field --- examples/dashboard/census.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/examples/dashboard/census.yml b/examples/dashboard/census.yml index bd9cef885..393fb66c9 100644 --- a/examples/dashboard/census.yml +++ b/examples/dashboard/census.yml @@ -15,16 +15,15 @@ axes: yaxis: metersnorth summary_fields: - - name: Counts - field: None - name: Race field: race - cat_colors: race_colors + cat_colors: + w: blue + b: green + a: red + h: orange + o: saddlebrown -race_colors: - w: blue - b: green - a: red - h: orange - o: saddlebrown + - name: Counts + field: None From 26bbfafa1c3330af7b9030b4cd6865a2e0166437 Mon Sep 17 00:00:00 2001 From: bcollins Date: Thu, 10 Mar 2016 15:17:20 -0600 Subject: [PATCH 06/22] added checks for categorical fields and refactored aggregate type menu --- examples/dashboard/dashboard.py | 87 +++++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 5d304deac..19a3fd7e8 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -11,6 +11,7 @@ import datashader as ds import datashader.transfer_functions as tf import pandas as pd +import numpy as np from bokeh.server.server import Server from bokeh.application import Application @@ -28,6 +29,8 @@ from webargs import fields from webargs.tornadoparser import use_args +from pdb import set_trace + # http request arguments for datashing HTTP request ds_args = { @@ -51,12 +54,33 @@ def get(self, args): plot_height=args['height'], x_range=(xmin, xmax), y_range=(ymin, ymax)) - agg = cvs.points(self.model.df, - self.model.active_axes[1], - self.model.active_axes[2], - self.model.aggregate_function(self.model.field)) - pix = tf.interpolate(agg, (255, 204, 204), 'red', - how=self.model.transfer_function) + + # handle categorical field + if self.model.field in self.model.categorical_fields: + agg = cvs.points(self.model.df, + self.model.active_axes[1], + self.model.active_axes[2], + ds.count_cat(self.model.field)) + + pix = tf.colorize(agg, self.model.colormap, + how=self.model.transfer_function) + + # handle ordinal field + elif self.model.field in self.model.ordinal_fields: + agg = cvs.points(self.model.df, + self.model.active_axes[1], + self.model.active_axes[2]) + + pix = tf.interpolate(agg, (255, 204, 204), 'red', + how=self.model.transfer_function) + # handle no field + else: + agg = cvs.points(self.model.df, + self.model.active_axes[1], + self.model.active_axes[2]) + + pix = tf.interpolate(agg, (255, 204, 204), 'red', + how=self.model.transfer_function) # serialize to image img_io = pix.to_bytesio() @@ -141,10 +165,24 @@ def load_config_file(self, config_path): # parse summary field self.fields = OrderedDict() + self.colormaps = OrderedDict() + self.ordinal_fields = [] + self.categorical_fields = [] for f in self.config['summary_fields']: self.fields[f['name']] = None if f['field'] == 'None' else f['field'] + + if 'cat_colors' in f.keys(): + self.colormaps[f['name']] = f['cat_colors'] + self.categorical_fields.append(f['field']) + + elif f['field'] != 'None': + self.ordinal_fields.append(f['field']) + self.field = list(self.fields.values())[0] + if self.colormaps: + self.colormap = self.colormaps[list(self.fields.keys())[0]] + def load_datasets(self): data_path = self.config['file'] print('Loading Data from {}...'.format(data_path)) @@ -164,6 +202,11 @@ def load_datasets(self): if data_path.endswith(".csv") : self.df = pd.read_csv(data_path, usecols=load_fields) + + # parse categorical fields + for f in self.categorical_fields: + self.df[f] = self.df[f].astype('category') + elif data_path.endswith(".castra"): import dask.dataframe as dd self.df = dd.from_castra(data_path) @@ -188,7 +231,7 @@ def create_layout(self): self.fig = Figure(tools='wheel_zoom,pan', x_range=self.x_range, y_range=self.y_range) self.fig.plot_height = 560 - self.fig.plot_width = 800 + self.fig.plot_width = 810 self.fig.axis.visible = False # add tiled basemap @@ -201,28 +244,34 @@ def create_layout(self): extra_url_vars=self.model.shader_url_vars) self.image_renderer = DynamicImageRenderer(image_source=self.image_source) self.fig.renderers.append(self.image_renderer) - + # add label layer self.label_source = WMTSTileSource(url=self.model.labels_url) self.label_renderer = TileRenderer(tile_source=self.label_source) self.fig.renderers.append(self.label_renderer) # add ui components + controls = [] axes_select = Select.create(name='Axes', options=self.model.axes) axes_select.on_change('value', self.on_axes_change) + controls.append(axes_select) - field_select = Select.create(name='Field', options=self.model.fields) - field_select.on_change('value', self.on_field_change) + self.field_select = Select.create(name='Field', options=self.model.fields) + self.field_select.on_change('value', self.on_field_change) + controls.append(self.field_select) - aggregate_select = Select.create(name='Aggregate', + self.aggregate_select = Select.create(name='Aggregate', options=self.model.aggregate_functions) - aggregate_select.on_change('value', self.on_aggregate_change) + self.aggregate_select.on_change('value', self.on_aggregate_change) + controls.append(self.aggregate_select) transfer_select = Select.create(name='Transfer Function', options=self.model.transfer_functions) transfer_select.on_change('value', self.on_transfer_function_change) + controls.append(transfer_select) + # add map components basemap_select = Select.create(name='Basemap', value='Toner', options=self.model.basemaps) basemap_select.on_change('value', self.on_basemap_change) @@ -238,10 +287,6 @@ def create_layout(self): show_labels_chk = CheckboxGroup(labels=["Show Labels"], active=[0]) show_labels_chk.on_click(self.on_labels_change) - - controls = [axes_select, field_select, aggregate_select, - transfer_select] - map_controls = [basemap_select, basemap_opacity_slider, image_opacity_slider, show_labels_chk] @@ -255,10 +300,20 @@ def update_image(self): self.image_renderer.image_source = ImageSource(url=self.model.service_url, extra_url_vars=self.model.shader_url_vars) + def on_field_change(self, attr, old, new): self.model.field = self.model.fields[new] self.update_image() + if not self.model.field: + self.aggregate_select.options = [dict(name="No Aggregates Available", value="")] + + elif self.model.field in self.model.categorical_fields: + self.aggregate_select.options = [dict(name="Categorical", value="count_cat")] + else: + opts = [dict(name=k, value=k) for k in self.model.aggregate_functions.keys()] + self.aggregate_select.options = opts + def on_basemap_change(self, attr, old, new): self.model.basemap = self.model.basemaps[new] self.tile_renderer.tile_source = WMTSTileSource(url=self.model.basemap) From 5b5db6f4638eb43440545efc4a6c1a2f8f1de179 Mon Sep 17 00:00:00 2001 From: bcollins Date: Thu, 10 Mar 2016 15:18:21 -0600 Subject: [PATCH 07/22] removed pdb --- examples/dashboard/dashboard.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 19a3fd7e8..e3d08e7ef 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -29,9 +29,6 @@ from webargs import fields from webargs.tornadoparser import use_args -from pdb import set_trace - - # http request arguments for datashing HTTP request ds_args = { 'width': fields.Int(missing=800), From 5b24f063939f18549a7b3d8a528851d8b7f3e936 Mon Sep 17 00:00:00 2001 From: bcollins Date: Fri, 11 Mar 2016 15:04:22 -0600 Subject: [PATCH 08/22] added hover support --- examples/dashboard/dashboard.py | 78 ++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index e3d08e7ef..b598083fa 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -5,6 +5,7 @@ import yaml import webbrowser import uuid +import math from collections import OrderedDict @@ -12,16 +13,17 @@ import datashader.transfer_functions as tf import pandas as pd import numpy as np +import pdb from bokeh.server.server import Server from bokeh.application import Application from bokeh.application.handlers import FunctionHandler -from bokeh.plotting import Figure +from bokeh.plotting import Figure from bokeh.models import (Range1d, ImageSource, WMTSTileSource, TileRenderer, DynamicImageRenderer, HBox, VBox) -from bokeh.models import Select, Slider, CheckboxGroup +from bokeh.models import Select, Slider, CheckboxGroup, CustomJS, ColumnDataSource, Square, HoverTool from tornado.ioloop import IOLoop from tornado.web import RequestHandler @@ -52,6 +54,11 @@ def get(self, args): x_range=(xmin, xmax), y_range=(ymin, ymax)) + hover_cvs = ds.Canvas(plot_width=math.ceil(args['width'] / self.model.hover_size), + plot_height=math.ceil(args['height'] / self.model.hover_size), + x_range=(xmin, xmax), + y_range=(ymin, ymax)) + # handle categorical field if self.model.field in self.model.categorical_fields: agg = cvs.points(self.model.df, @@ -59,6 +66,11 @@ def get(self, args): self.model.active_axes[2], ds.count_cat(self.model.field)) + hover_agg = hover_cvs.points(self.model.df, + self.model.active_axes[1], + self.model.active_axes[2], + ds.count_cat(self.model.field)) + pix = tf.colorize(agg, self.model.colormap, how=self.model.transfer_function) @@ -66,7 +78,13 @@ def get(self, args): elif self.model.field in self.model.ordinal_fields: agg = cvs.points(self.model.df, self.model.active_axes[1], - self.model.active_axes[2]) + self.model.active_axes[2], + self.model.aggregate_function(self.model.field)) + + hover_agg = hover_cvs.points(self.model.df, + self.model.active_axes[1], + self.model.active_axes[2], + self.model.aggregate_function(self.model.field)) pix = tf.interpolate(agg, (255, 204, 204), 'red', how=self.model.transfer_function) @@ -76,15 +94,25 @@ def get(self, args): self.model.active_axes[1], self.model.active_axes[2]) + hover_agg = hover_cvs.points(self.model.df, + self.model.active_axes[1], + self.model.active_axes[2]) pix = tf.interpolate(agg, (255, 204, 204), 'red', how=self.model.transfer_function) + def hover_callback(): + xs, ys = np.meshgrid(hover_agg.x_axis.values, + hover_agg.y_axis.values) + self.model.hover_source.data['x'] = xs.flatten() + self.model.hover_source.data['y'] = ys.flatten() + self.model.hover_source.data['value'] = hover_agg.values.flatten() + + server.get_sessions('/')[0].with_document_locked(hover_callback) # serialize to image img_io = pix.to_bytesio() self.write(img_io.getvalue()) self.set_header("Content-type", "image/png") - class AppState(object): """Simple value object to hold app state""" @@ -138,6 +166,10 @@ def __init__(self, config_file, app_port=5000): # set defaults self.load_datasets() + # hover + self.hover_source = ColumnDataSource(data=dict(x=[], y=[], val=[])) + self.hover_size = 8 + def load_config_file(self, config_path): '''load and parse yaml config file''' @@ -247,6 +279,31 @@ def create_layout(self): self.label_renderer = TileRenderer(tile_source=self.label_source) self.fig.renderers.append(self.label_renderer) + self.invisible_square = Square(x='x', + y='y', + fill_color=None, + line_color=None, + size=self.model.hover_size) + + self.visible_square = Square(x='x', + y='y', + fill_color='#79DCDE', + fill_alpha=.5, + line_color='#79DCDE', + line_alpha=1, + size=self.model.hover_size) + + cr = self.fig.add_glyph(self.model.hover_source, + self.invisible_square, + selection_glyph=self.visible_square, + nonselection_glyph=self.invisible_square) + + # Add a hover tool, that selects the circle + code = "source.set('selected', cb_data['index']);" + callback = CustomJS(args={'source': self.model.hover_source}, code=code) + self.hover_tool = HoverTool(tooltips=[(self.model.fields.keys()[0], "@value")], callback=callback, renderers=[cr], mode='mouse') + self.fig.add_tools(self.hover_tool) + # add ui components controls = [] axes_select = Select.create(name='Axes', @@ -281,6 +338,11 @@ def create_layout(self): end=100, step=1) basemap_opacity_slider.on_change('value', self.on_basemap_opacity_slider_change) + hover_size_slider = Slider(title="Hover Size (px)", value=8, start=4, + end=30, step=1) + hover_size_slider.on_change('value', self.on_hover_size_change) + controls.append(hover_size_slider) + show_labels_chk = CheckboxGroup(labels=["Show Labels"], active=[0]) show_labels_chk.on_click(self.on_labels_change) @@ -297,9 +359,9 @@ def update_image(self): self.image_renderer.image_source = ImageSource(url=self.model.service_url, extra_url_vars=self.model.shader_url_vars) - def on_field_change(self, attr, old, new): self.model.field = self.model.fields[new] + self.hover_tool.tooltips = [(new, '@value')] self.update_image() if not self.model.field: @@ -315,6 +377,12 @@ def on_basemap_change(self, attr, old, new): self.model.basemap = self.model.basemaps[new] self.tile_renderer.tile_source = WMTSTileSource(url=self.model.basemap) + def on_hover_size_change(self, attr, old, new): + self.model.hover_size = int(new) + self.invisible_square.size = int(new) + self.visible_square.size = int(new) + self.update_image() + def on_axes_change(self, attr, old, new): self.model.active_axes = self.model.axes[new] self.update_image() From ee1bc4bbb38d28aa4ceb0ce60a015f0b7a51aa9a Mon Sep 17 00:00:00 2001 From: bcollins Date: Thu, 17 Mar 2016 17:30:10 -0500 Subject: [PATCH 09/22] added mean aggregate downsample for hover layer --- examples/dashboard/dashboard.py | 61 ++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index b598083fa..1b93d749f 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -13,7 +13,6 @@ import datashader.transfer_functions as tf import pandas as pd import numpy as np -import pdb from bokeh.server.server import Server from bokeh.application import Application @@ -38,6 +37,7 @@ 'select': fields.Str(missing=""), } + class GetDataset(RequestHandler): """Handles http requests for datashading.""" @use_args(ds_args) @@ -54,10 +54,6 @@ def get(self, args): x_range=(xmin, xmax), y_range=(ymin, ymax)) - hover_cvs = ds.Canvas(plot_width=math.ceil(args['width'] / self.model.hover_size), - plot_height=math.ceil(args['height'] / self.model.hover_size), - x_range=(xmin, xmax), - y_range=(ymin, ymax)) # handle categorical field if self.model.field in self.model.categorical_fields: @@ -66,13 +62,9 @@ def get(self, args): self.model.active_axes[2], ds.count_cat(self.model.field)) - hover_agg = hover_cvs.points(self.model.df, - self.model.active_axes[1], - self.model.active_axes[2], - ds.count_cat(self.model.field)) - - pix = tf.colorize(agg, self.model.colormap, - how=self.model.transfer_function) + pix = tf.colorize(agg, + self.model.colormap, + how=self.model.transfer_function) # handle ordinal field elif self.model.field in self.model.ordinal_fields: @@ -81,11 +73,6 @@ def get(self, args): self.model.active_axes[2], self.model.aggregate_function(self.model.field)) - hover_agg = hover_cvs.points(self.model.df, - self.model.active_axes[1], - self.model.active_axes[2], - self.model.aggregate_function(self.model.field)) - pix = tf.interpolate(agg, (255, 204, 204), 'red', how=self.model.transfer_function) # handle no field @@ -94,18 +81,32 @@ def get(self, args): self.model.active_axes[1], self.model.active_axes[2]) - hover_agg = hover_cvs.points(self.model.df, - self.model.active_axes[1], - self.model.active_axes[2]) pix = tf.interpolate(agg, (255, 204, 204), 'red', how=self.model.transfer_function) def hover_callback(): - xs, ys = np.meshgrid(hover_agg.x_axis.values, - hover_agg.y_axis.values) - self.model.hover_source.data['x'] = xs.flatten() - self.model.hover_source.data['y'] = ys.flatten() - self.model.hover_source.data['value'] = hover_agg.values.flatten() + + def downsample(aggregate, factor): + ys, xs = aggregate.shape + crarr = aggregate[:ys-(ys % int(factor)),:xs-(xs % int(factor))] + return np.nanmean(np.concatenate([[crarr[i::factor,j::factor] + for i in range(factor)] + for j in range(factor)]), axis=0) + + hover_agg = downsample(agg.values, self.model.hover_size) + + sq_xs = np.linspace(self.model.map_extent[0], + self.model.map_extent[2], + agg.shape[1] / self.model.hover_size) + + sq_ys = np.linspace(self.model.map_extent[1], + self.model.map_extent[3], + agg.shape[0] / self.model.hover_size) + + agg_xs, agg_ys = np.meshgrid(sq_xs, sq_ys) + self.model.hover_source.data['x'] = agg_xs.flatten() + self.model.hover_source.data['y'] = agg_ys.flatten() + self.model.hover_source.data['value'] = hover_agg.flatten() server.get_sessions('/')[0].with_document_locked(hover_callback) # serialize to image @@ -279,6 +280,7 @@ def create_layout(self): self.label_renderer = TileRenderer(tile_source=self.label_source) self.fig.renderers.append(self.label_renderer) + # Add a hover tool self.invisible_square = Square(x='x', y='y', fill_color=None, @@ -298,10 +300,13 @@ def create_layout(self): selection_glyph=self.visible_square, nonselection_glyph=self.invisible_square) - # Add a hover tool, that selects the circle code = "source.set('selected', cb_data['index']);" callback = CustomJS(args={'source': self.model.hover_source}, code=code) - self.hover_tool = HoverTool(tooltips=[(self.model.fields.keys()[0], "@value")], callback=callback, renderers=[cr], mode='mouse') + + self.hover_tool = HoverTool(tooltips=[(self.model.fields.keys()[0], "@value")], + callback=callback, + renderers=[cr], + mode='mouse') self.fig.add_tools(self.hover_tool) # add ui components @@ -338,7 +343,7 @@ def create_layout(self): end=100, step=1) basemap_opacity_slider.on_change('value', self.on_basemap_opacity_slider_change) - hover_size_slider = Slider(title="Hover Size (px)", value=8, start=4, + hover_size_slider = Slider(title="Hover Size (px)", value=8, start=1, end=30, step=1) hover_size_slider.on_change('value', self.on_hover_size_change) controls.append(hover_size_slider) From a0f3cbcd6e0330b2ce627b87137f64780e248ba8 Mon Sep 17 00:00:00 2001 From: bcollins Date: Thu, 17 Mar 2016 17:31:15 -0500 Subject: [PATCH 10/22] moved slider start value back to 4 --- examples/dashboard/dashboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 1b93d749f..7def848e2 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -343,7 +343,7 @@ def create_layout(self): end=100, step=1) basemap_opacity_slider.on_change('value', self.on_basemap_opacity_slider_change) - hover_size_slider = Slider(title="Hover Size (px)", value=8, start=1, + hover_size_slider = Slider(title="Hover Size (px)", value=8, start=4, end=30, step=1) hover_size_slider.on_change('value', self.on_hover_size_change) controls.append(hover_size_slider) From 75cbd86351d39081e5bc499a233cdd9d78b0293b Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Fri, 18 Mar 2016 23:40:24 -0500 Subject: [PATCH 11/22] Added description of census data --- examples/README.md | 63 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/examples/README.md b/examples/README.md index f7edfc337..9f830f307 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,8 +8,8 @@ good network connection. The dataset is roughly 1.5 GB on disk. python download_sample_data.py ``` -The examples also require bokeh to be installed. Bokeh is available through -either conda or pip. +Datashader is an independent library, but most of the examples require +bokeh to be installed. Bokeh is available through either conda or pip: ``` conda install bokeh @@ -21,16 +21,6 @@ pip install bokeh ## Examples -### Dashboard - -An example interactive dashboard using [bokeh -server](http://bokeh.pydata.org/en/latest/docs/user_guide/server.html) -integrated with a datashading pipeline. To start, run: - -``` -python dashboard/dashboard.py --config dashboard/nyc_taxi.yml -``` - ### Notebooks Most of the examples are in the form of runnable Jupyter notebooks. Copies of @@ -74,3 +64,52 @@ map](https://blog.openstreetmap.org/2012/04/01/bulk-gps-point-data/). This dataset isn't provided by the download script, and is only included to demonstrate working with a large dataset. The run notebook can be viewed using the `anaconda.org` link provided above. + + + +### Dashboard + +An example interactive dashboard using +[bokeh server](http://bokeh.pydata.org/en/latest/docs/user_guide/server.html) +integrated with a datashading pipeline. Requires webargs: + +``` +pip install webargs +``` + +To start, run: + +``` +python dashboard/dashboard.py -c dashboard/nyc_taxi.yml +``` + +The 'nyc_taxi.yml' configuration file set up the dashboard to use the +NYC Taxi dataset downloaded above. If you have less than 16GB of RAM +on your machine, you will want to add the "-o" option to tell it to work +out of core instead of loading all data into memory, though doing so will +make interactive use substantially slower than if sufficient memory were +available. + +You can write similar configuration files for working with other +datasets of your own, while adding features to dashboard.py itself if +needed. As an example, a configuration file for the [2010 US Census +racial data](http://www.coopercenter.org/demographics/Racial-Dot-Map) +is also provided. To use the census data, you'll need to install dask +and castra: + +``` +conda install dask +conda install -c quanyuan castra +``` + +You'll also need to download the 1.3GB data file +[census.castra.zip]() +and unzip it into `examples/data/` (5GB unzipped), then run the +dashboard: + +``` +python dashboard/dashboard.py -c dashboard/census.yml +``` + +If you have other dashboards running, you'll need to add "-p 5001" (etc.) to select +a unique port number for the web page to use for communicating with the Bokeh server. From 70c8260623b82e870d410f3f9cc23642055c4cd2 Mon Sep 17 00:00:00 2001 From: "James A. Bednar" Date: Fri, 18 Mar 2016 23:40:24 -0500 Subject: [PATCH 12/22] Added outofcore option and port option --- examples/dashboard/dashboard.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 7def848e2..1dd9f6785 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -117,7 +117,7 @@ def downsample(aggregate, factor): class AppState(object): """Simple value object to hold app state""" - def __init__(self, config_file, app_port=5000): + def __init__(self, config_file, outofcore, app_port): self.load_config_file(config_file) @@ -165,7 +165,7 @@ def __init__(self, config_file, app_port=5000): self.shader_url_vars['cachebust'] = str(uuid.uuid4()) # set defaults - self.load_datasets() + self.load_datasets(outofcore) # hover self.hover_source = ColumnDataSource(data=dict(x=[], y=[], val=[])) @@ -213,7 +213,7 @@ def load_config_file(self, config_path): if self.colormaps: self.colormap = self.colormaps[list(self.fields.keys())[0]] - def load_datasets(self): + def load_datasets(self,outofcore): data_path = self.config['file'] print('Loading Data from {}...'.format(data_path)) @@ -240,6 +240,9 @@ def load_datasets(self): elif data_path.endswith(".castra"): import dask.dataframe as dd self.df = dd.from_castra(data_path) + if not outofcore: + self.df = self.df.cache(cache=dict) + else: raise IOError("Unknown data file type; .csv and .castra currently supported") @@ -412,12 +415,15 @@ def on_labels_change(self, new): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', help='yaml config file (e.g. nyc_taxi.yml)', required=True) + parser.add_argument('-p', '--port', help='port number to use for communicating with server; defaults to 5000', default=5000) + parser.add_argument('-o', '--outofcore', help='use out-of-core processing if available, for datasets larger than memory', + default=False, action='store_true') args = vars(parser.parse_args()) - APP_PORT = 5000 + APP_PORT = args['port'] def add_roots(doc): - model = AppState(args['config'], APP_PORT) + model = AppState(args['config'], args['outofcore'], APP_PORT) view = AppView(model) GetDataset.model = model doc.add_root(view.layout) From 740edd67d293a33712a543562302e15f879127b6 Mon Sep 17 00:00:00 2001 From: bcollins Date: Mon, 21 Mar 2016 12:00:03 -0500 Subject: [PATCH 13/22] added cmap back after merge --- examples/dashboard/dashboard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 1dd9f6785..d9b3fec94 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -73,7 +73,7 @@ def get(self, args): self.model.active_axes[2], self.model.aggregate_function(self.model.field)) - pix = tf.interpolate(agg, (255, 204, 204), 'red', + pix = tf.interpolate(agg, cmap=[(255, 204, 204), 'red'], how=self.model.transfer_function) # handle no field else: @@ -81,7 +81,7 @@ def get(self, args): self.model.active_axes[1], self.model.active_axes[2]) - pix = tf.interpolate(agg, (255, 204, 204), 'red', + pix = tf.interpolate(agg, cmap=[(255, 204, 204), 'red'], how=self.model.transfer_function) def hover_callback(): From 3d1f9a0c272d53b6ed163b9ba50c5aa814e54ce0 Mon Sep 17 00:00:00 2001 From: bcollins Date: Mon, 21 Mar 2016 16:24:51 -0500 Subject: [PATCH 14/22] added colorbar for numeric data --- examples/dashboard/dashboard.py | 67 ++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index d9b3fec94..39691dbc7 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -5,7 +5,7 @@ import yaml import webbrowser import uuid -import math +import pdb from collections import OrderedDict @@ -13,6 +13,7 @@ import datashader.transfer_functions as tf import pandas as pd import numpy as np +from xarray import DataArray from bokeh.server.server import Server from bokeh.application import Application @@ -37,7 +38,6 @@ 'select': fields.Str(missing=""), } - class GetDataset(RequestHandler): """Handles http requests for datashading.""" @use_args(ds_args) @@ -84,7 +84,7 @@ def get(self, args): pix = tf.interpolate(agg, cmap=[(255, 204, 204), 'red'], how=self.model.transfer_function) - def hover_callback(): + def update_plots(): def downsample(aggregate, factor): ys, xs = aggregate.shape @@ -93,6 +93,7 @@ def downsample(aggregate, factor): for i in range(factor)] for j in range(factor)]), axis=0) + # update hover layer hover_agg = downsample(agg.values, self.model.hover_size) sq_xs = np.linspace(self.model.map_extent[0], @@ -108,7 +109,21 @@ def downsample(aggregate, factor): self.model.hover_source.data['y'] = agg_ys.flatten() self.model.hover_source.data['value'] = hover_agg.flatten() - server.get_sessions('/')[0].with_document_locked(hover_callback) + # update legend + min_val = agg.values.min() + max_val = agg.values.max() + vals = np.vstack([np.linspace(min_val, max_val, 180)] * 18) + vals_arr = DataArray(vals) + img = tf.interpolate(vals_arr, + cmap=[(255, 204, 204), 'red'], + how=self.model.transfer_function) + self.model.legend_source.data['image'] = [img.values] + self.model.legend_source.data['x'] = [min_val] + self.model.legend_source.data['dw'] = [max_val - min_val] + self.model.legend_fig.x_range.start = min_val + self.model.legend_fig.x_range.end = max_val + + server.get_sessions('/')[0].with_document_locked(update_plots) # serialize to image img_io = pix.to_bytesio() self.write(img_io.getvalue()) @@ -171,6 +186,13 @@ def __init__(self, config_file, outofcore, app_port): self.hover_source = ColumnDataSource(data=dict(x=[], y=[], val=[])) self.hover_size = 8 + # legend + self.legend_source = ColumnDataSource(data=dict(image=[], + x=[0], + y=[0], + dw=[180], + dh=[18])) + def load_config_file(self, config_path): '''load and parse yaml config file''' @@ -261,8 +283,16 @@ def create_layout(self): self.y_range = Range1d(start=self.model.map_extent[1], end=self.model.map_extent[3], bounds=None) - self.fig = Figure(tools='wheel_zoom,pan', x_range=self.x_range, + self.fig = Figure(tools='wheel_zoom,pan', + x_range=self.x_range, + lod_threshold=None, y_range=self.y_range) + + self.fig.min_border_top = 0 + self.fig.min_border_bottom = 10 + self.fig.min_border_left = 0 + self.fig.min_border_right = 0 + self.fig.plot_height = 560 self.fig.plot_width = 810 self.fig.axis.visible = False @@ -310,6 +340,29 @@ def create_layout(self): callback=callback, renderers=[cr], mode='mouse') + + self.model.legend_fig = Figure(x_range=(0, 180), + plot_height=50, + plot_width=self.fig.plot_width, + lod_threshold=None, + toolbar_location=None, + y_range=(0,18)) + + self.model.legend_fig.min_border_top = 0 + self.model.legend_fig.min_border_bottom = 10 + self.model.legend_fig.min_border_left = 0 + self.model.legend_fig.min_border_right = 0 + self.model.legend_fig.yaxis.visible = False + self.model.legend_fig.grid.grid_line_alpha = 0 + + self.model.legend_fig.image_rgba(source=self.model.legend_source, + image='image', + x='x', + y='y', + dw='dw', + dh='dh', + dw_units='screen') + self.fig.add_tools(self.hover_tool) # add ui components @@ -359,7 +412,9 @@ def create_layout(self): self.controls = VBox(width=200, height=600, children=controls) self.map_controls = HBox(width=self.fig.plot_width, children=map_controls) - self.map_area = VBox(width=self.fig.plot_width, children=[self.map_controls, self.fig]) + self.map_area = VBox(width=self.fig.plot_width, children=[self.map_controls, + self.fig, + self.model.legend_fig]) self.layout = HBox(width=1024, children=[self.controls, self.map_area]) def update_image(self): From d82a076b6397d6a0b79e02eb6f36a8ec001f1e77 Mon Sep 17 00:00:00 2001 From: bcollins Date: Thu, 24 Mar 2016 11:49:21 -0500 Subject: [PATCH 15/22] added eq_hist, spread slider, and colorbars --- examples/dashboard/dashboard.py | 216 ++++++++++++++++++++++---------- 1 file changed, 147 insertions(+), 69 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 39691dbc7..206250875 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -24,6 +24,7 @@ DynamicImageRenderer, HBox, VBox) from bokeh.models import Select, Slider, CheckboxGroup, CustomJS, ColumnDataSource, Square, HoverTool +from bokeh.palettes import BrBG9, PiYG9 from tornado.ioloop import IOLoop from tornado.web import RequestHandler @@ -73,7 +74,7 @@ def get(self, args): self.model.active_axes[2], self.model.aggregate_function(self.model.field)) - pix = tf.interpolate(agg, cmap=[(255, 204, 204), 'red'], + pix = tf.interpolate(agg, cmap=self.model.color_ramp, how=self.model.transfer_function) # handle no field else: @@ -81,11 +82,21 @@ def get(self, args): self.model.active_axes[1], self.model.active_axes[2]) - pix = tf.interpolate(agg, cmap=[(255, 204, 204), 'red'], + pix = tf.interpolate(agg, cmap=self.model.color_ramp, how=self.model.transfer_function) + if self.model.spread_size > 0: + pix = tf.spread(pix, px=self.model.spread_size) + def update_plots(): + def downsample_categorical(aggregate, factor): + ys, xs, zs = aggregate.shape + crarr = aggregate[:ys-(ys % int(factor)),:xs-(xs % int(factor))] + return np.nanmean(np.concatenate([[crarr[i::factor,j::factor] + for i in range(factor)] + for j in range(factor)]), axis=0) + def downsample(aggregate, factor): ys, xs = aggregate.shape crarr = aggregate[:ys-(ys % int(factor)),:xs-(xs % int(factor))] @@ -93,9 +104,7 @@ def downsample(aggregate, factor): for i in range(factor)] for j in range(factor)]), axis=0) - # update hover layer - hover_agg = downsample(agg.values, self.model.hover_size) - + # update hover layer ---------------------------------------------------- sq_xs = np.linspace(self.model.map_extent[0], self.model.map_extent[2], agg.shape[1] / self.model.hover_size) @@ -107,21 +116,68 @@ def downsample(aggregate, factor): agg_xs, agg_ys = np.meshgrid(sq_xs, sq_ys) self.model.hover_source.data['x'] = agg_xs.flatten() self.model.hover_source.data['y'] = agg_ys.flatten() - self.model.hover_source.data['value'] = hover_agg.flatten() - - # update legend - min_val = agg.values.min() - max_val = agg.values.max() - vals = np.vstack([np.linspace(min_val, max_val, 180)] * 18) - vals_arr = DataArray(vals) - img = tf.interpolate(vals_arr, - cmap=[(255, 204, 204), 'red'], - how=self.model.transfer_function) - self.model.legend_source.data['image'] = [img.values] - self.model.legend_source.data['x'] = [min_val] - self.model.legend_source.data['dw'] = [max_val - min_val] - self.model.legend_fig.x_range.start = min_val - self.model.legend_fig.x_range.end = max_val + + if self.model.field in self.model.categorical_fields: + hover_agg = downsample_categorical(agg.values, self.model.hover_size) + cats = agg[agg.dims[2]].values.tolist() + tooltips = [] + for i, e in enumerate(cats): + self.model.hover_source.data[e] = hover_agg[:,:,i].flatten() + tooltips.append((e, '@{}'.format(e))) + self.model.hover_tool.tooltips = tooltips + + else: + hover_agg = downsample(agg.values, self.model.hover_size) + self.model.hover_source.data['value'] = hover_agg.flatten() + self.model.hover_tool.tooltips = [(self.model.field_title, '@value')] + + # update legend --------------------------------------------------------- + if self.model.field in self.model.categorical_fields: + + cat_dim = agg.dims[-1] + len_bar=900 + cats = agg[cat_dim].values.tolist() + total = agg.sum(dim=cat_dim) + min_val = int(total.min().data) + max_val = int(total.max().data) + scale = np.linspace(min_val, max_val, 180, dtype=total.dtype) + cats = agg.coords[agg.dims[-1]].values + ncats = len(cats) + data = np.zeros((180, ncats, ncats), dtype=total.dtype) + data[:, np.arange(ncats), np.arange(ncats)] = scale[:, None] + cbar = DataArray(data, dims=['value', 'fake', cat_dim], + coords=[scale, cats, cats]) + img = tf.colorize(cbar, self.model.colormap, how=self.model.transfer_function) + + dw = max_val - min_val + legend_fig = self.model.create_legend(img.values.T, + x=min_val, + y=0, + dh=18 * ncats, + dw=dw, + x_start=min_val, + x_end=max_val, + y_range=(0,18 * ncats)) + self.model.legend_vbox.children = [legend_fig] + + else: + min_val = np.nanmin(agg.values) + max_val = np.nanmax(agg.values) + vals = np.linspace(min_val, max_val, 180)[None, :] + vals_arr = DataArray(vals) + img = tf.interpolate(vals_arr, cmap=self.model.color_ramp, + how=self.model.transfer_function) + dw = max_val - min_val + legend_fig = self.model.create_legend(img.values, + x=min_val, + y=0, + dh=18, + dw=dw, + x_start=min_val, + x_end=max_val, + y_range=(0,18)) + + self.model.legend_vbox.children = [legend_fig] server.get_sessions('/')[0].with_document_locked(update_plots) # serialize to image @@ -135,6 +191,8 @@ class AppState(object): def __init__(self, config_file, outofcore, app_port): self.load_config_file(config_file) + self.plot_height = 560 + self.plot_width = 810 self.aggregate_functions = OrderedDict() self.aggregate_functions['Count'] = ds.count @@ -149,6 +207,7 @@ def __init__(self, config_file, outofcore, app_port): self.transfer_functions[u"\u221B - Cube Root"] = 'cbrt' self.transfer_functions['Log'] = 'log' self.transfer_functions['Linear'] = 'linear' + self.transfer_functions['Histogram Equalization'] = 'eq_hist' self.transfer_function = list(self.transfer_functions.values())[0] self.basemaps = OrderedDict() @@ -186,12 +245,14 @@ def __init__(self, config_file, outofcore, app_port): self.hover_source = ColumnDataSource(data=dict(x=[], y=[], val=[])) self.hover_size = 8 - # legend - self.legend_source = ColumnDataSource(data=dict(image=[], - x=[0], - y=[0], - dw=[180], - dh=[18])) + # spreading + self.spread_size = 1 + + # color ramps + self.color_ramps = OrderedDict() + self.color_ramps['BrBG'] = BrBG9 + self.color_ramps['PiYG'] = PiYG9 + self.color_ramp = list(self.color_ramps.values())[0] def load_config_file(self, config_path): '''load and parse yaml config file''' @@ -231,6 +292,7 @@ def load_config_file(self, config_path): self.ordinal_fields.append(f['field']) self.field = list(self.fields.values())[0] + self.field_title = list(self.fields.keys())[0] if self.colormaps: self.colormap = self.colormaps[list(self.fields.keys())[0]] @@ -252,7 +314,7 @@ def load_datasets(self,outofcore): load_fields = [f for f in self.fields.values() if f is not None] + axes_fields - if data_path.endswith(".csv") : + if data_path.endswith(".csv"): self.df = pd.read_csv(data_path, usecols=load_fields) # parse categorical fields @@ -268,6 +330,29 @@ def load_datasets(self,outofcore): else: raise IOError("Unknown data file type; .csv and .castra currently supported") + def create_legend(self, img, x, y, dw, dh, x_start, x_end, y_range): + legend_fig = Figure(x_range=(x_start, x_end), + plot_height=max(dh, 50), + plot_width=self.plot_width, + lod_threshold=None, + toolbar_location=None, + y_range=y_range) + + legend_fig.min_border_top = 0 + legend_fig.min_border_bottom = 10 + legend_fig.min_border_left = 0 + legend_fig.min_border_right = 0 + legend_fig.yaxis.visible = False + legend_fig.grid.grid_line_alpha = 0 + + legend_fig.image_rgba(image=[img], + x=[x], + y=[y], + dw=[dw], + dh=[dh], + dw_units='screen') + return legend_fig + class AppView(object): @@ -286,15 +371,14 @@ def create_layout(self): self.fig = Figure(tools='wheel_zoom,pan', x_range=self.x_range, lod_threshold=None, + plot_width=self.model.plot_width, + plot_height=self.model.plot_height, y_range=self.y_range) self.fig.min_border_top = 0 self.fig.min_border_bottom = 10 self.fig.min_border_left = 0 self.fig.min_border_right = 0 - - self.fig.plot_height = 560 - self.fig.plot_width = 810 self.fig.axis.visible = False # add tiled basemap @@ -315,18 +399,18 @@ def create_layout(self): # Add a hover tool self.invisible_square = Square(x='x', - y='y', - fill_color=None, - line_color=None, - size=self.model.hover_size) + y='y', + fill_color=None, + line_color=None, + size=self.model.hover_size) self.visible_square = Square(x='x', - y='y', - fill_color='#79DCDE', - fill_alpha=.5, - line_color='#79DCDE', - line_alpha=1, - size=self.model.hover_size) + y='y', + fill_color='#79DCDE', + fill_alpha=.5, + line_color='#79DCDE', + line_alpha=1, + size=self.model.hover_size) cr = self.fig.add_glyph(self.model.hover_source, self.invisible_square, @@ -335,35 +419,12 @@ def create_layout(self): code = "source.set('selected', cb_data['index']);" callback = CustomJS(args={'source': self.model.hover_source}, code=code) - - self.hover_tool = HoverTool(tooltips=[(self.model.fields.keys()[0], "@value")], + self.model.hover_tool = HoverTool(tooltips=[(self.model.fields.keys()[0], "@value")], callback=callback, renderers=[cr], mode='mouse') - - self.model.legend_fig = Figure(x_range=(0, 180), - plot_height=50, - plot_width=self.fig.plot_width, - lod_threshold=None, - toolbar_location=None, - y_range=(0,18)) - - self.model.legend_fig.min_border_top = 0 - self.model.legend_fig.min_border_bottom = 10 - self.model.legend_fig.min_border_left = 0 - self.model.legend_fig.min_border_right = 0 - self.model.legend_fig.yaxis.visible = False - self.model.legend_fig.grid.grid_line_alpha = 0 - - self.model.legend_fig.image_rgba(source=self.model.legend_source, - image='image', - x='x', - y='y', - dw='dw', - dh='dh', - dw_units='screen') - - self.fig.add_tools(self.hover_tool) + self.fig.add_tools(self.model.hover_tool) + self.model.legend_vbox = VBox() # add ui components controls = [] @@ -386,6 +447,10 @@ def create_layout(self): transfer_select.on_change('value', self.on_transfer_function_change) controls.append(transfer_select) + color_ramp_select = Select.create(name='Color Ramp', options=self.model.color_ramps) + color_ramp_select.on_change('value', self.on_color_ramp_change) + controls.append(color_ramp_select) + # add map components basemap_select = Select.create(name='Basemap', value='Toner', options=self.model.basemaps) @@ -399,11 +464,17 @@ def create_layout(self): end=100, step=1) basemap_opacity_slider.on_change('value', self.on_basemap_opacity_slider_change) + spread_size_slider = Slider(title="Spread Size (px)", value=0, start=0, + end=10, step=1) + spread_size_slider.on_change('value', self.on_spread_size_change) + controls.append(spread_size_slider) + hover_size_slider = Slider(title="Hover Size (px)", value=8, start=4, end=30, step=1) hover_size_slider.on_change('value', self.on_hover_size_change) controls.append(hover_size_slider) + show_labels_chk = CheckboxGroup(labels=["Show Labels"], active=[0]) show_labels_chk.on_click(self.on_labels_change) @@ -414,7 +485,7 @@ def create_layout(self): self.map_controls = HBox(width=self.fig.plot_width, children=map_controls) self.map_area = VBox(width=self.fig.plot_width, children=[self.map_controls, self.fig, - self.model.legend_fig]) + self.model.legend_vbox]) self.layout = HBox(width=1024, children=[self.controls, self.map_area]) def update_image(self): @@ -423,13 +494,12 @@ def update_image(self): extra_url_vars=self.model.shader_url_vars) def on_field_change(self, attr, old, new): + self.model.field_title = new self.model.field = self.model.fields[new] - self.hover_tool.tooltips = [(new, '@value')] self.update_image() if not self.model.field: self.aggregate_select.options = [dict(name="No Aggregates Available", value="")] - elif self.model.field in self.model.categorical_fields: self.aggregate_select.options = [dict(name="Categorical", value="count_cat")] else: @@ -446,6 +516,10 @@ def on_hover_size_change(self, attr, old, new): self.visible_square.size = int(new) self.update_image() + def on_spread_size_change(self, attr, old, new): + self.model.spread_size = int(new) + self.update_image() + def on_axes_change(self, attr, old, new): self.model.active_axes = self.model.axes[new] self.update_image() @@ -458,6 +532,10 @@ def on_transfer_function_change(self, attr, old, new): self.model.transfer_function = self.model.transfer_functions[new] self.update_image() + def on_color_ramp_change(self, attr, old, new): + self.model.color_ramp = self.model.color_ramps[new] + self.update_image() + def on_image_opacity_slider_change(self, attr, old, new): self.image_renderer.alpha = new / 100 From 3a9bc99a82937f9b8cea0410b5013b87d67e4eb8 Mon Sep 17 00:00:00 2001 From: bcollins Date: Thu, 24 Mar 2016 17:20:35 -0500 Subject: [PATCH 16/22] resized plot to better fit 15" monitor --- examples/dashboard/dashboard.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 206250875..249683161 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -191,8 +191,8 @@ class AppState(object): def __init__(self, config_file, outofcore, app_port): self.load_config_file(config_file) - self.plot_height = 560 - self.plot_width = 810 + self.plot_height = 600 + self.plot_width = 1124 self.aggregate_functions = OrderedDict() self.aggregate_functions['Count'] = ds.count @@ -486,7 +486,7 @@ def create_layout(self): self.map_area = VBox(width=self.fig.plot_width, children=[self.map_controls, self.fig, self.model.legend_vbox]) - self.layout = HBox(width=1024, children=[self.controls, self.map_area]) + self.layout = HBox(width=1366, children=[self.controls, self.map_area]) def update_image(self): self.model.shader_url_vars['cachebust'] = str(uuid.uuid4()) From c72e2a0fab401b91dc8f7e4cb17b1f29a65d4b44 Mon Sep 17 00:00:00 2001 From: bcollins Date: Thu, 24 Mar 2016 17:31:11 -0500 Subject: [PATCH 17/22] added log x_axis_type for non-linear transfer_functions --- examples/dashboard/dashboard.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 249683161..54fa2cc66 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -331,12 +331,15 @@ def load_datasets(self,outofcore): raise IOError("Unknown data file type; .csv and .castra currently supported") def create_legend(self, img, x, y, dw, dh, x_start, x_end, y_range): + + x_axis_type = 'linear' if self.transfer_function == 'linear' else 'log' legend_fig = Figure(x_range=(x_start, x_end), plot_height=max(dh, 50), plot_width=self.plot_width, lod_threshold=None, toolbar_location=None, - y_range=y_range) + y_range=y_range, + x_axis_type=x_axis_type) legend_fig.min_border_top = 0 legend_fig.min_border_bottom = 10 From 7ccdab3e422eba5ebc3de63d0af10cdd4ec1d079 Mon Sep 17 00:00:00 2001 From: bcollins Date: Fri, 25 Mar 2016 10:37:54 -0500 Subject: [PATCH 18/22] adjusted padding on ordinal colorbar, added older-style categorical legend to left-hand controls menu --- examples/dashboard/census.yml | 6 +++ examples/dashboard/dashboard.py | 91 ++++++++++++++++++--------------- 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/examples/dashboard/census.yml b/examples/dashboard/census.yml index 393fb66c9..40b85837f 100644 --- a/examples/dashboard/census.yml +++ b/examples/dashboard/census.yml @@ -24,6 +24,12 @@ summary_fields: a: red h: orange o: saddlebrown + cat_names: + w: White + b: Black + a: Asian + h: Hispanic + o: Other - name: Counts field: None diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 54fa2cc66..1fe2fdb77 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -24,6 +24,7 @@ DynamicImageRenderer, HBox, VBox) from bokeh.models import Select, Slider, CheckboxGroup, CustomJS, ColumnDataSource, Square, HoverTool +from bokeh.models import Plot, Text, Circle from bokeh.palettes import BrBG9, PiYG9 from tornado.ioloop import IOLoop @@ -133,32 +134,9 @@ def downsample(aggregate, factor): # update legend --------------------------------------------------------- if self.model.field in self.model.categorical_fields: - - cat_dim = agg.dims[-1] - len_bar=900 - cats = agg[cat_dim].values.tolist() - total = agg.sum(dim=cat_dim) - min_val = int(total.min().data) - max_val = int(total.max().data) - scale = np.linspace(min_val, max_val, 180, dtype=total.dtype) - cats = agg.coords[agg.dims[-1]].values - ncats = len(cats) - data = np.zeros((180, ncats, ncats), dtype=total.dtype) - data[:, np.arange(ncats), np.arange(ncats)] = scale[:, None] - cbar = DataArray(data, dims=['value', 'fake', cat_dim], - coords=[scale, cats, cats]) - img = tf.colorize(cbar, self.model.colormap, how=self.model.transfer_function) - - dw = max_val - min_val - legend_fig = self.model.create_legend(img.values.T, - x=min_val, - y=0, - dh=18 * ncats, - dw=dw, - x_start=min_val, - x_end=max_val, - y_range=(0,18 * ncats)) - self.model.legend_vbox.children = [legend_fig] + cat_legend = self.model.create_categorical_legend(self.model.colormap, self.model.colornames) + self.model.legend_side_vbox.children = [cat_legend] + self.model.legend_bottom_vbox.children = [] else: min_val = np.nanmin(agg.values) @@ -177,7 +155,8 @@ def downsample(aggregate, factor): x_end=max_val, y_range=(0,18)) - self.model.legend_vbox.children = [legend_fig] + self.model.legend_bottom_vbox.children = [legend_fig] + self.model.legend_side_vbox.children = [] server.get_sessions('/')[0].with_document_locked(update_plots) # serialize to image @@ -279,6 +258,7 @@ def load_config_file(self, config_path): # parse summary field self.fields = OrderedDict() self.colormaps = OrderedDict() + self.color_name_maps = OrderedDict() self.ordinal_fields = [] self.categorical_fields = [] for f in self.config['summary_fields']: @@ -287,6 +267,7 @@ def load_config_file(self, config_path): if 'cat_colors' in f.keys(): self.colormaps[f['name']] = f['cat_colors'] self.categorical_fields.append(f['field']) + self.color_name_maps[f['name']] = f['cat_names'] elif f['field'] != 'None': self.ordinal_fields.append(f['field']) @@ -296,6 +277,7 @@ def load_config_file(self, config_path): if self.colormaps: self.colormap = self.colormaps[list(self.fields.keys())[0]] + self.colornames = self.color_name_maps[list(self.fields.keys())[0]] def load_datasets(self,outofcore): data_path = self.config['file'] @@ -330,6 +312,30 @@ def load_datasets(self,outofcore): else: raise IOError("Unknown data file type; .csv and .castra currently supported") + def create_categorical_legend(self, colormap, colornames): + plot_options = {} + plot_options['x_range'] = Range1d(start=0, end=200) + plot_options['y_range'] = Range1d(start=0, end=100) + plot_options['plot_height'] = 120 + plot_options['plot_width'] = 190 + + plot_options['min_border_bottom'] = 0 + plot_options['min_border_left'] = 0 + plot_options['min_border_right'] = 0 + plot_options['min_border_top'] = 0 + plot_options['outline_line_width'] = 0 + plot_options['toolbar_location'] = None + + legend = Plot(**plot_options) + regions = list(colormap.keys()) + colors = list(colormap.values()) + for i, (region, color) in enumerate(zip(regions, colors)): + text_y = 95 - i * 20 + legend.add_glyph(Text(x=40, y=text_y-12, text=[colornames[region]], text_font_size='10pt', text_color='#666666')) + legend.add_glyph(Circle(x=15, y=text_y-5, fill_color=color, size=10, line_color=None, fill_alpha=0.8)) + + return legend + def create_legend(self, img, x, y, dw, dh, x_start, x_end, y_range): x_axis_type = 'linear' if self.transfer_function == 'linear' else 'log' @@ -343,8 +349,8 @@ def create_legend(self, img, x, y, dw, dh, x_start, x_end, y_range): legend_fig.min_border_top = 0 legend_fig.min_border_bottom = 10 - legend_fig.min_border_left = 0 - legend_fig.min_border_right = 0 + legend_fig.min_border_left = 15 + legend_fig.min_border_right = 15 legend_fig.yaxis.visible = False legend_fig.grid.grid_line_alpha = 0 @@ -427,7 +433,8 @@ def create_layout(self): renderers=[cr], mode='mouse') self.fig.add_tools(self.model.hover_tool) - self.model.legend_vbox = VBox() + self.model.legend_side_vbox = VBox() + self.model.legend_bottom_vbox = VBox() # add ui components controls = [] @@ -454,6 +461,18 @@ def create_layout(self): color_ramp_select.on_change('value', self.on_color_ramp_change) controls.append(color_ramp_select) + spread_size_slider = Slider(title="Spread Size (px)", value=0, start=0, + end=10, step=1) + spread_size_slider.on_change('value', self.on_spread_size_change) + controls.append(spread_size_slider) + + hover_size_slider = Slider(title="Hover Size (px)", value=8, start=4, + end=30, step=1) + hover_size_slider.on_change('value', self.on_hover_size_change) + controls.append(hover_size_slider) + + controls.append(self.model.legend_side_vbox) + # add map components basemap_select = Select.create(name='Basemap', value='Toner', options=self.model.basemaps) @@ -467,16 +486,6 @@ def create_layout(self): end=100, step=1) basemap_opacity_slider.on_change('value', self.on_basemap_opacity_slider_change) - spread_size_slider = Slider(title="Spread Size (px)", value=0, start=0, - end=10, step=1) - spread_size_slider.on_change('value', self.on_spread_size_change) - controls.append(spread_size_slider) - - hover_size_slider = Slider(title="Hover Size (px)", value=8, start=4, - end=30, step=1) - hover_size_slider.on_change('value', self.on_hover_size_change) - controls.append(hover_size_slider) - show_labels_chk = CheckboxGroup(labels=["Show Labels"], active=[0]) show_labels_chk.on_click(self.on_labels_change) @@ -488,7 +497,7 @@ def create_layout(self): self.map_controls = HBox(width=self.fig.plot_width, children=map_controls) self.map_area = VBox(width=self.fig.plot_width, children=[self.map_controls, self.fig, - self.model.legend_vbox]) + self.model.legend_bottom_vbox]) self.layout = HBox(width=1366, children=[self.controls, self.map_area]) def update_image(self): From ec16992502c60b908d7c09f46642208997dcfbbe Mon Sep 17 00:00:00 2001 From: bcollins Date: Fri, 25 Mar 2016 14:59:48 -0500 Subject: [PATCH 19/22] fixed logscale colorbar --- examples/dashboard/dashboard.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 1fe2fdb77..d67e5fef5 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -140,8 +140,21 @@ def downsample(aggregate, factor): else: min_val = np.nanmin(agg.values) + + if min_val == 0: + min_val = agg.data[agg.data > 0].min() + max_val = np.nanmax(agg.values) - vals = np.linspace(min_val, max_val, 180)[None, :] + + + if self.model.transfer_function == 'linear': + vals = np.linspace(min_val, max_val, 180)[None, :] + else: + vals = (np.logspace(0, + np.log1p(max_val-min_val), + base=np.e, num=180, + dtype=min_val.dtype) + min_val)[None,:] + vals_arr = DataArray(vals) img = tf.interpolate(vals_arr, cmap=self.model.color_ramp, how=self.model.transfer_function) From 92a68416152c10e4862a023971f74e7053fbd4f0 Mon Sep 17 00:00:00 2001 From: bcollins Date: Fri, 25 Mar 2016 16:00:42 -0500 Subject: [PATCH 20/22] switched out diverging colorramps for sequential --- examples/dashboard/dashboard.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index d67e5fef5..7d46c48f2 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -25,7 +25,7 @@ from bokeh.models import Select, Slider, CheckboxGroup, CustomJS, ColumnDataSource, Square, HoverTool from bokeh.models import Plot, Text, Circle -from bokeh.palettes import BrBG9, PiYG9 +from bokeh.palettes import GnBu9, OrRd9, PuRd9, YlGnBu9, Greys9 from tornado.ioloop import IOLoop from tornado.web import RequestHandler @@ -242,8 +242,11 @@ def __init__(self, config_file, outofcore, app_port): # color ramps self.color_ramps = OrderedDict() - self.color_ramps['BrBG'] = BrBG9 - self.color_ramps['PiYG'] = PiYG9 + self.color_ramps['Orange-Red'] = list(reversed(OrRd9))[2:] + self.color_ramps['Green-Blue'] = list(reversed(GnBu9))[2:] + self.color_ramps['Purple-Red'] = list(reversed(PuRd9))[2:] + self.color_ramps['Yellow-Green-Blue'] = list(reversed(YlGnBu9))[2:] + self.color_ramps['Grays'] = list(reversed(Greys9))[2:] self.color_ramp = list(self.color_ramps.values())[0] def load_config_file(self, config_path): From 602b10e5247875d5c155e63d47426384a55565d6 Mon Sep 17 00:00:00 2001 From: bcollins Date: Fri, 25 Mar 2016 16:01:16 -0500 Subject: [PATCH 21/22] removed pdb --- examples/dashboard/dashboard.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index 7d46c48f2..c6cec6caa 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -5,7 +5,6 @@ import yaml import webbrowser import uuid -import pdb from collections import OrderedDict From 0ef5ec347f358862d4576cf9c9565fa67a2995d4 Mon Sep 17 00:00:00 2001 From: bcollins Date: Fri, 25 Mar 2016 16:03:04 -0500 Subject: [PATCH 22/22] small import cleanup --- examples/dashboard/dashboard.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/dashboard/dashboard.py b/examples/dashboard/dashboard.py index c6cec6caa..83b6cd7da 100644 --- a/examples/dashboard/dashboard.py +++ b/examples/dashboard/dashboard.py @@ -22,7 +22,10 @@ from bokeh.models import (Range1d, ImageSource, WMTSTileSource, TileRenderer, DynamicImageRenderer, HBox, VBox) -from bokeh.models import Select, Slider, CheckboxGroup, CustomJS, ColumnDataSource, Square, HoverTool +from bokeh.models import (Select, Slider, CheckboxGroup, + CustomJS, ColumnDataSource, + Square, HoverTool) + from bokeh.models import Plot, Text, Circle from bokeh.palettes import GnBu9, OrRd9, PuRd9, YlGnBu9, Greys9