diff --git a/holoviews/plotting/bokeh/chart.py b/holoviews/plotting/bokeh/chart.py
index caa7c493fc..b242ac08e1 100644
--- a/holoviews/plotting/bokeh/chart.py
+++ b/holoviews/plotting/bokeh/chart.py
@@ -73,16 +73,16 @@ def get_data(self, element, ranges=None, empty=False):
             else:
                 ms = style.get('size', np.sqrt(6))**2
                 sizes = element.dimension_values(self.size_index)
-                if sizes.dtype.kind in ('i', 'f'):
-                    sizes = compute_sizes(sizes, self.size_fn,
-                                          self.scaling_factor,
-                                          self.scaling_method, ms)
-                    data[map_key] = np.sqrt(sizes)
-                    mapping['size'] = map_key
-                else:
+                sizes = compute_sizes(sizes, self.size_fn,
+                                      self.scaling_factor,
+                                      self.scaling_method, ms)
+                if sizes is None:
                     eltype = type(element).__name__
                     self.warning('%s dimension is not numeric, cannot '
                                  'use to scale %s size.' % (sdim, eltype))
+                else:
+                    data[map_key] = np.sqrt(sizes)
+                    mapping['size'] = map_key
 
         data[dims[xidx]] = [] if empty else element.dimension_values(xidx)
         data[dims[yidx]] = [] if empty else element.dimension_values(yidx)
diff --git a/holoviews/plotting/mpl/chart.py b/holoviews/plotting/mpl/chart.py
index 4e64916dc3..3cfe8eccb5 100644
--- a/holoviews/plotting/mpl/chart.py
+++ b/holoviews/plotting/mpl/chart.py
@@ -510,15 +510,15 @@ def _compute_styles(self, element, ranges, style):
         sdim = element.get_dimension(self.size_index)
         if sdim:
             sizes = element.dimension_values(self.size_index)
-            if sizes.dtype.kind in ('i', 'f'):
-                style['s'] = compute_sizes(sizes, self.size_fn, self.scaling_factor,
-                                           self.scaling_method, ms)
-                ms = style.pop('s') if 's' in style else plt.rcParams['lines.markersize']
-            else:
+            ms = style['s'] if 's' in style else plt.rcParams['lines.markersize']
+            sizes = compute_sizes(sizes, self.size_fn, self.scaling_factor,
+                                  self.scaling_method, ms)
+            if sizes is None:
                 eltype = type(element).__name__
                 self.warning('%s dimension is not numeric, cannot '
                              'use to scale %s size.' % (sdim, eltype))
-
+            else:
+                style['s'] = sizes
         style['edgecolors'] = style.pop('edgecolors', 'none')
 
 
diff --git a/holoviews/plotting/util.py b/holoviews/plotting/util.py
index 80f6e4ea4f..e8ea200513 100644
--- a/holoviews/plotting/util.py
+++ b/holoviews/plotting/util.py
@@ -98,6 +98,8 @@ def compute_sizes(sizes, size_fn, scaling_factor, scaling_method, base_size):
     base size and size_fn, which will be applied before
     scaling.
     """
+    if sizes.dtype.kind not in ('i', 'f'):
+        return None
     if scaling_method == 'area':
         pass
     elif scaling_method == 'width':