diff --git a/README.md b/README.md index 7a97241c..b577b957 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **NOTE** *This repository is waiting for adoption and there might be longer waiting times for answers. However, feel free to report bugs or ask questions in the issues. -Last known working versions of major dependencies can be found [here](https://github.com/automl/issues/* +Last known working versions of major dependencies can be found [here](https://github.com/automl/issues/).* ## Configuration Assessment, Visualization and Evaluation diff --git a/cave/analyzer/configurator/parallel_coordinates.py b/cave/analyzer/configurator/parallel_coordinates.py index 60086404..6c5e1832 100644 --- a/cave/analyzer/configurator/parallel_coordinates.py +++ b/cave/analyzer/configurator/parallel_coordinates.py @@ -5,6 +5,9 @@ import pandas as pd from ConfigSpace.configuration_space import ConfigurationSpace, Configuration from ConfigSpace.hyperparameters import NumericalHyperparameter, CategoricalHyperparameter +from bokeh.io import output_notebook +from bokeh.models.annotations import Title +from bokeh.plotting import show from bokeh.embed import components from bokeh.layouts import column from bokeh.models import Div @@ -235,3 +238,14 @@ def get_html(self, d=None, tooltip=None): result["tooltip"] = self.__doc__ d["Parallel Coordinates"] = result return result + + def get_jupyter(self): + bokeh_plots = self.plot_bokeh() + output_notebook() + if len(self.result) == 0: + if(bokeh_plots): + show(bokeh_plots) + return + for budget, bokeh_plot in bokeh_plots.items(): + plot = column( Div(text=budget), bokeh_plot) # to add a title + show(plot) \ No newline at end of file diff --git a/cave/analyzer/parameter_importance/base_parameter_importance.py b/cave/analyzer/parameter_importance/base_parameter_importance.py index 261a8ba9..5a57f287 100644 --- a/cave/analyzer/parameter_importance/base_parameter_importance.py +++ b/cave/analyzer/parameter_importance/base_parameter_importance.py @@ -109,3 +109,22 @@ def get_html(self, d=None, tooltip=None): "and the whiskers are quartiles."} return super().get_html(d, tooltip) + + def get_jupyter(self): + from IPython.core.display import HTML, display + import matplotlib.pyplot as plt + import matplotlib + from importlib import reload + matplotlib.use('nbAgg') # GUI backend + matplotlib = reload(matplotlib) + for b, data in self.result['Importances Per Parameter'].items(): + im_list = [] + for component, _ in data.items(): + if(component == "figure"): + im_list += [plt.imread(f) for f in data[component]] + f, axes = plt.subplots(1, len(im_list), figsize = (6*len(im_list),5)) + for img, ax in zip(im_list, axes): + ax.imshow(img) + ax.axis('off') + f.suptitle(b, fontsize=14) + plt.show() diff --git a/cave/analyzer/parameter_importance/fanova.py b/cave/analyzer/parameter_importance/fanova.py index 790b5279..a72ae9f0 100644 --- a/cave/analyzer/parameter_importance/fanova.py +++ b/cave/analyzer/parameter_importance/fanova.py @@ -99,19 +99,54 @@ def parse_pairwise(p): return result + def _plot_img_lst(self, plt, img_lst, max_cols, sup_title): + import math + rows = math.ceil( len(img_lst)/max_cols ) + figure, axes = plt.subplots(rows, max_cols, figsize = (5*max_cols,5*rows)) + i = 0 + for r in range(rows): + for c in range(max_cols): + if(rows > 1): + if(i < len(img_lst)): + axes[r, c].imshow(img_lst[i]) + axes[r, c].axis('off') + else: + if(i < len(img_lst)): + axes[c].imshow(img_lst[i]) + axes[c].axis('off') + i += 1 + + figure.suptitle(sup_title, fontsize=14) + return figure + def get_jupyter(self): - from IPython.core.display import HTML, Image, display - for b, result in self.result.items(): - error = self.result[b]['else'] if 'else' in self.result[b] else None - if error: - display(HTML(error)) - else: - # Show table - display(HTML(self.result[b]["Importance"]["table"])) - # Show plots - display(*list([Image(filename=d["figure"]) for d in self.result[b]['Marginals'].values()])) - display(*list([Image(filename=d["figure"]) for d in self.result[b]['Pairwise Marginals'].values()])) - # While working for a prettier solution, this might be an option: - # display(HTML(figure_to_html([d["figure"] for d in self.result[b]['Marginals'].values()] + - # [d["figure"] for d in self.result[b]['Pairwise Marginals'].values()], - # max_in_a_row=3, true_break_between_rows=True))) + # from IPython.core.display import HTML, Image, display + # for b, result in self.result['Importances Per Parameter'].items(): + # display(HTML(result["Importance"]["table"])) + # # Show plots + # display(*list([Image(filename=d["figure"]) for d in result['Marginals'].values()])) + # display(*list([Image(filename=d["figure"]) for d in result['Pairwise Marginals'].values()])) + # Reload matplotlib backend to avoid GUI blank plots + from IPython.core.display import HTML, display + import matplotlib + import matplotlib.pyplot as plt + from importlib import reload + matplotlib.use('nbAgg') # GUI backend + matplotlib = reload(matplotlib) + + for b, result in self.result['Importances Per Parameter'].items(): + # Show Table + display(HTML(result["Importance"]["table"])) + + # Show plots + marginals_lst, pairws_marginals_lst = [], [] + marginals_lst = [plt.imread(hp['figure']) for hp in result['Marginals'].values()] + pairws_marginals_lst = [plt.imread(hp['figure']) for hp in result['Pairwise Marginals'].values()] + + f_marginals = self._plot_img_lst(plt, marginals_lst, + max_cols = 2, + sup_title = b + ": Marginals") + f_pairwise_marginals = self._plot_img_lst(plt, pairws_marginals_lst,\ + max_cols = 2, + sup_title = b + ": Pairwise Marginals") + plt.show() diff --git a/cave/analyzer/parameter_importance/local_parameter_importance.py b/cave/analyzer/parameter_importance/local_parameter_importance.py index 27db01d5..5cf9849d 100644 --- a/cave/analyzer/parameter_importance/local_parameter_importance.py +++ b/cave/analyzer/parameter_importance/local_parameter_importance.py @@ -27,7 +27,7 @@ def get_name(self): def postprocess(self, pimp, output_dir): param_imp = pimp.evaluator.evaluated_parameter_importance plots = OrderedDict() - for p, i in [(k, v) for k, v in sorted(param_imp.items(), + for p, _ in [(k, v) for k, v in sorted(param_imp.items(), key=operator.itemgetter(1), reverse=True)]: plots[p] = os.path.join(output_dir, 'lpi', p + '.png') return OrderedDict([ @@ -35,8 +35,42 @@ def postprocess(self, pimp, output_dir): ]) def get_jupyter(self): + # from IPython.core.display import HTML, display, Image + # for b, budget_dict in self.result['Importances Per Parameter'].items(): + # for plots, d in budget_dict.items(): + # if(plots != 'Interactive Plots'): + # display(Image(filename = d["figure"])) + # Reload matplotlib backend to avoid GUI blank plots from IPython.core.display import HTML, display - display(HTML(figure_to_html(self.get_plots(), max_in_a_row=3, true_break_between_rows=True))) + import matplotlib + import matplotlib.pyplot as plt + from importlib import reload + import math + matplotlib.use('nbAgg') # GUI backend + matplotlib = reload(matplotlib) + + max_cols = 2 + for b, data in self.result['Importances Per Parameter'].items(): + img_lst = [ plt.imread( c['figure'] ) for c in data.values() if 'figure' in c ] + # Plot in grid + rows = math.ceil( len(img_lst)/max_cols ) + figure, axes = plt.subplots(rows, max_cols, figsize = (5*max_cols,4*rows)) + i = 0 + for r in range(rows): + for c in range(max_cols): + if(rows > 1): + if(i < len(img_lst)): + axes[r, c].imshow(img_lst[i]) + axes[r, c].axis('off') + else: + if(i < len(img_lst)): + axes[c].imshow(img_lst[i]) + axes[c].axis('off') + i += 1 + + figure.suptitle(b, fontsize=14) + plt.show() + if self.runscontainer.analyzing_options['Parameter Importance'].getboolean('whisker_quantiles_plot'): output_notebook() show(self.plot_whiskers()) diff --git a/cave/plot/configurator_footprint.py b/cave/plot/configurator_footprint.py index 96c492d3..c32c1d28 100644 --- a/cave/plot/configurator_footprint.py +++ b/cave/plot/configurator_footprint.py @@ -21,7 +21,7 @@ from bokeh.models.widgets import CheckboxButtonGroup, RadioButtonGroup, Button, Div from bokeh.plotting import figure, ColumnDataSource from sklearn.decomposition import PCA -from sklearn.manifold.mds import MDS +from sklearn.manifold import MDS from sklearn.preprocessing import StandardScaler from smac.epm.rf_with_instances import RandomForestWithInstances from smac.runhistory.runhistory import RunHistory