From 4c5391300224bb8285e5c0b9cf45a8d712097dc3 Mon Sep 17 00:00:00 2001 From: leaprovenzano Date: Tue, 6 Jun 2017 15:49:17 +0100 Subject: [PATCH 1/7] added the ability to optimize loss, and changes to population scoring added some docstring for DEvol.run --- devol.py | 124 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 86 insertions(+), 38 deletions(-) diff --git a/devol.py b/devol.py index 94fc40f..99bb99a 100644 --- a/devol.py +++ b/devol.py @@ -1,5 +1,6 @@ from __future__ import print_function + from genome_handler import GenomeHandler import numpy as np from keras.models import Sequential @@ -11,98 +12,145 @@ import csv from tqdm import trange, tqdm import sys +import operator + +METRIC_OPS = [operator.__lt__, operator.__gt__] +METRIC_OBJECTIVES = [min, max] + class DEvol: def __init__(self, genome_handler, data_path=""): self.genome_handler = genome_handler self.datafile = data_path or (datetime.now().ctime() + '.csv') - self.bssf = (None, 0.) # model, accuracy + self.bssf = (None, float('inf'), 0.) # model, loss, accuracy + print("Genome encoding and accuracy data stored at", self.datafile, "\n") + def set_objective(self, metric): + """set the metric and objective for this search should be 'accuracy' or 'loss'""" + if metric is 'acc': + metric = 'accuracy' + if not metric in ['loss', 'accuracy']: + raise ValueError( + 'Invalid metric name {} provided - should be "accuracy" or "loss"'.format(metric)) + self.metric = metric + self.objective = "max" if self.metric is "accuracy" else "min" + self.metric_index = 1 if self.metric is 'loss' else -1 + self.metric_op = METRIC_OPS[self.objective is 'max'] + self.metric_objective = METRIC_OBJECTIVES[self.objective is 'max'] + # Create a population and evolve - # Returns best model found in the form of (model, accuracy) - def run(self, dataset, num_generations, pop_size, epochs, fitness=None): + # Returns best model found in the form of (model, loss, accuracy) + def run(self, dataset, num_generations, pop_size, epochs, fitness=None, metric='accuracy'): + """run genetic search on dataset given number of generations and population size + + Args: + dataset : tuple or list of numpy arrays in form ((train_data, train_labels), (validation_data, validation_labels)) + num_generations (int): number of generations to search + pop_size (int): initial population size + epochs (int): epochs to run each search, passed to keras model.fit -currently searches are + curtailed if no improvement is seen in 1 epoch + fitness (None, optional): scoring function to be applied to population scores, will be called on a numpy array + which is a min/max scaled version of evaluated model metrics, so + It should accept a real number including 0. If left as default just the min/max + scaled values will be used. + metric (str, optional): must be "accuracy" or "loss" , defines what to optimize during search + + Returns: + (keras model, float, float ): best model found in the form of (model, loss, accuracy) + """ + self.set_objective(metric) generations = trange(num_generations, desc="Generations") (self.x_train, self.y_train), (self.x_test, self.y_test) = dataset # Generate initial random population members = [self.genome_handler.generate() for _ in range(pop_size)] fit = [] + # where to look for our metric in bssf.. + metric_index = 1 if self.metric is 'loss' else -1 for i in trange(len(members), desc="Gen 1 Models Fitness Eval"): - loss, acc, model = self.evaluate(members[i], epochs) - if acc > self.bssf[1]: - self.bssf = (model, acc) - fit.append(acc) - pop = Population(members, fit, fitness) + res = self.evaluate(members[i], epochs) + v = res[metric_index] + if self.metric_op(v, self.bssf[metric_index]): + self.bssf = res + fit.append(v) + fit = np.array(fit) - tqdm.write("Generation 1:\t\tmax: {0}\t\taverage: {1}\t\tstd: {2}".format(max(fit), np.mean(fit), np.std(fit))) + pop = Population(members, fit, fitness, obj=self.objective) + tqdm.write("Generation 1:\t\tbest {3}: {0:0.4f}\t\taverage: {1:0.4f}\t\tstd: {2:0.4f}".format(self.metric_objective(fit), + np.mean(fit), np.std(fit), self.metric)) # Evolve over generations for gen in generations: if gen == 0: continue members = [] - for i in range(int(pop_size*0.95)): # Crossover + for i in range(int(pop_size * 0.95)): # Crossover members.append(self.crossover(pop.select(), pop.select())) - members += pop.getBest(pop_size - int(pop_size*0.95)) - for i in range(len(members)): # Mutation + members += pop.getBest(pop_size - int(pop_size * 0.95)) + for i in range(len(members)): # Mutation members[i] = self.mutate(members[i], gen) fit = [] for i in trange(len(members), desc="Gen %i Models Fitness Eval" % (gen + 1)): - loss, acc, model = self.evaluate(members[i], epochs) - if acc > self.bssf[1]: - self.bssf = (model, acc) - fit.append(acc) - pop = Population(members, fit, fitness) + res = self.evaluate(members[i], epochs) + v = res[metric_index] + if self.metric_op(v, self.bssf[metric_index]): + self.bssf = res + fit.append(v) fit = np.array(fit) - tqdm.write("Generation {3}:\t\tmax: {0}\t\taverage: {1}\t\tstd: {2}".format(max(fit), np.mean(fit), np.std(fit), gen + 1)) - + pop = Population(members, fit, fitness, obj=self.objective) + + tqdm.write("Generation {3}:\t\tbest {4}: {0:0.4f}\t\taverage: {1:0.4f}\t\tstd: {2:0.4f}".format(self.metric_objective(fit), + np.mean(fit), np.std(fit), gen + 1, self.metric)) return self.bssf def evaluate(self, genome, epochs): model = self.genome_handler.decode(genome) loss, accuracy = None, None model.fit(self.x_train, self.y_train, validation_data=(self.x_test, self.y_test), - epochs=epochs, - verbose=0, - callbacks=[EarlyStopping(monitor='val_loss', patience=1, verbose=0)]) + epochs=epochs, + verbose=0, + callbacks=[EarlyStopping(monitor='val_loss', patience=1, verbose=0)]) loss, accuracy = model.evaluate(self.x_test, self.y_test, verbose=0) - # Record the stats with open(self.datafile, 'a') as csvfile: - writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + writer = csv.writer(csvfile, delimiter=',', + quotechar='"', quoting=csv.QUOTE_MINIMAL) row = list(genome) + [loss, accuracy] - writer.writerow(row) + writer.writerow(row) + return model, loss, accuracy - return loss, accuracy, model - def crossover(self, genome1, genome2): crossIndexA = rand.randint(0, len(genome1)) child = genome1[:crossIndexA] + genome2[crossIndexA:] return child - + def mutate(self, genome, generation): - num_mutations = max(3, generation / 4) # increase mutations as program continues + # increase mutations as program continues + num_mutations = max(3, generation / 4) return self.genome_handler.mutate(genome, num_mutations) + class Population: def __len__(self): return len(self.members) - def __init__(self, members, fitnesses, score): + def __init__(self, members, fitnesses, score, obj='max'): self.members = members - fitnesses -= min(fitnesses) - fitnesses /= max(fitnesses) - self.scores = list(map(score or self.score, fitnesses)) + scores = fitnesses - fitnesses.min() + scores /= scores.max() + if obj is 'min': + scores = 1 - scores + if score: + self.scores = score(scores) + else: + self.scores = scores self.s_fit = sum(self.scores) - def score(self, fitness): - return (fitness * 100)**4 - def getBest(self, n): - combined = [(self.members[i], self.scores[i]) \ - for i in range(len(self.members))] + combined = [(self.members[i], self.scores[i]) + for i in range(len(self.members))] sorted(combined, key=(lambda x: x[1]), reverse=True) return [x[0] for x in combined[:n]] From f6ac03deb7533aa0fc323fabf4f82a3ee902877f Mon Sep 17 00:00:00 2001 From: leaprovenzano Date: Tue, 6 Jun 2017 15:55:36 +0100 Subject: [PATCH 2/7] updated demo since run now returns loss as well as accuracy, fixed image shape incompatiability with tensorflow backend --- demo.ipynb | 113 +++++++++++++++++++++++++++++------------------------ 1 file changed, 63 insertions(+), 50 deletions(-) diff --git a/demo.ipynb b/demo.ipynb index f0ec89d..43811d8 100644 --- a/demo.ipynb +++ b/demo.ipynb @@ -3,15 +3,13 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Using Theano backend.\n" + "Using TensorFlow backend.\n" ] } ], @@ -19,7 +17,8 @@ "from keras.datasets import mnist\n", "from keras.utils.np_utils import to_categorical\n", "from devol import DEvol, GenomeHandler\n", - "import numpy as np" + "import numpy as np\n", + "from keras import backend as K\n" ] }, { @@ -32,15 +31,36 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [], + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(60000, 28, 28)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train = x_train.reshape(x_train.shape[0], 1, 28, 28).astype('float32') / 255\n", - "x_test = x_test.reshape(x_test.shape[0], 1, 28, 28).astype('float32') / 255\n", + "x_train.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# set image format so that it's compatable with backend\n", + "image_format = (28, 28, 1) if K.image_data_format() is 'channels_last' else (1, 28, 28 )\n", + "\n", + "x_train = x_train.reshape((x_train.shape[0],) + image_format).astype('float32') / 255\n", + "x_test = x_test.reshape((x_test.shape[0],) + image_format).astype('float32') / 255\n", "y_train = to_categorical(y_train)\n", "y_test = to_categorical(y_test)\n", "dataset = ((x_train, y_train), (x_test, y_test))" @@ -56,11 +76,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[A\u001b[A" + ] + } + ], "source": [ "max_conv_layers = 6\n", "max_dense_layers = 2 # including final softmax layer\n", @@ -85,62 +113,47 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false + "scrolled": false }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Generations: 0%| | 0/10 [00:00 Date: Tue, 6 Jun 2017 16:00:00 +0100 Subject: [PATCH 3/7] updated readme to include the bit about search using loss. --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index b78b0c2..6459762 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -# DEvol - Deep Neural Network Evolution +f# DEvol - Deep Neural Network Evolution DEvol (DeepEvolution) utilizes genetic programming to automatically architect a deep neural network with optimal hyperparameters for a given dataset using the Keras library. This approach should design an equal or superior model to what a human could design when working under the same constraints as are imposed upon the genetic program (e.g., maximum number of layers, maximum number of convolutional filters per layer, etc.). The current setup is designed for classification problems, though this could be extended to include any other output type as well. @@ -43,6 +43,6 @@ DEvol is pretty straightforward to use for basic classification problems. See `d 1. **Prep your dataset.** DEvol expects a classification problem with labels that are one-hot encoded as it uses `categorical_crossentropy` for its loss function. Otherwise, you can prep your data however you'd like. Just pass your input shape into `GenomeHandler`. 2. **Create a `GenomeHandler`.** The `GenomeHandler` defines the constraints that you apply to your models. Specify the maximum number of convolutional and dense layers, the max dense nodes and feature maps, and the input shape. You can also specify whether you'd like to allow batch_normalization, dropout, and max_pooling, which are included by default. You can also pass in a list of optimizers and activation functions you'd like to allow. -3. **Create and run the DEvol.** Pass your `GenomeHandler` to the `DEvol` constructor, and run it. Here you have a few more options such as the number of generations, the population size, epochs used for fitness evaluation, and an (optional) fitness function which converts a model's accuracy into a fitness score. +3. **Create and run the DEvol.** Pass your `GenomeHandler` to the `DEvol` constructor, and run it. Here you have a few more options such as the number of generations, the population size, epochs used for fitness evaluation, the evaluation metric to optimize (accuracy or loss) and an (optional) fitness function which converts a model's accuracy or loss into a fitness score. See `demo.ipynb` for a basic example. From 1d85344790d9dff609921806d77d7a158d7533e3 Mon Sep 17 00:00:00 2001 From: leaprovenzano Date: Wed, 7 Jun 2017 09:41:40 +0100 Subject: [PATCH 4/7] fixed typo in readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 6459762..aa421a5 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -f# DEvol - Deep Neural Network Evolution +# DEvol - Deep Neural Network Evolution DEvol (DeepEvolution) utilizes genetic programming to automatically architect a deep neural network with optimal hyperparameters for a given dataset using the Keras library. This approach should design an equal or superior model to what a human could design when working under the same constraints as are imposed upon the genetic program (e.g., maximum number of layers, maximum number of convolutional filters per layer, etc.). The current setup is designed for classification problems, though this could be extended to include any other output type as well. From 3624e38a9b6c81e7ad1710a8f6001f71b156e7f5 Mon Sep 17 00:00:00 2001 From: leaprovenzano Date: Wed, 7 Jun 2017 09:44:03 +0100 Subject: [PATCH 5/7] removed old inline comments above run method in devol.py --- devol.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/devol.py b/devol.py index 99bb99a..e8e12f4 100644 --- a/devol.py +++ b/devol.py @@ -40,8 +40,7 @@ def set_objective(self, metric): self.metric_op = METRIC_OPS[self.objective is 'max'] self.metric_objective = METRIC_OBJECTIVES[self.objective is 'max'] - # Create a population and evolve - # Returns best model found in the form of (model, loss, accuracy) + def run(self, dataset, num_generations, pop_size, epochs, fitness=None, metric='accuracy'): """run genetic search on dataset given number of generations and population size From 675cbba054a2e3bb3012ee70def11a32c8956145 Mon Sep 17 00:00:00 2001 From: leaprovenzano Date: Wed, 7 Jun 2017 10:08:51 +0100 Subject: [PATCH 6/7] added changes to fit return to include loss, also changed print function for python3 compatability --- demo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/demo.py b/demo.py index f739fb6..16569d3 100644 --- a/demo.py +++ b/demo.py @@ -1,3 +1,5 @@ +from __future__ import print_function + from keras.datasets import mnist from keras.utils.np_utils import to_categorical from devol import DEvol, GenomeHandler @@ -41,5 +43,5 @@ num_epochs = 1 devol = DEvol(genome_handler) -model, accuracy = devol.run(dataset, num_generations, population_size, num_epochs) -print model.summary() +model, loss, accuracy = devol.run(dataset, num_generations, population_size, num_epochs) +print(model.summary()) From 29328e6e595f8ae882419afd77e1c6a989a2dccc Mon Sep 17 00:00:00 2001 From: leaprovenzano Date: Fri, 9 Jun 2017 00:31:24 +0100 Subject: [PATCH 7/7] set image dim for backend to channels_last (tf style) for demos --- demo.ipynb | 13 ++++++++----- demo.py | 5 +++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/demo.ipynb b/demo.ipynb index 43811d8..28d7cfc 100644 --- a/demo.ipynb +++ b/demo.ipynb @@ -53,14 +53,16 @@ { "cell_type": "code", "execution_count": 16, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ - "# set image format so that it's compatable with backend\n", - "image_format = (28, 28, 1) if K.image_data_format() is 'channels_last' else (1, 28, 28 )\n", + "K.set_image_data_format(\"channels_last\")\n", "\n", - "x_train = x_train.reshape((x_train.shape[0],) + image_format).astype('float32') / 255\n", - "x_test = x_test.reshape((x_test.shape[0],) + image_format).astype('float32') / 255\n", + "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", + "x_train = x_train.reshape(x_train.shape[0], 28, 28, 1).astype('float32') / 255\n", + "x_test = x_test.reshape(x_test.shape[0], 28, 28, 1).astype('float32') / 255\n", "y_train = to_categorical(y_train)\n", "y_test = to_categorical(y_test)\n", "dataset = ((x_train, y_train), (x_test, y_test))" @@ -113,6 +115,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "collapsed": true, "scrolled": false }, "outputs": [], diff --git a/demo.py b/demo.py index 16569d3..e796197 100644 --- a/demo.py +++ b/demo.py @@ -10,9 +10,10 @@ # for many introductory deep learning examples. Here, we load the data and # prepare it for use by the GPU. We also do a one-hot encoding of the labels. +K.set_image_data_format("channels_last") (x_train, y_train), (x_test, y_test) = mnist.load_data() -x_train = x_train.reshape(x_train.shape[0], 1, 28, 28).astype('float32') / 255 -x_test = x_test.reshape(x_test.shape[0], 1, 28, 28).astype('float32') / 255 +x_train = x_train.reshape(x_train.shape[0], 28, 28, 1).astype('float32') / 255 +x_test = x_test.reshape(x_test.shape[0], 28, 28, 1).astype('float32') / 255 y_train = to_categorical(y_train) y_test = to_categorical(y_test) dataset = ((x_train, y_train), (x_test, y_test))