diff --git a/.idea/dictionaries/maxim.xml b/.idea/dictionaries/maxim.xml index 7e1dce0..dfeab49 100644 --- a/.idea/dictionaries/maxim.xml +++ b/.idea/dictionaries/maxim.xml @@ -2,6 +2,7 @@ hyperengine + keras theano diff --git a/.travis.yml b/.travis.yml index 7fababe..eb82dae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: - "3.6" install: - - pip install six==1.11.0 numpy==1.14.0 scipy==1.0.0 tensorflow==1.5.0 + - pip install six==1.11.0 numpy==1.14.0 scipy==1.0.0 tensorflow==1.5.0 keras==2.1.2 script: - cd hyperengine/tests diff --git a/hyperengine/__init__.py b/hyperengine/__init__.py index 75cbf43..617644d 100644 --- a/hyperengine/__init__.py +++ b/hyperengine/__init__.py @@ -5,6 +5,7 @@ from .bayesian import * from .model import * from .impl.tensorflow import * +from .impl.keras import * from . import base as util from . import spec diff --git a/hyperengine/examples/5_1_keras_mnist.py b/hyperengine/examples/5_1_keras_mnist.py new file mode 100644 index 0000000..47ed931 --- /dev/null +++ b/hyperengine/examples/5_1_keras_mnist.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +__author__ = 'maxim' + +import keras +from keras.datasets import mnist +from keras.models import Sequential +from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D + +import hyperengine as hype + + +num_classes = 10 +img_rows, img_cols = 28, 28 + +def prepare_data(): + (x_train, y_train), (x_test, y_test) = mnist.load_data() + + x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1) + x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1) + + x_train = x_train.astype('float32') / 255 + x_test = x_test.astype('float32') / 255 + + y_train = keras.utils.to_categorical(y_train, num_classes) + y_test = keras.utils.to_categorical(y_test, num_classes) + + return hype.Data(train=hype.DataSet(x_train, y_train), + validation=hype.DataSet(x_test, y_test), + test=hype.DataSet(x_test, y_test)) + +def build_model(params): + model = Sequential() + model.add(Conv2D(params.conv[0].size, + kernel_size=params.conv[0].kernel, + activation=params.conv[0].activation, + input_shape=(img_rows, img_cols, 1))) + model.add(Conv2D(params.conv[1].size, + kernel_size=params.conv[1].kernel, + activation=params.conv[1].activation)) + model.add(MaxPooling2D(pool_size=params.pooling.size)) + model.add(Dropout(params.pooling.dropout)) + model.add(Flatten()) + model.add(Dense(params.dense.size, activation=params.dense.activation)) + model.add(Dropout(params.dense.dropout)) + model.add(Dense(num_classes, activation='softmax')) + + model.compile(loss=keras.losses.categorical_crossentropy, + optimizer=keras.optimizers.Adadelta(lr=params.learning_rate), + metrics=['accuracy']) + return model + + +data = prepare_data() + +def solver_generator(params): + solver_params = { + 'batch_size': 2000, + 'epochs': 5, + } + model = build_model(params) + solver = hype.KerasSolver(model, data=data, hyper_params=params, **solver_params) + return solver + + +hyper_params_spec = hype.spec.new( + learning_rate = 10**hype.spec.uniform(-1, -3), + conv = [ + dict( + size = 32, + kernel = (3, 3), + activation = 'relu', + ), + dict( + size = 64, + kernel = (3, 3), + activation = 'relu', + ), + ], + pooling = dict( + size = (2, 2), + dropout = 0.25, + ), + dense = dict( + size = 128, + activation = 'relu', + dropout = 0.5, + ) +) +strategy_params = { + 'io_load_dir': 'temp-mnist/example-5-1', + 'io_save_dir': 'temp-mnist/example-5-1', +} + +tuner = hype.HyperTuner(hyper_params_spec, solver_generator, **strategy_params) +tuner.tune() diff --git a/hyperengine/impl/keras/__init__.py b/hyperengine/impl/keras/__init__.py new file mode 100644 index 0000000..331d3c5 --- /dev/null +++ b/hyperengine/impl/keras/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +__author__ = 'maxim' + +from .keras_solver import KerasSolver diff --git a/hyperengine/impl/keras/keras_solver.py b/hyperengine/impl/keras/keras_solver.py new file mode 100644 index 0000000..f2a18d4 --- /dev/null +++ b/hyperengine/impl/keras/keras_solver.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +__author__ = 'maxim' + + +import keras.backend as K +from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint +from keras.utils.layer_utils import count_params + +from ...base import * +from ...model.base_solver import reducers + + +class MyCallback(Callback): + def __init__(self): + super(MyCallback, self).__init__() + self._history = {} + + def history(self): + return self._history + + def val_accuracy_history(self): + return self._history['val_acc'] + + def on_epoch_begin(self, epoch, logs=None): + pass + + def on_epoch_end(self, epoch, logs=None): + logs = logs or {} + for k, v in logs.items(): + self._history.setdefault(k, []).append(v) + + def on_batch_begin(self, batch, logs=None): + pass + + def on_batch_end(self, batch, logs=None): + pass + + def on_train_begin(self, logs=None): + pass + + def on_train_end(self, logs=None): + pass + + +class KerasSolver(object): + def __init__(self, model, data, hyper_params=None, model_io=None, reducer='max', **params): + self._model = model + + self._train_set = data.train + self._val_set = data.validation + self._test_set = data.test + self._hyper_params = hyper_params + self._reducer = as_numeric_function(reducer, presets=reducers) + + self._model_io = model_io if model_io is not None else None + + self._epochs = params.get('epochs', 1) + self._batch_size = params.get('batch_size', 16) + self._eval_test = params.get('evaluate_test', False) + + + def train(self): + info('Start training. Model size: %dk' % (self._model_size() / 1000)) + info('Hyper params: %s' % smart_str(self._hyper_params)) + + early_stopping = EarlyStopping() + callback = MyCallback() + self._model.fit(x=self._train_set.x, + y=self._train_set.y, + validation_data=(self._val_set.x, self._val_set.y), + batch_size=self._batch_size, + epochs=self._epochs, + verbose=1, + callbacks=[callback, early_stopping]) + + return self._reducer(callback.val_accuracy_history()) + + + def terminate(self): + K.clear_session() + + + def _model_size(self): + return count_params(self._model.trainable_weights) diff --git a/hyperengine/impl/tensorflow/tensorflow_runner.py b/hyperengine/impl/tensorflow/tensorflow_runner.py index 19466d1..7859791 100644 --- a/hyperengine/impl/tensorflow/tensorflow_runner.py +++ b/hyperengine/impl/tensorflow/tensorflow_runner.py @@ -7,7 +7,7 @@ import tensorflow as tf from ...base import * -from ...model.base_runner import BaseRunner +from ...model import BaseRunner from .tf_util import graph_vars, get_total_dim diff --git a/setup.py b/setup.py index 8d03c83..731b68c 100644 --- a/setup.py +++ b/setup.py @@ -22,5 +22,6 @@ def read(file_): install_requires = ['numpy', 'scipy', 'six'], extras_require = { 'tf': ['tensorflow'], + 'keras': ['keras'], }, )