Error with space_eval

Has anyone else run into this? My model runs no problem, then chokes at the end with space_eval:

Train on 5120 samples, validate on 1280 samples
Epoch 1/1
17s - loss: 2.0797 - acc: 0.1191 - val_loss: 2.0794 - val_acc: 0.1375
Final validation accuracy: 0.1375
Train on 5120 samples, validate on 1280 samples
Epoch 1/1
17s - loss: 2.0441 - acc: 0.1602 - val_loss: 1.8211 - val_acc: 0.2500
Final validation accuracy: 0.25
Evalutation of best performing model:
1280/1280 [==============================] - 2s     
[1.8220701128244401, 0.24921874999999999]
Best performing model chosen hyper-parameters:
{'LSTM': 50, 'batch_size': 64}
Trial 0 vals: {'LSTM': [1], 'batch_size': [0]}
Traceback (most recent call last):
  File "to_stack.py", line 87, in <module>
    print(space_eval(space, vals))
  File "/.../lib/python2.7/site-packages/hyperopt/fmin.py", line 342, in space_eval
    rval = pyll.rec_eval(space, memo=memo)
  File "/.../lib/python2.7/site-packages/hyperopt/pyll/base.py", line 870, in rec_eval
    raise TypeError('switch argument was', switch_i)
TypeError: ('switch argument was', [0])

For reproducibility, I am working on Ubuntu 14.04.4 LTS (GNU/Linux 3.13.0-111-generic x86_64) with Python 2.7. I have downloaded the latest versions of keras (2.0.5) and hyperas (0.3)

My full code is below (I may import some things I don’t need with this MWE). I can provide train_pl.csv if necessary, but for now suffice to say the final shapes of x_train and x_test returned in data are (5120, 153, 1) and (1280, 153, 1), respectively, and each row of train_pl.csv is integer-valued.

from __future__ import print_function
from hyperopt import Trials, STATUS_OK, tpe, space_eval
import keras.optimizers
from keras.layers.core import Dense, Dropout, Activation
from keras.layers import LSTM
from keras.models import Sequential
from keras.utils import np_utils
from sklearn.preprocessing import MinMaxScaler
from hyperas import optim
from hyperas.distributions import choice, uniform
import numpy

def data():
    train_file='train_pl.csv'
    trainset = numpy.loadtxt(train_file, delimiter=",")

    X = trainset[:, 0:(trainset.shape[1]-2)]
    Y = (trainset[:,trainset.shape[1]-1]).astype(int)

    scaler = MinMaxScaler(feature_range=(0, 1))
    X_scaled = scaler.fit_transform(X)

    y_binary = np_utils.to_categorical(Y)

    num_per_class = int(float(X.shape[0])/(float(y_binary.shape[1])))

    to_take = numpy.random.choice(num_per_class, int(num_per_class*0.2), replace=False)
    class_split = numpy.array_split(X_scaled, y_binary.shape[1])
    val_list = [x[to_take] for x in class_split]
    big_list = [item for sublist in val_list for item in sublist]
    val_X = numpy.asarray(big_list)
    label_set = numpy.arange(0, y_binary.shape[1])
    val_Y = numpy.repeat(label_set, int(num_per_class*0.2))
    val_Y = np_utils.to_categorical(val_Y)

    setdiffval = set(range(num_per_class)) - set(to_take)
    setdiffval = list(setdiffval)

    X_train_vals = [x[setdiffval] for x in class_split]
    X_train = [item for sublist in X_train_vals for item in sublist]
    X_train = numpy.asarray(X_train)
    Y_train = numpy.repeat(label_set, int(num_per_class*0.8))
    Y_train = np_utils.to_categorical(Y_train)

    x_train = X_train
    x_test = val_X
    y_train = Y_train
    y_test = val_Y
    x_train = numpy.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
    x_test = numpy.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))

    return (x_train, y_train, x_test, y_test)

def model(x_train, y_train, x_test, y_test):
    model = Sequential()
    model.add(LSTM({{choice([10, 20, 50, 100])}}, input_shape=(x_train.shape[1], x_train.shape[2]), return_sequences=False))
    model.add(Activation('relu'))
    model.add(Dense(8))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', metrics=['accuracy'],
                  optimizer='adam')

    model.fit(x_train, y_train,
              batch_size={{choice([64, 128])}}, epochs=1,verbose=2,validation_data=(x_test, y_test))
    score, acc = model.evaluate(x_test, y_test, verbose=0)
    print('Final validation accuracy:', acc)

    return {'loss': -acc, 'status': STATUS_OK, 'model': model}


if __name__ == '__main__':
    trials=Trials()
    best_run, best_model, space = optim.minimize(model=model, data=data,algo=tpe.suggest,max_evals=2,trials=trials, eval_space=True, return_space=True)
    X_train, Y_train, X_test, Y_test = data()
    print("Evalutation of best performing model:")
    print(best_model.evaluate(X_test, Y_test))
    print("Best performing model chosen hyper-parameters:")
    print(best_run)
    for t, trial in enumerate(trials):
        vals = trial.get('misc').get('vals')
        print("Trial %s vals: %s" % (t, vals))
        print(space_eval(space, vals))](url)

[pkainz]

I guess you have a copy/paste error in your last line, which should read like

print(space_eval(space, vals))

However, the issue arises since up to PR #91 vals already contained the unpacked indices due to the alteration of the trials object in the optim.minimize function. It was ‘unpacking’ all values from 1-element lists, which could then be resolved to the space. In PR #102 we ensured the consistency between the returned trials object from hyperopt and removed the modifying code.

Could you try the following (temporary) workaround:

for t, trial in enumerate(trials):
        vals = trial.get('misc').get('vals')
        print("Trial %s vals: %s" % (t, vals))
        tmp = {}
        for k,v in list(vals.items()):
            tmp[k] = v[0]
        print(space_eval(space, tmp))

and let us know, if this works?

I will then prepare a fix for this.

[maxpumperla]

@caugusta that’s great to hear, looks good. will close this ticket then.