Problem with KerasClassifier

Hi @adriangb, congratulations for the work.

I am having a problem with a dataset that I am using in my work. KerasRegressor is working perfectly with me, but KerasClassifier presented a problem that did not happen with keras.wrappers.scikit_learn.

I took a simple example with the mnist dataset and the problem persisted.

If I change the import scikeras to keras.wrappers.scikit_learn the code works perfectly.

import numpy as np
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Flatten, Dropout

#from keras.wrappers.scikit_learn import KerasClassifier
from scikeras.wrappers import KerasClassifier

from sklearn.model_selection import GridSearchCV

#load dataset
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0

X = np.vstack((x_train, x_test)) 
y = np.hstack((y_train, y_test))

def build_model(optimizer='adam'):
    model = Sequential()
    model.add(Flatten(input_shape=(28, 28)))
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Dense(10, activation='softmax'))

    model.compile(optimizer=optimizer,
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

    return model
    
batch_size = [32, 256]
optimizer = ['adam', 'rmsprop']

param_grids = dict(batch_size = batch_size, optimizer = optimizer)
model = KerasClassifier(build_fn=build_model, verbose=1, batch_size=None, optimizer=None)

grid = GridSearchCV(estimator=model, param_grid=param_grids, n_jobs = 10)
result = grid.fit(X, y)

print("Best: {} using {}".format(result.best_score_, result.best_params_))

1750/1750 [==============================] - 3s 2ms/step - loss: 0.2233 - accuracy: 0.9342
438/438 [==============================] - 0s 562us/step
Traceback (most recent call last):
  File "mnist.py", line 39, in <module>
    result = grid.fit(X, y)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/utils/validation.py", line72, in inner_f
    return f(**kwargs)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/model_selection/_search.py", line 736, in fit
    self._run_search(evaluate_candidates)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/model_selection/_search.py", line 1188, in _run_search
    evaluate_candidates(ParameterGrid(self.param_grid))
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/model_selection/_search.py", line 708, in evaluate_candidates
    out = parallel(delayed(_fit_and_score)(clone(base_estimator),
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/joblib/parallel.py", line 1048, in__call__
    if self.dispatch_one_batch(iterator):
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/joblib/parallel.py", line 866, in dispatch_one_batch
    self._dispatch(tasks)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/joblib/parallel.py", line 784, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/joblib/_parallel_backends.py", line 208, in apply_async
    result = ImmediateResult(func)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/joblib/_parallel_backends.py", line 572, in __init__
    self.results = batch()
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/joblib/parallel.py", line 262, in __call__
    return [func(*args, **kwargs)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/joblib/parallel.py", line 262, in <listcomp>
    return [func(*args, **kwargs)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/model_selection/_validation.py", line 560, in _fit_and_score
    test_scores = _score(estimator, X_test, y_test, scorer)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/model_selection/_validation.py", line 607, in _score
    scores = scorer(estimator, X_test, y_test)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/metrics/_scorer.py", line 90, in __call__
    score = scorer(estimator, *args, **kwargs)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/metrics/_scorer.py", line 372, in _passthrough_scorer
    return estimator.score(*args, **kwargs)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/scikeras/wrappers.py", line 653, in score
    y_pred = self.predict(X, **kwargs)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/scikeras/wrappers.py", line 617, in predict
    y, _ = self._post_process_y(y_pred)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/scikeras/wrappers.py", line 887, in _post_process_y
    self.encoders_[i].inverse_transform(y_)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/preprocessing/_label.py", line 293, in inverse_transform
    y = column_or_1d(y, warn=True)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/utils/validation.py", line72, in inner_f
    return f(**kwargs)
  File "/home/thiago.cavalcante/anaconda3/envs/my_env_conda/lib/python3.8/site-packages/sklearn/utils/validation.py", line845, in column_or_1d
    raise ValueError(
ValueError: y should be a 1d array, got an array of shape (14000, 10) instead.

Shapes of X and y: X (70000, 28, 28) y (70000,)

Thanks a lot for attention.

[adriangb]

I’ll close this issue since it seems like it’s resolved. Let me know if there is any followup. Thanks for the bug report!

[adriangb]

Yes, that is because SciKeras no longer introspects your model’s loss function. To make categorical_crossentropy work, you would have to either:

1. One hot encode the target yourself, before passing it to SciKeras. This would be the same as if you were using plain Keras.
2. Pass the loss function to the constructor directly. Example below

def build_model():
    model = Sequential()
    model.add(Flatten(input_shape=(28, 28)))
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Dense(10, activation='softmax'))
    return model

model = KerasClassifier(
    model=build_model,
    loss="categorical_crossentropy",   # pass categorical_crossentropy
)

This applies even if you are compiling your own model:

def build_model():
    model = Sequential()
    model.add(Flatten(input_shape=(28, 28)))
    model.add(Dense(512, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Dense(10, activation='softmax'))
    model.compile(
        loss="categorical_crossentropy",  # loss matches constructor loss
        optimizer="sgd",  # override default optimizer
        metrics=["accuracy"]  # add a metric
    )
    return model

model = KerasClassifier(
    model=build_model,
    loss="categorical_crossentropy",   # pass categorical_crossentropy
)