Resumption fails with custom objective

Bug Description

I have tried to use customized metrics (e.g., using f1-score) to select the best model, but I found I cannot resume a previously finished (or killed) run by running the same code again (as described in https://autokeras.com/tutorial/faq/, as long as the “overwrite” argument of AutoModel is set to False (which is by default), running the same code again will continue the previous fit). To reproduce the bug, we can use the code snippet in https://github.com/keras-team/autokeras/issues/867 (but please note that here I let the “direction” be ‘max’, which is the correct objective direction for the f1-score) with the following steps:

Bug Reproduction

After finishing running the following code to use f1_score as customized metric to select the best model for 3 trails, if I would like to run for two more trails, I simply let “max_trials=5” in ImageClassifier and run the same snippet again, but there will be an error “ValueError: Unknown metric function: f1_score”.

Code for reproducing the bug:

from tensorflow.keras.datasets import mnist
import kerastuner

import autokeras as ak

from tensorflow.keras import backend as K


def recall_m(y_true, y_pred):
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    recall = true_positives / (possible_positives + K.epsilon())
    return recall

def precision_m(y_true, y_pred):
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives + K.epsilon())
    return precision

def f1_score(y_true, y_pred):
    precision = precision_m(y_true, y_pred)
    recall = recall_m(y_true, y_pred)
    return 2*((precision*recall)/(precision+recall+K.epsilon()))

# Prepare the dataset.
(x_train, y_train), (x_test, y_test) = mnist.load_data()
print(x_train.shape)  # (60000, 28, 28)
print(y_train.shape)  # (60000,)
print(y_train[:3])  # array([7, 2, 1], dtype=uint8)

# Initialize the ImageClassifier.
clf = ak.ImageClassifier(
    max_trials=3,
    objective=kerastuner.Objective('val_f1_score', direction='max'),
    metrics=[f1_score],
)
# Search for the best model.
clf.fit(x_train, y_train, epochs=10)
# Evaluate on the testing data.
print('Accuracy: {accuracy}'.format(
    accuracy=clf.evaluate(x_test, y_test)))

Expected Behavior

After running the above code, letting “max_trials=5” in ImageClassifier and running the same snippet again, an error “ValueError: Unknown metric function: f1_score” will occur.

The following is the whole traceback of the error:

Traceback (most recent call last):
File "", line 1, in
File "/snap/pycharm-professional/211/plugins/python/helpers/pydev/_pydev_bundle/pydev_umd.py", line 197, in runfile
pydev_imports.execfile(filename, global_vars, local_vars) # execute the script
File "/snap/pycharm-professional/211/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/home/colotu01/csi4900tf2/20200728_try/try_to_continue_fitting.py", line 50, in
metrics=[f1_score],
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/tasks/image.py", line 85, in init
**kwargs)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/tasks/image.py", line 24, in init
**kwargs)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/auto_model.py", line 136, in init
**kwargs)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/tuners/task_specific.py", line 102, in init
**kwargs)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/tuners/greedy.py", line 237, in init
**kwargs)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/engine/tuner.py", line 40, in init
super().init(oracle, hypermodel, **kwargs)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/kerastuner/engine/tuner.py", line 104, in init
overwrite=overwrite)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/kerastuner/engine/base_tuner.py", line 71, in init
self.directory, self.project_name, overwrite=overwrite)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/kerastuner/engine/oracle.py", line 312, in _set_project_dir
self.reload()
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/kerastuner/engine/oracle.py", line 337, in reload
super(Oracle, self).reload(self._get_oracle_fname())
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/kerastuner/engine/stateful.py", line 64, in reload
self.set_state(state)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/tuners/greedy.py", line 84, in set_state
self.hypermodel = graph.Graph.from_config(state['hypermodel'])
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/graph.py", line 205, in from_config
blocks = [blocks_module.deserialize(block) for block in config['blocks']]
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/graph.py", line 205, in
blocks = [blocks_module.deserialize(block) for block in config['blocks']]
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/blocks/init.py", line 37, in deserialize
printable_module_name='hypermodels')
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 360, in deserialize_keras_object
return cls.from_config(cls_config)
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/engine/head.py", line 72, in from_config
config['metrics'] = deserialize_metrics(config['metrics'])
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/autokeras/engine/head.py", line 23, in deserialize_metrics
deserialized.append(tf.keras.metrics.deserialize(metric))
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/python/keras/metrics.py", line 3443, in deserialize
printable_module_name='metric function')
File "/home/colotu01/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/python/keras/utils/generic_utils.py", line 378, in deserialize_keras_object
'Unknown ' + printable_module_name + ': ' + object_name)
ValueError: Unknown metric function: f1_score

To say a little bit about my guess about the cause of the error, which might be helpful

I found there is another problem when using customized metrics (e.g. the way above we use f1_score) to select the best model:

After the fitting process using the above code and exporting the model by:

model = clf.export_model() model.save(“model_autokeras.h5”)

then there will be an error “ValueError: Unknown metric function: f1_score” if we would like to load the saved model by the following code directly (which is in https://autokeras.com/tutorial/export/):

from tensorflow.keras.models import load_model loaded_model = load_model(“model_autokeras”, custom_objects=ak.CUSTOM_OBJECTS)

I guess that is because the .h5 file contains our customized function, so currently I found the following way can be a solution and the loaded_model works well for me:

my_custom_objects={‘f1_score’: f1_score} my_custom_objects.update(ak.CUSTOM_OBJECTS) loaded_model = load_model(“model_autokeras”, custom_objects=my_custom_objects)

So, I guess the error is caused by the same problem as in the above loading process, i.e., in the process of loading model or checkpoint we should pass {‘f1_score’: f1_score} to the custom_objects.

Setup Details

Include the details about the versions of:

• OS type and version: Ubuntu 18.04
• Python: 3.7.7
• autokeras: 1.0.5
• keras-tuner: 1.0.2rc1
• numpy: 1.18.5
• tensorflow: 2.3.0

[ywtaccelerator]

Hi, I have come up with a simple workaround for this bug and it seems to work well for me these days:

Using custom object scope: Import from tensorflow.keras.utils import CustomObjectScope and add with CustomObjectScope({'f1_score': f1_score}): before initializing the classifier, so the whole above code snippet will be:

from tensorflow.keras.datasets import mnist
import kerastuner

import autokeras as ak

from tensorflow.keras import backend as K
from tensorflow.keras.utils import CustomObjectScope

def recall_m(y_true, y_pred):
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    recall = true_positives / (possible_positives + K.epsilon())
    return recall

def precision_m(y_true, y_pred):
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives + K.epsilon())
    return precision

def f1_score(y_true, y_pred):
    precision = precision_m(y_true, y_pred)
    recall = recall_m(y_true, y_pred)
    return 2*((precision*recall)/(precision+recall+K.epsilon()))

with CustomObjectScope({'f1_score': f1_score}):
    # Prepare the dataset.
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    print(x_train.shape)  # (60000, 28, 28)
    print(y_train.shape)  # (60000,)
    print(y_train[:3])  # array([7, 2, 1], dtype=uint8)

    # Initialize the ImageClassifier.
    clf = ak.ImageClassifier(
        max_trials=3,
        objective=kerastuner.Objective('val_f1_score', direction='max'),
        metrics=[f1_score],
    )
    # Search for the best model.
    clf.fit(x_train, y_train, epochs=10)
    # Evaluate on the testing data.
    print('Accuracy: {accuracy}'.format(
        accuracy=clf.evaluate(x_test, y_test)))

So after running the above code, letting “max_trials=5” in ImageClassifier and running the same snippet again, the NAS process will resume and the code will run for two more trials.

Hope this could help you formally solve the bug, by the way, I am more interested in how to make use of multiple GPUs, because the NAS process is too time-consuming… I have tried some “tricks” but none of them works 😦

[haifeng-jin]

Yes, AutoKeras uses Keras under the hood and did not do much for metrics.