KeyError: metrics in talos.scan

talos.version = 1.0.0 tf.version = 2.2.0

The concise demo fails with 'KeyError: ‘metrics.’ The following code:

from keras.models import Sequential
from keras.layers import Dropout, Dense


import sys
import talos
x, y = talos.templates.datasets.breast_cancer()

x = talos.utils.rescale_meanzero(x)
def breast_cancer_model(x_train, y_train, x_val, y_val, params):

    model = Sequential()
    model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1],
                    activation=params['activation'],
                    kernel_initializer=params['kernel_initializer']))

    model.add(Dropout(params['dropout']))

    model.add(Dense(1, activation=params['last_activation'],
                    kernel_initializer=params['kernel_initializer']))

    model.compile(loss=params['losses'],
                  optimizer=params['optimizer'],
                  metrics=['acc', talos.utils.metrics.f1score])

    history = model.fit(x_train, y_train, 
                        validation_data=[x_val, y_val],
                        batch_size=params['batch_size'],
                        callbacks=[talos.utils.live()],
                        epochs=params['epochs'],
                        verbose=0)

    return history, model

p = {'first_neuron':[9,10,11],
     'hidden_layers':[0, 1, 2],
     'batch_size': [30],
     'epochs': [100],
     'dropout': [0],
     'kernel_initializer': ['uniform','normal'],
     'optimizer': ['Nadam', 'Adam'],
     'losses': ['binary_crossentropy'],
     'activation':['relu', 'elu'],
     'last_activation': ['sigmoid']}
t = talos.Scan(x=x,
               y=y,
               model=breast_cancer_model,
               params=p,
               experiment_name='breast_cancer',
               round_limit=10)

…results in the following:

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-1-7a8ddf94f1cb> in <module>
     47      'last_activation': ['sigmoid']}
     48 # and run the experiment
---> 49 t = talos.Scan(x=x,
     50                y=y,
     51                model=breast_cancer_model,

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/scan/Scan.py in __init__(self, x, y, params, model, experiment_name, x_val, y_val, val_split, random_method, seed, performance_target, fraction_limit, round_limit, time_limit, boolean_limit, reduction_method, reduction_interval, reduction_window, reduction_threshold, reduction_metric, minimize_loss, disable_progress_bar, print_params, clear_session, save_weights)
    194         # start runtime
    195         from .scan_run import scan_run
--> 196         scan_run(self)

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/scan/scan_run.py in scan_run(self)
     24         # otherwise proceed with next permutation
     25         from .scan_round import scan_round
---> 26         self = scan_round(self)
     27         self.pbar.update(1)
     28 

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/scan/scan_round.py in scan_round(self)
     17     # fit the model
     18     from ..model.ingest_model import ingest_model
---> 19     self.model_history, self.round_model = ingest_model(self)
     20     self.round_history.append(self.model_history.history)
     21 

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/talos/model/ingest_model.py in ingest_model(self)
      4     through Scan() model paramater.'''
      5 
----> 6     return self.model(self.x_train,
      7                       self.y_train,
      8                       self.x_val,

<ipython-input-1-7a8ddf94f1cb> in breast_cancer_model(x_train, y_train, x_val, y_val, params)
     27                   metrics=['acc', talos.utils.metrics.f1score])
     28 
---> 29     history = model.fit(x_train, y_train, 
     30                         validation_data=[x_val, y_val],
     31                         batch_size=params['batch_size'],

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
     64   def _method_wrapper(self, *args, **kwargs):
     65     if not self._in_multi_worker_mode():  # pylint: disable=protected-access
---> 66       return method(self, *args, **kwargs)
     67 
     68     # Running inside `run_distribute_coordinator` already.

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
    828       self.stop_training = False
    829       train_function = self.make_train_function()
--> 830       callbacks.on_train_begin()
    831       # Handle fault-tolerance for multi-worker.
    832       # TODO(omalleyt): Fix the ordering issues that mean this has to

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/tensorflow/python/keras/callbacks.py in on_train_begin(self, logs)
    445     logs = self._process_logs(logs)
    446     for callback in self.callbacks:
--> 447       callback.on_train_begin(logs)
    448 
    449   def on_train_end(self, logs=None):

/home/anaconda/anaconda3/envs/tf-gpu2/lib/python3.8/site-packages/kerasplotlib/traininglog.py in on_train_begin(self, logs)
     31 
     32     def on_train_begin(self, logs={}):
---> 33         self.base_metrics = [metric for metric in self.params['metrics'] if not metric.startswith('val_')]
     34         if self.figsize is None:
     35             self.figsize = (

KeyError: 'metrics'

[mikkokotila]

Use this instead:

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dropout, Dense

%matplotlib inline

import sys
sys.path.insert(0, '/Users/mikko/Documents/GitHub/talos')
import talos

# then we load the dataset
x, y = talos.templates.datasets.breast_cancer()

# and normalize every feature to mean 0, std 1
x = talos.utils.rescale_meanzero(x)


# first we have to make sure to input data and params into the function
def breast_cancer_model(x_train, y_train, x_val, y_val, params):

    model = Sequential()
    model.add(Dense(params['first_neuron'], input_dim=x_train.shape[1],
                    activation=params['activation'],
                    kernel_initializer=params['kernel_initializer']))
    
    model.add(Dropout(params['dropout']))

    model.add(Dense(1, activation=params['last_activation'],
                    kernel_initializer=params['kernel_initializer']))
    
    model.compile(loss=params['losses'],
                  optimizer=params['optimizer'],
                  metrics=['acc', talos.utils.metrics.f1score])
    
    history = model.fit(x_train, y_train, 
                        validation_data=(x_val, y_val),
                        batch_size=params['batch_size'],
                        epochs=params['epochs'],
                        verbose=0)

    return history, model

  
  # then we can go ahead and set the parameter space
p = {'first_neuron':[9,10,11],
     'hidden_layers':[0, 1, 2],
     'batch_size': [30],
     'epochs': [100],
     'dropout': [0],
     'kernel_initializer': ['uniform','normal'],
     'optimizer': ['Nadam', 'Adam'],
     'losses': ['binary_crossentropy'],
     'activation':['relu', 'elu'],
     'last_activation': ['sigmoid']}

# and run the experiment
t = talos.Scan(x=x,
               y=y,
               model=breast_cancer_model,
               params=p,
               experiment_name='breast_cancer',
               round_limit=10)

Closing here. Feel free to open new issue if anything.

[rpa1tera]

I’m having this same issue with KeyError: metrics. Does anyone have solved this problem yet?

did you solve it?