Pruning + Distributed optimization

Conditions

• Optuna version: 0.17.1
• Python version: 3.7.3
• OS: Win 10
• Machine Learning library to be optimized: tfkeras

Code to reproduce

def main():

    study = optuna.create_study(study_name='keras',
                                storage='mssql+pyodbc://...', 
                                load_if_exists=True, direction='maximize', pruner=optuna.pruners.MedianPruner(n_startup_trials=2))

    study.optimize(objective, n_trials=10, timeout=600)

    show_result(study)

Error:

TrialState.FAIL because of the following error: StorageInternalError('An exception is raised during the commit. This typically happens due to invalid data in the commit, e.g. exceeding max length. (The actual exception is as follows: ProgrammingError("(pyodbc.ProgrammingError) (\'Invalid parameter type.  param-index=0 param-type=numpy.float32\', \'HY105\')"))')
Traceback (most recent call last):
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1244, in _execute_context
    cursor, statement, parameters, context
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\default.py", line 550, in do_execute
    cursor.execute(statement, parameters)
pyodbc.ProgrammingError: ('Invalid parameter type.  param-index=0 param-type=numpy.float32', 'HY105')

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\optuna\storages\rdb\storage.py", line 804, in _commit
    session.commit()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 1027, in commit
    self.transaction.commit()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 494, in commit
    self._prepare_impl()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 473, in _prepare_impl
    self.session.flush()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 2459, in flush
    self._flush(objects)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 2597, in _flush
    transaction.rollback(_capture_exception=True)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
    compat.reraise(exc_type, exc_value, exc_tb)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 154, in reraise
    raise value
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 2557, in _flush
    flush_context.execute()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\unitofwork.py", line 422, in execute
    rec.execute(self)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\unitofwork.py", line 589, in execute
    uow,
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\persistence.py", line 236, in save_obj
    update,
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\persistence.py", line 996, in _emit_update_statements
    statement, multiparams
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 988, in execute
    return meth(self, multiparams, params)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\sql\elements.py", line 287, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1107, in _execute_clauseelement
    distilled_params,
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1248, in _execute_context
    e, statement, parameters, cursor, context
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1466, in _handle_dbapi_exception
    util.raise_from_cause(sqlalchemy_exception, exc_info)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 399, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
    raise value.with_traceback(tb)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1244, in _execute_context
    cursor, statement, parameters, context
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\default.py", line 550, in do_execute
    cursor.execute(statement, parameters)
sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('Invalid parameter type.  param-index=0 param-type=numpy.float32', 'HY105')
[SQL: UPDATE trials SET value=? WHERE trials.trial_id = ?]
[parameters: (0.06432292, 3598)]
(Background on this error at: http://sqlalche.me/e/f405)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\optuna\study.py", line 505, in _run_trial
    result = func(trial)
  File "Z:/Alex/tfkeras_integartion.py", line 89, in objective
    callbacks=callbacks,
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\keras\engine\training.py", line 694, in fit
    initial_epoch=initial_epoch)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1433, in fit_generator
    steps_name='steps_per_epoch')
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\keras\engine\training_generator.py", line 331, in model_iteration
    callbacks.on_epoch_end(epoch, epoch_logs)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\keras\callbacks.py", line 311, in on_epoch_end
    callback.on_epoch_end(epoch, logs)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\optuna\integration\tfkeras.py", line 62, in on_epoch_end
    self.trial.report(current_score, step=epoch)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\optuna\trial.py", line 378, in report
    self.storage.set_trial_value(self._trial_id, value)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\optuna\storages\rdb\storage.py", line 508, in set_trial_value
    self._commit(session)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\optuna\storages\rdb\storage.py", line 813, in _commit
    sys.exc_info()[2])
  File "C:\Users\alex.gomes\AppData\Roaming\Python\Python37\site-packages\six.py", line 692, in reraise
    raise value.with_traceback(tb)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\optuna\storages\rdb\storage.py", line 804, in _commit
    session.commit()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 1027, in commit
    self.transaction.commit()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 494, in commit
    self._prepare_impl()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 473, in _prepare_impl
    self.session.flush()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 2459, in flush
    self._flush(objects)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 2597, in _flush
    transaction.rollback(_capture_exception=True)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\langhelpers.py", line 68, in __exit__
    compat.reraise(exc_type, exc_value, exc_tb)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 154, in reraise
    raise value
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\session.py", line 2557, in _flush
    flush_context.execute()
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\unitofwork.py", line 422, in execute
    rec.execute(self)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\unitofwork.py", line 589, in execute
    uow,
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\persistence.py", line 236, in save_obj
    update,
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\orm\persistence.py", line 996, in _emit_update_statements
    statement, multiparams
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 988, in execute
    return meth(self, multiparams, params)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\sql\elements.py", line 287, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1107, in _execute_clauseelement
    distilled_params,
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1248, in _execute_context
    e, statement, parameters, cursor, context
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1466, in _handle_dbapi_exception
    util.raise_from_cause(sqlalchemy_exception, exc_info)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 399, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 153, in reraise
    raise value.with_traceback(tb)
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1244, in _execute_context
    cursor, statement, parameters, context
  File "C:\Users\alex.gomes\AppData\Local\Continuum\anaconda3\lib\site-packages\sqlalchemy\engine\default.py", line 550, in do_execute
    cursor.execute(statement, parameters)
optuna.structs.StorageInternalError: An exception is raised during the commit. This typically happens due to invalid data in the commit, e.g. exceeding max length. (The actual exception is as follows: ProgrammingError("(pyodbc.ProgrammingError) ('Invalid parameter type.  param-index=0 param-type=numpy.float32', 'HY105')"))

Hi,

I am trying to execute the pruning example “tfkeras_integration.py” distributed, all the code is pretty much the same besides the create_study with the storage parameter. However it does not work, the error above shows up. The data base string (storage) is 100% correct since I am using it in an another code.

Could anyone help me with this problem?

Thanks.

[alexacgomes]

It worked! Thanks a lot @toshihikoyanase

[toshihikoyanase]

Thank you for sharing your notebook.

I found that the type of the return values of test function was numpy.ndarray. Trial.report expects float values, so please cast the values to float when you report them.

def objective(trial): 
    # ...
    for e in range(epoch):
        for data_train_batch, ret_train_batch in dataset_train:
            state = data_train_batch[:-1]
            model_out = tf.nn.softmax(model(state))
            # ...

        intermediate_value = test(model, data_val, ret_day_val)  # The type of `intermediate_value` is `numpy.ndarray`
        trial.report(float(intermediate_value), step=e)  # `trial.report` accepts only float values as the first argument.