When will keras support float16 for BatchNorm layer?

I’ve encountered this issue https://github.com/keras-team/keras/issues/9582(url) and it still hasn’t solved the problem… I’m trying to utilize the GPU with floatx=float16 but fail to use batch normalization layer.

from keras.models import Sequential
from keras.layers import Conv2D, BatchNormalization
model = Sequential()
model.add(Conv2D(32,3, input_shape = (224,224,3)))
model.add(BatchNormalization())

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-2-f051fa3daa13> in <module>
      3 model = Sequential()
      4 model.add(Conv2D(32,3, input_shape = (224,224,3)))
----> 5 model.add(BatchNormalization())

/usr/local/lib/python3.5/dist-packages/keras/engine/sequential.py in add(self, layer)
    179                 self.inputs = network.get_source_inputs(self.outputs[0])
    180         elif self.outputs:
--> 181             output_tensor = layer(self.outputs[0])
    182             if isinstance(output_tensor, list):
    183                 raise TypeError('All layers in a Sequential model '

/usr/local/lib/python3.5/dist-packages/keras/engine/base_layer.py in __call__(self, inputs, **kwargs)
    455             # Actually call the layer,
    456             # collecting output(s), mask(s), and shape(s).
--> 457             output = self.call(inputs, **kwargs)
    458             output_mask = self.compute_mask(inputs, previous_mask)
    459 

/usr/local/lib/python3.5/dist-packages/keras/layers/normalization.py in call(self, inputs, training)
    183         normed_training, mean, variance = K.normalize_batch_in_training(
    184             inputs, self.gamma, self.beta, reduction_axes,
--> 185             epsilon=self.epsilon)
    186 
    187         if K.backend() != 'cntk':

/usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py in normalize_batch_in_training(x, gamma, beta, reduction_axes, epsilon)
   1862         return _fused_normalize_batch_in_training(
   1863             x, gamma, beta, reduction_axes,
-> 1864             epsilon=epsilon)
   1865     else:
   1866         if sorted(reduction_axes) == list(range(ndim(x)))[:-1]:

/usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py in _fused_normalize_batch_in_training(x, gamma, beta, reduction_axes, epsilon)
   1837         beta,
   1838         epsilon=epsilon,
-> 1839         data_format=tf_data_format)
   1840 
   1841 

/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_impl.py in fused_batch_norm(x, scale, offset, mean, variance, epsilon, data_format, is_training, name)
    907       data_format=data_format,
    908       is_training=is_training,
--> 909       name=name)
    910   return y, batch_mean, batch_var
    911 

/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gen_nn_ops.py in fused_batch_norm_v2(x, scale, offset, mean, variance, epsilon, data_format, is_training, name)
   3835         "FusedBatchNormV2", x=x, scale=scale, offset=offset, mean=mean,
   3836         variance=variance, epsilon=epsilon, data_format=data_format,
-> 3837         is_training=is_training, name=name)
   3838     _result = _op.outputs[:]
   3839     _inputs_flat = _op.inputs

/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
    607               _SatisfiesTypeConstraint(base_type,
    608                                        _Attr(op_def, input_arg.type_attr),
--> 609                                        param_name=input_name)
    610             attrs[input_arg.type_attr] = attr_value
    611             inferred_from[input_arg.type_attr] = input_name

/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py in _SatisfiesTypeConstraint(dtype, attr_def, param_name)
     58           "allowed values: %s" %
     59           (param_name, dtypes.as_dtype(dtype).name,
---> 60            ", ".join(dtypes.as_dtype(x).name for x in allowed_list)))
     61 
     62 

TypeError: Value passed to parameter 'scale' has DataType float16 not in list of allowed values: float32

[Golbstein]

I’ve created this layer: https://github.com/Golbstein/KerasExtras/blob/master/BN16.py

[omalleyt12]

@janaal1 Thanks for the issue! Can you please open this on the main tensorflow github?