ValueError while implementing sample code

Model:

Sample Code:

from smooth_tiled_predictions import predict_img_with_smooth_windowing
# CNN's receptive field's border size: size of patches
window_size = 32

# Amount of categories predicted per pixels.
nb_classes = 16

# Load an image. Convention is channel_last, such as having an input_img.shape of: (x, y, nb_channels), where nb_channels is of 3 for regular RGB images.
input_img = test

# Use the algorithm. The `pred_func` is passed and will process all the image 8-fold by tiling small patches with overlap, called once with all those image as a batch outer dimension.
# Note that model.predict(...) accepts a 4D tensor of shape (batch, x, y, nb_channels), such as a Keras model.
predictions_smooth = predict_img_with_smooth_windowing(
    input_img,
    window_size=window_size,
    subdivisions=2,  # Minimal amount of overlap for windowing. Must be an even number.
    nb_classes=nb_classes,
    pred_func=(
        lambda img_batch_subdiv: encoder.predict(image_to_neural_input(img_batch_subdiv))
    )
)

Error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
G:\Programs\anaconda3\envs\py35\lib\site-packages\numpy\lib\arraypad.py in _normalize_shape(ndarray, shape, cast_to_int)
   1035     try:
-> 1036         shape_arr = np.broadcast_to(shape_arr, (ndims, 2))
   1037     except ValueError:

G:\Programs\anaconda3\envs\py35\lib\site-packages\numpy\lib\stride_tricks.py in broadcast_to(array, shape, subok)
    172     """
--> 173     return _broadcast_to(array, shape, subok=subok, readonly=True)
    174 

G:\Programs\anaconda3\envs\py35\lib\site-packages\numpy\lib\stride_tricks.py in _broadcast_to(array, shape, subok, readonly)
    127         (array,), flags=['multi_index', 'refs_ok', 'zerosize_ok'] + extras,
--> 128         op_flags=[op_flag], itershape=shape, order='C').itviews[0]
    129     result = _maybe_view_as_subclass(array, broadcast)

ValueError: operands could not be broadcast together with remapped shapes [original->remapped]: (3,2) and requested shape (4,2)

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-34-4fa28d1e9a96> in <module>()
     17     nb_classes=nb_classes,
     18     pred_func=(
---> 19         lambda img_batch_subdiv: encoder.predict(image_to_neural_input(img_batch_subdiv))
     20     )
     21 )

E:\GitHub\MTP\implementation\autoencoders\smooth_tiled_predictions.py in predict_img_with_smooth_windowing(input_img, window_size, subdivisions, nb_classes, pred_func)
    226     http://blog.kaggle.com/2017/05/09/dstl-satellite-imagery-competition-3rd-place-winners-interview-vladimir-sergey/
    227     """
--> 228     pad = _pad_img(input_img, window_size, subdivisions)
    229     pads = _rotate_mirror_do(pad)
    230 

E:\GitHub\MTP\implementation\autoencoders\smooth_tiled_predictions.py in _pad_img(img, window_size, subdivisions)
     77     aug = int(round(window_size * (1 - 1.0/subdivisions)))
     78     more_borders = ((aug, aug), (aug, aug), (0, 0))
---> 79     ret = np.pad(img, pad_width=more_borders, mode='reflect')
     80     # gc.collect()
     81 

G:\Programs\anaconda3\envs\py35\lib\site-packages\numpy\lib\arraypad.py in pad(array, pad_width, mode, **kwargs)
   1299 
   1300     narray = np.array(array)
-> 1301     pad_width = _validate_lengths(narray, pad_width)
   1302 
   1303     allowedkwargs = {

G:\Programs\anaconda3\envs\py35\lib\site-packages\numpy\lib\arraypad.py in _validate_lengths(narray, number_elements)
   1078 
   1079     """
-> 1080     normshp = _normalize_shape(narray, number_elements)
   1081     for i in normshp:
   1082         chk = [1 if x is None else x for x in i]

G:\Programs\anaconda3\envs\py35\lib\site-packages\numpy\lib\arraypad.py in _normalize_shape(ndarray, shape, cast_to_int)
   1037     except ValueError:
   1038         fmt = "Unable to create correctly shaped tuple from %s"
-> 1039         raise ValueError(fmt % (shape,))
   1040 
   1041     # Cast if necessary

ValueError: Unable to create correctly shaped tuple from ((16, 16), (16, 16), (0, 0))

Any help is appreciated. Apology if i am creating a trivial issue

[thisisashukla]

On reading the code i realized that the issue is not of any serious concern. I was passing already windowed image (image of x*y*channels converted into n*window_size*window_size*channels) to the code. Instead we need to pass the complete image array x*y*channels. Also for my case I am using an autoencoder. So I am not doing any classification. My model instead outputs n*x*y*channels2. So I need to give channels2 as number of classes in the input argument. This may be useful for anybody working with autoencoders.

Thanks for the awesome piece of code

[Mahi-Mai]

@git-ankur-shukla Thank you for responding!

My code…

from keras import models
import cv2
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

from smooth_tiled_predictions import predict_img_with_smooth_windowing

model = models.load_model(local_model_path)

# Load an image. Convention is channel_last, such as having an input_img.shape of: (x, y, nb_channels), where nb_channels is of 3 for regular RGB images.
im = cv2.imread(file_pth)
im = np.expand_dims(im, axis=0)

# Use the algorithm. The `pred_func` is passed and will process all the image 8-fold by tiling small patches with overlap, called once with all those image as a batch outer dimension.
# Note that model.predict(...) accepts a 4D tensor of shape (batch, x, y, nb_channels), such as a Keras model.
predictions_smooth = predict_img_with_smooth_windowing(im,
                                                       window_size=256,
                                                       subdivisions=8,  # Minimal amount of overlap for windowing. Must be an even number
                                                       nb_classes=2,
                                                       pred_func=(lambda img_batch_subdiv: model.predict(image_to_neural_input(img_batch_subdiv)))
                                                      )

print(type(predictions_smooth))
print(predictions_smooth.shape)

And I get this stack-trace:

ValueError                                Traceback (most recent call last)
<ipython-input-7-8ddaaab1e3bb> in <module>
      9                                                        subdivisions=8,  # Minimal amount of overlap for windowing. Must be an even number
     10                                                        nb_classes=2,
---> 11                                                        pred_func=(lambda img_batch_subdiv: model.predict(image_to_neural_input(img_batch_subdiv)))
     12                                                       )
     13 

/repos/Kernel-Counting/U-Net/Prediction/smooth_tiled_predictions.py in predict_img_with_smooth_windowing(input_img, window_size, subdivisions, nb_classes, pred_func)
    221     http://blog.kaggle.com/2017/05/09/dstl-satellite-imagery-competition-3rd-place-winners-interview-vladimir-sergey/
    222     """
--> 223     pad = _pad_img(input_img, window_size, subdivisions)
    224     pads = _rotate_mirror_do(pad)
    225 

/repos/Kernel-Counting/U-Net/Prediction/smooth_tiled_predictions.py in _pad_img(img, window_size, subdivisions)
     77     aug = int(round(window_size * (1 - 1.0/subdivisions)))
     78     more_borders = ((aug, aug), (aug, aug), (0, 0))
---> 79     ret = np.pad(img, pad_width=more_borders, mode='reflect')
     80     # gc.collect()
     81 

<__array_function__ internals> in pad(*args, **kwargs)

/usr/local/anaconda/lib/python3.6/site-packages/numpy/lib/arraypad.py in pad(array, pad_width, mode, **kwargs)
    739 
    740     # Broadcast to shape (array.ndim, 2)
--> 741     pad_width = _as_pairs(pad_width, array.ndim, as_index=True)
    742 
    743     if callable(mode):

/usr/local/anaconda/lib/python3.6/site-packages/numpy/lib/arraypad.py in _as_pairs(x, ndim, as_index)
    514     # Converting the array with `tolist` seems to improve performance
    515     # when iterating and indexing the result (see usage in `pad`)
--> 516     return np.broadcast_to(x, (ndim, 2)).tolist()
    517 
    518 

<__array_function__ internals> in broadcast_to(*args, **kwargs)

/usr/local/anaconda/lib/python3.6/site-packages/numpy/lib/stride_tricks.py in broadcast_to(array, shape, subok)
    180            [1, 2, 3]])
    181     """
--> 182     return _broadcast_to(array, shape, subok=subok, readonly=True)
    183 
    184 

/usr/local/anaconda/lib/python3.6/site-packages/numpy/lib/stride_tricks.py in _broadcast_to(array, shape, subok, readonly)
    125     it = np.nditer(
    126         (array,), flags=['multi_index', 'refs_ok', 'zerosize_ok'] + extras,
--> 127         op_flags=['readonly'], itershape=shape, order='C')
    128     with it:
    129         # never really has writebackifcopy semantics

ValueError: operands could not be broadcast together with remapped shapes [original->remapped]: (3,2) and requested shape (4,2)

Also note that I made the changes suggested in this thread: https://github.com/Vooban/Smoothly-Blend-Image-Patches/issues/1