UCF101 datasets with DataLoader returns error when stacking examples in mini-batches

🐛 Bug

UCF101 dataset returns a RunTimeError when combined with standard DataLoader class. It returns the error when trying to stack multiple tensors in batches.

To Reproduce

import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torch.utils.data import DataLoader

tfs = transforms.Compose([
            transforms.Lambda(lambda x: x / 255.), # scale in [0, 1]
            transforms.Lambda(lambda x: x.permute(0, 3, 1, 2) ) # reshape into (T, C, H, W)
    ])


# root, root_labels are the directories containing data and labels
d = datasets.UCF101(root, root_labels, frames_per_clip=25, step_between_clips=25, train=False, transform=tfs)
dataset = DataLoader(d, batch_size=7, shuffle=True, drop_last=True)

for i, (v, a, l) in enumerate(dataset):  # <- RunTimeError occurs here
    pass

RuntimeError                              Traceback (most recent call last)
      1 print(len(dataset))
----> 2 for i, (v, a, l) in enumerate(dataset):
      3     pass

~/miniconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in __next__(self)
    343 
    344     def __next__(self):
--> 345         data = self._next_data()
    346         self._num_yielded += 1
    347         if self._dataset_kind == _DatasetKind.Iterable and \

~/miniconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in _next_data(self)
    383     def _next_data(self):
    384         index = self._next_index()  # may raise StopIteration
--> 385         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    386         if self._pin_memory:
    387             data = _utils.pin_memory.pin_memory(data)

~/miniconda3/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py in fetch(self, possibly_batched_index)
     45         else:
     46             data = self.dataset[possibly_batched_index]
---> 47         return self.collate_fn(data)

~/miniconda3/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py in default_collate(batch)
     77     elif isinstance(elem, container_abcs.Sequence):
     78         transposed = zip(*batch)
---> 79         return [default_collate(samples) for samples in transposed]
     80 
     81     raise TypeError(default_collate_err_msg_format.format(elem_type))

~/miniconda3/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py in <listcomp>(.0)
     77     elif isinstance(elem, container_abcs.Sequence):
     78         transposed = zip(*batch)
---> 79         return [default_collate(samples) for samples in transposed]
     80 
     81     raise TypeError(default_collate_err_msg_format.format(elem_type))

~/miniconda3/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py in default_collate(batch)
     53             storage = elem.storage()._new_shared(numel)
     54             out = elem.new(storage)
---> 55         return torch.stack(batch, 0, out=out)
     56     elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' \
     57             and elem_type.__name__ != 'string_':

RuntimeError: invalid argument 0: Sizes of tensors must match except in dimension 0. Got 2 and 1 in dimension 1 at /opt/conda/conda-bld/pytorch_1579022060824/work/aten/src/TH/generic/THTensor.cpp:612

Expected behavior

The iteration over the dataloader should return a video tensor of size (B, T, C, H, W) where B is batch size, T is the number of frames, C are the image channels and H and W the image dimensions.

Environment

• PyTorch / torchvision Version: 1.4.0 / 0.5.0
• OS (e.g., Linux): CentOS Linux 7
• How you installed PyTorch / torchvision (conda, pip, source): conda
• Python version: 3.7.3

[AndreaCossu]

Perfect, that’s working! I just added return to your last line:

def custom_collate(batch):
    filtered_batch = []
    for video, _, label in batch:
        filtered_batch.append((video, label))
    return torch.utils.data.dataloader.default_collate(filtered_batch)

Final working code

import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torch.utils.data import DataLoader

tfs = transforms.Compose([
             # scale in [0, 1]
             transforms.Lambda(lambda x: x / 255.),
             # reshape into (T, C, H, W)
             transforms.Lambda(lambda x: x.permute(0, 3, 1, 2) ) 
    ])


# root, root_labels are the directories containing data and labels
d = datasets.UCF101(root, root_labels, frames_per_clip=25, 
        step_between_clips=25, train=False, transform=tfs)
dataset = DataLoader(d, batch_size=7, shuffle=True, 
        drop_last=True, collate_fn=custom_collate)

for i, (v, l) in enumerate(dataset): 
    print(v.size())
    print(l)
    break

Thank you very much!!

[fmassa]

Oh, I see, yes, this is an issue with the audio signal.

I would propose that you write a custom collate_fn that you pass to your DataLoader which will remove the audio file, maybe something like

def custom_collate(batch):
    filtered_batch = []
    for video, _, label in batch:
        filtered_batch.append((video, label))
    torch.utils.data.dataloader.default_collate(filtered_batch)