search

fastai / fastai2

Temporary home for fastai v2 while it's being developed
https://dev.fast.ai
Apache License 2.0
645 stars 235 forks source link

ValueError: only one element tensors can be converted to Python scalars when loading into DataBunch #55

Closed jakubLangr closed 4 years ago

jakubLangr commented 4 years ago

Hello,

I have a pytorch data dataset:

x_t = torch.Tensor(np.ndarray([16,128,128,3]))
my_dataset = torch.utils.data.TensorDataset(x_t)
my_dloader = torch.utils.data.DataLoader(my_dataset)
img_dblock = DataBlock(blocks=(ImageBlock))
fa_dl = DataLoader(my_dloader)
img_dblock.databunch(fa_dl)

But the last line gives the following error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-133-994da2df3f53> in <module>
----> 1 img_dblock.databunch(fa_dl)

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/block.py in databunch(self, source, path, type_tfms, item_tfms, batch_tfms, **kwargs)
     80 
     81     def databunch(self, source, path='.', type_tfms=None, item_tfms=None, batch_tfms=None, **kwargs):
---> 82         dsrc = self.datasource(source, type_tfms=type_tfms)
     83         item_tfms  = _merge_tfms(self.default_item_tfms,  item_tfms)
     84         batch_tfms = _merge_tfms(self.default_batch_tfms, batch_tfms)

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/block.py in datasource(self, source, type_tfms)
     77         type_tfms = L([self.default_type_tfms, type_tfms, labellers]).map_zip(
     78             lambda tt,tfm,l: L(l) + _merge_tfms(tt, tfm))
---> 79         return DataSource(items, tfms=type_tfms, splits=splits, dl_type=self.dl_type, n_inp=self.n_inp)
     80 
     81     def databunch(self, source, path='.', type_tfms=None, item_tfms=None, batch_tfms=None, **kwargs):

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/core.py in __init__(self, items, tfms, tls, n_inp, dl_type, **kwargs)
    233     def __init__(self, items=None, tfms=None, tls=None, n_inp=None, dl_type=None, **kwargs):
    234         super().__init__(dl_type=dl_type)
--> 235         self.tls = L(tls if tls else [TfmdList(items, t, **kwargs) for t in L(ifnone(tfms,[None]))])
    236         self.n_inp = (1 if len(self.tls)==1 else len(self.tls)-1) if n_inp is None else n_inp
    237 

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/core.py in <listcomp>(.0)
    233     def __init__(self, items=None, tfms=None, tls=None, n_inp=None, dl_type=None, **kwargs):
    234         super().__init__(dl_type=dl_type)
--> 235         self.tls = L(tls if tls else [TfmdList(items, t, **kwargs) for t in L(ifnone(tfms,[None]))])
    236         self.n_inp = (1 if len(self.tls)==1 else len(self.tls)-1) if n_inp is None else n_inp
    237 

~/daisy-gan/venv/lib/python3.6/site-packages/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     39             return x
     40 
---> 41         res = super().__call__(*((x,) + args), **kwargs)
     42         res._newchk = 0
     43         return res

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/core.py in __init__(self, items, tfms, use_list, do_setup, as_item, split_idx, train_setup, splits, types)
    169                  splits=None, types=None):
    170         super().__init__(items, use_list=use_list)
--> 171         self.splits = L([slice(None),[]] if splits is None else splits).map(mask2idxs)
    172         if isinstance(tfms,TfmdList): tfms = tfms.tfms
    173         if isinstance(tfms,Pipeline): do_setup=False

~/daisy-gan/venv/lib/python3.6/site-packages/fastcore/foundation.py in map(self, f, *args, **kwargs)
    360              else f.format if isinstance(f,str)
    361              else f.__getitem__)
--> 362         return self._new(map(g, self))
    363 
    364     def filter(self, f, negate=False, **kwargs):

~/daisy-gan/venv/lib/python3.6/site-packages/fastcore/foundation.py in _new(self, items, *args, **kwargs)
    313     @property
    314     def _xtra(self): return None
--> 315     def _new(self, items, *args, **kwargs): return type(self)(items, *args, use_list=None, **kwargs)
    316     def __getitem__(self, idx): return self._get(idx) if is_indexer(idx) else L(self._get(idx), use_list=None)
    317     def copy(self): return self._new(self.items.copy())

~/daisy-gan/venv/lib/python3.6/site-packages/fastcore/foundation.py in __call__(cls, x, *args, **kwargs)
     39             return x
     40 
---> 41         res = super().__call__(*((x,) + args), **kwargs)
     42         res._newchk = 0
     43         return res

~/daisy-gan/venv/lib/python3.6/site-packages/fastcore/foundation.py in __init__(self, items, use_list, match, *rest)
    304         if items is None: items = []
    305         if (use_list is not None) or not _is_array(items):
--> 306             items = list(items) if use_list else _listify(items)
    307         if match is not None:
    308             if is_coll(match): match = len(match)

~/daisy-gan/venv/lib/python3.6/site-packages/fastcore/foundation.py in _listify(o)
    240     if isinstance(o, list): return o
    241     if isinstance(o, str) or _is_array(o): return [o]
--> 242     if is_iter(o): return list(o)
    243     return [o]
    244 

~/daisy-gan/venv/lib/python3.6/site-packages/fastcore/foundation.py in __call__(self, *args, **kwargs)
    206             if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    207         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 208         return self.fn(*fargs, **kwargs)
    209 
    210 # Cell

~/daisy-gan/venv/lib/python3.6/site-packages/fastcore/foundation.py in mask2idxs(mask)
    256     if len(mask)==0: return []
    257     it = mask[0]
--> 258     if hasattr(it,'item'): it = it.item()
    259     if isinstance(it,(bool,NoneType,np.bool_)): return [i for i,m in enumerate(mask) if m]
    260     return [int(i) for i in mask]

ValueError: only one element tensors can be converted to Python scalars

I can get e.g. the examples running just fine.

But I think this may be a bug, because at no point I am defining any masks. But it is totally possible I am doing something wrong the data model is still not 100% clear to me

The only other way I can think of converting a pytorch dataloader into a fastai2 one is like this:

tfa_dl = TfmdDL(fa_dl)
tfa_dl.one_batch()

But that also makes an error on the last line:

TypeError                                 Traceback (most recent call last)
<ipython-input-136-cfc6f26b1fa6> in <module>
----> 1 tfa_dl.one_batch()

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/load.py in one_batch(self)
    128     def one_batch(self):
    129         if self.n is not None and len(self)==0: raise ValueError(f'This DataLoader does not contain any batches')
--> 130         with self.fake_l.no_multiproc(): res = first(self)
    131         if hasattr(self, 'it'): delattr(self, 'it')
    132         return res

~/daisy-gan/venv/lib/python3.6/site-packages/fastcore/utils.py in first(x)
    172 def first(x):
    173     "First element of `x`, or None if missing"
--> 174     try: return next(iter(x))
    175     except StopIteration: return None
    176 

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/load.py in __iter__(self)
     96         self.before_iter()
     97         for b in _loaders[self.fake_l.num_workers==0](self.fake_l):
---> 98             if self.device is not None: b = to_device(b, self.device)
     99             yield self.after_batch(b)
    100         self.after_iter()

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/core.py in device(self)
     90     def device(self):
     91         if defaults.use_cuda==False: return 'cpu'
---> 92         if not getattr(self, '_device', None): self._one_pass()
     93         return self._device
     94 

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/core.py in _one_pass(self)
     41 
     42     def _one_pass(self):
---> 43         its = self.after_batch(self.do_batch([self.do_item(0)]))
     44         self._device = find_device(its)
     45         self._n_inp = 1 if not isinstance(its, (list,tuple)) or len(its)==1 else len(its)-1

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/load.py in do_item(self, s)
    117     def prebatched(self): return self.bs is None
    118     def do_item(self, s):
--> 119         try: return self.after_item(self.create_item(s))
    120         except SkipItemException: return None
    121     def chunkify(self, b): return b if self.prebatched else chunked(b, self.bs, self.drop_last)

~/daisy-gan/venv/lib/python3.6/site-packages/fastai2/data/load.py in create_item(self, s)
    123     def randomize(self): self.rng = random.Random(self.rng.randint(0,2**32-1))
    124     def retain(self, res, b):  return retain_types(res, b[0] if is_listy(b) else b)
--> 125     def create_item(self, s):  return next(self.it) if s is None else self.dataset[s]
    126     def create_batch(self, b): return (fa_collate,fa_convert)[self.prebatched](b)
    127     def do_batch(self, b): return self.retain(self.create_batch(self.before_batch(b)), b)

TypeError: 'DataLoader' object does not support indexing
jakubLangr commented 4 years ago

Ha, my bad it should be

my_dataset = torch.utils.data.TensorDataset(x_t)
my_dloader = torch.utils.data.DataLoader(my_dataset)
fa_dl = DataLoader(my_dloader)
img_dbunch = DataBunch(fa_dl)
# img_dblock.databunch(fa_dl)
img_dbunch.one_batch()

But it still does not actually show any images sadly. Just the values of the tensors—even with actual images.