muellerzr / fastdebug

In-place debugger for the fastai library and pytorch
https://muellerzr.github.io/fastdebug
Apache License 2.0
28 stars 3 forks source link

Issue running lr_find after importing fastdebug #5

Open kevinbird15 opened 3 years ago

kevinbird15 commented 3 years ago

fastai version: 2.3.1 fastdebug version: 0.1.3

Here is the code I'm trying to run:

from fastai.vision.all import *
from fastdebug import * 

path = untar_data(URLs.PETS)
path.ls()
files = get_image_files(path/"images")
len(files)
files[0],files[6]
def label_func(f): return f[0].isupper()
dls = ImageDataLoaders.from_name_func(path, files, label_func, item_tfms=Resize(224))
dls.show_batch()
learn = cnn_learner(dls, resnet34, metrics=error_rate)
learn.lr_find()

When I run lr_find, I get the following error:

---------------------------------------------------------------------------
CancelFitException                        Traceback (most recent call last)
~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastdebug/fastai/learner.py in _call_one(self, event_name)
    122         try:
--> 123             cb(event_name)
    124         except Exception as e:

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/callback/core.py in __call__(self, event_name)
     43         res = None
---> 44         if self.run and _run: res = getattr(self, event_name, noop)()
     45         if event_name=='after_fit': self.run=True #Reset self.run to True at each end of fit

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/callback/schedule.py in after_batch(self)
    182         if self.smooth_loss < self.best_loss: self.best_loss = self.smooth_loss
--> 183         if self.smooth_loss > 4*self.best_loss and self.stop_div: raise CancelFitException()
    184         if self.train_iter >= self.num_it: raise CancelFitException()

CancelFitException: 

During handling of the above exception, another exception occurred:

IndexError                                Traceback (most recent call last)
<ipython-input-13-d81c6bd29d71> in <module>
----> 1 learn.lr_find()

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/callback/schedule.py in lr_find(self, start_lr, end_lr, num_it, stop_div, show_plot, suggestions)
    220     n_epoch = num_it//len(self.dls.train) + 1
    221     cb=LRFinder(start_lr=start_lr, end_lr=end_lr, num_it=num_it, stop_div=stop_div)
--> 222     with self.no_logging(): self.fit(n_epoch, cbs=cb)
    223     if show_plot: self.recorder.plot_lr_find()
    224     if suggestions:

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in fit(self, n_epoch, lr, wd, cbs, reset_opt)
    216             self.opt.set_hypers(lr=self.lr if lr is None else lr)
    217             self.n_epoch = n_epoch
--> 218             self._with_events(self._do_fit, 'fit', CancelFitException, self._end_cleanup)
    219 
    220     def _end_cleanup(self): self.dl,self.xb,self.yb,self.pred,self.loss = None,(None,),(None,),None,None

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    158 
    159     def _with_events(self, f, event_type, ex, final=noop):
--> 160         try: self(f'before_{event_type}');  f()
    161         except ex: self(f'after_cancel_{event_type}')
    162         self(f'after_{event_type}');  final()

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in _do_fit(self)
    207         for epoch in range(self.n_epoch):
    208             self.epoch=epoch
--> 209             self._with_events(self._do_epoch, 'epoch', CancelEpochException)
    210 
    211     def fit(self, n_epoch, lr=None, wd=None, cbs=None, reset_opt=False):

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    158 
    159     def _with_events(self, f, event_type, ex, final=noop):
--> 160         try: self(f'before_{event_type}');  f()
    161         except ex: self(f'after_cancel_{event_type}')
    162         self(f'after_{event_type}');  final()

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in _do_epoch(self)
    201 
    202     def _do_epoch(self):
--> 203         self._do_epoch_train()
    204         self._do_epoch_validate()
    205 

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in _do_epoch_train(self)
    193     def _do_epoch_train(self):
    194         self.dl = self.dls.train
--> 195         self._with_events(self.all_batches, 'train', CancelTrainException)
    196 
    197     def _do_epoch_validate(self, ds_idx=1, dl=None):

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    158 
    159     def _with_events(self, f, event_type, ex, final=noop):
--> 160         try: self(f'before_{event_type}');  f()
    161         except ex: self(f'after_cancel_{event_type}')
    162         self(f'after_{event_type}');  final()

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in all_batches(self)
    164     def all_batches(self):
    165         self.n_iter = len(self.dl)
--> 166         for o in enumerate(self.dl): self.one_batch(*o)
    167 
    168     def _do_one_batch(self):

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in one_batch(self, i, b)
    189         b = self._set_device(b)
    190         self._split(b)
--> 191         self._with_events(self._do_one_batch, 'batch', CancelBatchException)
    192 
    193     def _do_epoch_train(self):

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    160         try: self(f'before_{event_type}');  f()
    161         except ex: self(f'after_cancel_{event_type}')
--> 162         self(f'after_{event_type}');  final()
    163 
    164     def all_batches(self):

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastai/learner.py in __call__(self, event_name)
    139 
    140     def ordered_cbs(self, event): return [cb for cb in self.cbs.sorted('order') if hasattr(cb, event)]
--> 141     def __call__(self, event_name): L(event_name).map(self._call_one)
    142 
    143     def _call_one(self, event_name):

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastcore/foundation.py in map(self, f, gen, *args, **kwargs)
    152     def range(cls, a, b=None, step=None): return cls(range_of(a, b=b, step=step))
    153 
--> 154     def map(self, f, *args, gen=False, **kwargs): return self._new(map_ex(self, f, *args, gen=gen, **kwargs))
    155     def argwhere(self, f, negate=False, **kwargs): return self._new(argwhere(self, f, negate, **kwargs))
    156     def filter(self, f=noop, negate=False, gen=False, **kwargs):

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastcore/basics.py in map_ex(iterable, f, gen, *args, **kwargs)
    664     res = map(g, iterable)
    665     if gen: return res
--> 666     return list(res)
    667 
    668 # Cell

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastcore/basics.py in __call__(self, *args, **kwargs)
    649             if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    650         fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 651         return self.func(*fargs, **kwargs)
    652 
    653 # Cell

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastdebug/fastai/learner.py in _call_one(self, event_name)
    123             cb(event_name)
    124         except Exception as e:
--> 125             callback_error(e, cb.__repr__(), event_name)
    126 
    127 # Cell

~/anaconda3/envs/fastaudio/lib/python3.7/site-packages/fastdebug/fastai/learner.py in callback_error(e, cb, event_name)
     37     Raises an error from when a Callback event failed, showing what event, the name of the Callback and the trace
     38     """
---> 39     e.args = [f"Exception raised in the {cb} Callback during {event_name}:\n\n{e.args[0]}"]
     40     raise e
     41 

IndexError: tuple index out of range

I will keep digging into this but wanted to start by confirming it's an issue that is reproducible

kevinbird15 commented 3 years ago

image

Here are the args at the lowest level as well. Let me know if anything else would be useful

muellerzr commented 3 years ago

@kevinbird15 is this still true on the latest fastai version? (2.4?)

kevinbird15 commented 3 years ago

Yes it is. Just tested on 2.4.1 and seeing the same error