File ~/anaconda3/envs/tapqir-env/lib/python3.8/site-packages/tapqir/main.py:460, in fit(model, cuda, nbatch_size, fbatch_size, learning_rate, num_iter, k_max, matlab, funsor, pykeops, overwrite, no_input, progress_bar)
458 model.init(learning_rate, nbatch_size, fbatch_size)
459 try:
--> 460 model.run(num_iter, progress_bar=progress_bar)
461 except CudaOutOfMemoryError:
462 logger.exception("Failed to fit the data")
File ~/anaconda3/envs/tapqir-env/lib/python3.8/site-packages/tapqir/models/model.py:223, in Model.run(self, num_iter, progress_bar)
221 # save a checkpoint every 200 iterations
222 if not self.iter % 200:
--> 223 self.save_checkpoint(writer)
224 if use_crit and self.converged:
225 logger.info(f"Iteration #{self.iter} model converged.")
File ~/anaconda3/envs/tapqir-env/lib/python3.8/site-packages/tapqir/models/model.py:273, in Model.save_checkpoint(self, writer)
271 self.converged = False
272 if len(self._rolling["-ELBO"]) == self._rolling["-ELBO"].maxlen:
--> 273 crit = all(
274 torch.tensor(self._rolling[p]).std()
275 / torch.tensor(self._rolling[p])[-50:].std()
276 < 1.05
277 for p in self.conv_params
278 )
279 if crit:
280 self.converged = True
File ~/anaconda3/envs/tapqir-env/lib/python3.8/site-packages/tapqir/models/model.py:274, in (.0)
271 self.converged = False
272 if len(self._rolling["-ELBO"]) == self._rolling["-ELBO"].maxlen:
273 crit = all(
--> 274 torch.tensor(self._rolling[p]).std()
275 / torch.tensor(self._rolling[p])[-50:].std()
276 < 1.05
277 for p in self.conv_params
278 )
279 if crit:
280 self.converged = True
Fitting the data ... Iteration #56800. Loaded a model checkpoint from /home/jchung/YNE30June2022/.tapqir/cosmos-model.tpqr
0% 0/100000 [00:01<?, ?it/s]
KeyError Traceback (most recent call last) File ~/anaconda3/envs/tapqir-env/lib/python3.8/site-packages/tapqir/gui.py:502, in fitCmd(b, layout, out, DEFAULTS) 500 DEFAULTS["priors"].update(layout["priors"].children[0].kwargs) 501 with out: --> 502 fit( 503 **layout.kwargs, 504 k_max=2, 505 funsor=False, 506 pykeops=True, 507 no_input=True, 508 progress_bar=tqdm_notebook, 509 ) 511 out.clear_output(wait=True)
File ~/anaconda3/envs/tapqir-env/lib/python3.8/site-packages/tapqir/main.py:460, in fit(model, cuda, nbatch_size, fbatch_size, learning_rate, num_iter, k_max, matlab, funsor, pykeops, overwrite, no_input, progress_bar) 458 model.init(learning_rate, nbatch_size, fbatch_size) 459 try: --> 460 model.run(num_iter, progress_bar=progress_bar) 461 except CudaOutOfMemoryError: 462 logger.exception("Failed to fit the data")
File ~/anaconda3/envs/tapqir-env/lib/python3.8/site-packages/tapqir/models/model.py:223, in Model.run(self, num_iter, progress_bar) 221 # save a checkpoint every 200 iterations 222 if not self.iter % 200: --> 223 self.save_checkpoint(writer) 224 if use_crit and self.converged: 225 logger.info(f"Iteration #{self.iter} model converged.")
File ~/anaconda3/envs/tapqir-env/lib/python3.8/site-packages/tapqir/models/model.py:273, in Model.save_checkpoint(self, writer) 271 self.converged = False 272 if len(self._rolling["-ELBO"]) == self._rolling["-ELBO"].maxlen: --> 273 crit = all( 274 torch.tensor(self._rolling[p]).std() 275 / torch.tensor(self._rolling[p])[-50:].std() 276 < 1.05 277 for p in self.conv_params 278 ) 279 if crit: 280 self.converged = True
File ~/anaconda3/envs/tapqir-env/lib/python3.8/site-packages/tapqir/models/model.py:274, in(.0)
271 self.converged = False
272 if len(self._rolling["-ELBO"]) == self._rolling["-ELBO"].maxlen:
273 crit = all(
--> 274 torch.tensor(self._rolling[p]).std()
275 / torch.tensor(self._rolling[p])[-50:].std()
276 < 1.05
277 for p in self.conv_params
278 )
279 if crit:
280 self.converged = True
KeyError: 'lamda_loc'