ebachelet / pyLIMA

GNU General Public License v3.0
30 stars 8 forks source link

PicklingError when starting a new fit #39

Closed Nijaid closed 1 year ago

Nijaid commented 4 years ago

I attempt to run two fits on the same event:

binary_par_model = microlmodels.create_model('FSBL', myevent, parallax=['Full', 2457211])
binary_par_model.parameters_boundaries[0] = [2457190, 2457230]

myevent.fit(binary_par_model, 'DE', DE_population_size=10, computational_pool=pool)

binary_model = microlmodels.create_model('FSBL', myevent)
binary_model.parameters_boundaries[0] = [2457190, 2457230]

myevent.fit(binary_model, 'DE', DE_population_size=10, computational_pool=pool)

The first one runs successfully, but the second one gives the error:

Start fit on Event, with model FSBL and method DE
check_event  : Everything looks fine, this event can be fitted
---------------------------------------------------------------------------
PicklingError                             Traceback (most recent call last)
<ipython-input-15-22572dab00e5> in <module>
      3 
----> 4 myevent.fit(binary_model, 'DE', DE_population_size=10, computational_pool=pool)

~/code/pylima/pyLIMA/pyLIMA/event.py in fit(self, model, method, DE_population_size, flux_estimation_MCMC, fix_parameters_dictionnary, grid_resolution, computational_pool, binary_regime)
    105         fit.mlfit(model, method, DE_population_size=DE_population_size, flux_estimation_MCMC=flux_estimation_MCMC,
    106                   fix_parameters_dictionnary=fix_parameters_dictionnary,
--> 107                   grid_resolution=grid_resolution, computational_pool=computational_pool, binary_regime=binary_regime)
    108 
    109         self.fits.append(fit)

~/code/pylima/pyLIMA/pyLIMA/microlfits.py in mlfit(self, model, method, DE_population_size, flux_estimation_MCMC, fix_parameters_dictionnary, grid_resolution, computational_pool, binary_regime)
    184             self.fit_results, self.fit_covariance, self.fit_time = self.trust_region_reflective()
    185         if self.method == 'DE':
--> 186             self.fit_results, self.fit_covariance, self.fit_time = self.differential_evolution(pool)
    187 
    188         if self.method == 'MCMC':

~/code/pylima/pyLIMA/pyLIMA/microlfits.py in differential_evolution(self, pool)
    522             atol=0.1, strategy='rand1bin',
    523             recombination=0.7, polish=True, init='latinhypercube',
--> 524             disp=True,workers = worker,
    525         )
    526 

/anaconda/envs/pylima/lib/python3.6/site-packages/scipy/optimize/_differentialevolution.py in differential_evolution(func, bounds, args, strategy, maxiter, popsize, tol, mutation, recombination, seed, callback, disp, polish, init, atol, updating, workers)
    270                                      updating=updating,
    271                                      workers=workers) as solver:
--> 272         ret = solver.solve()
    273 
    274     return ret

/anaconda/envs/pylima/lib/python3.6/site-packages/scipy/optimize/_differentialevolution.py in solve(self)
    672         if np.all(np.isinf(self.population_energies)):
    673             self.population_energies[:] = self._calculate_population_energies(
--> 674                 self.population)
    675             self._promote_lowest_energy()
    676 

/anaconda/envs/pylima/lib/python3.6/site-packages/scipy/optimize/_differentialevolution.py in _calculate_population_energies(self, population)
    773         try:
    774             calc_energies = list(self._mapwrapper(self.func,
--> 775                                                   parameters_pop[0:nfevs]))
    776             energies[0:nfevs] = calc_energies
    777         except (TypeError, ValueError):

/anaconda/envs/pylima/lib/python3.6/site-packages/scipy/_lib/_util.py in __call__(self, func, iterable)
    412         # only accept one iterable because that's all Pool.map accepts
    413         try:
--> 414             return self._mapfunc(func, iterable)
    415         except TypeError:
    416             # wrong number of arguments

/anaconda/envs/limasuite/lib/python3.6/multiprocessing/pool.py in map(self, func, iterable, chunksize)
    264         in a list that is returned.
    265         '''
--> 266         return self._map_async(func, iterable, mapstar, chunksize).get()
    267 
    268     def starmap(self, func, iterable, chunksize=None):

/anaconda/envs/limasuite/lib/python3.6/multiprocessing/pool.py in get(self, timeout)
    642             return self._value
    643         else:
--> 644             raise self._value
    645 
    646     def _set(self, i, obj):

/anaconda/envs/limasuite/lib/python3.6/multiprocessing/pool.py in _handle_tasks(taskqueue, put, outqueue, pool, cache)
    422                         break
    423                     try:
--> 424                         put(task)
    425                     except Exception as e:
    426                         job, idx = task[:2]

/anaconda/envs/limasuite/lib/python3.6/multiprocessing/connection.py in send(self, obj)
    204         self._check_closed()
    205         self._check_writable()
--> 206         self._send_bytes(_ForkingPickler.dumps(obj))
    207 
    208     def recv_bytes(self, maxlength=None):

/anaconda/envs/limasuite/lib/python3.6/multiprocessing/reduction.py in dumps(cls, obj, protocol)
     49     def dumps(cls, obj, protocol=None):
     50         buf = io.BytesIO()
---> 51         cls(buf, protocol).dump(obj)
     52         return buf.getbuffer()
     53 

PicklingError: Can't pickle <class 'pyLIMA.microloutputs.Fit_outputs'>: attribute lookup Fit_outputs on pyLIMA.microloutputs failed

This could be an error in my environment, since I haven't seen this issue brought up by others, or a problem with the nested class interfering with the multiprocessing pool, which is setup as:

import multiprocessing as multip
pool = multip.Pool(processes=6)

Any thoughts?

ebachelet commented 4 years ago

That is likely due to the multiprocessing. I would recomand to run two separare fits, i.e. create two events etc, while I am working on a fix. Another try is to kill and restart a new pool between the two fits.