search

spacetelescope / jdaviz

JWST astronomical data analysis tools in the Jupyter platform
https://jdaviz.readthedocs.io/
BSD 3-Clause "New" or "Revised" License
141 stars 74 forks source link

[BUG] Mosviz: Model Fitting doesn't work in Mosviz #752

Open PatrickOgle opened 3 years ago

PatrickOgle commented 3 years ago

Describe the bug All attempts to fit a model using the Model Fitting plugin in Mosviz fail, for even the simplest model (Constant1D), to either the full Dataset or a Subset. The following traceback is generated:

IndexError                                Traceback (most recent call last)
~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/ipyvue/VueTemplateWidget.py in _handle_event(self, _, content, buffers)
     55                 getattr(self, 'vue_' + event)(data, buffers)
     56             else:
---> 57                 getattr(self, 'vue_' + event)(data)
     58 
     59 

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/jdaviz/configs/default/plugins/model_fitting/model_fitting.py in vue_model_fitting(self, *args, **kwargs)
    359         self._fitted_spectrum = fitted_spectrum
    360 
--> 361         self.vue_register_spectrum({"spectrum": fitted_spectrum})
    362         self.app.fitted_models[self.model_label] = fitted_model
    363 

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/jdaviz/configs/default/plugins/model_fitting/model_fitting.py in vue_register_spectrum(self, event)
    498             # Remove the actual Glue data object from the data_collection
    499             self.data_collection.remove(self.data_collection[label])
--> 500         self.data_collection[label] = spectrum
    501 
    502         self.app.add_data_to_viewer('spectrum-viewer', label)

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data_collection.py in __setitem__(self, key, data)
    391                 self.remove(existing_data)
    392 
--> 393         self.append(data)
    394 
    395     def __iter__(self):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data_collection.py in append(self, data)
     80                 s.register()
     81             msg = DataCollectionAddMessage(self, data)
---> 82             self.hub.broadcast(msg)
     83 
     84         self._sync_link_manager()

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/hub.py in broadcast(self, message)
    213             logging.getLogger(__name__).info("Broadcasting %s", message)
    214             for subscriber, handler in self._find_handlers(message):
--> 215                 handler(message)
    216 
    217     def __getstate__(self):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/jdaviz/app.py in _on_data_added(self, msg)
   1022             the new data.
   1023         """
-> 1024         self._link_new_data()
   1025         data_item = self._create_data_item(msg.data.label)
   1026         self.state.data_items.append(data_item)

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/jdaviz/app.py in _link_new_data(self)
    257                 continue
    258             else:
--> 259                 self.data_collection.add_link(LinkSame(wc_old[0], wc_new[0]))
    260                 break
    261 

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data_collection.py in add_link(self, links)
    160            instances, or a :class:`~glue.core.link_helpers.LinkCollection`
    161         """
--> 162         self._link_manager.add_link(links)
    163 
    164     def remove_link(self, links):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/link_manager.py in add_link(self, link, update_external)
    187                 self._external_links.append(link)
    188                 if update_external:
--> 189                     self.update_externally_derivable_components()
    190 
    191     @contract(link=ComponentLink)

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/link_manager.py in update_externally_derivable_components(self, data)
    240                 d = DerivedComponent(data, link)
    241                 comps[cid] = d
--> 242             data._set_externally_derivable_components(comps)
    243 
    244         # Now update information about pixel-aligned data

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data.py in _set_externally_derivable_components(self, derivable_components)
   1028         if self.hub:
   1029             msg = ExternallyDerivableComponentsChangedMessage(self)
-> 1030             self.hub.broadcast(msg)
   1031 
   1032     def _set_pixel_aligned_data(self, pixel_aligned_data):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/hub.py in broadcast(self, message)
    213             logging.getLogger(__name__).info("Broadcasting %s", message)
    214             for subscriber, handler in self._find_handlers(message):
--> 215                 handler(message)
    216 
    217     def __getstate__(self):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/viewers/common/viewer.py in _update_data(self, message)
    271                 else:
    272                     if layer_artist.layer is message.data:
--> 273                         layer_artist.update()
    274 
    275     def _update_subset(self, message):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue_jupyter/table/viewer.py in update(self)
    157 
    158     def update(self):
--> 159         self._refresh()
    160 
    161     def clear(self):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue_jupyter/table/viewer.py in _refresh(self)
    151 
    152     def _refresh(self):
--> 153         self._table_viewer.redraw()
    154 
    155     def redraw(self):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue_jupyter/table/viewer.py in redraw(self)
    204             self.widget_table.selections = [subset.label for subset in subsets]
    205             self.widget_table.selection_colors = [subset.style.color for subset in subsets]
--> 206         self.widget_table._update()
    207 
    208     def apply_filter(self):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue_jupyter/table/viewer.py in _update(self)
     29     def _update(self):
     30         self._update_columns()
---> 31         self._update_items()
     32         self.total_length = len(self)
     33         self.options = {**self.options, 'totalItems': self.total_length}

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue_jupyter/table/viewer.py in _update_items(self)
     68 
     69     def _update_items(self):
---> 70         self.items = self._get_items()
     71 
     72     @traitlets.default('items')

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue_jupyter/table/viewer.py in _get_items(self)
    127 
    128         view = slice(i1, i2)
--> 129         masks = {k.label: k.to_mask(view) for k in self.data.subsets}
    130 
    131         items = []

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue_jupyter/table/viewer.py in <dictcomp>(.0)
    127 
    128         view = slice(i1, i2)
--> 129         masks = {k.label: k.to_mask(view) for k in self.data.subsets}
    130 
    131         items = []

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/subset.py in to_mask(self, view)
    180 
    181         """
--> 182         return self.data.get_mask(self.subset_state, view=view)
    183 
    184     @contract(value=bool)

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data.py in get_mask(self, subset_state, view)
   1383     def get_mask(self, subset_state, view=None):
   1384         try:
-> 1385             return subset_state.to_mask(self, view=view)
   1386         except IncompatibleAttribute:
   1387             return get_mask_with_key_joins(self, self._key_joins, subset_state, view=view)

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/subset.py in to_mask(self, data, view)
    771     @contract(data='isinstance(Data)', view='array_view')
    772     def to_mask(self, data, view=None):
--> 773         x = data[self.att, view]
    774         result = (x >= self.lo) & (x <= self.hi)
    775         return result

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data.py in __getitem__(self, key)
    569                 raise IncompatibleAttribute(_k)
    570 
--> 571         return self.get_data(key, view=view)
    572 
    573     def _ipython_key_completions_(self):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data.py in get_data(self, cid, view)
   1362 
   1363         if view is not None:
-> 1364             result = comp[view]
   1365         else:
   1366             result = comp.data

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/component.py in __getitem__(self, key)
    196 
    197     def __getitem__(self, key):
--> 198         return self._link.compute(self._data, key)
    199 
    200 

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/component_link.py in compute(self, data, view)
    164 
    165         # First we get the values of all the 'from' components.
--> 166         args = [data[join_component_view(f, view)] for f in self._from]
    167 
    168         # We keep track of the original shape of the arguments

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/component_link.py in <listcomp>(.0)
    164 
    165         # First we get the values of all the 'from' components.
--> 166         args = [data[join_component_view(f, view)] for f in self._from]
    167 
    168         # We keep track of the original shape of the arguments

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data.py in __getitem__(self, key)
    569                 raise IncompatibleAttribute(_k)
    570 
--> 571         return self.get_data(key, view=view)
    572 
    573     def _ipython_key_completions_(self):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data.py in get_data(self, cid, view)
   1362 
   1363         if view is not None:
-> 1364             result = comp[view]
   1365         else:
   1366             result = comp.data

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/component.py in __getitem__(self, key)
    196 
    197     def __getitem__(self, key):
--> 198         return self._link.compute(self._data, key)
    199 
    200 

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/component_link.py in compute(self, data, view)
    164 
    165         # First we get the values of all the 'from' components.
--> 166         args = [data[join_component_view(f, view)] for f in self._from]
    167 
    168         # We keep track of the original shape of the arguments

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/component_link.py in <listcomp>(.0)
    164 
    165         # First we get the values of all the 'from' components.
--> 166         args = [data[join_component_view(f, view)] for f in self._from]
    167 
    168         # We keep track of the original shape of the arguments

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data.py in __getitem__(self, key)
    569                 raise IncompatibleAttribute(_k)
    570 
--> 571         return self.get_data(key, view=view)
    572 
    573     def _ipython_key_completions_(self):

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/data.py in get_data(self, cid, view)
   1362 
   1363         if view is not None:
-> 1364             result = comp[view]
   1365         else:
   1366             result = comp.data

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/component.py in __getitem__(self, key)
    196 
    197     def __getitem__(self, key):
--> 198         return self._link.compute(self._data, key)
    199 
    200 

~/miniconda3/envs/jwebbinar7/lib/python3.8/site-packages/glue/core/component_link.py in compute(self, data, view)
    167 
    168         # We keep track of the original shape of the arguments
--> 169         original_shape = args[0].shape
    170         logger.debug("shape of first argument: %s", original_shape)
    171 

IndexError: list index out of range

To Reproduce Steps to reproduce the behavior:

  1. Go to Model Fitting
  2. Click on 1DSpectrum (or Subset)
  3. Add a constant model component 'C'
  4. Enter 'C' in the Model Equation Editor
  5. Press 'Fit'

Expected behavior The model should be fitted and plotted in the spectrum viewer

Screenshots N/A Desktop (please complete the following information):

Package versions (please complete the following information):

macOS-10.15.7-x86_64-i386-64bit Python 3.8.10 | packaged by conda-forge | (default, May 10 2021, 22:58:09) [Clang 11.1.0 ] Numpy 1.21.1 astropy 4.3 specutils 1.3 spectral-cube 0.5.0 pyyaml 5.4.1 click 8.0.1 asteval 0.9.25 idna 2.10 traitlets 5.0.5 bqplot 0.12.30 bqplot-image-gl 1.4.3 glue-core 1.0.1 glue-jupyter 0.7 glue-astronomy 0.2 echo 0.5 ipyvue 1.5.0 ipyvuetify 1.8.0 ipysplitpanes 0.2.0 ipygoldenlayout 0.4.0 voila 0.2.10 vispy 0.7.3 Jdaviz 1.2.dev365+ge29b827

Additional context (e.g. data files) ERS NIRSpec MOS PRISM dataset with cutouts, jwst 1.1.0-processed

Model fitting works in Specviz and Cubeviz, but has never been attempted for an x1d in Mosviz.

🐱

duytnguyendtn commented 3 years ago

This traceback looks similar to one we caught in #762. We should revisit this issue once that is merged

rosteen commented 3 years ago

It looks like this error is because the linking code in app.py attempts to link to the first dataset when new data is loaded, which means it's trying to link the fitted spectrum to an image. @astrofrog pointed out recently that the linking there could cause problems and well, here's a problem!

I don't think this will be solved by #762, will probably require a couple extra lines of code in app.py to do the linking more intelligently. There's a comment that it "Link[s] to the first dataset with compatible coordinates", but it really links to the first dataset with any world coordinates. We should change it to do what the comment says!

pllim commented 3 years ago

Perhaps #811 can shed some light on this as well?

rosteen commented 3 years ago

Maybe. I just tested with #762 and auto-link set to False and this still errors, with a (I think) slightly different trace. This looks like the error is cause by the glue-astronomy translator trying to get the mask attribute of the subset I'm fitting, but I haven't delved too deep into it yet.

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
~/opt/anaconda3/envs/viz_dev/lib/python3.9/site-packages/ipyvue/VueTemplateWidget.py in _handle_event(self, _, content, buffers)
     55                 getattr(self, 'vue_' + event)(data, buffers)
     56             else:
---> 57                 getattr(self, 'vue_' + event)(data)
     58 
     59 

~/projects/jdaviz/jdaviz/configs/default/plugins/model_fitting/model_fitting.py in vue_model_fitting(self, *args, **kwargs)
    359         self._fitted_spectrum = fitted_spectrum
    360 
--> 361         self.vue_register_spectrum({"spectrum": fitted_spectrum})
    362         self.app.fitted_models[self.model_label] = fitted_model
    363 

~/projects/jdaviz/jdaviz/configs/default/plugins/model_fitting/model_fitting.py in vue_register_spectrum(self, event)
    500         self.data_collection[label] = spectrum
    501 
--> 502         self.app.add_data_to_viewer('spectrum-viewer', label)

~/projects/jdaviz/jdaviz/app.py in add_data_to_viewer(self, viewer_reference, data_path, clear_other_data, ext)
    677         if data_id is not None:
    678             data_ids.append(data_id)
--> 679             self._update_selected_data_items(viewer_item['id'], data_ids)
    680         else:
    681             raise ValueError(

~/projects/jdaviz/jdaviz/app.py in _update_selected_data_items(self, viewer_id, selected_items)
    992                                               viewer_id=viewer_id,
    993                                               sender=self)
--> 994             self.hub.broadcast(add_data_message)
    995 
    996         # Remove any deselected data objects from viewer

~/projects/glue/glue/core/hub.py in broadcast(self, message)
    213             logging.getLogger(__name__).info("Broadcasting %s", message)
    214             for subscriber, handler in self._find_handlers(message):
--> 215                 handler(message)
    216 
    217     def __getstate__(self):

~/projects/jdaviz/jdaviz/configs/default/plugins/line_lists/line_lists.py in _on_viewer_data_changed(self, msg)
     95 
     96         try:
---> 97             viewer_data = self.app.get_viewer('spectrum-viewer').data()
     98         except TypeError:
     99             warn_message = SnackbarMessage("Line list plugin could not retrieve data from viewer",

~/projects/jdaviz/jdaviz/configs/specviz/plugins/viewers.py in data(self, cls)
     69                         handler, _ = data_translator.get_handler_for(_class)
     70                         try:
---> 71                             layer_data = handler.to_object(layer_data,
     72                                                            statistic=statistic)
     73                         except IncompatibleAttribute:

~/opt/anaconda3/envs/viz_dev/lib/python3.9/site-packages/glue_astronomy/translators/spectrum1d.py in to_object(self, data_or_subset, attribute, statistic)
    142             return data_kwargs
    143 
--> 144         data_kwargs = parse_attributes(
    145             [attribute] if not hasattr(attribute, '__len__') else attribute)
    146 

~/opt/anaconda3/envs/viz_dev/lib/python3.9/site-packages/glue_astronomy/translators/spectrum1d.py in parse_attributes(attributes)
    109                     mask = None
    110                 else:
--> 111                     mask = data.get_mask(subset_state=subset_state)
    112                     mask = ~mask
    113 

~/projects/glue/glue/core/data.py in get_mask(self, subset_state, view)
   1383     def get_mask(self, subset_state, view=None):
   1384         try:
-> 1385             return subset_state.to_mask(self, view=view)
   1386         except IncompatibleAttribute:
   1387             return get_mask_with_key_joins(self, self._key_joins, subset_state, view=view)

~/projects/glue/glue/core/subset.py in to_mask(self, data, view)
    771     @contract(data='isinstance(Data)', view='array_view')
    772     def to_mask(self, data, view=None):
--> 773         x = data[self.att, view]
    774         result = (x >= self.lo) & (x <= self.hi)
    775         return result

~/projects/glue/glue/core/data.py in __getitem__(self, key)
    569                 raise IncompatibleAttribute(_k)
    570 
--> 571         return self.get_data(key, view=view)
    572 
    573     def _ipython_key_completions_(self):

~/projects/glue/glue/core/data.py in get_data(self, cid, view)
   1364             result = comp[view]
   1365         else:
-> 1366             result = comp.data
   1367 
   1368         return result

~/projects/glue/glue/core/component.py in data(self)
    188     def data(self):
    189         """ Return the numerical data as a numpy array """
--> 190         return self._link.compute(self._data)
    191 
    192     @property

~/projects/glue/glue/core/component_link.py in compute(self, data, view)
    164 
    165         # First we get the values of all the 'from' components.
--> 166         args = [data[join_component_view(f, view)] for f in self._from]
    167 
    168         # We keep track of the original shape of the arguments

~/projects/glue/glue/core/component_link.py in <listcomp>(.0)
    164 
    165         # First we get the values of all the 'from' components.
--> 166         args = [data[join_component_view(f, view)] for f in self._from]
    167 
    168         # We keep track of the original shape of the arguments

~/projects/glue/glue/core/data.py in __getitem__(self, key)
    569                 raise IncompatibleAttribute(_k)
    570 
--> 571         return self.get_data(key, view=view)
    572 
    573     def _ipython_key_completions_(self):

~/projects/glue/glue/core/data.py in get_data(self, cid, view)
   1364             result = comp[view]
   1365         else:
-> 1366             result = comp.data
   1367 
   1368         return result

~/projects/glue/glue/core/component.py in data(self)
    188     def data(self):
    189         """ Return the numerical data as a numpy array """
--> 190         return self._link.compute(self._data)
    191 
    192     @property

~/projects/glue/glue/core/component_link.py in compute(self, data, view)
    167 
    168         # We keep track of the original shape of the arguments
--> 169         original_shape = args[0].shape
    170         logger.debug("shape of first argument: %s", original_shape)
    171 

IndexError: list index out of range
pllim commented 3 years ago

For future reference, I tried patching glue-astronomy in glue-viz/glue-astronomy#39 (to fix Cubeviz) but Tom said glue-astronomy is doing the right thing by raising exception, so it is up to Jdaviz to catch it.

rosteen commented 3 years ago

@PatrickOgle As I mentioned offline, I still see an error with some of the older simulated NIRSPEC data, but with the most recent simulated dataset the model fitting appears to work fine. Are you ok with closing this issue given that it seems to have been a now-fixed problem with the old simulated data?

PatrickOgle commented 3 years ago

I will have to see it work in my Jwebbinar notebook to be convinced...