get_fastccd_images cannot process all instances of recorded data

[ambarb]

For this

scans = [60144,60145,60146]
dark8s = [60148,60147,60147]

nth = 2 
print(f'Using S-{scans[nth]} with dark S-{dark8s[nth]} ')
stack = get_fastccd_images(db[scans[nth]], (db[dark8s[nth]], None, None), tag='fccd_image')  #fccd_image_lightfield  or fccd_image
images = stack[0]

the last data set has darks with requested number of images = 2500, but the scan was stopped before it finished (2133). This will be a problem for data collected with swimmer, data that we ask DAMA to edit the database because we are missing the table information, and so on. In this case, we have the table information. I'm not sure what happens if we ^C a motor scan....maybe its okay to use the data up to a certain point, but if the last point doesn't have the full number of images, then this will be problematic.

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-53-edc6e725dc72> in <module>()
      1 nth = 2
      2 print(f'Using S-{db[scans[nth]].start.scan_id} with dark S-{db[dark8s[nth]].start.scan_id} ')
----> 3 stack = get_fastccd_images(db[scans[nth]], (db[dark8s[nth]], None, None), tag='fccd_image')  #fccd_image_lightfield  or fccd_image
      4 images = stack[0]

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/utils.py in get_fastccd_images(light_header, dark_headers, flat, gain, tag, roi)
     92 
     93                 tt = ttime.time()
---> 94                 b = get_images_to_3D(bgnd_events, dtype=np.uint16)
     95                 logger.info("Image conversion took %.3f seconds",
     96                             ttime.time() - tt)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/utils.py in get_images_to_3D(images, dtype)
    173 
    174     """
--> 175     im = np.vstack([np.asarray(im, dtype=dtype) for im in images])
    176     return im
    177 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/utils.py in <listcomp>(.0)
    173 
    174     """
--> 175     im = np.vstack([np.asarray(im, dtype=dtype) for im in images])
    176     return im
    177 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/numpy/core/numeric.py in asarray(a, dtype, order)
    490 
    491     """
--> 492     return array(a, dtype, copy=False, order=order)
    493 
    494 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in <genexpr>(.0)
    224 
    225     def __iter__(self):
--> 226         return (self._get(i) for i in self.indices)
    227 
    228     def __len__(self):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in _get(self, key)
    204 
    205     def _get(self, key):
--> 206         return self._ancestor[key]
    207 
    208     def _map_index(self, key):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in __getitem__(self, i)
    185                 indices, new_length = key_to_indices(i, len(self))
    186                 if new_length is None:
--> 187                     return self._get(indices)
    188                 else:
    189                     return cls(self, indices, new_length, propagate_attrs)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/pims/base_frames.py in __getitem__(self, key)
    146         """__getitem__ is handled by Slicerator. In all pims readers, the data
    147         returning function is get_frame."""
--> 148         return self.get_frame(key)
    149 
    150     def __iter__(self):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/databroker/assets/handlers.py in get_frame(self, i)
     35 
     36     def get_frame(self, i):
---> 37         return Frame(self._dataset[self._start + i], frame_no=i)
     38 
     39     def __len__(self):

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/dataset.py in __getitem__(self, args)
    474 
    475         # Perform the dataspace selection.
--> 476         selection = sel.select(self.shape, args, dsid=self.id)
    477 
    478         if selection.nselect == 0:

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in select(shape, args, dsid)
     92 
     93     sel = SimpleSelection(shape)
---> 94     sel[args]
     95     return sel
     96 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in __getitem__(self, args)
    259             return self
    260 
--> 261         start, count, step, scalar = _handle_simple(self.shape,args)
    262 
    263         self._id.select_hyperslab(start, count, step)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in _handle_simple(shape, args)
    449         else:
    450             try:
--> 451                 x,y,z = _translate_int(int(arg), length)
    452                 s = True
    453             except TypeError:

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in _translate_int(exp, length)
    469 
    470     if not 0<=exp<length:
--> 471         raise ValueError("Index (%s) out of range (0-%s)" % (exp, length-1))
    472 
    473     return exp, 1, 1

ValueError: Index (2134) out of range (0-2133)

[ambarb]

We need to investigate further:

• [ ] Is the problem with fet_table() or is it within in the csxtools code
• [ ] Is this a problem using .data and/or catalogs?
• [ ] Need to expand test cases to be XPCS data (which is a separate problem) and scan data.

[ambarb]

stale