search

NSLS-II-CSX / csxtools

Useful python tools for CSX (23-ID)
http://nsls-ii-csx.github.io/csxtools
Other
4 stars 13 forks source link

Strange error related to dark image scan with x2 gain #75

Closed ambarb closed 1 year ago

ambarb commented 3 years ago

Possible corrupt data for gain2??? Using None ordb[354022] works for bgnd2, but the scan that is supposed to be used results in an error (db[135026]).

scans =   [134931, 134958, 134914, 134910, 135020, 135024, ]
bgnd1 =   [134934, 134961, 134917, 134913, 135023, 135027, ]#None #
bgnd2 =   [134933, 134960, 134916, 134912, 135022, 135026, ]#None #
bgnd8 =   [134932, 134959, 134915, 134911, 135021, 135025, ]

ii = 5 #which scan index

scan_no =  scans[ii] 
bgnd1 = db[bgnd1[ii]]
bgnd2 = db[bgnd2[ii]]
bgnd8 = db[bgnd8[ii],]

#images = get_fastccd_images(h, (bgnd8[0],None,None), flat=None) #  **works** 135024
#images = get_fastccd_images(h, (bgnd8[0],None, bgnd1), flat=None) # works 135024
images = get_fastccd_images(h, (bgnd8[0],bgnd2,None), flat=None) # doesn't work for 135024
#images = get_fastccd_images(h, (bgnd8[0], bgnd2, bgnd1), flat=None) # doesn't work for 135024

stack = get_images_to_4D(images) / ADUpPH
print('Raw image size is: ',stack.shape)
/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/utils.py:181: UserWarning: Images and get_images are deprecated. Use Header.data('fccd_image') instead.
  images = header.db.get_images(header, tag)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-227-35c7fa5e722f> in <module>()
      2 #images = get_fastccd_images(h, (bgnd8[0],None,None), flat=None) # works 135024
      3 #images = get_fastccd_images(h, (bgnd8[0],None, bgnd1), flat=None) # works 135024
----> 4 images = get_fastccd_images(h, (bgnd8[0],bgnd2,None), flat=None) # doesn't work for 135024
      5 #images = get_fastccd_images(h, (bgnd8[0], bgnd2, bgnd1), flat=None) # doesn't work for 135024
      6 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/utils.py in get_fastccd_images(light_header, dark_headers, flat, gain, tag, roi)
     92 
     93                 tt = ttime.time()
---> 94                 b = get_images_to_3D(bgnd_events, dtype=np.uint16)
     95                 logger.info("Image conversion took %.3f seconds",
     96                             ttime.time() - tt)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/utils.py in get_images_to_3D(images, dtype)
    173 
    174     """
--> 175     im = np.vstack([np.asarray(im, dtype=dtype) for im in images])
    176     return im
    177 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/utils.py in <listcomp>(.0)
    173 
    174     """
--> 175     im = np.vstack([np.asarray(im, dtype=dtype) for im in images])
    176     return im
    177 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/numpy/core/numeric.py in asarray(a, dtype, order)
    490 
    491     """
--> 492     return array(a, dtype, copy=False, order=order)
    493 
    494 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in <genexpr>(.0)
    224 
    225     def __iter__(self):
--> 226         return (self._get(i) for i in self.indices)
    227 
    228     def __len__(self):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in _get(self, key)
    204 
    205     def _get(self, key):
--> 206         return self._ancestor[key]
    207 
    208     def _map_index(self, key):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in __getitem__(self, i)
    185                 indices, new_length = key_to_indices(i, len(self))
    186                 if new_length is None:
--> 187                     return self._get(indices)
    188                 else:
    189                     return cls(self, indices, new_length, propagate_attrs)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/pims/base_frames.py in __getitem__(self, key)
    146         """__getitem__ is handled by Slicerator. In all pims readers, the data
    147         returning function is get_frame."""
--> 148         return self.get_frame(key)
    149 
    150     def __iter__(self):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/databroker/assets/handlers.py in get_frame(self, i)
     35 
     36     def get_frame(self, i):
---> 37         return Frame(self._dataset[self._start + i], frame_no=i)
     38 
     39     def __len__(self):

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/dataset.py in __getitem__(self, args)
    474 
    475         # Perform the dataspace selection.
--> 476         selection = sel.select(self.shape, args, dsid=self.id)
    477 
    478         if selection.nselect == 0:

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in select(shape, args, dsid)
     92 
     93     sel = SimpleSelection(shape)
---> 94     sel[args]
     95     return sel
     96 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in __getitem__(self, args)
    259             return self
    260 
--> 261         start, count, step, scalar = _handle_simple(self.shape,args)
    262 
    263         self._id.select_hyperslab(start, count, step)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in _handle_simple(shape, args)
    449         else:
    450             try:
--> 451                 x,y,z = _translate_int(int(arg), length)
    452                 s = True
    453             except TypeError:

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in _translate_int(exp, length)
    469 
    470     if not 0<=exp<length:
--> 471         raise ValueError("Index (%s) out of range (0-%s)" % (exp, length-1))
    472 
    473     return exp, 1, 1

ValueError: Index (1) out of range (0-0)
ambarb commented 3 years ago

Can load the raw x1 gain dark images.

-used with the above code

stack = get_fastccd_images(bgnd1, (None), flat=None)
print(type(stack))
#images = get_images_to_3D(stack) 
images = stack[0]
print(images.shape)

Can NOT load the raw x2 gain dark images.

-used with the above code

stack = get_fastccd_images(bgnd2, (None), flat=None)
print(type(stack))
#images = get_images_to_3D(stack) 
images = stack[0]
print(images.shape)
Processing without dark images
<class 'slicerator.Pipeline'>
/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/utils.py:181: UserWarning: Images and get_images are deprecated. Use Header.data('fccd_image') instead.
  images = header.db.get_images(header, tag)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-240-3226e65c5e69> in <module>()
     14 print(type(stack))
     15 #images = get_images_to_3D(stack)
---> 16 images = stack[0]
     17 print(images.shape)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in __getitem__(self, i)
    431         indices, new_length = key_to_indices(i, len(self))
    432         if new_length is None:
--> 433             return self._get(indices)
    434         else:
    435             return Slicerator(self, indices, new_length, self._propagate_attrs)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in _get(self, key)
    413         # We need to copy here: else any _proc_func that acts inplace would
    414         # change the ancestor value.
--> 415         return self._proc_func(copy(self._ancestor[key]))
    416 
    417     def __repr__(self):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in proc_func(x)
    591                 or isinstance(obj, Pipeline):
    592             def proc_func(x):
--> 593                 return func(x, *args, **kwargs)
    594 
    595             return Pipeline(obj, proc_func)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/utils.py in _correct_fccd_images(image, bgnd, flat, gain)
    191 @pipeline
    192 def _correct_fccd_images(image, bgnd, flat, gain):
--> 193     image = correct_images(image, bgnd, flat, gain)
    194     image = rotate90(image, 'cw')
    195     return image

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/csxtools/fastccd/images.py in correct_images(images, dark, flat, gain)
     50         flat = np.asarray(flat, dtype=np.float32)
     51 
---> 52     data = fastccd.correct_images(images, dark, flat, gain)
     53     t = ttime.time() - t
     54 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in <genexpr>(.0)
    224 
    225     def __iter__(self):
--> 226         return (self._get(i) for i in self.indices)
    227 
    228     def __len__(self):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in _get(self, key)
    204 
    205     def _get(self, key):
--> 206         return self._ancestor[key]
    207 
    208     def _map_index(self, key):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/slicerator.py in __getitem__(self, i)
    185                 indices, new_length = key_to_indices(i, len(self))
    186                 if new_length is None:
--> 187                     return self._get(indices)
    188                 else:
    189                     return cls(self, indices, new_length, propagate_attrs)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/pims/base_frames.py in __getitem__(self, key)
    146         """__getitem__ is handled by Slicerator. In all pims readers, the data
    147         returning function is get_frame."""
--> 148         return self.get_frame(key)
    149 
    150     def __iter__(self):

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/databroker/assets/handlers.py in get_frame(self, i)
     35 
     36     def get_frame(self, i):
---> 37         return Frame(self._dataset[self._start + i], frame_no=i)
     38 
     39     def __len__(self):

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

h5py/_objects.pyx in h5py._objects.with_phil.wrapper()

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/dataset.py in __getitem__(self, args)
    474 
    475         # Perform the dataspace selection.
--> 476         selection = sel.select(self.shape, args, dsid=self.id)
    477 
    478         if selection.nselect == 0:

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in select(shape, args, dsid)
     92 
     93     sel = SimpleSelection(shape)
---> 94     sel[args]
     95     return sel
     96 

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in __getitem__(self, args)
    259             return self
    260 
--> 261         start, count, step, scalar = _handle_simple(self.shape,args)
    262 
    263         self._id.select_hyperslab(start, count, step)

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in _handle_simple(shape, args)
    449         else:
    450             try:
--> 451                 x,y,z = _translate_int(int(arg), length)
    452                 s = True
    453             except TypeError:

/opt/conda_envs/analysis-2018-2.1/lib/python3.6/site-packages/h5py/_hl/selections.py in _translate_int(exp, length)
    469 
    470     if not 0<=exp<length:
--> 471         raise ValueError("Index (%s) out of range (0-%s)" % (exp, length-1))
    472 
    473     return exp, 1, 1

ValueError: Index (1) out of range (0-0)
ambarb commented 3 years ago

So need to try to load from file directly not using get_fccd_images() to understand if something bad happened to this file.

ambarb commented 1 year ago

probably bad file or data is missing and databroker refuses to open