I suspect that we need to make a similar change to XPD's profile.
This is leads to problems like in run '929ec727-3b1a-48d4-a05f-978431732a19' where because the key fs is not in the descriptor things barf with
In [8]: next(gen)
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/headersource/mongo_core.py in get_events_generator(descriptor, event_col, descriptor_col, descriptor_cache, run_start_col, run_start_cache, convert_arrays)
64 try:
---> 65 _dk = data_keys[k]
66 except KeyError as err:
KeyError: 'fs'
The above exception was the direct cause of the following exception:
MismatchedDataKeys Traceback (most recent call last)
<ipython-input-8-6e72e47198db> in <module>
----> 1 next(gen)
/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/_core.py in documents(self, stream_name, fields, fill)
324 stream_name=stream_name,
325 fill=fill)
--> 326 for payload in gen:
327 yield payload
328
/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/_core.py in get_documents(self, headers, stream_name, fields, fill, handler_registry)
1404 header=h, stream_name=stream_name,
1405 fields=fields)
-> 1406 for name, doc in gen:
1407 # Find all the res/datum backed keys
1408 if name == 'descriptor':
/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/eventsource/shim.py in docs_given_header(self, header, stream_name, fields)
157 yield 'descriptor', d
158 ev_gens.append(self.mds.get_events_generator(d))
--> 159 for ev in interlace_gens(*ev_gens):
160 event_data = ev['data'] # cache for perf
161 desc = ev['descriptor']
/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/eventsource/shim.py in interlace_gens(*gens)
45
46 for j, itr in enumerate(iters):
---> 47 safe_next(itr, j)
48 while heap:
49 _, j, val, itr = heapq.heappop(heap)
/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/eventsource/shim.py in safe_next(itr, indx)
39 def safe_next(itr, indx):
40 try:
---> 41 val = next(itr)
42 except StopIteration:
43 return
/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/headersource/base.py in get_events_generator(self, descriptor, convert_arrays)
191 # when we drop 2.7, this can be
192 # yield from evs
--> 193 for ev in evs:
194 yield ev
195
/opt/conda_envs/collection-2020-2.0rc7-1-xpdacq/lib/python3.7/site-packages/databroker/headersource/mongo_core.py in get_events_generator(descriptor, event_col, descriptor_col, descriptor_cache, run_start_col, run_start_cache, convert_arrays)
73 f"event['data'].keys(): {ev['data'].keys()}, "
74 "descriptor['data_keys'].keys(): "
---> 75 f"{descriptor['data_keys'].keys()}") from err
76 # convert any arrays stored directly in mds into ndarray
77 if convert_arrays:
MismatchedDataKeys: The documents are not valid. Either because they were recorded incorrectly in the first place, corrupted since, or exercising a yet-undiscovered bug in a reader. event['data'].keys() must equal descriptor['data_keys'].keys(). event['data'].keys(): dict_keys(['pe1c_image', 'pe1c_stats1_total', 'start_Grid_X', 'stop_Grid_X', 'Grid_Y', 'Grid_Y_user_setpoint', 'fs']), descriptor['data_keys'].keys(): dict_keys(['start_Grid_X', 'stop_Grid_X', 'pe1c_image', 'pe1c_stats1_total', 'Grid_Y', 'Grid_Y_user_setpoint'])
git blame say this was last touched by @mrakitin in May 2019.
I suspect that we need to make a similar change to XPD's profile.
This is leads to problems like in run
'929ec727-3b1a-48d4-a05f-978431732a19'
where because the keyfs
is not in the descriptor things barf withgit blame say this was last touched by @mrakitin in May 2019.