h = db[-1] failed with lots of output

[prjemian]

Wow! Figure this out. Successful scan set ended with:

|        97 | 12:53:24.6 |   10000000 |     368170 |     122133 |                                                                                                       
|        98 | 12:53:26.2 |   10000000 |     367589 |     118986 |                                                                                                       
|        99 | 12:53:27.9 |   10000000 |     367137 |     122621 |                                                                                                       
+-----------+------------+------------+------------+------------+                                                                                                       
|   seq_num |       time |    seconds |   I0_USAXS |   PD_USAXS |                                                                                                       
+-----------+------------+------------+------------+------------+                                                                                                       
|       100 | 12:53:29.4 |   10000000 |     367253 |     122889 |                                                                                                       
+-----------+------------+------------+------------+------------+
generator uascan ['590f705e'] (scan num: 29)
Out[2]:                                                                                                                                                                 
('1bcef68c-66bd-412d-94ee-871f78df0b63',                                                                                                                                
 '267f3490-35a6-4427-ae75-6fdad06df09c',                                                                                                                                
 '9fe3fbf9-8051-447f-9af6-b6184e718084',
 '590f705e-4910-4b38-a80d-ca8b7b36a752')

In [3]: exit                                                                                                                                                            

but could not mine the db for the last scan.

exception trace and console session

``` In [4]: h = db[-1] --------------------------------------------------------------------------- KeyError Traceback (most recent call last) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __getitem__(self, key) 38 try: ---> 39 return self.__data[key] 40 except KeyError: KeyError: '882d9a67432910a8c6359bc4fcb6f22f' During handling of the above exception, another exception occurred: KeyError Traceback (most recent call last) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in get(self, **kwargs) 229 try: --> 230 datasource = self.__cache[token] 231 logger.debug( /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/lru.py in __getitem__(self, key, cache_getitem) 13 def __getitem__(self, key, cache_getitem=Cache.__getitem__): ---> 14 value = cache_getitem(self, key) 15 self.__update(key) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __getitem__(self, key) 40 except KeyError: ---> 41 return self.__missing__(key) 42 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __missing__(self, key) 67 def __missing__(self, key): ---> 68 raise KeyError(key) 69 KeyError: '882d9a67432910a8c6359bc4fcb6f22f' During handling of the above exception, another exception occurred: AttributeError Traceback (most recent call last) in ----> 1 h = db[-1] /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/v1.py in __getitem__(self, key) 323 return [self[index] 324 for index in reversed(range(key.start, key.stop or 0, key.step or 1))] --> 325 entry = self._catalog[key] 326 return Header(entry) 327 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in __getitem__(self, key) 382 if not isinstance(key, list) and key in self._get_entries(): 383 # triggers reload_on_change --> 384 e = self._entries[key] 385 e._catalog = self 386 e._pmode = self.pmode /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/_drivers/mongo_normalized.py in __getitem__(self, name) 84 # still work because BlueskyRun supports those methods and will just 85 # return itself. ---> 86 return entry.get() # an instance of BlueskyRun 87 88 def _find_run_start_doc(self, name): /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in get(self, **kwargs) 234 datasource.name) 235 except KeyError: --> 236 datasource = super().get(**kwargs) 237 self.__cache[token] = datasource 238 return datasource /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/local.py in get(self, **user_parameters) 279 """Instantiate the DataSource for the given parameters""" 280 plugin, open_args = self._create_open_args(user_parameters) --> 281 data_source = plugin(**open_args) 282 data_source.catalog_object = self._catalog 283 data_source.name = self.name /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in __init__(self, get_run_start, get_run_stop, get_event_descriptors, get_event_pages, get_event_count, get_resource, get_resources, lookup_resource_for_datum, get_datum_pages, get_filler, entry, transforms, **kwargs) 991 self._run_stop_doc = None 992 --> 993 super().__init__(**kwargs) 994 logger.debug( 995 "Created %s named %r", /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in __init__(self, name, description, metadata, auth, ttl, getenv, getshell, persist_mode, storage_options, *args) 111 self.updated = time.time() 112 self._entries = self._make_entries_container() --> 113 self.force_reload() 114 115 @classmethod /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in force_reload(self) 168 def force_reload(self): 169 """Imperative reload data now""" --> 170 self._load() 171 self.updated = time.time() 172 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in _load(self) 1112 "Loaded %s named %r", 1113 self.__class__.__name__, -> 1114 self._entry.name) 1115 1116 def get(self, *args, **kwargs): AttributeError: 'NoneType' object has no attribute 'name' In [5]: db Out[5]: In [6]: db[-1] --------------------------------------------------------------------------- KeyError Traceback (most recent call last) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __getitem__(self, key) 38 try: ---> 39 return self.__data[key] 40 except KeyError: KeyError: '882d9a67432910a8c6359bc4fcb6f22f' During handling of the above exception, another exception occurred: KeyError Traceback (most recent call last) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in get(self, **kwargs) 229 try: --> 230 datasource = self.__cache[token] 231 logger.debug( /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/lru.py in __getitem__(self, key, cache_getitem) 13 def __getitem__(self, key, cache_getitem=Cache.__getitem__): ---> 14 value = cache_getitem(self, key) 15 self.__update(key) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __getitem__(self, key) 40 except KeyError: ---> 41 return self.__missing__(key) 42 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __missing__(self, key) 67 def __missing__(self, key): ---> 68 raise KeyError(key) 69 KeyError: '882d9a67432910a8c6359bc4fcb6f22f' During handling of the above exception, another exception occurred: AttributeError Traceback (most recent call last) in ----> 1 db[-1] /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/v1.py in __getitem__(self, key) 323 return [self[index] 324 for index in reversed(range(key.start, key.stop or 0, key.step or 1))] --> 325 entry = self._catalog[key] 326 return Header(entry) 327 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in __getitem__(self, key) 382 if not isinstance(key, list) and key in self._get_entries(): 383 # triggers reload_on_change --> 384 e = self._entries[key] 385 e._catalog = self 386 e._pmode = self.pmode /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/_drivers/mongo_normalized.py in __getitem__(self, name) 84 # still work because BlueskyRun supports those methods and will just 85 # return itself. ---> 86 return entry.get() # an instance of BlueskyRun 87 88 def _find_run_start_doc(self, name): /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in get(self, **kwargs) 234 datasource.name) 235 except KeyError: --> 236 datasource = super().get(**kwargs) 237 self.__cache[token] = datasource 238 return datasource /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/local.py in get(self, **user_parameters) 279 """Instantiate the DataSource for the given parameters""" 280 plugin, open_args = self._create_open_args(user_parameters) --> 281 data_source = plugin(**open_args) 282 data_source.catalog_object = self._catalog 283 data_source.name = self.name /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in __init__(self, get_run_start, get_run_stop, get_event_descriptors, get_event_pages, get_event_count, get_resource, get_resources, lookup_resource_for_datum, get_datum_pages, get_filler, entry, transforms, **kwargs) 991 self._run_stop_doc = None 992 --> 993 super().__init__(**kwargs) 994 logger.debug( 995 "Created %s named %r", /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in __init__(self, name, description, metadata, auth, ttl, getenv, getshell, persist_mode, storage_options, *args) 111 self.updated = time.time() 112 self._entries = self._make_entries_container() --> 113 self.force_reload() 114 115 @classmethod /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in force_reload(self) 168 def force_reload(self): 169 """Imperative reload data now""" --> 170 self._load() 171 self.updated = time.time() 172 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in _load(self) 1112 "Loaded %s named %r", 1113 self.__class__.__name__, -> 1114 self._entry.name) 1115 1116 def get(self, *args, **kwargs): AttributeError: 'NoneType' object has no attribute 'name' In [7]: db("590f705e-4910-4b38-a80d-ca8b7b36a752") Out[7]: In [8]: db["590f705e-4910-4b38-a80d-ca8b7b36a752"] --------------------------------------------------------------------------- KeyError Traceback (most recent call last) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __getitem__(self, key) 38 try: ---> 39 return self.__data[key] 40 except KeyError: KeyError: '882d9a67432910a8c6359bc4fcb6f22f' During handling of the above exception, another exception occurred: KeyError Traceback (most recent call last) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in get(self, **kwargs) 229 try: --> 230 datasource = self.__cache[token] 231 logger.debug( /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/lru.py in __getitem__(self, key, cache_getitem) 13 def __getitem__(self, key, cache_getitem=Cache.__getitem__): ---> 14 value = cache_getitem(self, key) 15 self.__update(key) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __getitem__(self, key) 40 except KeyError: ---> 41 return self.__missing__(key) 42 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __missing__(self, key) 67 def __missing__(self, key): ---> 68 raise KeyError(key) 69 KeyError: '882d9a67432910a8c6359bc4fcb6f22f' During handling of the above exception, another exception occurred: AttributeError Traceback (most recent call last) in ----> 1 db["590f705e-4910-4b38-a80d-ca8b7b36a752"] /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/v1.py in __getitem__(self, key) 323 return [self[index] 324 for index in reversed(range(key.start, key.stop or 0, key.step or 1))] --> 325 entry = self._catalog[key] 326 return Header(entry) 327 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in __getitem__(self, key) 382 if not isinstance(key, list) and key in self._get_entries(): 383 # triggers reload_on_change --> 384 e = self._entries[key] 385 e._catalog = self 386 e._pmode = self.pmode /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/_drivers/mongo_normalized.py in __getitem__(self, name) 84 # still work because BlueskyRun supports those methods and will just 85 # return itself. ---> 86 return entry.get() # an instance of BlueskyRun 87 88 def _find_run_start_doc(self, name): /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in get(self, **kwargs) 234 datasource.name) 235 except KeyError: --> 236 datasource = super().get(**kwargs) 237 self.__cache[token] = datasource 238 return datasource /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/local.py in get(self, **user_parameters) 279 """Instantiate the DataSource for the given parameters""" 280 plugin, open_args = self._create_open_args(user_parameters) --> 281 data_source = plugin(**open_args) 282 data_source.catalog_object = self._catalog 283 data_source.name = self.name /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in __init__(self, get_run_start, get_run_stop, get_event_descriptors, get_event_pages, get_event_count, get_resource, get_resources, lookup_resource_for_datum, get_datum_pages, get_filler, entry, transforms, **kwargs) 991 self._run_stop_doc = None 992 --> 993 super().__init__(**kwargs) 994 logger.debug( 995 "Created %s named %r", /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in __init__(self, name, description, metadata, auth, ttl, getenv, getshell, persist_mode, storage_options, *args) 111 self.updated = time.time() 112 self._entries = self._make_entries_container() --> 113 self.force_reload() 114 115 @classmethod /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in force_reload(self) 168 def force_reload(self): 169 """Imperative reload data now""" --> 170 self._load() 171 self.updated = time.time() 172 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in _load(self) 1112 "Loaded %s named %r", 1113 self.__class__.__name__, -> 1114 self._entry.name) 1115 1116 def get(self, *args, **kwargs): AttributeError: 'NoneType' object has no attribute 'name' In [9]: for h in db("590f705e-4910-4b38-a80d-ca8b7b36a752"): ...: print(h) ...: --------------------------------------------------------------------------- KeyError Traceback (most recent call last) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __getitem__(self, key) 38 try: ---> 39 return self.__data[key] 40 except KeyError: KeyError: '882d9a67432910a8c6359bc4fcb6f22f' During handling of the above exception, another exception occurred: KeyError Traceback (most recent call last) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in get(self, **kwargs) 229 try: --> 230 datasource = self.__cache[token] 231 logger.debug( /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/lru.py in __getitem__(self, key, cache_getitem) 13 def __getitem__(self, key, cache_getitem=Cache.__getitem__): ---> 14 value = cache_getitem(self, key) 15 self.__update(key) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __getitem__(self, key) 40 except KeyError: ---> 41 return self.__missing__(key) 42 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/cachetools/cache.py in __missing__(self, key) 67 def __missing__(self, key): ---> 68 raise KeyError(key) 69 KeyError: '882d9a67432910a8c6359bc4fcb6f22f' During handling of the above exception, another exception occurred: AttributeError Traceback (most recent call last) in ----> 1 for h in db("590f705e-4910-4b38-a80d-ca8b7b36a752"): 2 print(h) 3 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/v1.py in __iter__(self) 1403 def __iter__(self): 1404 # TODO Catalog.walk() fails. We should probably support Catalog.items(). -> 1405 for uid, entry in self._catalog._entries.items(): 1406 header = Header(entry) 1407 if self._data_key is None: /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/_collections_abc.py in __iter__(self) 742 def __iter__(self): 743 for key in self._mapping: --> 744 yield (key, self._mapping[key]) 745 746 ItemsView.register(dict_items) /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/_drivers/mongo_normalized.py in __getitem__(self, name) 84 # still work because BlueskyRun supports those methods and will just 85 # return itself. ---> 86 return entry.get() # an instance of BlueskyRun 87 88 def _find_run_start_doc(self, name): /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in get(self, **kwargs) 234 datasource.name) 235 except KeyError: --> 236 datasource = super().get(**kwargs) 237 self.__cache[token] = datasource 238 return datasource /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/local.py in get(self, **user_parameters) 279 """Instantiate the DataSource for the given parameters""" 280 plugin, open_args = self._create_open_args(user_parameters) --> 281 data_source = plugin(**open_args) 282 data_source.catalog_object = self._catalog 283 data_source.name = self.name /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in __init__(self, get_run_start, get_run_stop, get_event_descriptors, get_event_pages, get_event_count, get_resource, get_resources, lookup_resource_for_datum, get_datum_pages, get_filler, entry, transforms, **kwargs) 991 self._run_stop_doc = None 992 --> 993 super().__init__(**kwargs) 994 logger.debug( 995 "Created %s named %r", /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in __init__(self, name, description, metadata, auth, ttl, getenv, getshell, persist_mode, storage_options, *args) 111 self.updated = time.time() 112 self._entries = self._make_entries_container() --> 113 self.force_reload() 114 115 @classmethod /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/intake/catalog/base.py in force_reload(self) 168 def force_reload(self): 169 """Imperative reload data now""" --> 170 self._load() 171 self.updated = time.time() 172 /share1/bluesky/conda_envs/bluesky_2020_5/lib/python3.7/site-packages/databroker/core.py in _load(self) 1112 "Loaded %s named %r", 1113 self.__class__.__name__, -> 1114 self._entry.name) 1115 1116 def get(self, *args, **kwargs): AttributeError: 'NoneType' object has no attribute 'name' In [10]: ```

[prjemian]

Looking at the databroker releases page, we could try an update.

[prjemian]

Testing databroker 1.0.5 in a custom environment, this seems to resolve the problem.

(/share1/bluesky/conda_envs/broker) usaxs@usaxscontrol /share1/USAXS_data $ ll ~/.config/databroker/
total 0
lrwxrwxrwx 1 usaxs 57 Jan  9  2018 mongodb_config.yml -> ../../.ipython/profile_bluesky/startup/mongodb_config.yml
(/share1/bluesky/conda_envs/broker) usaxs@usaxscontrol /share1/USAXS_data $ $CONDA_PREFIX/bin/python
Python 3.8.3 (default, May 19 2020, 18:47:26) 
[GCC 7.3.0] :: Anaconda, Inc. on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import databroker
>>> db = databroker.Broker.named("/home/beams/USAXS/.config/databroker/mongodb_config.yml")
>>> db[-1]
<databroker.v1.Header object at 0x7f69981b97f0>
>>> h = _

``` >>> h.start {'uid': '68034a36-3466-4eac-ba95-c582a7ea6586', 'time': 1591818038.343237, 'versions': {'bluesky': '1.6.0', 'ophyd': '1.5.0', 'databroker': '1.0.2', 'apstools': '1.2.4rc3', 'epics': '3.4.2', 'numpy': '1.18.5', 'matplotlib': '3.2.1', 'spec2nexus': '2021.1.7', 'pyRestTable': '2020.0.2', 'pymongo': '3.9.0'}, 'scan_id': 12, 'EPICS_CA_MAX_ARRAY_BYTES': '1280000', 'login_id': 'usaxs@usaxscontrol.xray.aps.anl.gov', 'pid': 46092, 'beamline_id': 'APS 9-ID-C USAXS', 'proposal_id': 'testing', 'EPICS_HOST_ARCH': 'linux-x86_64', 'plan_type': 'generator', 'plan_name': 'SAXS', 'detectors': ['saxs_det'], 'num_points': 1, 'num_intervals': 0, 'plan_args': {'detectors': ["MyPilatusDetector(prefix='usaxs_pilatus1:', name='saxs_det', read_attrs=['hdf1'], configuration_attrs=['cam', 'cam.acquire_period', 'cam.acquire_time', 'cam.image_mode', 'cam.manufacturer', 'cam.model', 'cam.num_exposures', 'cam.trigger_mode', 'hdf1'])"], 'num': 1}, 'hints': {'dimensions': [[['time'], 'primary']]}, 'full_filename': '/share1/USAXS_data/2020-06/usaxs.mac', 'filename': 'usaxs.mac', 'line_number': 21, 'action': 'saxsExp', 'parameters': ['20.000', '22.000', '0', 'AirBlank'], 'archive': {'text': 'Command file: usaxs.mac\n====== ======================= =================================\nline # action parameters \n====== ======================= =================================\n2 CURRENT_EXPERIMENT_NAME NIST \n21 saxsExp 20.000, 22.000, 0, AirBlank \n23 saxsExp 19.000, 42.000, 0.093, Fer_TiFree\n25 saxsExp 20.000, 80.000, 0.098, Fer_028pTi\n27 saxsExp 20.000, 61.000, 0.058, Fer_1pTi \n29 saxsExp 20.000, 120.000, 0, TapeBlank \n31 saxsExp 20.000, 100.000, 1, SteelPowder \n35 waxsExp 20.000, 22.000, 0, AirBlank \n37 waxsExp 19.000, 42.000, 0.093, Fer_TiFree\n39 waxsExp 20.000, 80.000, 0.098, Fer_028pTi\n41 waxsExp 20.000, 61.000, 0.058, Fer_1pTi \n43 waxsExp 20.000, 120.000, 0, TapeBlank \n45 waxsExp 20.000, 100.000, 1, SteelPowder \n49 preUSAXStune \n51 USAXSscan 20.000, 22.000, 0, AirBlank \n53 USAXSscan 19.000, 42.000, 0.093, Fer_TiFree\n55 USAXSscan 20.000, 80.000, 0.098, Fer_028pTi\n57 USAXSscan 20.000, 61.000, 0.058, Fer_1pTi \n59 USAXSscan 20.000, 120.000, 0, TapeBlank \n61 USAXSscan 20.000, 100.000, 1, SteelPowder \n====== ======================= =================================\n', 'source': '/home/beams/USAXS/.ipython/profile_bluesky/startup/instrument/plans/command_list.py', 'is_file': True, 'line': 395, 'caller': 'execute_command_list', 'caller_code': ' logger.info(text)\n archive = instrument_archive(text)\n', 'source_contents': ['\n', '"""\n', 'run batch of scans from command list\n', '"""\n', '\n', '__all__ = """\n', ' beforeScanComputeOtherStuff\n', ' postCommandsListfile2WWW\n', ' before_command_list\n', ' after_command_list\n', ' before_plan\n', ' after_plan\n', ' parse_Excel_command_file\n', ' parse_text_command_file\n', ' command_list_as_table\n', ' get_command_list\n', ' summarize_command_file\n', ' run_command_file\n', ' execute_command_list\n', '""".split()\n', '\n', '\n', 'from ..session_logs import logger\n', 'logger.info(__file__)\n', '\n', 'from apstools.utils import ExcelDatabaseFileGeneric\n', 'from bluesky import plan_stubs as bps\n', 'from usaxs_support.nexus import reset_manager\n', 'from usaxs_support.surveillance import instrument_archive\n', 'import datetime\n', 'import os\n', 'import pyRestTable\n', '\n', 'from ..devices import constants\n', 'from ..devices import measure_background\n', 'from ..devices import saxs_det, waxs_det\n', 'from ..devices import terms\n', 'from ..devices import ti_filter_shutter\n', 'from ..devices import upd_controls, I0_controls, I00_controls, trd_controls\n', 'from ..devices import user_data\n', 'from ..utils.quoted_line import split_quoted_line\n', 'from .axis_tuning import instrument_default_tune_ranges\n', 'from .axis_tuning import update_EPICS_tuning_widths\n', 'from .axis_tuning import user_defined_settings\n', 'from .mode_changes import mode_SAXS, mode_USAXS, mode_WAXS, mode_Radiography\n', '\n', '\n', 'def beforeScanComputeOtherStuff():\n', ' """\n', ' things to be computed and set before each data collection starts\n', ' """\n', ' yield from bps.null() # TODO: remove this once you add the "other stuff"\n', '\n', '\n', 'def postCommandsListfile2WWW(commands):\n', ' """\n', ' post list of commands to WWW and archive the list for posterity\n', ' """\n', ' tbl_file = "commands.txt"\n', ' tbl = command_list_as_table(commands)\n', ' timestamp = datetime.datetime.now().isoformat().replace("T", " ")\n', ' file_contents = "bluesky command sequence\\n"\n', ' file_contents += f"written: {timestamp}\\n"\n', ' file_contents += str(tbl.reST())\n', '\n', ' # post for livedata page\n', ' # path = "/tmp"\n', ' path = "/share1/local_livedata"\n', ' with open(os.path.join(path, tbl_file), "w") as fp:\n', ' fp.write(file_contents)\n', '\n', ' # post to EPICS\n', ' yield from bps.mv(\n', ' user_data.macro_file, os.path.split(tbl_file)[-1],\n', ' user_data.macro_file_time, timestamp,\n', ' )\n', '\n', ' # keep this list for posterity\n', ' timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")\n', ' path = "/share1/log/macros"\n', ' posterity_file = f"{timestamp}-{tbl_file}"\n', ' with open(os.path.join(path, posterity_file), "w") as fp:\n', ' fp.write(file_contents)\n', '\n', '\n', '\n', 'def before_command_list(md={}, commands=None):\n', ' """\n', ' things to be done before a command list is run\n', ' """\n', ' from .scans import preUSAXStune\n', '\n', ' yield from bps.mv(\n', ' user_data.time_stamp, str(datetime.datetime.now()),\n', ' user_data.state, "Starting data collection",\n', ' user_data.collection_in_progress, 1,\n', ' ti_filter_shutter, "close",\n', ' terms.SAXS.collecting, 0,\n', ' terms.WAXS.collecting, 0,\n', ' )\n', ' if constants["MEASURE_DARK_CURRENTS"]:\n', ' yield from measure_background(\n', ' [upd_controls, I0_controls, I00_controls, trd_controls],\n', ' )\n', '\n', ' # reset the ranges to be used when tuning optical axes (issue #129)\n', ' # These routines are defined in file: 29-axis-tuning.py\n', ' yield from instrument_default_tune_ranges()\n', ' yield from user_defined_settings()\n', ' yield from update_EPICS_tuning_widths()\n', '\n', ' yield from beforeScanComputeOtherStuff() # 41-commands.py\n', '\n', ' if terms.preUSAXStune.run_tune_on_qdo.get():\n', ' logger.info("Runing preUSAXStune as requested at start of measurements")\n', ' yield from preUSAXStune(md=md)\n', '\n', ' if constants["SYNC_ORDER_NUMBERS"]:\n', ' order_number = max([\n', ' terms.FlyScan.order_number.get(),\n', ' saxs_det.hdf1.file_number.get(),\n', ' waxs_det.hdf1.file_number.get(),\n', ' ])\n', '\n', ' try:\n', ' yield from bps.mv(saxs_det.hdf1.file_number, order_number)\n', ' except NameError:\n', ' pass\n', ' try:\n', ' yield from bps.mv(waxs_det.hdf1.file_number, order_number)\n', ' except NameError:\n', ' pass\n', ' yield from bps.mv(terms.FlyScan.order_number, order_number)\n', '\n', ' if commands is not None:\n', ' postCommandsListfile2WWW(commands)\n', '\n', ' # force the next FlyScan to reload the metadata configuration\n', ' # which forces a (re)connection to the EPICS PVs\n', ' reset_manager()\n', '\n', '\n', 'def after_command_list(md={}):\n', ' """\n', ' things to be done after a command list is run\n', ' """\n', ' yield from bps.mv(\n', ' user_data.time_stamp, str(datetime.datetime.now()),\n', ' user_data.state, "USAXS macro file done", # exact text triggers the music\n', ' user_data.collection_in_progress, 0,\n', ' ti_filter_shutter, "close",\n', ' )\n', '\n', '\n', 'def before_plan(md={}):\n', ' """\n', ' things to be done before every data collection plan\n', ' """\n', ' from .scans import preSWAXStune, preUSAXStune\n', '\n', ' if terms.preUSAXStune.needed:\n', ' # tune at previous sample position\n', " # don't overexpose the new sample position\n", ' # select between positions \n', ' mode_now = terms.SAXS.UsaxsSaxsMode.get(as_string=True)\n', ' if mode_now == "USAXS in beam" : \n', ' yield from preUSAXStune(md=md)\n', ' else:\n', ' yield from preSWAXStune(md=md)\n', '\n', '\n', 'def after_plan(weight=1, md={}):\n', ' """\n', ' things to be done after every data collection plan\n', ' """\n', ' from .scans import preUSAXStune\n', '\n', ' yield from bps.mv( # increment it\n', ' terms.preUSAXStune.num_scans_last_tune,\n', ' terms.preUSAXStune.num_scans_last_tune.get() + weight\n', ' )\n', '\n', '\n', 'def parse_Excel_command_file(filename):\n', ' """\n', ' parse an Excel spreadsheet with commands, return as command list\n', '\n', ' TEXT view of spreadsheet (Excel file line numbers shown)::\n', '\n', ' [1] List of sample scans to be run\n', ' [2]\n', ' [3]\n', ' [4] scan sx sy thickness sample name\n', ' [5] FlyScan 0 0 0 blank\n', ' [6] FlyScan 5 2 0 blank\n', '\n', ' PARAMETERS\n', '\n', ' filename : str\n', ' Name of input Excel spreadsheet file. Can be relative or absolute path,\n', ' such as "actions.xslx", "../sample.xslx", or\n', ' "/path/to/overnight.xslx".\n', '\n', ' RETURNS\n', '\n', ' list of commands : list[command]\n', ' List of command tuples for use in ``execute_command_list()``\n', '\n', ' RAISES\n', '\n', ' FileNotFoundError\n', ' if file cannot be found\n', '\n', ' """\n', ' full_filename = os.path.abspath(filename)\n', ' assert os.path.exists(full_filename)\n', ' xl = ExcelDatabaseFileGeneric(full_filename)\n', '\n', ' commands = []\n', '\n', ' if len(xl.db) > 0:\n', ' for i, row in enumerate(xl.db.values()):\n', ' action, *values = list(row.values())\n', '\n', ' # trim off any None values from end\n', ' while len(values) > 0:\n', ' if values[-1] is not None:\n', ' break\n', ' values = values[:-1]\n', '\n', ' commands.append((action, values, i+1, list(row.values())))\n', '\n', ' return commands\n', '\n', '\n', 'def parse_text_command_file(filename):\n', ' """\n', ' parse a text file with commands, return as command list\n', '\n', ' * The text file is interpreted line-by-line.\n', ' * Blank lines are ignored.\n', ' * A pound sign (#) marks the rest of that line as a comment.\n', ' * All remaining lines are interpreted as commands with arguments.\n', '\n', ' Example of text file (no line numbers shown)::\n', '\n', ' #List of sample scans to be run\n', ' # pound sign starts a comment (through end of line)\n', '\n', ' # action value\n', ' mono_shutter open\n', '\n', ' # action x y width height\n', ' uslits 0 0 0.4 1.2\n', '\n', ' # action sx sy thickness sample name\n', ' FlyScan 0 0 0 blank\n', ' FlyScan 5 2 0 "empty container"\n', '\n', ' # action sx sy thickness sample name\n', ' SAXS 0 0 0 blank\n', '\n', ' # action value\n', ' mono_shutter close\n', '\n', ' PARAMETERS\n', '\n', ' filename : str\n', ' Name of input text file. Can be relative or absolute path,\n', ' such as "actions.txt", "../sample.txt", or\n', ' "/path/to/overnight.txt".\n', '\n', ' RETURNS\n', '\n', ' list of commands : list[command]\n', ' List of command tuples for use in ``execute_command_list()``\n', '\n', ' RAISES\n', '\n', ' FileNotFoundError\n', ' if file cannot be found\n', ' """\n', ' full_filename = os.path.abspath(filename)\n', ' assert os.path.exists(full_filename)\n', ' with open(full_filename, "r") as fp:\n', ' buf = fp.readlines()\n', '\n', ' commands = []\n', ' for i, raw_command in enumerate(buf):\n', ' row = raw_command.strip()\n', ' if row == "" or row.startswith("#"):\n', ' continue # comment or blank\n', '\n', ' else: # command line\n', ' action, *values = split_quoted_line(row)\n', ' commands.append((action, values, i+1, raw_command.rstrip()))\n', '\n', ' return commands\n', '\n', '\n', 'def command_list_as_table(commands):\n', ' """\n', ' format a command list as a pyRestTable.Table object\n', ' """\n', ' tbl = pyRestTable.Table()\n', ' tbl.addLabel("line #")\n', ' tbl.addLabel("action")\n', ' tbl.addLabel("parameters")\n', ' for command in commands:\n', ' action, args, line_number = command[:3]\n', ' row = [line_number, action, ", ".join(map(str, args))]\n', ' tbl.addRow(row)\n', ' return tbl\n', '\n', '\n', 'def get_command_list(filename):\n', ' """\n', ' return command list from either text or Excel file\n', ' """\n', ' full_filename = os.path.abspath(filename)\n', ' assert os.path.exists(full_filename)\n', ' try:\n', ' commands = parse_Excel_command_file(filename)\n', ' except Exception: # TODO: XLRDError\n', ' commands = parse_text_command_file(filename)\n', ' return commands\n', '\n', '\n', 'def summarize_command_file(filename):\n', ' """\n', ' print the command list from a text or Excel file\n', ' """\n', ' commands = get_command_list(filename)\n', ' logger.info((\n', ' f"Command file: {filename}\\n"\n', ' f"{command_list_as_table(commands)}"\n', ' ))\n', '\n', '\n', 'def run_command_file(filename, md={}):\n', ' """\n', ' plan: execute a list of commands from a text or Excel file\n', '\n', ' * Parse the file into a command list\n', ' * yield the command list to the RunEngine (or other)\n', ' """\n', ' commands = get_command_list(filename)\n', ' yield from execute_command_list(filename, commands)\n', '\n', '\n', 'def execute_command_list(filename, commands, md={}):\n', ' """\n', ' plan: execute the command list\n', '\n', ' The command list is a tuple described below.\n', '\n', ' * Only recognized commands will be executed.\n', ' * Unrecognized commands will be reported as comments.\n', '\n', ' PARAMETERS\n', '\n', ' filename : str\n', ' Name of input text file. Can be relative or absolute path,\n', ' such as "actions.txt", "../sample.txt", or\n', ' "/path/to/overnight.txt".\n', ' commands : list[command]\n', ' List of command tuples for use in ``execute_command_list()``\n', '\n', ' where\n', '\n', ' command : tuple\n', ' (action, OrderedDict, line_number, raw_command)\n', ' action: str\n', ' names a known action to be handled\n', ' parameters: list\n', ' List of parameters for the action.\n', ' The list is empty of there are no values\n', ' line_number: int\n', ' line number (1-based) from the input text file\n', ' raw_command: obj (str or list(str)\n', ' contents from input file, such as:\n', ' ``SAXS 0 0 0 blank``\n', ' """\n', ' from .scans import preUSAXStune, SAXS, USAXSscan, WAXS\n', '\n', ' full_filename = os.path.abspath(filename)\n', '\n', ' if len(commands) == 0:\n', ' yield from bps.null()\n', ' return\n', '\n', ' text = f"Command file: {filename}\\n"\n', ' text += str(command_list_as_table(commands))\n', ' logger.info(text)\n', ' archive = instrument_archive(text)\n', '\n', ' yield from before_command_list(md=md, commands=commands)\n', ' for command in commands:\n', ' action, args, i, raw_command = command\n', ' logger.info(f"file line {i}: {raw_command}")\n', '\n', ' _md = {}\n', ' _md["full_filename"] = full_filename\n', ' _md["filename"] = filename\n', ' _md["line_number"] = i\n', ' _md["action"] = action\n', ' _md["parameters"] = args # args is shorter than parameters, means the same thing here\n', ' _md["archive"] = archive\n', ' _md["iso8601"] = datetime.datetime.now().isoformat(" ")\n', '\n', ' _md.update(md or {}) # overlay with user-supplied metadata\n', '\n', ' action = action.lower()\n', ' if action == "preusaxstune":\n', ' yield from preUSAXStune(md=_md)\n', '\n', ' elif action in ("flyscan", "usaxsscan"):\n', ' sx = float(args[0])\n', ' sy = float(args[1])\n', ' sth = float(args[2])\n', ' snm = args[3]\n', ' _md.update(dict(sx=sx, sy=sy, thickness=sth, title=snm))\n', ' yield from USAXSscan(sx, sy, sth, snm, md=_md) # either step or fly scan\n', '\n', ' elif action in ("saxs", "saxsexp"):\n', ' sx = float(args[0])\n', ' sy = float(args[1])\n', ' sth = float(args[2])\n', ' snm = args[3]\n', ' _md.update(dict(sx=sx, sy=sy, thickness=sth, title=snm))\n', ' yield from SAXS(sx, sy, sth, snm, md=_md)\n', '\n', ' elif action in ("waxs", "waxsexp"):\n', ' sx = float(args[0])\n', ' sy = float(args[1])\n', ' sth = float(args[2])\n', ' snm = args[3]\n', ' _md.update(dict(sx=sx, sy=sy, thickness=sth, title=snm))\n', ' yield from WAXS(sx, sy, sth, snm, md=_md)\n', '\n', ' elif action in ("mode_Radiography"):\n', ' yield from mode_Radiography()\n', '\n', ' elif action in ("mode_saxs"):\n', ' yield from mode_SAXS()\n', '\n', ' elif action in ("mode_usaxs"):\n', ' yield from mode_USAXS()\n', '\n', ' elif action in ("mode_waxs"):\n', ' yield from mode_WAXS()\n', '\n', ' else:\n', ' logger.info(f"no handling for line {i}: {raw_command}")\n', '\n', ' yield from after_command_list(md=md)\n'], 'archive_file': '/share1/log/macros/20200610-143850-usaxs.txt', 'archive_timestamp': '20200610-143850'}, 'iso8601': '2020-06-10 14:40:06.578848', 'purpose': 'tuner', 'datetime': '2020-06-10 14:39:58.837976', 'sx': 20.0, 'sy': 22.0, 'thickness': 0.0, 'title': 'AirBlank', 'hdf5_file': 'AirBlank_0691.hdf', 'hdf5_path': '/share1/USAXS_data/2020-06/06_10_NIST_Static_saxs', 'method': 'areaDetectorAcquire', 'area_detector_name': 'saxs_det', '_name': 'Run Start'} >>> ```

[prjemian]

@jilavsky : An upgrade of databroker should fix this. Let me know when you are ready.

[prjemian]

We can do this upgrade this evening when SPEC is running overnight.

[prjemian]

Done.