yugangzhang
commented
6 years ago More info
In [1]: det=[xray_eye3]
In [2]: RE(count(det))
---------------------------------------------------------------------------
ServerSelectionTimeoutError Traceback (most recent call last)
<ipython-input-2-956d8f265817> in <module>()
----> 1 RE(count(det))
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/run_engine.py in __call__(self, plan, subs, raise_if_interrupted, **metadata_kw)
644 # it (unless it is a canceled error)
645 if exc is not None:
--> 646 raise exc
647
648 if raise_if_interrupted and self._interrupted:
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/run_engine.py in _run(self)
1084 self.log.error("Run aborted")
1085 self.log.error("%r", err)
-> 1086 raise err
1087 finally:
1088 # Some done_callbacks may still be alive in other threads.
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/run_engine.py in _run(self)
982 resp = self._response_stack.pop()
983 try:
--> 984 msg = self._plan_stack[-1].send(resp)
985 # We have exhausted the top generator
986 except StopIteration:
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in __call__(self, plan)
3325 plan = monitor_during_wrapper(plan, self.monitors)
3326 plan = fly_during_wrapper(plan, self.flyers)
-> 3327 return (yield from plan)
3328
3329
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in fly_during_wrapper(plan, flyers)
1482 plan1 = plan_mutator(plan, insert_after_open)
1483 plan2 = plan_mutator(plan1, insert_before_close)
-> 1484 return (yield from plan2)
1485
1486
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
162 continue
163 else:
--> 164 raise ex
165 # if inserting / mutating, put new generator on the stack
166 # and replace the current msg with the first element from the
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
115 ret = result_stack.pop()
116 try:
--> 117 msg = plan_stack[-1].send(ret)
118 except StopIteration as e:
119 # discard the exhausted generator
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
162 continue
163 else:
--> 164 raise ex
165 # if inserting / mutating, put new generator on the stack
166 # and replace the current msg with the first element from the
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
115 ret = result_stack.pop()
116 try:
--> 117 msg = plan_stack[-1].send(ret)
118 except StopIteration as e:
119 # discard the exhausted generator
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in monitor_during_wrapper(plan, signals)
1424 plan1 = plan_mutator(plan, insert_after_open)
1425 plan2 = plan_mutator(plan1, insert_before_close)
-> 1426 return (yield from plan2)
1427
1428
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
162 continue
163 else:
--> 164 raise ex
165 # if inserting / mutating, put new generator on the stack
166 # and replace the current msg with the first element from the
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
115 ret = result_stack.pop()
116 try:
--> 117 msg = plan_stack[-1].send(ret)
118 except StopIteration as e:
119 # discard the exhausted generator
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
162 continue
163 else:
--> 164 raise ex
165 # if inserting / mutating, put new generator on the stack
166 # and replace the current msg with the first element from the
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
115 ret = result_stack.pop()
116 try:
--> 117 msg = plan_stack[-1].send(ret)
118 except StopIteration as e:
119 # discard the exhausted generator
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in baseline_wrapper(plan, devices, name)
1763 return (yield from plan)
1764 else:
-> 1765 return (yield from plan_mutator(plan, insert_baseline))
1766
1767
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
162 continue
163 else:
--> 164 raise ex
165 # if inserting / mutating, put new generator on the stack
166 # and replace the current msg with the first element from the
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
115 ret = result_stack.pop()
116 try:
--> 117 msg = plan_stack[-1].send(ret)
118 except StopIteration as e:
119 # discard the exhausted generator
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in count(detectors, num, delay, md)
2081 return (yield from infinite_plan())
2082 else:
-> 2083 return (yield from finite_plan())
2084
2085
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/utils.py in dec_inner(*inner_args, **inner_kwargs)
920 plan = gen_func(*inner_args, **inner_kwargs)
921 plan = wrapper(plan, *args, **kwargs)
--> 922 return (yield from plan)
923 return dec_inner
924 return dec
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in stage_wrapper(plan, devices)
1609 return (yield from plan)
1610
-> 1611 return (yield from finalize_wrapper(inner(), unstage_devices()))
1612
1613
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in finalize_wrapper(plan, final_plan, pause_for_debug)
1110 cleanup = True
1111 try:
-> 1112 ret = yield from plan
1113 except GeneratorExit:
1114 cleanup = False
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in inner()
1607 def inner():
1608 yield from stage_devices()
-> 1609 return (yield from plan)
1610
1611 return (yield from finalize_wrapper(inner(), unstage_devices()))
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/utils.py in dec_inner(*inner_args, **inner_kwargs)
920 plan = gen_func(*inner_args, **inner_kwargs)
921 plan = wrapper(plan, *args, **kwargs)
--> 922 return (yield from plan)
923 return dec_inner
924 return dec
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in run_wrapper(plan, md)
910 metadata to be passed into the 'open_run' message
911 """
--> 912 rs_uid = yield from open_run(md)
913 yield from finalize_wrapper(plan,
914 close_run())
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in open_run(md)
1036 :func:`bluesky.plans.close_run`
1037 """
-> 1038 return (yield Msg('open_run', **(md or {})))
1039
1040
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
99 # if we have a stashed exception, pass it along
100 try:
--> 101 msg = plan_stack[-1].throw(exception)
102 except Exception as e:
103 # if we catch an exception,
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in pre_baseline()
1745 if msg.command == 'open_run':
1746 def pre_baseline():
-> 1747 ret = yield msg
1748 yield from trigger_and_read(devices, name=name)
1749 return ret
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
187 try:
188 # yield out the 'current message' and collect the return
--> 189 inner_ret = yield msg
190 except GeneratorExit:
191 # special case GeneratorExit. We must clean up all of our plans
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
99 # if we have a stashed exception, pass it along
100 try:
--> 101 msg = plan_stack[-1].throw(exception)
102 except Exception as e:
103 # if we catch an exception,
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/utils.py in single_gen(msg)
127 the input message
128 '''
--> 129 return (yield msg)
130
131
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
187 try:
188 # yield out the 'current message' and collect the return
--> 189 inner_ret = yield msg
190 except GeneratorExit:
191 # special case GeneratorExit. We must clean up all of our plans
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
187 try:
188 # yield out the 'current message' and collect the return
--> 189 inner_ret = yield msg
190 except GeneratorExit:
191 # special case GeneratorExit. We must clean up all of our plans
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
99 # if we have a stashed exception, pass it along
100 try:
--> 101 msg = plan_stack[-1].throw(exception)
102 except Exception as e:
103 # if we catch an exception,
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/utils.py in single_gen(msg)
127 the input message
128 '''
--> 129 return (yield msg)
130
131
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
187 try:
188 # yield out the 'current message' and collect the return
--> 189 inner_ret = yield msg
190 except GeneratorExit:
191 # special case GeneratorExit. We must clean up all of our plans
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/plans.py in plan_mutator(plan, msg_proc)
187 try:
188 # yield out the 'current message' and collect the return
--> 189 inner_ret = yield msg
190 except GeneratorExit:
191 # special case GeneratorExit. We must clean up all of our plans
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/run_engine.py in _run(self)
1033 # exceptions (coming in via throw) can be
1034 # raised
-> 1035 response = yield from coro(msg)
1036 # special case `CancelledError` and let the outer
1037 # exception block deal with it.
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/run_engine.py in _open_run(self, msg)
1233
1234 doc = dict(uid=self._run_start_uid, time=ttime.time(), **md)
-> 1235 yield from self.emit(DocumentNames.start, doc)
1236 self.log.debug("Emitted RunStart (uid=%r)", doc['uid'])
1237 yield from self._reset_checkpoint_state_coro()
~/conda_envs/dev/lib/python3.6/asyncio/coroutines.py in coro(*args, **kw)
208 @functools.wraps(func)
209 def coro(*args, **kw):
--> 210 res = func(*args, **kw)
211 if (base_futures.isfuture(res) or inspect.isgenerator(res) or
212 isinstance(res, CoroWrapper)):
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/run_engine.py in emit(self, name, doc)
2069 "Process blocking callbacks and schedule non-blocking callbacks."
2070 jsonschema.validate(doc, schemas[name])
-> 2071 self.dispatcher.process(name, doc)
2072
2073
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/run_engine.py in process(self, name, doc)
2089 doc : dict
2090 """
-> 2091 exceptions = self.cb_registry.process(name, name.name, doc)
2092 for exc, traceback in exceptions:
2093 warn("A %r was raised during the processing of a %s "
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/utils.py in process(self, sig, *args, **kwargs)
276 for cid, func in list(self.callbacks[sig].items()):
277 try:
--> 278 func(*args, **kwargs)
279 except ReferenceError:
280 self._remove_proxy(func)
~/conda_envs/dev/lib/python3.6/site-packages/bluesky/utils.py in __call__(self, *args, **kwargs)
357 mtd = self.func
358 # invoke the callable and return the result
--> 359 return mtd(*args, **kwargs)
360
361 def __eq__(self, other):
~/conda_envs/dev/lib/python3.6/site-packages/databroker/_core.py in insert(self, name, doc)
1085 """
1086 if name in {'start', 'stop'}:
-> 1087 return self.hs.insert(name, doc)
1088 else:
1089 return self.event_source_for_insert.insert(name, doc)
~/conda_envs/dev/lib/python3.6/site-packages/databroker/headersource/shim.py in insert(self, name, doc)
39
40 def insert(self, name, doc):
---> 41 return self.mds.insert(name, doc)
42
43 def find_last(self, num):
~/conda_envs/dev/lib/python3.6/site-packages/databroker/headersource/base.py in insert(self, name, doc)
528 def insert(self, name, doc):
529 if name != 'bulk_events':
--> 530 getattr(self, self._INS_METHODS[name])(**doc)
531 else:
532 for desc_uid, events in doc.items():
~/conda_envs/dev/lib/python3.6/site-packages/databroker/headersource/base.py in insert_run_start(self, time, uid, **kwargs)
405 the full document.
406 '''
--> 407 return self._api.insert_run_start(self._runstart_col,
408 self._RUNSTART_CACHE,
409 time=time,
~/conda_envs/dev/lib/python3.6/site-packages/databroker/headersource/mongo.py in _runstart_col(self)
76
77 self.__runstart_col.create_index([('uid', pymongo.DESCENDING)],
---> 78 unique=True)
79 self.__runstart_col.create_index([('time', pymongo.DESCENDING),
80 ('scan_id', pymongo.DESCENDING)],
~/conda_envs/dev/lib/python3.6/site-packages/pymongo/collection.py in create_index(self, keys, **kwargs)
1527 keys = helpers._index_list(keys)
1528 name = kwargs.setdefault("name", helpers._gen_index_name(keys))
-> 1529 self.__create_index(keys, kwargs)
1530 return name
1531
~/conda_envs/dev/lib/python3.6/site-packages/pymongo/collection.py in __create_index(self, keys, index_options)
1415 index.update(index_options)
1416
-> 1417 with self._socket_for_writes() as sock_info:
1418 if collation is not None:
1419 if sock_info.max_wire_version < 5:
~/conda_envs/dev/lib/python3.6/contextlib.py in __enter__(self)
79 def __enter__(self):
80 try:
---> 81 return next(self.gen)
82 except StopIteration:
83 raise RuntimeError("generator didn't yield") from None
~/conda_envs/dev/lib/python3.6/site-packages/pymongo/mongo_client.py in _get_socket(self, selector)
821 @contextlib.contextmanager
822 def _get_socket(self, selector):
--> 823 server = self._get_topology().select_server(selector)
824 try:
825 with server.get_socket(self.__all_credentials) as sock_info:
~/conda_envs/dev/lib/python3.6/site-packages/pymongo/topology.py in select_server(self, selector, server_selection_timeout, address)
212 return random.choice(self.select_servers(selector,
213 server_selection_timeout,
--> 214 address))
215
216 def select_server_by_address(self, address,
~/conda_envs/dev/lib/python3.6/site-packages/pymongo/topology.py in select_servers(self, selector, server_selection_timeout, address)
187 if server_timeout == 0 or now > end_time:
188 raise ServerSelectionTimeoutError(
--> 189 self._error_message(selector))
190
191 self._ensure_opened()
ServerSelectionTimeoutError: xf11id-srv1:27017: [Errno 111] Connection refused
We encountered a "SeverSelectionTimeoutError: xf11id-srv1:27017: [Errno 111] Connection refused" problem when trying to do any bluesky scan. We knew that we had a storage problem on srv1 since last night and we are moving data from srv1 to other server. Now, there are ~ 150 G available.