search

dask / dask-ec2

Start a cluster in EC2 for dask.distributed
106 stars 37 forks source link

Initializing client with address specified results in a "Connection refused" error #82

Closed arokem closed 7 years ago

arokem commented 7 years ago

When using the example notebooks, or initializing a client as per the instructions, I get the following error: 

In [3]: c = Client('127.0.0.1:8786')
distributed.utils - ERROR - Timed out trying to connect to 'tcp://127.0.0.1:8786' after 3 s: ConnectionRefusedError: [Errno 111] Connection refused
Traceback (most recent call last):
  File "/opt/anaconda/lib/python3.5/site-packages/distributed/comm/core.py", line 159, in connect
    quiet_exceptions=EnvironmentError)
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py", line 1015, in run
    value = future.result()
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/concurrent.py", line 237, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 3, in raise_exc_info
tornado.gen.TimeoutError: Timeout

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/anaconda/lib/python3.5/site-packages/distributed/utils.py", line 193, in f
    result[0] = yield gen.maybe_future(func(*args, **kwargs))
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py", line 1015, in run
    value = future.result()
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/concurrent.py", line 237, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 3, in raise_exc_info
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py", line 1021, in run
    yielded = self.gen.throw(*exc_info)
  File "/opt/anaconda/lib/python3.5/site-packages/distributed/client.py", line 473, in _start
    yield self.ensure_connected(timeout=timeout)
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py", line 1015, in run
    value = future.result()
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/concurrent.py", line 237, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 3, in raise_exc_info
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py", line 1021, in run
    yielded = self.gen.throw(*exc_info)
  File "/opt/anaconda/lib/python3.5/site-packages/distributed/client.py", line 501, in ensure_connected
    timeout=timeout)
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py", line 1015, in run
    value = future.result()
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/concurrent.py", line 237, in result
    raise_exc_info(self._exc_info)
  File "<string>", line 3, in raise_exc_info
  File "/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py", line 1021, in run
    yielded = self.gen.throw(*exc_info)
  File "/opt/anaconda/lib/python3.5/site-packages/distributed/comm/core.py", line 168, in connect
    _raise(error)
  File "/opt/anaconda/lib/python3.5/site-packages/distributed/comm/core.py", line 152, in _raise
    raise IOError(msg)
OSError: Timed out trying to connect to 'tcp://127.0.0.1:8786' after 3 s: ConnectionRefusedError: [Errno 111] Connection refused
---------------------------------------------------------------------------
TimeoutError                              Traceback (most recent call last)
/opt/anaconda/lib/python3.5/site-packages/distributed/comm/core.py in connect(addr, timeout, deserialize)
    158                                           future,
--> 159                                           quiet_exceptions=EnvironmentError)
    160         except EnvironmentError as e:

/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py in run(self)
   1014                     try:
-> 1015                         value = future.result()
   1016                     except Exception:

/opt/anaconda/lib/python3.5/site-packages/tornado/concurrent.py in result(self, timeout)
    236         if self._exc_info is not None:
--> 237             raise_exc_info(self._exc_info)
    238         self._check_done()

/opt/anaconda/lib/python3.5/site-packages/tornado/util.py in raise_exc_info(exc_info)

TimeoutError: Timeout

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
<ipython-input-3-7f40a6085691> in <module>()
----> 1 c = Client('127.0.0.1:8786')

/opt/anaconda/lib/python3.5/site-packages/distributed/client.py in __init__(self, address, start, loop, timeout, set_as_default, scheduler_file)
    382 
    383         if start:
--> 384             self.start(timeout=timeout)
    385 
    386         from distributed.channels import ChannelClient

/opt/anaconda/lib/python3.5/site-packages/distributed/client.py in start(self, **kwargs)
    417         self.loop.add_callback(pc.start)
    418         _global_client[0] = self
--> 419         sync(self.loop, self._start, **kwargs)
    420         self.status = 'running'
    421 

/opt/anaconda/lib/python3.5/site-packages/distributed/utils.py in sync(loop, func, *args, **kwargs)
    202         e.wait(1000000)
    203     if error[0]:
--> 204         six.reraise(*error[0])
    205     else:
    206         return result[0]

/opt/anaconda/lib/python3.5/site-packages/six.py in reraise(tp, value, tb)
    684         if value.__traceback__ is not tb:
    685             raise value.with_traceback(tb)
--> 686         raise value
    687 
    688 else:

/opt/anaconda/lib/python3.5/site-packages/distributed/utils.py in f()
    191                 raise RuntimeError("sync() called from thread of running loop")
    192             yield gen.moment
--> 193             result[0] = yield gen.maybe_future(func(*args, **kwargs))
    194         except Exception as exc:
    195             logger.exception(exc)

/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py in run(self)
   1013 
   1014                     try:
-> 1015                         value = future.result()
   1016                     except Exception:
   1017                         self.had_exception = True

/opt/anaconda/lib/python3.5/site-packages/tornado/concurrent.py in result(self, timeout)
    235             return self._result
    236         if self._exc_info is not None:
--> 237             raise_exc_info(self._exc_info)
    238         self._check_done()
    239         return self._result

/opt/anaconda/lib/python3.5/site-packages/tornado/util.py in raise_exc_info(exc_info)

/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py in run(self)
   1019 
   1020                     if exc_info is not None:
-> 1021                         yielded = self.gen.throw(*exc_info)
   1022                         exc_info = None
   1023                     else:

/opt/anaconda/lib/python3.5/site-packages/distributed/client.py in _start(self, timeout, **kwargs)
    471         self.scheduler_comm = None
    472 
--> 473         yield self.ensure_connected(timeout=timeout)
    474 
    475         self.coroutines.append(self._handle_report())

/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py in run(self)
   1013 
   1014                     try:
-> 1015                         value = future.result()
   1016                     except Exception:
   1017                         self.had_exception = True

/opt/anaconda/lib/python3.5/site-packages/tornado/concurrent.py in result(self, timeout)
    235             return self._result
    236         if self._exc_info is not None:
--> 237             raise_exc_info(self._exc_info)
    238         self._check_done()
    239         return self._result

/opt/anaconda/lib/python3.5/site-packages/tornado/util.py in raise_exc_info(exc_info)

/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py in run(self)
   1019 
   1020                     if exc_info is not None:
-> 1021                         yielded = self.gen.throw(*exc_info)
   1022                         exc_info = None
   1023                     else:

/opt/anaconda/lib/python3.5/site-packages/distributed/client.py in ensure_connected(self, timeout)
    499 
    500         comm = yield connect(self.scheduler.address,
--> 501                              timeout=timeout)
    502 
    503         yield self.scheduler.identity()

/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py in run(self)
   1013 
   1014                     try:
-> 1015                         value = future.result()
   1016                     except Exception:
   1017                         self.had_exception = True

/opt/anaconda/lib/python3.5/site-packages/tornado/concurrent.py in result(self, timeout)
    235             return self._result
    236         if self._exc_info is not None:
--> 237             raise_exc_info(self._exc_info)
    238         self._check_done()
    239         return self._result

/opt/anaconda/lib/python3.5/site-packages/tornado/util.py in raise_exc_info(exc_info)

/opt/anaconda/lib/python3.5/site-packages/tornado/gen.py in run(self)
   1019 
   1020                     if exc_info is not None:
-> 1021                         yielded = self.gen.throw(*exc_info)
   1022                         exc_info = None
   1023                     else:

/opt/anaconda/lib/python3.5/site-packages/distributed/comm/core.py in connect(addr, timeout, deserialize)
    166                 _raise(error)
    167         except gen.TimeoutError:
--> 168             _raise(error)
    169         else:
    170             break

/opt/anaconda/lib/python3.5/site-packages/distributed/comm/core.py in _raise(error)
    150         msg = ("Timed out trying to connect to %r after %s s: %s"
    151                % (addr, timeout, error))
--> 152         raise IOError(msg)
    153 
    154     while True:

OSError: Timed out trying to connect to 'tcp://127.0.0.1:8786' after 3 s: ConnectionRefusedError: [Errno 111] Connection refused

Potential fix to follow in an upcoming PR

arokem commented 7 years ago

Everything works fine in the notebooks when I use the fix implemented in #83

quasiben commented 7 years ago

We recently made some changes to dask-ec2, which version are you using ?

arokem commented 7 years ago

Whoops. I was on 0.4. Sorry about the noise. Closing...