Server crashes when reconnecting

[lirbank]

Server version: 2.0.0-beta-7 Client version: 2.0.0-beta-7

When stopping and starting the server (via cli), to test reconnect (https://github.com/rethinkdb/horizon/issues/772#issuecomment-241564417), I have to try multiple times before the server starts. It seems to be worse when the client is trying (though not sure), eg if I put the retry interval to 10 sec instead of five it seems to go better.

See below how many times I had to launch the server before I got it running:

🌄 Horizon ready for connections
^C^C$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:49532
   └── Drivers can connect to port 49531
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:49619
   └── Drivers can connect to port 49618
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:49689
   └── Drivers can connect to port 49688
Starting Horizon...
🌄 Horizon ready for connections
^C^C$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:51252
   └── Drivers can connect to port 51251
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:51339
   └── Drivers can connect to port 51338
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:51425
   └── Drivers can connect to port 51424
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:51512
   └── Drivers can connect to port 51511
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:51614
   └── Drivers can connect to port 51613
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:51716
   └── Drivers can connect to port 51715
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:51786
   └── Drivers can connect to port 51785
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:51907
   └── Drivers can connect to port 51906
Starting Horizon...
error: Unclean shutdown - killing RethinkDB child process
/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33
    if (!this._metadata.is_ready()) {
                       ^

TypeError: Cannot read property 'is_ready' of null
    at new Client (/Users/mikaellirbank/Git/lirbank/horizon/server/src/client.js:33:24)
    at WebSocketServer.websocket.Server.on.on (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:113:39)
    at emitOne (events.js:96:13)
    at WebSocketServer.emit (events.js:188:7)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:91:14
    at completeHybiUpgrade2 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:284:5)
    at /Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:297:16
    at Object.accept_protocol (/Users/mikaellirbank/Git/lirbank/horizon/server/src/server.js:36:5)
    at completeHybiUpgrade1 (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:294:32)
    at WebSocketServer.handleHybiUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:337:3)
    at WebSocketServer.handleUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:173:26)
    at Server.WebSocketServer._onServerUpgrade (/Users/mikaellirbank/Git/lirbank/horizon/server/node_modules/ws/lib/WebSocketServer.js:89:12)
    at emitThree (events.js:116:13)
    at Server.emit (events.js:194:7)
    at onParserExecuteCommon (_http_server.js:406:14)
    at HTTPParser.onParserExecute (_http_server.js:374:5)
$ hz serve --dev
App available at http://127.0.0.1:8181
RethinkDB
   ├── Admin interface: http://localhost:51945
   └── Drivers can connect to port 51944
Starting Horizon...
🌄 Horizon ready for connections

[danielmewes]

Thanks for the issue report @lirbank !

Pinging @Tryneus

[Tryneus]

Fix is up in PR #778. Thanks for reporting this, @lirbank!

[deontologician]

Fixed by #778