fix: Reconnect Ogmios sockets on unexpected disconnects, and logging fixes

[rhyslbw]

Context

Ogmios disconnects the WebSocket when attempting to submit some invalid payloads. Once disconnected, the server must be restarted. There are also a couple of bugs with the connection context error and close handlers, causing a bunyan error.

Proposed Solution

• Hoist the context init to DRY the code.
• Fix the handlers to always log a structured object
• Treat 1006 close events as an error, the rest as informational.
• Shutdown and re-initialize the client on unexpected close events.

Important Changes Introduced

Example of the bunyan error:

{"name":"cardano-graphql","hostname":"5fd0492a7031","pid":1,"level":30,"module":"Server","msg":"Initializing","time":"2022-09-08T09:13:13.201Z","v":0}
{"name":"cardano-graphql","hostname":"5fd0492a7031","pid":1,"level":30,"module":"HasuraClient","msg":"Initializing","time":"2022-09-08T09:13:13.729Z","v":0}
{"name":"cardano-graphql","hostname":"5fd0492a7031","pid":1,"level":30,"module":"CardanoNodeClient","msg":"Initializing. This can take a few minutes...","time":"2022-09-08T09:13:14.315Z","v":0}
{"name":"cardano-graphql","hostname":"5fd0492a7031","pid":1,"level":30,"module":"CardanoNodeClient","msg":"Initialized","time":"2022-09-08T09:13:18.058Z","v":0}
{"name":"cardano-graphql","hostname":"5fd0492a7031","pid":1,"level":30,"module":"MetadataFetchClient","msg":"Initializing","time":"2022-09-08T09:13:18.059Z","v":0}
{"name":"cardano-graphql","hostname":"5fd0492a7031","pid":1,"level":30,"module":"MetadataFetchClient","msg":"Initialized","time":"2022-09-08T09:13:18.189Z","v":0}
{"name":"cardano-graphql","hostname":"5fd0492a7031","pid":1,"level":30,"module":"ChainFollower","msg":"Initializing","time":"2022-09-08T09:13:18.189Z","v":0}
{"name":"cardano-graphql","hostname":"5fd0492a7031","pid":1,"level":30,"module":"ChainFollower","msg":"Initialized","time":"2022-09-08T09:13:18.202Z","v":0}
bunyan usage error: /app/node_modules/ws/lib/websocket.js:246: attempt to log with an unbound log method: `this` is: <ref *1> WebSocket {
  _events: [Object: null prototype] {},
  _eventsCount: 0,
  _maxListeners: undefined,
  _binaryType: 'nodebuffer',
  _closeCode: 1006,
  _closeFrameReceived: false,
  _closeFrameSent: false,
  _closeMessage: '',
  _closeTimer: null,
  _extensions: {},
  _protocol: '',
  _readyState: 3,
  _receiver: Receiver {
    _writableState: WritableState {
      objectMode: false,
      highWaterMark: 16384,
      finalCalled: false,
      needDrain: false,
      ending: true,
      ended: true,
      finished: true,
      destroyed: true,
      decodeStrings: true,
      defaultEncoding: 'utf8',
      length: 0,
      writing: false,
      corked: 0,
      sync: true,
      bufferProcessing: false,
      onwrite: [Function: bound onwrite],
      writecb: null,
      writelen: 0,
      afterWriteTickInfo: null,
      buffered: [],
      bufferedIndex: 0,
      allBuffers: true,
      allNoop: true,
      pendingcb: 0,
      prefinished: true,
      errorEmitted: false,
      emitClose: true,
      autoDestroy: true,
      errored: null,
      closed: true
    },
    _events: [Object: null prototype] {},
    _eventsCount: 0,
    _maxListeners: undefined,
    _binaryType: 'nodebuffer',
    _extensions: {},
    _isServer: false,
    _maxPayload: 134217728,
    _bufferedBytes: 0,
    _buffers: [],
    _compressed: false,
    _payloadLength: 0,
    _mask: undefined,
    _fragmented: 0,
    _masked: false,
    _fin: false,
    _opcode: 0,
    _totalPayloadLength: 0,
    _messageLength: 0,
    _fragments: [],
    _state: 0,
    _loop: false,
    [Symbol(kCapture)]: false,
    [Symbol(websocket)]: [Circular *1]
  },
  _sender: Sender {
    _extensions: {},
    _socket: Socket {
      connecting: false,
      _hadError: false,
      _parent: null,
      _host: 'cardano-node-ogmios',
      _readableState: [ReadableState],
      _events: [Object: null prototype],
      _eventsCount: 2,
      _maxListeners: undefined,
      _writableState: [WritableState],
      allowHalfOpen: false,
      _sockname: null,
      _pendingData: null,
      _pendingEncoding: '',
      server: null,
      _server: null,
      parser: null,
      _httpMessage: null,
      timeout: 0,
      write: [Function: writeAfterFIN],
      [Symbol(async_id_symbol)]: 299,
      [Symbol(kHandle)]: null,
      [Symbol(kSetNoDelay)]: true,
      [Symbol(lastWriteQueueSize)]: 0,
      [Symbol(timeout)]: null,
      [Symbol(kBuffer)]: null,
      [Symbol(kBufferCb)]: null,
      [Symbol(kBufferGen)]: null,
      [Symbol(kCapture)]: false,
      [Symbol(kBytesRead)]: 138,
      [Symbol(kBytesWritten)]: 233,
      [Symbol(RequestTimeout)]: undefined,
      [Symbol(websocket)]: undefined
    },
    _firstFragment: true,
    _compress: false,
    _bufferedBytes: 0,
    _deflating: false,
    _queue: []
  },
  _socket: Socket {
    connecting: false,
    _hadError: false,
    _parent: null,
    _host: 'cardano-node-ogmios',
    _readableState: ReadableState {
      objectMode: false,
      highWaterMark: 16384,
      buffer: BufferList { head: null, tail: null, length: 0 },
      length: 0,
      pipes: [],
      flowing: true,
      ended: true,
      endEmitted: true,
      reading: false,
      sync: false,
      needReadable: false,
      emittedReadable: false,
      readableListening: false,
      resumeScheduled: false,
      errorEmitted: false,
      emitClose: false,
      autoDestroy: false,
      destroyed: true,
      errored: null,
      closed: true,
      closeEmitted: true,
      defaultEncoding: 'utf8',
      awaitDrainWriters: null,
      multiAwaitDrain: false,
      readingMore: false,
      dataEmitted: true,
      decoder: null,
      encoding: null,
      [Symbol(kPaused)]: false
    },
    _events: [Object: null prototype] {
      end: [Function: onReadableStreamEnd],
      error: [Function: socketOnError]
    },
    _eventsCount: 2,
    _maxListeners: undefined,
    _writableState: WritableState {
      objectMode: false,
      highWaterMark: 16384,
      finalCalled: true,
      needDrain: false,
      ending: true,
      ended: true,
      finished: true,
      destroyed: true,
      decodeStrings: false,
      defaultEncoding: 'utf8',
      length: 0,
      writing: false,
      corked: 0,
      sync: false,
      bufferProcessing: false,
      onwrite: [Function: bound onwrite],
      writecb: null,
      writelen: 0,
      afterWriteTickInfo: null,
      buffered: [],
      bufferedIndex: 0,
      allBuffers: true,
      allNoop: true,
      pendingcb: 0,
      prefinished: true,
      errorEmitted: false,
      emitClose: false,
      autoDestroy: false,
      errored: null,
      closed: true,
      closeEmitted: false,
      writable: true
    },
    allowHalfOpen: false,
    _sockname: null,
    _pendingData: null,
    _pendingEncoding: '',
    server: null,
    _server: null,
    parser: null,
    _httpMessage: null,
    timeout: 0,
    write: [Function: writeAfterFIN],
    [Symbol(async_id_symbol)]: 299,
    [Symbol(kHandle)]: null,
    [Symbol(kSetNoDelay)]: true,
    [Symbol(lastWriteQueueSize)]: 0,
    [Symbol(timeout)]: null,
    [Symbol(kBuffer)]: null,
    [Symbol(kBufferCb)]: null,
    [Symbol(kBufferGen)]: null,
    [Symbol(kCapture)]: false,
    [Symbol(kBytesRead)]: 138,
    [Symbol(kBytesWritten)]: 233,
    [Symbol(RequestTimeout)]: undefined,
    [Symbol(websocket)]: undefined
  },
  _bufferedAmount: 0,
  _isServer: false,
  _redirects: 0,
  _url: 'ws://cardano-node-ogmios:1337/',
  _req: null,
  [Symbol(kCapture)]: false
}
{"name":"cardano-graphql","hostname":"5fd0492a7031","pid":1,"level":50,"module":"ChainFollower","code":1006,"msg":"","time":"2022-09-08T09:13:21.235Z","v":0}