pricing.js:530 JSON parse failse (broken chunk?)

[frankmasonus]

For whatever reason, fails always on XAU/USD

error: uncaughtException: Unexpected end of JSON input date=Mon Jan 22 2018 10:54:22 GMT-0500 (EST), pid=17684, uid=501, gid=20, cwd=/Users/me/depot/backoffice/utils, execPath=/Users/me/.nvm/versions/node/v8.6.0/bin/node, version=v8.6.0, argv=[/Users/me/.nvm/versions/node/v8.6.0/bin/node, /Users/me/depot/backoffice/utils/stream.js], rss=35946496, heapTotal=15269888, heapUsed=11493680, external=108305, loadavg=[2.79296875, 2.89208984375, 2.9814453125], uptime=142613 SyntaxError: Unexpected end of JSON input at JSON.parse () at /Users/me/depot/backoffice/utils/node_modules/@oanda/v20/pricing.js:530:32 at chunks.forEach.chunk (/Users/me/depot/backoffice/utils/node_modules/@oanda/v20/context.js:133:33) at Array.forEach () at IncomingMessage.response.on.d (/Users/me/depot/backoffice/utils/node_modules/@oanda/v20/context.js:130:32) at emitOne (events.js:115:13) at IncomingMessage.emit (events.js:210:7) at addChunk (_stream_readable.js:266:12) at readableAddChunk (_stream_readable.js:253:11) at IncomingMessage.Readable.push (_stream_readable.js:211:10) error: uncaughtException: Parse Error date=Mon Jan 22 2018 10:54:22 GMT-0500 (EST), pid=17684, uid=501, gid=20, cwd=/Users/me/depot/backoffice/utils, execPath=/Users/me/.nvm/versions/node/v8.6.0/bin/node, version=v8.6.0, argv=[/Users/me/.nvm/versions/node/v8.6.0/bin/node, /Users/me/depot/backoffice/utils/stream.js], rss=36257792, heapTotal=15269888, heapUsed=11751584, external=108539, loadavg=[2.79296875, 2.89208984375, 2.9814453125], uptime=142613 Error: Parse Error at TLSSocket.socketOnData (_http_client.js:454:20) at emitOne (events.js:115:13) at TLSSocket.emit (events.js:210:7) at addChunk (_stream_readable.js:266:12) at readableAddChunk (_stream_readable.js:253:11) at TLSSocket.Readable.push (_stream_readable.js:211:10) at TLSWrap.onread (net.js:587:20) Response { method: 'GET', path: '/v3/accounts/101-001-5495877-001/pricing/stream?instruments=,EUR/USD,GBP/USD,USD/JPY,XAU/USD,USD/CHF,EUR/CHF,AUD/USD,USD/CAD,NZD/USD,EUR/GBP&snapshot=true&', statusCode: '200', statusMessage: 'OK', contentType: 'application/octet-stream', rawBody: '{"type":"PRICE","time":"2018-01-22T15:54:22.661804860Z","bids":[{"price":"1334.848","liquidity":1000},{"price":"1334.768","liquidity":1000},{"price":"1334.728","liquidity":1000},{"price":"1334.648","liquidity":1000},{"price":"1334.548","liquidity":1000}],"asks":[{"price":"1335.138","liquidity":1000},{"price":"1335.218","liquidity":1000},{"price":"1335.25', body: null }

[qrpike]

Also seeing broken chunks in the HTTP stream. Any advice?

[giclo]

i just closed it with try/catch, I don't use it for production

[madurangae]

I'm seeing the same issue.

[tobemo]

This seems to occur once the response buffer of the http request reaches a certain length. If it gets too long split arbitrarily which, if not handled correctly, results in unparsable JSON.

Oanda sort off handles this. Notice that in line 140 of context.js responsBody is set to chunk. If this chunk is one of those unparsable ones that is split in two then in the next on data call (response.on('data', d => {) the remaining data will be appended resulting in a legal JSON string. But.. Oanda apparently didn't account for JSON.parse to throw an error.

A simple fix is to wrap line 650 of pricing.js in a try catch. The problem then solves itself.

This is how I adapted lines 125-141 of context.js but keep in mind that this code can't be used together with pricing.js. I alreay parse the JSON strings in here then handle it elsewhere without ever using pricing.js.

    // OLD
    responseBody = '';
    response.on('data', d => {
        responseBody += d;    // appending buffer to string converts it into string

        if (streamChunkHandler)
        {
            let chunks = responseBody.split("\n");

            chunks.forEach(chunk => {
                if (chunk.length > 0)
                {
                    streamChunkHandler(chunk);
                }

                responseBody = chunk;
            });
        }
    });

    // CUSTOM
    response.on('data', d => {
          responseBody += d;
          let chunks = responseBody.split('\n');
          for (const chunk of chunks) {
              if (chunk.length > 0) {
                  try {
                      let msg = JSON.parse(chunk);
                      if (msg.type == 'HEARTBEAT') {
                          heartbeat_handler(new PricingHeartbeat(msg)); // custom callback
                      }
                      else if (msg.type == 'PRICE') {
                          price_handler(new ClientPrice(msg)); //custom callback
                      }
                  }
                  catch (e) {
                      if (e instanceof SyntaxError) {
                          responseBody = chunk; // store bad chunk, next response.on(data) will be appended to this
                      } else {
                          throw e;
                      }
                  } 
              }
          }
    });