Not getting all bodies or message end event

[epinzur]

I'm seeing an issue on certain requests, where I'm not getting body events for bodies: 1 and 1.MIME, but I am getting body events for bodies 2 and 2.MIME. Also no message-end event is fired.

For now I've modified my code to fire the message event for any started messages when the fetch command ends, but I'd like to remove this, and be able to parse the missing data.

I created a test to reproduce this issue. I started debugging the code, and looking into a fix, but I'm not sure what the best approach will be.

I know that Parser.prototype._resUntagged = function() { is attempting to parse `... BODY[1] "Test!" BODY[1.MIME] {76}...``` as a single piece instead of two.

Please let me know of any questions.

Here is the test I made:

var assert = require('assert'),
    net = require('net'),
    Imap = require('../lib/Connection'),
    crypto = require('crypto');

var CRLF = '\r\n';

// generate data larger than highWaterMark
var body1mime = crypto.pseudoRandomBytes(38).toString('hex');
var body2 = crypto.pseudoRandomBytes(118).toString('hex').substr(0, 235);
var body2mime = crypto.pseudoRandomBytes(38).toString('hex').substr(0, 75);

var RESPONSES = [
    ['* CAPABILITY IMAP4rev1 UNSELECT NAMESPACE QUOTA CHILDREN',
        'A0 OK Thats all she wrote!',
        ''
    ].join(CRLF),
    ['* CAPABILITY IMAP4rev1 UNSELECT NAMESPACE QUOTA CHILDREN UIDPLUS MOVE',
        'A1 OK authenticated (Success)',
        ''
    ].join(CRLF),
    ['* NAMESPACE (("" "/")) NIL NIL',
        'A2 OK Success',
        ''
    ].join(CRLF),
    ['* LIST (\\Noselect) "/" "/"',
        'A3 OK Success',
        ''
    ].join(CRLF),
    ['* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen)',
        '* OK [PERMANENTFLAGS ()] Flags permitted.',
        '* OK [UIDVALIDITY 2] UIDs valid.',
        '* 685 EXISTS',
        '* 0 RECENT',
        '* OK [UIDNEXT 4422] Predicted next UID.',
        'A4 OK [READ-ONLY] INBOX selected. (Success)',
        ''
    ].join(CRLF),
    '* 1 FETCH (FLAGS (\\Flagged \\Seen $NotJunk) UID 6 INTERNALDATE "06-Jul-2015 14:18:53 +0000" BODY[1] "Test!" BODY[1.MIME] {76}'
    + CRLF
    + body1mime
    + CRLF
    + '* 1 FETCH (FLAGS (\\Flagged \\Seen $NotJunk) UID 6 INTERNALDATE "06-Jul-2015 14:18:53 +0000"  BODY[2] {235}'
    + CRLF
    + body2
    + CRLF
    + '* 1 FETCH (FLAGS (\\Flagged \\Seen $NotJunk) UID 6 INTERNALDATE "06-Jul-2015 14:18:53 +0000"   BODY[2.MIME] {75}'
    + CRLF
    + body2mime
    + CRLF
    + 'A5 OK UID FETCH completed'
    + CRLF
    + ''
    ,
    ['* BYE LOGOUT Requested',
        'A6 OK good day (Success)',
        ''
    ].join(CRLF)
];

var srv = net.createServer(function (sock) {
    sock.write('* OK asdf\r\n');
    var buf = '', lines;
    sock.on('data', function (data) {
        buf += data.toString('utf8');
        if (buf.indexOf(CRLF) > -1) {
            lines = buf.split(CRLF);
            buf = lines.pop();
            lines.forEach(function () {
                sock.write(RESPONSES.shift());
            });
        }
    });
});

var message1Ended = false;
var bodyData = {};
var result = null;

srv.listen(0, '127.0.0.1', function () {
    var port = srv.address().port;
    var imap = new Imap({
        user: 'foo',
        password: 'bar',
        host: '127.0.0.1',
        port: port,
        keepalive: false
    });
    imap.on('ready', function () {
        imap.openBox('INBOX', true, function () {
            var f = imap.fetch(6, {
                bodies: ['1', '1.MIME', '2', '2.MIME'],
                struct: true
            });
            f.on('message', function (msg, seqno) {
                var prefix = '(#' + seqno + ') ';
                msg.on('body', function (stream, info) {
                    var buffer = '';
                    stream.on('data', function (chunk) {
                        buffer += chunk.toString('utf8');
                    });
                    stream.once('end', function () {
                        bodyData[info.which] = buffer;
                    });
                });
                msg.once('attributes', function (attrs) {
                    result = attrs;
                });
                msg.once('end', function () {
                    message1Ended = true;
                });
            });
            f.on('end', function () {
                srv.close();
                imap.end();
            });
        });
    });
    imap.connect();
});

process.once('exit', function () {
    assert.deepEqual(message1Ended, true);
    assert.deepEqual(result, {
        uid: 6,
        date: new Date('06-Jul-2015 14:18:53 +0000'),
        flags: ['\\Flagged', '\\Seen', '$NotJunk']
    });
    assert.deepEqual(bodyData, {
        '1': "Test!",
        '1.MIME': body1mime,
        '2': body2,
        '2.MIME': body2mime
    });
});

[epinzur]

Also, I'm not 100% sure my test assertions are correct, but they are at least close.

[JustinBeaudry]

I'm not having an issue with the message body (though on my fetch I'm requesting just the TEXT), but I do also NOT receive the end event from imap.end(), the process just hangs or throws:

{ [Error: read ECONNRESET]
  code: 'ECONNRESET',
  errno: 'ECONNRESET',
  syscall: 'read',
  source: 'socket' }

[epinzur]

@mscdex have you had time to look at this?

[mscdex]

@rltvty Not yet unfortunately, I haven't forgotten about it though.

[lintaba]

Somehow it's connected to the TLS and the two sockets. With listening on socket and this._sock to close/end events, it works for me, but surely this is not a real solution. https://github.com/mscdex/node-imap/blob/master/lib/Connection.js#L178

  this._sock.once('close', function(had_err) {
    clearTimeout(self._tmrConn);
    clearTimeout(self._tmrAuth);
    clearTimeout(self._tmrKeepalive);
    self.state = 'disconnected';
    self.debug && self.debug('[connection] Closed');
    self.emit('close', had_err);
  });

  this._sock.once('end', function() {
    clearTimeout(self._tmrConn);
    clearTimeout(self._tmrAuth);
    clearTimeout(self._tmrKeepalive);
    self.state = 'disconnected';
    self.debug && self.debug('[connection] Ended');
    self.emit('end');
  });

[todddcls]

I'm seeing the same issue (attempting to parse two bodies at the same time). I've debugged it enough to know that Parser._resUntagged regex is picking up the latter BODY[1.MIME] {76} instead of the first BODY[1]. RE_BODYLITERAL is tested first, which why it picks up BODY[1.MIME] {76}. Otherwise parseFetch would have picked up the BODY[1](which it does via RE_BODYINLINEKEY regex).

Looking at the definition of RE_BODYINLINEKEY it looks like it expects that to be on a line by itself. I'm not familiar enough with the IMAP protocol to know if that's mandatory (and I can't make heads or tails out of the RFC). So I'm wondering if these are poorly formatted emails we're getting from the server?

[todddcls]

@mscdex The following fixes the issue, but is it legit?

Parser.prototype._resUntagged = function() { var m = RE_BODYLITERAL.exec(this._buffer); var body = /BODY[(.*)] /i.exec(this._buffer)

if (m && (!body || m.index < body.index)) {

[srinath-imaginea]

Seeing the same issue with the most recent version.

events.js:85
      throw er; // Unhandled 'error' event
            ^
Error: read ECONNRESET
    at exports._errnoException (util.js:746:11)
    at TCP.onread (net.js:550:26)

[ashwynh21]

I've been tinkering around this issue and realized that you will get all the bodies if you have a '' added to your bodies array, it should look something like this in the end:

var f = imap.fetch(6, { bodies: ['1', '1.MIME', '2', '2.MIME', ''], struct: true })

This worked for me, this issue has been open for a while