Executing after hook_queue next(OK) occasionally very slow or events fired much later

[TellusTalk]

Hi! I'm having Haraka to Http Post inbound Emails to our Server. A few times a day the Http Post is delay for quite some time.

[Log example 8 minutes "late" Oct 7 09:06:31 future haraka[9021]: [INFO] [-] [core] [HttpMail] [http_post_file] [Before http.request] trio@engelholm.se_2013-10-07T09:06:31.750Z_4B83744B-A217-4FC5-8A5D-BBA1BD1B0DE9

Oct 7 09:14:44 future haraka[9021]: [INFO] [-] [core] [HttpMail] [http_post_file] [post_response] post_response.statusCode = 200 : trio@engelholm.se_2013-10-07T09:06:31.750Z_4B83744B-A217-4FC5-8A5D-BBA1BD1B0DE9

Maybe I'm not implementing the plugin correct grateful for help. My plugin code below.

Thanks /Peter

var

fs = require('fs'),

http = require('http'),

querystring = require('querystring'),

logger = require('./logger'),

DROP_DIRECTORY_PATH = '/tellustalk/haraka/drop/',

RETRY_DIRECTORY_PATH = '/tellustalk/haraka/retry/';

exports.hook_queue = function (next, connection, params) {

'use strict';

function haraka_log(function_name_in, section_in, text_in) {

    var log_text = '[HttpMail]';

    log_text += ' [' + function_name_in + ']';

    log_text += ' [' + section_in + ']';

    log_text += ' ' + text_in;

    logger.loginfo(log_text);

}//function haraka_log

function move_file(filename_in) {

    fs.rename(DROP_DIRECTORY_PATH + filename_in, RETRY_DIRECTORY_PATH + filename_in, function (err) {

        if (err) {

            //throw err;

            haraka_log('move_file', 'fs.rename ... failed', filename_in + '\n' + JSON.stringify(err));

        } else {

            haraka_log('move_file', 'fs.rename ... success', filename_in);

        }

    });

}//function move_file

function delete_file(filename_in) {

    fs.unlink(DROP_DIRECTORY_PATH + filename_in, function (err) {

        if (err) {

            //throw err;

            haraka_log('delete_file', 'fs.unlink ... failed', filename_in + '\n' + JSON.stringify(err));

        } else {

            haraka_log('delete_file', 'fs.unlink ... success', filename_in);

        }

    });

}//function delete_file

function http_post_file(filename_in) {

    var

        post_options = {

            host: 'cloud.tellustalk.com',

            port: 80,

            path: '/http_mail/future?' + querystring.stringify({FileName: filename_in}),

            method: 'POST',

            headers: {'Content-Type': 'text/plain'}

        },

        post_request,

        read_stream;

    haraka_log('http_post_file', 'Before http.request', filename_in);

    post_request = http.request(post_options, function (post_response) {

        haraka_log('http_post_file', 'post_response', ' post_response.statusCode = ' + post_response.statusCode + ' : ' + filename_in);

        if (post_response.statusCode === 200) {

            delete_file(filename_in);

        } else {

            move_file(filename_in);//Posted later by retry script;

        }

    });//post_request = http.request(post_options, function(post_response) {

    post_request.on('error', function (err) {

        haraka_log('http_post_file post_request.on(\'error\' ...)', err.message, filename_in);

        move_file(filename_in);

    });//post_request.on('error', function(err) {

    read_stream = fs.createReadStream(DROP_DIRECTORY_PATH + filename_in);

    read_stream.pipe(post_request);

}//function http_post_file

var

    x_sender = connection.transaction.mail_from,

    x_receiver_list = connection.transaction.rcpt_to,

    filename = x_sender + '_' + new Date().toISOString() + '_' + connection.uuid,

    writeStream;

connection.transaction.add_header('x-sender', x_sender.toString());

x_receiver_list.forEach(function (value) {

    connection.transaction.add_header('x-receiver', value.toString());

});

haraka_log('main', 'filename', filename);

writeStream = fs.createWriteStream(DROP_DIRECTORY_PATH + filename);

connection.transaction.message_stream.pipe(writeStream, {dot_stuffing: true, ending_dot: true});

writeStream.on("close", function () {

    haraka_log('main writeStream.on("close"...', 'File Saved!', filename);

    http_post_file(filename);

    next(OK);

});//writeStream.on("close", function()

};//exports.hook_queue = function(next, connection, params) {

[TellusTalk]

This problem is not related to Haraka. It had to do with incorrect use of nodes http.request, which is utilizing socket pooling. The socket assigned to your request is not automatically returned to the pool. Make sure to do it manually or you will run out of sockets and performance will be severally degraded.

Use one of these options to solve this problem

• Disable socket pooling set: post_options.agent = false;
• Read the response post_response.read() and socket is returned back to pool
• post_response.resume(); Just returns the socket to the pool.

---

Example: The problem fixed using, post_response.resume();

post_request = http.request(post_options, function (post_response) {

    haraka_log('http_post_file', 'post_response', ' post_response.statusCode = ' + post_response.statusCode + ' : ' + filename_in);

    if (post_response.statusCode === 200) {

        delete_file(filename_in);

    } else {

        move_file(filename_in);//Posted later by retry script;

    }

    post_response.resume();  //This line solves the problem and returns the socket to the pool.

});//post_request = http.request(post_options, function(post_response) {