gmail sending mails again and again

[morwalz]

Seems haraka doesnt sending confirmation message on mail receive so google try to deliever it again

[smfreegard]

You need to supply far more information than that. e.g. some logs of the session would be useful. Also what plugins you are using.

[morwalz]

what kind of logs you need. I configured haraka server. and sending emails on haraka. It recieve emails still gmail try to deleiver again and again. and after 24 hours gmail says email bounced

[smfreegard]

Increase your logging by running:

echo LOGINFO > /path/to/haraka/config/loglevel

Then send a message into it from Gmail. Find the session UUID in the logs - it will look something like [0220EA5B-F9D5-4B6D-A07E-5E9A95894C99.1].

Grep the logs for the value before the '.' e.g. grep 0220EA5B-F9D5-4B6D-A07E-5E9A95894C99 and then paste it here.

[baudehlo]

I bet this is as simple as using the default config/plugins, which uses smtp_forward, and nothing listening on the other end.

Shankar: where do you want your mail to end up?

On Fri, Dec 5, 2014 at 4:29 PM, Steve Freegard notifications@github.com wrote:

Increase your logging by running:

echo LOGINFO > /path/to/haraka/config/loglevel

Then send a message into it from Gmail. Find the session UUID in the logs

• it will look something like [0220EA5B-F9D5-4B6D-A07E-5E9A95894C99.1].

Grep the logs for the value before the '.' e.g. grep 0220EA5B-F9D5-4B6D-A07E-5E9A95894C99 and then paste it here.

— Reply to this email directly or view it on GitHub https://github.com/baudehlo/Haraka/issues/769#issuecomment-65857396.

[morwalz]

[INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [access] pass:unlisted(mail) [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [access] pass:unlisted(conn), any, any, any [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] sender morwal89@gmail.com code=CONT msg="" [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [access] pass:unlisted(mail), unlisted(rcpt) [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [access] pass:unlisted(conn), any, any, any, any [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] hook=rcpt plugin=rcpt_to.in_host_list function=hook_rcpt params="morwal89@brownfolder.com" retval=OK msg="" [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] recipient morwal89@brownfolder.com code=CONT msg="" sender="morwal89@gmail.com" [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] I got email :) [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:64 => Recpt emailmorwal89@brownfolder.com [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:65 =>from addressmorwal89@gmail.com [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] hook=data plugin=saveAttachment/save_attach function=hook_data params="" retval=OK msg="" [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:64 => Recpt emailmorwal89@brownfolder.com [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:88 => Got attachment [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] message mid="CAJUrmed30MTW4FTxjaKnMFZ6y4qJV99b8_8jdR5zZ2196Jr3vQ@mail.gmail.com" size=1567047 rcpts=1/0/0 delay=0.001 code=CONT msg="" [INFO] [-] [save_disk] Inside Save Disk:[object Object] [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] hook=queue plugin=save_disk function=hook_queue params="" retval=OK msg="" [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] queue code=CONT msg="Message Queued (2D28EA69-BC9F-4539-A755-54FB0108E0E2.1)" [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] line 179: Logs entry updated

[morwalz]

I do not want to use smtp forward. I want to write received mails to disk or not even that. I commented out #queue/smtp_forward and used save_disk plugin (self written )

[smfreegard]

Ok - the log output looks like your plugin is returning next(OK) correctly and that Haraka is returning 250 Message Queued (2D28EA69-BC9F-4539-A755-54FB0108E0E2.1) in response.

However - I asked for the full session log and you've only given me the transaction log - please provide the output from:

grep 2D28EA69-BC9F-4539-A755-54FB0108E0E2 /path/to/log

Additionally a look at your save_disk plugin would also help.

[morwalz]

var fs = require('fs');
exports.hook_queue = function (next, connection, params) {
    this.loginfo("Inside Save Disk:" + connection);
    var ws = fs.createWriteStream('./tmp/mail' + (Date.now()) + '.eml');
    /*ws.once('close', function () {
        return next(OK);
    });*/
    connection.transaction.message_stream.pipe(ws);
    next(OK);
};

[morwalz]

loglevel: LOGINFO Starting up Haraka version 2.5.0 [INFO] [-] [core] Loading plugins [INFO] [-] [core] Loading plugin: access [INFO] [-] [core] loaded 6386 Public Suffixes [INFO] [-] [core] loaded TLD files: 1=442 2=4542 3=283 [INFO] [-] [access] loading connect.rdns_access.whitelist [INFO] [-] [access] loading mail_from.access.whitelist [INFO] [-] [access] loading rcpt_to.access.whitelist [INFO] [-] [access] loading connect.rdns_access.blacklist [INFO] [-] [access] loading mail_from.access.blacklist [INFO] [-] [access] loading rcpt_to.access.blacklist [INFO] [-] [access] loading connect.rdns_access.whitelist_regex [INFO] [-] [access] loading mail_from.access.whitelist_regex [INFO] [-] [access] loading rcpt_to.access.whitelist_regex [INFO] [-] [access] loading connect.rdns_access.blacklist_regex [INFO] [-] [access] loading mail_from.access.blacklist_regex [INFO] [-] [access] loading rcpt_to.access.blacklist_regex [INFO] [-] [access] skipping helo.checks.regexps [INFO] [-] [access] loading access.domains [INFO] [-] [core] Loading plugin: dnsbl [INFO] [-] [core] Loading plugin: data.headers [INFO] [-] [core] Loading plugin: helo.checks [INFO] [-] [core] Loading plugin: mail_from.is_resolvable [INFO] [-] [core] Loading plugin: max_unrecognized_commands [INFO] [-] [core] Loading plugin: rcpt_to.in_host_list [INFO] [-] [rcpt_to.in_host_list] loading host_list [INFO] [-] [rcpt_to.in_host_list] loading host_list_regex [INFO] [-] [core] Loading plugin: saveAttachment/save_attach [INFO] [-] [core] Loading plugin: save_disk

[smfreegard]

That log output isn't what I asked for ... please can you give me the session output.

The plugin looks OK; but you shouldn't really be returning next(OK) immediately after the pipe() - it should be done like so:

var fs = require('fs');
exports.hook_queue = function (next, connection, params) {
    var plugin = this;
    connection.loginfo(plugin, "Inside Save Disk:");
    var ws = fs.createWriteStream('./tmp/mail' + (Date.now()) + '.eml');
    ws.on('error', function (err) {
        connection.logerror(plugin, 'Error writing quarantine file: ' + err.message);
        return next();
    });
    ws.once('close', function () {
        return next(OK);
    });
    connection.transaction.message_stream.pipe(ws);
};

That way the message will not be accepted by Haraka until the file has been completely written to the pipe. Returning next(OK) before it means that the session could be reset before the writes have completed and that would be bad.

[morwalz]

I was like this only. this is not working, I am having one more plugin which is having hook on data attchment.

[smfreegard]

Your original had no error handling at all which is likely why it didn't work...

I need the log output that I keep asking for otherwise I can't tell you what the problem is. I'm going to bed soon so I suggest you hurry up if you want me to look at it before I do...

[msimerson]

@morwalz @smfreegard is asking you for this:

grep 2D28EA69-BC9F-4539-A755-54FB0108E0E2 /var/log/* where '*' is the name of the log file.

[baudehlo]

Also things get really complex when you want to process attachments because processing the attachment stream can't directly wait until done to continue on with the current transaction. This is a sort of Haraka "here be dragons" that I should write a tutorial on.

On Fri, Dec 5, 2014 at 5:15 PM, Steve Freegard notifications@github.com wrote:

Your original had no error handling at all which is likely why it didn't work...

I need the log output that I keep asking for otherwise I can't tell you what the problem is. I'm going to bed soon so I suggest you hurry up if you want me to look at it before I do...

— Reply to this email directly or view it on GitHub https://github.com/baudehlo/Haraka/issues/769#issuecomment-65863316.

[morwalz]

/root/dropboxmail/haraka_mail$ grep 2D28EA69-BC9F-4539-A755-54FB0108E0E2 log [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2] [core] connect ip=209.85.223.174 port=46833 local_ip=74.207.244.47 local_port=25 [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2] [access] pass:unlisted(conn) [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2] [access] pass:unlisted(conn), any [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2] [access] pass:unlisted(conn), any, any [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2] [helo.checks] helo_host: mail-ie0-f174.google.com, ips: 209.85.223.174, pass:match_re, bare_ip, dynamic, big_co(not), literal_mismatch, valid_hostname, rdns_match, forward_dns, mismatch [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [access] pass:unlisted(mail) [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [access] pass:unlisted(conn), any, any, any [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] sender morwal89@gmail.com code=CONT msg="" [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [access] pass:unlisted(mail), unlisted(rcpt) [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [access] pass:unlisted(conn), any, any, any, any [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] hook=rcpt plugin=rcpt_to.in_host_list function=hook_rcpt params="morwal89@brownfolder.com" retval=OK msg="" [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] recipient morwal89@brownfolder.com code=CONT msg="" sender="morwal89@gmail.com" [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] I got email :) [ { original: 'morwal89@brownfolder.com', [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:64 => Recpt emailmorwal89@brownfolder.com [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:65 =>from addressmorwal89@gmail.com [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] hook=data plugin=saveAttachment/save_attach function=hook_data params="" retval=OK msg="" [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:64 => Recpt emailmorwal89@brownfolder.com [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:88 => Got attachment [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:91 => Got links : 1 [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:136 => File uploading function calling [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] line 143: Save attchment to dropbox method now.. directly image/jpeg; name="IMG_20141109_110235 (1).jpg"IMG_20141109_110235 (1).jpg [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:64 => Recpt emailmorwal89@brownfolder.com [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:88 => Got attachment [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] message mid="CAJUrmed30MTW4FTxjaKnMFZ6y4qJV99b8_8jdR5zZ2196Jr3vQ@mail.gmail.com" size=1567047 rcpts=1/0/0 delay=0.001 code=CONT msg="" [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] hook=queue plugin=save_disk function=hook_queue params="" retval=OK msg="" [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] queue code=CONT msg="Message Queued (2D28EA69-BC9F-4539-A755-54FB0108E0E2.1)" [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:91 => Got links : 1 [NOTICE] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] disconnect ip=209.85.223.174 rdns="mail-ie0-f174.google.com" helo="mail-ie0-f174.google.com" relay=N early=N esmtp=Y tls=N pipe=Y txns=1 rcpts=1/0/0 msgs=1/0/0 bytes=1567047 lr="" time=0.906 [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] Line:136 => File uploading function calling [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] line 143: Save attchment to dropbox method now.. directly image/png; name="shankar.png"shankar.png [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] line 177: File uploaded successfully /Photos/shankar.png [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] line 179: Logs entry updated [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] line 179: Logs entry updated { _id: 548225d844ff30b424595cb2, [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] line 177: File uploaded successfully /Photos/IMG_20141109_110235 (1).jpg [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] line 179: Logs entry updated [INFO] [2D28EA69-BC9F-4539-A755-54FB0108E0E2.1] [core] line 179: Logs entry updated { _id: 548225d844ff30b424595cb2,

[smfreegard]

Ok - that log output looks totally normal except for the attachment hooks running after the disconnect.

Firstly - try using my version of your plugin which contains the additional error handling and returns next() only when the stream emits a close() event.

Can you post the code of your hook_data plugin that is hooking the attachments? I suspect you're going to have to make them a bit more complicated than they are already for this to work as you want it to.

[morwalz]

@baudehlo any help on this ? cant wait till the tutorial. please provide me if you have any raw code.

[morwalz]

I suspect you're going to have to make them a bit more complicated than they are already for this to work as you want it to.?? i didnt get. I cant really post the code on this plugin. But i will try sending next only after processing of data is completed. Right now i am just sending next after start processing data. Processing data takes longer then sending next. (async code)

[smfreegard]

Sure - but you can't do that and just expect it to work. You have to keep the remote end waiting until you have at least written the data to temporary files before you process it asynchronously.

If you had a look at the attachments plugin that I wrote (https://github.com/baudehlo/Haraka/blob/master/plugins/attachment.js) you'll see that it basically prevents the hook_data_post hook from running until all of the attachment hooks have completed successfully.

Basically you need something like this:

exports.start_attachment = function (connection, ctype, filename, body, stream) {
    var plugin = this;
    var txn = connection.transaction;

    function next(code, msg) {
       if (txn.notes.attachment_count === 0 && txn.notes.attachment_next) {
           return txn.notes.attachment_next();
        }
        else {
            return;
        }
    }

    // If we are going to do something with this attachment, then:
    stream.connection = connection;
    stream.pause();

    // For each async action we must increment attachment_count before we start
    // and decrement it once we have finished or hit an error.

    //  Example    
    var ws = fs.createWriteStream(...);
    txn.notes.attachment_count++;
    ws.on('error', function (err) {
        txn.notes.attachment_count--;
        connection.logerror(plugin, err.message);
        return next();
    });
    ws.on('close', function () {
        txn.notes.attachment_count--;
        return next();
    });

    // Start the pipe and resume the input stream
    stream.pipe(ws);
    stream.resume();
}

exports.hook_data = function (next, connection) {
    var plugin = this;
    var txn = connection.transaction;
    txn.parse_body = true; 
    txn.notes.attachment_count = 0;
    txn.attachment_hooks(function (ctype, filename, body, stream) {
        plugin.start_attachment(connection, ctype, filename, body, stream);
    });
    return next();
}

exports.hook_data_post = function (next, connection) {
    var txn = connection.transaction;
    // Wait for all of the attachment hooks to complete
    if (txn.notes.attachment_count > 0) {
        // We still have attachment hooks running...
        // Store the reference to next in notes, so they can run it on completion
        txn.notes.attachment_next = next;
    }
    else {
        next();
    }
}

[morwalz]

@smfreegard thank alot. your code helped me fix the issue.

[smfreegard]

Thanks for the feedback. Glad you got everything working.