GieltjE commented 4 years ago

Summary

Outbound scanning is currently to sparsely documented and non-functional (part bug, part feature i guess?)

Motivation

We are not rejecting/greylisting inbound email, this has proven to be problematic (lots of mail getting "lost" and users getting mad because the other mailserver is ancient/crap/listed on blocklists e.t.c.). However we are attempting to prevent becoming blacklisted ourselves (keyword attempt), as in we want to scan and reject outbound mail that has a certain score(uribl, surbl with url redirection e.t.c., basically normal spamfiltering minus spf, dkim ofcourse) or contains virusses.

Describe alternatives you've considered

Reading the documentation for the 10th time?

Additional context

https://rspamd.com/doc/tutorials/scanning_outbound.html <-- has no section for regular milter
It is not clear how to define a different weight to a symbol for just authenticated/trusted_ips.
https://rspamd.com/doc/configuration/settings.html <-- symbols/groups ([symbols|groups]_enabled, [symbols|groups]_disabled) should have an setting like [symbols|groups]_add and [symbols|groups]_remove to allow adding/removing specific items without having to figure out what was in them. e.g. the following just stops all filtering and doesn't add any headers:
```
local.d/settings.conf
authenticated {
priority = high;
authenticated = yes;

apply {
actions {
  reject = 5.0; # force rejection when trying to send spam
}
groups_enabled = ["antivirus","uribl","headers","surbl"];
}
}
```

Usefull parts:

local.d/settings.conf
authenticated {
  priority = high;
  authenticated = yes;

  apply {
    actions {
      reject = 5.0; # force rejection when trying to send spam
    }
    #groups_enabled = ["antivirus","uribl","headers","surbl"]; # breaks everything
  }
}

override.d/actions.conf
add_header = 5;
reject = null; # Do not ever reject
greylist = null; # Do not ever grelist

The above should cause no inbound rejection (functions properly), but sending an email causes no rejection (even when including an item on the openphish list).

Full config dump:

spamassassin {
}
dkim_signing {
    use_esld = true;
    allow_hdrfrom_mismatch = false;
    selector = "_dkim";
    symbol = "DKIM_SIGNED";
    sign_networks [
        "127.2.4.7",
    ]
    sign_inbound = true;
    allow_envfrom_empty = true;
    try_fallback = true;
    sign_authenticated = true;
    path = "/var/lib/rspamd/dkim/dkim.key";
    use_redis = false;
    allow_username_mismatch = false;
    sign_local = true;
    key_prefix = "DKIM_KEYS";
    use_domain = "header";
    allow_hdrfrom_multiple = false;
}
mx_check {
    enabled = true;
    key_prefix = "rmx";
    symbol_good_mx = "MX_GOOD";
    symbol_no_mx = "MX_MISSING";
    symbol_bad_mx = "MX_INVALID";
    timeout = 1;
    expire = 86400;
}
regexp {
  <snip>
}
arc {
    use_esld = true;
    allow_hdrfrom_mismatch = true;
    selector = "_dkim";
    key_prefix = "ARC_KEYS";
    sign_networks [
        "127.2.4.7",
    ]
    path = "/var/lib/rspamd/dkim/dkim.key";
    allow_envfrom_empty = true;
    try_fallback = true;
    sign_authenticated = true;
    symbol_sign = "ARC_SIGNED";
    use_redis = false;
    allow_username_mismatch = false;
    sign_inbound = true;
    use_domain = "recipient";
    sign_local = true;
    allow_hdrfrom_multiple = false;
}
maillist {
    symbol = "MAILLIST";
}
lua = "/usr/share/rspamd/rules/rspamd.lua";
surbl [
]
modules {
    path = "/usr/share/rspamd/plugins";
    try_path = "/etc/rspamd/plugins.d/";
    fallback_path = "/usr/share/rspamd/lua";
}
antivirus {
    clamav {
        scan_mime_parts = true;
        type = "clamav";
        servers = "/var/run/clamav/clamd.ctl";
        symbol = "CLAM_VIRUS";
    }
}
whitelist {
    rules {
     <snip>
    }
}
neural {
    train {
        learning_rate = 0.010000;
        max_usages = 20;
        max_iterations = 25;
        max_train = 1000;
    }
    enabled = true;
    timeout = 20;
    ann_expire = 1728000;
}
metric {
}
hfilter {
    rcpt_enabled = true;
    helo_enabled = true;
    from_enabled = true;
    hostname_enabled = true;
    url_enabled = true;
    mid_enabled = false;
}
phishing {
    redirector_domains [
        "https://maps.rspamd.com/rspamd/redirectors.inc.zst:REDIRECTOR_FALSE",
        "/etc/rspamd/local.d/maps.d/redirectors.inc:LOCAL_REDIRECTOR_FALSE",
        "/etc/rspamd/local.d/redirectors.inc:LOCAL_REDIRECTOR_FALSE",
        "fallback+file:///etc/rspamd/maps.d/redirectors.inc:REDIRECTOR_FALSE",
    ]
    openphish_map = "https://www.openphish.com/feed.txt";
    symbol = "PHISHING";
    openphish_enabled = true;
    openphish_premium = false;
    phishtank_enabled = true;
}
mime_types {
    file [
        "https://maps.rspamd.com/rspamd/mime_types.inc.zst",
        "/etc/rspamd/local.d/maps.d/mime_types.inc.local",
        "/var/lib/rspamd/mime_types.inc.local",
        "fallback+file:///etc/rspamd/maps.d/mime_types.inc",
    ]
    extension_map {
        pdf [
            "application/octet-stream",
            "application/pdf",
        ]
        html = "text/html";
        txt [
            "message/disposition-notification",
            "text/plain",
            "text/rfc822-headers",
        ]
    }
}
logging {
    filename = "/var/log/rspamd/rspamd.log";
    log_format = <<EOD
id: <$mid>,$if_qid{ qid: <$>,}$if_ip{ ip: $,}$if_user{ user: $,}$if_smtp_from{ from: <$>,}
(default: $is_spam ($action): [$scores] [$symbols_scores_params]),
len: $len, time: $time_real, dns req: $dns_req,
digest: <$digest>$if_smtp_rcpts{, rcpts: <$>}$if_mime_rcpts{, mime_rcpts: <$>}$if_filename{, file: $}$if_forced_action{, forced: $}$if_settings_id{, settings_id: $}
EOD;
    log_usec = false;
    color = false;
    type = "file";
    debug_modules [
    ]
    log_re_cache = true;
    level = "notice";
}
rspamd_update {
    enabled = false;
    key = "qxuogdh5eghytji1utkkte1dn3n81c3y5twe61uzoddzwqzuxxyb";
    rules = "sign+https://updates.rspamd.com/rspamd-2.ucl";
}
fuzzy_check {
  <snip>
    timeout = 2;
    min_bytes = 1000;
}
composites {
  <snip>
}
mid {
    source {
        url [
            "https://maps.rspamd.com/rspamd/mid.inc.zst",
            "/etc/rspamd/local.d/maps.d/mid.inc",
            "/etc/rspamd/local.d/mid.inc",
            "fallback+file:///etc/rspamd/maps.d/mid.inc",
        ]
    }
}
forged_recipients {
    symbol_sender = "FORGED_SENDER";
    symbol_rcpt = "FORGED_RECIPIENTS";
}
spamtrap {
    learn_fuzzy = false;
    enabled = false;
    learn_spam = false;
}
force_actions {
}
spf {
    spf_cache_expire = 86400;
    external_relay = "/etc/rspamd/local.d/relay.map";
    spf_cache_size = 2000;
}
clickhouse {
    limit = 1000;
    ipmask6 = 48;
    full_urls = false;
    timeout = 5;
    ipmask = 19;
}
group {
  <snip>
}
metadata_exporter {
    rules {
    }
}
multimap {
  <snip>
}
worker {
    fuzzy {
        backend = "redis";
        allow_update [
            "127.0.0.1",
            "::1",
        ]
        expire = 7776000;
        count = 4;
        bind_socket = "127.0.0.1:11335";
        sync = 60;
    }
}
worker {
    normal {
        enabled = false;
        bind_socket = "localhost:11333";
        mime = true;
    }
}
worker {
    controller {
        password = "xxxxxxxxxf";
        secure_ip = "127.0.0.1";
        secure_ip = "::1";
        enable_password = "xxxxxxxx;
        static_dir = "/usr/share/rspamd/www";
        count = 1;
        bind_socket = "localhost:11334";
    }
}
worker {
    rspamd_proxy {
        max_retries = 5;
        timeout = 120;
        spam_header = "X-Spam";
        quarantine_on_reject = false;
        reject_message = "Spam message rejected";
        discard_on_reject = false;
        milter = true;
        upstream {
            local {
                hosts = "localhost";
                self_scan = true;
                default = true;
            }
        }
        bind_socket = "localhost:11332";
        count = 1;
    }
}
worker {
    fuzzy {
        backend = "redis";
        allow_update [
            "localhost",
        ]
        count = -1;
        bind_socket = "localhost:11335";
        expire = 7776000;
    }
}
dmarc {
    report_settings {
        domain = "black-mail.nl";
        email = "reports@black-mail.nl";
        org_name = "Black-Mail.nl";
    }
    reporting = true;
    send_reports = true;
}
milter_headers {
    skip_local = false;
    use [
        "x-spamd-bar",
        "authentication-results",
        "x-spam-status",
        "spam-header",
        "x-virus",
        "fuzzy-hashes",
    ]
    authenticated_headers [
        "authentication-results",
        "x-spam-status",
        "spam-header",
        "x-virus",
    ]
    local_headers [
        "x-spamd-bar",
        "authentication-results",
        "x-spam-status",
        "spam-header",
        "x-virus",
    ]
    skip_authenticated = false;
    extended_spam_headers = true;
    routines {
        authentication-results {
            header = "Authentication-Results";
            remove = 1;
            add_smtp_user = true;
        }
        x-virus {
            symbols [
                "CLAM_VIRUS",
            ]
            remove = 1;
            status_infected = "Infected";
            header = "X-Virus";
            status_clean = "Clean";
        }
    }
}
external_services {
  <snip>
}
actions {
    add_header = 5;
    greylist = null;
    reject = null;
}
elastic {
    limit = 10;
    import_kibana = false;
    debug = false;
    timeout = 5;
    index_pattern = "rspamd-%Y.%m.%d";
}
options {
    cache_file = "/var/lib/rspamd/symbols.cache";
    map_watch_interval = 300;
    tempdir = "/tmp";
    history_rows = 200;
    soft_reject_on_timeout = false;
    task_timeout = 8;
    hs_cache_dir = "/var/lib/rspamd/";
    local_addrs = "127.0.0.0/8, ::ffff:127.0.0.0/104, ::1/128, fe80::/10, x.x.x.x/32";
    local_addrs [
        "127.2.4.7",
    ]
    url_tld = "/usr/share/rspamd/effective_tld_names.dat";
    max_urls = 10240;
    stats_file = "/var/lib/rspamd/stats.ucl";
    dns_max_requests = 64;
    pidfile = "/run/rspamd/rspamd.pid";
    rrd = "/var/lib/rspamd/rspamd.rrd";
    check_all_filters = true;
    max_lua_urls = 1024;
    explicit_modules [
        "settings",
        "bayes_expiry",
    ]
    control_socket = "/var/lib/rspamd/rspamd.sock mode=0600";
    allow_raw_input = true;
    dynamic_conf = "/var/lib/rspamd/rspamd_dynamic";
    dns {
        sockets = 16;
        timeout = 1;
        retransmits = 5;
    }
    raw_mode = false;
    filters = "chartable,dkim,regexp,fuzzy_check";
    classify_headers [
        "User-Agent",
        "X-Mailer",
        "Content-Type",
        "X-MimeOLE",
    ]
    words_decay = 600;
    history_file = "/var/lib/rspamd/rspamd.history";
    one_shot = false;
    map_file_watch_multiplier = 0.100000;
}
reputation {
}
emails [
]
asn {
    provider_info {
        ip6 = "asn6.rspamd.com";
        ip4 = "asn.rspamd.com";
    }
    provider_type = "rspamd";
}
settings {
    authenticated {
        authenticated = true;
        priority = "high";
        apply {
            actions {
                reject = 5;
            }
            groups_enabled [
                "antivirus",
                "uribl",
                "headers",
                "surbl",
            ]
        }
    }
}
chartable {
    symbol = "R_MIXED_CHARSET";
    threshold = 0.300000;
}
dcc {
    fuz2_max = 999999;
    fuz1_max = 999999;
    body_max = 999999;
    timeout = 5;
    retransmits = 2;
    symbol_bulk = "DCC_BULK";
    log_clean = false;
    symbol = "DCC_REJECT";
    symbol_fail = "DCC_FAIL";
    enabled = true;
    servers = "/var/lib/dcc/dccifd";
    socket = "/var/dcc/dccifd";
}
history_redis {
    nrows = 200;
    subject_privacy = false;
    compress = true;
    key_prefix = "rs_history";
}
classifier {
  <snip>
}
p0f {
  <snip>
}
metric_exporter {
}
url_redirector {
    top_urls_key = "rdr:top_urls";
    key_prefix = "rdr:";
    top_urls_count = 200;
    expire = 86400;
    redirectors_only = true;
    redirector_hosts_map = "/etc/rspamd/maps.d/redirectors.inc";
    max_size = 10000;
    nested_limit = 1;
    redirector_symbol = "MY_REDIRECTOR_SYMBOL";
    max_urls = 5;
    check_ssl = false;
    timeout = 10;
}
trie {
}
greylist {
    ipv6_mask = 64;
    whitelist_domains_url [
        "/etc/rspamd/local.d/greylist-whitelist-domains.inc",
        "/etc/rspamd/local.d/maps.d/greylist-whitelist-domains.inc",
    ]
    expire = 86400;
    enabled = false;
    ipv4_mask = 19;
    message = "Try again later";
    max_data_len = 10000;
    action = "soft reject";
    key_prefix = "rg";
    timeout = 300;
}
replies {
    symbol = "REPLY";
    message = "Message is reply to one we originated";
    expire = 1209600;
    key_prefix = "rr";
}
redis {
    servers = "127.0.0.1";
}
dkim {
    trusted_only = false;
    dkim_cache_size = 2000;
    dkim_cache_expire = 86400;
    time_jitter = 21600;
    skip_multi = false;
}
rbl {
    default_exclude_users = true;
    exclude_users = false;
    default_received = false;
    default_unknown = true;
    default_from = true;
    rbls {
  <snip>
    }
    url_whitelist [
        "https://maps.rspamd.com/rspamd/surbl-whitelist.inc.zst",
        "/etc/rspamd/local.d/maps.d/surbl-whitelist.inc.local",
        "/var/lib/rspamd/surbl-whitelist.inc.local",
        "fallback+file:///etc/rspamd/maps.d/surbl-whitelist.inc",
    ]
}
ratelimit {
    max_rcpt = 5;
    whitelisted_rcpts = "postmaster,mailer-daemon";
}
once_received {
    bad_host = "static";
    bad_host = "dynamic";
    good_host = "mail";
    symbol_strict = "ONCE_RECEIVED_STRICT";
    symbol_mx = "DIRECT_TO_MX";
    symbol = "ONCE_RECEIVED";
}

andryyy commented 4 years ago

Sounds more like you need help configuring it. Perhaps you should try the telegram channel. :)

GieltjE commented 4 years ago

Talked on IRC today, they suggested my config should work fine.

fatalbanana commented 4 years ago

So what will actually work is overriding actions like so:

# cat rspamd.conf.override 
actions {
  add_header = 5;
}

Setting reject to null unfortunately will not.

GieltjE commented 4 years ago

https://rspamd.com/doc/tutorials/writing_rules.html#configuration-files It's litteraly the way described in this example

stale[bot] commented 4 years ago

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

GieltjE commented 4 years ago

damn stale crapbot

stale[bot] commented 3 years ago

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

GieltjE commented 3 years ago

damn stale crapbot

stale[bot] commented 3 years ago

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

GieltjE commented 3 years ago

!stale

fatalbanana commented 3 years ago

https://rspamd.com/doc/tutorials/scanning_outbound.html <-- has no section for regular milter

There's not much to be done about it that comes to mind, possibly it's useful to mention non_smtpd_milters for Postfix users.

It is not clear how to define a different weight to a symbol for just authenticated/trusted_ips.

Using symbol in apply block: https://rspamd.com/doc/configuration/settings.html#settings-structure

https://rspamd.com/doc/configuration/settings.html <-- symbols/groups ([symbols|groups]_enabled, [symbols|groups]_disabled) should have an setting like [symbols|groups]_add and [symbols|groups]_remove to allow adding/removing specific items without having to figure out what was in them.

They are called symbols and symbols_disabled.

The above should cause no inbound rejection (functions properly), but sending an email causes no rejection (even when including an item on the openphish list).

The issue is about setting null in config not behaving the same as removing the action; it should be fixed here.

rspamd / rspamd

[Feature] Outbound scanning improvements #3415

Summary

Motivation

Describe alternatives you've considered

Additional context