Custom error page not shown when ModSecurity found something.

LeeShan87 commented 6 years ago

Hi All,

I want to show a custom error page to our clients, when and only when our WAF block their request. Something like: Your request made something nasty. If you think this was a false positive alert, please contact with our support.

Our current configuration: Nginx: 1.12.0 Modsecurity: v3/master Modsecurity-nginx: master

An example Nginx config:

worker_processes  auto;

events {
    worker_connections  1024;
    use epoll;
}

http {
 server {
        listen 80 default_server;
        server_name  localhost;
    # Error page will be shown, but nothing will be logged.
        error_page 403 404 /40x.html;
        location = /40x.html {
    # If I add the same ModSecurity configurations here too, then it will auditlog for this location too, 
        # but the default Nginx error page will be shown.
        # And it will not work as expected, if multiple ModSecurity rule configuration is used.
        modsecurity On;
        root /srv/http;
        internal;
        modsecurity_rules '
                SecRuleEngine On
                SecAuditEngine On
                SecAuditLogParts ABIJDEFHZ
                SecAuditLogType Serial
                SecAuditLog /tmp/modsec_audit.log
                SecDebugLog "/tmp/debug_log.txt"
                SecDebugLogLevel 9
                SecRule ARGS "test" "log,id:1,block,deny,status:403"
        ';
        }

        location / {
            # If ModSecurity found something, error page will not shown,
                    # if custom error page defined here.
            # But logging will be ok.
            error_page 403 404 /40x.html;
            location = /40x.html {
            root /srv/http;
            internal;
            }

            modsecurity On;
            modsecurity_rules '
                SecRuleEngine On
                SecAuditEngine On
                SecAuditLogParts ABIJDEFHZ
                SecAuditLogType Serial
                SecAuditLog /tmp/modsec_audit.log
                SecDebugLog "/tmp/debug_log.txt"
                SecDebugLogLevel 9
                SecRule ARGS "test" "log,id:1,block,deny,status:403"
            ';
       }
    }
}

I already tried: https://github.com/SpiderLabs/ModSecurity/issues/1459 https://github.com/SpiderLabs/ModSecurity-nginx/issues/55

AirisX commented 6 years ago

Hello,

logging actions are performed in the ngx_http_modsecurity_log_handler which registered on the NGX_HTTP_LOG_PHASE. This handler performed last in Nginx-connector module.

Let's say we defining the custom error page like this:

server {
        listen 80 default_server;
        server_name  localhost;

        error_page 403 404 /40x.html;

        location = /40x.html {
                root /srv/http;
                internal;
        }

        location / {
            modsecurity On;
            modsecurity_rules '
                SecRuleEngine On
                SecAuditEngine On
                SecAuditLogParts ABIJDEFHZ
                SecAuditLogType Serial
                SecAuditLog /tmp/modsec_audit.log
                SecDebugLog "/tmp/debug_log.txt"
                SecDebugLogLevel 9
                SecRule ARGS "test" "log,id:1,block,deny,status:403"
            ';
       }
    }

In this case after ModSecurity found something the request will be redirected to our custom page. Error page will be shown correctly. But there are no audit log entries because ngx_http_modsecurity_log_handler will not be executed after the redirection. And I think this is the main problem here.

AirisX commented 6 years ago

Hi @LeeShan87,

I'm trying to fix this problem here https://github.com/SpiderLabs/ModSecurity-nginx/pull/90. Could you apply this patch and check changes in you case?

met3or commented 6 years ago

@AirisX - Following the compilation recipe for CentOS 7 here: https://github.com/SpiderLabs/ModSecurity/wiki/Compilation-recipes#centos-7-minimal

And using the altered src/ngx_http_modsecurity_module.c file suggested in your patch, I am unable to compile nginx.

/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c:22:8: error: expected ‘=’, ‘,’, ‘;’, ‘asm’ or ‘__attribute__’ before string constant
 nclude "stdio.h"
        ^
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c: In function ‘ngx_http_modsecurity_process_intervention’:
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c:201:13: error: ‘retur’ undeclared (first use in this function)
             retur
             ^
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c:201:13: note: each undeclared identifier is reported only once for each function it appears in
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c:202:9: error: expected ‘;’ before ‘}’ token
         }
         ^
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c: At top level:
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c:389:5: error: ‘ngx_http_modsecurity_init’ undeclared here (not in a function)
     ngx_http_modsecurity_init,              /* postconfiguration */
     ^
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c: In function ‘ngx_http_modsecurity_init’:
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c:423:26: error: unused variable ‘h_log’ [-Werror=unused-variable]
     ngx_http_handler_pt *h_log;
                          ^
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c: At top level:
/opt/ModSecurity-nginx/src/ngx_http_modsecurity_module.c:419:1: error: ‘ngx_http_modsecurity_init’ defined but not used [-Werror=unused-function]
 ngx_http_modsecurity_init(ngx_conf_t *cf)
 ^
cc1: all warnings being treated as errors
make[1]: *** [objs/addon/src/ngx_http_modsecurity_module.o] Error 1
make[1]: Leaving directory `/opt/nginx-1.9.2'
make: *** [build] Error 2

Are the errors generated, happy to help test this patch for you as I'm interested in having custom error pages too, let me know if there are any updates to this :)

Thanks, met3or

AirisX commented 6 years ago

Hi @met3or,

Are you sure that proposed patch applied correctly? For example you have undeclared directive retur (there is no n at the end) that this patch doesn't contain. Please see the diff one more - https://github.com/SpiderLabs/ModSecurity-nginx/pull/90/files?diff=split

Thank you!

met3or commented 6 years ago

Hi @AirisX - I'll copy the file across so I'm using the exact one, I'll try another time and feedback :)

Thanks for getting back in touch!

met3or commented 6 years ago

Hi @AirisX - Apologies for my hasty attempt earlier, using the correct patch I am able to successfully log a ModSecurity rule whilst also displaying the custom error page.

Looks good so far! 👍

met3or commented 6 years ago

Hi @AirisX - Just to confirm that I'm able to get this working as I had expected however, when running modsecurity outside of the location block in the configuration the custom defined error page does not display.

I feel this is worth mentioning.

Otherwise experiencing great results!

AirisX commented 6 years ago

@met3or could you provide your config here?

AirisX commented 6 years ago

Hi @met3or,

In order to resolve this problem you need to set modsecurity off; in location with custom error page, like this:

error_page 403 404 /40x.html;

location = /40x.html {
    root /srv/http;
    modsecurity off;
    internal;
}

The problem is that if ModSecurity is enabled in the server context, all of its locations inherit these settings. When ModSecurity will finds something nasty and thrown 403 response, the internal redirection mechanism will redirects this response to a location with a custom error page. Since this location also includes ModSecurity, the custom error page is not displayed. The request is checked again (as you can see in audit log entries). Of course, this is not the expected behavior. To change this you should disable ModSecurity at a location that contains custom error page.

victorhora commented 6 years ago

Maybe recursive_error_pages would change the inheritance behaviour as suggested at https://github.com/SpiderLabs/ModSecurity/issues/1672#issuecomment-363987672

AirisX commented 6 years ago

@victorhora recursive_error_pages do not change the inheritance behaviour in this case.

msamad commented 6 years ago

Hi guys, Thank you for all the comments above, helped me work around ModSecurity for this issue. It is a bit ugly but works. If we set modsecurity off; then we lose the audit log. Keeping modsecurity on; and just skipping the modsecurity rules in error_page location works.

        location / {
            modsecurity On;
            .......... modsecurity rules ........

            error_page 403 /40x.html;
            location = /40x.html {
                root /srv/http;
                internal;
                modsecurity_rules '
                    SecRule REQUEST_URI "@beginsWith /" "id:1,pass,phase:1,skipAfter:END-RESPONSE-980-CORRELATION"
                    SecRule REQUEST_URI "@beginsWith /" "id:2,pass,phase:2,skipAfter:END-RESPONSE-980-CORRELATION"
                ';
            }
       }

So now, when ModSecurity throws 403 and the internal redirect happens, the rules don't run and a custom page is shown.

Any better way to skip the rules than mentioned above?

msamad commented 6 years ago

Sorry, my bad. Jumped to conclusion too early. The issue persists, the original audit log still does not appear. It was the audit log by skipAfter rules that was passing the test cases. :(

met3or commented 6 years ago

Hi guys, Thank you for all the comments above, helped me work around ModSecurity for this issue. It is a bit ugly but works. If we set modsecurity off; then we lose the audit log. Keeping modsecurity on; and just skipping the modsecurity rules in error_page location works.
        location / {
            modsecurity On;
            .......... modsecurity rules ........

            error_page 403 /40x.html;
            location = /40x.html {
                root /srv/http;
                internal;
                modsecurity_rules '
                    SecRule REQUEST_URI "@beginsWith /" "id:1,pass,phase:1,skipAfter:END-RESPONSE-980-CORRELATION"
                    SecRule REQUEST_URI "@beginsWith /" "id:2,pass,phase:2,skipAfter:END-RESPONSE-980-CORRELATION"
                ';
            }
       }
So now, when ModSecurity throws 403 and the internal redirect happens, the rules don't run and a custom page is shown.

Any better way to skip the rules than mentioned above?

I've tried using rules similar to those described in your post, and whilst they achieve what I want (the custom error page displays) I seem to lose the reason for the 403 in the logs.

Has anyone managed to find a suitable workaround to this?

hazcod commented 5 years ago

Any news on this? I do not want to lose audit logs.

msamad commented 5 years ago

So far it has been either audit log or the custom error page - I sacrificed custom error page for audit logging. Would be great to have both though

hazcod commented 5 years ago

I fixed it by storing my blocked page in my asset directory, and turning off modsecurity there. I now have a custom page and logging. 👍

msamad commented 5 years ago

@HazCod Would you mind sharing a snippet of config that works for you?

meigea commented 5 years ago

https://github.com/SpiderLabs/ModSecurity-nginx/issues/76#issuecomment-392988941

@msamad Have you got the good idea to solve this problem, what you just comment is like Drink poison to quench your thirst, lose the log sounds more dangerious.

msamad commented 5 years ago

@meigea https://github.com/SpiderLabs/ModSecurity-nginx/issues/76#issuecomment-392988941 was me jumping to conclusion too early. See https://github.com/SpiderLabs/ModSecurity-nginx/issues/76#issuecomment-393002226 right after that. With limited functionalities come greater sacrifices ;) Losing custom page is not great but better than losing the logs, haven't found any other way.

meigea commented 5 years ago

I tried it like this , i move the modsecurity.conf in location block and move the modsecury on after server closely. and i test it that it can work. then ... it work!

like this


server {
modsecurity on;
error_page 403 /error_pages/3/403.html; 
.... 
location / {
    modsecurity_rules_file /etc/nginx/modsecurity/site1.modsecurity.conf; 
    proxy_cache cya_waf_cache;
    proxy_pass http://192.168.2.110:9070;
    proxy_redirect off;
   ...
    proxy_intercept_errors on;
}

location /error_pages {
    ## not add any SecRule here
    alias /usr/local/src/htmls/error_templates/htmls;
}

meigea commented 5 years ago

@msamad i tried it like i just comment and it work well. you can test and have a try. https://github.com/SpiderLabs/ModSecurity-nginx/issues/76#issuecomment-434546436

my venv

Modsec3.0
Openresty1.13.7
Modsecurity-nginx3

msamad commented 5 years ago

@meigea thanks for that, I'll give it a try.

msamad commented 5 years ago

@meigea tried your configuration and it doesn't work - I lose modsecurity audit log. Are you sure you are not seeing info/debug logs instead of audit log. https://www.nginx.com/blog/modsecurity-logging-and-debugging/

meigea commented 5 years ago

@msamad i'm sorry. it's also no use. i see the log is that produced long ago. for log we can only do it that set the modsecurity on; modsecrule after server block closely.

sorry not to help ... i wait for your options .

i have seen the debug log but log is so many that i can't find the line that save auditlog in phaser:5, it seems that we can't judge the log diff with the debug txt of two kind of configure.

LeeShan87 commented 5 years ago

Hi all!

After almost a year since I opened this issue it is still opened. I found a suitable workaround, but as a developer I wouldn't accept it as an answer.

The simple answer, if you want to have auditlog with custom error page (and with ModSecurity CRS) all built and released with the current upstream you cannot achieve it.

Same goes for mirroring. :(

The basic issue is (my opinion) modsecurity configuration belongs to a location. So if you DENY a request it will be blocked. But error page and mirror location... well it's a different location. Your request will not get to it. (Ok you caught me cheating. But this way it's easier to present the issue.)

So what is my workaround?

I don't deny requests. I just REDIRECT them.

How it can be achived? (first point why I not recommend this solution) Modify the CRS evaluation rule. (It would be much more acceptable, if SecRuleUpdateActionById had already implemented in libmodsecurity.)

crs/rules/REQUEST-949-BLOCKING-EVALUATION.conf

SecRule TX:ANOMALY_SCORE "@ge %{tx.inbound_anomaly_score_threshold}" \
    "msg:'Inbound Anomaly Score Exceeded (Total Score: %{TX.ANOMALY_SCORE})',\
    severity:CRITICAL,\
    phase:request,\
    id:949110,\
    t:none,\
    redirect:somerandomstring/403.html,\
    log,\
    tag:'application-multi',\
    tag:'language-multi',\
    tag:'platform-multi',\
    tag:'attack-generic',\
    setvar:tx.inbound_tx_msg=%{tx.msg},\
    setvar:tx.inbound_anomaly_score=%{tx.anomaly_score}"

The second part in your nginx.conf (second part I don't recommend) Create a publicly available location with the previously created 'somerandomstring'. (This string can be use to fingerprint, if you are using libmodsecurity with nginx)

    server {
        location  ~* .*somerandomstring/.*$ {
               root /your/error/page/path;
               rewrite ^(.*somerandomstring)/(.*) /$2 break;
               # well this is another issue... You cannot easily kill all keep alive connections with nginx/libmodsecurity
               keepalive_requests 0;
               keepalive_timeout 0;
        }

         location / {
            modsecurity On;
            modsecurity_rules_file /path/to/your/modsec.conf;
        }
    }

This type of configuration workaround currently working on nearly 800 production servers.

How does this workaround looks like a security point of view?

When ModSecurity triggers the attacker will receive HTTP 302 response code with the location of the error page. It will be logged and this is what matters for a SecOps/Ops.

After this point it's up to the attacker to follow the redirection or not. If it follows the custom error page will be presented and the currently active keep alive connection will be terminated.

Don't forget libmodsecurity is still in childhood. We have to be cleaver and contribute in this great product.

PS.: @victorhora could you review this?

victorhora commented 5 years ago

I was trying to reproduce the scenario presented here but I'm not too sure if I got it right. I could not reproduce the scenario where audit logs are lost with the latest code from both libModSecurity and the nginx-connector.

Could someone please provide a minimalist nginx/ModSecurity configuration that presents this behaviour? Thanks :)

victorhora commented 5 years ago

Sorry @LeeShan87, I didn't see your comment prior to mine :)

So from your comment, I'm assuming the issue still persists with the latest code of libModSecurity? There were some recent big changes including SecRuleUpdateActionByID should be working (https://github.com/SpiderLabs/ModSecurity/commit/85ecd190d965ab962874dfd2312ff4bb81469a4d)

LeeShan87 commented 5 years ago

yeeeee ^^

This is a feature I missed a very log time ago. I will definitely try it out.

I will try to create a minimalist config for a fail and workaround solution.

(It's not so easy if you created a custom build and a config generator for ModSecurity :S)

LeeShan87 commented 5 years ago

I have created a minimalist fail and success config for this issue. setup: echo "test" > /tmp/40x.html

modsec-issue-76.zip

One of our current released build:

$(pwd)/nginx -V nginx version: nginx/1.13.8 built by gcc 4.7.2 (Debian 4.7.2-5) built with OpenSSL 1.0.1e 11 Feb 2013 (running with OpenSSL 1.0.1t 3 May 2016) TLS SNI support enabled configure arguments: --user=waf --group=waf --prefix=$(pwd) --conf-path=$(pwd)/etc/nginx.conf --error-log-path=/var/log/waf/error.log --http-client-body-temp-path=/var/lib/waf/body --http-fastcgi-temp-path=/var/lib/waf/fastcgi --http-log-path=/var/log/waf/access.log --http-proxy-temp-path=/var/lib/waf/proxy --lock-path=/var/lock/waf.lock --pid-path=/var/run/waf.pid --with-pcre-jit --with-http_gzip_static_module --with-http_ssl_module --with-threads --with-http_realip_module --without-http_browser_module --without-http_geo_module --without-http_limit_req_module --without-http_memcached_module --without-http_referer_module --without-http_scgi_module --without-http_split_clients_module --with-http_stub_status_module --without-http_ssi_module --without-http_userid_module --without-http_uwsgi_module --add-module=/build/waf-fsh03E/waf-1.1.0/debian/modules/nginx-echo --add-module=/build/waf-fsh03E/waf-1.1.0/debian/modules/ModSecurity-connector (Company policy: We prefer tagged git commits for releases)

I have changed the output of the command. (This is not the place of self promotion.)

LeeShan87 commented 5 years ago

I just had time to build a new version:

Centos 6
Nginx: 1.15.6
ModSecurity v3.0.3
ModSecurity-nginx: master

This issue is still exists.

But PR #90 seems to solve the problem. Thank you for your patch @AirisX . I'm very sorry too. I haven't noticed till now it's not merge to master :S. Shame on me

met3or commented 5 years ago

So far from testing the patch from @AirisX PR #90 it seems to do the trick nicely upon initial tests. Thanks for this!

met3or commented 5 years ago

The logs produced in PR #90 unfortunately falsely publish the http_code falsely (as mentioned in the PR notes). Despite this not being the end of the world, It'd be ideal to have a way to have this produced with accurate results in the log.

yilingyi commented 5 years ago

Hi guys, I tried it like this,auditlog can be output normally,it work！

server { listen 80; listen 443 ssl; servername ; . . . modsecurity on; modsecurity_rules_file modsecurity.conf;

location / {
    . . .
    error_page  405 @error_page_405;
}
location @error_page_405 {
    rewrite ^(.*)$ /deny.html;
    modsecurity_rules '
            SecRule REQUEST_URI "@beginsWith /" "id:1,pass,phase:2,log,ctl:ruleEngine=DetectionOnly"
            ';
    proxy_pass http://localhost;
    internal;
}
location /deny.html {   #deny.html is a error page
      root html;
      more_set_headers  'request_id: $request_id';
    }

}

shmurf commented 5 years ago

Hi guys, I tried it like this,auditlog can be output normally,it work！

server { listen 80; listen 443 ssl; servername ; . . . modsecurity on; modsecurity_rules_file modsecurity.conf;

location / {
    . . .
    error_page  405 @error_page_405;
}
location @error_page_405 {
    rewrite ^(.*)$ /deny.html;
    modsecurity_rules '
            SecRule REQUEST_URI "@beginsWith /" "id:1,pass,phase:2,log,ctl:ruleEngine=DetectionOnly"
            ';
    proxy_pass http://localhost;
    internal;
}
location /deny.html {   #deny.html is a error page
      root html;
      more_set_headers  'request_id: $request_id';
    }

}

Thanks for this!

I just implemented this in a test environment and it seems to be doing the trick. I initially disabled modsec completely to enable showing my custom error page but as mentioned it resulted in logs being disabled.

location = /error.html {
#modsecurity off;
  modsecurity_rules 'SecRule REQUEST_URI "@beginsWith /" "id:1,pass,phase:2,log,ctl:ruleEngine=DetectionOnly"';
  ssi on;
  internal;
  root /srv/www/errors;
}

If I'm understanding this fix correctly, it leaves modsec on but sets the mode to detection only so it doesn't continue to redirect to the nginx default error page. One think however is that the logs might not be 100% accurate, for starters it'll show Warning when in fact the request was initially blocked. This is probably because the log is recording the redirected request and not the initial request. However the response headers and rule triggers seem to all be intact.

dansiviter commented 4 years ago

Just hit this issue. Is there any ETA on a permanent fix? It seems the MR has stalled too.

zimmerle commented 4 years ago

Fixed on ModSecurity-nginx

owasp-modsecurity / ModSecurity-nginx

Custom error page not shown when ModSecurity found something. #76

my venv