[INSTALLATION] Everything installed but will not block any bots

[buzzsubash]

Hello, I have installed per the directions, which seemed very easy, but I am not able to get a successful blocking test. Everything appears to be in order, but will not work. I use an ubuntu server and lets encrypt SSL What can I look for to fix it?

1. Output while crawling

Subashs-MacBook-Pro ~ % curl -A "Xenu Link Sleuth/1.3.8" -I https://mydomain.com HTTP/1.1 200 OK Server: nginx Date: Mon, 30 Aug 2021 11:55:45 GMT Content-Type: text/html; charset=UTF-8 Connection: keep-alive Vary: Accept-Encoding Set-Cookie: ci_session=; expires=Mon, 30-Aug-2021 12:00:45 GMT; Max-Age=300; path=/; HttpOnly Expires: Thu, 19 Nov 1981 08:52:00 GMT Cache-Control: no-store, no-cache, must-revalidate Pragma: no-cache

======================================================================================= 2. nginx.conf file

root@buzzgeetha:/etc/nginx# cat nginx.conf

Server globals

user www-data; worker_processes auto; worker_rlimit_nofile 65535; error_log /var/log/nginx/error.log; pid /var/run/nginx.pid;

Worker config

events { worker_connections 1024; use epoll; multi_accept on; }

http {

Main settings

sendfile                        on;
tcp_nopush                      on;
tcp_nodelay                     on;
client_header_timeout           60s;
client_body_timeout             60s;
client_header_buffer_size       2k;
client_body_buffer_size         256k;
client_max_body_size            256m;
large_client_header_buffers     4 8k;
send_timeout                    60s;
keepalive_timeout               30s;
reset_timedout_connection       on;
server_tokens                   off;
server_name_in_redirect         off;
server_names_hash_max_size      512;
server_names_hash_bucket_size   512;

# Log format
log_format  main    '$remote_addr - $remote_user [$time_local] $request '
                    '"$status" $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';
log_format  bytes   '$body_bytes_sent';
#access_log          /var/log/nginx/access.log main;
access_log off;

# Mime settings
include             /etc/nginx/mime.types;
default_type        application/octet-stream;

# Compression
gzip                on;
gzip_static         on;
gzip_vary           on;
gzip_comp_level     6;
gzip_min_length     1024;
gzip_buffers        16 8k;
gzip_types          text/plain text/css text/javascript text/js text/xml application/json application/javascript application/x-javascript application/xml application/xml+rss application/x-font-ttf image/svg+xml font/opentype;
gzip_proxied        any;
gzip_disable        "MSIE [1-6]\.";

# Proxy settings
proxy_redirect      off;
proxy_set_header    Host            $host;
proxy_set_header    X-Real-IP       $remote_addr;
proxy_set_header    X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass_header   Set-Cookie;
proxy_buffers       32 4k;
proxy_connect_timeout   30s;
proxy_send_timeout  90s;
proxy_read_timeout  90s;

# Cloudflare https://www.cloudflare.com/ips
set_real_ip_from   103.21.244.0/22;
set_real_ip_from   103.22.200.0/22;
set_real_ip_from   103.31.4.0/22;
set_real_ip_from   104.16.0.0/12;
set_real_ip_from   108.162.192.0/18;
set_real_ip_from   131.0.72.0/22;
set_real_ip_from   141.101.64.0/18;
set_real_ip_from   162.158.0.0/15;
set_real_ip_from   172.64.0.0/13;
set_real_ip_from   173.245.48.0/20;
set_real_ip_from   188.114.96.0/20;
set_real_ip_from   190.93.240.0/20;
set_real_ip_from   197.234.240.0/22;
set_real_ip_from   198.41.128.0/17;
#set_real_ip_from   2400:cb00::/32;
#set_real_ip_from   2606:4700::/32;
#set_real_ip_from   2803:f800::/32;
#set_real_ip_from   2405:b500::/32;
#set_real_ip_from   2405:8100::/32;
#set_real_ip_from   2c0f:f248::/32;
#set_real_ip_from   2a06:98c0::/29;
real_ip_header     CF-Connecting-IP;

# SSL PCI Compliance
ssl_session_cache   shared:SSL:10m;
ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_ciphers        "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";

# Error pages
error_page          403          /error/403.html;
error_page          404          /error/404.html;
error_page          502 503 504  /error/50x.html;

# Cache settings
proxy_cache_path /var/cache/nginx levels=2 keys_zone=cache:10m inactive=60m max_size=1024m;
proxy_cache_key "$host$request_uri $cookie_user";
proxy_temp_path  /var/cache/nginx/temp;
proxy_ignore_headers Expires Cache-Control;
proxy_cache_use_stale error timeout invalid_header http_502;
proxy_cache_valid any 1d;

# Cache bypass
map $http_cookie $no_cache {
    default 0;
    ~SESS 1;
    ~wordpress_logged_in 1;
}

# File cache settings
open_file_cache          max=10000 inactive=30s;
open_file_cache_valid    60s;
open_file_cache_min_uses 2;
open_file_cache_errors   off;

# Wildcard include
include             /etc/nginx/conf.d/*.conf;

======================================================================================= 3. Sites available

root@buzzgeetha:/etc/nginx/sites-available# ls -l total 12 -rw-r--r-- 1 root root 2633 Aug 30 09:56 default -rw-r--r-- 1 root root 527 Aug 30 18:10 domain1.com -rw-r--r-- 1 root root 505 Aug 30 18:10 domain2.net root@buzzgeetha:/etc/nginx/sites-available#

For some reason, .vhost files were never generated.

4. Sites directory

root@buzzgeetha:/etc/nginx/sites-available# cat domain1.com server { listen 80 default_server; listen [::]:80;

    root /home/admin/web/domain1.com/public_html;
    index index.html index.htm index.nginx-debian.html;

    server_name domain1.com www.domain1.com;

##
# Nginx Bad Bot Blocker Includes
# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
##
include /etc/nginx/bots.d/ddos.conf; 
include /etc/nginx/bots.d/blockbots.conf;

    location / {
            try_files $uri $uri/ =404;
    }

} root@buzzgeetha:/etc/nginx/sites-available#

[buzzsubash]

5. Output of few IMPT config

root@buzzgeetha:/etc/nginx/conf.d# sudo nginx -V nginx version: nginx/1.21.1 built by gcc 7.5.0 (Ubuntu 7.5.0-3ubuntu1~18.04) built with OpenSSL 1.1.1 11 Sep 2018 TLS SNI support enabled configure arguments: --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx --with-compat --with-file-aio --with-threads --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_mp4_module --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-mail --with-mail_ssl_module --with-stream --with-stream_realip_module --with-stream_ssl_module --with-stream_ssl_preread_module --with-cc-opt='-g -O2 -fdebug-prefix-map=/data/builder/debuild/nginx-1.21.1/debian/debuild-base/nginx-1.21.1=. -fstack-protector-strong -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fPIC' --with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now -Wl,--as-needed -pie'

root@buzzgeetha:/etc/nginx/conf.d# ls -la /etc/nginx total 92 drwxr-xr-x 8 root root 4096 Aug 30 18:31 . drwxr-xr-x 110 root root 12288 Aug 30 21:16 .. drwxr-xr-x 2 root root 4096 Aug 30 09:56 bots.d drwxr-xr-x 2 root root 4096 Aug 30 21:21 conf.d -rw-r--r-- 1 root root 1077 Apr 6 2018 fastcgi.conf -rw-r--r-- 1 root root 1007 Jul 6 22:59 fastcgi_params -rw-r--r-- 1 root root 2837 Apr 6 2018 koi-utf -rw-r--r-- 1 root root 2223 Apr 6 2018 koi-win -rw-r--r-- 1 root root 5290 Jul 6 22:59 mime.types lrwxrwxrwx 1 root root 22 Jul 6 23:11 modules -> /usr/lib/nginx/modules drwxr-xr-x 2 root root 4096 Aug 30 17:35 modules-enabled -rw-r--r-- 1 root root 4869 Aug 30 18:31 nginx.conf -rw-r--r-- 1 root root 648 Jul 6 23:11 nginx.conf.dpkg-dist -rw-r--r-- 1 root root 180 Apr 6 2018 proxy_params -rw-r--r-- 1 root root 636 Jul 6 22:59 scgi_params drwxr-xr-x 2 root root 4096 Aug 30 21:16 sites-available drwxr-xr-x 2 root root 4096 Aug 30 18:01 sites-enabled drwxr-xr-x 2 root root 4096 Aug 30 09:54 snippets -rw-r--r-- 1 root root 664 Jul 6 22:59 uwsgi_params -rw-r--r-- 1 root root 3071 Apr 6 2018 win-utf root@buzzgeetha:/etc/nginx/conf.d# ls -la /etc/nginx/bots.d total 48 drwxr-xr-x 2 root root 4096 Aug 30 09:56 . drwxr-xr-x 8 root root 4096 Aug 30 18:31 .. -rw------- 1 root root 4723 Aug 30 09:55 bad-referrer-words.conf -rw------- 1 root root 5197 Aug 30 09:56 blacklist-ips.conf -rw------- 1 root root 3954 Aug 30 09:56 blacklist-user-agents.conf -rw------- 1 root root 3623 Aug 30 09:55 blockbots.conf -rw------- 1 root root 2971 Aug 30 09:55 custom-bad-referrers.conf -rw------- 1 root root 1748 Aug 30 09:55 ddos.conf -rw------- 1 root root 2974 Aug 30 09:56 whitelist-domains.conf -rw------- 1 root root 2292 Aug 30 09:56 whitelist-ips.conf root@buzzgeetha:/etc/nginx/conf.d# ls -ls /etc/nginx/conf.d total 540 4 -rw-r--r-- 1 root root 200 Aug 30 09:18 x.x.x.x.conf <<<<<<<< IP of server 4 -rw------- 1 root root 1914 Aug 30 18:41 botblocker-nginx-settings.conf 512 -rw------- 1 root root 523098 Aug 30 18:17 globalblacklist.conf 4 -rw-r--r-- 1 root root 482 Aug 2 20:55 phpmyadmin.inc 4 -rw-r--r-- 1 root root 306 Aug 2 20:55 phppgadmin.inc 4 -rw-r--r-- 1 root root 178 Aug 2 20:55 status.conf 4 -rw-r--r-- 1 root root 344 Aug 30 21:21 vesta.conf 4 -rw-r--r-- 1 root root 358 Aug 2 20:55 webmail.inc root@buzzgeetha:/etc/nginx/conf.d#

[buzzsubash]

Copy of vhost / website / host .conf file

/etc/nginx/conf.d# cat x.x.x.x.conf server { listen x.x.x.x:80 default; servername ;

access_log /var/log/nginx/x.x.x.x.log main;

location / {
    proxy_pass  http://x.x.x.x:8080;

} }

x.x.x.x refers to the host IP

========================================

Ubuntu

uname -a 4.15.0 #1 SMP Tue Aug 25 11:59:26 MSK 2020 x86_64 x86_64 x86_64 GNU/Linux

[buzzsubash]

root@buzzgeetha:/etc/nginx/conf.d# cat vesta.conf

include /home/admin/conf/web/domain1.com.nginx.conf; include /home/admin/conf/web/domain1.com.nginx.ssl.conf; include /home/admin/conf/web/assets.domain1.com.nginx.conf; include /home/admin/conf/web/domain2.net.nginx.conf; include /home/admin/conf/web/assets.domain2.net.nginx.conf; include /home/admin/conf/web/assets.domain1.com.nginx.ssl.conf; root@buzzgeetha:/etc/nginx/conf.d#

[buzzsubash]

This has been sorted out, I never ran the ./update after adding second domain. All good now.

[buzzsubash]

27 cd /home/admin/conf/web/ 28 ls -l 29 cd /etc/nginx/conf.d 30 cat vesta.conf 31 sudo wget https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/install-ngxblocker -O /usr/local/sbin/install-ngxblocker 32 sudo chmod +x /usr/local/sbin/install-ngxblocker 33 cd /usr/local/sbin 34 sudo ./install-ngxblocker 35 cd /usr/local/sbin/ 36 sudo ./install-ngxblocker -x 37 sudo chmod +x /usr/local/sbin/setup-ngxblocker 38 sudo chmod +x /usr/local/sbin/update-ngxblocker 39 cd /usr/local/sbin/ 40 sudo ./setup-ngxblocker 41 cd /usr/local/sbin/ 42 ls /home/admin/conf/web/ 43 sudo ./setup-ngxblocker -x -e conf -v /home/admin/conf/web/ 44 sudo nginx -t 45 sudo service nginx restart 46 sudo crontab -e 47 ./setup-ngxblocker -v /home/admin/conf/web/ 48 ./setup-ngxblocker -x -v /home/admin/conf/web/ 49 reboot 50 timedatectl