search

vozlt / nginx-module-vts

Nginx virtual host traffic status module
BSD 2-Clause "Simplified" License
3.23k stars 463 forks source link

How can I count the 5XX reqeusts numbers by Location or URI? #302

Open huangyonghome opened 1 week ago

huangyonghome commented 1 week ago

I have a prometheus rule to alert the 5XX request with nginx_server_requests metrics. But the metric only contains nginx $host information. I want to filter some URLs,Could anyone help me? Thank you very much!

the conf is below

  #开启nginx status监控
    vhost_traffic_status_zone;
    #vhost_traffic_status_filter_by_host on;
    vhost_traffic_status_filter_by_set_key $host  $uri;
    vhost_traffic_status_zone shared:vhost_traffic_status:32m;

I noticed there was a new metric nginx_filter_requests and it has three key informations(status,uri,domain) that I indeed need.

nginx_filter_requests{code="1xx", filter="/chat-assist/carrier", filterName="chat-assist-service-test1.xxx.com"

But I want to confirm that my configuration is correct，Are the statistical results of the indicators correct??

Addtions:

After running for several minites. When I running curl http://127.0.0.1:9913/metrics command. all metrics has gone away

[root@yy-t1-sre-public-nginx01 ~]# curl http://127.0.0.1:9913/metrics
# HELP nginx_vts_exporter_build_info A metric with a constant '1' value labeled by version, revision, branch, and goversion from which nginx_vts_exporter was built.
# TYPE nginx_vts_exporter_build_info gauge
nginx_vts_exporter_build_info{branch="HEAD",goversion="go1.10",revision="8aa2881c7050d9b28f2312d7ce99d93458611d04",version="0.10.3"} 1

After running for a while ,the Nginx Server CPU high usaged (4 cores).and one CPU core 100% usage with nginx error logs:

2024/10/29 15:30:51 [warn] 131018#131018: time maybe delayed, got current_msec:2940877417, shm_access_time:2940877437

When I commented the config vhost_traffic_status_filter_by_set_key $host $uri;, the CPU usage returned to original level.

u5surf commented 1 week ago

@huangyonghome Hi, Thanks submit the issue. With this nginx.conf which might imitate your configuration snippet and on which has a super tiny proxy server and upstream origin sets, but I could not reproduce the issue. We seems that the configuration might work fine at least. Of course it can be replied prometheus format response, also doesn't get higher that cpu utilization.

worker_processes  1;

events {
    worker_connections  1024;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    vhost_traffic_status_zone;
    vhost_traffic_status_filter_by_set_key $host  $uri;
    vhost_traffic_status_zone shared:vhost_traffic_status:32m;
    proxy_cache_path /var/cache/nginx keys_zone=zone1:1m max_size=1g inactive=24h;

    sendfile        on;
    keepalive_timeout  65;
    upstream backend {
        zone backend 1M;
        server localhost:8082;
    }
    # server
    server {
        listen 8081;
        location / {
             proxy_cache zone1;
             proxy_pass http://backend;
             access_log /usr/local/nginx/logs/access.log;
        }
    }
    # upstream origin
    server {
        root html;
        listen 8082;
        location / {
            index index.html index.html;
        }
    }
    # metric
    server {
       listen 9913;
       location /status {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format prometheus;
            vhost_traffic_status_bypass_stats on;
        }
    }
}
% curl localhost:9913/status
# HELP nginx_vts_info Nginx info
# TYPE nginx_vts_info gauge
nginx_vts_info{hostname="yugonomac-mini.local",module_version="v0.2.2",version="1.27.3"} 1
# HELP nginx_vts_start_time_seconds Nginx start time
# TYPE nginx_vts_start_time_seconds gauge
nginx_vts_start_time_seconds 1730373628.392
# HELP nginx_vts_main_connections Nginx connections
# TYPE nginx_vts_main_connections gauge
nginx_vts_main_connections{status="accepted"} 4
nginx_vts_main_connections{status="active"} 1
nginx_vts_main_connections{status="handled"} 4
nginx_vts_main_connections{status="reading"} 0
nginx_vts_main_connections{status="requests"} 4
nginx_vts_main_connections{status="waiting"} 0
nginx_vts_main_connections{status="writing"} 1
# HELP nginx_vts_main_shm_usage_bytes Shared memory [vhost_traffic_status] info
# TYPE nginx_vts_main_shm_usage_bytes gauge
nginx_vts_main_shm_usage_bytes{shared="max_size"} 33554432
nginx_vts_main_shm_usage_bytes{shared="used_size"} 7052
nginx_vts_main_shm_usage_bytes{shared="used_node"} 2
# HELP nginx_vts_server_bytes_total The request/response bytes
# TYPE nginx_vts_server_bytes_total counter
# HELP nginx_vts_server_requests_total The requests counter
# TYPE nginx_vts_server_requests_total counter
# HELP nginx_vts_server_request_seconds_total The request processing time in seconds
# TYPE nginx_vts_server_request_seconds_total counter
# HELP nginx_vts_server_request_seconds The average of request processing times in seconds
# TYPE nginx_vts_server_request_seconds gauge
# HELP nginx_vts_server_request_duration_seconds The histogram of request processing time
# TYPE nginx_vts_server_request_duration_seconds histogram
# HELP nginx_vts_server_cache_total The requests cache counter
# TYPE nginx_vts_server_cache_total counter
nginx_vts_server_bytes_total{host="_",direction="in"} 77
nginx_vts_server_bytes_total{host="_",direction="out"} 853
nginx_vts_server_requests_total{host="_",code="1xx"} 0
nginx_vts_server_requests_total{host="_",code="2xx"} 1
nginx_vts_server_requests_total{host="_",code="3xx"} 0
nginx_vts_server_requests_total{host="_",code="4xx"} 0
nginx_vts_server_requests_total{host="_",code="5xx"} 0
nginx_vts_server_request_seconds_total{host="_"} 0.000
nginx_vts_server_request_seconds{host="_"} 0.000
nginx_vts_server_cache_total{host="_",status="miss"} 0
nginx_vts_server_cache_total{host="_",status="bypass"} 0
nginx_vts_server_cache_total{host="_",status="expired"} 0
nginx_vts_server_cache_total{host="_",status="stale"} 0
nginx_vts_server_cache_total{host="_",status="updating"} 0
nginx_vts_server_cache_total{host="_",status="revalidated"} 0
nginx_vts_server_cache_total{host="_",status="hit"} 0
nginx_vts_server_cache_total{host="_",status="scarce"} 0
nginx_vts_server_bytes_total{host="*",direction="in"} 77
nginx_vts_server_bytes_total{host="*",direction="out"} 853
nginx_vts_server_requests_total{host="*",code="1xx"} 0
nginx_vts_server_requests_total{host="*",code="2xx"} 1
nginx_vts_server_requests_total{host="*",code="3xx"} 0
nginx_vts_server_requests_total{host="*",code="4xx"} 0
nginx_vts_server_requests_total{host="*",code="5xx"} 0
nginx_vts_server_request_seconds_total{host="*"} 0.000
nginx_vts_server_request_seconds{host="*"} 0.000
nginx_vts_server_cache_total{host="*",status="miss"} 0
nginx_vts_server_cache_total{host="*",status="bypass"} 0
nginx_vts_server_cache_total{host="*",status="expired"} 0
nginx_vts_server_cache_total{host="*",status="stale"} 0
nginx_vts_server_cache_total{host="*",status="updating"} 0
nginx_vts_server_cache_total{host="*",status="revalidated"} 0
nginx_vts_server_cache_total{host="*",status="hit"} 0
nginx_vts_server_cache_total{host="*",status="scarce"} 0
# HELP nginx_vts_filter_bytes_total The request/response bytes
# TYPE nginx_vts_filter_bytes_total counter
# HELP nginx_vts_filter_requests_total The requests counter
# TYPE nginx_vts_filter_requests_total counter
# HELP nginx_vts_filter_request_seconds_total The request processing time in seconds counter
# TYPE nginx_vts_filter_request_seconds_total counter
# HELP nginx_vts_filter_request_seconds The average of request processing times in seconds
# TYPE nginx_vts_filter_request_seconds gauge
# HELP nginx_vts_filter_request_duration_seconds The histogram of request processing time
# TYPE nginx_vts_filter_request_duration_seconds histogram
# HELP nginx_vts_filter_cache_total The requests cache counter
# TYPE nginx_vts_filter_cache_total counter
nginx_vts_filter_bytes_total{filter="/index.html",filter_name="localhost",direction="in"} 77
nginx_vts_filter_bytes_total{filter="/index.html",filter_name="localhost",direction="out"} 853
nginx_vts_filter_requests_total{filter="/index.html",filter_name="localhost",code="1xx"} 0
nginx_vts_filter_requests_total{filter="/index.html",filter_name="localhost",code="2xx"} 1
nginx_vts_filter_requests_total{filter="/index.html",filter_name="localhost",code="3xx"} 0
nginx_vts_filter_requests_total{filter="/index.html",filter_name="localhost",code="4xx"} 0
nginx_vts_filter_requests_total{filter="/index.html",filter_name="localhost",code="5xx"} 0
nginx_vts_filter_request_seconds_total{filter="/index.html",filter_name="localhost"} 0.000
nginx_vts_filter_request_seconds{filter="/index.html",filter_name="localhost"} 0.000
nginx_vts_filter_cache_total{filter="/index.html",filter_name="localhost",status="miss"} 0
nginx_vts_filter_cache_total{filter="/index.html",filter_name="localhost",status="bypass"} 0
nginx_vts_filter_cache_total{filter="/index.html",filter_name="localhost",status="expired"} 0
nginx_vts_filter_cache_total{filter="/index.html",filter_name="localhost",status="stale"} 0
nginx_vts_filter_cache_total{filter="/index.html",filter_name="localhost",status="updating"} 0
nginx_vts_filter_cache_total{filter="/index.html",filter_name="localhost",status="revalidated"} 0
nginx_vts_filter_cache_total{filter="/index.html",filter_name="localhost",status="hit"} 0
nginx_vts_filter_cache_total{filter="/index.html",filter_name="localhost",status="scarce"} 0

I come up with the possibility of this issue, the pattern of url(path) might have been quite huge, this your computer resource consume massively.