shreemaan-abhishek
commented
7 months ago please use english to communicate and provide detailed reproduction steps.
Open ShenFeng312 opened 7 months ago
shreemaan-abhishek
commented
7 months ago please use english to communicate and provide detailed reproduction steps.
ShenFeng312
commented
6 months ago my nginx config
# Configuration File - Nginx Server Configs
# This is a read-only file, do not try to modify it.
master_process on;
worker_processes auto;
worker_cpu_affinity auto;
# main configuration snippet starts
# main configuration snippet ends
error_log logs/error.log warn;
pid logs/nginx.pid;
worker_rlimit_nofile 20480;
events {
accept_mutex off;
worker_connections 10620;
}
worker_rlimit_core 16G;
worker_shutdown_timeout 240s;
env APISIX_PROFILE;
env PATH; # for searching external plugin runner's binary
lua {
}
http {
# put extra_lua_path in front of the builtin path
# so user can override the source code
lua_package_path "$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;/usr/local/apisix/?.lua;/usr/local/apisix/?/init.lua;;./?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;/usr/local/openresty/luajit/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/?/init.lua;;";
lua_package_cpath "$prefix/deps/lib64/lua/5.1/?.so;$prefix/deps/lib/lua/5.1/?.so;;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so;";
lua_max_pending_timers 16384;
lua_max_running_timers 4096;
lua_shared_dict internal-status 10m;
lua_shared_dict upstream-healthcheck 10m;
lua_shared_dict worker-events 10m;
lua_shared_dict lrucache-lock 10m;
lua_shared_dict balancer-ewma 10m;
lua_shared_dict balancer-ewma-locks 10m;
lua_shared_dict balancer-ewma-last-touched-at 10m;
lua_shared_dict etcd-cluster-health-check 10m; # etcd health check
# for custom shared dict
lua_ssl_verify_depth 5;
ssl_session_timeout 86400;
underscores_in_headers on;
lua_socket_log_errors off;
resolver 127.0.0.11 ipv6=off;
resolver_timeout 5;
lua_http10_buffering off;
lua_regex_match_limit 100000;
lua_regex_cache_max_entries 8192;
log_format main escape=default '$remote_addr - $remote_user [$time_local] $http_host "$request" $status $body_bytes_sent $request_time "$http_referer" "$http_user_agent" $upstream_addr $upstream_status $upstream_response_time "$upstream_scheme://$upstream_host$upstream_uri"';
uninitialized_variable_warn off;
access_log logs/access.log main buffer=16384 flush=3;
open_file_cache max=1000 inactive=60;
client_max_body_size 0;
keepalive_timeout 60s;
client_header_timeout 60s;
client_body_timeout 60s;
send_timeout 10s;
variables_hash_max_size 2048;
server_tokens off;
include mime.types;
charset utf-8;
# error_page
error_page 500 @50x.html;
real_ip_header X-Real-IP;
real_ip_recursive off;
set_real_ip_from 127.0.0.1;
set_real_ip_from unix:;
# http configuration snippet starts
# http configuration snippet ends
upstream apisix_backend {
server 0.0.0.1;
keepalive 320;
keepalive_requests 1000;
keepalive_timeout 60s;
# we put the static configuration above so that we can override it in the Lua code
balancer_by_lua_block {
apisix.http_balancer_phase()
}
}
upstream apisix_dubbo_backend {
server 0.0.0.1;
balancer_by_lua_block {
apisix.http_balancer_phase()
}
# dynamical keepalive doesn't work with dubbo as the connection here
# is managed by ngx_multi_upstream_module
multi 32;
keepalive 320;
keepalive_requests 1000;
keepalive_timeout 60s;
}
apisix_delay_client_max_body_check on;
apisix_mirror_on_demand on;
init_by_lua_block {
require "resty.core"
apisix = require("apisix")
local dns_resolver = { "127.0.0.11", }
local args = {
dns_resolver = dns_resolver,
}
apisix.http_init(args)
}
init_worker_by_lua_block {
apisix.http_init_worker()
}
exit_worker_by_lua_block {
apisix.http_exit_worker()
}
server {
listen 0.0.0.0:9092;
access_log off;
location / {
content_by_lua_block {
apisix.http_control()
}
}
location @50x.html {
set $from_error_page 'true';
content_by_lua_block {
require("apisix.error_handling").handle_500()
}
}
}
server {
listen 0.0.0.0:9080 default_server reuseport;
listen 0.0.0.0:9443 ssl default_server http2 reuseport;
server_name _;
ssl_certificate cert/ssl_PLACE_HOLDER.crt;
ssl_certificate_key cert/ssl_PLACE_HOLDER.key;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 10m;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
ssl_prefer_server_ciphers on;
ssl_session_tickets off;
# http server configuration snippet starts
# http server configuration snippet ends
location = /apisix/nginx_status {
allow 127.0.0.0/24;
deny all;
access_log off;
stub_status;
}
location /apisix/admin {
set $upstream_scheme 'http';
set $upstream_host $http_host;
set $upstream_uri '';
allow 0.0.0.0/0;
deny all;
content_by_lua_block {
apisix.http_admin()
}
}
ssl_certificate_by_lua_block {
apisix.http_ssl_phase()
}
proxy_ssl_name $upstream_host;
proxy_ssl_server_name on;
location / {
set $upstream_mirror_uri '';
set $upstream_upgrade '';
set $upstream_connection '';
set $upstream_scheme 'http';
set $upstream_host $http_host;
set $upstream_uri '';
set $ctx_ref '';
set $from_error_page '';
# http server location configuration snippet starts
# http server location configuration snippet ends
set $dubbo_service_name '';
set $dubbo_service_version '';
set $dubbo_method '';
access_by_lua_block {
apisix.http_access_phase()
}
proxy_http_version 1.1;
proxy_set_header Host $upstream_host;
proxy_set_header Upgrade $upstream_upgrade;
proxy_set_header Connection $upstream_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_header Date;
### the following x-forwarded-* headers is to send to upstream server
set $var_x_forwarded_for $remote_addr;
set $var_x_forwarded_proto $scheme;
set $var_x_forwarded_host $host;
set $var_x_forwarded_port $server_port;
if ($http_x_forwarded_for != "") {
set $var_x_forwarded_for "${http_x_forwarded_for}, ${realip_remote_addr}";
}
if ($http_x_forwarded_host != "") {
set $var_x_forwarded_host $http_x_forwarded_host;
}
if ($http_x_forwarded_port != "") {
set $var_x_forwarded_port $http_x_forwarded_port;
}
proxy_set_header X-Forwarded-For $var_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $var_x_forwarded_proto;
proxy_set_header X-Forwarded-Host $var_x_forwarded_host;
proxy_set_header X-Forwarded-Port $var_x_forwarded_port;
proxy_pass $upstream_scheme://apisix_backend$upstream_uri;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
body_filter_by_lua_block {
apisix.http_body_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
location @grpc_pass {
access_by_lua_block {
apisix.grpc_access_phase()
}
grpc_set_header Content-Type application/grpc;
grpc_socket_keepalive on;
grpc_pass $upstream_scheme://apisix_backend;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
body_filter_by_lua_block {
apisix.http_body_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
location @dubbo_pass {
access_by_lua_block {
apisix.dubbo_access_phase()
}
dubbo_pass_all_headers on;
dubbo_pass_body on;
dubbo_pass $dubbo_service_name $dubbo_service_version $dubbo_method apisix_dubbo_backend;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
body_filter_by_lua_block {
apisix.http_body_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
location @50x.html {
set $from_error_page 'true';
content_by_lua_block {
require("apisix.error_handling").handle_500()
}
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
}
# http end configuration snippet starts
# http end configuration snippet ends
}please use english to communicate and provide detailed reproduction steps. @shreemaan-abhishek OK
ShenFeng312
commented
6 months ago my route
{
"uri": "/xxxxxx",
"name": "获取组件列表",
"plugins": {
"dubbo-proxy": {
"method": "invoke",
"service_name": "xxxx",
"service_version": "1.0.0"
},
"prometheus": {},
"proxy-rewrite": {
"headers": {
"dubbo-method": "xxxx",
"dubbo-service": "xxxxx",
"parameter-types-desc": "xxxxxxx"
}
}
},
"upstream_id": "490548123844739786",
"status": 1
}my upstream
{
"type": "roundrobin",
"discovery_type": "nacos",
"discovery_args": {
"namespace_id": "gateway-dev"
},
"name": "xxxx",
"service_name": "xxx"
}@shreemaan-abhishek This doesn't seem to be caused by a timeout. Because it didn't reach the default 60s timeout, and it didn't even reach 1s, but it still timed out.
ShenFeng312
commented
6 months ago I test this with https://github.com/alibaba/tengine/tree/master/modules/mod_dubbo it was OK
sheharyaar
commented
5 months ago HI @ShenFeng312 , please checkout : #10765 and https://github.com/apache/apisix/issues/10765#issuecomment-1880112639
Does this work for you ?
ShenFeng312
commented
5 months ago HI @ShenFeng312 , please checkout : #10765 and #10765 (comment)
Does this work for you ?
It did not work for me. When the request payload is very short, the request is successful in such cases. Currently, it seems that the issue only arises when the payload is too long. Additionally, if I use Tengine directly, it can return success. @sheharyaar
sheharyaar
commented
5 months ago Where are you attaching the payload ? In the body or in the headers ? Currently, only payload via headers is supported. Checkout : https://github.com/apache/apisix/pull/10822
ShenFeng312
commented
5 months ago Where are you attaching the payload ? In the body or in the headers ? Currently, only payload via headers is supported. Checkout : #10822
This is unrelated to how the payload is set. You can look at my captured data. If there's an issue with the payload, it should be a problem with the Dubbo protocol. However, this is highly unlikely. Or should I declare the HTTP response content-length in the data returned by Dubbo? @sheharyaar
sheharyaar
commented
5 months ago I will take a look at the captured packet soon.
johndudo
commented
4 months ago I have the same trouble , I add "dubbo_buffer_size 8096k;" in config.yaml like this:
'8096k' is enough for the dubbo respose body but apisix still return the error:
ShenFeng312
commented
4 months ago @johndudo maybe u can use http-dubbo plugin
johndudo
commented
4 months ago @ShenFeng312 http-dubbo plugin? where 's the plugin :)? or you mean to write a plugin 'http-dubbo' to proxy dubbo request?
ShenFeng312
commented
4 months ago @johndudo https://github.com/apache/apisix/pull/10703 here
ShenFeng312
commented
1 month ago @sheharyaar @shreemaan-abhishek @johndudo Hello, I have found more information. Please assist me in reviewing it together.
When reading data from upstream, APISIX invokes ngx_http_multi_upstream_handler, which then proceeds to ngx_http_multi_upstream_process and ultimately to ngx_http_multi_upstream_read_handler.
In ngx_http_multi_upstream_read_handler, a timeout event is added:
n = c->recv(c, b->last, b->end - b->last);
if (n == NGX_AGAIN) {
ngx_add_timer(pc->read, u->conf->read_timeout);
}The purpose of this code is to handle read timeouts. However, I cannot understand why ngx_http_multi_upstream_handler is triggered immediately after each ngx_add_timer call.
Did APISIX make any modifications to the event execution that could be causing this issue?
shreemaan-abhishek
commented
1 month ago could be due to the openresty version upgrade?
ShenFeng312
commented
1 month ago @shreemaan-abhishek The bug is caused by differences between OpenResty and Nginx.
In Nginx's ngx_http_upstream.c:
if (!u->request_body_sent) {
u->request_body_sent = 1;
if (u->header_sent) {
return;
}
ngx_add_timer(c->read, u->conf->read_timeout);
if (c->read->ready) {
ngx_http_upstream_process_header(r, u);
return;
}
}The added read_timeout is u->conf->read_timeout.
However, in OpenResty, due to the HAVE_NGX_UPSTREAM_TIMEOUT_FIELDS macro, OpenResty directly uses u->read_timeout.
But the ngx_multi_upstream_module does not account for this difference.
Current Behavior
2023/12/08 08:10:07 [warn] 45#45: 57631 multi: multi connection detach not empty 00007F45BD856FA0 2023/12/08 08:10:07 [error] 45#45: 57627 upstream timed out (110: Operation timed out) while connecting to upstream, client: 39.99.246.117, server: _, subrequest: "/app/Service/getxxxx", upstream: "dubbo://39.101.204.9:20918" 39.99.246.117 - - [08/Dec/2023:08:10:07 +0000] 39.99.253.87:30004 "POST /app/Service/getxxxx HTTP/1.1" 504 164 0.026 "-" "PostmanRuntime/7.35.0" 39.101.204.9:20918 504 0.026 "http://39.99.253.87:30004/app/Service/getxxxx"
Expected Behavior
response success
Error Logs
2023/12/08 08:10:07 [warn] 45#45: 57631 multi: multi connection detach not empty 00007F45BD856FA0 2023/12/08 08:10:07 [error] 45#45: 57627 upstream timed out (110: Operation timed out) while connecting to upstream, client: 39.99.246.117, server: _, subrequest: "/app/Service/getxxxx", upstream: "dubbo://39.101.204.9:20918" 39.99.246.117 - - [08/Dec/2023:08:10:07 +0000] 39.99.253.87:30004 "POST /app/Service/getxxxx HTTP/1.1" 504 164 0.026 "-" "PostmanRuntime/7.35.0" 39.101.204.9:20918 504 0.026 "http://39.99.253.87:30004/app/Service/getxxxx"
tcpdump info
Steps to Reproduce
Environment