Open Lewisyixin opened 1 month ago
I have seen similar issue saying that this bug is fixed in 3.2TLS, but I can still reproduce in 3.2.2. https://github.com/apache/apisix/issues/10554
anyone can help?
Hi @Lewisyixin
Does this issue recur stably on version 3.2.2? Is it convenient to provide the configuration of the recurrence problem and the related logs?
HI @zhoujiexiong this issue is easy to reproduce. I got a clean vm and just do 1. yum 3.2.2 version apisix 2. config etcd 3. init 4. start 5.reload and seems no error logs(all logs are here after I start and reload)
2024/08/19 15:37:25 [warn] 14425#14425: 1 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14426#14426: 2 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14431#14431: 4 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14428#14428: 5 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14427#14427: 3 [lua] plugin.lua:202: load(): new plugins: {"limit-conn":true,"limit-count":true,"cas-auth":true,"syslog":true,"cors":true,"proxy-rewrite":true,"csrf":true,"mocking":true,"prometheus":true,"hmac-auth":true,"wolf-rbac":true,"fault-injection":true,"authz-keycloak":true,"response-rewrite":true,"authz-casdoor":true,"jwt-auth":true,"consumer-restriction":true,"ua-restriction":true,"grpc-transcode":true,"kafka-logger":true,"sls-logger":true,"degraphql":true,"openwhisk":true,"datadog":true,"server-info":true,"splunk-hec-logging":true,"serverless-post-function":true,"request-validation":true,"real-ip":true,"inspect":true,"public-api":true,"google-cloud-logging":true,"ai":true,"forward-auth":true,"basic-auth":true,"ldap-auth":true,"grpc-web":true,"rocketmq-logger":true,"ext-plugin-post-resp":true,"ext-plugin-post-req":true,"limit-req":true,"openfunction":true,"key-auth":true,"azure-functions":true,"body-transformer":true,"ext-plugin-pre-req":true,"elasticsearch-logger":true,"udp-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"request-id":true,"authz-casbin":true,"clickhouse-logger":true,"file-logger":true,"aws-lambda":true,"uri-blocker":true,"redirect":true,"openid-connect":true,"tcp-logger":true,"workflow":true,"zipkin":true,"client-control":true,"http-logger":true,"proxy-mirror":true,"proxy-cache":true,"gzip":true,"proxy-control":true,"api-breaker":true,"serverless-pre-function":true,"kafka-proxy":true,"loggly":true,"traffic-split":true,"referer-restriction":true,"skywalking-logger":true,"opa":true,"ip-restriction":true,"echo":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14431#14431: 4 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14426#14426: 2 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14428#14428: 5 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14425#14425: 1 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14426#14426: 21 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:25 [warn] 14431#14431: 34 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:25 [warn] 14428#14428: 53 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:25 [warn] 14425#14425: 75 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:25 [warn] 14427#14427: 3 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"mqtt-proxy":true,"syslog":true,"limit-conn":true}, context: init_worker_by_lua 2024/08/19 15:37:25 [warn] 14427#14427: 92 [lua] config_etcd.lua:168: main etcd watcher started, revision=16, context: ngx.timer 2024/08/19 15:37:30 [notice] 14506#14506: signal process started 2024/08/19 15:37:30 [emerg] 14513#14513: bind() to 127.0.0.1:9091 failed (98: Address already in use) 2024/08/19 15:37:30 [warn] 14508#14508: 452 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14507#14507: 453 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14509#14509: 454 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14510#14510: 455 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14508#14508: 452 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14507#14507: 453 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14508#14508: 474 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer 2024/08/19 15:37:30 [warn] 14507#14507: 495 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer 2024/08/19 15:37:30 [warn] 14510#14510: 455 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14509#14509: 454 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14510#14510: 532 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer 2024/08/19 15:37:30 [warn] 14509#14509: 549 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer 2024/08/19 15:37:30 [warn] 14513#14513: 584 [lua] plugin.lua:202: load(): new plugins: {"real-ip":true,"tcp-logger":true,"key-auth":true,"openid-connect":true,"csrf":true,"ai":true,"basic-auth":true,"ext-plugin-post-resp":true,"limit-count":true,"consumer-restriction":true,"loggly":true,"proxy-control":true,"uri-blocker":true,"kafka-proxy":true,"serverless-post-function":true,"inspect":true,"datadog":true,"grpc-transcode":true,"mocking":true,"ldap-auth":true,"authz-casdoor":true,"splunk-hec-logging":true,"google-cloud-logging":true,"proxy-rewrite":true,"public-api":true,"rocketmq-logger":true,"response-rewrite":true,"authz-keycloak":true,"traffic-split":true,"forward-auth":true,"ext-plugin-pre-req":true,"ip-restriction":true,"jwt-auth":true,"request-id":true,"api-breaker":true,"openfunction":true,"redirect":true,"grpc-web":true,"ext-plugin-post-req":true,"openwhisk":true,"azure-functions":true,"sls-logger":true,"http-logger":true,"example-plugin":true,"tencent-cloud-cls":true,"client-control":true,"request-validation":true,"clickhouse-logger":true,"file-logger":true,"gzip":true,"body-transformer":true,"workflow":true,"cas-auth":true,"proxy-cache":true,"prometheus":true,"serverless-pre-function":true,"syslog":true,"hmac-auth":true,"referer-restriction":true,"limit-req":true,"opa":true,"skywalking-logger":true,"degraphql":true,"limit-conn":true,"proxy-mirror":true,"zipkin":true,"wolf-rbac":true,"cors":true,"udp-logger":true,"authz-casbin":true,"ua-restriction":true,"kafka-logger":true,"server-info":true,"fault-injection":true,"elasticsearch-logger":true,"echo":true,"aws-lambda":true}, context: init_worker_by_lua 2024/08/19 15:37:30 [warn] 14513#14513: 584 [lua] plugin.lua:252: load_stream(): new plugins: {"ip-restriction":true,"limit-conn":true,"syslog":true,"mqtt-proxy":true}, context: init_worker_by_lua 2024/08/19 15:37:31 [warn] 14513#14513: 598 [lua] config_etcd.lua:168: main etcd watcher started, revision=17, context: ngx.timer
HI @zhoujiexiong any suggestions? Do I have to upgrade my apisix version?
HI @zhoujiexiong any suggestions? Do I have to upgrade my apisix version?
Hi @Lewisyixin
Could you compare it with the latest version? :)
Current Behavior
When a new deployed apisix reload, shutting down worker process will not disappear.
Expected Behavior
no shutting down worker process
Error Logs
No response
Steps to Reproduce
Environment
apisix version
): 3.2.2uname -a
): 3.10.0-514.26.2.el7.x86_64openresty -V
ornginx -V
): 1.21.4.2 :