search

unbit / uwsgi

uWSGI application server container
http://projects.unbit.it/uwsgi
Other
3.45k stars 688 forks source link

upgrading from 2.0.24 to 2.0.25.1 results in slow chain reload #2661

Open jjanyan opened 1 month ago

jjanyan commented 1 month ago

When chain reloading on 2.0.24 each process would stop and reload in 1-2 seconds. After upgrading it takes 60+ seconds.

uwsgi config ``` [uwsgi] ini = :standard processes=16 threads=4 wsgi-file = /var/www/cb/app/main.wsgi [standard] stats=127.0.0.1:7000 strict=true stats-http # distribute the load evenly. hope it helps thunder-lock buffer-size=32768 callable = application chdir=/var/www/cb/app disable-write-exception = true enable-threads = true http11-socket=127.0.0.1:5000 auto-procname = true procname-prefix = cbapp- ignore-sigpipe = true ignore-write-errors = true master=True lazy-apps=true touch-chain-reload=%d%s # deletes tmp files and sockets vacuum = 1 # drop out of root while running uid=www-data gid=www-data ```
old strace ``` 14:44:37.795616 epoll_wait(4, 0x7ffc96b94f8c, 1, -1) = -1 EINTR (Interrupted system call) 14:45:32.670559 --- SIGHUP {si_signo=SIGHUP, si_code=SI_USER, si_pid=512, si_uid=1000} --- 14:45:32.670608 write(2, "Gracefully killing worker 1 (pid"..., 42) = 42 14:45:32.678796 getpid() = 721 14:45:32.678900 tgkill(721, 1558, SIGRTMIN) = 0 14:45:32.678947 getpid() = 721 14:45:32.678971 tgkill(721, 1559, SIGRTMIN) = 0 14:45:32.679000 getpid() = 721 14:45:32.679022 tgkill(721, 1560, SIGRTMIN) = 0 14:45:32.679060 futex(0x7f13efb4a9d0, FUTEX_WAIT, 1558, NULL) = 0 14:45:32.694681 getpid() = 721 14:45:32.705369 getpid() = 721 14:45:32.705510 getpid() = 721 14:45:32.705630 getpid() = 721 14:45:32.705751 getpid() = 721 14:45:32.705819 getpid() = 721 14:45:32.705875 getpid() = 721 14:45:32.705970 getpid() = 721 14:45:32.706049 getpid() = 721 14:45:32.706169 getpid() = 721 14:45:32.706242 getpid() = 721 14:45:32.706437 rt_sigaction(SIGINT, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f13f5de73c0}, {sa_handler=0x560d23c38ce0, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7f13f5de73c0}, 8) = 0 14:45:32.932182 munmap(0x7f13f2ce0000, 262144) = 0 14:45:32.932458 munmap(0x7f13f2d20000, 262144) = 0 14:45:32.932652 munmap(0x7f13f2d60000, 262144) = 0 14:45:32.932859 munmap(0x7f13f2da0000, 262144) = 0 14:45:32.933062 munmap(0x7f13f2de0000, 262144) = 0 14:45:32.933265 munmap(0x7f13f2e20000, 262144) = 0 14:45:32.933491 munmap(0x7f13f2e60000, 262144) = 0 14:45:32.933716 munmap(0x7f13f2ea0000, 262144) = 0 14:45:32.933962 munmap(0x7f13f2ee0000, 262144) = 0 14:45:32.934190 munmap(0x7f13f2f20000, 262144) = 0 14:45:32.934414 munmap(0x7f13f3008000, 262144) = 0 14:45:32.934629 munmap(0x7f13f30a0000, 262144) = 0 14:45:32.936106 munmap(0x7f13e6072000, 262144) = 0 14:45:33.004145 munmap(0x7f13ea56c000, 2121784) = 0 14:45:33.005647 munmap(0x7f13ea368000, 2109520) = 0 14:45:33.006954 munmap(0x7f13f346b000, 262144) = 0 14:45:33.009365 munmap(0x7f13eadba000, 2101288) = 0 14:45:33.058559 munmap(0x7f13e626a000, 98624) = 0 14:45:33.058705 munmap(0x7f13e6283000, 288392) = 0 14:45:33.058849 munmap(0x7f13e6507000, 320176) = 0 14:45:33.058942 munmap(0x7f13e63f8000, 1064160) = 0 14:45:33.059023 munmap(0x7f13e6302000, 180408) = 0 14:45:33.059101 munmap(0x7f13e632f000, 778256) = 0 14:45:33.059170 munmap(0x7f13e62ca000, 225296) = 0 14:45:33.059250 munmap(0x7f13e64fc000, 43152) = 0 14:45:33.059344 munmap(0x7f13e63ee000, 36880) = 0 14:45:33.059410 munmap(0x7f13e67f5000, 114904) = 0 14:45:33.064186 munmap(0x7f13ec257000, 2101288) = 0 14:45:33.065172 munmap(0x7f13ec054000, 2105408) = 0 14:45:33.065300 munmap(0x7f13ebe51000, 2105424) = 0 14:45:33.065410 munmap(0x7f13ebc4f000, 2101320) = 0 14:45:33.065640 munmap(0x7f13ebc0f000, 262144) = 0 14:45:33.065799 munmap(0x7f13eb9cc000, 2105432) = 0 14:45:33.066870 munmap(0x7f13ea773000, 2105416) = 0 14:45:33.067328 munmap(0x7f13ea976000, 2101304) = 0 14:45:33.067453 munmap(0x7f13eab78000, 2101304) = 0 14:45:33.067582 munmap(0x7f13eafbc000, 2105416) = 0 14:45:33.067708 munmap(0x7f13eb1bf000, 2105400) = 0 14:45:33.068717 munmap(0x7f13eb3c2000, 2105424) = 0 14:45:33.069563 munmap(0x7f13f2fe0000, 17928) = 0 14:45:33.070780 munmap(0x7f13eb5c5000, 2109520) = 0 14:45:33.070942 munmap(0x7f13eb7c9000, 2105408) = 0 14:45:33.075051 sigaltstack(NULL, {ss_sp=0x560d241e7bc0, ss_flags=0, ss_size=16384}) = 0 14:45:33.075220 sigaltstack({ss_sp=NULL, ss_flags=SS_DISABLE, ss_size=0}, NULL) = 0 14:45:33.075484 getpid() = 721 14:45:33.075581 getpid() = 721 14:45:33.075680 getpid() = 721 14:45:33.093044 munmap(0x7f13ed34a000, 33554432) = 0 14:45:33.359632 exit_group(17) = ? 14:45:33.365747 +++ exited with 17 +++ ```
new strace ``` 14:48:05.356082 epoll_wait(4, 0x7ffcd7e33ddc, 1, -1) = -1 EINTR (Interrupted system call) 14:48:33.777746 --- SIGHUP {si_signo=SIGHUP, si_code=SI_USER, si_pid=317314, si_uid=1000} --- 14:48:33.777849 write(2, "Gracefully killing worker 1 (pid"..., 45) = 45 14:48:33.778048 futex(0x7fc20001b9d0, FUTEX_WAIT, 318213, NULL) = 0 14:49:23.093081 getpid() = 318184 14:49:23.098742 getpid() = 318184 14:49:23.099029 getpid() = 318184 14:49:23.099243 getpid() = 318184 14:49:23.099394 futex(0x7fc1e80ba320, FUTEX_WAKE_PRIVATE, 1) = 1 14:49:23.099580 getpid() = 318184 14:49:23.099720 getpid() = 318184 14:49:23.099869 getpid() = 318184 14:49:23.100078 getpid() = 318184 14:49:23.100357 futex(0x7fc20fcf2e74, FUTEX_WAKE_PRIVATE, 1) = 1 14:49:23.100532 futex(0x7fc20fcf2e78, FUTEX_WAKE_PRIVATE, 1) = 1 14:49:23.101716 futex(0x55a3f86e07c0, FUTEX_WAIT_BITSET_PRIVATE, 0, {tv_sec=3176029, tv_nsec=473378574}, FUTEX_BITSET_MATCH_ANY) = 0 14:49:23.131723 getpid() = 318184 14:49:23.131890 getpid() = 318184 14:49:23.131997 getpid() = 318184 14:49:23.132063 futex(0x7fc1e80ba320, FUTEX_WAKE_PRIVATE, 1) = 1 14:49:23.132255 futex(0x7fc20fcf2e70, FUTEX_WAKE_PRIVATE, 1) = 1 14:49:23.132398 futex(0x7fc20fcf2e78, FUTEX_WAKE_PRIVATE, 1) = 1 14:49:23.132605 close(7) = 0 14:49:23.133521 getsockname(8, {sa_family=AF_INET, sin_port=htons(42790), sin_addr=inet_addr("172.30.2.188")}, [16]) = 0 14:49:23.133739 getpeername(8, {sa_family=AF_INET, sin_port=htons(11211), sin_addr=inet_addr("172.30.1.171")}, [16]) = 0 14:49:23.134171 close(8) = 0 14:49:23.134386 rt_sigaction(SIGINT, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER|SA_ONSTACK, sa_restorer=0x7fc210287420}, {sa_handler=0x55a3f49fbcf0, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x7fc210287420}, 8) = 0 14:49:23.369283 munmap(0x7fc20dcb8000, 1048576) = 0 14:49:23.369836 munmap(0x7fc20ddb8000, 1048576) = 0 14:49:23.505724 munmap(0x7fc20d74d000, 32832) = 0 14:49:23.506792 munmap(0x7fc20d747000, 20568) = 0 14:49:23.512019 munmap(0x7fc20d766000, 16432) = 0 14:49:23.529880 munmap(0x7fc2008e6000, 98624) = 0 14:49:23.530110 munmap(0x7fc2008ff000, 288392) = 0 14:49:23.530206 munmap(0x7fc200b83000, 320176) = 0 14:49:23.530295 munmap(0x7fc200a74000, 1064160) = 0 14:49:23.530374 munmap(0x7fc20097e000, 180408) = 0 14:49:23.530455 munmap(0x7fc2009ab000, 778256) = 0 14:49:23.530533 munmap(0x7fc200946000, 225296) = 0 14:49:23.530630 munmap(0x7fc200b78000, 43152) = 0 14:49:23.530709 munmap(0x7fc200a6a000, 36880) = 0 14:49:23.536055 munmap(0x7fc20deea000, 16432) = 0 14:49:23.536783 munmap(0x7fc20dee5000, 16456) = 0 14:49:23.537428 munmap(0x7fc20debb000, 16472) = 0 14:49:23.537960 munmap(0x7fc20d791000, 16464) = 0 14:49:23.538908 munmap(0x7fc20d78c000, 16480) = 0 14:49:23.539735 munmap(0x7fc20d756000, 20560) = 0 14:49:23.542551 munmap(0x7fc20d75c000, 16448) = 0 14:49:23.543176 munmap(0x7fc20d761000, 16448) = 0 14:49:23.544433 munmap(0x7fc20d76b000, 16464) = 0 14:49:23.545103 munmap(0x7fc20d770000, 16448) = 0 14:49:23.546284 munmap(0x7fc20d775000, 24672) = 0 14:49:23.547537 munmap(0x7fc20d77c000, 20568) = 0 14:49:23.548474 munmap(0x7fc20d782000, 16456) = 0 14:49:23.549371 munmap(0x7fc20d787000, 16432) = 0 14:49:23.555353 close(10) = 0 14:49:23.620419 munmap(0x7fc1fd83f000, 3846144) = 0 14:49:23.621102 munmap(0x7fc20f0be000, 16384) = 0 14:49:23.621693 getpid() = 318184 14:49:23.621825 getpid() = 318184 14:49:23.621934 getpid() = 318184 14:49:23.646642 futex(0x7fc20bea5de0, FUTEX_WAKE_PRIVATE, 1) = 1 14:49:23.646811 futex(0x7fc20bea5d90, FUTEX_WAKE_PRIVATE, 1) = 1 14:49:23.647008 munmap(0x7fc207010000, 33554432) = 0 14:49:23.776661 exit_group(17) = ? 14:49:23.796898 +++ exited with 17 +++ ```

In the new strace there's

14:48:33.778048 futex(0x7fc20001b9d0, FUTEX_WAIT, 318213, NULL) = 0  # a full minute?
14:49:23.093081 getpid()                = 318184

Maybe we're doing it wrong and need to change our config somehow. Any insight would be appreciated it. In the meantime we're going to downgrade to 2.0.24