boniek83 commented 8 years ago

Threads stacktraces:

14893

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f8514 in ngx_http_push_stream_cleanup_request_context ()

3 0x00000000004fde3b in ngx_http_push_stream_websocket_reading ()

4 0x0000000000485f0f in ngx_http_request_handler ()

5 0x0000000000471ddc in ngx_epoll_process_events ()

6 0x0000000000469783 in ngx_process_events_and_timers ()

7 0x000000000046fef1 in ngx_worker_process_cycle ()

8 0x000000000046e909 in ngx_spawn_process ()

9 0x00000000004700c4 in ngx_start_worker_processes ()

10 0x0000000000470b64 in ngx_master_process_cycle ()

11 0x000000000044cbb7 in main ()

14894

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14895

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14896

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14897

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f30a9 in ngx_http_push_stream_throw_the_message_away ()

3 0x00000000004f61e8 in ngx_http_push_stream_ensure_qtd_of_messages.part.18 ()

4 0x00000000004f821f in ngx_http_push_stream_add_msg_to_channel ()

5 0x00000000004fb37e in ngx_http_push_stream_publisher_body_handler ()

6 0x000000000048edb0 in ngx_http_read_client_request_body ()

7 0x00000000004fcc6e in ngx_http_push_stream_publisher_handler ()

8 0x000000000048277e in ngx_http_core_content_phase ()

9 0x000000000047cd85 in ngx_http_core_run_phases ()

10 0x0000000000487e1f in ngx_http_process_request ()

11 0x000000000048868e in ngx_http_process_request_line ()

12 0x0000000000469c00 in ngx_event_process_posted ()

13 0x000000000046fef1 in ngx_worker_process_cycle ()

14 0x000000000046e909 in ngx_spawn_process ()

15 0x00000000004700c4 in ngx_start_worker_processes ()

16 0x0000000000470b64 in ngx_master_process_cycle ()

17 0x000000000044cbb7 in main ()

14898

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f30a9 in ngx_http_push_stream_throw_the_message_away ()

3 0x00000000004f61e8 in ngx_http_push_stream_ensure_qtd_of_messages.part.18 ()

4 0x00000000004f821f in ngx_http_push_stream_add_msg_to_channel ()

5 0x00000000004fb37e in ngx_http_push_stream_publisher_body_handler ()

6 0x000000000048edb0 in ngx_http_read_client_request_body ()

7 0x00000000004fcc6e in ngx_http_push_stream_publisher_handler ()

8 0x000000000048277e in ngx_http_core_content_phase ()

9 0x000000000047cd85 in ngx_http_core_run_phases ()

10 0x0000000000487e1f in ngx_http_process_request ()

11 0x000000000048868e in ngx_http_process_request_line ()

12 0x0000000000471ddc in ngx_epoll_process_events ()

13 0x0000000000469783 in ngx_process_events_and_timers ()

14 0x000000000046fef1 in ngx_worker_process_cycle ()

15 0x000000000046e909 in ngx_spawn_process ()

16 0x00000000004700c4 in ngx_start_worker_processes ()

17 0x0000000000470b64 in ngx_master_process_cycle ()

18 0x000000000044cbb7 in main ()

14899

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f8514 in ngx_http_push_stream_cleanup_request_context ()

3 0x00000000004fde3b in ngx_http_push_stream_websocket_reading ()

4 0x0000000000485f0f in ngx_http_request_handler ()

5 0x0000000000471ddc in ngx_epoll_process_events ()

6 0x0000000000469783 in ngx_process_events_and_timers ()

7 0x000000000046fef1 in ngx_worker_process_cycle ()

8 0x000000000046e909 in ngx_spawn_process ()

9 0x00000000004700c4 in ngx_start_worker_processes ()

10 0x0000000000470b64 in ngx_master_process_cycle ()

11 0x000000000044cbb7 in main ()

14900

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14901

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14902

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f30a9 in ngx_http_push_stream_throw_the_message_away ()

3 0x00000000004f61e8 in ngx_http_push_stream_ensure_qtd_of_messages.part.18 ()

4 0x00000000004f821f in ngx_http_push_stream_add_msg_to_channel ()

5 0x00000000004fb37e in ngx_http_push_stream_publisher_body_handler ()

6 0x000000000048edb0 in ngx_http_read_client_request_body ()

7 0x00000000004fcc6e in ngx_http_push_stream_publisher_handler ()

8 0x000000000048277e in ngx_http_core_content_phase ()

9 0x000000000047cd85 in ngx_http_core_run_phases ()

10 0x0000000000487e1f in ngx_http_process_request ()

11 0x000000000048868e in ngx_http_process_request_line ()

12 0x0000000000471ddc in ngx_epoll_process_events ()

13 0x0000000000469783 in ngx_process_events_and_timers ()

14 0x000000000046fef1 in ngx_worker_process_cycle ()

15 0x000000000046e909 in ngx_spawn_process ()

16 0x00000000004700c4 in ngx_start_worker_processes ()

17 0x0000000000470b64 in ngx_master_process_cycle ()

18 0x000000000044cbb7 in main ()

14903

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14904

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14905

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14906

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14907

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f8514 in ngx_http_push_stream_cleanup_request_context ()

3 0x00000000004fde3b in ngx_http_push_stream_websocket_reading ()

4 0x0000000000485f0f in ngx_http_request_handler ()

5 0x0000000000471ddc in ngx_epoll_process_events ()

6 0x0000000000469783 in ngx_process_events_and_timers ()

7 0x000000000046fef1 in ngx_worker_process_cycle ()

8 0x000000000046e909 in ngx_spawn_process ()

9 0x00000000004700c4 in ngx_start_worker_processes ()

10 0x0000000000470b64 in ngx_master_process_cycle ()

11 0x000000000044cbb7 in main ()

14908

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14909

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004fbac8 in ngx_http_push_stream_free_memory_of_expired_messages_and_channels_data ()

3 0x00000000004fbc98 in ngx_http_push_stream_memory_cleanup_timer_wake_handler ()

4 0x0000000000469b52 in ngx_event_expire_timers ()

5 0x00000000004697ab in ngx_process_events_and_timers ()

6 0x000000000046fef1 in ngx_worker_process_cycle ()

7 0x000000000046e909 in ngx_spawn_process ()

8 0x00000000004700c4 in ngx_start_worker_processes ()

9 0x0000000000470b64 in ngx_master_process_cycle ()

10 0x000000000044cbb7 in main ()

14910

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14911

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14912

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14913

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f8514 in ngx_http_push_stream_cleanup_request_context ()

3 0x00000000004fde3b in ngx_http_push_stream_websocket_reading ()

4 0x0000000000485f0f in ngx_http_request_handler ()

5 0x0000000000471ddc in ngx_epoll_process_events ()

6 0x0000000000469783 in ngx_process_events_and_timers ()

7 0x000000000046fef1 in ngx_worker_process_cycle ()

8 0x000000000046e909 in ngx_spawn_process ()

9 0x00000000004700c4 in ngx_start_worker_processes ()

10 0x0000000000470b64 in ngx_master_process_cycle ()

11 0x000000000044cbb7 in main ()

14914

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14915

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

14916

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

After some time nginx hangs when using pushstream module. Nginx version 1.10.1 pushstream module version 0.5.2.

shadeware commented 4 years ago

Sorry, I can't provide you with coredumps since they may contain confidential info.

However it was just found out that the hang happens at the same time with custom in-app garbage collector. This GC triggers on several threads simultaneously, and they all start sending DELETE requests to push-stream. Some of these requests may be duplicate (from several threads), some of them may attempt to delete non-existent channels (e.g. that existed before last crash).

wandenberg commented 4 years ago

@shadeware no problem. can you at least send me updated stacktrace of each process? (you can send directly to my email). Then I can see if the issue is due to the number of deletes in a short period, or these requests combined with other operations, ... And when it happens you have to restart the server, right? And confirm if you still use this setup latest version from master and nginx version is 1.13.3?

shadeware commented 4 years ago

can you at least send me updated stacktrace of each process

Actually nothing has been changed yet, so previous stackstraces should still be relevant, but I collected more samples: nginx-stacktraces.zip

And when it happens you have to restart the server, right?

I have to restart nginx, it's done automatically via monitoring script (stacktraces were also collected by this script).

And confirm if you still use this setup latest version from master and nginx version is 1.13.3?

Yep, nothing changed

$ strings /usr/sbin/nginx | fgrep 0.5.3
$ strings /usr/sbin/nginx | fgrep 0.5.4
0.5.4

$ nginx -V
nginx version: nginx/1.13.3
built with OpenSSL 1.0.1f 6 Jan 2014
TLS SNI support enabled
configure arguments: --with-cc-opt='-g -O2 -fPIE -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -D_FORTIFY_SOURCE=2' --with-ld-opt='-Wl,-Bsymbolic-functions -fPIE -pie -Wl,-z,relro -Wl,-z,now' --prefix=/usr/share/nginx --conf-path=/etc/nginx/nginx.conf --http-log-path=/var/log/nginx/access.log --error-log-path=/var/log/nginx/error.log --lock-path=/var/lock/nginx.lock --pid-path=/run/nginx.pid --http-client-body-temp-path=/var/lib/nginx/body --http-fastcgi-temp-path=/var/lib/nginx/fastcgi --http-proxy-temp-path=/var/lib/nginx/proxy --http-scgi-temp-path=/var/lib/nginx/scgi --http-uwsgi-temp-path=/var/lib/nginx/uwsgi --with-debug --with-pcre-jit --with-ipv6 --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-http_auth_request_module --with-http_addition_module --with-http_dav_module --with-http_geoip_module --with-http_gzip_static_module --with-http_image_filter_module  --add-module=/home/packages/nginx/nginx/debian/modules/../../../nginx-push-stream-module

wandenberg commented 4 years ago

Hi @shadeware, sorry for the delay. I am preparing a branch with the fix, would you be able to test it for me? Another question to confirm a suspicious, is your application connecting to multiple channels on the same long polling connection?

shadeware commented 4 years ago

would you be able to test it for me?

I'm afraid I won't. 1 week ago I fixed this problem by disabling all DELETE requests to pushstream, and it didn't hang once since that time. I can install new version just in case though.

Another question to confirm a suspicious, is your application connecting to multiple channels on the same long polling connection?

Yes, it does.

wandenberg commented 4 years ago

"Perfect". My thought is correct. I will complete the change and ask you to test if possible. The issue is due to the fact that different channels can use the same "lock". While deleting a channel we remove the long polling connection from all channels it is subscribed, and the way is done now, it might require to use the same "lock" twice, which causes a deadlock. If it is easy for you to reenable the delete function for a few minutes while testing the version I would appreciate it.

wandenberg commented 4 years ago

@shadeware here is the branch with the fix. Let me know when you start to test it and the result

wandenberg / nginx-push-stream-module

Nginx hangs #245

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f8514 in ngx_http_push_stream_cleanup_request_context ()

3 0x00000000004fde3b in ngx_http_push_stream_websocket_reading ()

4 0x0000000000485f0f in ngx_http_request_handler ()

5 0x0000000000471ddc in ngx_epoll_process_events ()

6 0x0000000000469783 in ngx_process_events_and_timers ()

7 0x000000000046fef1 in ngx_worker_process_cycle ()

8 0x000000000046e909 in ngx_spawn_process ()

9 0x00000000004700c4 in ngx_start_worker_processes ()

10 0x0000000000470b64 in ngx_master_process_cycle ()

11 0x000000000044cbb7 in main ()

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f9c09 in ngx_http_push_stream_send_old_messages ()

3 0x00000000004f9eab in ngx_http_push_stream_subscriber_assign_channel.isra.26 ()

4 0x00000000004fda07 in ngx_http_push_stream_websocket_handler ()

5 0x000000000048277e in ngx_http_core_content_phase ()

6 0x000000000047cd85 in ngx_http_core_run_phases ()

7 0x0000000000487e1f in ngx_http_process_request ()

8 0x000000000048868e in ngx_http_process_request_line ()

9 0x0000000000469c00 in ngx_event_process_posted ()

10 0x000000000046fef1 in ngx_worker_process_cycle ()

11 0x000000000046e909 in ngx_spawn_process ()

12 0x00000000004700c4 in ngx_start_worker_processes ()

13 0x0000000000470b64 in ngx_master_process_cycle ()

14 0x000000000044cbb7 in main ()

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f30a9 in ngx_http_push_stream_throw_the_message_away ()

3 0x00000000004f61e8 in ngx_http_push_stream_ensure_qtd_of_messages.part.18 ()

4 0x00000000004f821f in ngx_http_push_stream_add_msg_to_channel ()

5 0x00000000004fb37e in ngx_http_push_stream_publisher_body_handler ()

6 0x000000000048edb0 in ngx_http_read_client_request_body ()

7 0x00000000004fcc6e in ngx_http_push_stream_publisher_handler ()

8 0x000000000048277e in ngx_http_core_content_phase ()

9 0x000000000047cd85 in ngx_http_core_run_phases ()

10 0x0000000000487e1f in ngx_http_process_request ()

11 0x000000000048868e in ngx_http_process_request_line ()

12 0x0000000000469c00 in ngx_event_process_posted ()

13 0x000000000046fef1 in ngx_worker_process_cycle ()

14 0x000000000046e909 in ngx_spawn_process ()

15 0x00000000004700c4 in ngx_start_worker_processes ()

16 0x0000000000470b64 in ngx_master_process_cycle ()

17 0x000000000044cbb7 in main ()

0 0x00007f31b01ad790 in sem_wait () from /lib64/libpthread.so.0

1 0x000000000045a1b8 in ngx_shmtx_lock ()

2 0x00000000004f30a9 in ngx_http_push_stream_throw_the_message_away ()