search

accre / lstore

LStore - A fault-tolerant, performant distributed data storage framework.
http://www.lstore.org
Apache License 2.0
4 stars 5 forks source link

Race in gop_end_get_time #137

Open PerilousApricot opened 7 years ago

PerilousApricot commented 7 years ago 
==================
WARNING: ThreadSanitizer: data race (pid=34917)
  Read of size 8 at 0x7d740005c908 by thread T54:
    #0 gop_get_end_time /home/meloam/lstore/src/ibp/op.c:217:29 (libibp.so.0+0x00000000cbdf)
    #1 gop_write_block /home/meloam/lstore/src/ibp/op_cmd.c:470:20 (libibp.so.0+0x0000000159e8)
    #2 write_send /home/meloam/lstore/src/ibp/op_cmd.c:511:15 (libibp.so.0+0x0000000194ac)
    #3 hc_send_thread /home/meloam/lstore/src/gop/hconnection.c:320:56 (libgop.so.0+0x00000001772c)

  Previous write of size 8 at 0x7d740005c908 by thread T55 (mutexes: write M2665, write M137):
    #0 hc_recv_thread /home/meloam/lstore/src/gop/hconnection.c:485:35 (libgop.so.0+0x0000000198ce)

  Location is heap block of size 2456 at 0x7d740005c800 allocated by thread T14:
    #0 malloc /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:591 (globus-gridftp-server+0x00000044d5cf)
    #1 new_ibp_op /home/meloam/lstore/src/ibp/op.c:256:5 (libibp.so.0+0x00000000cf2b)
    #2 ibp_vec_write_gop /home/meloam/lstore/src/ibp/op.c:381:20 (libibp.so.0+0x00000000e742)
    #3 ds_ibp_writev /home/meloam/lstore/src/lio/ds/ibp.c:821:16 (liblio.so.0+0x000000029f0d)
    #4 seglun_rw_op /home/meloam/lstore/src/lio/segment/lun.c:1592:50 (liblio.so.0+0x000000187329)
    #5 seglun_rw_func /home/meloam/lstore/src/lio/segment/lun.c:1781:14 (liblio.so.0+0x00000018a12b)
    #6 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #7 gop_waitany /home/meloam/lstore/src/gop/gop.c:381:13 (libgop.so.0+0x000000010f19)
    #8 gop_waitany /home/meloam/lstore/src/gop/gop.c:364:13 (libgop.so.0+0x000000010801)
    #9 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1568:27 (liblio.so.0+0x00000014f581)
    #10 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #11 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Mutex M2665 (0x7d90012fa0a8) created at:
    #0 pthread_mutex_init /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1119 (globus-gridftp-server+0x000000433f55)
    #1 apr_thread_mutex_create /home/meloam/lstore/vendor/apr-accre/locks/unix/thread_mutex.c:71:14 (liblio.so.0+0x0000001a8b26)
    #2 create_host_connection /home/meloam/lstore/src/gop/hconnection.c:651:10 (libgop.so.0+0x00000001b827)
    #3 spawn_new_connection /home/meloam/lstore/src/gop/hportal.c:682:12 (libgop.so.0+0x0000000207e9)
    #4 check_hportal_connections /home/meloam/lstore/src/gop/hportal.c:783:9 (libgop.so.0+0x000000021914)
    #5 gop_hp_submit /home/meloam/lstore/src/gop/hportal.c:892:5 (libgop.so.0+0x00000002282c)
    #6 gop_hp_que_op_submit /home/meloam/lstore/src/gop/hportal.c:935:12 (libgop.so.0+0x000000022d8b)
    #7 _ibp_submit_op /home/meloam/lstore/src/ibp/config.c:413:9 (libibp.so.0+0x00000000719f)
    #8 _opque_start_execution /home/meloam/lstore/src/gop/opque.c:443:13 (libgop.so.0+0x0000000264b3)
    #9 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:217:9 (libgop.so.0+0x00000000efd1)
    #10 gop_waitall /home/meloam/lstore/src/gop/gop.c:427:13 (libgop.so.0+0x000000011ca3)
    #11 slun_row_replace_fix /home/meloam/lstore/src/lio/segment/lun.c:703:15 (liblio.so.0+0x000000178d09)
    #12 _seglun_grow /home/meloam/lstore/src/lio/segment/lun.c:893:21 (liblio.so.0+0x00000017bd0c)
    #13 _slun_truncate /home/meloam/lstore/src/lio/segment/lun.c:1061:15 (liblio.so.0+0x00000017e4d6)
    #14 seglun_rw_func /home/meloam/lstore/src/lio/segment/lun.c:1749:26 (liblio.so.0+0x00000018985d)
    #15 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #16 gop_waitany /home/meloam/lstore/src/gop/gop.c:381:13 (libgop.so.0+0x000000010f19)
    #17 gop_waitany /home/meloam/lstore/src/gop/gop.c:364:13 (libgop.so.0+0x000000010801)
    #18 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1568:27 (liblio.so.0+0x00000014f581)
    #19 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #20 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Mutex M137 (0x7d900000ab28) created at:
    #0 pthread_mutex_init /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1119 (globus-gridftp-server+0x000000433f55)
    #1 apr_thread_mutex_create /home/meloam/lstore/vendor/apr-accre/locks/unix/thread_mutex.c:71:14 (liblio.so.0+0x0000001a8b26)
    #2 tbx_pc_new /home/meloam/lstore/src/toolbox/pigeon_coop.c:326:29 (libtoolbox.so.0+0x00000001e9d4)
    #3 gop_init_opque_system /home/meloam/lstore/src/gop/opque.c:114:24 (libgop.so.0+0x0000000234da)
    #4 gop_construct_fn /home/meloam/lstore/src/gop/constructor.c:27:5 (libgop.so.0+0x00000000d2f5)
    #5 _dl_init_internal <null> (ld-linux-x86-64.so.2+0x00000000f3a2)
    #6 <null> <null> (libltdl.so.7+0x0000000066a1)
    #7 globus_extension_activate <null> (libglobus_common.so.0+0x000000073628)
    #8 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #9 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #10 main <null> (globus-gridftp-server+0x0000004acbc2)

  Thread T54 (tid=35026, running) created by thread T15 at:
    #0 pthread_create /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (globus-gridftp-server+0x000000425266)
    #1 create_host_connection /home/meloam/lstore/src/gop/hconnection.c:658:5 (libgop.so.0+0x00000001b983)
    #2 spawn_new_connection /home/meloam/lstore/src/gop/hportal.c:682:12 (libgop.so.0+0x0000000207e9)
    #3 check_hportal_connections /home/meloam/lstore/src/gop/hportal.c:783:9 (libgop.so.0+0x000000021914)
    #4 gop_hp_submit /home/meloam/lstore/src/gop/hportal.c:892:5 (libgop.so.0+0x00000002282c)
    #5 gop_hp_que_op_submit /home/meloam/lstore/src/gop/hportal.c:935:12 (libgop.so.0+0x000000022d8b)
    #6 _ibp_submit_op /home/meloam/lstore/src/ibp/config.c:413:9 (libibp.so.0+0x00000000719f)
    #7 _opque_start_execution /home/meloam/lstore/src/gop/opque.c:443:13 (libgop.so.0+0x0000000264b3)
    #8 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:217:9 (libgop.so.0+0x00000000efd1)
    #9 gop_waitall /home/meloam/lstore/src/gop/gop.c:427:13 (libgop.so.0+0x000000011ca3)
    #10 slun_row_replace_fix /home/meloam/lstore/src/lio/segment/lun.c:703:15 (liblio.so.0+0x000000178d09)
    #11 _seglun_grow /home/meloam/lstore/src/lio/segment/lun.c:893:21 (liblio.so.0+0x00000017bd0c)
    #12 _slun_truncate /home/meloam/lstore/src/lio/segment/lun.c:1061:15 (liblio.so.0+0x00000017e4d6)
    #13 seglun_rw_func /home/meloam/lstore/src/lio/segment/lun.c:1749:26 (liblio.so.0+0x00000018985d)
    #14 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #15 gop_waitany /home/meloam/lstore/src/gop/gop.c:381:13 (libgop.so.0+0x000000010f19)
    #16 gop_waitany /home/meloam/lstore/src/gop/gop.c:364:13 (libgop.so.0+0x000000010801)
    #17 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1568:27 (liblio.so.0+0x00000014f581)
    #18 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #19 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Thread T55 (tid=35027, running) created by thread T15 at:
    #0 pthread_create /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (globus-gridftp-server+0x000000425266)
    #1 create_host_connection /home/meloam/lstore/src/gop/hconnection.c:661:9 (libgop.so.0+0x00000001ba89)
    #2 spawn_new_connection /home/meloam/lstore/src/gop/hportal.c:682:12 (libgop.so.0+0x0000000207e9)
    #3 check_hportal_connections /home/meloam/lstore/src/gop/hportal.c:783:9 (libgop.so.0+0x000000021914)
    #4 gop_hp_submit /home/meloam/lstore/src/gop/hportal.c:892:5 (libgop.so.0+0x00000002282c)
    #5 gop_hp_que_op_submit /home/meloam/lstore/src/gop/hportal.c:935:12 (libgop.so.0+0x000000022d8b)
    #6 _ibp_submit_op /home/meloam/lstore/src/ibp/config.c:413:9 (libibp.so.0+0x00000000719f)
    #7 _opque_start_execution /home/meloam/lstore/src/gop/opque.c:443:13 (libgop.so.0+0x0000000264b3)
    #8 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:217:9 (libgop.so.0+0x00000000efd1)
    #9 gop_waitall /home/meloam/lstore/src/gop/gop.c:427:13 (libgop.so.0+0x000000011ca3)
    #10 slun_row_replace_fix /home/meloam/lstore/src/lio/segment/lun.c:703:15 (liblio.so.0+0x000000178d09)
    #11 _seglun_grow /home/meloam/lstore/src/lio/segment/lun.c:893:21 (liblio.so.0+0x00000017bd0c)
    #12 _slun_truncate /home/meloam/lstore/src/lio/segment/lun.c:1061:15 (liblio.so.0+0x00000017e4d6)
    #13 seglun_rw_func /home/meloam/lstore/src/lio/segment/lun.c:1749:26 (liblio.so.0+0x00000018985d)
    #14 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #15 gop_waitany /home/meloam/lstore/src/gop/gop.c:381:13 (libgop.so.0+0x000000010f19)
    #16 gop_waitany /home/meloam/lstore/src/gop/gop.c:364:13 (libgop.so.0+0x000000010801)
    #17 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1568:27 (liblio.so.0+0x00000014f581)
    #18 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #19 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Thread T14 (tid=34934, running) created by main thread at:
    #0 pthread_create /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (globus-gridftp-server+0x000000425266)
    #1 apr_thread_pool_create /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:380:14 (libgop.so.0+0x00000005bae4)
    #2 lio_create_nl /home/meloam/lstore/src/lio/lio_config.c:920:30 (liblio.so.0+0x000000039595)
    #3 lio_create /home/meloam/lstore/src/lio/lio_config.c:1101:10 (liblio.so.0+0x00000003e8b6)
    #4 lio_init /home/meloam/lstore/src/lio/lio_config.c:1397:14 (liblio.so.0+0x00000004125b)
    #5 activate /home/meloam/lstore/binding/gridftp/src/thunk.c:38:5 (libglobus_gridftp_server_lstore.so+0x000000006810)
    #6 globus_l_gfs_lstore_activate /home/meloam/lstore/binding/gridftp/src/lstore_dsi.c:452:18 (libglobus_gridftp_server_lstore.so+0x000000003961)
    #7 globus_module_activate_proxy <null> (libglobus_common.so.0+0x000000047d96)
    #8 globus_extension_activate <null> (libglobus_common.so.0+0x0000000737ce)
    #9 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #10 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #11 main <null> (globus-gridftp-server+0x0000004acbc2)

SUMMARY: ThreadSanitizer: data race /home/meloam/lstore/src/ibp/op.c:217:29 in gop_get_end_time
==================
tacketar commented 7 years ago

Fixed with commit f7b476d6251dd77c8af63a4cc0b3a16f4013e820