Multiple races around pigeon holes

[PerilousApricot]

WARNING: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) (pid=26670)
    #0 pthread_mutex_lock /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:3600 (globus-gridftp-server+0x000000431ab0)
    #1 release_pigeon_hole /home/meloam/lstore/src/toolbox/pigeon_hole.c:117:5 (libtoolbox.so.0+0x00000001f0b1)
    #2 tbx_pch_release /home/meloam/lstore/src/toolbox/pigeon_coop.c:166:5 (libtoolbox.so.0+0x00000001c75c)
    #3 gop_generic_free /home/meloam/lstore/src/gop/gop.c:709:5 (libgop.so.0+0x0000000148f6)
    #4 _ibp_op_free /home/meloam/lstore/src/ibp/config.c:394:5 (libibp.so.0+0x000000006f84)
    #5 _ds_ibp_op_free /home/meloam/lstore/src/lio/ds/ibp.c:583:5 (liblio.so.0+0x000000028b96)
    #6 gop_free /home/meloam/lstore/src/gop/gop.c:303:9 (libgop.so.0+0x00000000fc84)
    #7 seglun_rw_op /home/meloam/lstore/src/lio/segment/lun.c:1676:49 (liblio.so.0+0x000000188cd2)
    #8 seglun_rw_func /home/meloam/lstore/src/lio/segment/lun.c:1781:14 (liblio.so.0+0x00000018a12b)
    #9 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #10 gop_waitany /home/meloam/lstore/src/gop/gop.c:381:13 (libgop.so.0+0x000000010f19)
    #11 gop_waitany /home/meloam/lstore/src/gop/gop.c:364:13 (libgop.so.0+0x000000010801)
    #12 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1568:27 (liblio.so.0+0x00000014f581)
    #13 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #14 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Location is heap block of size 8192 at 0x7d9000008000 allocated by main thread:
    #0 malloc /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:591 (globus-gridftp-server+0x00000044d5cf)
    #1 allocator_alloc /home/meloam/lstore/vendor/apr-accre/memory/unix/apr_pools.c:349:17 (liblio.so.0+0x0000001a8ea7)
    #2 tbx_pc_new /home/meloam/lstore/src/toolbox/pigeon_coop.c:325:27 (libtoolbox.so.0+0x00000001e921)
    #3 gop_init_opque_system /home/meloam/lstore/src/gop/opque.c:114:24 (libgop.so.0+0x0000000234da)
    #4 gop_construct_fn /home/meloam/lstore/src/gop/constructor.c:27:5 (libgop.so.0+0x00000000d2f5)
    #5 _dl_init_internal <null> (ld-linux-x86-64.so.2+0x00000000f3a2)
    #6 <null> <null> (libltdl.so.7+0x0000000066a1)
    #7 globus_extension_activate <null> (libglobus_common.so.0+0x000000073628)
    #8 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #9 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #10 main <null> (globus-gridftp-server+0x0000004acbc2)

  Mutex M0 (0x7d90000080a8) created at:
    [failed to restore the stack]

SUMMARY: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) /home/meloam/lstore/src/toolbox/pigeon_hole.c:117:5 in release_pigeon_hole
==================
==================
WARNING: ThreadSanitizer: data race (pid=26670)
  Read of size 4 at 0x7d0c0000e294 by thread T85:
    #0 release_pigeon_hole /home/meloam/lstore/src/toolbox/pigeon_hole.c:118:5 (libtoolbox.so.0+0x00000001f131)
    #1 tbx_pch_release /home/meloam/lstore/src/toolbox/pigeon_coop.c:166:5 (libtoolbox.so.0+0x00000001c75c)
    #2 gop_generic_free /home/meloam/lstore/src/gop/gop.c:709:5 (libgop.so.0+0x0000000148f6)
    #3 _ibp_op_free /home/meloam/lstore/src/ibp/config.c:394:5 (libibp.so.0+0x000000006f84)
    #4 _ds_ibp_op_free /home/meloam/lstore/src/lio/ds/ibp.c:583:5 (liblio.so.0+0x000000028b96)
    #5 gop_free /home/meloam/lstore/src/gop/gop.c:303:9 (libgop.so.0+0x00000000fc84)
    #6 seglun_rw_op /home/meloam/lstore/src/lio/segment/lun.c:1676:49 (liblio.so.0+0x000000188cd2)
    #7 seglun_rw_func /home/meloam/lstore/src/lio/segment/lun.c:1781:14 (liblio.so.0+0x00000018a12b)
    #8 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #9 gop_waitany /home/meloam/lstore/src/gop/gop.c:381:13 (libgop.so.0+0x000000010f19)
    #10 gop_waitany /home/meloam/lstore/src/gop/gop.c:364:13 (libgop.so.0+0x000000010801)
    #11 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1568:27 (liblio.so.0+0x00000014f581)
    #12 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #13 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Previous write of size 4 at 0x7d0c0000e294 by thread T103 (mutexes: write M17026556031434920):
    #0 reserve_pigeon_hole /home/meloam/lstore/src/toolbox/pigeon_hole.c:158:22 (libtoolbox.so.0+0x00000001f6d7)
    #1 tbx_pch_reserve /home/meloam/lstore/src/toolbox/pigeon_coop.c:231:16 (libtoolbox.so.0+0x00000001d487)
    #2 gop_init /home/meloam/lstore/src/gop/gop.c:690:11 (libgop.so.0+0x00000001447d)
    #3 init_tp_op /home/meloam/lstore/src/gop/thread_pool_op.c:290:5 (libgop.so.0+0x00000002aee8)
    #4 gop_tp_op_new /home/meloam/lstore/src/gop/thread_pool_op.c:329:5 (libgop.so.0+0x00000002b3b7)
    #5 seglun_write /home/meloam/lstore/src/lio/segment/lun.c:1831:11 (liblio.so.0+0x00000018aa96)
    #6 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1656:19 (liblio.so.0+0x000000150a2d)
    #7 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #8 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Location is heap block of size 48 at 0x7d0c0000e280 allocated by main thread:
    #0 malloc /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:591 (globus-gridftp-server+0x00000044d5cf)
    #1 new_pigeon_hole /home/meloam/lstore/src/toolbox/pigeon_hole.c:193:32 (libtoolbox.so.0+0x00000001f967)
    #2 tbx_pc_new /home/meloam/lstore/src/toolbox/pigeon_coop.c:325:27 (libtoolbox.so.0+0x00000001e921)
    #3 gop_init_opque_system /home/meloam/lstore/src/gop/opque.c:114:24 (libgop.so.0+0x0000000234da)
    #4 gop_construct_fn /home/meloam/lstore/src/gop/constructor.c:27:5 (libgop.so.0+0x00000000d2f5)
    #5 _dl_init_internal <null> (ld-linux-x86-64.so.2+0x00000000f3a2)
    #6 <null> <null> (libltdl.so.7+0x0000000066a1)
    #7 globus_extension_activate <null> (libglobus_common.so.0+0x000000073628)
    #8 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #9 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #10 main <null> (globus-gridftp-server+0x0000004acbc2)

  Mutex M17026556031434920 is already destroyed.

  Thread T85 (tid=27082, running) created by thread T18 at:
    #0 pthread_create /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (globus-gridftp-server+0x000000425266)
    #1 add_task /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:575:14 (libgop.so.0+0x00000005c165)
    #2 _opque_start_execution /home/meloam/lstore/src/gop/opque.c:443:13 (libgop.so.0+0x0000000264b3)
    #3 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:217:9 (libgop.so.0+0x00000000efd1)
    #4 gop_waitany /home/meloam/lstore/src/gop/gop.c:368:13 (libgop.so.0+0x000000010885)
    #5 cache_rw_pages /home/meloam/lstore/src/lio/segment/cache.c:498:15 (liblio.so.0+0x000000110b0d)
    #6 cache_flush_range_gop_func /home/meloam/lstore/src/lio/segment/cache.c:2826:35 (liblio.so.0+0x000000130e3c)
    #7 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #8 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Thread T103 (tid=28838, running) created by thread T18 at:
    #0 pthread_create /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (globus-gridftp-server+0x000000425266)
    #1 add_task /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:575:14 (libgop.so.0+0x00000005c165)
    #2 _opque_start_execution /home/meloam/lstore/src/gop/opque.c:443:13 (libgop.so.0+0x0000000264b3)
    #3 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:217:9 (libgop.so.0+0x00000000efd1)
    #4 gop_waitany /home/meloam/lstore/src/gop/gop.c:368:13 (libgop.so.0+0x000000010885)
    #5 cache_rw_pages /home/meloam/lstore/src/lio/segment/cache.c:498:15 (liblio.so.0+0x000000110b0d)
    #6 cache_flush_range_gop_func /home/meloam/lstore/src/lio/segment/cache.c:2826:35 (liblio.so.0+0x000000130e3c)
    #7 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #8 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

SUMMARY: ThreadSanitizer: data race /home/meloam/lstore/src/toolbox/pigeon_hole.c:118:5 in release_pigeon_hole
==================
WARNING: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) (pid=26670)
    #0 pthread_mutex_lock /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:3600 (globus-gridftp-server+0x000000431ab0)
    #1 tbx_pch_reserve /home/meloam/lstore/src/toolbox/pigeon_coop.c:224:5 (libtoolbox.so.0+0x00000001d2a6)
    #2 gop_init /home/meloam/lstore/src/gop/gop.c:690:11 (libgop.so.0+0x00000001447d)
    #3 init_opque /home/meloam/lstore/src/gop/opque.c:241:5 (libgop.so.0+0x000000024b25)
    #4 gop_opque_new /home/meloam/lstore/src/gop/opque.c:269:5 (libgop.so.0+0x000000024e4c)
    #5 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1503:9 (liblio.so.0+0x00000014e02c)
    #6 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #7 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Location is heap block of size 8192 at 0x7d9000004000 allocated by main thread:
    #0 malloc /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:591 (globus-gridftp-server+0x00000044d5cf)
    #1 allocator_alloc /home/meloam/lstore/vendor/apr-accre/memory/unix/apr_pools.c:349:17 (liblio.so.0+0x0000001a8ea7)
    #2 gop_init_opque_system /home/meloam/lstore/src/gop/opque.c:114:24 (libgop.so.0+0x0000000234da)
    #3 gop_construct_fn /home/meloam/lstore/src/gop/constructor.c:27:5 (libgop.so.0+0x00000000d2f5)
    #4 _dl_init_internal <null> (ld-linux-x86-64.so.2+0x00000000f3a2)
    #5 <null> <null> (libltdl.so.7+0x0000000066a1)
    #6 globus_extension_activate <null> (libglobus_common.so.0+0x000000073628)
    #7 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #8 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #9 main <null> (globus-gridftp-server+0x0000004acbc2)

  Mutex M0 (0x7d90000040a8) created at:
    [failed to restore the stack]

SUMMARY: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) /home/meloam/lstore/src/toolbox/pigeon_coop.c:224:5 in tbx_pch_reserve
==================
==================
WARNING: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) (pid=26670)
    #0 pthread_mutex_lock /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:3600 (globus-gridftp-server+0x000000431ab0)
    #1 tbx_pch_release /home/meloam/lstore/src/toolbox/pigeon_coop.c:154:5 (libtoolbox.so.0+0x00000001c39a)
    #2 gop_generic_free /home/meloam/lstore/src/gop/gop.c:709:5 (libgop.so.0+0x0000000148f6)
    #3 _ibp_op_free /home/meloam/lstore/src/ibp/config.c:394:5 (libibp.so.0+0x000000006f84)
    #4 _ds_ibp_op_free /home/meloam/lstore/src/lio/ds/ibp.c:583:5 (liblio.so.0+0x000000028b96)
    #5 gop_free /home/meloam/lstore/src/gop/gop.c:303:9 (libgop.so.0+0x00000000fc84)
    #6 seglun_rw_op /home/meloam/lstore/src/lio/segment/lun.c:1676:49 (liblio.so.0+0x000000188cd2)
    #7 seglun_rw_func /home/meloam/lstore/src/lio/segment/lun.c:1781:14 (liblio.so.0+0x00000018a12b)
    #8 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #9 gop_waitany /home/meloam/lstore/src/gop/gop.c:381:13 (libgop.so.0+0x000000010f19)
    #10 gop_waitany /home/meloam/lstore/src/gop/gop.c:364:13 (libgop.so.0+0x000000010801)
    #11 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1568:27 (liblio.so.0+0x00000014f581)
    #12 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #13 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Location is heap block of size 8192 at 0x7d9000004000 allocated by main thread:
    #0 malloc /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:591 (globus-gridftp-server+0x00000044d5cf)
    #1 allocator_alloc /home/meloam/lstore/vendor/apr-accre/memory/unix/apr_pools.c:349:17 (liblio.so.0+0x0000001a8ea7)
    #2 gop_init_opque_system /home/meloam/lstore/src/gop/opque.c:114:24 (libgop.so.0+0x0000000234da)
    #3 gop_construct_fn /home/meloam/lstore/src/gop/constructor.c:27:5 (libgop.so.0+0x00000000d2f5)
    #4 _dl_init_internal <null> (ld-linux-x86-64.so.2+0x00000000f3a2)
    #5 <null> <null> (libltdl.so.7+0x0000000066a1)
    #6 globus_extension_activate <null> (libglobus_common.so.0+0x000000073628)
    #7 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #8 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #9 main <null> (globus-gridftp-server+0x0000004acbc2)

  Mutex M0 (0x7d90000040a8) created at:
    [failed to restore the stack]

SUMMARY: ThreadSanitizer: use of an invalid mutex (e.g. uninitialized or destroyed) /home/meloam/lstore/src/toolbox/pigeon_coop.c:154:5 in tbx_pch_release
=========
=========
WARNING: ThreadSanitizer: data race (pid=26670)
  Atomic read of size 1 at 0x7d90000040a8 by thread T89:
    #0 pthread_mutex_unlock /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:3613 (globus-gridftp-server+0x000000431c6e)
    #1 tbx_pch_reserve /home/meloam/lstore/src/toolbox/pigeon_coop.c:240:13 (libtoolbox.so.0+0x00000001d66c)
    #2 gop_init /home/meloam/lstore/src/gop/gop.c:690:11 (libgop.so.0+0x00000001447d)
    #3 init_opque /home/meloam/lstore/src/gop/opque.c:241:5 (libgop.so.0+0x000000024b25)
    #4 gop_opque_new /home/meloam/lstore/src/gop/opque.c:269:5 (libgop.so.0+0x000000024e4c)
    #5 segjerase_write_func /home/meloam/lstore/src/lio/segment/jerasure.c:1503:9 (liblio.so.0+0x00000014e02c)
    #6 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #7 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Previous write of size 1 at 0x7d90000040a8 by thread T3:
    #0 pthread_mutex_destroy /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1135 (globus-gridftp-server+0x00000042c41e)
    #1 tbx_pc_destroy /home/meloam/lstore/src/toolbox/pigeon_coop.c:279:5 (libtoolbox.so.0+0x00000001def1)
    #2 gop_shutdown /home/meloam/lstore/src/gop/opque.c:129:9 (libgop.so.0+0x0000000235cf)
    #3 gop_destruct_fn /home/meloam/lstore/src/gop/constructor.c:31:5 (libgop.so.0+0x00000000d323)
    #4 _dl_fini <null> (ld-linux-x86-64.so.2+0x00000000fa19)
    #5 globus_l_callback_thread_poll <null> (libglobus_common.so.0+0x00000002517e)
    #6 globus_l_thread_pool_thread_start <null> (libglobus_common.so.0+0x000000063084)
    #7 thread_starter <null> (libglobus_thread_pthread.so+0x000000003b30)

  Location is heap block of size 8192 at 0x7d9000004000 allocated by main thread:
    #0 malloc /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:591 (globus-gridftp-server+0x00000044d5cf)
    #1 allocator_alloc /home/meloam/lstore/vendor/apr-accre/memory/unix/apr_pools.c:349:17 (liblio.so.0+0x0000001a8ea7)
    #2 gop_init_opque_system /home/meloam/lstore/src/gop/opque.c:114:24 (libgop.so.0+0x0000000234da)
    #3 gop_construct_fn /home/meloam/lstore/src/gop/constructor.c:27:5 (libgop.so.0+0x00000000d2f5)
    #4 _dl_init_internal <null> (ld-linux-x86-64.so.2+0x00000000f3a2)
    #5 <null> <null> (libltdl.so.7+0x0000000066a1)
    #6 globus_extension_activate <null> (libglobus_common.so.0+0x000000073628)
    #7 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #8 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #9 main <null> (globus-gridftp-server+0x0000004acbc2)

  Thread T89 (tid=28833, running) created by thread T18 at:
    #0 pthread_create /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (globus-gridftp-server+0x000000425266)
    #1 add_task /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:575:14 (libgop.so.0+0x00000005c165)
    #2 _opque_start_execution /home/meloam/lstore/src/gop/opque.c:443:13 (libgop.so.0+0x0000000264b3)
    #3 _gop_start_execution /home/meloam/lstore/src/gop/gop.c:217:9 (libgop.so.0+0x00000000efd1)
    #4 gop_waitany /home/meloam/lstore/src/gop/gop.c:368:13 (libgop.so.0+0x000000010885)
    #5 cache_rw_pages /home/meloam/lstore/src/lio/segment/cache.c:498:15 (liblio.so.0+0x000000110b0d)
    #6 cache_flush_range_gop_func /home/meloam/lstore/src/lio/segment/cache.c:2826:35 (liblio.so.0+0x000000130e3c)
    #7 thread_pool_exec_fn /home/meloam/lstore/src/gop/thread_pool_op.c:232:14 (libgop.so.0+0x00000002a7b1)
    #8 thread_pool_func /home/meloam/lstore/vendor/apr-util-accre/misc/apr_thread_pool.c:271:13 (libgop.so.0+0x00000005bd0b)

  Thread T3 (tid=26674, running) created by main thread at:
    #0 pthread_create /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:902 (globus-gridftp-server+0x000000425266)
    #1 globus_l_pthread_thread_create <null> (libglobus_thread_pthread.so+0x0000000032df)
    #2 globus_thread_create <null> (libglobus_common.so.0+0x000000060f71)
    #3 globus_i_thread_start <null> (libglobus_common.so.0+0x0000000633c9)
    #4 globus_l_callback_activate <null> (libglobus_common.so.0+0x0000000206cc)
    #5 globus_module_activate_proxy <null> (libglobus_common.so.0+0x000000047d96)
    #6 globus_module_activate <null> (libglobus_common.so.0+0x000000048782)
    #7 globus_l_callback_activate <null> (libglobus_common.so.0+0x00000001816d)
    #8 globus_module_activate_proxy <null> (libglobus_common.so.0+0x000000047d96)
    #9 globus_module_activate <null> (libglobus_common.so.0+0x000000048782)
    #10 globus_l_common_activate <null> (libglobus_common.so.0+0x00000002daa6)
    #11 globus_module_activate_proxy <null> (libglobus_common.so.0+0x000000047d96)
    #12 globus_module_activate <null> (libglobus_common.so.0+0x000000048782)
    #13 main <null> (globus-gridftp-server+0x0000004ac8f1)

SUMMARY: ThreadSanitizer: data race /home/meloam/lstore/src/toolbox/pigeon_coop.c:240:13 in tbx_pch_reserve
=========

[PerilousApricot]

WARNING: ThreadSanitizer: destroy of a locked mutex (pid=31427)
    #0 pthread_mutex_destroy /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1135 (globus-gridftp-server+0x00000042c41e)
    #1 tbx_pc_destroy /home/meloam/lstore/src/toolbox/pigeon_coop.c:279:5 (libtoolbox.so.0+0x00000001def1)
    #2 gop_shutdown /home/meloam/lstore/src/gop/opque.c:129:9 (libgop.so.0+0x0000000235cf)
    #3 gop_destruct_fn /home/meloam/lstore/src/gop/constructor.c:31:5 (libgop.so.0+0x00000000d323)
    #4 _dl_fini <null> (ld-linux-x86-64.so.2+0x00000000fa19)
    #5 globus_l_callback_thread_poll <null> (libglobus_common.so.0+0x00000002517e)
    #6 globus_l_thread_pool_thread_start <null> (libglobus_common.so.0+0x000000063084)
    #7 thread_starter <null> (libglobus_thread_pthread.so+0x000000003b30)

  and:
    #0 pthread_mutex_destroy /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1135 (globus-gridftp-server+0x00000042c41e)
    #1 tbx_pc_destroy /home/meloam/lstore/src/toolbox/pigeon_coop.c:279:5 (libtoolbox.so.0+0x00000001def1)
    #2 gop_shutdown /home/meloam/lstore/src/gop/opque.c:129:9 (libgop.so.0+0x0000000235cf)
    #3 gop_destruct_fn /home/meloam/lstore/src/gop/constructor.c:31:5 (libgop.so.0+0x00000000d323)
    #4 _dl_fini <null> (ld-linux-x86-64.so.2+0x00000000fa19)
    #5 globus_l_callback_thread_poll <null> (libglobus_common.so.0+0x00000002517e)
    #6 globus_l_thread_pool_thread_start <null> (libglobus_common.so.0+0x000000063084)
    #7 thread_starter <null> (libglobus_thread_pthread.so+0x000000003b30)

  Location is heap block of size 8192 at 0x7d9000004000 allocated by main thread:
    #0 malloc /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:591 (globus-gridftp-server+0x00000044d5cf)
    #1 allocator_alloc /home/meloam/lstore/vendor/apr-accre/memory/unix/apr_pools.c:349:17 (liblio.so.0+0x0000001a8ea7)
    #2 gop_init_opque_system /home/meloam/lstore/src/gop/opque.c:114:24 (libgop.so.0+0x0000000234da)
    #3 gop_construct_fn /home/meloam/lstore/src/gop/constructor.c:27:5 (libgop.so.0+0x00000000d2f5)
    #4 _dl_init_internal <null> (ld-linux-x86-64.so.2+0x00000000f3a2)
    #5 <null> <null> (libltdl.so.7+0x0000000066a1)
    #6 globus_extension_activate <null> (libglobus_common.so.0+0x000000073628)
    #7 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #8 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #9 main <null> (globus-gridftp-server+0x0000004acbc2)

  Mutex M171 (0x7d90000040a8) created at:
    #0 pthread_mutex_init /home/meloam/llvm/llvm-3.9.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1119 (globus-gridftp-server+0x000000433f55)
    #1 apr_thread_mutex_create /home/meloam/lstore/vendor/apr-accre/locks/unix/thread_mutex.c:71:14 (liblio.so.0+0x0000001a8b26)
    #2 gop_init_opque_system /home/meloam/lstore/src/gop/opque.c:114:24 (libgop.so.0+0x0000000234da)
    #3 gop_construct_fn /home/meloam/lstore/src/gop/constructor.c:27:5 (libgop.so.0+0x00000000d2f5)
    #4 _dl_init_internal <null> (ld-linux-x86-64.so.2+0x00000000f3a2)
    #5 <null> <null> (libltdl.so.7+0x0000000066a1)
    #6 globus_extension_activate <null> (libglobus_common.so.0+0x000000073628)
    #7 globus_i_gfs_data_new_dsi <null> (libglobus_gridftp_server.so.6+0x0000000317cb)
    #8 globus_i_gfs_data_init <null> (libglobus_gridftp_server.so.6+0x000000031155)
    #9 main <null> (globus-gridftp-server+0x0000004acbc2)

SUMMARY: ThreadSanitizer: destroy of a locked mutex /home/meloam/lstore/src/toolbox/pigeon_coop.c:279:5 in tbx_pc_destroy