search

XRPLF / rippled

Decentralized cryptocurrency blockchain daemon implementing the XRP Ledger protocol in C++
https://xrpl.org
ISC License
4.52k stars 1.47k forks source link

Crash during execution, running on FreeBSD #102

Closed vinniefalco closed 10 years ago

vinniefalco commented 11 years ago

During execution, the server experiences a crash. There is a core dump, followed by the Sustain process restarting the server.

BobWay commented 11 years ago

(gdb) bt

0 0x0000000802926a4c in thr_kill () from /lib/libc.so.7

1 0x00000008029c38ab in abort () from /lib/libc.so.7

2 0x00000008023f3f54 in __gnu_cxx::__verbose_terminate_handler ()

from /usr/lib/libstdc++.so.6

3 0x00000008023f8453 in std::set_unexpected () from /usr/lib/libstdc++.so.6

4 0x00000008023f8493 in std::terminate () from /usr/lib/libstdc++.so.6

5 0x00000008023f8391 in __cxa_rethrow () from /usr/lib/libstdc++.so.6

6 0x0000000000f26698 in SHAMap::getStack (this=0x8049f7420, id=@0x7fffff1ef240,

include_nonmatching_leaf=true, partialOk=false) at ripple_SHAMap.cpp:97

7 0x0000000000f2f6f1 in SHAMap::peekNextItem (this=0x8049f7420, id=@0x7fffff1ef240,

type=@0x7fffff1ef184) at ripple_SHAMap.cpp:472

8 0x0000000000f2fb28 in SHAMap::peekNextItem (this=0x8049f7420, id=@0x7fffff1ef240)

at ripple_SHAMap.cpp:463

9 0x0000000000cc4e35 in Ledger::getNextLedgerIndex (this=0x8048217e0,

uHash=@0x7fffff1ef240) at Ledger.cpp:1247

10 0x0000000000cc4f34 in LedgerEntrySet::getNextLedgerIndex (this=0x804871120,

uHash=@0x7fffff1ef940) at LedgerEntrySet.cpp:987

11 0x0000000000cc5199 in LedgerEntrySet::getNextLedgerIndex (this=0x804871120,

uHash=@0x7fffff1ef940, uEnd=@0x7fffff1ef920) at LedgerEntrySet.cpp:1006

12 0x0000000000cd076c in OfferCreateTransactor::takeOffers (this=0x8088b0800,

bOpenLedger=true, bPassive=false, bSell=true, uBookBase=@0x7fffff1f2ed0, 
uTakerAccountID=@0x8088b081c, sleTakerAccount=@0x7fffff1f2f80, 
saTakerPays=@0x7fffff1f2e30, saTakerGets=@0x7fffff1f2e80, 
saTakerPaid=@0x7fffff1f2de0, saTakerGot=@0x7fffff1f2d90, bUnfunded=@0x7fffff1f2f7f)
at OfferCreateTransactor.cpp:141

13 0x0000000000cd42fe in OfferCreateTransactor::doApply (this=0x8088b0800)

at OfferCreateTransactor.cpp:523

14 0x0000000000e05700 in Transactor::apply (this=0x8088b0800) at Transactor.cpp:243

15 0x0000000000e08d9f in TransactionEngine::applyTransaction (this=0x804871118,

txn=@0x804df3220, params=17, didApply=@0x7fffff1f8fff) at TransactionEngine.cpp:99

16 0x0000000000cddb1e in LedgerMaster::doTransaction (this=0x8048710f0,

txn=@0x804b16ee0, params=17, didApply=@0x7fffff1f8fff) at LedgerMaster.cpp:165

17 0x0000000000cde0a6 in NetworkOPs::processTransaction (this=0x8048714f0,

trans=@0x7fffff1f99e0, bAdmin=false, callback=@0x7fffff1f99c0) at NetworkOPs.cpp:345

18 0x0000000000de6921 in NetworkOPs::processTransaction (this=0x8048714f0,

transaction=@0x7fffff1f9b00, bAdmin=false) at NetworkOPs.h:136

19 0x0000000000f99fe8 in checkTransaction (flags=0, stx=@0x7fffff1f9b70,

peer=@0x7fffff1f9b60) at ripple_Peer.cpp:1077

20 0x0000000000ffac12 in boost::_bi::list4boost::arg<1, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr >::operator()<void (*)(Job&, int, boost::shared_ptr, boost::weak_ptr), boost::_bi::list1<Job&> > (this=0x8053f1e58, f=@0x8053f1e50,

a=@0x7fffff1f9bd0) at bind.hpp:457

21 0x0000000000ffacae in boost::_bi::bind_t<void, void (*)(Job&, int, boost::shared_ptr, boost::weak_ptr), boost::_bi::list4boost::arg<1, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >::operator() (this=0x8053f1e50, a1=@0x7fffff1f9cc0)

at bind_template.hpp:32

22 0x0000000000ffacd8 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, void (*)(Job&, int, boost::shared_ptr, boost::weak_ptr), boost::_bi::list4boost::arg<1, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >, void, Job&>::invoke (function_obj_ptr=@0x7fffff1f9cd8, a0=@0x7fffff1f9cc0)

at function_template.hpp:153

23 0x000000000101c5a9 in boost::function1<void, Job&>::operator() (this=0x7fffff1f9cd0,

a0=@0x7fffff1f9cc0) at function_template.hpp:760

24 0x0000000000f8a7f3 in Job::doJob (this=0x7fffff1f9cc0) at ripple_Job.cpp:36

25 0x0000000000fc295e in JobQueue::threadEntry (this=0x804871b60)

---Type to continue, or q to quit--- at ripple_JobQueue.cpp:282

26 0x0000000000ff0ba2 in boost::_mfi::mf0<void, JobQueue>::operator() (

this=0x8048951b8, p=0x804871b60) at mem_fn_template.hpp:49

27 0x0000000000ff0d61 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x8048951c8, f=@0x8048951b8,

a=@0x7fffff1f9f5f) at bind.hpp:253

28 0x0000000000ff0da6 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x8048951b8)

at bind_template.hpp:20

29 0x0000000000ff0dcc in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (

this=0x804895100) at thread.hpp:61

30 0x0000000801e2d062 in thread_proxy () from /usr/local/lib/libboost_thread.so

31 0x00000008027c0511 in pthread_getprio () from /lib/libthr.so.3

32 0x0000000000000000 in ?? ()

Cannot access memory at address 0x7fffff1fa000

BobWay commented 11 years ago

(gdb) bt

0 0x0000000802926a4c in thr_kill () from /lib/libc.so.7

1 0x00000008029c38ab in abort () from /lib/libc.so.7

2 0x00000008023f3f54 in __gnu_cxx::__verbose_terminate_handler ()

from /usr/lib/libstdc++.so.6

3 0x00000008023f8453 in std::set_unexpected () from /usr/lib/libstdc++.so.6

4 0x00000008023f8493 in std::terminate () from /usr/lib/libstdc++.so.6

5 0x00000008023f83fa in __cxa_throw () from /usr/lib/libstdc++.so.6

6 0x0000000000f2996d in SHAMap::getNodePointer (this=0x809837720, id=@0x7fffffff3d50,

hash=@0x8051f0600) at ripple_SHAMap.cpp:234

7 0x0000000000f320c3 in SHAMap::walkToPointer (this=0x809837720, id=@0x7fffffff4ae0)

at ripple_SHAMap.cpp:195

8 0x0000000000f32373 in SHAMap::peekItem (this=0x809837720, id=@0x7fffffff4ae0)

at ripple_SHAMap.cpp:549

9 0x0000000000cc4195 in Ledger::getSLE (this=0x804914ce0, uHash=@0x7fffffff4ae0)

at Ledger.cpp:1131

10 0x0000000000cc7302 in LedgerEntrySet::entryCache (this=0x7fffffffb2e8,

letType=ltDIR_NODE, index=@0x7fffffff4ae0) at LedgerEntrySet.cpp:95

11 0x0000000000ccd2c9 in LedgerEntrySet::dirAdd (this=0x7fffffffb2e8,

uNodeDir=@0x7fffffff4b80, uRootIndex=@0x7fffffff4ae0, uLedgerIndex=@0x7fffffff4b00, 
fDescriber=@0x7fffffff7b60) at LedgerEntrySet.cpp:635

12 0x0000000000cd54c5 in OfferCreateTransactor::doApply (this=0x80553ab80)

at OfferCreateTransactor.cpp:623

13 0x0000000000e05700 in Transactor::apply (this=0x80553ab80) at Transactor.cpp:243

14 0x0000000000e08d9f in TransactionEngine::applyTransaction (this=0x7fffffffb2e0,

txn=@0x80481e620, params=tapRETRY, didApply=@0x7fffffffaa57)
at TransactionEngine.cpp:99

15 0x0000000000fb0459 in LedgerConsensus::applyTransaction (this=0x805009120,

engine=@0x7fffffffb2e0, txn=@0x7fffffffb3f0, ledger=@0x7fffffffbe00, 
openLedger=false, retryAssured=true) at ripple_LedgerConsensus.cpp:1059

16 0x0000000000fb0980 in LedgerConsensus::applyTransactions (this=0x805009120,

set=@0x7fffffffdb00, applyLedger=@0x7fffffffbe00, checkLedger=@0x7fffffffbe00, 
failedTransactions=@0x7fffffffbc60, openLgr=false) at ripple_LedgerConsensus.cpp:1106

17 0x0000000000fb2a99 in LedgerConsensus::accept (this=0x805009120, set=@0x7fffffffdb00)

at ripple_LedgerConsensus.cpp:1205

18 0x0000000000ffedc1 in boost::_mfi::mf2<void, LedgerConsensus, boost::shared_ptr const&, boost::shared_ptr >::callboost::shared_ptr<LedgerConsensus, boost::shared_ptr const, boost::shared_ptr > (this=0x7fffffffdae0,

u=@0x7fffffffdaf0, b1=@0x7fffffffdb00, b2=@0x7fffffffd980) at mem_fn_template.hpp:271

19 0x0000000000ffee3c in boost::_mfi::mf2<void, LedgerConsensus, boost::shared_ptr const&, boost::shared_ptr >::operator()boost::shared_ptr

(this=0x7fffffffdae0, u=@0x7fffffffdaf0, a1=@0x7fffffffdb00, a2=@0x7fffffffd980)
at mem_fn_template.hpp:286

20 0x0000000000ffeebb in boost::_bi::list3boost::_bi::value<boost::shared_ptr, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::shared_ptr >::operator()<boost::_mfi::mf2<void, LedgerConsensus, boost::shared_ptr const&, boost::shared_ptr >, boost::_bi::list0> (this=0x7fffffffdaf0,

f=@0x7fffffffdae0, a=@0x7fffffffd9df) at bind.hpp:392

21 0x0000000000ffef26 in boost::_bi::bind_t<void, boost::_mfi::mf2<void, LedgerConsensus, boost::shared_ptr const&, boost::shared_ptr >, boost::_bi::list3boost::_bi::value<boost::shared_ptr, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::shared_ptr > >::operator() (

this=0x7fffffffdae0) at bind_template.hpp:20

22 0x0000000000ffefc9 in boost::asio::asio_handler_invoke<boost::_bi::bind_t<void, boost::_mfi::mf2<void, LedgerConsensus, boost::shared_ptr const&, boost::shared_ptr >, boost::_bi::list3boost::_bi::value<boost::shared_ptr, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::shared_ptr > > > (function=@0x7fffffffdae0) at handler_invoke_hook.hpp:64

23 0x0000000000fff00b in boost_asio_handler_invoke_helpers::invoke<boost::_bi::bind_t<voi---Type to continue, or q to quit---

d, boost::_mfi::mf2<void, LedgerConsensus, boost::shared_ptr const&, boost::shared_ptr >, boost::_bi::list3boost::_bi::value<boost::shared_ptr, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::shared_ptr > >, boost::_bi::bind_t<void, boost::_mfi::mf2<void, LedgerConsensus, boost::shared_ptr const&, boost::shared_ptr >, boost::_bi::list3boost::_bi::value<boost::shared_ptr, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::shared_ptr > > > (function=@0x7fffffffdb70, context=@0x7fffffffdb70) at handler_invoke_helpers.hpp:39

24 0x0000000000fff0f8 in boost::asio::detail::completion_handler<boost::_bi::bind_t<void, boost::_mfi::mf2<void, LedgerConsensus, boost::shared_ptr const&, boost::shared_ptr >, boost::_bi::list3boost::_bi::value<boost::shared_ptr, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::shared_ptr > > >::do_complete (owner=0x80481e380, base=0x809830c40)

at completion_handler.hpp:67

25 0x0000000000d13932 in boost::asio::detail::task_io_service_operation::complete (

this=0x809830c40, owner=@0x80481e380, ec=@0x7fffffffddf0, bytes_transferred=0)
at task_io_service_operation.hpp:37

26 0x0000000000de464f in boost::asio::detail::task_io_service::do_run_one (

this=0x80481e380, lock=@0x7fffffffdd30, this_thread=@0x7fffffffdd80, 
private_op_queue=@0x7fffffffdd60, ec=@0x7fffffffddf0) at task_io_service.ipp:366

27 0x0000000000de4861 in boost::asio::detail::task_io_service::run (this=0x80481e380,

ec=@0x7fffffffddf0) at task_io_service.ipp:146

28 0x0000000000de4976 in boost::asio::io_service::run (this=0x804871008)

at io_service.ipp:59

29 0x0000000000f1ad31 in Application::run (this=0x804871000)

at ripple_Application.cpp:606

30 0x0000000000cc0c95 in startServer () at main.cpp:41

31 0x0000000000d11f06 in main (argc=2, argv=0x7fffffffe6c8) at main.cpp:263

BobWay commented 11 years ago

We rebuilt the latest dev branch on FreeBSD. Ran it with the existing database and config. It hung almost immediately.

ripple# ./rippled --net 2013-Jun-26 23:29:20 WRN Launching child 1 Loading: "/root/NewCoin/public/rippled.cfg" 2013-Jun-26 23:29:20 ripple_JobQueue:NFO Auto-tuning to 6 validation/transaction/proposal threads 2013-Jun-26 23:29:20 FTL Unable to open logfile "log/debug.log" 2013-Jun-26 23:29:20 ripple_Application:NFO SQLite used for nodes 2013-Jun-26 23:29:20 ripple_Application:NFO Root master seed: snoPBrXtMeMyMHUVTgbuqAfg1SUTb 2013-Jun-26 23:29:20 ripple_Application:NFO Root account: rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh 2013-Jun-26 23:29:20 LedgerMaster:NFO PushLedger: AB868A6CFEEC779C2FF845C0AF00A642259986AF40C01976A7F842B6918936C7 2013-Jun-26 23:29:20 LedgerMaster:NFO StashAccepted: 2D81EC3F8AB517E3148872F7155122187F85FF3A995963A504FE597A43E791AC NodeIdentity: n9LprKw7B9H3Yf8bwZvkQsgPKTsUwBAFXkdQ5wGrHqdd9TQGtawR 2013-Jun-26 23:29:20 ripple_UniqueNodeList:NFO Bootstrapping UNL: loading from 'validators.txt'. 2013-Jun-26 23:29:20 ripple_UniqueNodeList:NFO Node Public: n9KPnVLn7ewVzHvn218DcEYsnWLzKerTDwhpofhk4Ym1RUq4TeGw RIP1 2013-Jun-26 23:29:20 ripple_UniqueNodeList:NFO Node Public: n9LFzWuhKNvXStHAuemfRKFVECLApowncMAM5chSCL9R5ECHGN4V RIP2 2013-Jun-26 23:29:20 ripple_UniqueNodeList:NFO Node Public: n94rSdgTyBNGvYg8pZXGuNt59Y5bGAZGxbxyvjDaqD9ceRAgD85P RIP3 2013-Jun-26 23:29:20 ripple_UniqueNodeList:NFO Node Public: n9LeQeDcLDMZKjx1TZtrXoLBLo5q1bR1sUQrWG7tEADFU6R27UBp RIP4

ripple# gdb rippled -p 22718 ... [Switching to Thread 804a0a580 (LWP 100424/rippled)] 0x0000000802a8c39c in __error () from /lib/libthr.so.3

(gdb) thread apply all where

Thread 10 (Thread 804a041c0 (LWP 100243/rippled)):

0 0x0000000802c07ce3 in _pthread_mutex_init_calloc_cb () from /lib/libc.so.7

1 0x0000000802c0a5c0 in _pthread_mutex_init_calloc_cb () from /lib/libc.so.7

2 0x0000000802c0c19c in _malloc_prefork () from /lib/libc.so.7

3 0x0000000802c11889 in free () from /lib/libc.so.7

4 0x0000000000cd55b2 in sqlite3MemFree () at ripple_SHAMapSyncFilters.cpp:110

5 0x0000000000cd642c in sqlite3_free () at ripple_SHAMapSyncFilters.cpp:110

6 0x0000000000cd6508 in sqlite3DbFree () at ripple_SHAMapSyncFilters.cpp:110

7 0x0000000000d015c6 in sqlite3VdbeClearObject () at ripple_SHAMapSyncFilters.cpp:110

8 0x0000000000d0160f in sqlite3VdbeDelete () at ripple_SHAMapSyncFilters.cpp:110

9 0x0000000000d01404 in sqlite3VdbeFinalize () at ripple_SHAMapSyncFilters.cpp:110

10 0x0000000000d02871 in sqlite3_finalize () at ripple_SHAMapSyncFilters.cpp:110

11 0x0000000000e850e0 in SqliteDatabase::executeSQL (this=0x804e3ee80, sql=0x804a994d8 "PRAGMA cache_size=-4096;",

fail_ok=false) at ripple_SqliteDatabase.cpp:97

12 0x0000000000dd17ee in Database::executeSQL (this=0x804e3ee80, strSql=@0x7fffffffd300, fail_okay=false)

at ripple_Database.h:49

13 0x0000000000faad3d in Application::setup (this=0x804a70000) at ripple_Application.cpp:524

14 0x0000000000d7337c in setupServer () at main.cpp:19

15 0x0000000000dcc7dd in main (argc=2, argv=0x7fffffffeae0) at main.cpp:264

Thread 9 (Thread 804a8d540 (LWP 100094/rippled)):

0 0x0000000802a8c39c in __error () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in pthread_cond_signal () from /lib/libthr.so.3

2 0x0000000000be9044 in boost::condition_variable::wait (this=0x804a70b50, m=@0x7fffffbfed20)

at condition_variable.hpp:55

3 0x0000000000bd4c49 in JobQueue::threadEntry (this=0x804a70b48) at ripple_JobQueue.cpp:246

4 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a6c7b8, p=0x804a70b48)

at mem_fn_template.hpp:49

5 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a6c7c8, f=@0x804a6c7b8, a=@0x7fffffbfef5f) at bind.hpp:253

6 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a6c7b8) at bind_template.hpp:20

7 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (this=0x804a6c700) at thread.hpp:61

8 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

9 0x0000000802a82511 in pthread_getprio () from /lib/libthr.so.3

10 0x0000000000000000 in ?? ()

Error accessing memory address 0x7fffffbff000: Bad address.

Thread 8 (Thread 804a8d380 (LWP 100129/rippled)):

0 0x0000000802a8c39c in __error () from /lib/libthr.so.3

1 0x0000000802a8c08c in __error () from /lib/libthr.so.3

2 0x0000000802a873bc in pthread_mutex_getprioceiling () from /lib/libthr.so.3

3 0x0000000000cd5a75 in pthreadMutexEnter () at ripple_SHAMapSyncFilters.cpp:110

4 0x0000000000cd5870 in sqlite3_mutex_enter () at ripple_SHAMapSyncFilters.cpp:110

5 0x0000000000d359e5 in sqlite3LockAndPrepare () at ripple_SHAMapSyncFilters.cpp:110

6 0x0000000000d35bfd in sqlite3_prepare_v2 () at ripple_SHAMapSyncFilters.cpp:110

7 0x0000000000e8510d in SqliteDatabase::executeSQL (this=0x804e3ee80,

sql=0x804c177a8 "DELETE FROM AccountTransactions WHERE LedgerSeq = 2;", fail_ok=false)
at ripple_SqliteDatabase.cpp:99

---Type to continue, or q to quit---

8 0x0000000000dd17ee in Database::executeSQL (this=0x804e3ee80, strSql=@0x7fffff9fd490, fail_okay=false)

at ripple_Database.h:49

9 0x0000000000d981e6 in Ledger::saveAcceptedLedger (this=0x804b12160, fromConsensus=false) at Ledger.cpp:549

10 0x0000000000e044d9 in boost::_mfi::mf2<void, Ledger, Job&, bool>::callboost::shared_ptr<Ledger, Job, bool> (

this=0x804c13bb0, u=@0x804c13bc0, b1=@0x7fffff9fdcc0, b2=@0x7fffff9fdb14) at mem_fn_template.hpp:271

11 0x0000000000e0451b in boost::_mfi::mf2<void, Ledger, Job&, bool>::operator()boost::shared_ptr (

this=0x804c13bb0, u=@0x804c13bc0, a1=@0x7fffff9fdcc0, a2=false) at mem_fn_template.hpp:286

12 0x0000000000e04594 in boost::_bi::list3boost::_bi::value<boost::shared_ptr, boost::arg<1>, boost::_bi::value >::operator()<boost::_mfi::mf2<void, Ledger, Job&, bool>, boost::_bi::list1<Job&> > (this=0x804c13bc0,

f=@0x804c13bb0, a=@0x7fffff9fdbd0) at bind.hpp:392

13 0x0000000000e045de in boost::_bi::bind_t<void, boost::_mfi::mf2<void, Ledger, Job&, bool>, boost::_bi::list3boost::_bi::value<boost::shared_ptr, boost::arg<1>, boost::_bi::value > >::operator() (this=0x804c13bb0,

a1=@0x7fffff9fdcc0) at bind_template.hpp:32

14 0x0000000000e04608 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf2<void, Ledger, Job&, bool>, boost::_bi::list3boost::_bi::value<boost::shared_ptr, boost::arg<1>, boost::_bi::value > >, void, Job&>::invoke (function_obj_ptr=@0x7fffff9fdcd8, a0=@0x7fffff9fdcc0) at function_template.hpp:153

15 0x0000000000be51b9 in boost::function1<void, Job&>::operator() (this=0x7fffff9fdcd0, a0=@0x7fffff9fdcc0)

at function_template.hpp:760

16 0x0000000000bd4433 in Job::doJob (this=0x7fffff9fdcc0) at ripple_Job.cpp:41

17 0x0000000000bd4e3e in JobQueue::threadEntry (this=0x804a70b48) at ripple_JobQueue.cpp:268

18 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a946b8, p=0x804a70b48)

at mem_fn_template.hpp:49

19 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a946c8, f=@0x804a946b8, a=@0x7fffff9fdf5f) at bind.hpp:253

20 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a946b8) at bind_template.hpp:20

21 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (this=0x804a94600) at thread.hpp:61

22 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

23 0x0000000802a82511 in pthread_getprio () from /lib/libthr.so.3

24 0x0000000000000000 in ?? ()

Error accessing memory address 0x7fffff9fe000: Bad address.

Thread 7 (Thread 804a8d1c0 (LWP 100142/rippled)):

0 0x0000000802a8c39c in __error () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in pthread_cond_signal () from /lib/libthr.so.3

2 0x0000000000be9044 in boost::condition_variable::wait (this=0x804a70b50, m=@0x7fffff7fcd20)

at condition_variable.hpp:55

3 0x0000000000bd4c49 in JobQueue::threadEntry (this=0x804a70b48) at ripple_JobQueue.cpp:246

4 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a945b8, p=0x804a70b48)

at mem_fn_template.hpp:49

5 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a945c8, f=@0x804a945b8, a=@0x7fffff7fcf5f) at bind.hpp:253

6 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a945b8) at bind_template.hpp:20

7 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (this=0x804a94500) at thread.hpp:61

8 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

9 0x0000000802a82511 in pthread_getprio () from /lib/libthr.so.3

10 0x0000000000000000 in ?? ()

Error accessing memory address 0x7fffff7fd000: Bad address.

---Type to continue, or q to quit--- Thread 6 (Thread 804a0ae40 (LWP 100326/rippled)):

0 0x0000000802a8c39c in __error () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in pthread_cond_signal () from /lib/libthr.so.3

2 0x0000000000be9044 in boost::condition_variable::wait (this=0x804a70b50, m=@0x7fffff5fbd20)

at condition_variable.hpp:55

3 0x0000000000bd4c49 in JobQueue::threadEntry (this=0x804a70b48) at ripple_JobQueue.cpp:246

4 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a944b8, p=0x804a70b48)

at mem_fn_template.hpp:49

5 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a944c8, f=@0x804a944b8, a=@0x7fffff5fbf5f) at bind.hpp:253

6 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a944b8) at bind_template.hpp:20

7 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (this=0x804a94400) at thread.hpp:61

8 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

9 0x0000000802a82511 in pthread_getprio () from /lib/libthr.so.3

10 0x0000000000000000 in ?? ()

Error accessing memory address 0x7fffff5fc000: Bad address.

Thread 5 (Thread 804a0ac80 (LWP 100375/rippled)):

0 0x0000000802a8c39c in __error () from /lib/libthr.so.3

1 0x0000000802a8c08c in __error () from /lib/libthr.so.3

2 0x0000000802a873bc in pthread_mutex_getprioceiling () from /lib/libthr.so.3

3 0x0000000000cd5a75 in pthreadMutexEnter () at ripple_SHAMapSyncFilters.cpp:110

4 0x0000000000cd5870 in sqlite3_mutex_enter () at ripple_SHAMapSyncFilters.cpp:110

5 0x0000000000cd60aa in sqlite3Malloc () at ripple_SHAMapSyncFilters.cpp:110

6 0x0000000000ce23cd in pcache1Alloc () at ripple_SHAMapSyncFilters.cpp:110

7 0x0000000000ce2550 in pcache1AllocPage () at ripple_SHAMapSyncFilters.cpp:110

8 0x0000000000ce300e in pcache1Fetch () at ripple_SHAMapSyncFilters.cpp:110

9 0x0000000000ce1900 in sqlite3PcacheFetch () at ripple_SHAMapSyncFilters.cpp:110

10 0x0000000000ce8405 in sqlite3PagerAcquire () at ripple_SHAMapSyncFilters.cpp:110

11 0x0000000000ceff5c in btreeGetPage () at ripple_SHAMapSyncFilters.cpp:110

12 0x0000000000cf007e in getAndInitPage () at ripple_SHAMapSyncFilters.cpp:110

13 0x0000000000cf3827 in moveToChild () at ripple_SHAMapSyncFilters.cpp:110

14 0x0000000000cf438d in sqlite3BtreeMovetoUnpacked () at ripple_SHAMapSyncFilters.cpp:110

15 0x0000000000d0aa89 in sqlite3VdbeExec () at ripple_SHAMapSyncFilters.cpp:110

16 0x0000000000d03241 in sqlite3Step () at ripple_SHAMapSyncFilters.cpp:110

17 0x0000000000d0342c in sqlite3_step () at ripple_SHAMapSyncFilters.cpp:110

18 0x0000000000e843e8 in SqliteStatement::step (this=0x1479378) at ripple_SqliteDatabase.cpp:397

19 0x000000000105b0a5 in HashedObjectStore::bulkWriteSQLite (this=0x804a708a8) at ripple_HashedObjectStore.cpp:349

20 0x00000000010a351a in boost::_mfi::mf1<void, HashedObjectStore, Job&>::operator() (this=0x7fffff3facd8,

p=0x804a708a8, a1=@0x7fffff3facc0) at mem_fn_template.hpp:165

21 0x00000000010a6b77 in boost::_bi::list2boost::_bi::value<HashedObjectStore*, boost::arg<1> >::operator()<boost::_mfi::mf1<void, HashedObjectStore, Job&>, boost::_bi::list1<Job&> > (this=0x7fffff3face8, f=@0x7fffff3facd8,

a=@0x7fffff3fabd0) at bind.hpp:313

22 0x00000000010a6bbe in boost::_bi::bind_t<void, boost::_mfi::mf1<void, HashedObjectStore, Job&>, boost::_bi::list2boost::_bi::value<HashedObjectStore*, boost::arg<1> > >::operator() (this=0x7fffff3facd8, a1=@0x7fffff3facc0)

at bind_template.hpp:32

23 0x00000000010a6be5 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf1<void, HashedObjectStore, Job&>, boost::_bi::list2boost::_bi::value<HashedObjectStore*, boost::arg<1> > >, void, Job&>::invoke (function_obj_ptr=@0x7fffff3facd8, a0=@0x7fffff3facc0) at function_template.hpp:153

---Type to continue, or q to quit---

24 0x0000000000be51b9 in boost::function1<void, Job&>::operator() (this=0x7fffff3facd0, a0=@0x7fffff3facc0)

at function_template.hpp:760

25 0x0000000000bd4433 in Job::doJob (this=0x7fffff3facc0) at ripple_Job.cpp:41

26 0x0000000000bd4e3e in JobQueue::threadEntry (this=0x804a70b48) at ripple_JobQueue.cpp:268

27 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a943b8, p=0x804a70b48)

at mem_fn_template.hpp:49

28 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a943c8, f=@0x804a943b8, a=@0x7fffff3faf5f) at bind.hpp:253

29 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a943b8) at bind_template.hpp:20

30 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (this=0x804a94300) at thread.hpp:61

31 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

32 0x0000000802a82511 in pthread_getprio () from /lib/libthr.so.3

33 0x0000000000000000 in ?? ()

Error accessing memory address 0x7fffff3fb000: Bad address.

Thread 4 (Thread 804a0aac0 (LWP 100395/rippled)):

0 0x0000000802a8c39c in __error () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in pthread_cond_signal () from /lib/libthr.so.3

2 0x0000000000be9044 in boost::condition_variable::wait (this=0x804a70b50, m=@0x7fffff1f9d20)

at condition_variable.hpp:55

3 0x0000000000bd4c49 in JobQueue::threadEntry (this=0x804a70b48) at ripple_JobQueue.cpp:246

4 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a942b8, p=0x804a70b48)

at mem_fn_template.hpp:49

5 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a942c8, f=@0x804a942b8, a=@0x7fffff1f9f5f) at bind.hpp:253

6 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a942b8) at bind_template.hpp:20

7 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (this=0x804a94200) at thread.hpp:61

8 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

9 0x0000000802a82511 in pthread_getprio () from /lib/libthr.so.3

10 0x0000000000000000 in ?? ()

Error accessing memory address 0x7fffff1fa000: Bad address.

Thread 3 (Thread 804a0a900 (LWP 100402/rippled)):

0 0x0000000802a8c39c in __error () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in pthread_cond_signal () from /lib/libthr.so.3

2 0x000000080232c343 in boost::condition_variable::timed_wait () from /usr/local/lib/libboost_thread.so

3 0x00000008023266de in boost::this_thread::sleep () from /usr/local/lib/libboost_thread.so

4 0x0000000000e5756f in boost::this_thread::sleepboost::posix_time::time_duration (rel_time=@0x7ffffeff8b30)

at thread_data.hpp:138

5 0x0000000000e57a49 in LoadManager::threadEntry (this=0x804a80d00) at ripple_LoadManager.cpp:406

6 0x0000000000e079a2 in boost::_mfi::mf0<void, LoadManager>::operator() (this=0x804a941b8, p=0x804a80d00)

at mem_fn_template.hpp:49

7 0x0000000000e08201 in boost::_bi::list1boost::_bi::value<LoadManager* >::operator()<boost::_mfi::mf0<void, LoadManager>, boost::_bi::list0> (this=0x804a941c8, f=@0x804a941b8, a=@0x7ffffeff8f5f) at bind.hpp:253

8 0x0000000000e08246 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, LoadManager>, boost::_bi::list1boost::_bi::value<LoadManager* > >::operator() (this=0x804a941b8) at bind_template.hpp:20

9 0x0000000000e0826c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, LoadManager>, boost::_bi::list1boost::_bi::value<LoadManager* > > >::run (this=0x804a94100) at thread.hpp:61

---Type to continue, or q to quit---

10 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

11 0x0000000802a82511 in pthread_getprio () from /lib/libthr.so.3

12 0x0000000000000000 in ?? ()

Error accessing memory address 0x7ffffeff9000: Bad address.

Thread 2 (Thread 804a0a740 (LWP 100406/rippled)):

0 0x0000000802a8c39c in __error () from /lib/libthr.so.3

1 0x0000000802a8c08c in __error () from /lib/libthr.so.3

2 0x0000000802a873bc in pthread_mutex_getprioceiling () from /lib/libthr.so.3

3 0x0000000802c0d2fe in _malloc_prefork () from /lib/libc.so.7

4 0x0000000802c11b3d in free () from /lib/libc.so.7

5 0x0000000000c5067a in ~SerializedType (this=0x804a3d500) at ripple_SerializedTypes.h:71

6 0x0000000000c3f167 in boost::checked_delete (x=0x804a3d500) at checked_delete.hpp:34

7 0x0000000000c3f185 in delete_clone (s=0x804a3d500) at ripple_SerializedTypes.h:160

8 0x0000000000c48da5 in boost::heap_clone_allocator::deallocate_clone (r=0x804a3d500)

at clone_allocator.hpp:74

9 0x0000000000c48dc5 in boost::ptr_container_detail::reversible_ptr_container<boost::ptr_container_detail::sequence_config<SerializedType, std::vector<void, std::allocator<void> > >, boost::heap_clone_allocator>::null_clone_allocator::deallocate_clone (x=0x804a3d500) at reversible_ptr_container.hpp:126

10 0x0000000000c48de5 in boost::ptr_container_detail::reversible_ptr_container<boost::ptr_container_detail::sequence_config<SerializedType, std::vector<void, std::allocator<void> > >, boost::heap_clone_allocator>::null_policy_deallocate_clone (x=0x804a3d500) at reversible_ptr_container.hpp:276

11 0x0000000000c48e11 in boost::ptr_container_detail::reversible_ptr_container<boost::ptr_container_detail::sequence_config<SerializedType, std::vector<void, std::allocator<void> > >, boost::heap_clone_allocator>::remove<boost::void_ptr_iterator<__gnu_cxx::__normal_iterator<void*, std::vector<void, std::allocator<void*> > >, SerializedType> > (

this=0x804ab1570, i={iter_ = {_M_current = 0x804ad5400}}) at reversible_ptr_container.hpp:250

12 0x0000000000c48e43 in boost::ptr_container_detail::reversible_ptr_container<boost::ptr_container_detail::sequence_config<SerializedType, std::vector<void, std::allocator<void> > >, boost::heap_clone_allocator>::remove<boost::void_ptr_iterator<__gnu_cxx::__normal_iterator<void*, std::vector<void, std::allocator<void*> > >, SerializedType> > (

this=0x804ab1570, first={iter_ = {_M_current = 0x804ad5400}}, last={iter_ = {_M_current = 0x804ad5480}})
at reversible_ptr_container.hpp:257

13 0x0000000000c48e91 in boost::ptr_container_detail::reversible_ptr_container<boost::ptr_container_detail::sequence_config<SerializedType, std::vector<void, std::allocator<void> > >, boost::heap_clone_allocator>::remove_all (

this=0x804ab1570) at reversible_ptr_container.hpp:218

14 0x0000000000c52c86 in ~reversible_ptr_container (this=0x804ab1570) at reversible_ptr_container.hpp:465

15 0x0000000000c52ce5 in ~ptr_sequence_adapter (this=0x804ab1570) at ptr_sequence_adapter.hpp:132

16 0x0000000000c52d05 in ~ptr_vector (this=0x804ab1570) at ptr_vector.hpp:35

17 0x0000000000e60405 in ~STObject (this=0x804ab1560) at ripple_SerializedObject.h:48

18 0x0000000000e607da in ~SerializedLedgerEntry (this=0x804ab1560) at ripple_SerializedLedger.h:28

19 0x0000000000df3223 in boost::detail::sp_ms_deleter::destroy (this=0x804ab1558)

at make_shared.hpp:56

20 0x0000000000df3249 in boost::detail::sp_ms_deleter::operator() (this=0x804ab1558)

at make_shared.hpp:86

21 0x0000000000e05c61 in boost::detail::sp_counted_impl_pd<SerializedLedgerEntry*, boost::detail::sp_ms_deleter >::dispose (this=0x804ab1540) at sp_counted_impl.hpp:148

22 0x0000000000b9bb2c in boost::detail::sp_counted_base::release (this=0x804ab1540) at sp_counted_base_gcc_x86.hpp:145

23 0x0000000000b9bb84 in ~shared_count (this=0x7ffffedf7428) at shared_count.hpp:305

24 0x0000000000dcfc29 in ~shared_ptr (this=0x7ffffedf7420) at shared_ptr.hpp:165

25 0x0000000000e1b13b in boost::shared_ptr::reset (this=0x804b18308) at shared_ptr.hpp:392

26 0x0000000001025470 in TaggedCache<uint256, SerializedLedgerEntry, UptimeTimerAdapter>::sweep (this=0x804a709a8)

at ripple_TaggedCache.h:240

---Type to continue, or q to quit---

27 0x0000000000fa9789 in Application::sweep (this=0x804a70000) at ripple_Application.cpp:682

28 0x0000000000ff58e2 in boost::_mfi::mf0<void, Application>::operator() (this=0x7ffffedf7bc0, p=0x804a70000)

at mem_fn_template.hpp:49

29 0x0000000000ff5c61 in boost::_bi::list1boost::_bi::value<Application* >::operator()<boost::_mfi::mf0<void, Application>, boost::_bi::list1<boost::system::error_code const&> > (this=0x7ffffedf7bd0, f=@0x7ffffedf7bc0,

a=@0x7ffffedf7ab0) at bind.hpp:253

30 0x0000000000ff5cae in boost::_bi::bind_t<void, boost::_mfi::mf0<void, Application>, boost::_bi::list1boost::_bi::value<Application* > >::operator()boost::system::error_code (this=0x7ffffedf7bc0, a1=@0x7ffffedf7bd8)

at bind_template.hpp:47

31 0x0000000000ff5ccd in boost::asio::detail::binder1<boost::_bi::bind_t<void, boost::_mfi::mf0<void, Application>, boost::_bi::list1boost::_bi::value<Application* > >, boost::system::error_code>::operator() (this=0x7ffffedf7bc0)

at bind_handler.hpp:46

32 0x0000000000ff5d66 in boost::asio::asio_handler_invoke<boost::asio::detail::binder1<boost::_bi::bind_t<void, boost::_mfi::mf0<void, Application>, boost::_bi::list1boost::_bi::value<Application* > >, boost::system::error_code> > (

function=
    {handler_ = {f_ = {f_ = {__pfn = 0x139, __delta = 0}}, l_ = {boost::_bi::storage1<boost::_bi::value<Application* >> = {a1_ = {t_ = 0x804a70000}}, <No data fields>}}, arg1_ = {m_val = 0, m_cat = 0x802317810}})
at handler_invoke_hook.hpp:64

33 0x0000000000ff5dc5 in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::binder1<boost::_bi::bind_t<void, boost::_mfi::mf0<void, Application>, boost::_bi::list1boost::_bi::value<Application > >, boost::system::error_code>, boost::_bi::bind_t<void, boost::_mfi::mf0<void, Application>, boost::_bi::list1boost::_bi::value<Application > > >

(function=@0x7ffffedf7c40, context=@0x7ffffedf7c40) at handler_invoke_helpers.hpp:39

34 0x0000000000ffe42e in boost::asio::detail::wait_handler<boost::_bi::bind_t<void, boost::_mfi::mf0<void, Application>, boost::_bi::list1boost::_bi::value<Application* > > >::do_complete (owner=0x804a1e380, base=0x804a99ac0)

at wait_handler.hpp:68

35 0x0000000000eb77b2 in boost::asio::detail::task_io_service_operation::complete (this=0x804a99ac0,

owner=@0x804a1e380, ec=@0x7ffffedf7ea0, bytes_transferred=0) at task_io_service_operation.hpp:37

36 0x0000000000f40f8f in boost::asio::detail::task_io_service::do_run_one (this=0x804a1e380, lock=@0x7ffffedf7de0,

this_thread=@0x7ffffedf7e30, private_op_queue=@0x7ffffedf7e10, ec=@0x7ffffedf7ea0) at task_io_service.ipp:366

37 0x0000000000f63381 in boost::asio::detail::task_io_service::run (this=0x804a1e380, ec=@0x7ffffedf7ea0)

at task_io_service.ipp:146

38 0x0000000000f63496 in boost::asio::io_service::run (this=0x804a70018) at io_service.ipp:59

39 0x0000000000faba1f in runAux (svc=@0x804a70018) at ripple_Application.cpp:346

40 0x0000000000ff7b6b in boost::_bi::list1boost::reference_wrapper::operator()<void (*)(boost::asio::io_service&), boost::_bi::list0> (this=0x804a6ffc0, f=@0x804a6ffb8, a=@0x7ffffedf7f5f) at bind.hpp:253

41 0x0000000000ff7bb6 in boost::_bi::bind_t<void, void (*)(boost::asio::io_service&), boost::_bi::list1<boost::reference_wrapper > >::operator() (this=0x804a6ffb8) at bind_template.hpp:20

42 0x0000000000ff7bdc in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(boost::asio::io_service&), boost::_bi::list1<boost::reference_wrapper > > >::run (this=0x804a6ff00) at thread.hpp:61

43 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

44 0x0000000802a82511 in pthread_getprio () from /lib/libthr.so.3

45 0x0000000000000000 in ?? ()

Error accessing memory address 0x7ffffedf8000: Bad address.

Thread 1 (Thread 804a0a580 (LWP 100424/rippled)):

0 0x0000000802a8c39c in __error () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in pthread_cond_signal () from /lib/libthr.so.3

2 0x0000000000f30c8e in boost::asio::detail::posix_event::waitboost::asio::detail::scoped_lock (this=0x7ffffebf6e80, lock=@0x7ffffebf6e40) at posix_event.hpp:80

3 0x0000000000f41026 in boost::asio::detail::task_io_service::do_run_one (this=0x804a1e400, lock=@0x7ffffebf6e40,

this_thread=@0x7ffffebf6e90, private_op_queue=@0x7ffffebf6e70, ec=@0x7ffffebf6f00) at task_io_service.ipp:377

---Type to continue, or q to quit---

4 0x0000000000f63381 in boost::asio::detail::task_io_service::run (this=0x804a1e400, ec=@0x7ffffebf6f00)

at task_io_service.ipp:146

5 0x0000000000f63496 in boost::asio::io_service::run (this=0x804a3d1b0) at io_service.ipp:59

6 0x0000000000f63558 in boost::asio::detail::resolver_service_base::work_io_service_runner::operator() (

this=0x804a3d5a8) at resolver_service_base.ipp:32

7 0x0000000000f63579 in boost::asio::detail::posix_thread::funcboost::asio::detail::resolver_service_base::work_io_service_runner::run (this=0x804a3d5a0) at posix_thread.hpp:82

8 0x0000000000f01971 in boost_asio_detail_posix_thread_function (arg=0x804a3d5a0) at posix_thread.ipp:64

9 0x0000000802a82511 in pthread_getprio () from /lib/libthr.so.3

10 0x0000000000000000 in ?? ()

Error accessing memory address 0x7ffffebf7000: Bad address.

JoelKatz commented 11 years ago

The last one, that only happens on startup, should be fixed with https://github.com/ripple/rippled/commit/e7da485373e538874f1e7bf0d8ba9d5a2cb947f4 Currently pushed to develop. Should make it to master in a day or so.

BobWay commented 11 years ago

Latest develop build. New gdb executable

stalled on startup

2013-Jun-27 02:12:24 NetworkOPs:NFO Net LCL 360485AFE76C4307BE979C1585689492986FCB88CFEB3EEE4F10A725D9E68FCA 2013-Jun-27 02:12:24 NetworkOPs:NFO Acquiring consensus ledger 360485AFE76C4307BE979C1585689492986FCB88CFEB3EEE4F10A725D9E68FCA 2013-Jun-27 02:12:24 NetworkOPs:NFO Consensus time for ledger 8 2013-Jun-27 02:12:24 NetworkOPs:NFO LCL is 3DD5E5C7E75861850E5C42B0836C0E6C991C96712A5844B29E5A31A8D62EFE80 2013-Jun-27 02:12:24 ripple_LedgerConsensus:NFO Entering consensus process, watching 2013-Jun-27 02:12:24 ripple_LedgerConsensus:WRN Need consensus ledger 360485AFE76C4307BE979C1585689492986FCB88CFEB3EEE4F10A725D9E68FCA 2013-Jun-27 02:12:24 ripple_LedgerConsensus:NFO Entering consensus with: 3DD5E5C7E75861850E5C42B0836C0E6C991C96712A5844B29E5A31A8D62EFE80 2013-Jun-27 02:12:24 ripple_LedgerConsensus:NFO Correct LCL is: 360485AFE76C4307BE979C1585689492986FCB88CFEB3EEE4F10A725D9E68FCA 2013-Jun-27 02:12:24 ripple_LedgerConsensus:NFO Have the consensus ledger 360485AFE76C4307BE979C1585689492986FCB88CFEB3EEE4F10A725D9E68FCA 2013-Jun-27 02:12:24 ripple_TaggedCache:NFO AcceptedLedger is growing fast 6 of 4 aging at 40 of 60 2013-Jun-27 02:12:33 ripple_LoadManager:WRN Server stalled for 10 seconds. 2013-Jun-27 02:12:34 ripple_TaggedCache:NFO AcceptedLedger is growing fast 6 of 4 aging at 40 of 60 2013-Jun-27 02:12:43 ripple_LoadManager:WRN Server stalled for 20 seconds. 2013-Jun-27 02:12:44 ripple_TaggedCache:NFO AcceptedLedger is growing fast 6 of 4 aging at 40 of 60 2013-Jun-27 02:12:53 ripple_LoadManager:WRN Server stalled for 30 seconds. 2013-Jun-27 02:13:03 ripple_LoadManager:WRN Server stalled for 40 seconds. 2013-Jun-27 02:13:13 ripple_LoadManager:WRN Server stalled for 50 seconds.

(gdb) bt

0 0x0000000802c6ec7c in kevent () from /lib/libc.so.7

1 0x0000000000f40855 in boost::asio::detail::kqueue_reactor::run (this=0x804f55160,

block=true, ops=...) at /usr/local/include/boost/asio/detail/impl/kqueue_reactor.ipp:393

2 0x0000000000f40e8a in boost::asio::detail::task_io_service::do_run_one (

this=0x804e40580, lock=..., this_thread=..., private_op_queue=..., ec=...)
at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:350

3 0x0000000000f63381 in boost::asio::detail::task_io_service::run (this=0x804e40580,

ec=...) at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:146

4 0x0000000000f63496 in boost::asio::io_service::run (this=0x804fdee00)

at /usr/local/include/boost/asio/impl/io_service.ipp:59

5 0x0000000000f634c6 in websocketpp::endpoint_base::run_internal (this=0x804fdee00)

at Subtrees/websocket/src/endpoint.hpp:53

6 0x0000000000f81fae in websocketpp::role::server<websocketpp::endpoint<websocketpp::role::server, websocketpp::socket::autotls, websocketpp::log::logger> >::start_listen (

this=0x804fdee10, e=..., num_threads=0) at Subtrees/websocket/src/roles/server.hpp:347

7 0x0000000000f820ce in websocketpp::role::server<websocketpp::endpoint<websocketpp::role::server, websocketpp::socket::autotls, websocketpp::log::logger> >::listen (this=0x804fdee10,

e=..., num_threads=1) at Subtrees/websocket/src/roles/server.hpp:266

8 0x0000000000e82771 in WSDoor::startListening (this=0x804a737f0)

at ./src/cpp/ripple/WSDoor.cpp:51

9 0x0000000000eec862 in boost::_mfi::mf0<void, WSDoor>::operator() (this=0x804a6fbb8,

p=0x804a737f0) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

10 0x0000000000eecef1 in boost::_bi::list1boost::_bi::value<WSDoor* >::operator()<boost::_mfi::mf0<void, WSDoor>, boost::_bi::list0> (this=0x804a6fbc8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

11 0x0000000000eecf36 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, WSDoor>, boost::_bi::list1boost::_bi::value<WSDoor* > >::operator() (this=0x804a6fbb8)

at /usr/local/include/boost/bind/bind_template.hpp:20

12 0x0000000000eecf5c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, WSDoor>, boost::_bi::list1boost::_bi::value<WSDoor* > > >::run (this=0x804a6fb00)

at /usr/local/include/boost/thread/detail/thread.hpp:61

13 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

14 0x0000000802a82511 in ?? () from /lib/libthr.so.3

15 0x00007ffffe5f5000 in ?? ()

Cannot access memory at address 0x7ffffe7f5000 (gdb)

BobWay commented 11 years ago

2013-Jun-27 02:22:24 ripple_LoadManager:WRN Server stalled for 210 seconds. 2013-Jun-27 02:22:34 ripple_LoadManager:WRN Server stalled for 220 seconds. 2013-Jun-27 02:22:40 ripple_TaggedCache:NFO HashedObjectStore is growing fast 8589 of 8192 aging at 28 of 30 2013-Jun-27 02:22:44 ripple_LoadManager:WRN Server stalled for 230 seconds. 2013-Jun-27 02:22:50 ripple_TaggedCache:NFO HashedObjectStore is growing fast 8856 of 8192 aging at 27 of 30 2013-Jun-27 02:22:54 ripple_LoadManager:WRN Server stalled for 240 seconds. 2013-Jun-27 02:23:00 ripple_TaggedCache:NFO HashedObjectStore is growing fast 8695 of 8192 aging at 28 of 30 2013-Jun-27 02:23:04 ripple_LoadManager:WRN Server stalled for 250 seconds. 2013-Jun-27 02:23:14 ripple_LoadManager:WRN Server stalled for 260 seconds. 2013-Jun-27 02:23:24 ripple_LoadManager:WRN Server stalled for 270 seconds. 2013-Jun-27 02:23:34 ripple_LoadManager:WRN Server stalled for 280 seconds. 2013-Jun-27 02:23:44 ripple_LoadManager:WRN Server stalled for 290 seconds. 2013-Jun-27 02:23:54 ripple_LoadManager:WRN Server stalled for 300 seconds. 2013-Jun-27 02:24:04 ripple_LoadManager:WRN Server stalled for 310 seconds. 2013-Jun-27 02:24:10 ripple_TaggedCache:NFO HashedObjectStore is growing fast 13747 of 8192 aging at 17 of 30 2013-Jun-27 02:24:14 ripple_LoadManager:WRN Server stalled for 320 seconds. 2013-Jun-27 02:24:20 ripple_TaggedCache:NFO HashedObjectStore is growing fast 13317 of 8192 aging at 18 of 30 2013-Jun-27 02:24:24 ripple_LoadManager:WRN Server stalled for 330 seconds. 2013-Jun-27 02:24:24 ripple_LoadMonitor:WRN Job: timerEntry ExecutionTime: 736912 2013-Jun-27 02:24:24 ripple_LoadMonitor:WRN Job: gotLedgerData ExecutionTime: 738731 2013-Jun-27 02:24:28 ripple_LoadMonitor:WRN Job: gotLedgerData ExecutionTime: 723737 2013-Jun-27 02:24:28 ripple_LoadMonitor:WRN Job: PeerImp::ledgerdata ExecutionTime: 723700 2013-Jun-27 02:24:28 ripple_Peer:NFO Peer: Connect: Error: system: Operation timed out: system:60 2013-Jun-27 02:24:28 ripple_Peer:NFO Peer: Connect: Error: system: Operation timed out: system:60 2013-Jun-27 02:24:28 ripple_Peer:NFO Peer: Connect: Outbound: 0000000804A9B400: 176.31.181.172 51235 2013-Jun-27 02:24:28 ripple_LedgerConsensus:NFO Have the consensus ledger 360485AFE76C4307BE979C1585689492986FCB88CFEB3EEE4F10A725D9E68FCA 2013-Jun-27 02:24:28 LedgerTiming:WRN CLC::shouldClose range Trans=no, Prop: 4/0, Secs: 738000 (last:2017) 2013-Jun-27 02:24:28 ripple_LedgerConsensus:NFO initial position 0000000000000000000000000000000000000000000000000000000000000000 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606702 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606702 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606702 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606702 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606711 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606711 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606711 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606711 2013-Jun-27 02:24:28 TransactionEngine:NFO applyTransaction: terResult=terNO_ACCOUNT : -96 : The source account does not exist. 2013-Jun-27 02:24:28 NetworkOPs:NFO TransactionResult: terNO_ACCOUNT: The source account does not exist. 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606732 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606732 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606732 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606732 2013-Jun-27 02:24:28 TransactionEngine:NFO applyTransaction: terResult=terNO_ACCOUNT : -96 : The source account does not exist. 2013-Jun-27 02:24:28 NetworkOPs:NFO TransactionResult: terNO_ACCOUNT: The source account does not exist. 2013-Jun-27 02:24:28 ripple_Peer:NFO Connect peer: success. 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606738 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606738 2013-Jun-27 02:24:28 TransactionEngine:NFO applyTransaction: terResult=terNO_ACCOUNT : -96 : The source account does not exist. 2013-Jun-27 02:24:28 NetworkOPs:NFO TransactionResult: terNO_ACCOUNT: The source account does not exist. 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606738 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606739 2013-Jun-27 02:24:28 ripple_Peer:NFO Peer: Write: Error: 0000000804A9C400: bytes=0: system: Broken pipe: system:32 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606744 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606744 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606745 2013-Jun-27 02:24:28 ripple_Validations:WRN Received stale validation now=425607408, close=425606745 2013-Jun-27 02:24:28 TransactionEngine:NFO applyTransaction: terResult=terNO_ACCOUNT : -96 : The source account does not exist. 2013-Jun-27 02:24:28 NetworkOPs:NFO TransactionResult: terNO_ACCOUNT: The source account does not exist. 2013-Jun-27 02:24:28 TransactionEngine:NFO applyTransaction: terResult=terNO_ACCOUNT : -96 : The source account does not exist. 2013-Jun-27 02:24:28 NetworkOPs:NFO TransactionResult: terNO_ACCOUNT: The source account does not exist. 2013-Jun-27 02:24:28 ripple_LoadManager:NFO {

BobWay commented 11 years ago

(gdb) continue Continuing. [New Thread 804a0a040 (LWP 100439)] [Thread 804a0a040 (LWP 100439) exited] [Thread 804a09e80 (LWP 100439) exited] [Thread 804a09cc0 (LWP 100439) exited] [Thread 804a09b00 (LWP 100439) exited] [Thread 804a0a3c0 (LWP 100439) exited] [Thread 804a0a040 (LWP 100439) exited] [Thread 804a09e80 (LWP 100439) exited] [Thread 804a09cc0 (LWP 100439) exited] [Thread 804a09b00 (LWP 100439) exited] [Thread 804a0a3c0 (LWP 100439) exited] [Thread 804a0a040 (LWP 100439) exited] [Thread 804a09e80 (LWP 100439) exited] [Thread 804a09cc0 (LWP 100439) exited] [Thread 804a09b00 (LWP 100439) exited] [Thread 804a0a3c0 (LWP 100439) exited] [Thread 804a0a040 (LWP 100439) exited] [Thread 804a09e80 (LWP 100439) exited] [Thread 804a09cc0 (LWP 100439) exited] [Thread 804a09b00 (LWP 100439) exited] [Thread 804a0a3c0 (LWP 100439) exited] [Thread 804a0a040 (LWP 100439) exited] [Thread 804a09e80 (LWP 100439) exited] [Thread 804a09cc0 (LWP 100439) exited] [Thread 804a09b00 (LWP 100439) exited] [Thread 804a0a3c0 (LWP 100439) exited] [Thread 804a0a040 (LWP 100439) exited] [Thread 804a09e80 (LWP 100439) exited] [Thread 804a09cc0 (LWP 100439) exited] [Thread 804a09b00 (LWP 100439) exited] [Thread 804a0a3c0 (LWP 100439) exited] [Thread 804a0a040 (LWP 100439) exited]

Program received signal SIGABRT, Aborted. [Switching to Thread 804a0aac0 (LWP 100375)] 0x0000000802be8a4c in thr_kill () from /lib/libc.so.7 (gdb)

BobWay commented 11 years ago

2013-Jun-27 02:28:00 ripple_LedgerConsensus:NFO initial position 0000000000000000000000000000000000000000000000000000000000000000 2013-Jun-27 02:28:02 ripple_LedgerConsensus:NFO Proposers:4 nw:70 thrV:3 thrC:3 2013-Jun-27 02:28:02 LedgerTiming:NFO normal consensus 2013-Jun-27 02:28:02 ripple_LedgerConsensus:NFO Converge cutoff (4 participants) 2013-Jun-27 02:28:02 ripple_LedgerConsensus:NFO CNF newLCL A002496D48136B5693F76B1FE79627BE30C02E6E3DF8512D149403F9A8F6BC69 2013-Jun-27 02:28:02 LedgerMaster:NFO StashAccepted: A002496D48136B5693F76B1FE79627BE30C02E6E3DF8512D149403F9A8F6BC69 2013-Jun-27 02:28:02 LedgerMaster:NFO Advancing accepted ledger to 1146728 with >= 2 validations 2013-Jun-27 02:28:04 NFO STATE->full 2013-Jun-27 02:28:04 NetworkOPs:NFO Consensus time for ledger 1146729 2013-Jun-27 02:28:04 NetworkOPs:NFO LCL is A002496D48136B5693F76B1FE79627BE30C02E6E3DF8512D149403F9A8F6BC69 2013-Jun-27 02:28:04 ripple_LedgerConsensus:NFO Entering consensus process, watching 2013-Jun-27 02:28:10 ripple_TaggedCache:NFO AcceptedLedger is growing fast 6 of 4 aging at 40 of 60 2013-Jun-27 02:28:19 OfferCreateTransactor:NFO OfferCreate: take against book: 4627DFFCFF8B5A265EDBD8AE8C14A52325DBFEDAF4F5C32E0000000000000000 for 140143840/XRP -> 2.768/USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B 2013-Jun-27 02:28:19 OfferCreateTransactor:NFO OfferCreate: offer not fully consumed: saTakerPays=2.768/USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B saTakerGets=140143840/XRP 2013-Jun-27 02:28:19 OfferCreateTransactor:NFO OfferCreate: adding to book: DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE0000000000000000 : USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B -> XRP/rrrrrrrrrrrrrrrrrrrrrhoLvTp terminate called after throwing an instance of 'SHAMapMissingNode' what(): SHAMapMissingNode

BobWay commented 11 years ago

(gdb) bt

0 0x0000000802be8a4c in thr_kill () from /lib/libc.so.7

1 0x0000000802c858ab in abort () from /lib/libc.so.7

2 0x00000008026b5f54 in __gnu_cxx::__verbose_terminate_handler() ()

from /usr/lib/libstdc++.so.6

3 0x00000008026ba453 in ?? () from /usr/lib/libstdc++.so.6

4 0x00000008026ba493 in std::terminate() () from /usr/lib/libstdc++.so.6

5 0x00000008026ba3fa in __cxa_throw () from /usr/lib/libstdc++.so.6

6 0x0000000000d8432d in SHAMap::getNodePointer (this=0x8097db0e0, id=..., hash=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:236

7 0x0000000000d87973 in SHAMap::walkToPointer (this=0x8097db0e0, id=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:197

8 0x0000000000d87c23 in SHAMap::peekItem (this=0x8097db0e0, id=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:551

9 0x0000000000d93755 in Ledger::getSLE (this=0x8096a48a0, uHash=...)

at ./src/cpp/ripple/Ledger.cpp:1134

10 0x0000000000faf546 in LedgerEntrySet::entryCache (this=0x804a70118, letType=ltDIR_NODE,

index=...) at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:99

11 0x0000000000fb8eb9 in LedgerEntrySet::dirAdd(unsigned long long&, uint256 const&, uint256 const&, boost::function<void (boost::shared_ptr const&)>) (

this=0x804a70118, uNodeDir=@0x7fffff1f2b30: 0, uRootIndex=..., uLedgerIndex=..., 
fDescriber=...) at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:639

12 0x0000000000fc1506 in OfferCreateTransactor::doApply (this=0x80550a380)

at ./src/cpp/ripple/OfferCreateTransactor.cpp:659

13 0x0000000000e73c60 in Transactor::apply (this=0x80550a380)

at ./src/cpp/ripple/Transactor.cpp:251

14 0x0000000000e772f9 in TransactionEngine::applyTransaction (this=0x804a70118, txn=...,

params=(tapNO_CHECK_SIGN | tapOPEN_LEDGER), didApply=@0x7fffff1f8fff: false)
at ./src/cpp/ripple/TransactionEngine.cpp:102

15 0x000000000107b5fe in LedgerMaster::doTransaction (this=0x804a700f0, txn=...,

params=(tapNO_CHECK_SIGN | tapOPEN_LEDGER), didApply=@0x7fffff1f8fff: false)
at ./src/cpp/ripple/LedgerMaster.cpp:170

16 0x0000000001080e1f in NetworkOPs::processTransaction(boost::shared_ptr, bool, bool, boost::function<void (boost::shared_ptr, TER)>) (this=0x804a704d8,

trans=..., bAdmin=false, bFailHard=false, callback=...)
at ./src/cpp/ripple/NetworkOPs.cpp:350

17 0x0000000000e648c3 in NetworkOPs::processTransaction (this=0x804a704d8, transaction=...,

bAdmin=false, bFailHard=false) at ./src/cpp/ripple/NetworkOPs.h:142

18 0x0000000000fcb4ce in checkTransaction (flags=0, stx=..., peer=...)

at ./src/cpp/ripple/ripple_Peer.cpp:1081

19 0x0000000001000e52 in boost::_bi::list4boost::arg<1, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr >::operator()<void (*)(Job&, int, boost::shared_ptr, boost::weak_ptr), boost::_bi::list1<Job&> > (this=0x805404828,

f=@0x805404820: 0xfcb260 <checkTransaction(Job&, int, boost::shared_ptr<SerializedTransaction>&, boost::weak_ptr<Peer>&)>, a=...) at /usr/local/include/boost/bind/bind.hpp:457

20 0x0000000001000eee in boost::_bi::bind_t<void, void (*)(Job&, int, boost::shared_ptr, boost::weak_ptr), boost::_bi::list4boost::arg<1, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >::operator() (this=0x805404820, a1=...)

at /usr/local/include/boost/bind/bind_template.hpp:32

21 0x0000000001000f18 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, void (*)(Job&, int, boost::shared_ptr, boost::weak_ptr), boost::_bi::list4boost::arg<1, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >, void, Job&>::invoke

(function_obj_ptr=..., a0=...)
at /usr/local/include/boost/function/function_template.hpp:153

22 0x0000000000be51b9 in boost::function1<void, Job&>::operator() (this=0x7fffff1f9cd0,

---Type to continue, or q to quit--- a0=...) at /usr/local/include/boost/function/function_template.hpp:760

23 0x0000000000bd4433 in Job::doJob (this=0x7fffff1f9cc0)

at modules/ripple_core/functional/ripple_Job.cpp:41

24 0x0000000000bd4e3e in JobQueue::threadEntry (this=0x804a70b48)

at modules/ripple_core/functional/ripple_JobQueue.cpp:268

25 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a942b8,

p=0x804a70b48) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

26 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a942c8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

27 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a942b8)

at /usr/local/include/boost/bind/bind_template.hpp:20

28 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (

this=0x804a94200) at /usr/local/include/boost/thread/detail/thread.hpp:61

29 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

30 0x0000000802a82511 in ?? () from /lib/libthr.so.3

31 0x0000000000000000 in ?? ()

(gdb)

BobWay commented 11 years ago

(gdb) thread apply all where

Thread 11 (Thread 804a041c0 (LWP 100442)):

0 0x0000000802a8c39c in ?? () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in ?? () from /lib/libthr.so.3

2 0x0000000000bd1d8b in boost::recursive_mutex::lock (this=0x804a090d8)

at /usr/local/include/boost/thread/pthread/recursive_mutex.hpp:133

3 0x0000000000bd1e66 in boost::unique_lockboost::recursive_mutex::lock (

this=0x7fffffffa230) at /usr/local/include/boost/thread/locks.hpp:412

4 0x0000000000bd1eac in boost::unique_lockboost::recursive_mutex::unique_lock (

this=0x7fffffffa230, m_=...) at /usr/local/include/boost/thread/locks.hpp:290

5 0x0000000000e6a22d in InboundLedger::awaitData (this=0x804a090a0)

at ./src/cpp/ripple/ripple_InboundLedger.cpp:171

6 0x0000000001054009 in InboundLedgers::awaitLedgerData (this=0x804a703e0, ledgerHash=...)

at ./src/cpp/ripple/ripple_InboundLedgers.cpp:74

7 0x0000000000fc4e23 in PeerImp::recvLedger (this=0x804a9a400, packet_ptr=...,

MasterLockHolder=...) at ./src/cpp/ripple/ripple_Peer.cpp:2072

8 0x0000000000fce7a9 in PeerImp::processReadBuffer (this=0x804a9a400)

at ./src/cpp/ripple/ripple_Peer.cpp:833

9 0x0000000000fcfdbc in PeerImp::handleReadBody (this=0x804a9a400, error=...)

at ./src/cpp/ripple/ripple_Peer.cpp:631

10 0x0000000000ff65bb in boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>::callboost::shared_ptr<PeerImp, boost::system::error_code const> (this=0x7fffffffd330,

u=..., b1=...) at /usr/local/include/boost/bind/mem_fn_template.hpp:156

11 0x0000000000ff65f1 in boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>::operator()boost::shared_ptr (this=0x7fffffffd330, u=..., a1=...)

at /usr/local/include/boost/bind/mem_fn_template.hpp:171

12 0x0000000000ff67b7 in boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()>::operator()<boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2<boost::system::error_code const&, unsigned long const&> > (

this=0x7fffffffd340, f=..., a=...) at /usr/local/include/boost/bind/bind.hpp:313

13 0x0000000000ff6806 in boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >::operator()<boost::system::error_code, unsigned long> (

this=0x7fffffffd330, a1=..., a2=@0x7fffffffd360: 585)
at /usr/local/include/boost/bind/bind_template.hpp:102

14 0x0000000000ff6835 in boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >, boost::system::error_code, unsigned long>::operator() (this=0x7fffffffd330) at /usr/local/include/boost/asio/detail/bind_handler.hpp:118

15 0x0000000000ff68d9 in boost::asio::asio_handler_invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >, boost::system::error_code, unsigned long> > (function=...)

at /usr/local/include/boost/asio/handler_invoke_hook.hpp:64

16 0x000000000100196b in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > (function=..., context=...)

at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:39

17 0x00000000010019bd in boost::asio::detail::asio_handler_invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >, boost::system::error_code, unsigned long> (function=...,

---Type to continue, or q to quit--- this_handler=0x7fffffffd4b0) at /usr/local/include/boost/asio/detail/bind_handler.hpp:153

18 0x00000000010019e5 in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >, boost::system::error_code, unsigned long>, boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >, boost::system::error_code, unsigned long> > (function=..., context=...)

at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:39

19 0x000000000102c53b in boost::asio::detail::strand_service::dispatch<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >, boost::system::error_code, unsigned long> > (this=0x804b78400,

impl=@0x7fffffffd998: 0x8056d3330, handler=...)
at /usr/local/include/boost/asio/detail/impl/strand_service.hpp:66

20 0x000000000102c786 in boost::asio::io_service::strand::dispatch<boost::asio::detail::binder2<boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> >, boost::system::error_code, unsigned long> > (this=0x7fffffffd990, handler=...)

at /usr/local/include/boost/asio/strand.hpp:150

21 0x000000000102c811 in boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > >::operator()<boost::system::error_code, unsigned long> (this=0x7fffffffd990, arg1=...,

arg2=@0x7fffffffd988: 585) at /usr/local/include/boost/asio/detail/wrapped_handler.hpp:79

22 0x0000000001031aa1 in boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > >::operator() (this=0x7fffffffd970, ec=...,

bytes_transferred=585, start=0) at /usr/local/include/boost/asio/impl/read.hpp:274

23 0x0000000001031bd1 in boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1::call_handler<boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > > (this=0x7fffffffd940, handler=..., ec=...,

bytes_transferred=@0x7fffffffd7b0: 585)
at /usr/local/include/boost/asio/ssl/detail/read_op.hpp:59

24 0x0000000001031786 in boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >::operator() (

this=0x7fffffffd930, ec=..., bytes_transferred=0, start=0)
at /usr/local/include/boost/asio/ssl/detail/io.hpp:260

25 0x0000000001033138 in boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_se---Type to continue, or q to quit---

rvice::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>::operator() (this=0x7fffffffd930) at /usr/local/include/boost/asio/detail/bind_handler.hpp:118

26 0x0000000001033159 in boost::asio::detail::rewrapped_handler<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > >::operator() (this=0x7fffffffd910)

at /usr/local/include/boost/asio/detail/wrapped_handler.hpp:171

27 0x00000000010331f9 in boost::asio::asio_handler_invoke<boost::asio::detail::rewrapped_handler<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > (function=...)

at /usr/local/include/boost/asio/handler_invoke_hook.hpp:64

28 0x0000000001033250 in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::rewrapped_handler<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > >, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > (function=..., context=...)

at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:39

29 0x00000000010332bd in boost::asio::detail::asio_handler_invoke<boost::asio::detail::rewrapped_handler<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > >, boost::asio::detail::binder2<boost::asio::ssl::detail:---Type to continue, or q to quit---

:io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > (function=..., this_handler=0x7fffffffda70) at /usr/local/include/boost/asio/detail/wrapped_handler.hpp:238

30 0x00000000010332e5 in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::rewrapped_handler<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > >, boost::asio::detail::rewrapped_handler<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > (function=..., context=...)

at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:39

31 0x00000000010333b4 in boost::asio::detail::completion_handler<boost::asio::detail::rewrapped_handler<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > >::do_complete (owner=0x804a1e300, base=0x804a6d800)

at /usr/local/include/boost/asio/detail/completion_handler.hpp:67

32 0x0000000001033610 in boost::asio::detail::strand_service::dispatch<boost::asio::detail::rewrapped_handler<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const---Type to continue, or q to quit---

&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > (this=0x804b78400, impl=@0x7fffffffdf98: 0x8056d3330, handler=...) at /usr/local/include/boost/asio/detail/impl/strand_service.hpp:92

33 0x00000000010336fe in boost::asio::io_service::strand::dispatch<boost::asio::detail::rewrapped_handler<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > (this=0x7fffffffdf90, handler=...)

at /usr/local/include/boost/asio/strand.hpp:150

34 0x000000000103379b in boost::asio::detail::asio_handler_invoke<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > (function=..., this_handler=0x7fffffffdf90)

at /usr/local/include/boost/asio/detail/wrapped_handler.hpp:204

35 0x0000000001033805 in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > (function=...,

context=...) at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:39

36 0x0000000001033831 in boost::asio::detail::asio_handler_invoke<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::stra---Type to continue, or q to quit---

nd, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > (function=..., this_handler=0x7fffffffdf70) at /usr/local/include/boost/asio/impl/read.hpp:474

37 0x0000000001033865 in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > > (function=..., context=...)

at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:39

38 0x0000000001033891 in boost::asio::ssl::detail::asio_handler_invoke<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > > (function=..., this_handler=0x7fffffffdf30)

at /usr/local/include/boost/asio/ssl/detail/io.hpp:305

39 0x00000000010338c5 in boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > >, boost::system::error_code, unsigned long>, boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::sharedptr, boost::arg<1> ()()> > > > > > (function=..., context=...)

at /usr/local/include/boost/asio/detail/handler_invoke_helpers.hpp:39

40 0x00000000010339a4 in boost::asio::detail::reactive_socket_recv_op<boost::asio::mutable_bu---Type to continue, or q to quit---

ffers_1, boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service >, boost::asio::ssl::detail::read_opboost::asio::mutable_buffers_1, boost::asio::detail::read_op<boost::asio::ssl::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service > >, boost::asio::mutable_buffers_1, boost::asio::detail::transfer_all_t, boost::asio::detail::wrapped_handler<boost::asio::io_service::strand, boost::_bi::bind_t<void, boost::_mfi::mf1<void, PeerImp, boost::system::error_code const&>, boost::_bi::list2boost::_bi::value<boost::shared_ptr, boost::arg<1> (*)()> > > > > >::do_complete ( owner=0x804a1e300, base=0x804a6d800) at /usr/local/include/boost/asio/detail/reactive_socket_recv_op.hpp:110

41 0x0000000000eb77b2 in boost::asio::detail::task_io_service_operation::complete (

this=0x804a6d800, owner=..., ec=..., bytes_transferred=0)
at /usr/local/include/boost/asio/detail/task_io_service_operation.hpp:37

42 0x0000000000f40f8f in boost::asio::detail::task_io_service::do_run_one (

this=0x804a1e300, lock=..., this_thread=..., private_op_queue=..., ec=...)
at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:366

43 0x0000000000f63381 in boost::asio::detail::task_io_service::run (this=0x804a1e300,

ec=...) at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:146

44 0x0000000000f63496 in boost::asio::io_service::run (this=0x804a70008)

at /usr/local/include/boost/asio/impl/io_service.ipp:59

45 0x0000000000fa9946 in Application::run (this=0x804a70000)

at ./src/cpp/ripple/ripple_Application.cpp:652

46 0x0000000000d9df75 in startServer () at ./src/cpp/ripple/main.cpp:47

47 0x0000000000dcc7ec in main (argc=2, argv=0x7fffffffeae0) at ./src/cpp/ripple/main.cpp:266

Thread 10 (Thread 804a8d540 (LWP 100129)):

0 0x0000000802a8c39c in ?? () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in ?? () from /lib/libthr.so.3

2 0x0000000000bd1d8b in boost::recursive_mutex::lock (this=0x804e37a98)

at /usr/local/include/boost/thread/pthread/recursive_mutex.hpp:133

3 0x0000000000bd1e66 in boost::unique_lockboost::recursive_mutex::lock (

this=0x7fffffbfabc0) at /usr/local/include/boost/thread/locks.hpp:412

4 0x0000000000bd1eac in boost::unique_lockboost::recursive_mutex::unique_lock (

this=0x7fffffbfabc0, m_=...) at /usr/local/include/boost/thread/locks.hpp:290

5 0x000000000105b8ba in HashedObjectStore::retrieveSQLite (this=0x804a708a8, hash=...)

at ./src/cpp/ripple/ripple_HashedObjectStore.cpp:443

6 0x0000000000e62677 in HashedObjectStore::retrieve (this=0x804a708a8, hash=...)

at ./src/cpp/ripple/ripple_HashedObjectStore.h:42

7 0x0000000000d827df in SHAMap::fetchNodeExternalNT (this=0x804ff9820, id=..., hash=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:827

8 0x0000000000d83428 in SHAMap::getNodePointerNT (this=0x804ff9820, id=..., hash=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:249

9 0x0000000000d83486 in SHAMap::getNodePointerNT (this=0x804ff9820, id=..., hash=...,

filter=0x7fffffbfbcc0) at ./src/cpp/ripple/ripple_SHAMap.cpp:264

10 0x0000000000d84048 in SHAMap::getMissingNodes (this=0x804ff9820, nodeIDs=...,

hashes=..., max=238, filter=0x7fffffbfbcc0) at ./src/cpp/ripple/ripple_SHAMapSync.cpp:55

11 0x0000000000e90221 in InboundLedger::trigger (this=0x804a09260, peer=...)

at ./src/cpp/ripple/ripple_InboundLedger.cpp:466

12 0x0000000001053da9 in InboundLedgers::gotLedgerData (this=0x804a703e0, hash=...,

packet_ptr=..., wPeer=...) at ./src/cpp/ripple/ripple_InboundLedgers.cpp:179

13 0x0000000000fffa1b in boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >::operator() (this=0x804c70700,

p=0x804a703e0, a1=..., a2=..., a3=..., a4=...)
at /usr/local/include/boost/bind/mem_fn_template.hpp:506

14 0x0000000000fffb64 in boost::_bi::list5boost::_bi::value<InboundLedgers*, boost::arg<1>, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr >::operator()<boost::_mfi::mf4<void, InboundLedgers, J---Type to continue, or q to quit---

ob&, uint256, boost::shared_ptrprotocol::TMLedgerData, boost::weak_ptr >, boost::_bi::list1<Job&> > (this=0x804c70710, f=..., a=...) at /usr/local/include/boost/bind/bind.hpp:525

15 0x0000000000fffbfe in boost::_bi::bind_t<void, boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >, boost::_bi::list5boost::_bi::value<InboundLedgers*, boost::arg<1>, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >::operator() (this=0x804c70700, a1=...)

at /usr/local/include/boost/bind/bind_template.hpp:32

16 0x0000000000fffc28 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >, boost::_bi::list5boost::_bi::value<InboundLedgers*, boost::arg<1>, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >, void, Job&>::invoke (

function_obj_ptr=..., a0=...)
at /usr/local/include/boost/function/function_template.hpp:153

17 0x0000000000be51b9 in boost::function1<void, Job&>::operator() (this=0x7fffffbfecd0,

a0=...) at /usr/local/include/boost/function/function_template.hpp:760

18 0x0000000000bd4433 in Job::doJob (this=0x7fffffbfecc0)

at modules/ripple_core/functional/ripple_Job.cpp:41

19 0x0000000000bd4e3e in JobQueue::threadEntry (this=0x804a70b48)

at modules/ripple_core/functional/ripple_JobQueue.cpp:268

20 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a6c7b8,

p=0x804a70b48) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

21 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a6c7c8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

22 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a6c7b8)

at /usr/local/include/boost/bind/bind_template.hpp:20

23 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (

this=0x804a6c700) at /usr/local/include/boost/thread/detail/thread.hpp:61

24 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

25 0x0000000802a82511 in ?? () from /lib/libthr.so.3

26 0x0000000000000000 in ?? ()

Thread 9 (Thread 804a8d380 (LWP 100142)):

0 0x0000000802a8c39c in ?? () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in ?? () from /lib/libthr.so.3

2 0x0000000000bd1d8b in boost::recursive_mutex::lock (this=0x804a08f18)

at /usr/local/include/boost/thread/pthread/recursive_mutex.hpp:133

3 0x0000000000bd1e66 in boost::unique_lockboost::recursive_mutex::lock (

this=0x7fffff9fd980) at /usr/local/include/boost/thread/locks.hpp:412

4 0x0000000000bd1eac in boost::unique_lockboost::recursive_mutex::unique_lock (

this=0x7fffff9fd980, m_=...) at /usr/local/include/boost/thread/locks.hpp:290

5 0x000000000104effa in PeerSet::invokeOnTimer (this=0x804a08ee0)

at ./src/cpp/ripple/ripple_PeerSet.cpp:47

6 0x000000000104f221 in PeerSet::TimerJobEntry (ptr=...)

at ./src/cpp/ripple/ripple_PeerSet.cpp:91

7 0x00000000010b1e29 in boost::_bi::list2boost::arg<1, boost::_bi::valueboost::shared_ptr >::operator()<void (*)(Job&, boost::shared_ptr), boost::_bi::list1<Job&> > (this=0x7fffff9fdce0,

f=@0x7fffff9fdcd8: 0x104f200 <PeerSet::TimerJobEntry(Job&, boost::shared_ptr<PeerSet>)>, 
a=...) at /usr/local/include/boost/bind/bind.hpp:313

8 0x00000000010b1e9e in boost::_bi::bind_t<void, void (*)(Job&, boost::shared_ptr), boost::_bi::list2boost::arg<1, boost::_bi::valueboost::shared_ptr > >::operator() (this=0x7fffff9fdcd8, a1=...) at /usr/local/include/boost/bind/bind_template.hpp:32

---Type to continue, or q to quit---

9 0x00000000010b1ec5 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, void (*)(Job&, boost::shared_ptr), boost::_bi::list2boost::arg<1, boost::_bi::valueboost::shared_ptr > >, void, Job&>::invoke (function_obj_ptr=...,

a0=...) at /usr/local/include/boost/function/function_template.hpp:153

10 0x0000000000be51b9 in boost::function1<void, Job&>::operator() (this=0x7fffff9fdcd0,

a0=...) at /usr/local/include/boost/function/function_template.hpp:760

11 0x0000000000bd4433 in Job::doJob (this=0x7fffff9fdcc0)

at modules/ripple_core/functional/ripple_Job.cpp:41

12 0x0000000000bd4e3e in JobQueue::threadEntry (this=0x804a70b48)

at modules/ripple_core/functional/ripple_JobQueue.cpp:268

13 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a946b8,

p=0x804a70b48) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

14 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a946c8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

15 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a946b8)

at /usr/local/include/boost/bind/bind_template.hpp:20

16 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (

this=0x804a94600) at /usr/local/include/boost/thread/detail/thread.hpp:61

17 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

18 0x0000000802a82511 in ?? () from /lib/libthr.so.3

19 0x0000000000000000 in ?? ()

Thread 8 (Thread 804a8d1c0 (LWP 100235)):

0 0x0000000802a8c39c in ?? () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in ?? () from /lib/libthr.so.3

2 0x0000000000be9044 in boost::condition_variable::wait (this=0x804a70b50, m=...)

at /usr/local/include/boost/thread/pthread/condition_variable.hpp:55

3 0x0000000000bd4c49 in JobQueue::threadEntry (this=0x804a70b48)

at modules/ripple_core/functional/ripple_JobQueue.cpp:246

4 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a945b8,

p=0x804a70b48) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

5 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a945c8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

6 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a945b8)

at /usr/local/include/boost/bind/bind_template.hpp:20

7 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (

this=0x804a94500) at /usr/local/include/boost/thread/detail/thread.hpp:61

8 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

9 0x0000000802a82511 in ?? () from /lib/libthr.so.3

10 0x0000000000000000 in ?? ()

Thread 7 (Thread 804a0ae40 (LWP 100243)):

0 0x0000000802a8c39c in ?? () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in ?? () from /lib/libthr.so.3

2 0x0000000000bd1d8b in boost::recursive_mutex::lock (this=0x804e37a98)

at /usr/local/include/boost/thread/pthread/recursive_mutex.hpp:133

3 0x0000000000bd1e66 in boost::unique_lockboost::recursive_mutex::lock (

this=0x7fffff5f7bc0) at /usr/local/include/boost/thread/locks.hpp:412

4 0x0000000000bd1eac in boost::unique_lockboost::recursive_mutex::unique_lock (

this=0x7fffff5f7bc0, m_=...) at /usr/local/include/boost/thread/locks.hpp:290

5 0x000000000105b8ba in HashedObjectStore::retrieveSQLite (this=0x804a708a8, hash=...)

---Type to continue, or q to quit--- at ./src/cpp/ripple/ripple_HashedObjectStore.cpp:443

6 0x0000000000e62677 in HashedObjectStore::retrieve (this=0x804a708a8, hash=...)

at ./src/cpp/ripple/ripple_HashedObjectStore.h:42

7 0x0000000000d827df in SHAMap::fetchNodeExternalNT (this=0x804ab1260, id=..., hash=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:827

8 0x0000000000d83428 in SHAMap::getNodePointerNT (this=0x804ab1260, id=..., hash=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:249

9 0x0000000000d83486 in SHAMap::getNodePointerNT (this=0x804ab1260, id=..., hash=...,

filter=0x7fffff5f8cc0) at ./src/cpp/ripple/ripple_SHAMap.cpp:264

10 0x0000000000d84048 in SHAMap::getMissingNodes (this=0x804ab1260, nodeIDs=...,

hashes=..., max=239, filter=0x7fffff5f8cc0) at ./src/cpp/ripple/ripple_SHAMapSync.cpp:55

11 0x0000000000e90221 in InboundLedger::trigger (this=0x804a090a0, peer=...)

at ./src/cpp/ripple/ripple_InboundLedger.cpp:466

12 0x00000000010538e6 in InboundLedgers::gotLedgerData (this=0x804a703e0, hash=...,

packet_ptr=..., wPeer=...) at ./src/cpp/ripple/ripple_InboundLedgers.cpp:134

13 0x0000000000fffa1b in boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >::operator() (this=0x804ad5a60,

p=0x804a703e0, a1=..., a2=..., a3=..., a4=...)
at /usr/local/include/boost/bind/mem_fn_template.hpp:506

14 0x0000000000fffb64 in boost::_bi::list5boost::_bi::value<InboundLedgers*, boost::arg<1>, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr >::operator()<boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >, boost::_bi::list1<Job&> > (this=0x804ad5a70, f=..., a=...) at /usr/local/include/boost/bind/bind.hpp:525

15 0x0000000000fffbfe in boost::_bi::bind_t<void, boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >, boost::_bi::list5boost::_bi::value<InboundLedgers*, boost::arg<1>, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >::operator() (this=0x804ad5a60, a1=...)

at /usr/local/include/boost/bind/bind_template.hpp:32

16 0x0000000000fffc28 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >, boost::_bi::list5boost::_bi::value<InboundLedgers*, boost::arg<1>, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >, void, Job&>::invoke (

function_obj_ptr=..., a0=...)
at /usr/local/include/boost/function/function_template.hpp:153

17 0x0000000000be51b9 in boost::function1<void, Job&>::operator() (this=0x7fffff5fbcd0,

a0=...) at /usr/local/include/boost/function/function_template.hpp:760

18 0x0000000000bd4433 in Job::doJob (this=0x7fffff5fbcc0)

at modules/ripple_core/functional/ripple_Job.cpp:41

19 0x0000000000bd4e3e in JobQueue::threadEntry (this=0x804a70b48)

at modules/ripple_core/functional/ripple_JobQueue.cpp:268

20 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a944b8,

p=0x804a70b48) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

21 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a944c8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

22 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a944b8)

at /usr/local/include/boost/bind/bind_template.hpp:20

23 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (

this=0x804a94400) at /usr/local/include/boost/thread/detail/thread.hpp:61

24 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

25 0x0000000802a82511 in ?? () from /lib/libthr.so.3

26 0x0000000000000000 in ?? ()

---Type to continue, or q to quit---

Thread 6 (Thread 804a0ac80 (LWP 100326)):

0 0x0000000802c88acc in read () from /lib/libc.so.7

1 0x0000000802a84760 in read () from /lib/libthr.so.3

2 0x0000000000cde143 in seekAndRead ()

3 0x0000000000cde203 in unixRead ()

4 0x0000000000cd4bb3 in sqlite3OsRead ()

5 0x0000000000ce5cd8 in readDbPage ()

6 0x0000000000ce859d in sqlite3PagerAcquire ()

7 0x0000000000ceff5c in btreeGetPage ()

8 0x0000000000cf007e in getAndInitPage ()

9 0x0000000000cf3827 in moveToChild ()

10 0x0000000000cf438d in sqlite3BtreeMovetoUnpacked ()

11 0x0000000000d0a479 in sqlite3VdbeExec ()

12 0x0000000000d03241 in sqlite3Step ()

13 0x0000000000d0342c in sqlite3_step ()

14 0x0000000000e843e8 in SqliteStatement::step (

this=0x1479398 <HashedObjectStore::retrieveSQLite(uint256 const&)::pSt>)
at ./src/cpp/ripple/ripple_SqliteDatabase.cpp:397

15 0x000000000105baed in HashedObjectStore::retrieveSQLite (this=0x804a708a8, hash=...)

at ./src/cpp/ripple/ripple_HashedObjectStore.cpp:449

16 0x0000000000e62677 in HashedObjectStore::retrieve (this=0x804a708a8, hash=...)

at ./src/cpp/ripple/ripple_HashedObjectStore.h:42

17 0x0000000000d827df in SHAMap::fetchNodeExternalNT (this=0x805717ba0, id=..., hash=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:827

18 0x0000000000d83428 in SHAMap::getNodePointerNT (this=0x805717ba0, id=..., hash=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:249

19 0x0000000000d83486 in SHAMap::getNodePointerNT (this=0x805717ba0, id=..., hash=...,

filter=0x7fffff3f7cc0) at ./src/cpp/ripple/ripple_SHAMap.cpp:264

20 0x0000000000d84048 in SHAMap::getMissingNodes (this=0x805717ba0, nodeIDs=...,

hashes=..., max=238, filter=0x7fffff3f7cc0) at ./src/cpp/ripple/ripple_SHAMapSync.cpp:55

21 0x0000000000e90221 in InboundLedger::trigger (this=0x804a08ee0, peer=...)

at ./src/cpp/ripple/ripple_InboundLedger.cpp:466

22 0x0000000001053da9 in InboundLedgers::gotLedgerData (this=0x804a703e0, hash=...,

packet_ptr=..., wPeer=...) at ./src/cpp/ripple/ripple_InboundLedgers.cpp:179

23 0x0000000000fffa1b in boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >::operator() (this=0x804fd4b20,

p=0x804a703e0, a1=..., a2=..., a3=..., a4=...)
at /usr/local/include/boost/bind/mem_fn_template.hpp:506

24 0x0000000000fffb64 in boost::_bi::list5boost::_bi::value<InboundLedgers*, boost::arg<1>, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr >::operator()<boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >, boost::_bi::list1<Job&> > (this=0x804fd4b30, f=..., a=...) at /usr/local/include/boost/bind/bind.hpp:525

25 0x0000000000fffbfe in boost::_bi::bind_t<void, boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >, boost::_bi::list5boost::_bi::value<InboundLedgers*, boost::arg<1>, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >::operator() (this=0x804fd4b20, a1=...)

at /usr/local/include/boost/bind/bind_template.hpp:32

26 0x0000000000fffc28 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf4<void, InboundLedgers, Job&, uint256, boost::shared_ptr, boost::weak_ptr >, boost::_bi::list5boost::_bi::value<InboundLedgers*, boost::arg<1>, boost::_bi::value, boost::_bi::valueboost::shared_ptr, boost::_bi::valueboost::weak_ptr > >, void, Job&>::invoke (

function_obj_ptr=..., a0=...)
at /usr/local/include/boost/function/function_template.hpp:153

---Type to continue, or q to quit---

27 0x0000000000be51b9 in boost::function1<void, Job&>::operator() (this=0x7fffff3facd0,

a0=...) at /usr/local/include/boost/function/function_template.hpp:760

28 0x0000000000bd4433 in Job::doJob (this=0x7fffff3facc0)

at modules/ripple_core/functional/ripple_Job.cpp:41

29 0x0000000000bd4e3e in JobQueue::threadEntry (this=0x804a70b48)

at modules/ripple_core/functional/ripple_JobQueue.cpp:268

30 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a943b8,

p=0x804a70b48) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

31 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a943c8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

32 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a943b8)

at /usr/local/include/boost/bind/bind_template.hpp:20

33 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (

this=0x804a94300) at /usr/local/include/boost/thread/detail/thread.hpp:61

34 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

35 0x0000000802a82511 in ?? () from /lib/libthr.so.3

36 0x0000000000000000 in ?? ()

Thread 5 (Thread 804a0aac0 (LWP 100375)):

0 0x0000000802a8c39c in ?? () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in ?? () from /lib/libthr.so.3

2 0x0000000000bd1d8b in boost::recursive_mutex::lock (this=0x804a09298)

at /usr/local/include/boost/thread/pthread/recursive_mutex.hpp:133

3 0x0000000000bd1e66 in boost::unique_lockboost::recursive_mutex::lock (

this=0x7fffff1f9980) at /usr/local/include/boost/thread/locks.hpp:412

4 0x0000000000bd1eac in boost::unique_lockboost::recursive_mutex::unique_lock (

this=0x7fffff1f9980, m_=...) at /usr/local/include/boost/thread/locks.hpp:290

5 0x000000000104effa in PeerSet::invokeOnTimer (this=0x804a09260)

at ./src/cpp/ripple/ripple_PeerSet.cpp:47

6 0x000000000104f221 in PeerSet::TimerJobEntry (ptr=...)

at ./src/cpp/ripple/ripple_PeerSet.cpp:91

7 0x00000000010b1e29 in boost::_bi::list2boost::arg<1, boost::_bi::valueboost::shared_ptr >::operator()<void (*)(Job&, boost::shared_ptr), boost::_bi::list1<Job&> > (this=0x7fffff1f9ce0,

f=@0x7fffff1f9cd8: 0x104f200 <PeerSet::TimerJobEntry(Job&, boost::shared_ptr<PeerSet>)>, 
a=...) at /usr/local/include/boost/bind/bind.hpp:313

8 0x00000000010b1e9e in boost::_bi::bind_t<void, void (*)(Job&, boost::shared_ptr), boost::_bi::list2boost::arg<1, boost::_bi::valueboost::shared_ptr > >::operator() (this=0x7fffff1f9cd8, a1=...) at /usr/local/include/boost/bind/bind_template.hpp:32

9 0x00000000010b1ec5 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, void (*)(Job&, boost::shared_ptr), boost::_bi::list2boost::arg<1, boost::_bi::valueboost::shared_ptr > >, void, Job&>::invoke (function_obj_ptr=...,

a0=...) at /usr/local/include/boost/function/function_template.hpp:153

10 0x0000000000be51b9 in boost::function1<void, Job&>::operator() (this=0x7fffff1f9cd0,

a0=...) at /usr/local/include/boost/function/function_template.hpp:760

11 0x0000000000bd4433 in Job::doJob (this=0x7fffff1f9cc0)

at modules/ripple_core/functional/ripple_Job.cpp:41

12 0x0000000000bd4e3e in JobQueue::threadEntry (this=0x804a70b48)

at modules/ripple_core/functional/ripple_JobQueue.cpp:268

13 0x0000000000be2812 in boost::_mfi::mf0<void, JobQueue>::operator() (this=0x804a942b8,

p=0x804a70b48) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

14 0x0000000000be29d1 in boost::_bi::list1boost::_bi::value<JobQueue* >::operator()<boost::_mfi::mf0<void, JobQueue>, boost::_bi::list0> (this=0x804a942c8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

---Type to continue, or q to quit---

15 0x0000000000be2a16 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > >::operator() (this=0x804a942b8)

at /usr/local/include/boost/bind/bind_template.hpp:20

16 0x0000000000be2a3c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, JobQueue>, boost::_bi::list1boost::_bi::value<JobQueue* > > >::run (

this=0x804a94200) at /usr/local/include/boost/thread/detail/thread.hpp:61

17 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

18 0x0000000802a82511 in ?? () from /lib/libthr.so.3

19 0x0000000000000000 in ?? ()

Thread 4 (Thread 804a0a900 (LWP 100395)):

0 0x0000000802a8c39c in ?? () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in ?? () from /lib/libthr.so.3

2 0x000000080232c343 in boost::condition_variable::timed_wait(boost::unique_lockboost::mutex&, boost::posix_time::ptime const&) () from /usr/local/lib/libboost_thread.so

3 0x00000008023266de in boost::this_thread::sleep(boost::posix_time::ptime const&) ()

from /usr/local/lib/libboost_thread.so

4 0x0000000000e5756f in boost::this_thread::sleepboost::posix_time::time_duration (

rel_time=...) at /usr/local/include/boost/thread/pthread/thread_data.hpp:138

5 0x0000000000e57a49 in LoadManager::threadEntry (this=0x804a80d00)

at ./src/cpp/ripple/ripple_LoadManager.cpp:406

6 0x0000000000e079a2 in boost::_mfi::mf0<void, LoadManager>::operator() (this=0x804a941b8,

p=0x804a80d00) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

7 0x0000000000e08201 in boost::_bi::list1boost::_bi::value<LoadManager* >::operator()<boost::_mfi::mf0<void, LoadManager>, boost::_bi::list0> (this=0x804a941c8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

8 0x0000000000e08246 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, LoadManager>, boost::_bi::list1boost::_bi::value<LoadManager* > >::operator() (this=0x804a941b8)

at /usr/local/include/boost/bind/bind_template.hpp:20

9 0x0000000000e0826c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, LoadManager>, boost::_bi::list1boost::_bi::value<LoadManager* > > >::run (

this=0x804a94100) at /usr/local/include/boost/thread/detail/thread.hpp:61

10 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

11 0x0000000802a82511 in ?? () from /lib/libthr.so.3

12 0x0000000000000000 in ?? ()

Thread 3 (Thread 804a0a740 (LWP 100401)):

0 0x0000000802c6ec7c in kevent () from /lib/libc.so.7

1 0x0000000000f40855 in boost::asio::detail::kqueue_reactor::run (this=0x804a14be0,

block=true, ops=...) at /usr/local/include/boost/asio/detail/impl/kqueue_reactor.ipp:393

2 0x0000000000f40e8a in boost::asio::detail::task_io_service::do_run_one (

this=0x804a1e380, lock=..., this_thread=..., private_op_queue=..., ec=...)
at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:350

3 0x0000000000f63381 in boost::asio::detail::task_io_service::run (this=0x804a1e380,

ec=...) at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:146

4 0x0000000000f63496 in boost::asio::io_service::run (this=0x804a70018)

at /usr/local/include/boost/asio/impl/io_service.ipp:59

5 0x0000000000faba5f in runAux (svc=...) at ./src/cpp/ripple/ripple_Application.cpp:346

6 0x0000000000ff7bab in boost::_bi::list1boost::reference_wrapper::operator()<void (*)(boost::asio::io_service&), boost::_bi::list0> (this=0x804a6ffc0,

f=@0x804a6ffb8: 0xfaba40 <runAux(boost::asio::io_service&)>, a=...)
at /usr/local/include/boost/bind/bind.hpp:253

7 0x0000000000ff7bf6 in boost::_bi::bind_t<void, void (*)(boost::asio::io_service&), boost::_bi::list1<boost::reference_wrapper > >::operator() (

this=0x804a6ffb8) at /usr/local/include/boost/bind/bind_template.hpp:20

8 0x0000000000ff7c1c in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(boost::asio::io_service&), boost::_bi::list1<boost::reference_wrapper > > >:---Type to continue, or q to quit---

:run (this=0x804a6ff00) at /usr/local/include/boost/thread/detail/thread.hpp:61

9 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

10 0x0000000802a82511 in ?? () from /lib/libthr.so.3

11 0x0000000000000000 in ?? ()

Thread 2 (Thread 804a0a580 (LWP 100427)):

0 0x0000000802a8c39c in ?? () from /lib/libthr.so.3

1 0x0000000802a8a4d1 in ?? () from /lib/libthr.so.3

2 0x0000000000f30c8e in boost::asio::detail::posix_event::waitboost::asio::detail::scoped_lock (this=0x7ffffebf6e80, lock=...)

at /usr/local/include/boost/asio/detail/posix_event.hpp:80

3 0x0000000000f41026 in boost::asio::detail::task_io_service::do_run_one (

this=0x804a1e400, lock=..., this_thread=..., private_op_queue=..., ec=...)
at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:377

4 0x0000000000f63381 in boost::asio::detail::task_io_service::run (this=0x804a1e400,

ec=...) at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:146

5 0x0000000000f63496 in boost::asio::io_service::run (this=0x804a3d1b0)

at /usr/local/include/boost/asio/impl/io_service.ipp:59

6 0x0000000000f63558 in boost::asio::detail::resolver_service_base::work_io_service_runner::operator() (this=0x804a3d5a8)

at /usr/local/include/boost/asio/detail/impl/resolver_service_base.ipp:32

7 0x0000000000f63579 in boost::asio::detail::posix_thread::funcboost::asio::detail::resolver_service_base::work_io_service_runner::run (this=0x804a3d5a0)

at /usr/local/include/boost/asio/detail/posix_thread.hpp:82

8 0x0000000000f01971 in boost::asio::detail::boost_asio_detail_posix_thread_function (

arg=0x804a3d5a0) at /usr/local/include/boost/asio/detail/impl/posix_thread.ipp:64

9 0x0000000802a82511 in ?? () from /lib/libthr.so.3

10 0x0000000000000000 in ?? ()

Thread 1 (Thread 804a0a200 (LWP 100438)):

0 0x0000000802c6ec7c in kevent () from /lib/libc.so.7

1 0x0000000000f40855 in boost::asio::detail::kqueue_reactor::run (this=0x804f55160,

block=true, ops=...) at /usr/local/include/boost/asio/detail/impl/kqueue_reactor.ipp:393

2 0x0000000000f40e8a in boost::asio::detail::task_io_service::do_run_one (

this=0x804e40580, lock=..., this_thread=..., private_op_queue=..., ec=...)
at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:350

3 0x0000000000f63381 in boost::asio::detail::task_io_service::run (this=0x804e40580,

ec=...) at /usr/local/include/boost/asio/detail/impl/task_io_service.ipp:146

4 0x0000000000f63496 in boost::asio::io_service::run (this=0x804fdee00)

at /usr/local/include/boost/asio/impl/io_service.ipp:59

5 0x0000000000f634c6 in websocketpp::endpoint_base::run_internal (this=0x804fdee00)

at Subtrees/websocket/src/endpoint.hpp:53

6 0x0000000000f81fae in websocketpp::role::server<websocketpp::endpoint<websocketpp::role::server, websocketpp::socket::autotls, websocketpp::log::logger> >::start_listen (

this=0x804fdee10, e=..., num_threads=0) at Subtrees/websocket/src/roles/server.hpp:347

7 0x0000000000f820ce in websocketpp::role::server<websocketpp::endpoint<websocketpp::role::server, websocketpp::socket::autotls, websocketpp::log::logger> >::listen (this=0x804fdee10,

e=..., num_threads=1) at Subtrees/websocket/src/roles/server.hpp:266

8 0x0000000000e82771 in WSDoor::startListening (this=0x804a737f0)

at ./src/cpp/ripple/WSDoor.cpp:51

9 0x0000000000eec862 in boost::_mfi::mf0<void, WSDoor>::operator() (this=0x804a6fbb8,

p=0x804a737f0) at /usr/local/include/boost/bind/mem_fn_template.hpp:49

10 0x0000000000eecef1 in boost::_bi::list1boost::_bi::value<WSDoor* >::operator()<boost::_mfi::mf0<void, WSDoor>, boost::_bi::list0> (this=0x804a6fbc8, f=..., a=...)

at /usr/local/include/boost/bind/bind.hpp:253

11 0x0000000000eecf36 in boost::_bi::bind_t<void, boost::_mfi::mf0<void, WSDoor>, boost::_bi::list1boost::_bi::value<WSDoor* > >::operator() (this=0x804a6fbb8)

---Type to continue, or q to quit--- at /usr/local/include/boost/bind/bind_template.hpp:20

12 0x0000000000eecf5c in boost::detail::thread_data<boost::_bi::bind_t<void, boost::_mfi::mf0<void, WSDoor>, boost::_bi::list1boost::_bi::value<WSDoor* > > >::run (this=0x804a6fb00)

at /usr/local/include/boost/thread/detail/thread.hpp:61

13 0x0000000802325062 in thread_proxy () from /usr/local/lib/libboost_thread.so

14 0x0000000802a82511 in ?? () from /lib/libthr.so.3

15 0x00007ffffe5f5000 in ?? ()

Cannot access memory at address 0x7ffffe7f5000 (gdb)

alexdupre commented 11 years ago

I experienced the "terminate called after throwing an instance of 'ripple::SHAMapMissingNode'" with the same backtrace in first comment with latest develop branch.

JoelKatz commented 11 years ago

@alexdupre What really puzzles me is this -- this bug seems to have something to do with corruption of the current ledger or a node in the current ledger not being in the local database. You would expect a bug like that to cause random failures in all kinds of code paths that access the current ledger. But we always see this very same stack trace -- always the OfferCreateTransactor calling LES::getNextLedgerIndex calling SHAMap::getStack.

If this happens again, please give me the last two or three lines from your debug log file.

alexdupre commented 11 years ago

I still have the log file. Here are the log lines for the last 5 seconds before crashing:

2013-Jun-29 08:02:50 ripple_LedgerConsensus:NFO initial position 0000000000000000000000000000000000000000000000000000000000000000 2013-Jun-29 08:02:52 ripple_LedgerConsensus:NFO Proposers:4 nw:70 thrV:3 thrC:3 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG CCTime: seq1164496: 425808170 has 4, 3 required 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Close time consensus reached: 425808170 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Checking for TX consensus: agree=4, disagree=0 2013-Jun-29 08:02:52 LedgerTiming:NFO normal consensus 2013-Jun-29 08:02:52 ripple_LedgerConsensus:NFO Converge cutoff (4 participants) 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Report: Prop=no val=no corLCL=yes fail=no 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Report: Prev = 584242561C93BBD385666145F269E010D1823D029AFD6649E4222331E73B1EC6:1164495 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Report: TxSt = 0000000000000000000000000000000000000000000000000000000000000000, close 425808170 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Applying consensus set transactions to the last closed ledger 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Pass: 0 Txns: 0 retriable 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Pass: 0 finished 0 changes 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Pass: 1 Txns: 0 final 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Pass: 1 finished 0 changes 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Report: NewL = 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7:1164496 2013-Jun-29 08:02:52 ripple_LedgerConsensus:NFO CNF newLCL 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Applying transactions from current open ledger 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Pass: 0 Txns: 0 retriable 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Pass: 0 finished 0 changes 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Pass: 1 Txns: 0 final 2013-Jun-29 08:02:52 ripple_LedgerConsensus:DBG Pass: 1 finished 0 changes 2013-Jun-29 08:02:52 LedgerMaster:NFO StashAccepted: 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 2013-Jun-29 08:02:52 NetworkOPs:DBG recvValidation 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 from 23.23.201.55 2013-Jun-29 08:02:52 ripple_Validations:DBG Val for 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 from n9LeQeDcLDMZKjx1TZtrXoLBLo5q1bR1sUQrWG7tEADFU6R27UBp added trusted/ current 2013-Jun-29 08:02:52 LedgerMaster:DBG no prior missing ledger, not resuming 2013-Jun-29 08:02:52 NetworkOPs:DBG recvValidation 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 from 23.23.201.55 2013-Jun-29 08:02:52 ripple_Validations:DBG Val for 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 from n9KPnVLn7ewVzHvn218DcEYsnWLzKerTDwhpofhk4Ym1RUq4TeGw added trusted/ current 2013-Jun-29 08:02:52 LedgerMaster:NFO Advancing accepted ledger to 1164496 with >= 2 validations 2013-Jun-29 08:02:52 LedgerMaster:DBG Publishing ledger 1164496 2013-Jun-29 08:02:52 LedgerMaster:DBG Ledger 1164496 accepted :441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 2013-Jun-29 08:02:52 LedgerMaster:DBG Ledger 1157719 is needed 2013-Jun-29 08:02:53 ripple_Peer:DBG Got new transaction from peer 2013-Jun-29 08:02:53 NetworkOPs:DBG recvValidation 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 from 23.23.201.55 2013-Jun-29 08:02:53 ripple_Validations:DBG Val for 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 from n9LFzWuhKNvXStHAuemfRKFVECLApowncMAM5chSCL9R5ECHGN4V added trusted/ current 2013-Jun-29 08:02:53 NetworkOPs:DBG recvValidation 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 from 23.23.201.55 2013-Jun-29 08:02:53 ripple_Validations:DBG Val for 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 from n94rSdgTyBNGvYg8pZXGuNt59Y5bGAZGxbxyvjDaqD9ceRAgD85P added trusted/current 2013-Jun-29 08:02:53 OfferCreateTransactor:NFO OfferCreate: take against book: DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE0000000000000000 for 7.57/USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B -> 514003000/XRP 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers: BEFORE saTakerGets=7.57/USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG takeOffers: bSell: 1: against book: DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE0000000000000000 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG takeOffers: possible counter offer found: uTipQuality=5549954497759356847 uTipIndex=DFA3B6DDAB58C7E8E5D944E736DA4B7046C30E4F460FD9DE4D056635F68727AF 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG takeOffers: done: dir=1 uTakeQuality=5549907494970524709 < uTipQuality=5549954497759356847 saTakerRate=0.00000001472754050073637/1/1 < saTipRate=0.00000001519756838905775/1/1 bPassive=0 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG takeOffers: tesSUCCESS 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG takeOffers< tesSUCCESS 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers=0 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers: saPaid=0/USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers: saGot=0/XRP 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers: AFTER saTakerPays=514003000/XRP 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers: AFTER saTakerGets=7.57/USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers: saTakerPays=514003000/XRP 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers: saTakerGets=7.57/USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers: mTxnAccountID=rNAAy9xnjuU6McAjVFtMyFbDNKzTXQ9wbV 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: takeOffers: FUNDS=7.579526387225548/USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B 2013-Jun-29 08:02:53 OfferCreateTransactor:NFO OfferCreate: offer not fully consumed: saTakerPays=514003000/XRP saTakerGets=7.57/USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B 2013-Jun-29 08:02:53 OfferCreateTransactor:NFO OfferCreate: adding to book: 4627DFFCFF8B5A265EDBD8AE8C14A52325DBFEDAF4F5C32E0000000000000000 : XRP/rrrrrrrrrrrrrrrrrrrrrhoLvTp -> USD/rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: sfAccount=rNAAy9xnjuU6McAjVFtMyFbDNKzTXQ9wbV 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: uPaysIssuerID=rrrrrrrrrrrrrrrrrrrrrhoLvTp 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: uGetsIssuerID=rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: uPaysCurrency=XRP 2013-Jun-29 08:02:53 OfferCreateTransactor:DBG OfferCreate: uGetsCurrency=USD 2013-Jun-29 08:02:53 OfferCreateTransactor:NFO OfferCreate: final terResult=tesSUCCESS sleOffer= { "Account" : "rNAAy9xnjuU6McAjVFtMyFbDNKzTXQ9wbV", "BookDirectory" : "4627DFFCFF8B5A265EDBD8AE8C14A52325DBFEDAF4F5C32E5C181F7812066000", "BookNode" : "0000000000000000", "Flags" : 131072, "LedgerEntryType" : "Offer", "OwnerNode" : "0000000000000000", "PreviousTxnID" : "0000000000000000000000000000000000000000000000000000000000000000", "PreviousTxnLgrSeq" : 0, "Sequence" : 1285, "TakerGets" : { "currency" : "USD", "issuer" : "rvYAfWj5gh67oV6fW32ZzP3Aw4Eubs59B", "value" : "7.57" }, "TakerPays" : "514003000", "index" : "029EF22BD3034839A9676C0F8F35C87203BBE895E5979B2C3D4B23053CB2623A" }

2013-Jun-29 08:02:53 TransactionEngine:NFO applyTransaction: terResult=tesSUCCESS : 0 : The transaction was applied. 2013-Jun-29 08:02:53 TransactionEngine:NFO applyTransaction: taaCREATE: { 029EF22BD3034839A9676C0F8F35C87203BBE895E5979B2C3D4B23053CB2623A, {, , Offer, 131072, rNAAy9xnjuU6McAjVFtMyFbDNKzTX Q9wbV, 1285, 514003000, 7.57, 4627DFFCFF8B5A265EDBD8AE8C14A52325DBFEDAF4F5C32E5C181F7812066000, 0, 0, CC0FF27788BAAC227F9043DF2D6617C945DE601F7B010894421D86DF5251F11A, 1164497, } } 2013-Jun-29 08:02:53 TransactionEngine:NFO applyTransaction: taaMODIFY: { 0A016E805D09CD4BBC2E142BF047DB6EDB9F8ED2E713DEA2996E50BFE7EC4DBC, {, , DirectoryNode, 0, rNAAy9xnjuU6McAjVFtMyFbDNK zTXQ9wbV, , , , , , , 0A016E805D09CD4BBC2E142BF047DB6EDB9F8ED2E713DEA2996E50BFE7EC4DBC, , } } 2013-Jun-29 08:02:53 TransactionEngine:NFO applyTransaction: taaCREATE: { 4627DFFCFF8B5A265EDBD8AE8C14A52325DBFEDAF4F5C32E5C181F7812066000, {, , DirectoryNode, 0, , 000000000000000000000000 0000000000000000, 0000000000000000000000000000000000000000, 0000000000000000000000005553440000000000, 0A20B3C85F482532A9578DBB3950B85CA06594D1, 6636088651489370112, , 4627DFFCFF8B5A265EDBD8 AE8C14A52325DBFEDAF4F5C32E5C181F7812066000, , } } 2013-Jun-29 08:02:53 TransactionEngine:NFO applyTransaction: taaMODIFY: { 9F249BAF1A390202DBB788ACB584784BB4AD944BA7A35310F380AC3153AE6197, {, , AccountRoot, 0, rNAAy9xnjuU6McAjVFtMyFbDNKzT XQ9wbV, 1286, 13322306639, 11, CC0FF27788BAAC227F9043DF2D6617C945DE601F7B010894421D86DF5251F11A, 1164497, , , , , , , } } 2013-Jun-29 08:02:53 NetworkOPs:NFO Transaction is now included in open ledger 2013-Jun-29 08:02:53 ripple_LoadMonitor:NFO Job: LedgerMaster::asyncAccept ExecutionTime: 635 2013-Jun-29 08:02:54 NetworkOPs:DBG L: 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 t=4, n=2 2013-Jun-29 08:02:54 NetworkOPs:NFO Consensus time for ledger 1164497 2013-Jun-29 08:02:54 NetworkOPs:NFO LCL is 441EE344F4707F34D7A489F9C2C5E429DD27BBDF651BCED9E80E89EEB9A3D0F7 2013-Jun-29 08:02:54 ripple_LedgerConsensus:DBG Creating consensus object 2013-Jun-29 08:02:54 ripple_LedgerConsensus:NFO Entering consensus process, watching 2013-Jun-29 08:02:54 NetworkOPs:DBG Initiating consensus engine 2013-Jun-29 08:02:54 ripple_LedgerConsensus:NFO initial position F8140EEFD39CAA4DE488EE1AC5296076C39FE76A13D4B1DAE118B79F98834246 2013-Jun-29 08:02:54 ripple_LoadMonitor:NFO Job: LedgerMaster::asyncAccept ExecutionTime: 630 2013-Jun-29 08:02:55 ripple_Peer:DBG Got new transaction from peer 2013-Jun-29 08:02:55 OfferCreateTransactor:NFO OfferCreate: take against book: FAE6F85BC792DE5D6E89D97862792CE1D01F47A3DD18F0E20000000000000000 for 500000000/XRP -> 48.45/CNY/razqQKzJRdB4UxFPWf5NEpEG3WMkmwgcXA 2013-Jun-29 08:02:55 OfferCreateTransactor:DBG OfferCreate: takeOffers: BEFORE saTakerGets=500000000/XRP 2013-Jun-29 08:02:55 OfferCreateTransactor:DBG takeOffers: bSell: 1: against book: FAE6F85BC792DE5D6E89D97862792CE1D01F47A3DD18F0E20000000000000000 2013-Jun-29 08:02:55 ripple_TaggedCache:NFO LedgerCache is growing fast 298 of 32 aging at 3 of 30 2

alexdupre commented 11 years ago

I got a different backtrace, now in OfferCancelTransactor:

Thread 8 (Thread 804809000 (LWP 100256)):

0 0x00000008042d638c in thr_kill () from /lib/libc.so.7

1 0x000000080437899b in abort () from /lib/libc.so.7

2 0x0000000803bf89e7 in __gnu_cxx::__verbose_terminate_handler () at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/vterminate.cc:95

3 0x0000000803bf5e16 in cxxabiv1::terminate (handler=) at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_terminate.cc:40

4 0x0000000803bf5e63 in std::terminate () at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_terminate.cc:50

5 0x0000000803bf5f5e in cxxabiv1::cxa_throw (obj=0x83ef58c40, tinfo=, dest=) at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_throw.cc:83

6 0x000000000106d63a in ripple::SHAMap::getNodePointer (this=0x8060909a0, id=..., hash=...) at ./src/cpp/ripple/ripple_SHAMap.cpp:236

7 0x000000000106d497 in ripple::SHAMap::walkToPointer (this=0x8060909a0, id=...) at ./src/cpp/ripple/ripple_SHAMap.cpp:197

8 0x000000000106ec3f in ripple::SHAMap::peekItem (this=0x8060909a0, id=...) at ./src/cpp/ripple/ripple_SHAMap.cpp:551

9 0x00000000010213e7 in ripple::Ledger::getSLE (this=0x804eed560, uHash=...) at ./src/cpp/ripple/Ledger.cpp:1137

10 0x000000000122804e in ripple::LedgerEntrySet::entryCache (this=0x8049400e0, letType=ripple::ltDIR_NODE, index=...) at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:99

11 0x000000000122cb01 in ripple::LedgerEntrySet::dirDelete (this=0x8049400e0, bKeepRoot=false, uNodeDir=@0x7fffff5f8190: 0, uRootIndex=..., uLedgerIndex=..., bStable=true, bSoft=true) at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:735

12 0x000000000122e09e in ripple::LedgerEntrySet::offerDelete (this=0x8049400e0, sleOffer=..., uOfferIndex=..., uOwnerID=...) at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:1056

13 0x0000000001078607 in ripple::OfferCancelTransactor::doApply (this=0x804ef6580) at ./src/cpp/ripple/OfferCancelTransactor.cpp:41

14 0x00000000010ca305 in ripple::Transactor::apply (this=0x804ef6580) at ./src/cpp/ripple/Transactor.cpp:251

15 0x00000000010c5d3b in ripple::TransactionEngine::applyTransaction (this=0x8049400d8, txn=..., params=(ripple::tapNO_CHECK_SIGN | ripple::tapOPEN_LEDGER), didApply=@0x7fffff5fa23f: false) at ./src/cpp/ripple/TransactionEngine.cpp:102

16 0x00000000012be90c in ripple::LedgerMaster::doTransaction (this=0x8049400d0, txn=..., params=(ripple::tapNO_CHECK_SIGN | ripple::tapOPEN_LEDGER), didApply=@0x7fffff5fa23f: false) at ./src/cpp/ripple/LedgerMaster.cpp:170

17 0x000000000125e0ad in ripple::NetworkOPs::processTransaction(boost::shared_ptrripple::Transaction, bool, bool, std::tr1::function<void (boost::shared_ptr, ripple::TER)>) (this=0x804940478, trans=...,

bAdmin=false, bFailHard=false, callback=...) at ./src/cpp/ripple/NetworkOPs.cpp:337

18 0x000000000107b942 in ripple::NetworkOPs::processTransaction (this=0x804940478, transaction=..., bAdmin=false, bFailHard=false) at ./src/cpp/ripple/NetworkOPs.h:142

19 0x00000000011c5192 in ripple::checkTransaction (flags=0, stx=..., peer=...) at ./src/cpp/ripple/ripple_Peer.cpp:1082

20 0x00000000012031c2 in std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer))(ripple::Job&, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer)>::call<ripple::Job&, 0, 1, 2, 3>(std::tr1::tupleripple::Job& const&, std::tr1::_Index_tuple<0, 1, 2, 3>) (this=0x805745ca0, args=...) at /usr/local/lib/gcc46/include/c++/tr1/functional:1153

21 0x00000000011faf46 in std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer))(ripple::Job&, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer)>::operator()ripple::Job(ripple::Job&) (this=0x805745ca0) at /usr/local/lib/gcc46/include/c++/tr1/functional:1207

22 0x00000000011f1dac in std::tr1::_Function_handler<void (ripple::Job&), std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer))(ripple::Job&, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer)> >::_M_invoke(std::tr1::_Any_data const&, ripple::Job&) (functor=..., args#0=...) at /usr/local/lib/gcc46/include/c++/tr1/functional:1684

23 0x0000000000ebcb48 in std::tr1::function<void (ripple::Job&)>::operator()(ripple::Job&) const (this=0x7fffff5facf0, __args#0=...) at /usr/local/lib/gcc46/include/c++/tr1/functional:2040

24 0x0000000000eb5022 in ripple::Job::doJob (this=0x7fffff5face0) at modules/ripple_core/functional/ripple_Job.cpp:41

25 0x0000000000eb6bb6 in ripple::JobQueue::threadEntry (this=0x804940a48) at modules/ripple_core/functional/ripple_JobQueue.cpp:268

26 0x0000000000ec616e in std::tr1::_Mem_fn<void (ripple::JobQueue::*)()>::operator() (this=0x80481d9d0, __object=0x804940a48) at /usr/local/lib/gcc46/include/c++/tr1/functional:572

27 0x0000000000ec6062 in std::tr1::_Bind<std::tr1::_Memfn<void (ripple::JobQueue::*)()> (ripple::JobQueue)>::call<, 0>(std::tr1::tuple<> const&, std::tr1::_Index_tuple<0>) (this=0x80481d9d0, args=...)

at /usr/local/lib/gcc46/include/c++/tr1/functional:1153

28 0x0000000000ec5db2 in std::tr1::_Bind<std::tr1::_Memfn<void (ripple::JobQueue::)()> (ripple::JobQueue_)>::operator()<>() (this=0x80481d9d0) at /usr/local/lib/gcc46/include/c++/tr1/functional:1207

29 0x0000000000ec59a0 in boost::detail::thread_data<std::tr1::_Bind<std::tr1::_Memfn<void (ripple::JobQueue::)()> (ripple::JobQueue*)> >::run() (this=0x80481d900) at /usr/local/include/boost/thread/detail/thread.hpp:78

30 0x0000000803311933 in thread_proxy () from /usr/local/lib/libboost_thread.so.5

31 0x00000008023520a4 in ?? () from /lib/libthr.so.3

32 0x0000000000000000 in ?? ()

Last log lines:

2013-Jul-05 09:40:17 OfferCancelTransactor:DBG OfferCancel: uAccountSequenceNext=147628 uOfferSequence=147147 2013-Jul-05 09:40:17 OfferCancelTransactor:WRN OfferCancel: uOfferSequence=147147 2013-Jul-05 09:40:17 TransactionEngine:NFO applyTransaction: terResult=tesSUCCESS : 0 : The transaction was applied. 2013-Jul-05 09:40:17 TransactionEngine:NFO applyTransaction: taaMODIFY: { 5B990836B96EE489F45FB62755479EAF0D80F17ECA627A07F1493ECD26634484, {, , DirectoryNode, 0, rLs4G81Z45GbNZgHXWHawuDogzWxAFCKW3, , , , , , , 135F93C06E0A230DBCF843DD10 E5DCE302C1EB2EF6E13DFF194CA9C8DB73232A, 442, 441} } 2013-Jul-05 09:40:17 TransactionEngine:NFO applyTransaction: taaMODIFY: { A0083FBE1409241D31E598D48915E59AD782690D816C5C13104E9D8BBF77BE95, {, , AccountRoot, 0, rLs4G81Z45GbNZgHXWHawuDogzWxAFCKW3, 147628, 209314575424, 113, B100B8619A652 6415C777C27ED71CCB949BDF3C87AF9E4B349C7AD3377FA681E, 1219003, , , , , , , } } 2013-Jul-05 09:40:17 TransactionEngine:NFO applyTransaction: taaDELETE: { D780BB63C22A1AF3EA6131BEDC53CB87512D4137AFB39A0B30B1E2F24142501A, {, , Offer, 0, rLs4G81Z45GbNZgHXWHawuDogzWxAFCKW3, 147147, 154.14138, 0.459, FE4D53B02BC5D46DE095 166E0667A0F3797F8A782F8A203B570BEE43DE15B000, 0, 441, BF856833F6ADF5813ED202FA224E4F812ED23FBC18F9D4ABADE584393F5801AF, 1214132, } } 2013-Jul-05 09:40:17 TransactionEngine:NFO applyTransaction: taaMODIFY: { DB7B89154788F6713C0A882A04141327C1641345F5D2F34337056ACB6A483556, {, , DirectoryNode, 0, rLs4G81Z45GbNZgHXWHawuDogzWxAFCKW3, , , , , , , 135F93C06E0A230DBCF843DD10 E5DCE302C1EB2EF6E13DFF194CA9C8DB73232A, 443, 441} } 2013-Jul-05 09:40:17 TransactionEngine:NFO applyTransaction: taaDELETE: { FE4D53B02BC5D46DE095166E0667A0F3797F8A782F8A203B570BEE43DE15B000, {, , DirectoryNode, 0, , 000000000000000000000000494C530000000000, 92D705968936C419CE614BF264B5EE B1CEA47FF4, 0000000000000000000000004254430000000000, 92D705968936C419CE614BF264B5EEB1CEA47FF4, 6272368881299730432, , FE4D53B02BC5D46DE095166E0667A0F3797F8A782F8A203B570BEE43DE15B000, , } } 2013-Jul-05 09:40:17 NetworkOPs:NFO Transaction is now included in open ledger 2013-Jul-05 09:40:17 OfferCancelTransactor:DBG OfferCancel: uAccountSequenceNext=99 uOfferSequence=93 2013-Jul-05 09:40:17 OfferCancelTransactor:WRN OfferCancel: uOfferSequence=93

JoelKatz commented 11 years ago

Interesting. The basic failure is the same -- a node in the open ledger wasn't found in any of the caches or in the database. Needless to say, a server should always have available, at worst in the HashedObjectStore, every node in its own open ledger. Do we only see this on FreeBSD? It's really hard to imagine how this could be an OS-specific bug.

I have a crazy theory -- does this only happen early in the server's run time? Or can the server be stable for some time and then this happens?

alexdupre commented 11 years ago

I'm quite sure it's not FreeBSD specific...yesterday Vinnie was affected by this issue, I doubt he was running the server on FreeBSD, since the reported message was "getNodePointer() threw SHAMapMissingNode with a call stack that has no try".

alexdupre commented 11 years ago

Another different backtrace for the same assertion:

Thread 8 (Thread 804809000 (LWP 100308)):

0 0x00000008042d638c in thr_kill () from /lib/libc.so.7

1 0x000000080437899b in abort () from /lib/libc.so.7

2 0x0000000803bf89e7 in __gnu_cxx::__verbose_terminate_handler () at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/vterminate.cc:95

3 0x0000000803bf5e16 in cxxabiv1::terminate (handler=) at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_terminate.cc:40

4 0x0000000803bf5e63 in std::terminate () at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_terminate.cc:50

5 0x0000000803bf5f5e in cxxabiv1::cxa_throw (obj=0x8057fb440, tinfo=, dest=) at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_throw.cc:83

6 0x000000000106d63a in ripple::SHAMap::getNodePointer (this=0x80548fc60, id=..., hash=...) at ./src/cpp/ripple/ripple_SHAMap.cpp:236

7 0x000000000106d497 in ripple::SHAMap::walkToPointer (this=0x80548fc60, id=...) at ./src/cpp/ripple/ripple_SHAMap.cpp:197

8 0x000000000106ec3f in ripple::SHAMap::peekItem (this=0x80548fc60, id=...) at ./src/cpp/ripple/ripple_SHAMap.cpp:551

9 0x00000000010213e7 in ripple::Ledger::getSLE (this=0x804b537e0, uHash=...) at ./src/cpp/ripple/Ledger.cpp:1137

10 0x000000000122804e in ripple::LedgerEntrySet::entryCache (this=0x8049400e0, letType=ripple::ltACCOUNT_ROOT, index=...) at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:99

11 0x000000000103371a in ripple::TransactionEngine::entryCache (this=0x8049400d8, type=ripple::ltACCOUNT_ROOT, index=...) at ./src/cpp/ripple/TransactionEngine.h:78

12 0x00000000010c9ff2 in ripple::Transactor::apply (this=0x804ff2c00) at ./src/cpp/ripple/Transactor.cpp:213

13 0x00000000010c5d3b in ripple::TransactionEngine::applyTransaction (this=0x8049400d8, txn=..., params=(ripple::tapNO_CHECK_SIGN | ripple::tapOPEN_LEDGER), didApply=@0x7fffff5fa23f: false) at ./src/cpp/ripple/TransactionEngine.cpp:102

14 0x00000000012be90c in ripple::LedgerMaster::doTransaction (this=0x8049400d0, txn=..., params=(ripple::tapNO_CHECK_SIGN | ripple::tapOPEN_LEDGER), didApply=@0x7fffff5fa23f: false) at ./src/cpp/ripple/LedgerMaster.cpp:170

15 0x000000000125e0ad in ripple::NetworkOPs::processTransaction(boost::shared_ptrripple::Transaction, bool, bool, std::tr1::function<void (boost::shared_ptr, ripple::TER)>) (this=0x804940478, trans=...,

bAdmin=false, bFailHard=false, callback=...) at ./src/cpp/ripple/NetworkOPs.cpp:337

16 0x000000000107b942 in ripple::NetworkOPs::processTransaction (this=0x804940478, transaction=..., bAdmin=false, bFailHard=false) at ./src/cpp/ripple/NetworkOPs.h:142

17 0x00000000011c5192 in ripple::checkTransaction (flags=0, stx=..., peer=...) at ./src/cpp/ripple/ripple_Peer.cpp:1082

18 0x00000000012031c2 in std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer))(ripple::Job&, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer)>::call<ripple::Job&, 0, 1, 2, 3>(std::tr1::tupleripple::Job& const&, std::tr1::_Index_tuple<0, 1, 2, 3>) (this=0x806c22b20, args=...) at /usr/local/lib/gcc46/include/c++/tr1/functional:1153

19 0x00000000011faf46 in std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer))(ripple::Job&, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer)>::operator()ripple::Job(ripple::Job&) (this=0x806c22b20) at /usr/local/lib/gcc46/include/c++/tr1/functional:1207

20 0x00000000011f1dac in std::tr1::_Function_handler<void (ripple::Job&), std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer))(ripple::Job&, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer)> >::_M_invoke(std::tr1::_Any_data const&, ripple::Job&) (functor=..., args#0=...) at /usr/local/lib/gcc46/include/c++/tr1/functional:1684

21 0x0000000000ebcb48 in std::tr1::function<void (ripple::Job&)>::operator()(ripple::Job&) const (this=0x7fffff5facf0, __args#0=...) at /usr/local/lib/gcc46/include/c++/tr1/functional:2040

22 0x0000000000eb5022 in ripple::Job::doJob (this=0x7fffff5face0) at modules/ripple_core/functional/ripple_Job.cpp:41

23 0x0000000000eb6bb6 in ripple::JobQueue::threadEntry (this=0x804940a48) at modules/ripple_core/functional/ripple_JobQueue.cpp:268

24 0x0000000000ec616e in std::tr1::_Mem_fn<void (ripple::JobQueue::*)()>::operator() (this=0x80481d9d0, __object=0x804940a48) at /usr/local/lib/gcc46/include/c++/tr1/functional:572

25 0x0000000000ec6062 in std::tr1::_Bind<std::tr1::_Memfn<void (ripple::JobQueue::*)()> (ripple::JobQueue)>::call<, 0>(std::tr1::tuple<> const&, std::tr1::_Index_tuple<0>) (this=0x80481d9d0, args=...)

at /usr/local/lib/gcc46/include/c++/tr1/functional:1153

26 0x0000000000ec5db2 in std::tr1::_Bind<std::tr1::_Memfn<void (ripple::JobQueue::)()> (ripple::JobQueue_)>::operator()<>() (this=0x80481d9d0) at /usr/local/lib/gcc46/include/c++/tr1/functional:1207

27 0x0000000000ec59a0 in boost::detail::thread_data<std::tr1::_Bind<std::tr1::_Memfn<void (ripple::JobQueue::)()> (ripple::JobQueue*)> >::run() (this=0x80481d900) at /usr/local/include/boost/thread/detail/thread.hpp:78

28 0x0000000803311933 in thread_proxy () from /usr/local/lib/libboost_thread.so.5

29 0x00000008023520a4 in ?? () from /lib/libthr.so.3

30 0x0000000000000000 in ?? ()

alexdupre commented 11 years ago

Another one..just to confirm that is not "always the OfferCreateTransactor calling LES::getNextLedgerIndex calling SHAMap::getStack":

Thread 8 (Thread 804809000 (LWP 100325)):

0 0x00000008042d638c in thr_kill () from /lib/libc.so.7

1 0x000000080437899b in abort () from /lib/libc.so.7

2 0x0000000803bf89e7 in __gnu_cxx::__verbose_terminate_handler () at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/vterminate.cc:95

3 0x0000000803bf5e16 in cxxabiv1::terminate (handler=) at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_terminate.cc:40

4 0x0000000803bf5e63 in std::terminate () at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_terminate.cc:50

5 0x0000000803bf5f5e in cxxabiv1::cxa_throw (obj=0x804bfa9c0, tinfo=, dest=) at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_throw.cc:83

6 0x000000000106d63a in ripple::SHAMap::getNodePointer (this=0x804bff360, id=..., hash=...) at ./src/cpp/ripple/ripple_SHAMap.cpp:236

7 0x000000000106d497 in ripple::SHAMap::walkToPointer (this=0x804bff360, id=...) at ./src/cpp/ripple/ripple_SHAMap.cpp:197

8 0x000000000106ec3f in ripple::SHAMap::peekItem (this=0x804bff360, id=...) at ./src/cpp/ripple/ripple_SHAMap.cpp:551

9 0x00000000010213e7 in ripple::Ledger::getSLE (this=0x87c08d9e0, uHash=...) at ./src/cpp/ripple/Ledger.cpp:1137

10 0x000000000122804e in ripple::LedgerEntrySet::entryCache (this=0x8049400e0, letType=ripple::ltRIPPLE_STATE, index=...) at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:99

11 0x000000000123072b in ripple::LedgerEntrySet::rippleCredit (this=0x8049400e0, uSenderID=..., uReceiverID=..., saAmount=..., bCheckIssuer=true)

at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:1438

12 0x00000000012315e4 in ripple::LedgerEntrySet::rippleSend (this=0x8049400e0, uSenderID=..., uReceiverID=..., saAmount=..., saActual=...)

at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:1567

13 0x00000000012321d8 in ripple::LedgerEntrySet::accountSend (this=0x8049400e0, uSenderID=..., uReceiverID=..., saAmount=...) at ./src/cpp/ripple/ripple_LedgerEntrySet.cpp:1640

14 0x00000000011d5f32 in ripple::OfferCreateTransactor::takeOffers (this=0x8052f2880, bOpenLedger=true, bPassive=false, bSell=true, uBookBase=..., uTakerAccountID=...,

sleTakerAccount=..., saTakerPays=..., saTakerGets=..., saTakerPaid=..., saTakerGot=..., bUnfunded=@0x7fffff5f430f: false) at ./src/cpp/ripple/OfferCreateTransactor.cpp:316

15 0x00000000011d8316 in ripple::OfferCreateTransactor::doApply (this=0x8052f2880) at ./src/cpp/ripple/OfferCreateTransactor.cpp:559

16 0x00000000010ca305 in ripple::Transactor::apply (this=0x8052f2880) at ./src/cpp/ripple/Transactor.cpp:251

17 0x00000000010c5d3b in ripple::TransactionEngine::applyTransaction (this=0x8049400d8, txn=..., params=(ripple::tapNO_CHECK_SIGN | ripple::tapOPEN_LEDGER),

didApply=@0x7fffff5fa23f: false) at ./src/cpp/ripple/TransactionEngine.cpp:102

18 0x00000000012be90c in ripple::LedgerMaster::doTransaction (this=0x8049400d0, txn=..., params=(ripple::tapNO_CHECK_SIGN | ripple::tapOPEN_LEDGER), didApply=@0x7fffff5fa23f: false)

at ./src/cpp/ripple/LedgerMaster.cpp:170

19 0x000000000125e0ad in ripple::NetworkOPs::processTransaction(boost::shared_ptrripple::Transaction, bool, bool, std::tr1::function<void (boost::shared_ptr, ripple::TER)>) (this=0x804940478, trans=..., bAdmin=false, bFailHard=false, callback=...) at ./src/cpp/ripple/NetworkOPs.cpp:337

20 0x000000000107b942 in ripple::NetworkOPs::processTransaction (this=0x804940478, transaction=..., bAdmin=false, bFailHard=false) at ./src/cpp/ripple/NetworkOPs.h:142

21 0x00000000011c5192 in ripple::checkTransaction (flags=0, stx=..., peer=...) at ./src/cpp/ripple/ripple_Peer.cpp:1082

22 0x00000000012031c2 in std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer))(ripple::Job&, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer)>::__call<ripple::Job&, 0, 1, 2, 3>(std::tr1::tupleripple::Job& const&, std::tr1::_Index_tuple<0, 1, 2, 3>) (

this=0x8053d9bb0, __args=...) at /usr/local/lib/gcc46/include/c++/tr1/functional:1153

23 0x00000000011faf46 in std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer))(ripple::Job&, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer)>::operator()ripple::Job(ripple::Job&) (this=0x8053d9bb0)

at /usr/local/lib/gcc46/include/c++/tr1/functional:1207

24 0x00000000011f1dac in std::tr1::_Function_handler<void (ripple::Job&), std::tr1::_Bind<void (*(std::tr1::_Placeholder<1>, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer))(ripple::Job&, int, boost::shared_ptrripple::SerializedTransaction, boost::weak_ptrripple::Peer)> >::_M_invoke(std::tr1::_Any_data const&, ripple::Job&) (

__functor=..., __args#0=...) at /usr/local/lib/gcc46/include/c++/tr1/functional:1684

25 0x0000000000ebcb48 in std::tr1::function<void (ripple::Job&)>::operator()(ripple::Job&) const (this=0x7fffff5facf0, __args#0=...)

at /usr/local/lib/gcc46/include/c++/tr1/functional:2040

26 0x0000000000eb5022 in ripple::Job::doJob (this=0x7fffff5face0) at modules/ripple_core/functional/ripple_Job.cpp:41

27 0x0000000000eb6bb6 in ripple::JobQueue::threadEntry (this=0x804940a48) at modules/ripple_core/functional/ripple_JobQueue.cpp:268

...

JoelKatz commented 11 years ago

Take a look at this stack trace, from one of our client servers running Linux:

5 0x00007fa100aaa17e in __cxa_throw () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6

6 0x00000000010e2224 in SHAMap::getNodePointer (this=0x7fa0eac07fc0, id=..., hash=...)

at ./src/cpp/ripple/ripple_SHAMap.cpp:236

7 0x0000000001044a77 in SHAMap::compare (this=0x121acad0, otherMap=..., differences=..., maxCount=16383)

at ./src/cpp/ripple/ripple_SHAMapDelta.cpp:141

8 0x000000000133e3f2 in LedgerConsensus::createDisputes (this=0x1b89a430, m1=..., m2=...)

at ./src/cpp/ripple/ripple_LedgerConsensus.cpp:290

9 0x000000000133eaf7 in LedgerConsensus::mapComplete (this=0x1b89a430, hash=..., map=..., acquired=true)

at ./src/cpp/ripple/ripple_LedgerConsensus.cpp:350

10 0x00000000012ed53b in NetworkOPs::mapComplete (this=0x2f39098, hash=..., map=...)

at ./src/cpp/ripple/NetworkOPs.cpp:1068

11 0x0000000001306863 in TACompletionHandler (hash=..., map=...) at ./src/cpp/ripple/ripple_TransactionAcquire.cpp:22

So it looks like this is a bug that causes nodes to somehow disappear or the server to think a map is complete when it's missing nodes. This particular one is interested because it occurred in proposed transaction sets. This rules out the ledger logic (these maps aren't part of a ledger) or database corruption (these maps aren't stored in the database).

alexdupre commented 11 years ago

Bug still present in 0.12.0 release:

Thread 16 (Thread 804807400 (LWP 100079)):

0 0x000000080437f38c in thr_kill () from /lib/libc.so.7

1 0x000000080442199b in abort () from /lib/libc.so.7

2 0x0000000803ca19e7 in __gnu_cxx::__verbose_terminate_handler () at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/vterminate.cc:95

3 0x0000000803c9ee16 in cxxabiv1::terminate (handler=) at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_terminate.cc:40

4 0x0000000803c9ee63 in std::terminate () at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_terminate.cc:50

5 0x0000000803c9ef5e in cxxabiv1::cxa_throw (obj=0x8806bca000, tinfo=, dest=) at ../../.././../gcc-4.6.3/libstdc++-v3/libsupc++/eh_throw.cc:83

6 0x0000000000fc353a in beast::Throwripple::SHAMapMissingNode (e=...) at Subtrees/beast/modules/beast_basics/../beast_core/diagnostic/beast_Throw.h:33

7 0x0000000000fab3ae in ripple::SHAMap::getNodePointer (this=0x88075f0460, id=..., hash=...) at modules/ripple_app/shamap/ripple_SHAMap.cpp:233

8 0x0000000000fabc7d in ripple::SHAMap::onlyBelow (this=0x88075f0460, node=0x880e85cd20) at modules/ripple_app/shamap/ripple_SHAMap.cpp:373

9 0x0000000000fad0f1 in ripple::SHAMap::delItem (this=0x88075f0460, id=...) at modules/ripple_app/shamap/ripple_SHAMap.cpp:644

10 0x000000000100bb50 in ripple::TransactionEngine::txnWrite (this=0x7fffffffa1c0) at modules/ripple_app/tx/TransactionEngine.cpp:52

11 0x000000000100c5f6 in ripple::TransactionEngine::applyTransaction (this=0x7fffffffa1c0, txn=..., params=ripple::tapRETRY, didApply=@0x7fffffff9967: true)

at modules/ripple_app/tx/TransactionEngine.cpp:168

12 0x00000000011e3ee5 in ripple::LedgerConsensus::applyTransaction (this=0x880e7ab120, engine=..., txn=..., ledger=..., openLedger=false, retryAssured=true)

at modules/ripple_app/consensus/ripple_LedgerConsensus.cpp:1069

13 0x00000000011e4403 in ripple::LedgerConsensus::applyTransactions (this=0x880e7ab120, set=..., applyLedger=..., checkLedger=..., failedTransactions=..., openLgr=false)

at modules/ripple_app/consensus/ripple_LedgerConsensus.cpp:1116

14 0x00000000011e4e9b in ripple::LedgerConsensus::accept (this=0x880e7ab120, set=...) at modules/ripple_app/consensus/ripple_LedgerConsensus.cpp:1217

...

JoelKatz commented 11 years ago

How long had the server been running when this happened? What platform? What database back end?

This is strange. We're applying transactions to a ledger because a consensus round completed. The transaction requires deleting a node from the state SHAMap tree. When we go to find other nodes that have a common parent to this node's parent, and we encounter a missing node. Of course, we're not supposed to use a ledger for the consensus process unless we have every node in it in the local database.

This could be caused by something bizarre like a hardware problem or compiler bug, but that seems unlikely. Also possible is a bug in the database back end. If it was a logic error in the SHAMap code, we'd expect lots of servers to crash at once, so I don't think it's that, but logic errors in the node caching or ledger acceptance code could do it.

alexdupre commented 11 years ago

Well, this is the full history, yesterday I had two SHAMapMissingNode exceptions:

alexdupre commented 11 years ago

Related to node caching, both HyperLevelDB and MDB use BatchWriter to write NodeObjects asynchronously. Could it happen that the TaggedCache already purged the NodeObject while the backend has not persisted it yet?

JoelKatz commented 11 years ago

No. The TaggedCache can't purge an object while the BatchWriter holds a reference to it, and the BatchWriter won't discard its reference until the write is complete.

alexdupre commented 11 years ago

For the records, I got another one, with 0.12.0 release running for ~3 days starting from a clean HyperLevelDB, so we can exclude db corruption at shutdown.

6 0x0000000000fc353a in beast::Throwripple::SHAMapMissingNode (e=...) at Subtrees/beast/modules/beast_basics/../beast_core/diagnostic/beast_Throw.h:33

7 0x0000000000fab3ae in ripple::SHAMap::getNodePointer (this=0x8048232a0, id=..., hash=...) at modules/ripple_app/shamap/ripple_SHAMap.cpp:233

8 0x0000000000fab1f3 in ripple::SHAMap::walkToPointer (this=0x8048232a0, id=...) at modules/ripple_app/shamap/ripple_SHAMap.cpp:194

9 0x0000000000faca35 in ripple::SHAMap::peekItem (this=0x8048232a0, id=...) at modules/ripple_app/shamap/ripple_SHAMap.cpp:549

10 0x0000000000f3869d in ripple::Ledger::getSLE (this=0x804d2ee20, uHash=...) at modules/ripple_app/ledger/Ledger.cpp:1131

11 0x0000000001152412 in ripple::LedgerEntrySet::entryCache (this=0x7fffffffa1c8, letType=ripple::ltOFFER, index=...) at modules/ripple_app/ledger/ripple_LedgerEntrySet.cpp:99

12 0x0000000000f495c8 in ripple::TransactionEngine::entryCache (this=0x7fffffffa1c0, type=ripple::ltOFFER, index=...) at modules/ripple_app/tx/TransactionEngine.h:78

13 0x0000000000fb5e8f in ripple::OfferCancelTransactor::doApply (this=0x804ff6d00) at modules/ripple_app/tx/OfferCancelTransactor.cpp:35

14 0x00000000010104b7 in ripple::Transactor::apply (this=0x804ff6d00) at modules/ripple_app/tx/Transactor.cpp:251

15 0x000000000100bed3 in ripple::TransactionEngine::applyTransaction (this=0x7fffffffa1c0, txn=..., params=ripple::tapRETRY, didApply=@0x7fffffff9967: false)

at modules/ripple_app/tx/TransactionEngine.cpp:102

16 0x00000000011e3ee5 in ripple::LedgerConsensus::applyTransaction (this=0x80a443620, engine=..., txn=..., ledger=..., openLedger=false, retryAssured=true)

at modules/ripple_app/consensus/ripple_LedgerConsensus.cpp:1069

17 0x00000000011e4403 in ripple::LedgerConsensus::applyTransactions (this=0x80a443620, set=..., applyLedger=..., checkLedger=..., failedTransactions=..., openLgr=false)

at modules/ripple_app/consensus/ripple_LedgerConsensus.cpp:1116

18 0x00000000011e4e9b in ripple::LedgerConsensus::accept (this=0x80a443620, set=...) at modules/ripple_app/consensus/ripple_LedgerConsensus.cpp:1217

vinniefalco commented 10 years ago

Can we close this?

BobWay commented 10 years ago

Yes, but I don't seem to have the close button anymore.

ghost commented 10 years ago

yes, you can close it, thanks!

ÔÚ 2013-12-19 03:28:02£¬"Vinnie Falco" notifications@github.com дµÀ£º

Can we close this?

¡ª Reply to this email directly or view it on GitHub.