search

cloudius-systems / osv

OSv, a new operating system for the cloud.
osv.io
Other
4.06k stars 603 forks source link

Partial balloon copy failure #398

Open avikivity opened 9 years ago

avikivity commented 9 years ago

0 0x00000000003d2a72 in cli_hlt () at /home/avi/osv/arch/x64/processor.hh:242

1 halt_no_interrupts () at /home/avi/osv/arch/x64/arch.hh:48

2 osv::halt () at /home/avi/osv/core/power.cc:34

3 0x0000000000223df0 in abort (fmt=fmt@entry=0x5c99c0 "Assertion failed: %s (%s: %s: %d)\n") at /home/avi/osv/runtime.cc:139

4 0x0000000000223e99 in __assert_fail (expr=, file=, line=, func=) at /home/avi/osv/runtime.cc:145

5 0x0000000000372709 in page_fault (ef=0xffff80003fd03038) at /home/avi/osv/arch/x64/mmu.cc:34

6

7 0x0000000000337047 in elf::object::symtab_len (this=this@entry=0xffffa00038740c00) at /home/avi/osv/core/elf.cc:737

8 0x0000000000337159 in elf::object::lookup_addr (this=0xffffa00038740c00, addr=addr@entry=0x100001438964 MoveAndUpdateClosure::copy_partial_obj()+292) at /home/avi/osv/core/elf.cc:767

9 0x00000000003394f7 in operator() (ml=..., __closure=) at /home/avi/osv/core/elf.cc:1147

10 with_modules<elf::program::lookup_addr(void const*)::__lambda6> (f=..., this=0xffffa0003fc05400) at /home/avi/osv/include/osv/elf.hh:597

11 elf::program::lookup_addr (this=, addr=addr@entry=0x100001438964 MoveAndUpdateClosure::copy_partial_obj()+292) at /home/avi/osv/core/elf.cc:1153

12 0x00000000003da98c in osv::lookup_name_demangled (addr=addr@entry=0x100001438964 MoveAndUpdateClosure::copy_partial_obj()+292, buf=buf@entry=0xffff800035bf87a0 "???+3610006",

len=len@entry=1024) at /home/avi/osv/core/demangle.cc:44

13 0x0000000000223ca1 in print_backtrace () at /home/avi/osv/runtime.cc:97

14 0x0000000000223e57 in abort (fmt=fmt@entry=0x5c99c0 "Assertion failed: %s (%s: %s: %d)\n") at /home/avi/osv/runtime.cc:131

15 0x0000000000223e99 in __assert_fail (expr=, file=, line=, func=) at /home/avi/osv/runtime.cc:145

16 0x0000000000326273 in mmu::jvm_balloon_vma::add_partial (this=this@entry=0xffffa00094d2ef00, partial=partial@entry=524288,

eff=eff@entry=0x78b8aadd0 <error: Cannot access memory at address 0x78b8aadd0>) at /home/avi/osv/core/mmu.cc:1409

17 0x0000000000341924 in jvm_balloon_fault (b=std::shared_ptr (count 3, weak 0) 0xffffa0003e0d1c00, ef=ef@entry=0xffff800035bf9068, vma=vma@entry=0xffffa00094d2ef00)

at /home/avi/osv/java/jvm_balloon.cc:400

18 0x0000000000329e04 in mmu::jvm_balloon_vma::fault (this=0xffffa00094d2ef00, fault_addr=32431996928, ef=0xffff800035bf9068) at /home/avi/osv/core/mmu.cc:1430

19 0x0000000000325da5 in mmu::vm_fault (addr=, addr@entry=32431999832, ef=ef@entry=0xffff800035bf9068) at /home/avi/osv/core/mmu.cc:1217

20 0x00000000003726aa in page_fault (ef=0xffff800035bf9068) at /home/avi/osv/arch/x64/mmu.cc:38

21

22 0x000000000036e973 in repmovsb (n=, src=, dest=) at /home/avi/osv/arch/x64/string.cc:100

23 memcpy_repmov (dest=0x78c480000, src=0x78d190b58, n=524288) at /home/avi/osv/arch/x64/string.cc:225

24 0x0000100001438965 in MoveAndUpdateClosure::copy_partial_obj() ()

25 0x0000100001439093 in PSParallelCompact::fill_region(ParCompactionManager*, unsigned long) ()

26 0x000010000142b62c in ParCompactionManager::drain_region_stacks() ()

27 0x0000100001410c9a in DrainStacksCompactionTask::do_it(GCTaskManager*, unsigned int) ()

28 0x0000100001148c62 in GCTaskThread::run() ()

29 0x00001000013d0f22 in java_start(Thread*) ()

30 0x0000000000423136 in operator() (__closure=0xffffa00037057700) at /home/avi/osv/libc/pthread.cc:79

31 std::_Function_handler<void(), pthread_private::pthread::pthread(void ()(void), void, sigset_t, const pthread_private::thread_attr*)::__lambda5>::_M_invoke(const std::_Any_data &) (

__functor=...) at /home/avi/osv/external/x64/gcc.bin/usr/include/c++/4.8.2/functional:2071

32 0x00000000003be11b in main (this=0xffff800035bf4040) at /home/avi/osv/core/sched.cc:915

33 sched::thread_main_c (t=0xffff800035bf4040) at /home/avi/osv/arch/x64/arch-switch.hh:137

34 0x0000000000372516 in thread_main () at /home/avi/osv/arch/x64/entry.S:113

1406 bool jvm_balloon_vma::add_partial(size_t partial, unsigned char *eff) 1407 { 1408 if (_effective_jvm_addr) { 1409 assert(eff == _effective_jvm_addr); 1410 } else { 1411 _effective_jvm_addr= eff; 1412 } 1413 1414 _partial_copy += partial;

(gdb) p partial $1 = 524288 (gdb) p eff $2 = (unsigned char ) 0x78b8aadd0 <error: Cannot access memory at address 0x78b8aadd0> (gdb) p _effective_jvm_addr $3 = (unsigned char ) 0x77e6c4470 "" (gdb)

p _this $4 = { mmu::vma = { _vptr.vma = 0x638690 <vtable for mmu::jvm_balloon_vma+16>, _range = { _start = 32419872768, _end = 32551993344 }, _perm = 3, _flags = 81, _map_dirty = true, _page_ops = 0x9b0598 mmu::page_allocator_noinit, _vma_list_hook = { <boost::intrusive::detail::generic_hook<boost::intrusive::get_set_nodealgo<void, false>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)1, 0>> = { boost::intrusive::detail::no_default_definer = {}, boost::intrusive::rbtreenode<void*> = { parent = 0xffffa00061893e30, left = 0x0, right = 0x0, color_ = boost::intrusive::rbtree_node<void*>::black_t }, }, } }, members of mmu::jvm_balloon_vma: _balloon = std::shared_ptr (count 3, weak 0) 0xffffa0003e0d1c00, _jvm_addr = 0x78c5bb928 "", _effective_jvm_addr = 0x77e6c4470 "", _partial_addr = 0, _partial_vma = 0x0, _partial_copy = 488632, _real_perm = 3, _real_flags = 17, _real_size = 132120576 }

p *_balloon ._M_ptr $9 = { _jvm_addr = 0x7c249a3a8 "T", _jref = 0xffffa00033fc8860, _alignment = 2097152, _balloon_size = 134217728 }

to reproduce: tomcat + json-servlet + wrk, for a long time

glommer commented 9 years ago

Thanks Avi. I will take a look soon

On Thu, Jul 17, 2014 at 3:06 PM, Avi Kivity notifications@github.com wrote:

0 0x00000000003d2a72 in cli_hlt () at

/home/avi/osv/arch/x64/processor.hh:242

1 https://github.com/cloudius-systems/osv/pull/1 halt_no_interrupts ()

at /home/avi/osv/arch/x64/arch.hh:48

2 https://github.com/cloudius-systems/osv/pull/2 osv::halt () at

/home/avi/osv/core/power.cc:34

3 https://github.com/cloudius-systems/osv/pull/3 0x0000000000223df0 in

abort (fmt=fmt@entry=0x5c99c0 "Assertion failed: %s (%s: %s: %d)\n") at /home/avi/osv/runtime.cc:139

4 https://github.com/cloudius-systems/osv/issues/4 0x0000000000223e99

in __assert_fail (expr=, file=, line=, func=) at /home/avi/osv/runtime.cc:145

5 https://github.com/cloudius-systems/osv/issues/5 0x0000000000372709

in page_fault (ef=0xffff80003fd03038) at /home/avi/osv/arch/x64/mmu.cc:34

6 https://github.com/cloudius-systems/osv/issues/6

7 https://github.com/cloudius-systems/osv/issues/7 0x0000000000337047

in elf::object::symtab_len (this=this@entry=0xffffa00038740c00) at /home/avi/osv/core/elf.cc:737

8 https://github.com/cloudius-systems/osv/issues/8 0x0000000000337159

in elf::object::lookup_addr (this=0xffffa00038740c00, addr=addr@entry=0x100001438964 MoveAndUpdateClosure::copy_partial_obj()+292) at /home/avi/osv/core/elf.cc:767

9 https://github.com/cloudius-systems/osv/issues/9 0x00000000003394f7

in operator() (ml=..., __closure=) at /home/avi/osv/core/elf.cc:1147

10 https://github.com/cloudius-systems/osv/issues/10 with_modules

(f=..., this=0xffffa0003fc05400) at /home/avi/osv/include/osv/elf.hh:597

11 https://github.com/cloudius-systems/osv/issues/11

elf::program::lookup_addr (this=, addr=addr@entry=0x100001438964 MoveAndUpdateClosure::copy_partial_obj()+292) at /home/avi/osv/core/elf.cc:1153

12 https://github.com/cloudius-systems/osv/issues/12

0x00000000003da98c in osv::lookup_name_demangled (addr=addr@entry=0x100001438964 MoveAndUpdateClosure::copy_partial_obj()+292, buf=buf@entry=0xffff800035bf87a0 "???+3610006", len=len@entry=1024) at /home/avi/osv/core/demangle.cc:44

13 https://github.com/cloudius-systems/osv/issues/13

0x0000000000223ca1 in print_backtrace () at /home/avi/osv/runtime.cc:97

14 https://github.com/cloudius-systems/osv/issues/14

0x0000000000223e57 in abort (fmt=fmt@entry=0x5c99c0 "Assertion failed: %s (%s: %s: %d)\n") at /home/avi/osv/runtime.cc:131

15 https://github.com/cloudius-systems/osv/issues/15

0x0000000000223e99 in __assert_fail (expr=, file=, line=, func=) at /home/avi/osv/runtime.cc:145

16 https://github.com/cloudius-systems/osv/issues/16

0x0000000000326273 in mmu::jvm_balloon_vma::add_partial (this=this@entry=0xffffa00094d2ef00, partial=partial@entry=524288, eff=eff@entry=0x78b8aadd0 ) at /home/avi/osv/core/mmu.cc:1409

17 https://github.com/cloudius-systems/osv/issues/17

0x0000000000341924 in jvm_balloon_fault (b=std::shared_ptr (count 3, weak 0) 0xffffa0003e0d1c00, ef=ef@entry=0xffff800035bf9068, vma=vma@entry =0xffffa00094d2ef00) at /home/avi/osv/java/jvm_balloon.cc:400

18 https://github.com/cloudius-systems/osv/issues/18

0x0000000000329e04 in mmu::jvm_balloon_vma::fault (this=0xffffa00094d2ef00, fault_addr=32431996928, ef=0xffff800035bf9068) at /home/avi/osv/core/mmu.cc:1430

19 https://github.com/cloudius-systems/osv/issues/19

0x0000000000325da5 in mmu::vm_fault (addr=, addr@entry=32431999832, ef=ef@entry=0xffff800035bf9068) at /home/avi/osv/core/mmu.cc:1217

20 https://github.com/cloudius-systems/osv/issues/20

0x00000000003726aa in page_fault (ef=0xffff800035bf9068) at /home/avi/osv/arch/x64/mmu.cc:38

21 https://github.com/cloudius-systems/osv/issues/21

22 https://github.com/cloudius-systems/osv/issues/22

0x000000000036e973 in repmovsb (n=, src=, dest=) at /home/avi/osv/arch/x64/string.cc:100

23 https://github.com/cloudius-systems/osv/issues/23 memcpy_repmov

(dest=0x78c480000, src=0x78d190b58, n=524288) at /home/avi/osv/arch/x64/string.cc:225

24 https://github.com/cloudius-systems/osv/issues/24

0x0000100001438965 in MoveAndUpdateClosure::copy_partial_obj() ()

25 https://github.com/cloudius-systems/osv/issues/25

0x0000100001439093 in PSParallelCompact::fillregion(ParCompactionManager, unsigned long) ()

26 https://github.com/cloudius-systems/osv/issues/26

0x000010000142b62c in ParCompactionManager::drain_region_stacks() ()

27 https://github.com/cloudius-systems/osv/issues/27

0x0000100001410c9a in DrainStacksCompactionTask::doit(GCTaskManager, unsigned int) ()

28 https://github.com/cloudius-systems/osv/issues/28

0x0000100001148c62 in GCTaskThread::run() ()

29 https://github.com/cloudius-systems/osv/issues/29

0x00001000013d0f22 in java_start(Thread*) ()

30 https://github.com/cloudius-systems/osv/issues/30

0x0000000000423136 in operator() (__closure=0xffffa00037057700) at /home/avi/osv/libc/pthread.cc:79

31 https://github.com/cloudius-systems/osv/issues/31

std::_Function_handler::_M_invoke(const std::_Any_data &) ( __functor=...) at /home/avi/osv/external/x64/gcc.bin/usr/include/c++/4.8.2/functional:2071

32 https://github.com/cloudius-systems/osv/pull/32 0x00000000003be11b

in main (this=0xffff800035bf4040) at /home/avi/osv/core/sched.cc:915

33 https://github.com/cloudius-systems/osv/issues/33

sched::thread_main_c (t=0xffff800035bf4040) at /home/avi/osv/arch/x64/arch-switch.hh:137

34 https://github.com/cloudius-systems/osv/issues/34

0x0000000000372516 in thread_main () at /home/avi/osv/arch/x64/entry.S:113

1406 bool jvm_balloon_vma::add_partial(size_t partial, unsigned char *eff) 1407 { 1408 if (_effective_jvm_addr) { 1409 assert(eff == _effective_jvm_addr); 1410 } else { 1411 _effective_jvm_addr= eff; 1412 } 1413 1414 _partial_copy += partial;

(gdb) p partial $1 = 524288 (gdb) p eff $2 = (unsigned char ) 0x78b8aadd0 (gdb) p _effective_jvm_addr $3 = (unsigned char ) 0x77e6c4470 "" (gdb)

p

_this $4 = {mmu::vma = { _vptr.vma = 0x638690 , _range = { _start = 32419872768, _end = 32551993344 }, _perm = 3, _flags = 81, _map_dirty = true, _page_ops = 0x9b0598 mmu::page_allocator_noinit, _vma_list_hook = { <boost::intrusive::detail::generic_hook<boost::intrusive::get_set_nodealgo<void, false>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)1, 0>> = { boost::intrusive::detail::no_default_definer = {}, boost::intrusive::rbtreenode<void*> = { parent = 0xffffa00061893e30, left = 0x0, right = 0x0, color_ = boost::intrusive::rbtree_node::black_t }, }, } }, members of mmu::jvm_balloon_vma: _balloon = std::shared_ptr (count 3, weak 0) 0xffffa0003e0d1c00, _jvm_addr = 0x78c5bb928 "", _effective_jvm_addr = 0x77e6c4470 "", _partial_addr = 0, _partial_vma = 0x0, _partial_copy = 488632, _real_perm = 3, _real_flags = 17, _real_size = 132120576 }

p *_balloon ._M_ptr $9 = { _jvm_addr = 0x7c249a3a8 "T", _jref = 0xffffa00033fc8860, _alignment = 2097152, _balloon_size = 134217728 }

to reproduce: tomcat + json-servlet + wrk, for a long time

— Reply to this email directly or view it on GitHub https://github.com/cloudius-systems/osv/issues/398.