search

eclipse-openj9 / openj9

Eclipse OpenJ9: A Java Virtual Machine for OpenJDK that's optimized for small footprint, fast start-up, and high throughput. Builds on Eclipse OMR (https://github.com/eclipse/omr) and combines with the Extensions for OpenJDK for OpenJ9 repo.
Other
3.28k stars 722 forks source link

MauveSingleInvocLoad crash vmState=0x00000000 #14753

Open pshipton opened 2 years ago

pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Test_openjdk8_j9_special.system_s390x_linux_Personal_testList_0/1 - rh7-390-1 MauveSingleInvocLoad_special_5m_21 -XX:+UseCompressedOops -Xjit -Xgcpolicy:gencon -Xaggressive

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk8_j9_special.system_s390x_linux_Personal_testList_0/1/system_test_output.tar.gz

LT  stderr Unhandled exception
LT  stderr Type=Segmentation error vmState=0x00000000
LT  stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=f1953270 Signal_Code=00000001
LT  stderr Handler1=000003FFB7C46858 Handler2=000003FFB7B32840 InaccessibleAddress=0000000000000000
LT  stderr gpr0=FFFFFFFFFFFFFFD0 gpr1=FFFFFFFFFFE6E7F4 gpr2=FFFFFFFFFFFFFFD0 gpr3=0000000000161D00
LT  stderr gpr4=0000000000000001 gpr5=0000000000000000 gpr6=0000000000127D00 gpr7=000003FFB7A7DCC8
LT  stderr gpr8=000003FF6969A017 gpr9=0000000000000000 gpr10=0000000000000000 gpr11=0000000031EAD410
LT  stderr gpr12=0000000000128D00 gpr13=000003FFB7E2CE68 gpr14=000003FFB7C9A980 gpr15=000003FFB7A7D808
LT  stderr psw=000003FFB7C9B022 mask=0705000180000000 fpc=00f8ff00 bea=000003FFB7C9B010
LT  stderr fpr0 3ec33d2d00000000 (f: 0.000000, d: 2.293464e-06)
LT  stderr fpr1 48409d8000000000 (f: 0.000000, d: 1.130774e+40)
LT  stderr fpr2 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr3 3c45485a00000000 (f: 0.000000, d: 2.307467e-18)
LT  stderr fpr4 48409d8000000000 (f: 0.000000, d: 1.130774e+40)
LT  stderr fpr5 3e924c9acb8ae3cc (f: 3414877184.000000, d: 2.726799e-07)
LT  stderr fpr6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr7 3e3a38c500000000 (f: 0.000000, d: 6.105228e-09)
LT  stderr fpr8 00000000824bd190 (f: 2186007040.000000, d: 1.080031e-314)
LT  stderr fpr9 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr12 0000000000000001 (f: 1.000000, d: 4.940656e-324)
LT  stderr fpr13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr Module=/home/jenkins/workspace/Test_openjdk8_j9_special.system_s390x_linux_Personal_testList_0/openjdkbinary/j2sdk-image/jre/lib/s390x/default/libj9vm29.so
LT  stderr Module_base_address=000003FFB7C00000
LT  stderr Target=2_90_20220319_289 (Linux 3.10.0-1160.42.2.el7.s390x)
LT  stderr CPU=s390x (4 logical CPUs) (0x1ec1b1000 RAM)
LT  stderr ----------- Stack Backtrace -----------
LT  stderr _ZN32VM_BytecodeInterpreterCompressed3runEP10J9VMThread+0x692 (0x000003FFB7C9B022 [libj9vm29.so+0x9b022])
LT  stderr bytecodeLoopCompressed+0xc0 (0x000003FFB7C9A980 [libj9vm29.so+0x9a980])
LT  stderr c_cInterpreter+0x64 (0x000003FFB7D8C54C [libj9vm29.so+0x18c54c])
LT  stderr ---------------------------------------
pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Test_openjdk8_j9_sanity.system_s390x_linux_Nightly_testList_0/243 - rh7-390-1 MauveSingleInvocLoad_J9_5m_1 -Xcompressedrefs -Xjit -Xgcpolicy:gencon

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk8_j9_sanity.system_s390x_linux_Nightly_testList_0/243/system_test_output.tar.gz

LT  stderr Unhandled exception
LT  stderr Type=Segmentation error vmState=0x00000000
LT  stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=f23eba70 Signal_Code=00000001
LT  stderr Handler1=000003FFA81C6858 Handler2=000003FFA80B2840 InaccessibleAddress=0000000000000000
LT  stderr gpr0=FFFFFFFFFFFFFFD0 gpr1=FFFFFFFFFFE6E8D4 gpr2=FFFFFFFFFFFFFFD0 gpr3=000000000034AE00
LT  stderr gpr4=0000000000000001 gpr5=0000000000000000 gpr6=0000000000127D00 gpr7=000003FFA3B7DCC8
LT  stderr gpr8=000003FF5569AAD7 gpr9=0000000000000000 gpr10=0000000000000000 gpr11=000000002FF548E0
LT  stderr gpr12=0000000030067778 gpr13=000003FFA83ACD88 gpr14=000003FFA821A980 gpr15=000003FFA3B7D808
LT  stderr psw=000003FFA821B022 mask=0705000180000000 fpc=00f8ff00 bea=000003FFA821B010
LT  stderr fpr0 3f63ef5400000000 (f: 0.000000, d: 2.433456e-03)
LT  stderr fpr1 47a5a68000000000 (f: 0.000000, d: 1.438915e+37)
LT  stderr fpr2 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr3 389dc9d600000000 (f: 0.000000, d: 5.602579e-36)
LT  stderr fpr4 47a5a68000000000 (f: 0.000000, d: 1.438915e+37)
LT  stderr fpr5 3e924925cb8ae3cc (f: 3414877184.000000, d: 2.724787e-07)
LT  stderr fpr6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr7 3e3a332500000000 (f: 0.000000, d: 6.100112e-09)
LT  stderr fpr8 00000000be33e190 (f: 3191071232.000000, d: 1.576599e-314)
LT  stderr fpr9 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr Module=/home/jenkins/workspace/Test_openjdk8_j9_sanity.system_s390x_linux_Nightly_testList_0/openjdkbinary/j2sdk-image/jre/lib/s390x/default/libj9vm29.so
LT  stderr Module_base_address=000003FFA8180000
LT  stderr Target=2_90_20220321_260 (Linux 3.10.0-1160.42.2.el7.s390x)
LT  stderr CPU=s390x (4 logical CPUs) (0x1ec1b1000 RAM)
LT  stderr ----------- Stack Backtrace -----------
LT  stderr _ZN32VM_BytecodeInterpreterCompressed3runEP10J9VMThread+0x692 (0x000003FFA821B022 [libj9vm29.so+0x9b022])
LT  stderr bytecodeLoopCompressed+0xc0 (0x000003FFA821A980 [libj9vm29.so+0x9a980])
LT  stderr c_cInterpreter+0x64 (0x000003FFA830C54C [libj9vm29.so+0x18c54c])
pshipton commented 2 years ago

@tajila this looks like a regression, crashing in the VM. I'll run some grinders to get a repo rate and maybe try to track down when it was introduced, but it would be worth taking a look at the crash to see if we can identify the problem.

pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Grinder/727/ - failed 1/30

tajila commented 2 years ago

DebugInterpreter run - https://openj9-jenkins.osuosl.org/job/Grinder/728/ - passed

pshipton commented 2 years ago

Started a new DebugInterpreter run with more iterations https://openj9-jenkins.osuosl.org/job/Grinder/729/

pshipton commented 2 years ago

Based on the 1/30 failure rate I got, it's not as hot as I thought it might be.

pshipton commented 2 years ago

@tajila One failure with the DebugIntpreter https://openj9-jenkins.osuosl.org/job/Grinder_iteration_4/32/

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Grinder_iteration_4/32/system_test_output.tar.gz

tajila commented 2 years ago

We are crashing upon entry to jitted method sun/util/locale/provider/LocaleProviderAdapter.getAdapter

I dont have a native stack trace yet as the s390x machines seem to be inaccessible right now

tajila commented 2 years ago

Native stack trace

#12 <signal handler called>
#13 j2iVirtualMethod (interfaceVTableIndex=18446744073709551568, receiver=0x0, _pc=<synthetic pointer>, _sp=<synthetic pointer>, this=0x3ffb097dcc0) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/openj9/runtime/vm/BytecodeInterpreter.hpp:728
#14 VM_DebugBytecodeInterpreterCompressed::run (this=this@entry=0x3ffb097dcc0, vmThread=<optimized out>) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/openj9/runtime/vm/BytecodeInterpreter.hpp:10021
#15 0x000003ffb0f68300 in debugBytecodeLoopCompressed (currentThread=<optimized out>) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/openj9/runtime/vm/BytecodeInterpreter.inc:112
#16 0x000003ffb100c54c in c_cInterpreter () at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/build/linux-s390x-normal-server-release/vm/runtime/vm/zcinterp.s:278
#17 0x000003ffb0ea3a88 in sidecarInvokeReflectMethodImpl (currentThread=currentThread@entry=0x34ae00, methodRef=methodRef@entry=0x494690, recevierRef=recevierRef@entry=0x4946b0, argsRef=argsRef@entry=0x4946a8) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/openj9/runtime/vm/callin.cpp:1198
#18 0x000003ffb0ea4a54 in sidecarInvokeReflectMethod (currentThread=0x34ae00, methodRef=0x494690, recevierRef=0x4946b0, argsRef=0x4946a8) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/openj9/runtime/vm/callin.cpp:1342
#19 0x000003ffb0268344 in JVM_InvokeMethod_Impl (env=0x34ae00, method=0x494690, obj=0x4946b0, args=0x4946a8) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/openj9/runtime/sunvmi/sunvmi.c:363
#20 0x000003ff92c0a018 in ?? ()
#21 0x000003ffb0e9ecec in runJavaThread (currentThread=0x3ff92c0a7e8, currentThread@entry=0x34ae00) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/openj9/runtime/vm/callin.cpp:645
#22 0x000003ffb0f19fde in javaProtectedThreadProc (portLibrary=portLibrary@entry=0x3ffb14c3f58 <j9portLibrary>, entryarg=entryarg@entry=0x34ae00) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/openj9/runtime/vm/vmthread.c:2070
#23 0x000003ffb0db39ae in omrsig_protect (portLibrary=0x3ffb14c3f58 <j9portLibrary>, fn=fn@entry=0x3ffb0f19f10 <javaProtectedThreadProc>, fn_arg=fn_arg@entry=0x34ae00, handler=0x3ffb0ec6858 <structuredSignalHandler>, handler_arg=handler_arg@entry=0x34ae00, flags=506, result=0x3ffb097ee38)
    at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/omr/port/unix/omrsignal.c:425
#24 0x000003ffb0f15d50 in javaThreadProc (entryarg=0x3ffac014050) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/openj9/runtime/vm/vmthread.c:349
#25 0x000003ffb0d05a14 in thread_wrapper (arg=0x3ff3c009678) at /home/jenkins/workspace/Build_JDK8_s390x_linux_Nightly/omr/thread/common/omrthread.c:1724
#26 0x000003ffb1808312 in start_thread () from /lib64/libpthread.so.0
#27 0x000003ffb160e232 in thread_start () from /lib64/libc.so.6
tajila commented 2 years ago

So it looks like we successfully call the JIT method, but then transistion to the interpreter for a subsequent call and fail the invoke virtual because the receiver is NULL.

tajila commented 2 years ago

@0xdaryl Can the JIT team continue this investigation

0xdaryl commented 2 years ago

@joransiu : could you assign this for investigation please?

joransiu commented 2 years ago

@Spencer-Comin , can you please take a look? Thanks!

@tajila : Any chance you still have the diagnostic output from: https://github.com/eclipse-openj9/openj9/issues/14753#issuecomment-1075645879?

tajila commented 2 years ago

@joransiu Unfortunately I no longer have the diagnostic output

pshipton commented 2 years ago

New 10x x 8 grinder of MauveSingleInvocLoad_J9_5m_1 on jdk8 with -XX:+DebugInterpreter https://openj9-jenkins.osuosl.org/view/Test/job/Grinder/827/

pshipton commented 2 years ago

There is a crash in https://openj9-jenkins.osuosl.org/job/Grinder_iteration_2/60 - ub18-390-2

vij-singh commented 2 years ago

@Spencer-Comin @joransiu Any new news on this one?

Spencer-Comin commented 2 years ago

@vij-singh Not yet, still trying to hunt down how the receiver is set to NULL

pshipton commented 2 years ago

This doesn't seem like it will be fixed shortly. It doesn't occur much in the builds, moving to the Backlog.

Spencer-Comin commented 2 years ago

Looking into the failure reproduced by @pshipton to find how the receiver argument to j2iVirtualMethod becomes NULL:

The actionData variable is passed as the receiver argument in VM_DebugBytecodeInterpreterCompressed::run [1]. On entering run, actionData is set to vmThread->returnValue2 [2]. Checking the value of vmThread in the coredump we see

(gdb) print *((J9VMThread *)0x0000000000f5fe00)
$1 = {..., returnValue = 13, returnValue2 = 0, ...}

confirming that vmThread->returnValue2 is NULL. Moving up a few stack frames to where currentThread->returnValue2 and currentThread->returnValue are set before calling c_cinterpreter [3], there are a couple inconsistencies:

; at sidecarInvokeReflectMethodImpl +1846
stg       %r9, 0x128(%r11)       ; currentThread->returnValue2 = %r9
lghi      %r1, 1
stg       %r1, 0x120(%r11)       ; currentThread->returnValue = 1 
lgr       %r2, %r11              ; load %r11 as the first param to the call (currentThread)
brasl     %r14, 0x3ff82c8d078    ; call c_cinterpreter
...
; at c_cInterpreter +0
stmg      %r6, %r15, 0x30(%r15)  ; store %r6 through %r15 at stack pointer + 0x30

First currentThread->returnValue is set to 1, not 13 as we see in run. Second, we need to verify the value of currentThread->returnValue2. Looking into the disassembly, currentThread->returnValue2 is in %r9 before calling c_cInterpreter, which is saved in c_cInterpreters stack frame. This means we can find the its value before c_cinterpreter is called by looking at c_cinterpreter's stack frame:

saved %r6:  0x0000000001454800
saved %r7:  0x000003ff82d67480
saved %r8:  0x000000008ab65058
saved %r9:  0x0000000001454c28
saved %r10: 0x000000000df7ed38
saved %r11: 0x0000000000f5fe00  ; J9VMThread 
saved %r12: 0x000003ff6a37d9c0  ; {sun/reflect/NativeMethodAccessorImpl.invoke} +1234
saved %r13: 0x0000000001454800
saved %r14: 0x000003ff82b23a88  ; {sidecarInvokeReflectMethodImpl} +1872)
saved %r15: 0x000003ff8277e1f8

%r9 is not the NULL value we see for currentThread->returnValue2 in VM_DebugBytecodeInterpreterCompressed::run in the coredump. Inspecting the saved value of %r9 we find a J9 method:

> !j9method 0x0000000001454c28
J9Method at 0x1454c28 {
  Fields for J9Method:
    0x0: U8* bytecodes = !j9x 0x000003FF04370C6C // "*+?"
    0x8: struct J9ConstantPool* constantPool = !j9constantpool 0x00000000014549E0 (flags = 0x0)
    0x10: void* methodRunAddress = !j9x 0x0000000000000006
    0x18: volatile void* extra = !j9x 0x0000000000001767
}
Signature: gnu/testlet/java/util/Calendar/getInstance.test(Lgnu/testlet/TestHarness;)V !bytecodes 0x0000000001454C28

This method appears to be correct from the Java stack we get looking at the vm thread in jdmpview:

> !stackslots 0x0000000000f5fe00
<f5fe00> *** BEGIN STACK WALK, flags = 00400001 walkThread = 0x0000000000F5FE00 ***
<f5fe00>    Method: java/util/Calendar.createCalendar(Ljava/util/TimeZone;Ljava/util/Locale;)Ljava/util/Calendar; !j9method 0x0000000000F2C5D0
...
WARNING: CorruptData encountered iterating o-slots. walkThread = 0x0000000000F5FE00
...
<f5fe00>    Method: java/util/Calendar.getInstance(Ljava/util/Locale;)Ljava/util/Calendar; !j9method 0x0000000000F2C590
...
<f5fe00>    Method: gnu/testlet/java/util/Calendar/getInstance.testMethod3(Lgnu/testlet/TestHarness;)V !j9method 0x0000000001454C88
...
<f5fe00>    Method: gnu/testlet/java/util/Calendar/getInstance.test(Lgnu/testlet/TestHarness;)V !j9method 0x0000000001454C28
...
<f5fe00>    Method: sun/reflect/NativeMethodAccessorImpl.invoke0(Ljava/lang/reflect/Method;Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; !j9method 0x0000000000D67798
...
<f5fe00>    Method: sun/reflect/NativeMethodAccessorImpl.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; !j9method 0x0000000000D67758
...
<f5fe00>    Method: sun/reflect/DelegatingMethodAccessorImpl.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; !j9method 0x0000000000D68FE8
...
<f5fe00>    Method: java/lang/reflect/Method.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; !j9method 0x0000000000CCA800
...
<f5fe00>    Method: net/adoptopenjdk/loadTest/adaptors/MauveAdaptor.executeTest()Lnet/adoptopenjdk/loadTest/adaptors/AdaptorInterface$ResultStatus; !j9method 0x000000000107E818
...
<f5fe00>    Method: net/adoptopenjdk/loadTest/LoadTestRunner$2.run()V !j9method 0x000000000107FA68
...
<f5fe00>    Method: java/util/concurrent/ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V !j9method 0x0000000000FFC138
...
<f5fe00>    Method: java/util/concurrent/ThreadPoolExecutor$Worker.run()V !j9method 0x000000000109A100
...
<f5fe00>    Method: java/lang/Thread.run()V !j9method 0x0000000000CB7A60
...
<f5fe00> <end of stack>
<f5fe00> *** END STACK WALK (rc = NONE) ***

(full stack dump at [4])

There are a couple things to look into here that might help solve this issue:

  1. How are currentThread->returnValue2 and currentThread->returnValue changed between calling c_cinterpreter and VM_DebugBytecodeInterpreterCompressed::run?
  2. What is going on with CorruptData warning that shows up in jdmpview when dumping the Java stack?

[1]. https://github.com/eclipse-openj9/openj9/blob/3d06b2f9c2c60e7655ec1beb6d6708fe5acf0642/runtime/vm/BytecodeInterpreter.hpp#L10020-L10022 [2]. https://github.com/eclipse-openj9/openj9/blob/3d06b2f9c2c60e7655ec1beb6d6708fe5acf0642/runtime/vm/BytecodeInterpreter.hpp#L9389 [3]. https://github.com/eclipse-openj9/openj9/blob/3d06b2f9c2c60e7655ec1beb6d6708fe5acf0642/runtime/vm/callin.cpp#L1215-L1217 [4]. jdmpview.full.txt

pshipton commented 2 years ago

Originally reported in https://github.com/eclipse-openj9/openj9/issues/15303, copying the details here.

https://openj9-jenkins.osuosl.org/job/Test_openjdk8_j9_special.system_s390x_linux_Personal_testList_1/12 - ub18-390-1 MauveSingleInvocLoad_special_5m_8 -Xaggressive -Xgcpolicy:gencon -Xjit -Xnocompressedrefs

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk8_j9_special.system_s390x_linux_Personal_testList_1/12/system_test_output.tar.gz

LT  16:43:41.662 - Starting thread. Suite=0 thread=0
LT  stderr Unhandled exception
LT  stderr Type=Segmentation error vmState=0x00000000
LT  stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
LT  stderr Handler1=000003FF8B9C6958 Handler2=000003FF8B8B2E10 InaccessibleAddress=0000000000000000
LT  stderr gpr0=0000000000000000 gpr1=FFFFFFFFFFFFFFD0 gpr2=000003FF8BBAE010 gpr3=000003FF8C7AC400
LT  stderr gpr4=0000000000000001 gpr5=0000000000000005 gpr6=000003FF8C2BDC00 gpr7=000003FF8B77DC88
LT  stderr gpr8=000003FEC169BF77 gpr9=0000000000000000 gpr10=0000000000000000 gpr11=000003FF8C02D438
LT  stderr gpr12=000003FF0BF2AF78 gpr13=000003FF8BBAE658 gpr14=000003FF8BA42436 gpr15=000003FF8B77D7D0
LT  stderr psw=000003FF8BA42AE0 mask=0705000180000000 fpc=00f8ff00 bea=000003FF8BA42ACE
LT  stderr fpr0 000003ff8c159b00 (f: 2350226176.000000, d: 2.171963e-311)
LT  stderr fpr1 462918008b77d3a8 (f: 2339886080.000000, d: 9.940662e+29)
LT  stderr fpr2 000003ff8aeb98fa (f: 2330695936.000000, d: 2.171953e-311)
LT  stderr fpr3 000003ff8b77d5a8 (f: 2339886592.000000, d: 2.171958e-311)
LT  stderr fpr4 0000000000000043 (f: 67.000000, d: 3.310240e-322)
LT  stderr fpr5 000003ff8bcdba40 (f: 2345515520.000000, d: 2.171961e-311)
LT  stderr fpr6 0000000000040000 (f: 262144.000000, d: 1.295163e-318)
LT  stderr fpr7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr8 0000000000041000 (f: 266240.000000, d: 1.315400e-318)
LT  stderr fpr9 000003ffe897b170 (f: 3902255360.000000, d: 2.172730e-311)
LT  stderr fpr10 000003ff8b73f000 (f: 2339631104.000000, d: 2.171958e-311)
LT  stderr fpr11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr12 000002aa2773ebc0 (f: 661908416.000000, d: 1.447528e-311)
LT  stderr fpr13 000002aa28e41f60 (f: 686038912.000000, d: 1.447540e-311)
LT  stderr fpr14 000003ffd13f90b4 (f: 3510604032.000000, d: 2.172536e-311)
LT  stderr fpr15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr Module=/home/jenkins/workspace/Test_openjdk8_j9_special.system_s390x_linux_Personal_testList_1/openjdkbinary/j2sdk-image/jre/lib/s390x/default/libj9vm29.so
LT  stderr Module_base_address=000003FF8B980000
LT  stderr Target=2_90_20220611_366 (Linux 4.15.0-171-generic)
LT  stderr CPU=s390x (4 logical CPUs) (0x1f70fd000 RAM)
LT  stderr ----------- Stack Backtrace -----------
LT  stderr _ZN26VM_BytecodeInterpreterFull3runEP10J9VMThread+0x698 (0x000003FF8BA42AE0 [libj9vm29.so+0xc2ae0])
LT  stderr bytecodeLoopFull+0xd6 (0x000003FF8BA42436 [libj9vm29.so+0xc2436])
LT  stderr c_cInterpreter+0x64 (0x000003FF8BB0D1DC [libj9vm29.so+0x18d1dc])
LT  stderr ---------------------------------------
r30shah commented 2 years ago

@tajila Looking at the update from @Spencer-Comin and inspecting the core-dump, I see that there is some discrepancies between the values set in vmThread fields in sidecarInvokeReflectMethodImpl function and the point where it crashes with null receiver. We might be missing something or looking at the different direction than what you looked initially, but only JIT compiled method that we see in the failing stack is sun/reflect/NativeMethodAccessorImpl.invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; but Spencer's comment in https://github.com/eclipse-openj9/openj9/issues/14753#issuecomment-1152763727 suggests that receiver was corrupted in vmThread in between calls from sidecarInvokeReflectMethodImpl to VM_DebugBytecodeInterpreterCompressed::run . So there is no point in checking the frames below sidecarInvokeReflectMethodImpl.

Do you remember how you found out the jitted method you mentioned in https://github.com/eclipse-openj9/openj9/issues/14753#issuecomment-1075672991 ?

pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Test_openjdk8_j9_sanity.system_s390x_linux_Nightly_testList_2/379/ MauveSingleInvocLoad_J9_5m_0

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk8_j9_sanity.system_s390x_linux_Nightly_testList_2/379/system_test_output.tar.gz

LT  04:21:45.963 - Starting thread. Suite=0 thread=0
LT  stderr Unhandled exception
LT  stderr Type=Segmentation error vmState=0x00000000
LT  stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
LT  stderr Handler1=000003FFA28C6930 Handler2=000003FFA27B30F0 InaccessibleAddress=0000000000000000
LT  stderr gpr0=0000000000000000 gpr1=FFFFFFFFFFE6D934 gpr2=FFFFFFFFFFFFFFD0 gpr3=00000000015C5C00
LT  stderr gpr4=0000000000000001 gpr5=0000000000000005 gpr6=0000000001591400 gpr7=000003FFA26FDC88
LT  stderr gpr8=000003FF516A448F gpr9=0000000000000000 gpr10=0000000000000000 gpr11=000003FF9C02D4F8
LT  stderr gpr12=000000000DF27728 gpr13=000003FFA2AAE240 gpr14=000003FFA291AE96 gpr15=000003FFA26FD7C8
LT  stderr psw=000003FFA291B53A mask=0705000180000000 fpc=00f8ff00 bea=000003FFA291B528
LT  stderr fpr0 000003ff1c003ad0 (f: 469777088.000000, d: 2.171034e-311)
LT  stderr fpr1 47d65780a26fd2c8 (f: 2725237504.000000, d: 1.187894e+38)
LT  stderr fpr2 000003ffa1db9e72 (f: 2715524608.000000, d: 2.172143e-311)
LT  stderr fpr3 000003ffa26fce80 (f: 2725236224.000000, d: 2.172148e-311)
LT  stderr fpr4 0000000000000043 (f: 67.000000, d: 3.310240e-322)
LT  stderr fpr5 000003ffa2bdba40 (f: 2730342912.000000, d: 2.172151e-311)
LT  stderr fpr6 0000000000040000 (f: 262144.000000, d: 1.295163e-318)
LT  stderr fpr7 8000800080008000 (f: 2147516416.000000, d: -6.953462e-310)
LT  stderr fpr8 0000000000041000 (f: 266240.000000, d: 1.315400e-318)
LT  stderr fpr9 000003ffdf47bcd0 (f: 3746020608.000000, d: 2.172652e-311)
LT  stderr fpr10 000003ffa26bf000 (f: 2724982784.000000, d: 2.172148e-311)
LT  stderr fpr11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr fpr12 000002aa0775c540 (f: 125158720.000000, d: 1.447263e-311)
LT  stderr fpr13 000002aa26bdbe70 (f: 649969280.000000, d: 1.447522e-311)
LT  stderr fpr14 000003ffd3bfa7ac (f: 3552552960.000000, d: 2.172557e-311)
LT  stderr fpr15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
LT  stderr Module=/home/jenkins/workspace/Test_openjdk8_j9_sanity.system_s390x_linux_Nightly_testList_2/openjdkbinary/j2sdk-image/jre/lib/s390x/default/libj9vm29.so
LT  stderr Module_base_address=000003FFA2880000
LT  stderr Target=2_90_20221004_403 (Linux 4.15.0-171-generic)
LT  stderr CPU=s390x (4 logical CPUs) (0x1f70fd000 RAM)
LT  stderr ----------- Stack Backtrace -----------
LT  stderr _ZN32VM_BytecodeInterpreterCompressed3runEP10J9VMThread+0x692 (0x000003FFA291B53A [libj9vm29.so+0x9b53a])
LT  stderr bytecodeLoopCompressed+0xd6 (0x000003FFA291AE96 [libj9vm29.so+0x9ae96])
LT  stderr c_cInterpreter+0x64 (0x000003FFA2A0D2E4 [libj9vm29.so+0x18d2e4])
LT  stderr ---------------------------------------