search

eclipse-openj9 / openj9

Eclipse OpenJ9: A Java Virtual Machine for OpenJDK that's optimized for small footprint, fast start-up, and high throughput. Builds on Eclipse OMR (https://github.com/eclipse/omr) and combines with the Extensions for OpenJDK for OpenJ9 repo.
Other
3.28k stars 721 forks source link

DaaLoadTest_all_special_5m_6 crash vmState=0x00000000 #15066

Open pshipton opened 2 years ago

pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Test_openjdk18_j9_special.system_x86-64_windows_Personal_testList_0/5 DaaLoadTest_all_special_5m_6 -Xgcpolicy:optavgpause -Xjit:count=0 -Xnocompressedrefs

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk18_j9_special.system_x86-64_windows_Personal_testList_0/5/system_test_output.tar.gz

DLT 11:12:49.602 - Completed 33.6%. Number of tests started=53 (+13)
DLT stderr Unhandled exception
DLT stderr Type=Segmentation error vmState=0x00000000
DLT stderr Windows_ExceptionCode=c0000005 J9Generic_Signal=00000004 ExceptionAddress=00007FFABD289280 ContextFlags=0010005f
DLT stderr Handler1=00007FFABDDE7060 Handler2=00007FFABDCCA960 InaccessibleReadAddress=0000000000000000
DLT stderr RDI=000000801F867509 RSI=0000000000000000 RAX=0000000000000000 RBX=000000801F8673A8
DLT stderr RCX=0000000000000002 RDX=0000000000000000 R8=0000000000000000 R9=000000801F56F550
DLT stderr R10=00007FF70219FFA0 R11=000000801E30C098 R12=0000000000000000 R13=000000801E03A700
DLT stderr R14=000000801F56F550 R15=000000801F8673A8
DLT stderr Unhandled exception
DLT stderr Type=Segmentation error vmState=0x00000000
DLT stderr RIP=00007FFABD289280 RSP=000000801F56F4A0 RBP=0000008077DF1E60 EFLAGS=0000000000010246
DLT stderr FS=0053 ES=002B DS=002B
DLT stderr Windows_ExceptionCode=c0000005 J9Generic_Signal=00000004 ExceptionAddress=00007FFABD289280 ContextFlags=0010005f
DLT stderr Handler1=00007FFABDDE7060 Handler2=00007FFABDCCA960 InaccessibleReadAddress=0000000000000000
DLT stderr RDI=000000801F867509 RSI=0000000000000000 RAX=0000000000000000 RBX=000000801F8673A8
DLT stderr RCX=0000000000000002 RDX=0000000000000000 R8=0000000000000000 R9=000000801F36F040
DLT stderr R10=00007FF702FEEAD0 R11=000000801E309E88 R12=0000000000000000 R13=000000801E03BF00
DLT stderr R14=000000801F36F040 R15=000000801F8673A8
DLT stderr RIP=00007FFABD289280 RSP=000000801F36EF90 RBP=0000008077DF1E60 EFLAGS=0000000000010246
DLT stderr FS=0053 ES=002B DS=002B
DLT stderr XMM0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM1 0000008008fdc974 (f: 150849904.000000, d: 2.716900e-312)
DLT stderr XMM2 0000008008fdc974 (f: 150849904.000000, d: 2.716900e-312)
DLT stderr XMM3 000000003a1b594e (f: 974870848.000000, d: 4.816502e-315)
DLT stderr XMM4 000000004579f000 (f: 1165619200.000000, d: 5.758924e-315)
DLT stderr XMM5 c2fe000041300000 (f: 1093664768.000000, d: -5.277656e+14)
DLT stderr XMM6 bfa87a4b027e9b95 (f: 41851796.000000, d: -4.780802e-02)
DLT stderr XMM7 4006000000000000 (f: 0.000000, d: 2.750000e+00)
DLT stderr XMM8 0000000040400000 (f: 1077936128.000000, d: 5.325712e-315)
DLT stderr XMM9 0000000000e98aa1 (f: 15305377.000000, d: 7.561861e-317)
DLT stderr XMM10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM1 0000008008fdc974 (f: 150849904.000000, d: 2.716900e-312)
DLT stderr XMM2 0000008008fdc974 (f: 150849904.000000, d: 2.716900e-312)
DLT stderr XMM3 00000000ba77557e (f: 3128382720.000000, d: 1.545626e-314)
DLT stderr XMM4 0000000045b5b800 (f: 1169537024.000000, d: 5.778281e-315)
DLT stderr XMM5 c2fe000041400000 (f: 1094713344.000000, d: -5.277656e+14)
DLT stderr XMM6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM8 0000000040400000 (f: 1077936128.000000, d: 5.325712e-315)
DLT stderr XMM9 0000000000ddefc2 (f: 14544834.000000, d: 7.186103e-317)
DLT stderr XMM10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr Module=F:\Users\jenkins\workspace\Test_openjdk18_j9_special.system_x86-64_windows_Personal_testList_0\openjdkbinary\j2sdk-image\bin\default\j9jit29.dll
DLT stderr Module_base_address=00007FFABD010000 Offset_in_DLL=0000000000279280
DLT stderr Target=2_90_20220515_28 (Windows Server 2012 R2 6.3 build 9600)
DLT stderr CPU=amd64 (8 logical CPUs) (0x1ffb9c000 RAM)
DLT stderr ----------- Stack Backtrace -----------
DLT stderr Module=F:\Users\jenkins\workspace\Test_openjdk18_j9_special.system_x86-64_windows_Personal_testList_0\openjdkbinary\j2sdk-image\bin\default\j9jit29.dll
DLT stderr Module_base_address=00007FFABD010000 Offset_in_DLL=0000000000279280
DLT stderr Target=2_90_20220515_28 (Windows Server 2012 R2 6.3 build 9600)
DLT stderr CPU=amd64 (8 logical CPUs) (0x1ffb9c000 RAM)
DLT stderr ----------- Stack Backtrace -----------
DLT stderr Java_java_lang_invoke_ThunkTuple_initialInvokeExactThunk+0x1af360 (0x00007FFABD289280 [j9jit29+0x279280])
DLT stderr (0x00007FFABDDDC0AF [j9vm29+0xbc0af])
DLT stderr Java_java_lang_ref_Reference_waitForReferenceProcessingImpl+0x297a (0x00007FFABCA704BA [jclse29+0x104ba])
DLT stderr Java_java_lang_J9VMInternals_getStackTrace+0x39 (0x00007FFABCA6D8C9 [jclse29+0xd8c9])
DLT stderr (0x00007FFAA9D499F1)
DLT stderr (0x00007FF70219FC80)
DLT stderr (0x000000801F56FA00)
DLT stderr (0x0000000000000001)
DLT stderr (0x000000801F56FA00)
DLT stderr (0x00007FFAA9E9F340)
DLT stderr (0x000000801E03A700)
DLT stderr (0x00007FF702ADD970)
DLT stderr (0x0000000000000010)
DLT stderr (0x0000000000000018)
DLT stderr JVM_OnUnload+0x23d10 (0x00007FFABDC9D0C0 [j9trc29+0x2d0c0])
DLT stderr (0x000000801E03A700)
DLT stderr (0x000000801F56FA00)
DLT stderr ---------------------------------------
DLT stderr Java_java_lang_invoke_ThunkTuple_initialInvokeExactThunk+0x1af360 (0x00007FFABD289280 [j9jit29+0x279280])
DLT stderr (0x00007FFABDDDC0AF [j9vm29+0xbc0af])
DLT stderr Java_java_lang_ref_Reference_waitForReferenceProcessingImpl+0x297a (0x00007FFABCA704BA [jclse29+0x104ba])
DLT stderr Java_java_lang_J9VMInternals_getStackTrace+0x39 (0x00007FFABCA6D8C9 [jclse29+0xd8c9])
DLT stderr (0x00007FFAA9D499F1)
DLT stderr (0x00007FF702FEE798)
DLT stderr (0x000000801F36F4F0)
DLT stderr (0x0000000000000001)
DLT stderr (0x000000801F36F4F0)
DLT stderr (0x00007FFAA9E9F340)
DLT stderr (0x000000801E03BF00)
DLT stderr (0x000000801DDA9800)
DLT stderr (0x00007FF701F47A48)
DLT stderr (0x00007FF7027D65E8)
DLT stderr JVM_OnUnload+0x23d10 (0x00007FFABDC9D0C0 [j9trc29+0x2d0c0])
DLT stderr (0x000000801E03BF00)
DLT stderr (0x000000801F36F4F0)
DLT stderr ---------------------------------------
pshipton commented 2 years ago

30x (6x x 5) grinder https://openj9-jenkins.osuosl.org/job/Grinder/811/

pshipton commented 2 years ago

Failed 2/30 in the grinder. Trying another without the -Xjit:count=0, https://openj9-jenkins.osuosl.org/job/Grinder/812/

pshipton commented 2 years ago

Failed 1/40 with the addition of -Xjit. Trying with gencon, https://openj9-jenkins.osuosl.org/job/Grinder/813/

pshipton commented 2 years ago

0/50 failed with gencon.

pshipton commented 2 years ago

javacore shows:

1XMCURTHDINFO  Current thread
3XMTHREADINFO      "load-3" J9VMThread:0x000000801E03BF00, omrthread_t:0x0000008078CC78C0, java/lang/Thread:0x00007FF701D47F20, state:R, prio=5
3XMJAVALTHREAD            (java/lang/Thread getId:0x1D, isDaemon:false)
3XMJAVALTHRCCL            jdk/internal/loader/ClassLoaders$AppClassLoader(0x00007FF7018E4C60)
3XMTHREADINFO1            (native thread ID:0xF70, native priority:0x5, native policy:UNKNOWN, vmstate:R, vm thread flags:0x00000020)
3XMCPUTIME               CPU usage total: 0.500000000 secs, user: 0.218750000 secs, system: 0.281250000 secs, current category="Application"
3XMHEAPALLOC             Heap bytes allocated since last GC cycle=0 (0x0)
3XMTHREADINFO3           Java callstack:
4XESTACKTRACE                at java/lang/J9VMInternals.getStackTrace(Native Method)
4XESTACKTRACE                at java/lang/Throwable.getInternalStackTrace(Throwable.java:292)
4XESTACKTRACE                at java/lang/Throwable.getStackTrace(Throwable.java:217)
4XESTACKTRACE                at jdk/internal/reflect/AccessorUtils.isIllegalArgument(AccessorUtils.java:58)
4XESTACKTRACE                at jdk/internal/reflect/DirectMethodHandleAccessor.isIllegalArgument(DirectMethodHandleAccessor.java:196)
4XESTACKTRACE                at jdk/internal/reflect/DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:113(Compiled Code))
4XESTACKTRACE                at java/lang/reflect/Method.invoke(Method.java:577(Compiled Code))
4XESTACKTRACE                at org/junit/runners/model/FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50(Compiled Code))
4XESTACKTRACE                at org/junit/internal/runners/model/ReflectiveCallable.run(ReflectiveCallable.java:12(Compiled Code))
4XESTACKTRACE                at org/junit/runners/model/FrameworkMethod.invokeExplosively(FrameworkMethod.java:47(Compiled Code))

Native stack

ntdll!ZwWaitForSingleObject+0xa
KERNELBASE!WaitForSingleObjectEx+0x98
j9prt29!omrdump_create+0x300
j9dmp29!doSystemDump+0xa3
j9dmp29!protectedDumpFunction+0x15
j9prt29!runInTryExcept+0x16
j9prt29!omrsig_protect+0x210
j9dmp29!runDumpAgent+0x2f2
j9dmp29!triggerDumpAgents+0x3a6
j9vm29!generateDiagnosticFiles+0x1ef
j9prt29!runInTryExcept+0x16
j9prt29!omrsig_protect+0x210
j9vm29!vmSignalHandler+0x1d2
j9prt29!mainVectoredExceptionHandler+0x154
ntdll!RtlRestoreContext+0x182
ntdll!RtlRaiseException+0xe33
ntdll!KiUserExceptionDispatcher+0x3a
j9jit29!getCurrentByteCodeIndexAndIsSameReceiver+0x160
j9vm29!iterateStackTrace+0x8ff
jclse29!getStackTrace+0x10a

@tajila can someone take a look, I've marked this as a blocker for the next release.

pshipton commented 2 years ago

Sorry, I just noticed this looks like JIT after all. @0xdaryl @hzongaro can someone take a look, I've marked this as a blocker for the next release.

hzongaro commented 2 years ago

Taking a look. . . .

hzongaro commented 2 years ago

I was able to reproduce this 1/50 in a Grinder run with the nightly build from April 21 https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Build_JDK18_x86-64_windows_Nightly/56/OpenJ9-JDK18-x86-64_windows-20220421-202221.tar.gz - which was deleted from Artifactory within the past 24 hours. Continuing to look at it. . . .

pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Test_openjdk18_j9_special.system_aarch64_mac_Personal_testList_0/6/ DaaLoadTest_all_special_5m_1 -Xgcpolicy:optthruput -Xjit:count=0 -Xnocompressedref

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk18_j9_special.system_aarch64_mac_Personal_testList_0/6/system_test_output.tar.gz

DLT 00:35:30.329 - Completed 7.9%. Number of tests started=101
DLT stderr Unhandled exception
DLT stderr Type=Segmentation error vmState=0x00000000
DLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000002
DLT stderr Handler1=00000001024A3FD4 Handler2=000000010266AD9C InaccessibleAddress=0000000000000000
DLT stderr x0=00000001493D4168 x1=0000000000000000 x2=0000000000000000 x3=000000016EA0E820
DLT stderr x4=00000001397A6540 x5=000000016EA0E57C x6=0000000280695A08 x7=0000000102689978
DLT stderr x8=0000000000020001 x9=0000000000000000 x10=000000013A1E6200 x11=0000000020200020
DLT stderr x12=0000000020200020 x13=00000000FFFFFFFF x14=00000000916BE3F2 x15=0000000000008897
DLT stderr x16=000000018E806940 x17=0000000020200000 x18=00000002831CFB60 x19=000000016EA0E820
DLT stderr x20=00000001493D4168 x21=0000000141A5BAE8 x22=00000001493D42EA x23=0000000139B5C100
DLT stderr x24=0000000000000000 x25=00000001493D42E8 x26=0000000000000000 x27=00000001493D4168
DLT stderr x28=0000000107C918B4 x29(FP)=000000016EA0E880 x30(LR)=000000010249A7C4 x31(SP)=000000016EA0E780
DLT stderr PC=0000000106A9D9F8 SP=000000016EA0E780
DLT stderr v0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v1 0000002000000020 (f: 32.000000, d: 6.790387e-313)
DLT stderr v2 3fe7d05f00000000 (f: 0.000000, d: 7.441859e-01)
DLT stderr v3 3fd2e8e36ae11e23 (f: 1793138176.000000, d: 2.954644e-01)
DLT stderr v4 00000001493d5468 (f: 1228756096.000000, d: 2.729082e-314)
DLT stderr v5 0000000141ace2c8 (f: 1101849344.000000, d: 2.666382e-314)
DLT stderr v6 00000001141acb18 (f: 337300256.000000, d: 2.288644e-314)
DLT stderr v7 00000001141acb18 (f: 337300256.000000, d: 2.288644e-314)
DLT stderr v8 3f81020000000000 (f: 0.000000, d: 8.304596e-03)
DLT stderr v9 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v16 bfd0000000000000 (f: 0.000000, d: -2.500000e-01)
DLT stderr v17 3fd55fc0aae55555 (f: 2867156224.000000, d: 3.339693e-01)
DLT stderr v18 bf64dd770e84a012 (f: 243572752.000000, d: -2.547009e-03)
DLT stderr v19 3fe62e42fefa39ef (f: 4277811712.000000, d: 6.931472e-01)
DLT stderr v20 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v21 ffffffffffffffff (f: 4294967296.000000, d: nan)
DLT stderr v22 ffffffffffffffff (f: 4294967296.000000, d: nan)
DLT stderr v23 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v24 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v25 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v26 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v27 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v28 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr v29 00000072fff0de39 (f: 4293975552.000000, d: 2.440290e-312)
DLT stderr v30 008007e16ecbc408 (f: 1858847744.000000, d: 2.853574e-306)
DLT stderr v31 0000000040400000 (f: 1077936128.000000, d: 5.325712e-315)
DLT stderr Module=/Users/jenkins/workspace/Test_openjdk18_j9_special.system_aarch64_mac_Personal_testList_0/openjdkbinary/j2sdk-image/lib/default/libj9jit29.dylib
DLT stderr Module_base_address=0000000106840000 Symbol=getCurrentByteCodeIndexAndIsSameReceiver
DLT stderr Symbol_address=0000000106A9D8F4
DLT stderr Target=2_90_20220522_29 (Mac OS X 11.4)
DLT stderr CPU=aarch64 (8 logical CPUs) (0x400000000 RAM)
DLT stderr ----------- Stack Backtrace -----------
DLT stderr ---------------------------------------
pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Test_openjdk18_j9_special.system_x86-64_linux_Personal_testList_3/6 DaaLoadTest_daa2_special_5m_6

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk18_j9_special.system_x86-64_linux_Personal_testList_3/6/system_test_output.tar.gz

DLT 12:00:42.646 - Completed 48.0%. Number of tests started=27 (+2)
DLT stderr Unhandled exception
DLT stderr Type=Segmentation error vmState=0x00000000
DLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
DLT stderr Handler1=00007F430A3AB880 Handler2=00007F430B8465C0 InaccessibleAddress=0000000000000000
DLT stderr RDI=0000000000000000 RSI=0000000000000000 RAX=0000000000000000 RBX=00007F42615EC5B8
DLT stderr RCX=00007F4261ABF178 RDX=00007F42615EC719 R8=0000000000000001 R9=0000000000000000
DLT stderr R10=00007F420403413C R11=00007F4204032F00 R12=00007F4261ABF178 R13=00007F42615EC719
DLT stderr R14=0000000000000001 R15=00007F4304013030
DLT stderr RIP=00007F43095D3A60 GS=0000 FS=0000 RSP=00007F4261ABF098
DLT stderr EFlags=0000000000010206 CS=0033 RBP=0000000000000000 ERR=0000000000000004
DLT stderr TRAPNO=000000000000000E OLDMASK=0000000000000000 CR2=0000000000000000
DLT stderr xmm0 00007f42314fefac (f: 827322304.000000, d: 6.913079e-310)
DLT stderr xmm1 00007f4283b507f0 (f: 2209679360.000000, d: 6.913147e-310)
DLT stderr xmm2 00000000b7654321 (f: 3076866816.000000, d: 1.520174e-314)
DLT stderr xmm3 00000000394aa265 (f: 961192576.000000, d: 4.748922e-315)
DLT stderr xmm4 00000000bdf1779e (f: 3186718720.000000, d: 1.574448e-314)
DLT stderr xmm5 0000003038bdf5c5 (f: 951973312.000000, d: 1.023261e-312)
DLT stderr xmm6 00000000be6191e7 (f: 3194065408.000000, d: 1.578078e-314)
DLT stderr xmm7 0000003000000020 (f: 32.000000, d: 1.018558e-312)
DLT stderr xmm8 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr xmm9 403e000000000000 (f: 0.000000, d: 3.000000e+01)
DLT stderr xmm10 ff000000ff0000ff (f: 4278190336.000000, d: -5.486129e+303)
DLT stderr xmm11 0000ff0000000000 (f: 0.000000, d: 1.385239e-309)
DLT stderr xmm12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr xmm13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr xmm14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr xmm15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr Module=/home/jenkins/workspace/Test_openjdk18_j9_special.system_x86-64_linux_Personal_testList_3/openjdkbinary/j2sdk-image/lib/default/libj9jit29.so
DLT stderr Module_base_address=00007F4309120000
DLT stderr Target=2_90_20220522_30 (Linux 3.10.0-1127.13.1.el7.x86_64)
DLT stderr CPU=amd64 (4 logical CPUs) (0x1e8cf4000 RAM)
DLT stderr ----------- Stack Backtrace -----------
DLT stderr getInlinedMethod+0x0 (0x00007F43095D3A60 [libj9jit29.so+0x4b3a60])
DLT stderr getFirstInlinedCallSiteWithByteCodeInfo+0x4b (0x00007F43095D3C0B [libj9jit29.so+0x4b3c0b])
DLT stderr getCurrentByteCodeIndexAndIsSameReceiver+0x97 (0x00007F43095D3D77 [libj9jit29.so+0x4b3d77])
DLT stderr iterateStackTrace+0x34b (0x00007F430A39F08B [libj9vm29.so+0x3108b])
DLT stderr getStackTrace+0xc5 (0x00007F4308C5CB55 [libjclse29.so+0x21b55])
DLT stderr Java_java_lang_J9VMInternals_getStackTrace+0x2e (0x00007F4308C5885E [libjclse29.so+0x1d85e])
DLT stderr  (0x00007F4269545731 [<unknown>+0x0])
DLT stderr ---------------------------------------
hzongaro commented 2 years ago

Looking at the x86_64-linux failure mentioned in the previous comment, the getStackTrace in operating on a NullPointerException that has this walkback field:

(kca) whatis 0x00007f42843a98e0
Obj - {[J}T
(kca) obj 0x00007f42843a98e0
Object is an Array of:
  Class Path/Name: {[J} J9Class 0x00007f4304118700
      ClassObject: 0x00007f4283781d60
           Access: Public Final Abstract  (10411)
      ClassLoader: 0x00007f430407d558  Object: 0x00007f42837af090 {jdk/internal/loader/ClassLoaders$BootClassLoader}T
     SubClassLink: 0x00007f4304118300  {[I}
        Hierarchy: (depth 1)
                   {java/lang/Object} J9Class 0x00007f430410dc00
       Interfaces: 
                   {java/lang/Cloneable} J9Class 0x00007f4304116900
                   {java/io/Serializable} J9Class 0x00007f4304110200
        J9Methods: 0x00007f4304118500 (0 methods)
Dump of Contiguous Array Elements (32 elements):
0x00007f42843a98f8: 00007f4269549c77 00007f42695418bf   [ w.TiB... ..TiB... ]
0x00007f42843a9908: 00007f426961832c 00007f4269826ba0   [ ,.aiB... .k.iB... ]
0x00007f42843a9918: 00007f42698a3c99 00007f426982e7fc   [ .<.iB... ...iB... ]
0x00007f42843a9928: 00007f4269a49d1b 00007f42698ab2ef   [ ...iB... ...iB... ]
0x00007f42843a9938: 00007f4269824295 00007f4269591a7a   [ .B.iB... z.YiB... ]
0x00007f42843a9948: 00007f4269823278 00007f4269809520   [ x2.iB... ...iB... ]
0x00007f42843a9958: 00007f4269809046 00007f4269806601   [ F..iB... .f.iB... ]
0x00007f42843a9968: 00007f42698a1adc 00007f4269df9773   [ ...iB... s..iB... ]
0x00007f42843a9978: 00007f4269df5435 00007f4269d6477e   [ 5T.iB... ~G.iB... ]
0x00007f42843a9988: 00007f4269f63985 00007f4269df561d   [ .9.iB... .V.iB... ]
0x00007f42843a9998: 00007f4269806601 00007f42698a1adc   [ .f.iB... ...iB... ]
0x00007f42843a99a8: 00007f4269df9773 00007f4269df5435   [ s..iB... 5T.iB... ]
0x00007f42843a99b8: 00007f4269d6477e 00007f42695d43c6   [ ~G.iB... .C]iB... ]
0x00007f42843a99c8: 00007f4269d0313b 00007f426942dd40   [ ;1.iB... @.BiB... ]
0x00007f42843a99d8: 00007f4269426fec 00007f423192998e   [ .oBiB... ...1B... ]
0x00007f42843a99e8: 00007f42319caeb5 00007f42314ba259   [ ...1B... Y.K1B... ]

Looking at the first few entries, I'm not sure what the first address is referring to:

Ptr Unknown!
(kca) whatis 0x00007f42695418bf
{java/lang/invoke/LambdaForm$DMH/0x000000000400da60.invokeVirtual} +319
(kca) whatis 0x00007f426961832c
{java/lang/invoke/LambdaForm$MH/0x0000000004031190.invoke} +332
(kca) whatis 0x00007f4269826ba0
{org/junit/runners/model/FrameworkMethod$1.runReflectiveCall} +768
(kca) whatis 0x00007f42698a3c99
{org/junit/internal/runners/model/ReflectiveCallable.run} +57
(kca) whatis 0x00007f426982e7fc
{org/junit/runners/model/FrameworkMethod.invokeExplosively} +148
(kca) whatis 0x00007f4269a49d1b
{org/junit/internal/runners/statements/InvokeMethod.evaluate} +155
(kca) whatis 0x00007f42698ab2ef
{org/junit/internal/runners/statements/ExpectException.evaluate} +79
(kca) whatis 0x00007f4269824295

In any event, I'm able to reproduce the failure locally, intermittently, so I'll continue to try to understand what's going wrong. . . .

hzongaro commented 2 years ago

Further update regarding the core file from the x86 Linux failure. The crash happens in getInlinedMethod because inlinedCallSite is NULL.

It looks like in this case it holds a pointer to the J9JITExceptionTable for net/openj9/test/binaryData/TestByteArray2Float.testMaximumValidOffset:

(kca) whatis regs
$rax  = 0x0000000000000000 Null
$rbx  = 0x00007f42615ec5b8 MetaData - {net/openj9/test/binaryData/TestByteArray2Float.testMaximumValidOffset}
$rcx  = 0x00007f4261abf178 Ptr Unknown
$rdx  = 0x00007f42615ec719 Ptr Unknown
...

Looking at that method, it looks like it was compiled twice. The J9JITExceptionTable it's looking at appears to be the one for the method compiled at cold, which had no methods inlined. The method compiled at warm, however, had a method inlined at two call sites.

(kca) m {net/openj9/test/binaryData/TestByteArray2Float.testMaximumValidOffset*}
    J9Class            J9Method               Start          Len {ClassPath/Name.MethodName}
--------------------------------------------------------------------------------------------
(0x00007f4204069c00 0x00007f4204069a00) 0x00007f42694ad724    67 {net/openj9/test/binaryData/TestByteArray2Float.testMaximumValidOffset()V}
(0x00007f4204069c00 0x00007f4204069a00) 0x00007f42696ca024  1746 {net/openj9/test/binaryData/TestByteArray2Float.testMaximumValidOffset()V}

(kca) m verbose 0x00007f42694ad724
         Method Signature: {net/openj9/test/binaryData/TestByteArray2Float.testMaximumValidOffset()V}
                 MetaData: 0x00007f42615ec5b8 (optLevel: cold)
               Frame Size: 408 bytes
                   Access: Public
         J9Class/J9Method: 0x00007f4204069c00 / 0x00007f4204069a00
               MethodInfo: 0x00007f426186d4f0
                 BodyInfo: 0x00007f426186d580 (Flags: Loops UsesGCR)
Compiled Method Start/End: 0x00007f42694ad724 / 0x00007f42694ad767 (67 bytes)
            No inlined calls
          Exception Table: (0 entries)
   Error reading the GCMap data.

(kca) m verbose 0x00007f42696ca024
         Method Signature: {net/openj9/test/binaryData/TestByteArray2Float.testMaximumValidOffset()V}
                 MetaData: 0x00007f426161ac38 (optLevel: warm)
               Frame Size: 120 bytes
                   Access: Public
         J9Class/J9Method: 0x00007f4204069c00 / 0x00007f4204069a00
               MethodInfo: 0x00007f426186d4f0
                 BodyInfo: 0x00007f42641b6280 (Flags: Loops)
Compiled Method Start/End: 0x00007f42696ca024 / 0x00007f42696ca6f6 (1746 bytes)
            Inlined Calls: J9Method, Index, Caller, ByteCode, {Class.Method} (2 inlined method(s))
                           0x00007f43048e5d78    0      -1      91 ( 5b)    {com/ibm/dataaccess/ByteArrayUnmarshaller.readFloat}
                           0x00007f43048e5d78    1      -1     157 ( 9d)    {com/ibm/dataaccess/ByteArrayUnmarshaller.readFloat}
          Exception Table: (0 entries)
GCMap StackAllocMap: No contiguous stack allocated objects!
GCMap Data: (32 maps)
                    Address    caller ByteCode [RegMap]    [StackMap]    [LiveMonitorMap]
            0x00007f42696ca09d     -1        3 0000000b 01
                  Registers: rax rbx rdx
                Stack Slots: P0x80
                  ...

I'm wondering whether the NullPointerException occurred in the warm-compiled method, but somehow the call to jitGetExceptionTableFromPC might be returning the J9JITExceptionTable for the cold-compiled method - possibly because it's retrieving stale cache information?

hzongaro commented 2 years ago

Peter mentioned in an earlier comment a Grinder run with -Xjit that failed 1/40. I believe that the way the options processing in the JIT works, specifying the jit option with suboptions and then specifying the jit option again without any suboptions will not override the suboptions that were previously specified - so -Xjit:count=0 -Xjit is the same as just specifying -Xjit:count=0.

I did some (internal) Grinder runs on Windows, Mac aarch64 and x86 Linux to double-check the frequency of failures - one set of runs with -Xjit:limit={*} as the EXTRA_OPTIONS parameter, to override the -Xjit:count=0, and a second set of runs with -Xjit:limit={*},count=0 (just to make sure the limit={*} didn't have any real effect).

Number of failures

-Xjit:limit={*} -Xjit:limit={*},count=0
Windows/x86 0/100 4/100
Mac/aarch64 0/100 1/100
Linux/x86 0/100 1/100

Given that this is an intermittent problem and the failure rate without -Xjit:count=0 seems to be even lower, perhaps the Blocker tag could be removed. @pshipton Peter, any thoughts?

pshipton commented 2 years ago

Removed the blocker label.

pshipton commented 2 years ago

Since it's not a blocker, and doesn't show up much, and it doesn't seem like it's going to be resolved shortly, moving to the Backlog.

hzongaro commented 2 years ago

it doesn't seem like it's going to be resolved shortly

Sad to say, no, not likely. I'm wondering about the possibility of some sort of race condition in the JIT's method metadata reclamation. Looking at the metadata from several core files that iterateStackTrace was operating on, I consistently see this:

(kca) struct J9JITExceptionTable 0x00007f4787ed0db8
J9JITExceptionTable (216 bytes)
                            struct J9UTF8*  className = 0x00007f4780102890 (offset: 0)
                           struct J9UTF8*  methodName = 0x00007f47801029ce (offset: 8) // Warning: ptr is not correctly aligned!
                      struct J9UTF8*  methodSignature = 0x00007f47ad45945a (offset: 16) // Warning: ptr is not correctly aligned!
   ...
                                            UDATA  flags = 0x0000000000000004 (offset: 96)
                       UDATA  registerSaveDescription = 0x0000000000000202 (offset: 104)
                                  void*  gcStackAtlas = 0x0000000000000000 (offset: 112)
                                  void*  inlinedCalls = 0x0000000000000000 (offset: 120)
   ...
                                 JIT_METADATA_IS_STUB = 0x4 / 0x0 (constant)

The flags field says it's a stub, which I understand means that the metadata was reclaimed, but iterateStackTrace is walking through inline calls, so I'm wondering whether the metadata could have been reclaimed while iterateStackTrace was working with the metadata.

In recent grinder runs on Windows-x86 and Linux-x86, I saw 1/100 failures with -Xjit:count=0 and 0/100 failures with -Xjit:count=0,disableMetadataReclamation - but the frequency of failures is so low and so easily perturbed, that that isn't very strong evidence. Further, in a recent run on Mac/OS-aarch64, I saw 0/100 with both -Xjit:count=0 and -Xjit:count=0,disableMetadataReclamation.

JasonFengJ9 commented 2 years ago

Adding an internal occurrence job/Test_openjdk18_j9_special.system_x86-64_windows_testList_4/58/(win10x64vs6)

openjdk version "18.0.1.1" 2022-04-22
IBM Semeru Runtime Open Edition 18.0.1.1+2 (build 18.0.1.1+2)
Eclipse OpenJ9 VM 18.0.1.1+2 (build master-79f0b73fa, JRE 18 Windows 10 amd64-64-Bit Compressed References 20220702_17 (JIT enabled, AOT enabled)
OpenJ9   - 79f0b73fa
OMR      - d018241d7
JCL      - 8c019e1aa5 based on jdk-18.0.1.1+2)

[2022-07-02T22:46:59.343Z] Running test DaaLoadTest_daa2_special_5m_1 ...
[2022-07-02T22:46:59.687Z] ===============================================
[2022-07-02T22:47:00.031Z] DaaLoadTest_daa2_special_5m_1 Start Time: Sat Jul  2 18:46:59 2022 Epoch Time (ms): 1656802019727
[2022-07-02T22:47:00.031Z] "C:/Users/jenkins/workspace/Test_openjdk18_j9_special.system_x86-64_windows_testList_4/openjdkbinary/j2sdk-image\\bin\\java" -Xshareclasses:destroyAll; "C:/Users/jenkins/workspace/Test_openjdk18_j9_special.system_x86-64_windows_testList_4/openjdkbinary/j2sdk-image\\bin\\java" -Xshareclasses:groupAccess,destroyAll; echo "cache cleanup done";
[2022-07-02T22:47:00.796Z] JVMSHRC005I No shared class caches available
[2022-07-02T22:47:01.546Z] JVMSHRC005I No shared class caches available
[2022-07-02T22:47:01.546Z] cache cleanup done
[2022-07-02T22:47:01.546Z] variation: Mode103
[2022-07-02T22:47:01.890Z] JVM_OPTIONS:  -Xgcpolicy:optthruput -Xjit:count=0 -Xnocompressedrefs 

[2022-07-02T22:48:05.033Z] DLT stderr Unhandled exception
[2022-07-02T22:48:05.033Z] DLT stderr Type=Segmentation error vmState=0x00000000
[2022-07-02T22:48:05.033Z] DLT stderr Windows_ExceptionCode=c0000005 J9Generic_Signal=00000004 ExceptionAddress=00007FFB1F8737C0 ContextFlags=0010005f
[2022-07-02T22:48:05.033Z] DLT stderr Handler1=00007FFB22037060 Handler2=00007FFB27FFA960 InaccessibleReadAddress=0000000000000000
[2022-07-02T22:48:05.033Z] DLT stderr RDI=0000018C43F90A49 RSI=0000000000000000 RAX=0000000000000000 RBX=0000018C43F908E8
[2022-07-02T22:48:05.033Z] DLT stderr RCX=0000000000000002 RDX=0000000000000000 R8=0000000000000000 R9=000000477A7FEF70
[2022-07-02T22:48:05.033Z] DLT stderr R10=00007FFB38510000 R11=000000477A7FEE50 R12=0000000000000000 R13=0000018C44EA2400
[2022-07-02T22:48:05.033Z] DLT stderr R14=000000477A7FEF70 R15=0000018C43F908E8
[2022-07-02T22:48:05.033Z] DLT stderr RIP=00007FFB1F8737C0 RSP=000000477A7FEEC0 RBP=0000018C26D73CC0 EFLAGS=0000000000010246
[2022-07-02T22:48:05.033Z] DLT stderr FS=0053 ES=002B DS=002B
[2022-07-02T22:48:05.033Z] DLT stderr XMM0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
[2022-07-02T22:48:05.033Z] DLT stderr XMM1 0000018c3091caa0 (f: 814860928.000000, d: 8.407129e-312)
[2022-07-02T22:48:05.033Z] DLT stderr XMM2 0000018c3091caa0 (f: 814860928.000000, d: 8.407129e-312)
[2022-07-02T22:48:05.033Z] DLT stderr XMM3 00000000bb5a4294 (f: 3143254784.000000, d: 1.552974e-314)
[2022-07-02T22:48:05.033Z] DLT stderr XMM4 0000000045077000 (f: 1158115328.000000, d: 5.721850e-315)
[2022-07-02T22:48:05.033Z] DLT stderr XMM5 c2fe000041300000 (f: 1093664768.000000, d: -5.277656e+14)
[2022-07-02T22:48:05.033Z] DLT stderr XMM6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
[2022-07-02T22:48:05.033Z] DLT stderr XMM7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
[2022-07-02T22:48:05.033Z] DLT stderr XMM8 0000018c41400000 (f: 1094713344.000000, d: 8.408512e-312)
[2022-07-02T22:48:05.033Z] DLT stderr XMM9 00000ee97ffa747d (f: 2147120256.000000, d: 8.100719e-311)
[2022-07-02T22:48:05.033Z] DLT stderr XMM10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
[2022-07-02T22:48:05.033Z] DLT stderr XMM11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
[2022-07-02T22:48:05.033Z] DLT stderr XMM12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
[2022-07-02T22:48:05.033Z] DLT stderr XMM13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
[2022-07-02T22:48:05.033Z] DLT stderr XMM14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
[2022-07-02T22:48:05.033Z] DLT stderr XMM15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
[2022-07-02T22:48:05.033Z] DLT stderr Module=C:\Users\jenkins\workspace\Test_openjdk18_j9_special.system_x86-64_windows_testList_4\openjdkbinary\j2sdk-image\bin\default\j9jit29.dll
[2022-07-02T22:48:05.033Z] DLT stderr Module_base_address=00007FFB1F600000 Offset_in_DLL=00000000002737c0
[2022-07-02T22:48:05.033Z] DLT stderr Target=2_90_20220702_17 (Windows 10 10.0 build 19042)
[2022-07-02T22:48:05.033Z] DLT stderr CPU=amd64 (16 logical CPUs) (0x3fff06000 RAM)
[2022-07-02T22:48:05.033Z] DLT stderr ----------- Stack Backtrace -----------
[2022-07-02T22:48:05.033Z] DLT stderr Java_java_lang_invoke_ThunkTuple_initialInvokeExactThunk+0x1ab0d0 (0x00007FFB1F8737C0 [j9jit29+0x2737c0])
[2022-07-02T22:48:05.033Z] DLT stderr (0x00007FFB2202C0AF [j9vm29+0xbc0af])
[2022-07-02T22:48:05.033Z] DLT stderr Java_java_lang_ref_Reference_waitForReferenceProcessingImpl+0x295a (0x00007FFB2560049A [jclse29+0x1049a])
[2022-07-02T22:48:05.033Z] DLT stderr Java_java_lang_J9VMInternals_getStackTrace+0x39 (0x00007FFB255FD8C9 [jclse29+0xd8c9])
[2022-07-02T22:48:05.033Z] DLT stderr (0x00007FFB0C4BF0F1)
[2022-07-02T22:48:05.033Z] DLT stderr (0x00007FF37AAB0A90)
[2022-07-02T22:48:05.033Z] DLT stderr (0x000000477A7FF420)
[2022-07-02T22:48:05.033Z] DLT stderr (0x0000000000000001)
[2022-07-02T22:48:05.033Z] DLT stderr (0x000000477A7FF420)
[2022-07-02T22:48:05.033Z] DLT stderr (0x00007FFB0C48F740)
[2022-07-02T22:48:05.033Z] DLT stderr (0x0000018C44EA2400)
[2022-07-02T22:48:05.033Z] DLT stderr (0x00000000000006BE)
[2022-07-02T22:48:05.033Z] DLT stderr (0x00007FF37A16E400)
[2022-07-02T22:48:05.033Z] DLT stderr ---------------------------------------
[2022-07-02T22:48:05.033Z] DLT stderr JVMDUMP039I Processing dump event "gpf", detail "" at 2022/07/02 18:47:59 - please wait.

[2022-07-02T22:48:14.580Z] DaaLoadTest_daa2_special_5m_1_FAILED
pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Test_openjdk18_j9_special.system_s390x_linux_Release_testList_3/12 DaaLoadTest_daa2_special_5m_6

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk18_j9_special.system_s390x_linux_Release_testList_3/12/system_test_output.tar.gz

DLT 22:37:24.729 - Completed 7.4%. Number of tests started=28
DLT stderr Unhandled exception
DLT stderr Type=Segmentation error vmState=0x00000000
DLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=20295670 Signal_Code=00000001
DLT stderr Handler1=000003FFA33434E0 Handler2=000003FFA3231778 InaccessibleAddress=0000000000000000
DLT stderr gpr0=0000000000000000 gpr1=000003FF00000000 gpr2=0000000000000000 gpr3=0000000000000000
DLT stderr gpr4=000003FFA22CEACE gpr5=000003FFA307E310 gpr6=000003FE9C08F4C0 gpr7=000003FF9C808D00
DLT stderr gpr8=0000000000000000 gpr9=000003FEF9F5E6D9 gpr10=000003FEF9F5E578 gpr11=0000000000000000
DLT stderr gpr12=000003FF00B6B030 gpr13=000003FF00000001 gpr14=000003FFA22CEAEE gpr15=000003FFA307E078
DLT stderr psw=000003FFA22CE8F8 mask=0705000180000000 fpc=0008ff00 bea=000003FFA22CEAE8
DLT stderr fpr0 4215563700000000 (f: 0.000000, d: 2.291019e+10)
DLT stderr fpr1 3e638ec300000000 (f: 0.000000, d: 3.642899e-08)
DLT stderr fpr2 3e3ab28300000000 (f: 0.000000, d: 6.215952e-09)
DLT stderr fpr3 3b7de14300000000 (f: 0.000000, d: 3.954576e-22)
DLT stderr fpr4 bdf54d2f00000000 (f: 0.000000, d: -3.099776e-10)
DLT stderr fpr5 3e92498100000000 (f: 0.000000, d: 2.724994e-07)
DLT stderr fpr6 3ecccd3d00000000 (f: 0.000000, d: 3.433432e-06)
DLT stderr fpr7 3e3a33ba00000000 (f: 0.000000, d: 6.100642e-09)
DLT stderr fpr8 000003ffa339c970 (f: 2738473216.000000, d: 2.172155e-311)
DLT stderr fpr9 ce34561ba421c756 (f: 2753677056.000000, d: -5.482672e+68)
DLT stderr fpr10 000003ff9c0139d0 (f: 2617326080.000000, d: 2.172095e-311)
DLT stderr fpr11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr fpr12 0005e37e6b405495 (f: 1799378048.000000, d: 8.189173e-309)
DLT stderr fpr13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr fpr14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr fpr15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr Module=/home/jenkins/workspace/Test_openjdk18_j9_special.system_s390x_linux_Release_testList_3/openjdkbinary/j2sdk-image/lib/default/libj9jit29.so
DLT stderr Module_base_address=000003FFA1D80000
DLT stderr Target=2_90_20220710_18 (Linux 3.10.0-1160.42.2.el7.s390x)
DLT stderr CPU=s390x (4 logical CPUs) (0x1ec1b1000 RAM)
DLT stderr ----------- Stack Backtrace -----------
DLT stderr getInlinedMethod+0x0 (0x000003FFA22CE8F8 [libj9jit29.so+0x54e8f8])
DLT stderr getFirstInlinedCallSiteWithByteCodeInfo+0x56 (0x000003FFA22CEAEE [libj9jit29.so+0x54eaee])
DLT stderr getCurrentByteCodeIndexAndIsSameReceiver+0xd6 (0x000003FFA22CECD6 [libj9jit29.so+0x54ecd6])
DLT stderr iterateStackTrace+0x4e4 (0x000003FFA3334C24 [libj9vm29.so+0x34c24])
DLT stderr getStackTrace+0xfe (0x000003FFA17A3E06 [libjclse29.so+0x23e06])
DLT stderr Java_java_lang_J9VMInternals_getStackTrace+0x3e (0x000003FFA179F3CE [libjclse29.so+0x1f3ce])
DLT stderr  (0x000003FF00B6AFC4 [<unknown>+0x0])
DLT stderr ---------------------------------------
pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Test_openjdk18_j9_special.system_ppc64le_linux_Personal_testList_3/16 DaaLoadTest_daa2_special_5m_6

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk18_j9_special.system_ppc64le_linux_Personal_testList_3/16/system_test_output.tar.gz

DLT 22:01:46.973 - Completed 7.0%. Number of tests started=47
DLT stderr Unhandled exception
DLT stderr Type=Segmentation error vmState=0x00000000
DLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
DLT stderr Handler1=00003FFFB0DBEC10 Handler2=00003FFFB0CE8440
DLT stderr R0=00003FFFA4D49DB8 R1=00003FFEF23BB4E0 R2=00003FFFA56FF000 R3=0000000000000000
DLT stderr R4=0000000000000000 R5=00003FFEF21FDB99 R6=00003FFEF23BB5E8 R7=0000000000010510
DLT stderr R8=0000000000000008 R9=0000000000000000 R10=0000000000000000 R11=00003FFE9003B020
DLT stderr R12=00003FFFA4D49FC0 R13=00003FFEF23C6900 R14=0000000000000000 R15=0000000000000137
DLT stderr R16=00003FFE9C01E4B2 R17=00003FFFACB01A00 R18=00003FFFB02B4640 R19=0000000000000001
DLT stderr R20=00003FFEF21FDB97 R21=0000000000000000 R22=0000000000000000 R23=0000000000000000
DLT stderr R24=00003FFFAC0CF280 R25=0000000000000002 R26=00003FFE90007768 R27=00003FFFAC0134F0
DLT stderr R28=00003FFEF23BB5E8 R29=0000000000000000 R30=00003FFEF21FDA38 R31=0000000000000000
DLT stderr NIP=00003FFFA4D49B20 MSR=800000010280F033 ORIG_GPR3=00000000000081C8 CTR=00003FFFA4D49FC0
DLT stderr LINK=00003FFFA4D49DE8 XER=0000000000000000 CCR=0000000022824840 SOFTE=0000000000000001
DLT stderr TRAP=0000000000000300 DAR=0000000000000000 dsisr=0000000040000000 RESULT=0000000000000000
DLT stderr FPR0 00003fffb1351220 (f: 2973045248.000000, d: 3.476613e-310)
DLT stderr FPR1 4046c04a80000000 (f: 2147483648.000000, d: 4.550227e+01)
DLT stderr FPR2 3ff2489940000000 (f: 1073741824.000000, d: 1.142724e+00)
DLT stderr FPR3 3fee666660000000 (f: 1610612736.000000, d: 9.500000e-01)
DLT stderr FPR4 3fe3333340000000 (f: 1073741824.000000, d: 6.000000e-01)
DLT stderr FPR5 3fca181e80000000 (f: 2147483648.000000, d: 2.038611e-01)
DLT stderr FPR6 3fe5555560000000 (f: 1610612736.000000, d: 6.666667e-01)
DLT stderr FPR7 3f1c7e7740000000 (f: 1073741824.000000, d: 1.086960e-04)
DLT stderr FPR8 4028000000000000 (f: 0.000000, d: 1.200000e+01)
DLT stderr FPR9 3fd99999a0000000 (f: 2684354560.000000, d: 4.000000e-01)
DLT stderr FPR10 3f20718080000000 (f: 2147483648.000000, d: 1.254529e-04)
DLT stderr FPR11 3ff0000000000000 (f: 0.000000, d: 1.000000e+00)
DLT stderr FPR12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR13 4008000000000000 (f: 0.000000, d: 3.000000e+00)
DLT stderr FPR14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR16 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR17 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR18 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR19 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR20 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR21 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR22 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR23 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR24 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR25 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR26 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR27 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR28 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR29 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR30 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr FPR31 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr Module=/home/jenkins/workspace/Test_openjdk18_j9_special.system_ppc64le_linux_Personal_testList_3/openjdkbinary/j2sdk-image/lib/default/libj9jit29.so
DLT stderr Module_base_address=00003FFFA47C0000
DLT stderr Target=2_90_20220724_64 (Linux 3.10.0-1160.25.1.el7.ppc64le)
DLT stderr CPU=ppc64le (8 logical CPUs) (0x1de290000 RAM)
DLT stderr ----------- Stack Backtrace -----------
DLT stderr protectedBacktrace+0x24 (0x00003FFFB0CE2A74 [libj9prt29.so+0x32a74])
DLT stderr omrsig_protect+0x3f4 (0x00003FFFB0CE9914 [libj9prt29.so+0x39914])
DLT stderr omrintrospect_backtrace_thread_raw+0xd8 (0x00003FFFB0CE3088 [libj9prt29.so+0x33088])
DLT stderr protectedIntrospectBacktraceThread+0x24 (0x00003FFFB0CE2774 [libj9prt29.so+0x32774])
DLT stderr omrsig_protect+0x3f4 (0x00003FFFB0CE9914 [libj9prt29.so+0x39914])
DLT stderr omrintrospect_backtrace_thread+0x8c (0x00003FFFB0CE286C [libj9prt29.so+0x3286c])
DLT stderr generateDiagnosticFiles+0xd4 (0x00003FFFB0DBEA44 [libj9vm29.so+0x3ea44])
DLT stderr omrsig_protect+0x3f4 (0x00003FFFB0CE9914 [libj9prt29.so+0x39914])
DLT stderr structuredSignalHandler+0x198 (0x00003FFFB0DBEDA8 [libj9vm29.so+0x3eda8])
DLT stderr mainSynchSignalHandler+0x294 (0x00003FFFB0CE86D4 [libj9prt29.so+0x386d4])
DLT stderr __kernel_sigtramp_rt64+0x0 (0x00003FFFB1710478)
DLT stderr getFirstInlinedCallSiteWithByteCodeInfo+0x48 (0x00003FFFA4D49DB8 [libj9jit29.so+0x589db8])
DLT stderr getCurrentByteCodeIndexAndIsSameReceiver+0x114 (0x00003FFFA4D4A0D4 [libj9jit29.so+0x58a0d4])
DLT stderr iterateStackTrace+0x384 (0x00003FFFB0DAE894 [libj9vm29.so+0x2e894])
DLT stderr getStackTrace+0x104 (0x00003FFFB02B6234 [libjclse29.so+0x26234])
DLT stderr Java_java_lang_J9VMInternals_getStackTrace+0x5c (0x00003FFFB02B148C [libjclse29.so+0x2148c])
DLT stderr  (0x00003FFF1261480C [<unknown>+0x0])
DLT stderr runJavaThread+0x240 (0x00003FFFB0D966D0 [libj9vm29.so+0x166d0])
DLT stderr javaProtectedThreadProc+0x138 (0x00003FFFB0E2D268 [libj9vm29.so+0xad268])
DLT stderr omrsig_protect+0x3f4 (0x00003FFFB0CE9914 [libj9prt29.so+0x39914])
DLT stderr javaThreadProc+0x60 (0x00003FFFB0E28B80 [libj9vm29.so+0xa8b80])
DLT stderr thread_wrapper+0x190 (0x00003FFFB0C7CBC0 [libj9thr29.so+0xcbc0])
DLT stderr start_thread+0x104 (0x00003FFFB1648CD4 [libpthread.so.0+0x8cd4])
DLT stderr clone+0xe4 (0x00003FFFB1547F14 [libc.so.6+0x127f14])
DLT stderr ---------------------------------------
pshipton commented 2 years ago

https://openj9-jenkins.osuosl.org/job/Test_openjdk18_j9_special.system_x86-64_windows_Personal_testList_0/17 DaaLoadTest_daa2_special_5m_6

https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk18_j9_special.system_x86-64_windows_Personal_testList_0/17/system_test_output.tar.gz

DLT 23:32:37.693 - Starting thread. Suite=0 thread=5
DLT stderr Unhandled exception
DLT stderr Type=Segmentation error vmState=0x00000000
DLT stderr Windows_ExceptionCode=c0000005 J9Generic_Signal=00000004 ExceptionAddress=00007FFB6DFF3CC0 ContextFlags=0010005f
DLT stderr Handler1=00007FFB6E7CA990 Handler2=00007FFB6EA2AA10 InaccessibleReadAddress=0000000000000000
DLT stderr RDI=000000475B710949 RSI=0000000000000000 RAX=0000000000000000 RBX=000000475B7107E8
DLT stderr RCX=0000000000000002 RDX=0000000000000000 R8=0000000000000000 R9=000000475B3CF2B0
DLT stderr R10=00007FF6DAA96F80 R11=000000475A0D1EC8 R12=0000000000000000 R13=0000004759DEDB00
DLT stderr R14=000000475B3CF2B0 R15=000000475B7107E8
DLT stderr RIP=00007FFB6DFF3CC0 RSP=000000475B3CF200 RBP=0000004748B3E3F0 EFLAGS=0000000000010246
DLT stderr FS=0053 ES=002B DS=002B
DLT stderr XMM0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM1 000000475418f0ac (f: 1410920576.000000, d: 1.513588e-312)
DLT stderr XMM2 000000475418f0ac (f: 1410920576.000000, d: 1.513588e-312)
DLT stderr XMM3 000000003ad344cb (f: 986924224.000000, d: 4.876054e-315)
DLT stderr XMM4 000000004b2323bf (f: 1260594048.000000, d: 6.228162e-315)
DLT stderr XMM5 c2fe000041b80000 (f: 1102577664.000000, d: -5.277657e+14)
DLT stderr XMM6 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM8 0000004741400000 (f: 1094713344.000000, d: 1.512026e-312)
DLT stderr XMM9 000000004b095440 (f: 1258902528.000000, d: 6.219805e-315)
DLT stderr XMM10 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM13 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr XMM15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
DLT stderr Module=F:\Users\jenkins\workspace\Test_openjdk18_j9_special.system_x86-64_windows_Personal_testList_0\openjdkbinary\j2sdk-image\bin\default\j9jit29.dll
DLT stderr Module_base_address=00007FFB6DD80000 Offset_in_DLL=0000000000273cc0
DLT stderr Target=2_90_20220807_50 (Windows Server 2012 R2 6.3 build 9600)
DLT stderr CPU=amd64 (8 logical CPUs) (0x1ffb9c000 RAM)
DLT stderr ----------- Stack Backtrace -----------
DLT stderr Java_java_lang_invoke_ThunkTuple_initialInvokeExactThunk+0x1ab4f0 (0x00007FFB6DFF3CC0 [j9jit29+0x273cc0])
DLT stderr (0x00007FFB6E7BF82F [j9vm29+0xbf82f])
DLT stderr Java_java_lang_ref_Reference_waitForReferenceProcessingImpl+0x295a (0x00007FFB6D71049A [jclse29+0x1049a])
DLT stderr Java_java_lang_J9VMInternals_getStackTrace+0x39 (0x00007FFB6D70D8C9 [jclse29+0xd8c9])
DLT stderr (0x00007FFB5A930EB1)
DLT stderr (0x00007FF6DABDC950)
DLT stderr (0x000000475B3CF760)
DLT stderr (0x0000000000000001)
DLT stderr (0x000000475B3CF760)
DLT stderr (0x00007FFB5AB44360)
DLT stderr (0x0000004759DEDB00)
DLT stderr (0x0000004758601700)
DLT stderr (0x00007FF6DAB70FE0)
DLT stderr (0x0000000000000001)
DLT stderr JVM_OnUnload+0x23d10 (0x00007FFB6E69D0C0 [j9trc29+0x2d0c0])
DLT stderr (0x0000004759DEDB00)
DLT stderr (0x000000475B3CF760)
DLT stderr ---------------------------------------