Closed pshipton closed 2 years ago
@0xdaryl fyi. This occurred in the last weekend testing on the 0.28 branches. Might be worth taking a look before moving it out.
Possible dup of https://github.com/eclipse-openj9/openj9/issues/13179 but this was closed as a dup of https://github.com/eclipse-openj9/openj9/issues/13431 which is aarch64 and resolved.
https://openj9-jenkins.osuosl.org/job/Test_openjdk17_j9_special.system_ppc64le_linux_Release_testList_3/11 - ub18-ppcle-2
MathLoadTest_bigdecimal_special_5m_16
-XX:+UseCompressedOops -Xgcpolicy:balanced -Xjit:counts=- - - - - - 1 1 1 1000 250 250 - - - 10000 100000 10000,gcOnResolve,rtResolve,sampleInterval=2,scorchingSampleThreshold=10000,quickProfile -Xcheck:gc:vmthreads:all:quiet
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00007F5566F354D0 Handler2=00007F5566E6C300
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00007F5566F354D0 Handler2=00007F5566E6C300
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00007F5566F354D0 Handler2=00007F5566E6C300
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00007F5566F354D0 Handler2=00007F5566E6C300
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00007F5566F354D0 Handler2=00007F5566E6C300
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00007F5566F354D0 Handler2=00007F5566E6C300
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00007F5566F354D0 Handler2=00007F5566E6C300
MLT stderr R0=00007F55433D619C R1=00007F55216CB690 R2=00007F556670F200 R3=000000000060FB00
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000087DA4DA8 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000030 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00007F556641C7C8
MLT stderr R12=000000000029EF00 R13=00007F55216D68E0 R14=0000000000620EF0 R15=000000000060FB00
MLT stderr R16=00007F5523E10038 R17=FFFFFFFFFFFFFFFF R18=0000000082690508 R19=0000000083C80CF8
MLT stderr R20=0000000081C134E0 R21=0000000083032B38 R22=0000000083304C90 R23=0000000083C0DD40
MLT stderr R24=00000000833A8AC8 R25=0000000080AA9ED8 R26=0000000080AA9ED8 R27=000000008829A6A8
MLT stderr R28=0000000088302970 R29=000000008829A0D0 R30=0000000000714900 R31=0000000087DA4DA8
MLT stderr R0=00007F55433D619C R1=00007F55213FB690 R2=00007F556670F200 R3=000000000061B300
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000087DA4DA8 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000034 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00007F556641C7C8
MLT stderr R12=000000000029EF00 R13=00007F55214068E0 R14=00000000006318D0 R15=000000000061B300
MLT stderr R16=00007F5523E10038 R17=FFFFFFFFFFFFFFFF R18=0000000080000007 R19=0000000083C80CF8
MLT stderr R20=00000000000020A8 R21=00000000817457F0 R22=00000000833098C8 R23=0000000083C0DD40
MLT stderr R24=00000000833BDE20 R25=0000000080AA9ED8 R26=0000000080AA9ED8 R27=00000000882306C0
MLT stderr R28=0000000081594B00 R29=00000000882300E8 R30=0000000000714900 R31=0000000087DA4DA8
MLT stderr R0=00007F55433D619C R1=00007F5522A2B690 R2=00007F556670F200 R3=00000000005E6E00
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000087DA4DA8 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000034 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00007F556641C7C8
MLT stderr R12=000000000029EF00 R13=00007F5522A368E0 R14=000000000062C030 R15=00000000005E6E00
MLT stderr R16=00007F5523E10038 R17=FFFFFFFFFFFFFFFF R18=0000000081597FE8 R19=0000000083C80CF8
MLT stderr R20=0000000081C134E0 R21=0000000083744E20 R22=000000008442F3E0 R23=0000000083C0DD40
MLT stderr R24=000000008442F3F8 R25=0000000080AA9ED8 R26=0000000080AA9ED8 R27=00000000882C9068
MLT stderr R28=0000000081594D88 R29=000000008442F1B8 R30=0000000000714900 R31=0000000087DA4DA8
MLT stderr NIP=00007F55664060A0 MSR=800000010280F033 ORIG_GPR3=00000000000081C8 CTR=00007F556641C7C8
MLT stderr LINK=00007F556641C7DC XER=0000000020000000 CCR=0000000042004248 SOFTE=0000000000000001
MLT stderr TRAP=0000000000000300 DAR=0000000000000017 dsisr=0000000040000000 RESULT=0000000000000000
MLT stderr R0=00007F55433D619C R1=00007F552148B690 R2=00007F556670F200 R3=0000000000618E00
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000087DA4DA8 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000037 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00007F556641C7C8
MLT stderr R12=000000000029EF00 R13=00007F55214968E0 R14=00000000006422B0 R15=0000000000618E00
MLT stderr R16=00007F5523E10038 R17=FFFFFFFFFFFFFFFF R18=0000000080000007 R19=0000000083C80CF8
MLT stderr R20=0000000000000000 R21=0000000000000000 R22=0000000082D02EC0 R23=0000000083C0DD40
MLT stderr R24=0000000082D187B8 R25=0000000080AA9ED8 R26=0000000080AA9ED8 R27=0000000088220848
MLT stderr R28=00000000832C7828 R29=0000000088220270 R30=0000000000714900 R31=0000000087DA4DA8
MLT stderr R0=00007F55433D619C R1=00007F552163B690 R2=00007F556670F200 R3=0000000000612000
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000087DA4DA8 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000036 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00007F556641C7C8
MLT stderr R12=000000000029EF00 R13=00007F55216468E0 R14=00000000006F9050 R15=0000000000612000
MLT stderr R16=00007F5523E10038 R17=FFFFFFFFFFFFFFFF R18=0000000080000007 R19=0000000083C80CF8
MLT stderr R20=0000000000000000 R21=0000000080AC4C20 R22=0000000084300178 R23=0000000083C0DD40
MLT stderr R24=00000000843255F8 R25=0000000080AA9ED8 R26=0000000080AA9ED8 R27=000000008826EF50
MLT stderr R28=0000000083F06E20 R29=000000008826E978 R30=0000000000714900 R31=0000000087DA4DA8
MLT stderr R0=00007F55433D619C R1=00007F5522BDB690 R2=00007F556670F200 R3=00000000005E6100
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000087DA4DA8 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000037 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00007F556641C7C8
MLT stderr R12=000000000029EF00 R13=00007F5522BE68E0 R14=00000000005EE1B0 R15=00000000005E6100
MLT stderr R16=00007F5523E10038 R17=FFFFFFFFFFFFFFFF R18=0000000080000007 R19=0000000083C80CF8
MLT stderr R20=00000000000020A8 R21=000000008171E948 R22=00000000833000F8 R23=0000000083C0DD40
MLT stderr R24=00000000833B91A8 R25=0000000080AA9ED8 R26=0000000080AA9ED8 R27=000000008823D590
MLT stderr R28=00000000811D7E78 R29=000000008823CFB8 R30=0000000000714900 R31=0000000087DA4DA8
MLT stderr NIP=00007F55664060A0 MSR=800000010280F033 ORIG_GPR3=00000000000081C8 CTR=00007F556641C7C8
MLT stderr LINK=00007F556641C7DC XER=0000000020000000 CCR=0000000042004248 SOFTE=0000000000000001
MLT stderr TRAP=0000000000000300 DAR=0000000000000017 dsisr=0000000040000000 RESULT=0000000000000000
MLT stderr R0=00007F55433D619C R1=00007F552190B690 R2=00007F556670F200 R3=00000000005FE700
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000087DA4DA8 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000032 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00007F556641C7C8
MLT stderr R12=000000000029EF00 R13=00007F55219168E0 R14=000000000064D3F0 R15=00000000005FE700
MLT stderr R16=00007F5523E10038 R17=FFFFFFFFFFFFFFFF R18=00000000824F5090 R19=0000000083C80CF8
MLT stderr R20=0000000081C134E0 R21=00000000833BA8E0 R22=0000000083313000 R23=0000000083C0DD40
MLT stderr R24=0000000083315690 R25=0000000080AA9ED8 R26=0000000080AA9ED8 R27=00000000882527D0
MLT stderr R28=0000000081594A78 R29=00000000882521F8 R30=0000000000714900 R31=0000000087DA4DA8
MLT stderr NIP=00007F55664060A0 MSR=800000010280F033 ORIG_GPR3=00000000000081C8 CTR=00007F556641C7C8
MLT stderr LINK=00007F556641C7DC XER=0000000020000000 CCR=0000000044004248 SOFTE=0000000000000001
MLT stderr TRAP=0000000000000300 DAR=0000000000000017 dsisr=0000000040000000 RESULT=0000000000000000
MLT stderr NIP=00007F55664060A0 MSR=800000010280F033 ORIG_GPR3=00000000000081C8 CTR=00007F556641C7C8
MLT stderr LINK=00007F556641C7DC XER=0000000020000000 CCR=0000000042004248 SOFTE=0000000000000001
MLT stderr TRAP=0000000000000300 DAR=0000000000000017 dsisr=0000000040000000 RESULT=0000000000000000
MLT stderr NIP=00007F55664060A0 MSR=800000010280F033 ORIG_GPR3=00000000000081C8 CTR=00007F556641C7C8
MLT stderr LINK=00007F556641C7DC XER=0000000020000000 CCR=0000000042004248 SOFTE=0000000000000001
MLT stderr TRAP=0000000000000300 DAR=0000000000000017 dsisr=0000000040000000 RESULT=0000000000000000
MLT stderr NIP=00007F55664060A0 MSR=800000010280F033 ORIG_GPR3=00000000000081C8 CTR=00007F556641C7C8
MLT stderr LINK=00007F556641C7DC XER=0000000020000000 CCR=0000000042004448 SOFTE=0000000000000001
MLT stderr TRAP=0000000000000300 DAR=0000000000000017 dsisr=0000000040000000 RESULT=0000000000000000
MLT stderr NIP=00007F55664060A0 MSR=800000010280F033 ORIG_GPR3=00000000000081C8 CTR=00007F556641C7C8
MLT stderr LINK=00007F556641C7DC XER=0000000020000000 CCR=0000000042004248 SOFTE=0000000000000001
MLT stderr TRAP=0000000000000300 DAR=0000000000000017 dsisr=0000000040000000 RESULT=0000000000000000
MLT stderr FPR0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR1 4050285700000000 (f: 0.000000, d: 6.463031e+01)
MLT stderr FPR2 c032f703035cfc16 (f: 56425496.000000, d: -1.896489e+01)
MLT stderr FPR3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR4 3fce840b4ac4e4d2 (f: 1254417664.000000, d: 2.384047e-01)
MLT stderr FPR5 bfe7154748bef6c8 (f: 1220474624.000000, d: -7.213475e-01)
MLT stderr FPR6 3fe62e42fefa39ef (f: 4277811712.000000, d: 6.931472e-01)
MLT stderr FPR7 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR8 0074007300650074 (f: 6619252.000000, d: 1.780215e-306)
MLT stderr FPR9 006d006500740049 (f: 7602249.000000, d: 1.290611e-306)
MLT stderr FPR10 003200330030005f (f: 3145823.000000, d: 1.001327e-307)
MLT stderr FPR11 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR12 0000000000000001 (f: 1.000000, d: 4.940656e-324)
MLT stderr FPR13 4020800000000000 (f: 0.000000, d: 8.250000e+00)
MLT stderr FPR14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR16 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR17 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR18 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR19 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR20 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR21 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR22 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR23 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR24 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR25 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR26 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR27 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR28 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR29 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR30 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR31 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR1 4050e674c0000000 (f: 3221225472.000000, d: 6.760088e+01)
MLT stderr FPR2 bffce61cf8ef36fd (f: 4176426752.000000, d: -1.806180e+00)
MLT stderr FPR3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR4 3fce840b4ac4e4d2 (f: 1254417664.000000, d: 2.384047e-01)
MLT stderr FPR5 bfe7154748bef6c8 (f: 1220474624.000000, d: -7.213475e-01)
MLT stderr FPR6 3fe62e42fefa39ef (f: 4277811712.000000, d: 6.931472e-01)
MLT stderr FPR7 bfc1aa2bc79c8100 (f: 3348922624.000000, d: -1.380057e-01)
MLT stderr FPR8 0074007300650074 (f: 6619252.000000, d: 1.780215e-306)
MLT stderr FPR9 006d006500740049 (f: 7602249.000000, d: 1.290611e-306)
MLT stderr FPR10 003100330030005f (f: 3145823.000000, d: 9.456997e-308)
MLT stderr FPR11 3f27b8ae6f573b37 (f: 1867987712.000000, d: 1.809800e-04)
MLT stderr FPR12 0000000000000001 (f: 1.000000, d: 4.940656e-324)
MLT stderr FPR13 4020800000000000 (f: 0.000000, d: 8.250000e+00)
MLT stderr FPR14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR16 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR17 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR18 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR19 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR20 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR21 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR22 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR23 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR24 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR25 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR26 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR27 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR28 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR29 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR30 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR31 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR1 404fa35ee0000000 (f: 3758096384.000000, d: 6.327633e+01)
MLT stderr FPR2 3fe62e42fefa39ef (f: 4277811712.000000, d: 6.931472e-01)
MLT stderr FPR3 3fc7466496cb03de (f: 2529887232.000000, d: 1.818357e-01)
MLT stderr FPR4 3fcc71c51d8e78af (f: 495876256.000000, d: 2.222220e-01)
MLT stderr FPR5 3fd2492494229359 (f: 2485293824.000000, d: 2.857143e-01)
MLT stderr FPR6 3fe62e42fefa39ef (f: 4277811712.000000, d: 6.931472e-01)
MLT stderr FPR7 bfd01eae7f513a67 (f: 2136029824.000000, d: -2.518727e-01)
MLT stderr FPR8 0074007300650074 (f: 6619252.000000, d: 1.780215e-306)
MLT stderr FPR9 006d006500740049 (f: 7602249.000000, d: 1.290611e-306)
MLT stderr FPR10 003400340030005f (f: 3145823.000000, d: 1.112581e-307)
MLT stderr FPR11 3efa7a9b1cd8b59d (f: 483964320.000000, d: 2.525227e-05)
MLT stderr FPR12 0000000000000001 (f: 1.000000, d: 4.940656e-324)
MLT stderr FPR13 4020800000000000 (f: 0.000000, d: 8.250000e+00)
MLT stderr FPR14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR16 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR17 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR18 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR19 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR20 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR21 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR22 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR23 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR24 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR25 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR26 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR27 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR28 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR29 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR30 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR31 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR1 405011af80000000 (f: 2147483648.000000, d: 6.427634e+01)
MLT stderr FPR2 bffce61cf8ef36fd (f: 4176426752.000000, d: -1.806180e+00)
MLT stderr FPR3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR4 3fcc71c51d8e78af (f: 495876256.000000, d: 2.222220e-01)
MLT stderr FPR5 3fd2492494229359 (f: 2485293824.000000, d: 2.857143e-01)
MLT stderr FPR6 3fe62e42fefa39ef (f: 4277811712.000000, d: 6.931472e-01)
MLT stderr FPR7 bfb1973c5a611ccc (f: 1516313856.000000, d: -6.871392e-02)
MLT stderr FPR8 0074007300650074 (f: 6619252.000000, d: 1.780215e-306)
MLT stderr FPR9 006d006500740049 (f: 7602249.000000, d: 1.290611e-306)
MLT stderr FPR10 003600320030005f (f: 3145823.000000, d: 1.223833e-307)
MLT stderr FPR11 3ef25b7d71a199cc (f: 1906416128.000000, d: 1.750696e-05)
MLT stderr FPR12 0000000000000001 (f: 1.000000, d: 4.940656e-324)
MLT stderr FPR13 4020800000000000 (f: 0.000000, d: 8.250000e+00)
MLT stderr FPR14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR16 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR17 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR18 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR19 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR20 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR21 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR22 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR23 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR24 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR25 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR26 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR27 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR28 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR29 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR30 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR31 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr Module=/home/jenkins/workspace/Test_openjdk17_j9_special.system_ppc64le_linux_Release_testList_3/openjdkbinary/j2sdk-image/lib/default/libj9jit29.so
... more
The only two failures like this so far have been on Power. Requesting @zl-wang to assign for triage please.
@ymanton please give this a crack ...
Looks like the crash happens while we are executing code for an inlined method that has been unloaded. Unloaded method calls jitCheckCast
with a class pointer that we pulled from the PTOC, but that class pointer is -1
because the PTOC slot was presumably clobbered once the class was unloaded. Control seems to fall through to the unloaded inlined method without passing through any sort of guard so one wasn't generated at compile time nor was one patched in once the class was unloaded.
that -1 is not a clobbering value. it really confirmed that class has been unloaded. on class unloading, we intentionally did:
setTOCSlot(hash[idx]._tocIndex * sizeof(uintptr_t), -1);
this means inlined guard is missing. @0xdaryl
I can't manually reproduce this, but I ran it and collected a trace file anyway since whether or not it crashes seems to depend on timing. In the core file, KCA has the inlining table as:
Inlined Calls: J9Method, Index, Caller, ByteCode, {Class.Method} (4 inlined method(s))
0x000000000069a098 0 -1 58 ( 3a) {java/lang/invoke/LambdaForm$MH/0x00000000cc04b030.linkToTargetMethod}
0x0000000000699bf9 1 0 13 ( d) Unloaded Method!
0x00000000002fb098 2 -1 71 ( 47) {java/lang/invoke/LambdaForm$MH/0x000000007c357b60.linkToTargetMethod}
0x00000000004a9fc8 3 2 5 ( 5) {java/lang/invoke/LambdaForm$MH/0x000000007c5f91b0.invoke}
In the trace log the inlining table is:
Call Stack Info
CalleeIndex CallerIndex ByteCodeIndex CalleeMethod
0 -1 58 java/lang/invoke/LambdaForm$MH/0x000000002803e150.linkToTargetMethod(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;
1 0 13 java/lang/invoke/LambdaForm$MH/0x000000002803d7e0.invoke(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;
2 -1 71 java/lang/invoke/LambdaForm$MH/0x00000000d846f450.linkToTargetMethod(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;
3 2 5 java/lang/invoke/LambdaForm$MH/0x00000000d83cb490.invoke(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;
Unloaded method has index 1, class name 0x000000002803d7e0
.
From the trace log it seems we don't generate any kind of guard for this method during inlining:
[ 80] O^O INLINER: Inlining qwerty
java/lang/invoke/LambdaForm$MH/0x000000002803d7e0.invoke(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;
into
java/lang/invoke/LambdaForm$MH/0x000000002803e150.linkToTargetMethod(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;
with a virtual guard kind=NoGuard type=NonoverriddenTest partialInline=0
@jdmpapin : please investigate this.
@ymanton : are you able to attach the log you last generated to this issue?
This was very puzzling, since customization was the only thing I knew could change the form
of a MethodHandle, and I found the method handle in the core, but its form
wasn't customized. Then I found a couple of other places that can change the form
(removing ensureInitialized()
from a DirectMethodHandle
, removing counting logic from a CountingWrapper
), but neither of those applied either, because the handle was a BoundMethodHandle$Species_LLL
. Nonetheless, something had definitely changed—the J9Method
implementing the handle was different from the one that had been in the inlining table (which can be seen by ignoring the low tag indicating that the method has been unloaded)
So I wrote some special instrumentation to find the change. When compiling JUnitAdaptor$1.addResult
, it would find the invokedynamic
at -1:58, save a number of references into new JNI global refs, and generate a series of ZEROCHK
trees checking for the first reference that differs in this chain of dependent loads: callSiteTableEntry[1].form.vmentry.vmtarget
. I hacked ZEROCHK
in that compilation to fail with an illegal instruction so that I could get a crash as soon as a difference was detected. Thankfully, I managed to get such a failure in my grinder, and from the core I determined that the call site table entry, MethodHandle
(appendix), and LambdaForm
were all the same as they were at compile time, but LambdaForm.vmentry
had been updated to refer to a different instance of MemberName
It turns out that the logic that compiles LambdaForm
to bytecode is racy, and it's possible to redundantly generate the same bytecode multiple times for a single LambdaForm
. For most purposes this is fine, since all of the generated classes are generally interchangeable, but if one is observed by the JIT, then the JIT can generate code that refers to it, and if the reference is later replaced, it can create an opportunity for the earlier one to get unloaded
The race condition is technically platform-agnostic, but I think it's more likely to appear on architectures (such as Power) that have a weaker memory model than x86
Here is a repro that fails reliably on my (x86 Linux) machine when run like so:
$ java -Xms32m -Xmx32m -Xshareclasses:none -Xjit:'limit={*.jitme*},count=1,initialOptLevel=warm,inhibitRecompilation,rtResolve' Repro
A few things to note about this repro:
invokedynamic
instructionWhile the failure I looked at in this issue was not caused by an update to MethodHandle.form
, it's still possible for such an update to cause a similar failure. Here is a second repro that fails reliably on my machine when run like so:
$ jdmpHackDelay=1 java -Xms32m -Xmx32m -Xshareclasses:none -Xjit:'limit={*.jitme*},count=0,initialOptLevel=warm,inhibitRecompilation,rtResolve' Repro
This is the same command line as above, but with an additional environment variable set, and with count=0
instead of count=1
. Like the first repro, the class file has been modified to get the right invokedynamic
instruction. Also like the first repro, reliable failure requires the included JCL patch, but this one needs the included JIT patch as well. Again, the patches only affect timing
I have a fix that prevents unloading in both of these cases and allows the generated code to run correctly. At the moment it's undergoing some broader testing, and I hope to have a PR open for it soon
https://openj9-jenkins.osuosl.org/job/Test_openjdk17_j9_special.system_ppc64le_linux_Release_testList_4/16 - cent7-ppcle-5
MathLoadTest_bigdecimal_special_5m_16
-XX:+UseCompressedOops -Xgcpolicy:balanced -Xjit:counts=- - - - - - 1 1 1 1000 250 250 - - - 10000 100000 10000,gcOnResolve,rtResolve,sampleInterval=2,scorchingSampleThreshold=10000,quickProfile -Xcheck:gc:vmthreads:all:quiet
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00003FFF93975C20 Handler2=00003FFF938AC300
MLT stderr R0=00003FFF69D2B2A0 R1=00003FFF6130B750 R2=00003FFF934DF100 R3=00000000005EFB00
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000089F35518 R6=0000000000000000 R7=00003FFF69D2B2A8
MLT stderr R8=0000000000000000 R9=0000000000000001 R10=00003FFF6A067D00 R11=00003FFF931F1318
MLT stderr R12=0000000000474900 R13=00003FFF61316900 R14=0000000000603230 R15=00000000005EFB00
MLT stderr R16=00003FFF62B80038 R17=FFFFFFFFFFFFFFFF R18=0000000000000000 R19=0000000000000000
MLT stderr R20=00003FFF6130F900 R21=0000000089D12D38 R22=0000000089D00198 R23=0000000000000000
MLT stderr R24=0000000089F353E8 R25=0000000000677D00 R26=0000000089DE1DE0 R27=00000000893731D0
MLT stderr R28=000000008910B300 R29=0000000089373188 R30=0000000089F35518 R31=0000000089F35518
MLT stderr NIP=00003FFF931DA6F0 MSR=800000010280F033 ORIG_GPR3=00000000000081C8 CTR=00003FFF931F1318
MLT stderr LINK=00003FFF931F132C XER=0000000020000000 CCR=0000000042004428 SOFTE=0000000000000001
MLT stderr TRAP=0000000000000300 DAR=0000000000000017 dsisr=0000000040000000 RESULT=0000000000000000
MLT stderr FPR0 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR1 4052467840000000 (f: 1073741824.000000, d: 7.310109e+01)
MLT stderr FPR2 3fd70e0000000000 (f: 0.000000, d: 3.602295e-01)
MLT stderr FPR3 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR4 3fe3333340000000 (f: 1073741824.000000, d: 6.000000e-01)
MLT stderr FPR5 3fd70ebd60000000 (f: 1610612736.000000, d: 3.602746e-01)
MLT stderr FPR6 3fe5555560000000 (f: 1610612736.000000, d: 6.666667e-01)
MLT stderr FPR7 3f268ec900000000 (f: 0.000000, d: 1.721020e-04)
MLT stderr FPR8 4033000000000000 (f: 0.000000, d: 1.900000e+01)
MLT stderr FPR9 3fd999bdc0000000 (f: 3221225472.000000, d: 4.000086e-01)
MLT stderr FPR10 3f8aa68840000000 (f: 1073741824.000000, d: 1.301295e-02)
MLT stderr FPR11 41cdcd6500000000 (f: 0.000000, d: 1.000000e+09)
MLT stderr FPR12 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR13 3fdb9b2820000000 (f: 536870912.000000, d: 4.313450e-01)
MLT stderr FPR14 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR15 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR16 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR17 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR18 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR19 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR20 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR21 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR22 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR23 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR24 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR25 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR26 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR27 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR28 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR29 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR30 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr FPR31 0000000000000000 (f: 0.000000, d: 0.000000e+00)
MLT stderr Module=/home/jenkins/workspace/Test_openjdk17_j9_special.system_ppc64le_linux_Release_testList_4/openjdkbinary/j2sdk-image/lib/default/libj9jit29.so
MLT stderr Module_base_address=00003FFF92620000
MLT stderr Target=2_90_20211219_18 (Linux 3.10.0-1160.25.1.el7.ppc64le)
MLT stderr CPU=ppc64le (8 logical CPUs) (0x1de290000 RAM)
MLT stderr ----------- Stack Backtrace -----------
MLT stderr (0x00003FFF938A69E4 [libj9prt29.so+0x369e4])
MLT stderr (0x00003FFF938AD708 [libj9prt29.so+0x3d708])
MLT stderr (0x00003FFF938A6AA8 [libj9prt29.so+0x36aa8])
MLT stderr (0x00003FFF938A6C10 [libj9prt29.so+0x36c10])
MLT stderr (0x00003FFF938A66E4 [libj9prt29.so+0x366e4])
MLT stderr (0x00003FFF938AD708 [libj9prt29.so+0x3d708])
MLT stderr (0x00003FFF938A67D8 [libj9prt29.so+0x367d8])
MLT stderr (0x00003FFF93975AA0 [libj9vm29.so+0x45aa0])
MLT stderr (0x00003FFF938AD708 [libj9prt29.so+0x3d708])
MLT stderr (0x00003FFF93975DD4 [libj9vm29.so+0x45dd4])
MLT stderr (0x00003FFF938AC560 [libj9prt29.so+0x3c560])
MLT stderr __kernel_sigtramp_rt64+0x0 (0x00003FFF94A30478)
MLT stderr (0x00003FFF931FBF90 [libj9jit29.so+0xbdbf90])
MLT stderr (0x00003FFF93949E30 [libj9vm29.so+0x19e30])
MLT stderr (0x00003FFF939E7530 [libj9vm29.so+0xb7530])
MLT stderr (0x00003FFF938AD708 [libj9prt29.so+0x3d708])
MLT stderr (0x00003FFF939E26C4 [libj9vm29.so+0xb26c4])
MLT stderr (0x00003FFF938413E8 [libj9thr29.so+0x113e8])
MLT stderr (0x00003FFF94968CD4 [libpthread.so.0+0x8cd4])
MLT stderr clone+0xe4 (0x00003FFF94867F14 [libc.so.6+0x127f14])
MLT stderr ---------------------------------------
https://openj9-jenkins.osuosl.org/job/Test_openjdk17_j9_special.system_ppc64le_linux_Release_testList_4/18
MathLoadTest_bigdecimal_special_5m_16
-XX:+UseCompressedOops -Xgcpolicy:balanced -Xjit:counts=- - - - - - 1 1 1 1000 250 250 - - - 10000 100000 10000,gcOnResolve,rtResolve,sampleInterval=2,scorchingSampleThreshold=10000,quickProfile -Xcheck:gc:vmthreads:all:quiet
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00003FFFAC845C20 Handler2=00003FFFAC77C300
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00003FFFAC845C20 Handler2=00003FFFAC77C300
MLT stderr J9Generic_Signal_Number=00000018 Signal_Number=0000000b Error_Value=00000000 Signal_Code=00000001
MLT stderr Handler1=00003FFFAC845C20 Handler2=00003FFFAC77C300
MLT stderr R0=00003FFF88BA8D1C R1=00003FFF44A7B740 R2=00003FFFA7FBF100 R3=0000000000625C00
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000081701DD0 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000033 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00003FFFA7CD1318
MLT stderr R12=000000000026DA00 R13=00003FFF44A86900 R14=000000000063B600 R15=0000000000625C00
MLT stderr R16=00003FFF85A00038 R17=FFFFFFFFFFFFFFFF R18=0000000000000000 R19=0000000081605FE8
MLT stderr R20=0000000000000000 R21=0000000081F7E8A8 R22=000000008490ED80 R23=0000000083A8BE00
MLT stderr R24=00000000849B15C0 R25=0000000081D0D770 R26=0000000081D0D770 R27=00000000876C0810
MLT stderr R28=0000000080F02110 R29=00000000876C0238 R30=0000000000748B00 R31=0000000081701DD0
MLT stderr R0=00003FFF88BA8D1C R1=00003FFF44CFB740 R2=00003FFFA7FBF100 R3=0000000000612200
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000081701DD0 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000033 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00003FFFA7CD1318
MLT stderr R12=000000000026DA00 R13=00003FFF44D06900 R14=0000000000635D70 R15=0000000000612200
MLT stderr R16=00003FFF85A00038 R17=FFFFFFFFFFFFFFFF R18=0000000000000000 R19=0000000081605FE8
MLT stderr R20=0000000000000000 R21=0000000081B033B8 R22=000000008202E8F0 R23=0000000083A8BE00
MLT stderr R24=000000008202EC68 R25=0000000081D0D770 R26=0000000081D0D770 R27=000000008765E690
MLT stderr R28=00000000870FF6D8 R29=000000008765E0B8 R30=0000000000748B00 R31=0000000081701DD0
MLT stderr R0=00003FFF88BA8D1C R1=00003FFF44DFB740 R2=00003FFFA7FBF100 R3=0000000000609800
MLT stderr R4=FFFFFFFFFFFFFFFF R5=0000000081701DD0 R6=000000000000001A R7=00000000000000D0
MLT stderr R8=0000000000000036 R9=0000000000000001 R10=FFFFFFFFFFFFFFFF R11=00003FFFA7CD1318
MLT stderr R12=000000000026DA00 R13=00003FFF44E06900 R14=0000000000646720 R15=0000000000609800
MLT stderr R16=00003FFF85A00038 R17=FFFFFFFFFFFFFFFF R18=0000000000000000 R19=0000000081605FE8
MLT stderr R20=0000000000000000 R21=0000000000000000 R22=00000000808268D8 R23=0000000083A8BE00
MLT stderr R24=0000000080851628 R25=0000000081D0D770 R26=0000000081D0D770 R27=00000000876506F8
MLT stderr R28=0000000087D8E730 R29=0000000087650120 R30=0000000000748B00 R31=0000000081701DD0
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
MLT stderr Unhandled exception
MLT stderr Type=Segmentation error vmState=0x00000000
...
https://openj9-jenkins.osuosl.org/job/Test_openjdk17_j9_special.system_ppc64le_linux_Release_testList_1/6 MathLoadTest_bigdecimal_special_5m_9 -Xaggressive -Xgcpolicy:gencon -Xjit -Xnocompressedrefs
https://openj9-artifactory.osuosl.org/artifactory/ci-openj9/Test/Test_openjdk17_j9_special.system_ppc64le_linux_Release_testList_1/6/system_test_output.tar.gz