search

facebookincubator / velox

A C++ vectorized database acceleration library aimed to optimizing query engines and data processing systems.
https://velox-lib.io/
Apache License 2.0
3.2k stars 1.06k forks source link

Flaky test: MultiFragmentTest.exchangeStatsOnFailure #9853

Closed zhli1142015 closed 2 weeks ago

zhli1142015 commented 2 weeks ago

Bug description

E20240517 10:41:09.074805 133061 Exceptions.h:69] Line: /home/runner/work/velox/velox/velox/velox/exec/tests/MultiFragmentTest.cpp:1855, Function:operator(), Expression:  Forced failure after 11 pages, Source: RUNTIME, ErrorCode: INVALID_STATE
E20240517 10:41:10.367101 133060 Exceptions.h:69] Line: /home/runner/work/velox/velox/velox/velox/exec/ExchangeQueue.cpp:103, Function:dequeueLocked, Expression:  Exception: VeloxRuntimeError
Error Source: RUNTIME
Error Code: INVALID_STATE
Reason: Forced failure after 11 pages
Retriable: False
Function: operator()
File: /home/runner/work/velox/velox/velox/velox/exec/tests/MultiFragmentTest.cpp
Line: 1855
Stack trace:
# 0  _ZN8facebook5velox7process10StackTraceC1Ei
# 1  _ZN8facebook5velox14VeloxException5State4makeIZNS1_C4EPKcmS5_St17basic_string_viewIcSt11char_traitsIcEES9_S9_S9_bNS1_4TypeES9_EUlRT_E_EESt10shared_ptrIKS2_ESA_SB_
# 2  _ZN8facebook5velox14VeloxExceptionC1EPKcmS3_St17basic_string_viewIcSt11char_traitsIcEES7_S7_S7_bNS1_4TypeES7_
# 3  _ZN8facebook5velox17VeloxRuntimeErrorC2EPKcmS3_St17basic_string_viewIcSt11char_traitsIcEES7_S7_S7_bS7_
# 4  _ZN8facebook5velox6detail14veloxCheckFailINS0_17VeloxRuntimeErrorERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEEvRKNS1_18VeloxCheckFailArgsET0_
# 5  _ZZN8facebook5velox4exec12_GLOBAL__N_145MultiFragmentTest_exchangeStatsOnFailure_Test8TestBodyEvENKUlPvE_clES4_
# 6  _ZSt13__invoke_implIvRZN8facebook5velox4exec12_GLOBAL__N_145MultiFragmentTest_exchangeStatsOnFailure_Test8TestBodyEvEUlPvE_JS5_EET_St14__invoke_otherOT0_DpOT1_
# 7  _ZSt10__invoke_rIvRZN8facebook5velox4exec12_GLOBAL__N_145MultiFragmentTest_exchangeStatsOnFailure_Test8TestBodyEvEUlPvE_JS5_EENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EES9_E4typeEOSA_DpOSB_
# 8  _ZNSt17_Function_handlerIFvPvEZN8facebook5velox4exec12_GLOBAL__N_145MultiFragmentTest_exchangeStatsOnFailure_Test8TestBodyEvEUlS0_E_E9_M_invokeERKSt9_Any_dataOS0_
# 9  _ZNKSt8functionIFvPvEEclES0_
# 10 _ZZN8facebook5velox6common8testutil9TestValue3setIvEEvRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt8functionIFvPT_EEENKUlPvE_clESI_
# 11 _ZSt13__invoke_implIvRZN8facebook5velox6common8testutil9TestValue3setIvEEvRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt8functionIFvPT_EEEUlPvE_JSJ_EESF_St14__invoke_otherOT0_DpOT1_
# 12 _ZSt10__invoke_rIvRZN8facebook5velox6common8testutil9TestValue3setIvEEvRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt8functionIFvPT_EEEUlPvE_JSJ_EENSt9enable_ifIX16is_invocable_r_vISF_T0_DpT1_EESF_E4typeEOSN_DpOSO_
# 13 _ZNSt17_Function_handlerIFvPvEZN8facebook5velox6common8testutil9TestValue3setIvEEvRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt8functionIFvPT_EEEUlS0_E_E9_M_invokeERKSt9_Any_dataOS0_
# 14 _ZNKSt8functionIFvPvEEclES0_
# 15 _ZN8facebook5velox6common8testutil9TestValue6adjustERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPv
# 16 _ZZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEEENKUlSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteISD_EESaISG_EElSA_IlSaIlEEE_clESI_lSK_
# 17 _ZSt13__invoke_implIvRZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEEEUlSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteISE_EESaISH_EElSB_IlSaIlEEE_JSJ_lSL_EET_St14__invoke_otherOT0_DpOT1_
# 18 _ZSt10__invoke_rIvRZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEEEUlSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteISE_EESaISH_EElSB_IlSaIlEEE_JSJ_lSL_EENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EESP_E4typeEOSQ_DpOSR_
# 19 _ZNSt17_Function_handlerIFvSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteIS3_EESaIS6_EElS0_IlSaIlEEEZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEEEUlS8_lSA_E_E9_M_invokeERKSt9_Any_dataOS8_OlOSA_
# 20 _ZNKSt8functionIFvSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteIS3_EESaIS6_EElS0_IlSaIlEEEEclES8_lSA_
# 21 _ZN8facebook5velox4exec12OutputBuffer7getDataEimlSt8functionIFvSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteIS7_EESaISA_EElS4_IlSaIlEEEES3_IFbvEE
# 22 _ZN8facebook5velox4exec19OutputBufferManager7getDataERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEimlSt8functionIFvSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteISF_EESaISI_EElSC_IlSaIlEEEESB_IFbvEE
# 23 _ZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE
# 24 _ZN8facebook5velox4exec14ExchangeClient7requestEOSt6vectorINS2_11RequestSpecESaIS4_EE
# 25 _ZZN8facebook5velox4exec14ExchangeClient7requestEOSt6vectorINS2_11RequestSpecESaIS4_EEENKUlOT_E_clINS1_14ExchangeSource8ResponseEEEDaS9_
# 26 _ZZN5folly7futures6detail10wrapInvokeIN8facebook5velox4exec14ExchangeSource8ResponseEZNS5_14ExchangeClient7requestEOSt6vectorINS8_11RequestSpecESaISA_EEEUlOT_E_EEDaONS_3TryISE_EEOT0_ENKUlvE_clEv
# 27 _ZN5folly7futures6detail19InvokeResultWrapperIvE10wrapResultIZNS1_10wrapInvokeIN8facebook5velox4exec14ExchangeSource8ResponseEZNS8_14ExchangeClient7requestEOSt6vectorINSB_11RequestSpecESaISD_EEEUlOT_E_EEDaONS_3TryISH_EEOT0_EUlvE_EENSK_INS_4UnitEEESH_
# 28 _ZN5folly7futures6detail10wrapInvokeIN8facebook5velox4exec14ExchangeSource8ResponseEZNS5_14ExchangeClient7requestEOSt6vectorINS8_11RequestSpecESaISA_EEEUlOT_E_EEDaONS_3TryISE_EEOT0_
# 29 _ZZNO5folly6FutureIN8facebook5velox4exec14ExchangeSource8ResponseEE9thenValueIZNS3_14ExchangeClient7requestEOSt6vectorINS8_11RequestSpecESaISA_EEEUlOT_E_EENS0_INS_7futures6detail19valueCallableResultIS5_SE_E10value_typeEEESF_ENUlONS_8Executor9KeepAliveISN_EEONS_3TryIS5_EEE_clESQ_ST_
# 30 _ZN5folly7futures6detail17CoreCallbackStateINS_4UnitEZNOS_6FutureIN8facebook5velox4exec14ExchangeSource8ResponseEE9thenValueIZNS7_14ExchangeClient7requestEOSt6vectorINSC_11RequestSpecESaISE_EEEUlOT_E_EENS4_INS1_19valueCallableResultIS9_SI_E10value_typeEEESJ_EUlONS_8Executor9KeepAliveISP_EEONS_3TryIS9_EEE_E6invokeIJSR_SU_EEEDaDpOT_
# 31 _ZN5folly7futures6detail27detail_msvc_15_7_workaround6invokeINS1_25tryExecutorCallableResultIN8facebook5velox4exec14ExchangeSource8ResponseEZNOS_6FutureIS9_E9thenValueIZNS7_14ExchangeClient7requestEOSt6vectorINSD_11RequestSpecESaISF_EEEUlOT_E_EENSA_INS1_19valueCallableResultIS9_SJ_E10value_typeEEESK_EUlONS_8Executor9KeepAliveISQ_EEONS_3TryIS9_EEE_EENS1_17CoreCallbackStateINS_4UnitESX_EES9_Li0EEEDcSJ_RT0_ST_ONSU_IT1_EE
# 32 _ZZZN5folly7futures6detail10FutureBaseIN8facebook5velox4exec14ExchangeSource8ResponseEE18thenImplementationIZNOS_6FutureIS7_E9thenValueIZNS5_14ExchangeClient7requestEOSt6vectorINSD_11RequestSpecESaISF_EEEUlOT_E_EENSA_INS1_19valueCallableResultIS7_SJ_E10value_typeEEESK_EUlONS_8Executor9KeepAliveISQ_EEONS_3TryIS7_EEE_NS1_25tryExecutorCallableResultIS7_SX_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSA_INS11_10value_typeEEEE4typeESK_S11_NS1_18InlineContinuationEENUlST_SW_E_clEST_SW_ENKUlvE_clEv
# 33 _ZN5folly11makeTryWithIZZNS_7futures6detail10FutureBaseIN8facebook5velox4exec14ExchangeSource8ResponseEE18thenImplementationIZNOS_6FutureIS8_E9thenValueIZNS6_14ExchangeClient7requestEOSt6vectorINSE_11RequestSpecESaISG_EEEUlOT_E_EENSB_INS2_19valueCallableResultIS8_SK_E10value_typeEEESL_EUlONS_8Executor9KeepAliveISR_EEONS_3TryIS8_EEE_NS2_25tryExecutorCallableResultIS8_SY_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSB_INS12_10value_typeEEEE4typeESL_S12_NS2_18InlineContinuationEENUlSU_SX_E_clESU_SX_EUlvE_EENS11_IXsrNS_5isTryINS_13invoke_detail6traitsISK_E6resultIEEEE5valueES1G_E4typeESL_
# 34 _ZZN5folly7futures6detail10FutureBaseIN8facebook5velox4exec14ExchangeSource8ResponseEE18thenImplementationIZNOS_6FutureIS7_E9thenValueIZNS5_14ExchangeClient7requestEOSt6vectorINSD_11RequestSpecESaISF_EEEUlOT_E_EENSA_INS1_19valueCallableResultIS7_SJ_E10value_typeEEESK_EUlONS_8Executor9KeepAliveISQ_EEONS_3TryIS7_EEE_NS1_25tryExecutorCallableResultIS7_SX_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSA_INS11_10value_typeEEEE4typeESK_S11_NS1_18InlineContinuationEENUlST_SW_E_clEST_SW_
# 35 _ZZN5folly7futures6detail4CoreIN8facebook5velox4exec14ExchangeSource8ResponseEE11setCallbackIZNS1_10FutureBaseIS7_E18thenImplementationIZNOS_6FutureIS7_E9thenValueIZNS5_14ExchangeClient7requestEOSt6vectorINSG_11RequestSpecESaISI_EEEUlOT_E_EENSD_INS1_19valueCallableResultIS7_SM_E10value_typeEEESN_EUlONS_8Executor9KeepAliveIST_EEONS_3TryIS7_EEE_NS1_25tryExecutorCallableResultIS7_S10_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSD_INS14_10value_typeEEEE4typeESN_S14_NS1_18InlineContinuationEEUlSW_SZ_E_EEvSN_OSt10shared_ptrINS_14RequestContextEES1A_ENUlRNS1_8CoreBaseESW_PNS_17exception_wrapperEE_clES1H_SW_S1J_
# 36 _ZN5folly6detail8function14FunctionTraitsIFvRNS_7futures6detail8CoreBaseEONS_8Executor9KeepAliveIS7_EEPNS_17exception_wrapperEEE9callSmallIZNS4_4CoreIN8facebook5velox4exec14ExchangeSource8ResponseEE11setCallbackIZNS4_10FutureBaseISL_E18thenImplementationIZNOS_6FutureISL_E9thenValueIZNSJ_14ExchangeClient7requestEOSt6vectorINSU_11RequestSpecESaISW_EEEUlOT_E_EENSR_INS4_19valueCallableResultISL_S10_E10value_typeEEES11_EUlSA_ONS_3TryISL_EEE_NS4_25tryExecutorCallableResultISL_S1A_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSR_INS1E_10value_typeEEEE4typeES11_S1E_NS4_18InlineContinuationEEUlSA_S19_E_EEvS11_OSt10shared_ptrINS_14RequestContextEES1K_EUlS6_SA_SC_E_EEvS6_SA_SC_RNS1_4DataE
# 37 _ZN5folly6detail8function14FunctionTraitsIFvRNS_7futures6detail8CoreBaseEONS_8Executor9KeepAliveIS7_EEPNS_17exception_wrapperEEEclES6_SA_SC_
# 38 _ZZN5folly7futures6detail8CoreBase10doCallbackEONS_8Executor9KeepAliveIS3_EENS1_5StateEENUlS6_E0_clES6_
# 39 _ZZNO5folly8Executor9KeepAliveIS0_E3addIZNS_7futures6detail8CoreBase10doCallbackEOS2_NS5_5StateEEUlS7_E0_EEvOT_ENUlvE_clEv
# 40 _ZN5folly6detail8function14FunctionTraitsIFvvEE9callSmallIZNOS_8Executor9KeepAliveIS6_E3addIZNS_7futures6detail8CoreBase10doCallbackEOS8_NSB_5StateEEUlSD_E0_EEvOT_EUlvE_EEvRNS1_4DataE
# 41 _ZN5folly6detail8function14FunctionTraitsIFvvEEclEv
# 42 _ZN5folly18ThreadPoolExecutor7runTaskERKSt10shared_ptrINS0_6ThreadEEONS0_4TaskE
# 43 _ZN5folly21CPUThreadPoolExecutor9threadRunESt10shared_ptrINS_18ThreadPoolExecutor6ThreadEE
# 44 _ZSt13__invoke_implIvRMN5folly18ThreadPoolExecutorEFvSt10shared_ptrINS1_6ThreadEEERPS1_JRS4_EET_St21__invoke_memfun_derefOT0_OT1_DpOT2_
# 45 _ZSt8__invokeIRMN5folly18ThreadPoolExecutorEFvSt10shared_ptrINS1_6ThreadEEEJRPS1_RS4_EENSt15__invoke_resultIT_JDpT0_EE4typeEOSC_DpOSD_
# 46 _ZNSt5_BindIFMN5folly18ThreadPoolExecutorEFvSt10shared_ptrINS1_6ThreadEEEPS1_S4_EE6__callIvJEJLm0ELm1EEEET_OSt5tupleIJDpT0_EESt12_Index_tupleIJXspT1_EEE
# 47 _ZNSt5_BindIFMN5folly18ThreadPoolExecutorEFvSt10shared_ptrINS1_6ThreadEEEPS1_S4_EEclIJEvEET0_DpOT_
# 48 _ZN5folly6detail8function14FunctionTraitsIFvvEE9callSmallISt5_BindIFMNS_18ThreadPoolExecutorEFvSt10shared_ptrINS7_6ThreadEEEPS7_SA_EEEEvRNS1_4DataE
# 49 _ZN5folly6detail8function14FunctionTraitsIFvvEEclEv
# 50 _ZZN5folly18NamedThreadFactory9newThreadEONS_8FunctionIFvvEEEENUlvE_clEv
# 51 _ZSt13__invoke_implIvZN5folly18NamedThreadFactory9newThreadEONS0_8FunctionIFvvEEEEUlvE_JEET_St14__invoke_otherOT0_DpOT1_
# 52 _ZSt8__invokeIZN5folly18NamedThreadFactory9newThreadEONS0_8FunctionIFvvEEEEUlvE_JEENSt15__invoke_resultIT_JDpT0_EE4typeEOS8_DpOS9_
# 53 _ZNSt6thread8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS2_8FunctionIFvvEEEEUlvE_EEE9_M_invokeIJLm0EEEEvSt12_Index_tupleIJXspT_EEE
# 54 _ZNSt6thread8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS2_8FunctionIFvvEEEEUlvE_EEEclEv
# 55 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS3_8FunctionIFvvEEEEUlvE_EEEEE6_M_runEv
# 56 0x00000000000dc252
# 57 0x0000000000094ac2
# 58 0x000000000012684f
, Source: RUNTIME, ErrorCode: INVALID_STATE
E[2024](https://github.com/facebookincubator/velox/actions/runs/9126575411/job/25095128587?pr=9850#step:10:2025)0517 10:41:10.367776 133062 Exceptions.h:69] Line: /home/runner/work/velox/velox/velox/velox/exec/ExchangeQueue.cpp:103, Function:dequeueLocked, Expression:  Exception: VeloxRuntimeError
Error Source: RUNTIME
Error Code: INVALID_STATE
Reason: Forced failure after 11 pages
Retriable: False
Function: operator()
File: /home/runner/work/velox/velox/velox/velox/exec/tests/MultiFragmentTest.cpp
Line: 1855
Stack trace:

System information

https://github.com/facebookincubator/velox/actions/runs/9126575411/job/25095128587?pr=9850

Relevant logs

E20240517 10:41:09.074805 133061 Exceptions.h:69] Line: /home/runner/work/velox/velox/velox/velox/exec/tests/MultiFragmentTest.cpp:1855, Function:operator(), Expression:  Forced failure after 11 pages, Source: RUNTIME, ErrorCode: INVALID_STATE
E20240517 10:41:10.367101 133060 Exceptions.h:69] Line: /home/runner/work/velox/velox/velox/velox/exec/ExchangeQueue.cpp:103, Function:dequeueLocked, Expression:  Exception: VeloxRuntimeError
Error Source: RUNTIME
Error Code: INVALID_STATE
Reason: Forced failure after 11 pages
Retriable: False
Function: operator()
File: /home/runner/work/velox/velox/velox/velox/exec/tests/MultiFragmentTest.cpp
Line: 1855
Stack trace:
# 0  _ZN8facebook5velox7process10StackTraceC1Ei
# 1  _ZN8facebook5velox14VeloxException5State4makeIZNS1_C4EPKcmS5_St17basic_string_viewIcSt11char_traitsIcEES9_S9_S9_bNS1_4TypeES9_EUlRT_E_EESt10shared_ptrIKS2_ESA_SB_
# 2  _ZN8facebook5velox14VeloxExceptionC1EPKcmS3_St17basic_string_viewIcSt11char_traitsIcEES7_S7_S7_bNS1_4TypeES7_
# 3  _ZN8facebook5velox17VeloxRuntimeErrorC2EPKcmS3_St17basic_string_viewIcSt11char_traitsIcEES7_S7_S7_bS7_
# 4  _ZN8facebook5velox6detail14veloxCheckFailINS0_17VeloxRuntimeErrorERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEEvRKNS1_18VeloxCheckFailArgsET0_
# 5  _ZZN8facebook5velox4exec12_GLOBAL__N_145MultiFragmentTest_exchangeStatsOnFailure_Test8TestBodyEvENKUlPvE_clES4_
# 6  _ZSt13__invoke_implIvRZN8facebook5velox4exec12_GLOBAL__N_145MultiFragmentTest_exchangeStatsOnFailure_Test8TestBodyEvEUlPvE_JS5_EET_St14__invoke_otherOT0_DpOT1_
# 7  _ZSt10__invoke_rIvRZN8facebook5velox4exec12_GLOBAL__N_145MultiFragmentTest_exchangeStatsOnFailure_Test8TestBodyEvEUlPvE_JS5_EENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EES9_E4typeEOSA_DpOSB_
# 8  _ZNSt17_Function_handlerIFvPvEZN8facebook5velox4exec12_GLOBAL__N_145MultiFragmentTest_exchangeStatsOnFailure_Test8TestBodyEvEUlS0_E_E9_M_invokeERKSt9_Any_dataOS0_
# 9  _ZNKSt8functionIFvPvEEclES0_
# 10 _ZZN8facebook5velox6common8testutil9TestValue3setIvEEvRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt8functionIFvPT_EEENKUlPvE_clESI_
# 11 _ZSt13__invoke_implIvRZN8facebook5velox6common8testutil9TestValue3setIvEEvRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt8functionIFvPT_EEEUlPvE_JSJ_EESF_St14__invoke_otherOT0_DpOT1_
# 12 _ZSt10__invoke_rIvRZN8facebook5velox6common8testutil9TestValue3setIvEEvRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt8functionIFvPT_EEEUlPvE_JSJ_EENSt9enable_ifIX16is_invocable_r_vISF_T0_DpT1_EESF_E4typeEOSN_DpOSO_
# 13 _ZNSt17_Function_handlerIFvPvEZN8facebook5velox6common8testutil9TestValue3setIvEEvRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESt8functionIFvPT_EEEUlS0_E_E9_M_invokeERKSt9_Any_dataOS0_
# 14 _ZNKSt8functionIFvPvEEclES0_
# 15 _ZN8facebook5velox6common8testutil9TestValue6adjustERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPv
# 16 _ZZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEEENKUlSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteISD_EESaISG_EElSA_IlSaIlEEE_clESI_lSK_
# 17 _ZSt13__invoke_implIvRZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEEEUlSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteISE_EESaISH_EElSB_IlSaIlEEE_JSJ_lSL_EET_St14__invoke_otherOT0_DpOT1_
# 18 _ZSt10__invoke_rIvRZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEEEUlSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteISE_EESaISH_EElSB_IlSaIlEEE_JSJ_lSL_EENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EESP_E4typeEOSQ_DpOSR_
# 19 _ZNSt17_Function_handlerIFvSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteIS3_EESaIS6_EElS0_IlSaIlEEEZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEEEUlS8_lSA_E_E9_M_invokeERKSt9_Any_dataOS8_OlOSA_
# 20 _ZNKSt8functionIFvSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteIS3_EESaIS6_EElS0_IlSaIlEEEEclES8_lSA_
# 21 _ZN8facebook5velox4exec12OutputBuffer7getDataEimlSt8functionIFvSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteIS7_EESaISA_EElS4_IlSaIlEEEES3_IFbvEE
# 22 _ZN8facebook5velox4exec19OutputBufferManager7getDataERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEimlSt8functionIFvSt6vectorISt10unique_ptrIN5folly5IOBufESt14default_deleteISF_EESaISI_EElSC_IlSaIlEEEESB_IFbvEE
# 23 _ZN8facebook5velox4exec4test12_GLOBAL__N_119LocalExchangeSource7requestEjNSt6chrono8durationIlSt5ratioILl1ELl1000000EEEE
# 24 _ZN8facebook5velox4exec14ExchangeClient7requestEOSt6vectorINS2_11RequestSpecESaIS4_EE
# 25 _ZZN8facebook5velox4exec14ExchangeClient7requestEOSt6vectorINS2_11RequestSpecESaIS4_EEENKUlOT_E_clINS1_14ExchangeSource8ResponseEEEDaS9_
# 26 _ZZN5folly7futures6detail10wrapInvokeIN8facebook5velox4exec14ExchangeSource8ResponseEZNS5_14ExchangeClient7requestEOSt6vectorINS8_11RequestSpecESaISA_EEEUlOT_E_EEDaONS_3TryISE_EEOT0_ENKUlvE_clEv
# 27 _ZN5folly7futures6detail19InvokeResultWrapperIvE10wrapResultIZNS1_10wrapInvokeIN8facebook5velox4exec14ExchangeSource8ResponseEZNS8_14ExchangeClient7requestEOSt6vectorINSB_11RequestSpecESaISD_EEEUlOT_E_EEDaONS_3TryISH_EEOT0_EUlvE_EENSK_INS_4UnitEEESH_
# 28 _ZN5folly7futures6detail10wrapInvokeIN8facebook5velox4exec14ExchangeSource8ResponseEZNS5_14ExchangeClient7requestEOSt6vectorINS8_11RequestSpecESaISA_EEEUlOT_E_EEDaONS_3TryISE_EEOT0_
# 29 _ZZNO5folly6FutureIN8facebook5velox4exec14ExchangeSource8ResponseEE9thenValueIZNS3_14ExchangeClient7requestEOSt6vectorINS8_11RequestSpecESaISA_EEEUlOT_E_EENS0_INS_7futures6detail19valueCallableResultIS5_SE_E10value_typeEEESF_ENUlONS_8Executor9KeepAliveISN_EEONS_3TryIS5_EEE_clESQ_ST_
# 30 _ZN5folly7futures6detail17CoreCallbackStateINS_4UnitEZNOS_6FutureIN8facebook5velox4exec14ExchangeSource8ResponseEE9thenValueIZNS7_14ExchangeClient7requestEOSt6vectorINSC_11RequestSpecESaISE_EEEUlOT_E_EENS4_INS1_19valueCallableResultIS9_SI_E10value_typeEEESJ_EUlONS_8Executor9KeepAliveISP_EEONS_3TryIS9_EEE_E6invokeIJSR_SU_EEEDaDpOT_
# 31 _ZN5folly7futures6detail27detail_msvc_15_7_workaround6invokeINS1_25tryExecutorCallableResultIN8facebook5velox4exec14ExchangeSource8ResponseEZNOS_6FutureIS9_E9thenValueIZNS7_14ExchangeClient7requestEOSt6vectorINSD_11RequestSpecESaISF_EEEUlOT_E_EENSA_INS1_19valueCallableResultIS9_SJ_E10value_typeEEESK_EUlONS_8Executor9KeepAliveISQ_EEONS_3TryIS9_EEE_EENS1_17CoreCallbackStateINS_4UnitESX_EES9_Li0EEEDcSJ_RT0_ST_ONSU_IT1_EE
# 32 _ZZZN5folly7futures6detail10FutureBaseIN8facebook5velox4exec14ExchangeSource8ResponseEE18thenImplementationIZNOS_6FutureIS7_E9thenValueIZNS5_14ExchangeClient7requestEOSt6vectorINSD_11RequestSpecESaISF_EEEUlOT_E_EENSA_INS1_19valueCallableResultIS7_SJ_E10value_typeEEESK_EUlONS_8Executor9KeepAliveISQ_EEONS_3TryIS7_EEE_NS1_25tryExecutorCallableResultIS7_SX_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSA_INS11_10value_typeEEEE4typeESK_S11_NS1_18InlineContinuationEENUlST_SW_E_clEST_SW_ENKUlvE_clEv
# 33 _ZN5folly11makeTryWithIZZNS_7futures6detail10FutureBaseIN8facebook5velox4exec14ExchangeSource8ResponseEE18thenImplementationIZNOS_6FutureIS8_E9thenValueIZNS6_14ExchangeClient7requestEOSt6vectorINSE_11RequestSpecESaISG_EEEUlOT_E_EENSB_INS2_19valueCallableResultIS8_SK_E10value_typeEEESL_EUlONS_8Executor9KeepAliveISR_EEONS_3TryIS8_EEE_NS2_25tryExecutorCallableResultIS8_SY_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSB_INS12_10value_typeEEEE4typeESL_S12_NS2_18InlineContinuationEENUlSU_SX_E_clESU_SX_EUlvE_EENS11_IXsrNS_5isTryINS_13invoke_detail6traitsISK_E6resultIEEEE5valueES1G_E4typeESL_
# 34 _ZZN5folly7futures6detail10FutureBaseIN8facebook5velox4exec14ExchangeSource8ResponseEE18thenImplementationIZNOS_6FutureIS7_E9thenValueIZNS5_14ExchangeClient7requestEOSt6vectorINSD_11RequestSpecESaISF_EEEUlOT_E_EENSA_INS1_19valueCallableResultIS7_SJ_E10value_typeEEESK_EUlONS_8Executor9KeepAliveISQ_EEONS_3TryIS7_EEE_NS1_25tryExecutorCallableResultIS7_SX_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSA_INS11_10value_typeEEEE4typeESK_S11_NS1_18InlineContinuationEENUlST_SW_E_clEST_SW_
# 35 _ZZN5folly7futures6detail4CoreIN8facebook5velox4exec14ExchangeSource8ResponseEE11setCallbackIZNS1_10FutureBaseIS7_E18thenImplementationIZNOS_6FutureIS7_E9thenValueIZNS5_14ExchangeClient7requestEOSt6vectorINSG_11RequestSpecESaISI_EEEUlOT_E_EENSD_INS1_19valueCallableResultIS7_SM_E10value_typeEEESN_EUlONS_8Executor9KeepAliveIST_EEONS_3TryIS7_EEE_NS1_25tryExecutorCallableResultIS7_S10_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSD_INS14_10value_typeEEEE4typeESN_S14_NS1_18InlineContinuationEEUlSW_SZ_E_EEvSN_OSt10shared_ptrINS_14RequestContextEES1A_ENUlRNS1_8CoreBaseESW_PNS_17exception_wrapperEE_clES1H_SW_S1J_
# 36 _ZN5folly6detail8function14FunctionTraitsIFvRNS_7futures6detail8CoreBaseEONS_8Executor9KeepAliveIS7_EEPNS_17exception_wrapperEEE9callSmallIZNS4_4CoreIN8facebook5velox4exec14ExchangeSource8ResponseEE11setCallbackIZNS4_10FutureBaseISL_E18thenImplementationIZNOS_6FutureISL_E9thenValueIZNSJ_14ExchangeClient7requestEOSt6vectorINSU_11RequestSpecESaISW_EEEUlOT_E_EENSR_INS4_19valueCallableResultISL_S10_E10value_typeEEES11_EUlSA_ONS_3TryISL_EEE_NS4_25tryExecutorCallableResultISL_S1A_EEEENSt9enable_ifIXntsrNT0_13ReturnsFutureE5valueENSR_INS1E_10value_typeEEEE4typeES11_S1E_NS4_18InlineContinuationEEUlSA_S19_E_EEvS11_OSt10shared_ptrINS_14RequestContextEES1K_EUlS6_SA_SC_E_EEvS6_SA_SC_RNS1_4DataE
# 37 _ZN5folly6detail8function14FunctionTraitsIFvRNS_7futures6detail8CoreBaseEONS_8Executor9KeepAliveIS7_EEPNS_17exception_wrapperEEEclES6_SA_SC_
# 38 _ZZN5folly7futures6detail8CoreBase10doCallbackEONS_8Executor9KeepAliveIS3_EENS1_5StateEENUlS6_E0_clES6_
# 39 _ZZNO5folly8Executor9KeepAliveIS0_E3addIZNS_7futures6detail8CoreBase10doCallbackEOS2_NS5_5StateEEUlS7_E0_EEvOT_ENUlvE_clEv
# 40 _ZN5folly6detail8function14FunctionTraitsIFvvEE9callSmallIZNOS_8Executor9KeepAliveIS6_E3addIZNS_7futures6detail8CoreBase10doCallbackEOS8_NSB_5StateEEUlSD_E0_EEvOT_EUlvE_EEvRNS1_4DataE
# 41 _ZN5folly6detail8function14FunctionTraitsIFvvEEclEv
# 42 _ZN5folly18ThreadPoolExecutor7runTaskERKSt10shared_ptrINS0_6ThreadEEONS0_4TaskE
# 43 _ZN5folly21CPUThreadPoolExecutor9threadRunESt10shared_ptrINS_18ThreadPoolExecutor6ThreadEE
# 44 _ZSt13__invoke_implIvRMN5folly18ThreadPoolExecutorEFvSt10shared_ptrINS1_6ThreadEEERPS1_JRS4_EET_St21__invoke_memfun_derefOT0_OT1_DpOT2_
# 45 _ZSt8__invokeIRMN5folly18ThreadPoolExecutorEFvSt10shared_ptrINS1_6ThreadEEEJRPS1_RS4_EENSt15__invoke_resultIT_JDpT0_EE4typeEOSC_DpOSD_
# 46 _ZNSt5_BindIFMN5folly18ThreadPoolExecutorEFvSt10shared_ptrINS1_6ThreadEEEPS1_S4_EE6__callIvJEJLm0ELm1EEEET_OSt5tupleIJDpT0_EESt12_Index_tupleIJXspT1_EEE
# 47 _ZNSt5_BindIFMN5folly18ThreadPoolExecutorEFvSt10shared_ptrINS1_6ThreadEEEPS1_S4_EEclIJEvEET0_DpOT_
# 48 _ZN5folly6detail8function14FunctionTraitsIFvvEE9callSmallISt5_BindIFMNS_18ThreadPoolExecutorEFvSt10shared_ptrINS7_6ThreadEEEPS7_SA_EEEEvRNS1_4DataE
# 49 _ZN5folly6detail8function14FunctionTraitsIFvvEEclEv
# 50 _ZZN5folly18NamedThreadFactory9newThreadEONS_8FunctionIFvvEEEENUlvE_clEv
# 51 _ZSt13__invoke_implIvZN5folly18NamedThreadFactory9newThreadEONS0_8FunctionIFvvEEEEUlvE_JEET_St14__invoke_otherOT0_DpOT1_
# 52 _ZSt8__invokeIZN5folly18NamedThreadFactory9newThreadEONS0_8FunctionIFvvEEEEUlvE_JEENSt15__invoke_resultIT_JDpT0_EE4typeEOS8_DpOS9_
# 53 _ZNSt6thread8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS2_8FunctionIFvvEEEEUlvE_EEE9_M_invokeIJLm0EEEEvSt12_Index_tupleIJXspT_EEE
# 54 _ZNSt6thread8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS2_8FunctionIFvvEEEEUlvE_EEEclEv
# 55 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5folly18NamedThreadFactory9newThreadEONS3_8FunctionIFvvEEEEUlvE_EEEEE6_M_runEv
# 56 0x00000000000dc252
# 57 0x0000000000094ac2
# 58 0x000000000012684f
, Source: RUNTIME, ErrorCode: INVALID_STATE
E20240517 10:41:10.367776 133062 Exceptions.h:69] Line: /home/runner/work/velox/velox/velox/velox/exec/ExchangeQueue.cpp:103, Function:dequeueLocked, Expression:  Exception: VeloxRuntimeError
Error Source: RUNTIME
Error Code: INVALID_STATE
Reason: Forced failure after 11 pages
Retriable: False
Function: operator()
File: /home/runner/work/velox/velox/velox/velox/exec/tests/MultiFragmentTest.cpp
Line: 1855
Stack trace: