search

cms-sw / cmssw

CMS Offline Software
http://cms-sw.github.io/
Apache License 2.0
1.09k stars 4.33k forks source link

[ROOT6] Crash in cling #44438

Open makortel opened 8 months ago

makortel commented 8 months ago

Workflow 24834.0 step3 segfaulted in CMSSW_14_1_ROOT6_X_2024-03-15-2300 on el8_amd64_gcc12

Thread 3 (Thread 0x14e67117c700 (LWP 1569915) "cmsRun"):
#3  0x000014e6c1310f40 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x000014e6af4bd23f in llvm::orc::ExecutionSession::lookup(std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolStringPtr, llvm::orc::SymbolState) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#6  0x000014e6af4d3126 in llvm::orc::LLJIT::lookupLinkerMangled(llvm::orc::JITDylib&, llvm::orc::SymbolStringPtr) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#7  0x000014e6add66027 in cling::IncrementalJIT::getSymbolAddress(llvm::StringRef, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#8  0x000014e6add605c5 in cling::IncrementalExecutor::getPointerToGlobalFromJIT(llvm::StringRef) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#9  0x000014e6adc622f7 in TClingCallFunc::compile_wrapper(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#10 0x000014e6adc6e891 in TClingCallFunc::make_wrapper() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#11 0x000014e6adc6edae in TClingCallFunc::IFacePtr() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#12 0x000014e6c896a01f in edm::FunctionWithDict::FunctionWithDict(TMethod*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libFWCoreReflection.so
#13 0x000014e6c896a74f in edm::TypeWithDict::functionMemberByName(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libFWCoreReflection.so
#14 0x000014e6922707dd in reco::findMethod(edm::TypeWithDict const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::variant<signed char, unsigned char, short, unsigned short, int, unsigned int, long, unsigned long, double, float, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::variant<signed char, unsigned char, short, unsigned short, int, unsigned int, long, unsigned long, double, float, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, std::vector<std::variant<signed char, unsigned char, short, unsigned short, int, unsigned int, long, unsigned long, double, float, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::variant<signed char, unsigned char, short, unsigned short, int, unsigned int, long, unsigned long, double, float, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >&, char const*, int&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#15 0x000014e69224e48e in reco::parser::MethodSetter::push(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::variant<signed char, unsigned char, short, unsigned short, int, unsigned int, long, unsigned long, double, float, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::variant<signed char, unsigned char, short, unsigned short, int, unsigned int, long, unsigned long, double, float, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&, char const*, bool) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#16 0x000014e69224ef21 in reco::parser::MethodSetter::operator()(char const*, char const*) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#17 0x000014e69226f60f in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::alternative<boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::contiguous<boost::spirit::classic::sequence<boost::spirit::classic::alpha_parser, boost::spirit::classic::kleene_star<boost::spirit::classic::chset<char> > > >, boost::spirit::classic::chlit<char> >, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> >, boost::spirit::classic::kleene_star<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::chlit<char> > >, reco::parser::MethodSetter>, boost::spirit::classic::sequence<boost::spirit::classic::action<boost::spirit::classic::contiguous<boost::spirit::classic::sequence<boost::spirit::classic::alpha_parser, boost::spirit::classic::kleene_star<boost::spirit::classic::chset<char> > > >, reco::parser::MethodSetter>, boost::spirit::classic::optional<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::chlit<char> > > > >, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#18 0x000014e69226f09c in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::kleene_star<boost::spirit::classic::alternative<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > > > > >, reco::parser::ExpressionVarSetter>, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#19 0x000014e692267c1e in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::action<boost::spirit::classic::chlit<char>, reco::parser::FunctionSetterCommit> >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::chlit<char> > >, reco::parser::ExpressionFunctionSetter> >, boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::action<boost::spirit::classic::chlit<char>, reco::parser::FunctionSetterCommit> >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::chlit<char> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::chlit<char> > >, reco::parser::ExpressionFunctionSetter> >, boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::action<boost::spirit::classic::chlit<char>, reco::parser::FunctionSetterCommit> >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::chlit<char> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::chlit<char> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::chlit<char> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::chlit<char> > >, reco::parser::ExpressionFunctionSetter> >, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> >, boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> >, boost::spirit::classic::chlit<char> > >, boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> >, reco::parser::ExpressionUnaryOperatorSetter<std::negate<double> > > >, boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#20 0x000014e69226661a in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::kleene_star<boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::ExpressionBinaryOperatorSetter<reco::parser::power_of<double> > > > >, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#21 0x000014e692266b6a in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::kleene_star<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::ExpressionBinaryOperatorSetter<std::multiplies<double> > >, boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::ExpressionBinaryOperatorSetter<std::divides<double> > > >, boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::ExpressionBinaryOperatorSetter<reco::parser::int_div_remainder<double> > > > > >, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#22 0x000014e69226d73a in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::kleene_star<boost::spirit::classic::alternative<boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::ExpressionBinaryOperatorSetter<std::plus<double> > >, boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::ExpressionBinaryOperatorSetter<std::minus<double> > > > > >, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#23 0x000014e69226812b in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::alternative<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> >, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#24 0x000014e69226567b in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::TrinarySelectorSetter>, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#25 0x000014e69226492d in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::alternative<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> >, boost::spirit::classic::sequence<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> >, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::chlit<char> > > >, boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::chlit<char>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::UnaryCutSetter<std::logical_not<bool> > > >, boost::spirit::classic::action<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, reco::parser::ExpressionSelectorSetter> >, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#26 0x000014e692264cb1 in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::kleene_star<boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::BinaryCutSetter<std::logical_and<bool> > > > >, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#27 0x000014e69226513a in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::kleene_star<boost::spirit::classic::action<boost::spirit::classic::sequence<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, boost::spirit::classic::assertive_parser<reco::parser::SyntaxErrors, boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t> > >, reco::parser::BinaryCutSetter<std::logical_or<bool> > > > >, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#28 0x000014e69226482a in boost::spirit::classic::impl::concrete_parser<boost::spirit::classic::action<boost::spirit::classic::rule<boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t, boost::spirit::classic::nil_t>, reco::parser::CutSetter>, boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> >, boost::spirit::classic::nil_t>::do_parse_virtual(boost::spirit::classic::scanner<char const*, boost::spirit::classic::scanner_policies<boost::spirit::classic::skipper_iteration_policy<boost::spirit::classic::iteration_policy>, boost::spirit::classic::match_policy, boost::spirit::classic::action_policy> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#29 0x000014e69225fead in reco::parser::cutParser(edm::TypeWithDict const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<reco::parser::SelectorBase>&, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libCommonToolsUtils.so
#30 0x000014e64bf3b66d in StringCutObjectSelector<reco::Vertex, false>::StringCutObjectSelector(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool) [clone .constprop.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginCommonToolsRecoAlgos_plugins.so
#31 0x000014e64bf3ebc6 in ObjectSelectorBase<SingleElementCollectionSelector<std::vector<reco::Vertex, std::allocator<reco::Vertex> >, StringCutObjectSelector<reco::Vertex, false>, std::vector<reco::Vertex, std::allocator<reco::Vertex> >, std::vector<reco::Vertex const*, std::allocator<reco::Vertex const*> >, helper::SelectionPointerAdder<std::vector<reco::Vertex const*, std::allocator<reco::Vertex const*> > > >, std::vector<reco::Vertex, std::allocator<reco::Vertex> >, NonNullNumberSelector, helper::NullPostProcessor<std::vector<reco::Vertex, std::allocator<reco::Vertex> > >, helper::CollectionStoreManager<std::vector<reco::Vertex, std::allocator<reco::Vertex> >, helper::IteratorToObjectConverter<std::vector<reco::Vertex, std::allocator<reco::Vertex> > > >, helper::ObjectSelectorBase<std::vector<reco::Vertex, std::allocator<reco::Vertex> >, edm::stream::EDFilter<> >, reco::modules::SingleElementCollectionSelectorEventSetupInit<SingleElementCollectionSelector<std::vector<reco::Vertex, std::allocator<reco::Vertex> >, StringCutObjectSelector<reco::Vertex, false>, std::vector<reco::Vertex, std::allocator<reco::Vertex> >, std::vector<reco::Vertex const*, std::allocator<reco::Vertex const*> >, helper::SelectionPointerAdder<std::vector<reco::Vertex const*, std::allocator<reco::Vertex const*> > > > > >::ObjectSelectorBase(edm::ParameterSet const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginCommonToolsRecoAlgos_plugins.so
#32 0x000014e64bf3ecf0 in edm::stream::ProducingModuleAdaptor<SingleObjectSelectorBase<std::vector<reco::Vertex, std::allocator<reco::Vertex> >, StringCutObjectSelector<reco::Vertex, false>, edm::stream::EDFilter<>, std::vector<reco::Vertex, std::allocator<reco::Vertex> >, std::vector<reco::Vertex const*, std::allocator<reco::Vertex const*> >, helper::NullPostProcessor<std::vector<reco::Vertex, std::allocator<reco::Vertex> > >, helper::CollectionStoreManager<std::vector<reco::Vertex, std::allocator<reco::Vertex> >, helper::IteratorToObjectConverter<std::vector<reco::Vertex, std::allocator<reco::Vertex> > > >, helper::ObjectSelectorBase<std::vector<reco::Vertex, std::allocator<reco::Vertex> >, edm::stream::EDFilter<> >, helper::SelectionPointerAdder<std::vector<reco::Vertex const*, std::allocator<reco::Vertex const*> > > >, edm::stream::EDFilterBase, edm::stream::EDFilterAdaptorBase>::setupStreamModules() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginCommonToolsRecoAlgos_plugins.so
#33 0x000014e6c8c890aa in edm::stream::ProducingModuleAdaptorBase<edm::stream::EDFilterBase>::doPreallocate(edm::PreallocationConfiguration const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#34 0x000014e6c8c6b7a6 in edm::Maker::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#35 0x000014e6c8be2778 in edm::Factory::makeModule(edm::MakeModuleParams const&, edm::ModuleTypeResolverMaker const*, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so

Thread 1 (Thread 0x14e6c567f640 (LWP 1569309) "cmsRun"):
#2  0x000014e6c130d4e0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  sz_index2size_lookup_impl (index=6) at include/jemalloc/internal/sz.h:238
#5  sz_index2size_lookup (index=6) at include/jemalloc/internal/sz.h:243
#6  sz_index2size (index=6) at include/jemalloc/internal/sz.h:251
#7  do_rallocx (ptr=0x14e64d05da60, size=184, flags=0, is_realloc=<optimized out>) at src/jemalloc.c:3522
#8  0x000014e6b1fda0c4 in llvm::SmallVectorBase<unsigned int>::grow_pod(void*, unsigned long, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#9  0x000014e6b1e5b901 in (anonymous namespace)::GetCFGOnlyPasses::passEnumerate(llvm::PassInfo const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#10 0x000014e6b1e68fce in llvm::PassRegistry::enumerateWith(llvm::PassRegistrationListener*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#11 0x000014e6b1e5ce78 in llvm::AnalysisUsage::setPreservesCFG() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#12 0x000014e6af78b62c in (anonymous namespace)::X86LowerAMXTypeLegacyPass::getAnalysisUsage(llvm::AnalysisUsage&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#13 0x000014e6b1e1e83f in llvm::PMTopLevelManager::findAnalysisUsage(llvm::Pass*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#14 0x000014e6b1e231db in llvm::PMTopLevelManager::schedulePass(llvm::Pass*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#15 0x000014e6b065af35 in llvm::TargetPassConfig::addPass(llvm::Pass*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#16 0x000014e6af74aa84 in (anonymous namespace)::X86PassConfig::addIRPasses() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#17 0x000014e6b065dead in llvm::TargetPassConfig::addISelPasses() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#18 0x000014e6b043ba24 in addPassesToGenerateCode(llvm::LLVMTargetMachine&, llvm::legacy::PassManagerBase&, bool, llvm::MachineModuleInfoWrapperPass&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#19 0x000014e6b043ca49 in llvm::LLVMTargetMachine::addPassesToEmitMC(llvm::legacy::PassManagerBase&, llvm::MCContext*&, llvm::raw_pwrite_stream&, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#20 0x000014e6af4a28a3 in llvm::orc::SimpleCompiler::operator()(llvm::Module&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#21 0x000014e6af4c6762 in llvm::orc::IRCompileLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, llvm::orc::ThreadSafeModule) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#22 0x000014e6af5266ba in llvm::orc::IRTransformLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, llvm::orc::ThreadSafeModule) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#23 0x000014e6af5266ba in llvm::orc::IRTransformLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, llvm::orc::ThreadSafeModule) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#24 0x000014e6af4ca2e7 in llvm::orc::BasicIRLayerMaterializationUnit::materialize(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#25 0x000014e6af4bb24b in llvm::orc::MaterializationTask::run() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#26 0x000014e6af4a2e6d in void llvm::detail::UniqueFunctionBase<void, std::unique_ptr<llvm::orc::Task, std::default_delete<llvm::orc::Task> > >::CallImpl<void (*)(std::unique_ptr<llvm::orc::Task, std::default_delete<llvm::orc::Task> >)>(void*, std::unique_ptr<llvm::orc::Task, std::default_delete<llvm::orc::Task> >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#27 0x000014e6af4bbfc2 in llvm::orc::ExecutionSession::dispatchOutstandingMUs() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#28 0x000014e6af4bc06a in llvm::orc::ExecutionSession::lookup(llvm::orc::LookupKind, std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolLookupSet, llvm::orc::SymbolState, llvm::unique_function<void (llvm::Expected<llvm::DenseMap<llvm::orc::SymbolStringPtr, llvm::JITEvaluatedSymbol, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void>, llvm::detail::DenseMapPair<llvm::orc::SymbolStringPtr, llvm::JITEvaluatedSymbol> > >)>, std::function<void (llvm::DenseMap<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> >, llvm::DenseMapInfo<llvm::orc::JITDylib*, void>, llvm::detail::DenseMapPair<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> > > > const&)>) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#29 0x000014e6af4bcf92 in llvm::orc::ExecutionSession::lookup(std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolLookupSet, llvm::orc::LookupKind, llvm::orc::SymbolState, std::function<void (llvm::DenseMap<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> >, llvm::DenseMapInfo<llvm::orc::JITDylib*, void>, llvm::detail::DenseMapPair<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> > > > const&)>) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#30 0x000014e6af4bd1e8 in llvm::orc::ExecutionSession::lookup(std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolStringPtr, llvm::orc::SymbolState) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#31 0x000014e6af4d3126 in llvm::orc::LLJIT::lookupLinkerMangled(llvm::orc::JITDylib&, llvm::orc::SymbolStringPtr) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#32 0x000014e6add66027 in cling::IncrementalJIT::getSymbolAddress(llvm::StringRef, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#33 0x000014e6add5f8b2 in cling::IncrementalExecutor::getAddressOfGlobal(llvm::StringRef, bool*) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#34 0x000014e6c7d30185 in ROOT::Internal::GetSymInLibImt(char const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCore.so
#35 0x000014e6c7d31084 in ROOT::Internal::DisableParBranchProcessing() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCore.so
#36 0x000014e6c8894d41 in TTree::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libTree.so
#37 0x000014e66fd4d581 in edm::roottree::getEntry(TTree*, long long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#38 0x000014e66fd35486 in edm::RootFile::RootFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::ProcessConfiguration const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<edm::InputFile>, std::shared_ptr<edm::EventSkipperByID>, bool, int, int, unsigned int, unsigned int, int, edm::InputSource::ProcessingMode, edm::RunHelperBase*, bool, bool, edm::ProductSelectorRules const&, edm::InputType, std::shared_ptr<edm::BranchIDListHelper>, edm::ProcessBlockHelper*, std::shared_ptr<edm::ThinnedAssociationsHelper>, std::vector<edm::BranchID, std::allocator<edm::BranchID> > const*, std::shared_ptr<edm::DuplicateChecker>, bool, edm::ProcessHistoryRegistry&, std::vector<std::shared_ptr<edm::IndexIntoFile>, std::allocator<std::shared_ptr<edm::IndexIntoFile> > > const&, unsigned long, std::vector<edm::Hash<2>, std::allocator<edm::Hash<2> > >&, bool, bool, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#39 0x000014e66fd50f86 in edm::RootPrimaryFileSequence::makeRootFile(std::shared_ptr<edm::InputFile>) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#40 0x000014e66fd4f052 in edm::RootInputFileSequence::initTheFile(bool, bool, edm::InputSource*, char const*, edm::InputType) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#41 0x000014e66fd4fc08 in edm::RootPrimaryFileSequence::RootPrimaryFileSequence(edm::ParameterSet const&, edm::PoolSource&, edm::InputFileCatalog const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#42 0x000014e66fd20102 in edm::PoolSource::PoolSource(edm::ParameterSet const&, edm::InputSourceDescription const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#43 0x000014e66fd210ce in edmplugin::PluginFactory<edm::InputSource* (edm::ParameterSet const&, edm::InputSourceDescription const&)>::PMaker<edm::PoolSource>::create(edm::ParameterSet const&, edm::InputSourceDescription const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#44 0x000014e6c8bec9e5 in edm::InputSourceFactory::makeInputSource(edm::ParameterSet const&, edm::InputSourceDescription const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#45 0x000014e6c8b85d2f in edm::makeInput(unsigned int, edm::ParameterSet&, edm::CommonParams const&, std::shared_ptr<edm::ProductRegistry>, std::shared_ptr<edm::BranchIDListHelper>, std::shared_ptr<edm::ProcessBlockHelper> const&, std::shared_ptr<edm::ThinnedAssociationsHelper>, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_amd64_gcc12/CMSSW_14_1_ROOT6_X_2024-03-15-2300/pyRelValMatrixLogs/run/24834.0_TTbar_14TeV+2026D98/step3_TTbar_14TeV+2026D98.log#/

makortel commented 8 months ago

assign core

makortel commented 8 months ago

type root

cmsbuild commented 8 months ago

New categories assigned: core

@Dr15Jones,@makortel,@smuzaffar you have been requested to review this Pull request/Issue and eventually sign? Thanks

cmsbuild commented 8 months ago

cms-bot internal usage

cmsbuild commented 8 months ago

A new Issue was created by @makortel.

@Dr15Jones, @smuzaffar, @rappoccio, @sextonkennedy, @antoniovilela, @makortel can you please review it and eventually sign/assign? Thanks.

cms-bot commands are listed here

makortel commented 8 months ago

FYI @pcanal @vgvassilev

I'm not sure if this crash could be related to https://github.com/cms-sw/cmssw/issues/43802

dan131riley commented 8 months ago

I'm not sure if this crash could be related to #43802

I would guess that it is related to but perhaps not identical. In particular, #43802 does not seem to be a code generation issue, as it isn't easily reproducible on a different ARM platform (Apple M1). So I suspect a concurrency issue, one that is very rare on x86-64 and ARM implementations with somewhat conservative memory ordering, but more common on the Cavium Thunder implementations in the OpenLab.

pcanal commented 8 months ago

Yes, it likely a thread issue. It appears that:

#31 0x000014e6af4d3126 in llvm::orc::LLJIT::lookupLinkerMangled(llvm::orc::JITDylib&, llvm::orc::SymbolStringPtr) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
#32 0x000014e6add66027 in cling::IncrementalJIT::getSymbolAddress(llvm::StringRef, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCling.so
[TSystem::DynFindSymbol  and TCling::FindSym removed by LTO]
#34 0x000014e6c7d30185 in ROOT::Internal::GetSymInLibImt(char const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02828/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-03-15-2300/external/el8_amd64_gcc12/lib/libCore.so

both is a modifying (i.e. non-read-only) query into Cling and is not taking the ROOT lock (the 1st thread correctly takes the lock). The solution is likely to add the lock (R__LOCKGUARD(gInterpreterMutex);) to TCling::FindSym

pcanal commented 7 months ago

The PR https://github.com/root-project/root/pull/15113 contains the commit root-project/root@c1a0840 (#15113) that should solve this problem.

dan131riley commented 6 months ago

el8_aarch64_gcc12, CMSSW_14_1_ROOT6_X_2024-05-15-2300, WF 11024.2 step3. Does this include the latest patches from @pcanal ? Clearer stack trace, no missing intermediates, and very likely related to #43802.

Thread 5 (Thread 0x40005bd79220 (LWP 1218936) "cmsRun"):
#4  0x0000400010a84af4 in __hypotf_finite () from /lib64/libm.so.6
#5  0x000040005626d388 in mkfit::MkFinder::selectHitIndicesV2(mkfit::LayerOfHits const&, int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCore.so
#6  0x0000400056257050 in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCore.so
#7  0x0000400056257b70 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCore.so
#8  0x0000400056259958 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCore.so
#9  0x00004000105c8774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000f6944f00, this=0x4000114eb480) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#10 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x4000114eb480) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#11 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#12 0x000040005625c55c in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCore.so
#13 0x000040005625cafc in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCore.so
#14 0x00004000105c8774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000f6947500, this=0x4000114eb480) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#15 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x4000114eb480) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#16 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#17 0x0000400056258c20 in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCore.so
#18 0x00004000105c8774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x40005bd77d00, this=0x4000114eb480) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#19 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x4000114eb480) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#20 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#21 0x0000400056253874 in void tbb::detail::d2::parallel_for_each<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}>(__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, __gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1} const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCore.so
#22 0x00004000562538f4 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCore.so
#23 0x00004000561dba60 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerMkFitCMS.so
#24 0x0000400056139e68 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerMkFitPlugins.so
#25 0x00004000105c1724 in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:757
#26 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#27 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:758
#28 0x000040005613f3dc in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerMkFitPlugins.so
#29 0x000040000eb53708 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so

Thread 4 (Thread 0x40005b369220 (LWP 1218935) "cmsRun"):
#4  0x000040006fd7eeec in TkConvValidator::analyze(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginValidationRecoEgammaPlugins.so
#5  0x000040006fd568fc in non-virtual thunk to DQMOneEDAnalyzer<>::accumulate(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginValidationRecoEgammaPlugins.so
#6  0x000040000eb5e4b8 in edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so

Thread 3 (Thread 0x40005a959220 (LWP 1218934) "cmsRun"):
#4  0x000040003d605cf0 in SiStripRecHit2D& std::vector<SiStripRecHit2D, std::allocator<SiStripRecHit2D> >::emplace_back<Point3DBase<float, LocalTag> const&, LocalError const&, GeomDet const&, edm::Ref<edmNew::DetSetVector<SiStripCluster>, SiStripCluster, edmNew::DetSetVector<SiStripCluster>::FindForDetSetVector> const&>(Point3DBase<float, LocalTag> const&, LocalError const&, GeomDet const&, edm::Ref<edmNew::DetSetVector<SiStripCluster>, SiStripCluster, edmNew::DetSetVector<SiStripCluster>::FindForDetSetVector> const&) [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#5  0x000040003d5fa704 in TkStripMeasurementDet::simpleRecHits(TrajectoryStateOnSurface const&, MeasurementTrackerEvent const&, std::vector<SiStripRecHit2D, std::allocator<SiStripRecHit2D> >&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#6  0x000040003d5f5108 in TkGluedMeasurementDet::measurements(TrajectoryStateOnSurface const&, MeasurementEstimator const&, MeasurementTrackerEvent const&, tracking::TempMeasurements&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerMeasurementDetPlugins.so
#7  0x000040003d6f7b68 in LayerMeasurements::groupedMeasurements(DetLayer const&, TrajectoryStateOnSurface const&, Propagator const&, MeasurementEstimator const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libTrackingToolsMeasurementDet.so
#8  0x00004000bfdccfdc in GroupedCkfTrajectoryBuilder::advanceOneLayer(TrajectorySeed const&, TempTrajectory&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const [clone .constprop.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#9  0x00004000bfdc30d0 in GroupedCkfTrajectoryBuilder::groupedLimitedCandidates(TrajectorySeed const&, TempTrajectory const&, TrajectoryFilter const*, Propagator const*, bool, std::vector<TempTrajectory, std::allocator<TempTrajectory> >&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#10 0x00004000bfdc49f4 in GroupedCkfTrajectoryBuilder::buildTrajectories(TrajectorySeed const&, std::vector<Trajectory, std::allocator<Trajectory> >&, unsigned int&, TrajectoryFilter const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerCkfPatternPlugins.so
#11 0x00004000beedb4d4 in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&)::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerCkfPattern.so
#12 0x00004000beedc56c in cms::CkfTrackCandidateMakerBase::produceBase(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libRecoTrackerCkfPattern.so
#13 0x000040000eb67614 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so

Thread 1 (Thread 0x40001076d170 (LWP 1209912) "cmsRun"):
#3  0x0000400015d9cf2c in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x000040000fd23790 in TPluginManager::RemoveHandler(char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCore.so
#6  0x000040000fd238c8 in TPluginManager::AddHandler(char const*, char const*, char const*, char const*, char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCore.so
#7  0x0000400023ca9fa4 in cling::IncrementalExecutor::executeWrapper(llvm::StringRef, cling::Value*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#8  0x0000400023c37540 in cling::Interpreter::RunFunction(clang::FunctionDecl const*, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#9  0x0000400023c37bec in cling::Interpreter::EvaluateInternal(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, cling::CompilationOptions, cling::Value*, cling::Transaction**, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#10 0x0000400023d14fb0 in cling::MetaSema::actOnxCommand(llvm::StringRef, llvm::StringRef, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#11 0x0000400023d21b04 in cling::MetaParser::isXCommand(cling::MetaSema::ActionResult&, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#12 0x0000400023d239b4 in cling::MetaParser::isCommand(cling::MetaSema::ActionResult&, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#13 0x0000400023d0ea3c in cling::MetaProcessor::process(llvm::StringRef, cling::Interpreter::CompilationResult&, cling::Value*, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#14 0x0000400023b4e9ac in HandleInterpreterException(cling::MetaProcessor*, char const*, cling::Interpreter::CompilationResult&, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#15 0x0000400023b61884 in TCling::ProcessLine(char const*, TInterpreter::EErrorCode*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#16 0x0000400023b61d44 in TCling::ProcessLineSynch(char const*, TInterpreter::EErrorCode*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#17 0x000040000fceaa28 in TApplication::ExecuteFile(char const*, int*, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCore.so
#18 0x0000400023b60654 in TCling::ExecuteMacro(char const*, TInterpreter::EErrorCode*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCling.so
#19 0x000040000fd40cfc in TROOT::Macro(char const*, int*, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCore.so
#20 0x000040000fd24880 in TPluginManager::LoadHandlerMacros(char const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCore.so
#21 0x000040000fd24edc in TPluginManager::LoadHandlersFromPluginDirs(char const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCore.so
#22 0x000040000fd25f54 in TPluginManager::FindHandler(char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libCore.so
#23 0x000040000f557bdc in ROOT::Math::Factory::CreateMinimizer(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libMathCore.so
#24 0x000040000f5595a0 in ROOT::Fit::FitConfig::CreateMinimizer() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libMathCore.so
#25 0x000040000f570f70 in ROOT::Fit::Fitter::DoInitMinimizer() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libMathCore.so
#26 0x000040000f577dac in bool ROOT::Fit::Fitter::DoMinimization<ROOT::Fit::Chi2FCN<ROOT::Math::IBaseFunctionMultiDimTempl<double>, ROOT::Math::IParametricFunctionMultiDimTempl<double> > >(std::unique_ptr<ROOT::Fit::Chi2FCN<ROOT::Math::IBaseFunctionMultiDimTempl<double>, ROOT::Math::IParametricFunctionMultiDimTempl<double> >, std::default_delete<ROOT::Fit::Chi2FCN<ROOT::Math::IBaseFunctionMultiDimTempl<double>, ROOT::Math::IParametricFunctionMultiDimTempl<double> > > >, ROOT::Math::IBaseFunctionMultiDimTempl<double> const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libMathCore.so
#27 0x000040000f573be0 in ROOT::Fit::Fitter::DoLeastSquareFit(ROOT::EExecutionPolicy const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libMathCore.so
#28 0x0000400016394610 in TFitResultPtr HFit::Fit<TGraph>(TGraph*, TF1*, Foption_t&, ROOT::Math::MinimizerOptions const&, char const*, ROOT::Fit::DataRange&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libHist.so
#29 0x000040001638ad6c in ROOT::Fit::FitObject(TGraph*, TF1*, Foption_t&, ROOT::Math::MinimizerOptions const&, char const*, ROOT::Fit::DataRange&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libHist.so
#30 0x000040001643a108 in TGraph::Fit(TF1*, char const*, char const*, double, double) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/external/el8_aarch64_gcc12/lib/libHist.so
#31 0x000040008be050cc in CSCSimHitMatcher::fitHitsInChamber(unsigned int, float&, float&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libValidationMuonHits.so
#32 0x000040008bef9700 in CSCStubResolutionValidation::analyze(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libValidationMuonCSCDigis.so
#33 0x000040000eb67614 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02837/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-05-15-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so

Current Modules:

Module: CSCDigiValidation:cscDigiValidation (crashed)
Module: TkConvValidator:tkConversionValidation
Module: CkfTrackCandidateMaker:lowPtTripletStepTrackCandidates
Module: MkFitProducer:pixelLessStepTrackCandidatesMkFit

A fatal system signal has occurred: segmentation violation
timeout: the monitored command dumped core
makortel commented 6 months ago

el8_aarch64_gcc12, CMSSW_14_1_ROOT6_X_2024-05-15-2300, WF 11024.2 step3. Does this include the latest patches from @pcanal ?

I think it should. The ROOT master branch was updated in https://github.com/cms-sw/cmsdist/pull/9187 that was merged in CMSSW_14_1_ROOT6_X_2024-05-13-2300.

iarspider commented 6 months ago

Occured again in CMSSW_14_1_X_2024-05-28-1100:

Thread 1 (Thread 0x40003ebe8010 (LWP 4085649) "cmsRun"):
#0  0x00004000402e091c in poll () from /lib64/libc.so.6
#1  0x0000400045366700 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#2  0x0000400045366934 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x000040017afff01c in ?? ()
#5  0x00004000538d0c10 in cling::Interpreter::RunFunction(clang::FunctionDecl const*, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#6  0x00004000538d0c10 in cling::Interpreter::RunFunction(clang::FunctionDecl const*, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#7  0x00004000538d12bc in cling::Interpreter::EvaluateInternal(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, cling::CompilationOptions, cling::Value*, cling::Transaction**, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#8  0x00004000539a3054 in cling::MetaSema::actOnxCommand(llvm::StringRef, llvm::StringRef, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#9  0x00004000539af944 in cling::MetaParser::isXCommand(cling::MetaSema::ActionResult&, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#10 0x00004000539b1804 in cling::MetaParser::isCommand(cling::MetaSema::ActionResult&, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#11 0x000040005399cafc in cling::MetaProcessor::process(llvm::StringRef, cling::Interpreter::CompilationResult&, cling::Value*, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#12 0x00004000537f0d7c in HandleInterpreterException(cling::MetaProcessor*, char const*, cling::Interpreter::CompilationResult&, cling::Value*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#13 0x0000400053803770 in TCling::ProcessLine(char const*, TInterpreter::EErrorCode*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#14 0x0000400053803ce4 in TCling::ProcessLineSynch(char const*, TInterpreter::EErrorCode*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#15 0x000040003f5f8898 in TApplication::ExecuteFile(char const*, int*, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCore.so
#16 0x0000400053802808 in TCling::ExecuteMacro(char const*, TInterpreter::EErrorCode*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCling.so
#17 0x000040003f64dc5c in TROOT::Macro(char const*, int*, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCore.so
#18 0x000040003f631730 in TPluginManager::LoadHandlerMacros(char const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCore.so
#19 0x000040003f631d8c in TPluginManager::LoadHandlersFromPluginDirs(char const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCore.so
#20 0x000040003f632e04 in TPluginManager::FindHandler(char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libCore.so
#21 0x000040003ecd7ccc in ROOT::Math::Factory::CreateMinimizer(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libMathCore.so
#22 0x000040003ecd9690 in ROOT::Fit::FitConfig::CreateMinimizer() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libMathCore.so
#23 0x000040003ecf11e0 in ROOT::Fit::Fitter::DoInitMinimizer() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libMathCore.so
#24 0x000040003ecf74bc in bool ROOT::Fit::Fitter::DoMinimization<ROOT::Fit::PoissonLikelihoodFCN<ROOT::Math::IBaseFunctionMultiDimTempl<double>, ROOT::Math::IParametricFunctionMultiDimTempl<double> > >(std::unique_ptr<ROOT::Fit::PoissonLikelihoodFCN<ROOT::Math::IBaseFunctionMultiDimTempl<double>, ROOT::Math::IParametricFunctionMultiDimTempl<double> >, std::default_delete<ROOT::Fit::PoissonLikelihoodFCN<ROOT::Math::IBaseFunctionMultiDimTempl<double>, ROOT::Math::IParametricFunctionMultiDimTempl<double> > > >, ROOT::Math::IBaseFunctionMultiDimTempl<double> const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libMathCore.so
#25 0x000040003ecf49b0 in ROOT::Fit::Fitter::DoBinnedLikelihoodFit(bool, ROOT::EExecutionPolicy const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libMathCore.so
#26 0x0000400047fc98a4 in TFitResultPtr HFit::Fit<TH1>(TH1*, TF1*, Foption_t&, ROOT::Math::MinimizerOptions const&, char const*, ROOT::Fit::DataRange&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libHist.so
#27 0x0000400047fbf414 in ROOT::Fit::FitObject(TH1*, TF1*, Foption_t&, ROOT::Math::MinimizerOptions const&, char const*, ROOT::Fit::DataRange&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libHist.so
#28 0x0000400048088f40 in TH1::Fit(TF1*, char const*, char const*, double, double) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_X_2024-05-28-1100/external/el8_aarch64_gcc12/lib/libHist.so
#29 0x00004000833a802c in PVFitter::runFitter() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libRecoVertexBeamSpotProducer.so
#30 0x00004000833a0adc in BeamFitter::runPVandTrkFitter() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libRecoVertexBeamSpotProducer.so
#31 0x00004000df1d9304 in AlcaBeamMonitor::globalEndLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/pluginAlcaBeamMonitor.so
#32 0x00004000df1e58ec in virtual thunk to edm::one::impl::LuminosityBlockCacheHolder<edm::one::EDProducerBase, alcabeammonitor::BeamSpotInfo>::doEndLuminosityBlock_(edm::LuminosityBlock const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/pluginAlcaBeamMonitor.so
#33 0x000040003e42bc08 in edm::one::EDProducerBase::doEndLuminosityBlock(edm::LumiTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#34 0x000040003e413600 in edm::WorkerT<edm::one::EDProducerBase>::implDoEnd(edm::LumiTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#35 0x000040003e35a438 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::LuminosityBlockPrincipal, (edm::BranchActionType)3> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::LuminosityBlockPrincipal, (edm::BranchActionType)3>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::LuminosityBlockPrincipal, (edm::BranchActionType)3>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#36 0x000040003e35f760 in edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::LuminosityBlockPrincipal, (edm::BranchActionType)3> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::LuminosityBlockPrincipal, (edm::BranchActionType)3> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#37 0x000040003e7c4bd8 in tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libFWCoreConcurrency.so
#38 0x000040003ef3a774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x40012c766a00, this=0x400040a73300) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#39 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400040a73300) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#40 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#41 0x000040003e316bf4 in edm::FinalWaitingTask::wait() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#42 0x000040003e326168 in edm::EventProcessor::processRuns() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#43 0x000040003e31c03c in edm::EventProcessor::runToCompletion() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02839/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-27-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#44 0x00000000004071a4 in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#45 0x000040003ef330d8 in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:688
#46 0x0000000000408aac in main::{lambda()#1}::operator()() const ()
#47 0x0000000000404804 in main ()