search

cms-sw / cmssw

CMS Offline Software
http://cms-sw.github.io/
Apache License 2.0
1.08k stars 4.3k forks source link

Crashes in mkfit::MkFinder::bkFitFitTracks() #37017

Closed makortel closed 2 years ago

makortel commented 2 years ago

In CMSSW_12_3_X_2022-02-21-1100 several workflows crash in mkFit. I'm listing stack traces of only some below

136.817

Thread 4 (Thread 0x2ae3b32a5700 (LWP 26168) "cmsRun"):
#2  0x00002ae36cb62e20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002ae366a72c4c in sincosf () from /lib64/libm.so.6
#5  0x00002ae3aa665356 in mkfit::helixAtZ(Matriplex::Matriplex<float, 6, 1, 4> const&, Matriplex::Matriplex<int, 1, 1, 4> const&, Matriplex::Matriplex<float, 1, 1, 4> const&, Matriplex::Matriplex<float, 6, 1, 4>&, Matriplex::Matriplex<float, 6, 6, 4>&, int, mkfit::PropagationFlags) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002ae3aa666b2b in mkfit::propagateHelixToZMPlex(Matriplex::MatriplexSym<float, 6, 4> const&, Matriplex::Matriplex<float, 6, 1, 4> const&, Matriplex::Matriplex<int, 1, 1, 4> const&, Matriplex::Matriplex<float, 1, 1, 4> const&, Matriplex::MatriplexSym<float, 6, 4>&, Matriplex::Matriplex<float, 6, 1, 4>&, int, mkfit::PropagationFlags, Matriplex::Matriplex<int, 1, 1, 4> const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002ae3aa65defb in mkfit::MkFinder::bkFitFitTracks(mkfit::EventOfHits const&, mkfit::SteeringParams const&, int, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002ae3aa64c792 in mkfit::MkBuilder::fit_cands(mkfit::MkFinder*, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#9  0x00002ae3aa653497 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#10 0x00002ae36616eb8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2ae44e70af00, this=0x2ae369ded400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#11 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2ae369ded400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#12 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#13 0x00002ae3aa648e6a in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#14 0x00002ae3aa653209 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#15 0x00002ae36616eb8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2ae44e70a700, this=0x2ae369ded400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#16 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2ae369ded400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#17 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#18 0x00002ae3aa64947a in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#19 0x00002ae36616eb8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2ae3b329d440, this=0x2ae369ded400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#20 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2ae369ded400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#21 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#22 0x00002ae3aa64c695 in mkfit::MkBuilder::backwardFit() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#23 0x00002ae3aa5dfff7 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCMS.so
#24 0x00002ae3aa579e26 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#25 0x00002ae36615d9ce in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:771
#26 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#27 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:772
#28 0x00002ae3aa57bbc4 in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#29 0x00002ae3649f4b73 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#30 0x00002ae3649ef7df in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#31 0x00002ae36494af15 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#32 0x00002ae36494b20b in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr const*, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#33 0x00002ae36494d7f5 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#34 0x00002ae3646e37d5 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreConcurrency.so

Thread 3 (Thread 0x2ae3b28a4700 (LWP 26167) "cmsRun"):
#3  0x00002ae36cb6656b in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x00002ae3aa65dc64 in mkfit::MkFinder::bkFitFitTracks(mkfit::EventOfHits const&, mkfit::SteeringParams const&, int, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002ae3aa64c792 in mkfit::MkBuilder::fit_cands(mkfit::MkFinder*, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002ae3aa653497 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002ae36616eb8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2ae44ea3fe00, this=0x2ae369ded480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2ae369ded480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#11 0x00002ae3aa648e6a in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#12 0x00002ae3aa653209 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so

Current Modules:
Module: none (crashed)
Module: none
Module: SeedCreatorFromRegionConsecutiveHitsEDProducer:initialStepSeedsPreSplitting
Module: MkFitProducer:initialStepTrackCandidatesMkFitPreSplitting

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/slc7_amd64_gcc10/CMSSW_12_3_X_2022-02-21-1100/pyRelValMatrixLogs/run/136.817_RunDoubleEG2017E+RunDoubleEG2017E+HLTDR2_2017+RECODR2_2017reHLT_skimDoubleEG_Prompt+HARVEST2017_skimDoubleEG/step3_RunDoubleEG2017E+RunDoubleEG2017E+HLTDR2_2017+RECODR2_2017reHLT_skimDoubleEG_Prompt+HARVEST2017_skimDoubleEG.log#/

136.822

Thread 5 (Thread 0x2b28bc800700 (LWP 17547) "cmsRun"):
#2  0x00002b28751e0e20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002b286f0f7c50 in sincosf () from /lib64/libm.so.6
#5  0x00002b28b2af5381 in mkfit::helixAtZ(Matriplex::Matriplex<float, 6, 1, 4> const&, Matriplex::Matriplex<int, 1, 1, 4> const&, Matriplex::Matriplex<float, 1, 1, 4> const&, Matriplex::Matriplex<float, 6, 1, 4>&, Matriplex::Matriplex<float, 6, 6, 4>&, int, mkfit::PropagationFlags) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b28b2af6b2b in mkfit::propagateHelixToZMPlex(Matriplex::MatriplexSym<float, 6, 4> const&, Matriplex::Matriplex<float, 6, 1, 4> const&, Matriplex::Matriplex<int, 1, 1, 4> const&, Matriplex::Matriplex<float, 1, 1, 4> const&, Matriplex::MatriplexSym<float, 6, 4>&, Matriplex::Matriplex<float, 6, 1, 4>&, int, mkfit::PropagationFlags, Matriplex::Matriplex<int, 1, 1, 4> const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b28b2aedefb in mkfit::MkFinder::bkFitFitTracks(mkfit::EventOfHits const&, mkfit::SteeringParams const&, int, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b28b2adc792 in mkfit::MkBuilder::fit_cands(mkfit::MkFinder*, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#9  0x00002b28b2ae3497 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#10 0x00002b286e7f3b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b295b8f7a00, this=0x2b287243d500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#11 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b287243d500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#12 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#13 0x00002b28b2ad8e6a in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#14 0x00002b28b2ae3209 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#15 0x00002b286e7f3b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b295b8f0500, this=0x2b287243d500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#16 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b287243d500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#17 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#18 0x00002b28b2ad947a in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#19 0x00002b286e7f3b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b28bc7f8440, this=0x2b287243d500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#20 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b287243d500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#21 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#22 0x00002b28b2adc695 in mkfit::MkBuilder::backwardFit() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#23 0x00002b28b2a6fff7 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCMS.so
#24 0x00002b28b2a09e26 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#25 0x00002b286e7e29ce in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:771
#26 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#27 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:772
#28 0x00002b28b2a0bbc4 in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#29 0x00002b286d079b73 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#30 0x00002b286d0747df in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#31 0x00002b286cfcff15 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#32 0x00002b286cfd020b in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr const*, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#33 0x00002b286cfd27f5 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#34 0x00002b286cd687d5 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreConcurrency.so

Thread 3 (Thread 0x2b28bad34700 (LWP 17545) "cmsRun"):
#2  0x00002b28751e0e20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002b28b2aec8f6 in mkfit::MkFinder::selectHitIndices(mkfit::LayerOfHits const&, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#5  0x00002b28b2ade08a in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b28b2ade46f in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b28b2ae21cd in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b286e7f3b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b295b685d00, this=0x2b287243d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b287243d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#11 0x00002b28b2ad9ce8 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#12 0x00002b28b2ae1f89 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#13 0x00002b286e7f3b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b295b686c00, this=0x2b287243d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b287243d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#15 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#16 0x00002b28b2ad983a in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#17 0x00002b286e7f3b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b28bad2c440, this=0x2b287243d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#18 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b287243d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#19 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#20 0x00002b28b2adc303 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#21 0x00002b28b2a6fde1 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCMS.so
#22 0x00002b28b2a09e26 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#23 0x00002b286e7e29ce in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:771
#24 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#25 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:772
#26 0x00002b28b2a0bbc4 in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#27 0x00002b286d079b73 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#28 0x00002b286d0747df in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#29 0x00002b286cfcff15 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#30 0x00002b286cfd020b in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr const*, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#31 0x00002b286cfd27f5 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#32 0x00002b286cd687d5 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreConcurrency.so

Thread 1 (Thread 0x2b2871780b00 (LWP 16774) "cmsRun"):
#3  0x00002b28751e456b in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x00002b28b2aedc64 in mkfit::MkFinder::bkFitFitTracks(mkfit::EventOfHits const&, mkfit::SteeringParams const&, int, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b28b2adc792 in mkfit::MkBuilder::fit_cands(mkfit::MkFinder*, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b28b2ae3497 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b286e7f3b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b28bd006200, this=0x2b287243d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b287243d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#11 0x00002b28b2ad8e6a in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#12 0x00002b28b2ae3209 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#13 0x00002b286e7f3b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b295b8f7e00, this=0x2b287243d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b287243d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#15 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#16 0x00002b286cf40988 in edm::EventProcessor::processLumis(std::shared_ptr<void> const&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so

Current Modules:
Module: none (crashed)
Module: MkFitProducer:initialStepTrackCandidatesMkFitPreSplitting
Module: MkFitProducer:initialStepTrackCandidatesMkFitPreSplitting
Module: none

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/slc7_amd64_gcc10/CMSSW_12_3_X_2022-02-21-1100/pyRelValMatrixLogs/run/136.822_RunSingleEl2017E+RunSingleEl2017E+HLTDR2_2017+RECODR2_2017reHLT_Prompt_L1TEgDQM+HARVEST2017_L1TEgDQM/step3_RunSingleEl2017E+RunSingleEl2017E+HLTDR2_2017+RECODR2_2017reHLT_Prompt_L1TEgDQM+HARVEST2017_L1TEgDQM.log#/

makortel commented 2 years ago

assign reconstruction

cmsbuild commented 2 years ago

New categories assigned: reconstruction

@slava77,@jpata,@clacaputo you have been requested to review this Pull request/Issue and eventually sign? Thanks

cmsbuild commented 2 years ago

A new Issue was created by @makortel Matti Kortelainen.

@Dr15Jones, @perrotta, @dpiparo, @makortel, @smuzaffar, @qliphy can you please review it and eventually sign/assign? Thanks.

cms-bot commands are listed here

makortel commented 2 years ago

FYI @cms-sw/tracking-pog-l2 @osschar @slava77 @dan131riley

makortel commented 2 years ago

Here are two other workflows that I looked that (working around GitHub's apparent limit of 2^16 characters in one comment)

136.824

Thread 4 (Thread 0x2b5f7d346700 (LWP 1091) "cmsRun"):
#2  0x00002b5f372b2e20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002b5f7470222c in void mkfit::MatriplexErrParPackerSlurpIn<mkfit::Hit, float>::pack<Matriplex::MatriplexSym<float, 3, 4>, Matriplex::Matriplex<float, 3, 1, 4> >(Matriplex::MatriplexSym<float, 3, 4>&, Matriplex::Matriplex<float, 3, 1, 4>&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#5  0x00002b5f747003cf in mkfit::MkFinder::findCandidatesCloneEngine(mkfit::LayerOfHits const&, mkfit::CandCloner&, int, int, mkfit::FindingFoos const&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b5f746ef0f0 in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b5f746ef46f in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b5f746f31cd in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#9  0x00002b5f301c8b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b6016beb600, this=0x2b5f33e3d400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#10 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b5f33e3d400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#11 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#12 0x00002b5f746eace8 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#13 0x00002b5f746f2f89 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so

Thread 3 (Thread 0x2b5f7c945700 (LWP 1090) "cmsRun"):
#3  0x00002b5f372b656b in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x00002b5f746fec64 in mkfit::MkFinder::bkFitFitTracks(mkfit::EventOfHits const&, mkfit::SteeringParams const&, int, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b5f746ed792 in mkfit::MkBuilder::fit_cands(mkfit::MkFinder*, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b5f746f4497 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b5f301c8b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b6016cb7f00, this=0x2b5f33e3d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b5f33e3d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#11 0x00002b5f746e9e6a in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#12 0x00002b5f746f4209 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#13 0x00002b5f301c8b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b5f7e860200, this=0x2b5f33e3d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b5f33e3d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#15 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#16 0x00002b5f746ea47a in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#17 0x00002b5f301c8b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b5f7c93d440, this=0x2b5f33e3d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#18 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b5f33e3d480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#19 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#20 0x00002b5f746ed695 in mkfit::MkBuilder::backwardFit() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#21 0x00002b5f74680ff7 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCMS.so
#22 0x00002b5f7461ae26 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#23 0x00002b5f301b79ce in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:771
#24 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#25 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:772
#26 0x00002b5f7461cbc4 in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#27 0x00002b5f2ea4eb73 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#28 0x00002b5f2ea497df in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#29 0x00002b5f2e9a4f15 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#30 0x00002b5f2e9a520b in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr const*, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#31 0x00002b5f2e9a77f5 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#32 0x00002b5f2e73d7d5 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreConcurrency.so

Thread 1 (Thread 0x2b5f33155b00 (LWP 842) "cmsRun"):
#2  0x00002b5f372b2e20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002b5f746ee703 in mkfit::MkBuilder::find_tracks_unroll_candidates(std::vector<std::pair<int, int>, std::allocator<std::pair<int, int> > >&, int, int, int, int, bool, mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#5  0x00002b5f746eed5f in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b5f746ef46f in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b5f746f31cd in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b5f301c8b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b5f7e889b00, this=0x2b5f33e3d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b5f33e3d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#11 0x00002b5f746eace8 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#12 0x00002b5f746f2f89 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#13 0x00002b5f301c8b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b5f7e86de00, this=0x2b5f33e3d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b5f33e3d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#15 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#16 0x00002b5f746ea83a in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#17 0x00002b5f301c8b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x7ffe5ca60800, this=0x2b5f33e3d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#18 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b5f33e3d380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#19 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#20 0x00002b5f746ed303 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#21 0x00002b5f74680de1 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCMS.so
#22 0x00002b5f7461ae26 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#23 0x00002b5f301b79ce in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:771
#24 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#25 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:772
#26 0x00002b5f7461cbc4 in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#27 0x00002b5f2ea4eb73 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#28 0x00002b5f2ea497df in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#29 0x00002b5f2e9a4f15 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#30 0x00002b5f2e9a520b in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr const*, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#31 0x00002b5f2e9a77f5 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#32 0x00002b5f2e73d7d5 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreConcurrency.so

Current Modules:
Module: MkFitProducer:initialStepTrackCandidatesMkFitPreSplitting (crashed)
Module: none
Module: none
Module: MkFitProducer:initialStepTrackCandidatesMkFitPreSplitting

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/slc7_amd64_gcc10/CMSSW_12_3_X_2022-02-21-1100/pyRelValMatrixLogs/run/136.824_RunSinglePh2017E+RunSinglePh2017E+HLTDR2_2017+RECODR2_2017reHLT_skimSinglePh_Prompt+HARVEST2017_skimSinglePh/step3_RunSinglePh2017E+RunSinglePh2017E+HLTDR2_2017+RECODR2_2017reHLT_skimSinglePh_Prompt+HARVEST2017_skimSinglePh.log#/

136.867

Thread 5 (Thread 0x2b18c8400700 (LWP 331) "cmsRun"):
#3  0x00002b187e54e56b in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x00002b18bd3e0c64 in mkfit::MkFinder::bkFitFitTracks(mkfit::EventOfHits const&, mkfit::SteeringParams const&, int, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b18bd3cf792 in mkfit::MkBuilder::fit_cands(mkfit::MkFinder*, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b18bd3d668f in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b1877114b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b1962150700, this=0x2b187aded480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b187aded480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#11 0x00002b18bd3cbe6a in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#12 0x00002b18bd3d6209 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#13 0x00002b1877114b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b1962151400, this=0x2b187aded480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b187aded480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#15 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#16 0x00002b18bd3cc47a in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#17 0x00002b1877114b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b18c83f8440, this=0x2b187aded480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#18 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b187aded480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#19 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#20 0x00002b18bd3cf695 in mkfit::MkBuilder::backwardFit() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#21 0x00002b18bd362ff7 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCMS.so
#22 0x00002b18bd2fce26 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#23 0x00002b18771039ce in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:771
#24 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#25 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:772
#26 0x00002b18bd2febc4 in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#27 0x00002b187599ab73 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#28 0x00002b18759957df in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#29 0x00002b18758f0f15 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#30 0x00002b18758f120b in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr const*, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#31 0x00002b18758f37f5 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#32 0x00002b18756897d5 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreConcurrency.so

Thread 4 (Thread 0x2b18c7428700 (LWP 330) "cmsRun"):
#2  0x00002b187e54ae20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002b18bd385420 in void std::__unguarded_linear_insert<__gnu_cxx::__normal_iterator<mkfit::IdxChi2List*, std::vector<mkfit::IdxChi2List, std::allocator<mkfit::IdxChi2List> > >, __gnu_cxx::__ops::_Val_comp_iter<bool (*)(mkfit::IdxChi2List const&, mkfit::IdxChi2List const&)> >(__gnu_cxx::__normal_iterator<mkfit::IdxChi2List*, std::vector<mkfit::IdxChi2List, std::allocator<mkfit::IdxChi2List> > >, __gnu_cxx::__ops::_Val_comp_iter<bool (*)(mkfit::IdxChi2List const&, mkfit::IdxChi2List const&)>) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#5  0x00002b18bd3855cf in void std::__insertion_sort<__gnu_cxx::__normal_iterator<mkfit::IdxChi2List*, std::vector<mkfit::IdxChi2List, std::allocator<mkfit::IdxChi2List> > >, __gnu_cxx::__ops::_Iter_comp_iter<bool (*)(mkfit::IdxChi2List const&, mkfit::IdxChi2List const&)> >(__gnu_cxx::__normal_iterator<mkfit::IdxChi2List*, std::vector<mkfit::IdxChi2List, std::allocator<mkfit::IdxChi2List> > >, __gnu_cxx::__normal_iterator<mkfit::IdxChi2List*, std::vector<mkfit::IdxChi2List, std::allocator<mkfit::IdxChi2List> > >, __gnu_cxx::__ops::_Iter_comp_iter<bool (*)(mkfit::IdxChi2List const&, mkfit::IdxChi2List const&)>) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b18bd383e7b in mkfit::CandCloner::processSeedRange(int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b18bd383f28 in mkfit::CandCloner::doWork(int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b18bd3d10f8 in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#9  0x00002b18bd3d146f in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#10 0x00002b18bd3d51cd in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#11 0x00002b1877114b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b1962090100, this=0x2b187aded500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#12 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b187aded500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#13 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#14 0x00002b18bd3ccce8 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#15 0x00002b18bd3d4f89 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so

Thread 3 (Thread 0x2b18c6a27700 (LWP 329) "cmsRun"):
#2  0x00002b187e54ae20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002b18bd3e7710 in mkfit::helixAtRFromIterativeCCS(Matriplex::Matriplex<float, 6, 1, 4> const&, Matriplex::Matriplex<int, 1, 1, 4> const&, Matriplex::Matriplex<float, 1, 1, 4> const&, Matriplex::Matriplex<float, 6, 1, 4>&, Matriplex::Matriplex<float, 6, 6, 4>&, Matriplex::Matriplex<int, 1, 1, 4>&, int, mkfit::PropagationFlags) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#5  0x00002b18bd3e8b8b in mkfit::propagateHelixToRMPlex(Matriplex::MatriplexSym<float, 6, 4> const&, Matriplex::Matriplex<float, 6, 1, 4> const&, Matriplex::Matriplex<int, 1, 1, 4> const&, Matriplex::Matriplex<float, 1, 1, 4> const&, Matriplex::MatriplexSym<float, 6, 4>&, Matriplex::Matriplex<float, 6, 1, 4>&, int, mkfit::PropagationFlags, Matriplex::Matriplex<int, 1, 1, 4> const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b18bd3c98fd in mkfit::kalmanPropagateAndComputeChi2(Matriplex::MatriplexSym<float, 6, 4> const&, Matriplex::Matriplex<float, 6, 1, 4> const&, Matriplex::Matriplex<int, 1, 1, 4> const&, Matriplex::MatriplexSym<float, 3, 4> const&, Matriplex::Matriplex<float, 3, 1, 4> const&, Matriplex::Matriplex<float, 1, 1, 4>&, Matriplex::Matriplex<float, 6, 1, 4>&, int, mkfit::PropagationFlags, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b18bd3e2422 in mkfit::MkFinder::findCandidatesCloneEngine(mkfit::LayerOfHits const&, mkfit::CandCloner&, int, int, mkfit::FindingFoos const&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b18bd3d10f0 in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#9  0x00002b18bd3d146f in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#10 0x00002b18bd3d51cd in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#11 0x00002b1877114b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b19620ebe00, this=0x2b187aded400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#12 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b187aded400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#13 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#14 0x00002b18bd3ccce8 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#15 0x00002b18bd3d4f89 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so

Thread 1 (Thread 0x2b187a0a1b00 (LWP 32315) "cmsRun"):
#2  0x00002b187e54ae20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00002b18bd3d076a in mkfit::MkBuilder::find_tracks_unroll_candidates(std::vector<std::pair<int, int>, std::allocator<std::pair<int, int> > >&, int, int, int, int, bool, mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#5  0x00002b18bd3d0d5f in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#6  0x00002b18bd3d146f in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#7  0x00002b18bd3d51cd in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#8  0x00002b1877114b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b1962126900, this=0x2b187aded380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b187aded380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#11 0x00002b18bd3ccce8 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#12 0x00002b18bd3d4f89 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#13 0x00002b1877114b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x2b1962124b00, this=0x2b187aded380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b187aded380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#15 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#16 0x00002b18bd3cc83a in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#17 0x00002b1877114b8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x7ffe6b208780, this=0x2b187aded380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#18 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x2b187aded380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#19 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#20 0x00002b18bd3cf303 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCore.so
#21 0x00002b18bd362de1 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libRecoTrackerMkFitCMS.so
#22 0x00002b18bd2fce26 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#23 0x00002b18771039ce in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:771
#24 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#25 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:772
#26 0x00002b18bd2febc4 in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/pluginRecoTrackerMkFitPlugins.so
#27 0x00002b187599ab73 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#28 0x00002b18759957df in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#29 0x00002b18758f0f15 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#30 0x00002b18758f120b in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr const*, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#31 0x00002b18758f37f5 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreFramework.so
#32 0x00002b18756897d5 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc10/cms/cmssw/CMSSW_12_3_X_2022-02-21-1100/lib/slc7_amd64_gcc10/libFWCoreConcurrency.so

Current Modules:
Module: MkFitProducer:detachedTripletStepTrackCandidatesMkFit (crashed)
Module: MkFitProducer:highPtTripletStepTrackCandidatesMkFit
Module: none
Module: none

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/slc7_amd64_gcc10/CMSSW_12_3_X_2022-02-21-1100/pyRelValMatrixLogs/run/136.867_RunSingleMu2018B+RunSingleMu2018B+HLTDR2_2018+RECODR2_2018reHLT_skimSingleMu_Offline_Lumi+HARVEST2018_L1TMuDQM/step3_RunSingleMu2018B+RunSingleMu2018B+HLTDR2_2018+RECODR2_2018reHLT_skimSingleMu_Offline_Lumi+HARVEST2018_L1TMuDQM.log#/

makortel commented 2 years ago

Common theme in these 4 workflows appears to be the crash in mkfit::MkFinder::bkFitFitTracks().

makortel commented 2 years ago

ASAN reports

==31171==ERROR: AddressSanitizer: invalid alignment requested in aligned_alloc: 64, alignment must be a power of two and the requested size 0x68 must be a multiple of alignment (thread T0)
    #0 0x2aaf96beffaf in __interceptor_aligned_alloc ../../../../libsanitizer/asan/asan_malloc_linux.cpp:192
    #1 0x2aafb6658bd7 in mkfit::MkBuilder::populate() (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libRecoTrackerMkFitCore.so+0x1babd7)
    #2 0x2aafb6390920 in MkFitProducer::MkFitProducer(edm::ParameterSet const&) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/pluginRecoTrackerMkFitPlugins.so+0x15a920)
    #3 0x2aafb63abcc4 in edm::WorkerMaker<MkFitProducer>::makeModule(edm::ParameterSet const&) const (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/pluginRecoTrackerMkFitPlugins.so+0x175cc4)
    #4 0x2aaf97d4107d in edm::Maker::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x82f07d)
    #5 0x2aaf979b1f66 in edm::Factory::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x49ff66)
    #6 0x2aaf97a05c59 in edm::ModuleRegistry::getModule(edm::MakeModuleParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x4f3c59)
    #7 0x2aaf97d4ec81 in edm::WorkerRegistry::getWorker(edm::WorkerParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x83cc81)
    #8 0x2aaf97d43246 in edm::WorkerManager::getWorker(edm::ParameterSet&, edm::ProductRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x831246)
    #9 0x2aaf97d47be8 in edm::WorkerManager::addToUnscheduledWorkers(edm::ParameterSet&, edm::ProductRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x835be8)
    #10 0x2aaf97ca5341 in edm::StreamSchedule::StreamSchedule(std::shared_ptr<edm::TriggerResultInserter>, std::vector<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> > > >&, std::vector<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> > > >&, std::shared_ptr<edm::ModuleRegistry>, edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProductRegistry&, edm::BranchIDListHelper&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration>, edm::StreamID, edm::ProcessContext const*) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x793341)
    #11 0x2aaf97c20cfa in edm::Schedule::Schedule(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::ProductRegistry&, edm::BranchIDListHelper&, edm::ProcessBlockHelperBase&, edm::ThinnedAssociationsHelper&, edm::SubProcessParentageHelper const*, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration>, bool, edm::PreallocationConfiguration const&, edm::ProcessContext const*) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x70ecfa)
    #12 0x2aaf97c5c77e in edm::ScheduleItems::initSchedule(edm::ParameterSet&, bool, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ProcessBlockHelperBase&) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x74a77e)
    #13 0x2aaf977be36e in edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x2ac36e)
    #14 0x2aaf977c7760 in edm::EventProcessor::EventProcessor(std::shared_ptr<edm::ProcessDesc>, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy) (/cvmfs/cms-ib.cern.ch/week1/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/lib/slc7_amd64_gcc11/libFWCoreFramework.so+0x2b5760)
    #15 0x40e36b in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (/cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/bin/slc7_amd64_gcc11/cmsRun+0x40e36b)
    #16 0x2aaf99d77897 in tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/slc7_amd64_gcc11/external/tbb/v2021.4.0-a1ac0a4b6fb3ff931b4e97030094b2d8/tbb-v2021.4.0/src/tbb/arena.cpp:698
    #17 0x411e7f in main::{lambda()#1}::operator()() const (/cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/bin/slc7_amd64_gcc11/cmsRun+0x411e7f)
    #18 0x40c615 in main (/cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc11/cms/cmssw/CMSSW_12_3_ASAN_X_2022-02-21-1100/bin/slc7_amd64_gcc11/cmsRun+0x40c615)
    #19 0x2aaf9abf3554 in __libc_start_main (/lib64/libc.so.6+0x22554)

(e.g. https://cmssdt.cern.ch/SDT/cgi-bin/logreader/slc7_amd64_gcc11/CMSSW_12_3_ASAN_X_2022-02-21-1100/pyRelValMatrixLogs/run/136.78_RunHLTPhy2017B+RunHLTPhy2017B+HLTDR2_2017+RECODR2_2017reHLT_Prompt+HARVEST2017/step3_RunHLTPhy2017B+RunHLTPhy2017B+HLTDR2_2017+RECODR2_2017reHLT_Prompt+HARVEST2017.log#/)

that may be related (or not).

slava77 commented 2 years ago

it seems to do with https://github.com/cms-sw/cmssw/blob/a96a2c72916dbf8ff3dffbe9ff58e9767d90062c/RecoTracker/MkFitCore/src/Config.cc#L7-L14

mkfit::ConfigWrapper::initializeForCMSSW calls PropagationConfig::set_as_default from different MkFitProducer instances

slava77 commented 2 years ago

the stack trace was 

#0  0x00007fffaf99ec64 in mkfit::MkFinder::bkFitFitTracks (this=this@entry=0x7ffe4a99c000, eventofhits=..., st_par=..., N_proc=N_proc@entry=4, chiDebug=chiDebug@entry=false)
    at ...CMSSW_12_3_X_2022-02-21-1100-sign1132/src/RecoTracker/MkFitCore/src/MkFinder.cc:1675
#1  0x00007fffaf98d792 in mkfit::MkBuilder::fit_cands (this=0x7fffdc6e1380, mkfndr=0x7ffe4a99c000, start_cand=<optimized out>, end_cand=308, region=<optimized out>)
    at ...CMSSW_12_3_X_2022-02-21-1100-sign1132/src/RecoTracker/MkFitCore/src/MkBuilder.cc:1331
#2  0x00007fffaf994497 in operator() (cands=..., __closure=0x7ffff27b5e50)
    at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/blocked_range.h:62
#3  tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::<lambda(int)>::<lambda(const tbb::detail::d1::blocked_range<int>&)>, const tbb::detail::d1::auto_partitioner>::run_body (r=..., this=0x7ffff27b5e00)
    at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/parallel_for.h:119
#4  tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::<lambda(int)>::<lambda(const tbb::detail::d1::blocked_range<int>&)>, const tbb::detail::d1::auto_partitioner>, tbb::detail::d1::blocked_range<int> >
    (ed=..., range=..., start=warning: RTTI symbol not found for class 'tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>'
..., this=<optimized out>)
    at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/partitioner.h:447
#5  tbb::detail::d1::partition_type_base<tbb::detail::d1::auto_partition_type>::execute<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::<lambda(int)>::<lambda(const tbb::detail::d1::blocked_range<int>&)>, const tbb::detail::d1::auto_partitioner>, tbb::detail::d1::blocked_range<int> > (ed=..., range=..., start=warning: RTTI symbol not found for class 'tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>'
...,
    this=<optimized out>) at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/partitioner.h:288
#6  tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::<lambda(int)>::<lambda(const tbb::detail::d1::blocked_range<int>&)>, const tbb::detail::d1::auto_partitioner>::execute(tbb::detail::d1::execution_data &) (this=0x7ffff27b5e00, ed=...)
    at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/parallel_for.h:172
#7  0x00007ffff638606c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7ffff27b5e00, waiter=..., this=0x7ffff27cb380)
    at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#8  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7ffff27cb380)
    at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#9  tbb::detail::r1::arena::process (this=<optimized out>, tls=...)
    at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:138
#10 0x00007ffff63925b3 in tbb::detail::r1::market::process (j=..., this=0x7ffff27cb580)
    at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/market.cpp:597
#11 tbb::detail::r1::rml::private_worker::run (this=0x7ffff07f4100)
    at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/private_server.cpp:267
#12 0x00007f0f3f0edb8c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x7f0f3b42b500, this=0x7f0f3b5cb400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:322
#13 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x7f0f3b5cb400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#14 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.cpp:168
#15 0x00007f0efa59ae6a in tbb::detail::d1::execute_and_wait (w_ctx=..., wait_ctx=..., t_ctx=..., t=warning: RTTI symbol not found for class 'tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>'
#16 tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::<lambda(int)>::<lambda(const tbb::detail::d1::blocked_range<int>&)>, const tbb::detail::d1::auto_partitioner>::run (partitioner=..., context=..., body=..., range=...) at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f766
0/include/oneapi/tbb/parallel_for.h:114
#17 tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::<lambda(int)>::<lambda(const tbb::detail::d1::blocked_range<int>&)>, const tbb::detail:
:d1::auto_partitioner>::run (partitioner=..., body=..., range=...) at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/one
api/tbb/parallel_for.h:103
#18 tbb::detail::d1::parallel_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::backwardFit()::<lambda(int)>::<lambda(const tbb::detail::d1::blocked_range<int>&)> > (body=..., ran
ge=...) at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/parallel_for.h:231
#19 operator() (region=<optimized out>, __closure=<optimized out>) at ...CMSSW_12_3_X_2022-02-21-1100-sign1132/src/RecoTracker/MkFitCore/src/MkBuilder.cc:1
274
#20 tbb::detail::d2::parallel_for_each_operator_selector<mkfit::MkBuilder::backwardFit()::<lambda(int)> >::call<int const&, tbb::detail::d2::feeder_impl<mkfit::MkBuilder::backwardFit()::<l
ambda(int)>, int> > (item=<optimized out>, body=...) at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/parall
el_for_each.h:95
#21 tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::<lambda(int)>, int>::oper
ator() (range=..., range=..., this=<optimized out>) at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/paralle
l_for_each.h:400
#22 tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<long unsigned int>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::
allocator<int> > >, mkfit::MkBuilder::backwardFit()::<lambda(int)>, int>, const tbb::detail::d1::auto_partitioner>::run_body (r=..., this=<optimized out>) at /cvmfs/cms-ib.cern.ch/nweek-02
721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/parallel_for.h:119
#23 tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<l
ong unsigned int>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int> >, mkfit::MkBuilder::backwardFit()::<lambda(int)>, int>, const tbb::
detail::d1::auto_partitioner>, tbb::detail::d1::blocked_range<long unsigned int> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<long unsigned int>, tbb::detail::d2::parallel_f
or_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::<lambda(int)>, int>, const tbb::detail::d1::auto_partiti
oner> &, tbb::detail::d1::blocked_range<unsigned long> &, tbb::detail::d1::execution_data &) (this=0x7f0f3b43ad78, start=warning: RTTI symbol not found for class 'tbb::detail::d1::start_fo
r<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuil
der::backwardFit()::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>'
#24 0x00007f0efa5a5209 in tbb::detail::d1::partition_type_base<tbb::detail::d1::auto_partition_type>::execute<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<long unsigned int>,
tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int> >, mkfit::MkBuilder::backwardFit()::<lambda(int)>, int>, const tbb::detail::d1::auto_pa
rtitioner>, tbb::detail::d1::blocked_range<long unsigned int> > (ed=..., range=..., start=warning: RTTI symbol not found for class 'tbb::detail::d1::start_for<tbb::detail::d1::blocked_rang
e<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::backwardFit()::{lambda(int
)#1}, int>, tbb::detail::d1::auto_partitioner const>'
#25 tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<long unsigned int>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::
allocator<int> > >, mkfit::MkBuilder::backwardFit()::<lambda(int)>, int>, const tbb::detail::d1::auto_partitioner>::execute(tbb::detail::d1::execution_data &) (this=0x7f0f3b43ad00, ed=...)
 at /cvmfs/cms-ib.cern.ch/nweek-02721/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/include/oneapi/tbb/parallel_for.h:172
#26 0x00007f0f3f0db06c in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x7f0f3b43ad00, waiter=..., this=0x7f0f3b5cb400) at /data
/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2
021.4.0/src/tbb/task_dispatcher.h:322
#27 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x7f0f3b5cb400) at /data/cmsbld/jenkins/workspace/auto-builds/CM
SSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/task_dispatcher.h:463
#28 tbb::detail::r1::arena::process (this=<optimized out>, tls=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc
7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/arena.cpp:138
#29 0x00007f0f3f0e75b3 in tbb::detail::r1::market::process (j=..., this=0x7f0f3b5cb580) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_
pre5-build/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/market.cpp:597
#30 tbb::detail::r1::rml::private_worker::run (this=0x7f0f39534080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build/BUILD/slc
7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/private_server.cpp:267
#31 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x7f0f39534080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_12_3_0_pre5-slc7_amd64_gcc10/build/CMSSW_12_3_0_pre5-build
/BUILD/slc7_amd64_gcc10/external/tbb/v2021.4.0-75e6d730601d8461f20893321f4f7660/tbb-v2021.4.0/src/tbb/private_server.cpp:221
#32 0x00007f0f3e2ebea5 in start_thread () from /lib64/libpthread.so.0
#33 0x00007f0f3e0139fd in clone () from /lib64/libc.so.6

print *this showed m_prop_config = 0x0, m_iteration_params = 0x0, m_iteration_layer_config = 0x0, m_iteration_hit_mask = 0x0

slava77 commented 2 years ago

should this issue be closed?