[ARM] Segfault in mkfit::MkBuilder::findTracksCloneEngine

[makortel]

Workflow 136.813 step 3 segfaulted in CMSSW_13_2_X_2023-06-22-2300 on el8_aarch64_gcc11 with

Thread 5 (Thread 0x40006ea09260 (LWP 950352) "cmsRun"):
#2  0x000040001dca33f8 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x000040005f1f43e4 in tensorflow::Tensor::~Tensor() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#5  0x000040005f2cf34c in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::Process(tensorflow::SimplePropagatorState::TaggedNode, long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#6  0x000040004729e4a4 in tensorflow::thread::ThreadPool::Schedule(std::function<void ()>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#7  0x000040004de49bdc in std::_Function_handler<void (std::function<void ()>), tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tensorflow::thread::ThreadPoolOptions const&)::{lambda(std::function<void ()>)#6}>::_M_invoke(std::_Any_data const&, std::function<void ()>&&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#8  0x000040005f2c2e3c in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ScheduleReady(absl::lts_20210324::InlinedVector<tensorflow::SimplePropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::SimplePropagatorState::TaggedNode> >*, tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#9  0x000040005f2c790c in tensorflow::(anonymous namespace)::ExecutorImpl::RunAsync(tensorflow::Executor::Args const&, std::function<void (tensorflow::Status const&)>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#10 0x000040004de590e0 in tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tensorflow::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#11 0x000040004de5b170 in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tensorflow::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#12 0x000040004445cde4 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libPhysicsToolsTensorFlow.so
#13 0x000040004445cfdc in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::thread::ThreadPoolInterface*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libPhysicsToolsTensorFlow.so
#14 0x00004000657497e8 in MkFitOutputConverter::computeDNNs(std::vector<TrackCandidate, std::allocator<TrackCandidate> > const&, std::vector<TrajectoryStateOnSurface, std::allocator<TrajectoryStateOnSurface> > const&, reco::BeamSpot const*, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const*, tensorflow::Session const*, std::vector<float, std::allocator<float> > const&, bool) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#15 0x000040006574b43c in MkFitOutputConverter::convertCandidates(MkFitOutputWrapper const&, mkfit::EventOfHits const&, MkFitClusterIndexToHit const&, MkFitClusterIndexToHit const&, edm::View<TrajectorySeed> const&, MagneticField const&, Propagator const&, Propagator const&, TkClonerImpl const&, std::vector<DetLayer const*, std::allocator<DetLayer const*> > const&, std::vector<mkfit::Track, std::allocator<mkfit::Track> > const&, reco::BeamSpot const*, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const*, tensorflow::Session const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#16 0x000040006574d51c in MkFitOutputConverter::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#17 0x0000400016af05c0 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#18 0x0000400016aea280 in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 4 (Thread 0x40006a6e9260 (LWP 950351) "cmsRun"):
#2  0x000040001dca33f8 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x0000400042cf5998 in ROOT::Math::SVector<double, 6u>& ROOT::Math::SVector<double, 6u>::operator=<ROOT::Math::BinaryOp<ROOT::Math::AddOp<double>, ROOT::Math::VecExpr<ROOT::Math::BinaryOp<ROOT::Math::AddOp<double>, ROOT::Math::SVector<double, 6u>, ROOT::Math::VecExpr<ROOT::Math::BinaryOpCopyL<ROOT::Math::MulOp<double>, ROOT::Math::Constant<double>, ROOT::Math::SVector<double, 6u>, double>, double, 6u>, double>, double, 6u>, ROOT::Math::VecExpr<ROOT::Math::BinaryOpCopyL<ROOT::Math::MulOp<double>, ROOT::Math::Constant<double>, ROOT::Math::SVector<double, 6u>, double>, double, 6u>, double> >(ROOT::Math::VecExpr<ROOT::Math::BinaryOp<ROOT::Math::AddOp<double>, ROOT::Math::VecExpr<ROOT::Math::BinaryOp<ROOT::Math::AddOp<double>, ROOT::Math::SVector<double, 6u>, ROOT::Math::VecExpr<ROOT::Math::BinaryOpCopyL<ROOT::Math::MulOp<double>, ROOT::Math::Constant<double>, ROOT::Math::SVector<double, 6u>, double>, double, 6u>, double>, double, 6u>, ROOT::Math::VecExpr<ROOT::Math::BinaryOpCopyL<ROOT::Math::MulOp<double>, ROOT::Math::Constant<double>, ROOT::Math::SVector<double, 6u>, double>, double, 6u>, double>, double, 6u> const&) [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libTrackPropagationRungeKutta.so
#5  0x0000400042cf7248 in RKOneCashKarpStep<double, 6>::operator()(double, ROOT::Math::SVector<double, 6u> const&, RKDerivative<double, 6> const&, RKDistance<double, 6> const&, double) [clone .constprop.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libTrackPropagationRungeKutta.so
#6  0x0000400042cf7c4c in RKAdaptiveSolver<double, RKOneCashKarpStep, 6>::operator()(double, ROOT::Math::SVector<double, 6u> const&, double, RKDerivative<double, 6> const&, RKDistance<double, 6> const&, float) [clone .constprop.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libTrackPropagationRungeKutta.so
#7  0x0000400042cfa218 in RKPropagatorInS::propagateWithPath(FreeTrajectoryState const&, Plane const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libTrackPropagationRungeKutta.so
#8  0x0000400042cf6114 in Propagator::propagateWithPath(TrajectoryStateOnSurface const&, Plane const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libTrackPropagationRungeKutta.so
#9  0x0000400042cc6934 in PropagatorWithMaterial::propagateWithPath(TrajectoryStateOnSurface const&, Plane const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libTrackingToolsMaterialEffects.so
#10 0x00004000213bbff4 in Propagator::propagateWithPath(TrajectoryStateOnSurface const&, Surface const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libTrackingToolsGeomPropagators.so
#11 0x0000400063dbdc44 in KFTrajectoryFitter::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libTrackingToolsTrackFitters.so
#12 0x0000400063d4d1b4 in (anonymous namespace)::KFFittingSmoother::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginTrackingToolsTrackFittersPlugins.so
#13 0x0000400063d46d78 in (anonymous namespace)::FlexibleKFFittingSmoother::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginTrackingToolsTrackFittersPlugins.so
#14 0x000040008aa27dbc in TrackProducerAlgorithm<reco::Track>::buildTrack(TrajectoryFitter const*, Propagator const*, std::vector<AlgoProductTraits<reco::Track>::AlgoProduct, std::allocator<AlgoProductTraits<reco::Track>::AlgoProduct> >&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >&, TrajectoryStateOnSurface&, TrajectorySeed const&, float, reco::BeamSpot const&, edm::RefToBase<TrajectorySeed>, int, signed char) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerTrackProducer.so
#15 0x000040008a8ff61c in TrackProducerAlgorithm<reco::Track>::runWithCandidate(TrackingGeometry const*, MagneticField const*, std::vector<TrackCandidate, std::allocator<TrackCandidate> > const&, TrajectoryFitter const*, Propagator const*, TransientTrackingRecHitBuilder const*, reco::BeamSpot const&, std::vector<AlgoProductTraits<reco::Track>::AlgoProduct, std::allocator<AlgoProductTraits<reco::Track>::AlgoProduct> >&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerTrackProducerPlugins.so
#16 0x000040008a8fddac in TrackProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerTrackProducerPlugins.so
#17 0x0000400016b04ccc in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#18 0x0000400016aeaeb0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 3 (Thread 0x400069cd9260 (LWP 950350) "cmsRun"):
#3  0x000040001dca7cc8 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x000040006585b300 in void tbb::detail::d2::concurrent_queue<mkfit::MkFinder*, tbb::detail::d1::cache_aligned_allocator<mkfit::MkFinder*> >::internal_push<mkfit::MkFinder* const&>(mkfit::MkFinder* const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#6  0x0000400065865128 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#7  0x000040006586758c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#8  0x0000400018548774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000e8555800, this=0x400019423e80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400019423e80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#11 0x0000400065869918 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#12 0x0000400065869f6c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#13 0x0000400018548774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000e8556800, this=0x400019423e80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400019423e80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#15 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#16 0x000040006585d1bc in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#17 0x0000400018548774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x400069cd7d40, this=0x400019423e80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#18 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400019423e80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#19 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#20 0x000040006585d858 in void tbb::detail::d2::parallel_for_each<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}>(__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, __gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1} const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#21 0x000040006585f4a0 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#22 0x00004000657eb940 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCMS.so
#23 0x00004000657592c0 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#24 0x00004000185416e4 in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/arena.cpp:757
#25 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#26 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/arena.cpp:758
#27 0x000040006575e3c4 in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#28 0x0000400016af05c0 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#29 0x0000400016aea280 in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 1 (Thread 0x4000186ecf30 (LWP 948590) "cmsRun"):
#2  0x000040001dca33f8 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x0000400065879d5c in mkfit::helixAtRFromIterativeCCS(Matriplex::Matriplex<float, 6, 1, 8> const&, Matriplex::Matriplex<int, 1, 1, 8> const&, Matriplex::Matriplex<float, 1, 1, 8> const&, Matriplex::Matriplex<float, 6, 1, 8>&, Matriplex::Matriplex<float, 6, 6, 8>&, Matriplex::Matriplex<int, 1, 1, 8>&, int, mkfit::PropagationFlags const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#5  0x000040006587b6f8 in mkfit::propagateHelixToRMPlex(Matriplex::MatriplexSym<float, 6, 8> const&, Matriplex::Matriplex<float, 6, 1, 8> const&, Matriplex::Matriplex<int, 1, 1, 8> const&, Matriplex::Matriplex<float, 1, 1, 8> const&, Matriplex::MatriplexSym<float, 6, 8>&, Matriplex::Matriplex<float, 6, 1, 8>&, Matriplex::Matriplex<int, 1, 1, 8>&, int, mkfit::PropagationFlags const&, Matriplex::Matriplex<int, 1, 1, 8> const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#6  0x0000400065828590 in mkfit::MkBase::propagateTracksToR(float, int, mkfit::PropagationFlags const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#7  0x0000400065864d60 in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#8  0x0000400065865100 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#9  0x000040006586758c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#10 0x0000400018548774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000e82f7f00, this=0x400019423d80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#11 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400019423d80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#12 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#13 0x0000400065869918 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#14 0x0000400065869f6c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#15 0x0000400018548774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000e8308800, this=0x400019423d80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#16 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400019423d80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#17 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#18 0x000040006585d1bc in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#19 0x0000400018548774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0xffffe50f0dc0, this=0x400019423d80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#20 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400019423d80) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#21 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#22 0x000040006585d858 in void tbb::detail::d2::parallel_for_each<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}>(__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, __gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1} const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#23 0x000040006585f4a0 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#24 0x00004000657eb940 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCMS.so
#25 0x00004000657592c0 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#26 0x00004000185416e4 in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/arena.cpp:757
#27 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#28 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins_a/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/arena.cpp:758
#29 0x000040006575e3c4 in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#30 0x0000400016af05c0 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#31 0x0000400016aea280 in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02790/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Current Modules:
Module: MkFitProducer:highPtTripletStepTrackCandidatesMkFit (crashed)
Module: MkFitProducer:highPtTripletStepTrackCandidatesMkFit
Module: TrackProducer:lowPtQuadStepTracks
Module: MkFitOutputConverter:highPtTripletStepTrackCandidates

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_aarch64_gcc11/CMSSW_13_2_X_2023-06-22-2300/pyRelValMatrixLogs/run/136.813_RunZeroBias2017D/step3_RunZeroBias2017D.log#/

[cmsbuild]

A new Issue was created by @makortel Matti Kortelainen.

@Dr15Jones, @perrotta, @dpiparo, @rappoccio, @makortel, @smuzaffar can you please review it and eventually sign/assign? Thanks.

cms-bot commands are listed here

[makortel]

assign reconstruction

[cmsbuild]

New categories assigned: reconstruction

@mandrenguyen,@clacaputo you have been requested to review this Pull request/Issue and eventually sign? Thanks

[makortel]

FYI @cms-sw/tracking-pog-l2

(I'm mostly documenting the stack trace)

[slava77]

type tracking

[slava77]

@makortel please remind how to access the ARM machine for testing. It would be nice to include the instructions with all non-standard arch runtime issues.

[VinInn]

nested tbb parallel for: usually just overhead as there are no free threads in a standard CMSSW job. Still, have we ever run MKFit on a 256 core AMPERE machine with 256 threads and one stream?

[makortel]

please remind how to access the ARM machine for testing.

There is lxplus8-arm. I fully expect the segfault to not be easily reproducible.

[VinInn]

maybe running a tracking only wf (maybe just one iteration) with one stream and 8 (or more) threads (if cores available)

[VinInn]

One gets 10 core on lxplus8-arm ( It's a (partion of a ) Neoverse N1)

[dan131riley]

One gets 10 core on lxplus8-arm ( It's a (partion of a ) Neoverse N1)

Probably won’t reproduce. The Cavium cores at the openlab seem to be much more aggressive at taking advantage of the relaxed ARM memory model, uncovering bugs that can’t be reproduced on other ARM implementations where the memory model implementation isn’t as relaxed as the spec allows (I’ve tried on an Apple M1).

[osschar]

Hmmh, the actual crash seems to be in void tbb::detail::d2::concurrent_queue<mkfit::MkFinder*, tbb::detail::d1::cache_aligned_allocator<mkfit::MkFinder*> >::internal_push<mkfit::MkFinder* const&>(mkfit::MkFinder* const&), apparently in a deleter here: https://github.com/cms-sw/cmssw/blob/master/RecoTracker/MkFitCore/src/Pool.h#L37 happening via release here: https://github.com/cms-sw/cmssw/blob/master/RecoTracker/MkFitCore/src/MkBuilder.cc#L950

There is a to-do note in MkFitProuducer where pools of shared helper objects are populated: https://github.com/cms-sw/cmssw/blob/master/RecoTracker/MkFit/plugins/MkFitProducer.cc#L100 but nothing seems obviously wrong in the code (Pool.h / MkBuilder.cc) that would give wight to the above comment :)

What's the priority on this?

[makortel]

Hmmh, the actual crash seems to be in void tbb::detail::d2::concurrent_queue<mkfit::MkFinder*, tbb::detail::d1::cache_aligned_allocator<mkfit::MkFinder*> >::internal_push<mkfit::MkFinder* const&>(mkfit::MkFinder* const&),

We had updated oneTBB to 2021.9.0 a few IBs before, and there was another crash elsewhere in TBB code (https://github.com/cms-sw/cmssw/issues/42093, on x86).

What's the priority on this?

As an isolated crash on ARM not particularly urgent (although we just observed another occurrance). I'm honestly now suspecting the TBB update, but would like to see a bit more evidence before rolling it back.

[makortel]

I'm honestly now suspecting the TBB update, but would like to see a bit more evidence before rolling it back.

We are going to roll back TBB to 2021.8.0 for 13_2_0_pre3, and I opened also an issue in https://github.com/oneapi-src/oneTBB/issues/1139 . Let's see if this reproduces after the rollback, or after we re-update to 2021.9.0 in 13_3_X.

[aandvalenzuela]

Just for the record on the related crashes, although probably not needed anymore after the revert. There is a new occurrence in workflow 136.874 (step 2) in CMSSW_13_2_X_2023-06-26-2300 on el8_aarch64_gcc1:

Thread 1 (Thread 0x400032efd060 (LWP 2717126) "cmsRun"):
#0  0x00004000333da76c in poll () from /lib64/libc.so.6
#1  0x0000400039d6e0ec in full_read.constprop () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_2_X_2023-06-26-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#2  0x0000400039d3c574 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_2_X_2023-06-26-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  0x0000400039d37d58 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_2_X_2023-06-26-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x000040008006b318 in void tbb::detail::d2::concurrent_queue<mkfit::MkFinder*, tbb::detail::d1::cache_aligned_allocator<mkfit::MkFinder*> >::internal_push<mkfit::MkFinder* const&>(mkfit::MkFinder* const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#6  0x0000400080075128 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#7  0x000040008007758c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#8  0x0000400032d58774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x40008f545000, this=0x400033c33d80) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#9  tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400033c33d80) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#11 0x0000400080079918 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#12 0x0000400080079f6c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#13 0x0000400032d58774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x400033c3c400, this=0x400033c33d80) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400033c33d80) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#15 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#16 0x000040008006d1bc in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#17 0x0000400032d58774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0xfffff1ae8200, this=0x400033c33d80) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#18 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400033c33d80) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#19 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#20 0x000040008006d858 in void tbb::detail::d2::parallel_for_each<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}>(__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, __gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1} const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#21 0x000040008006f4a0 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#22 0x000040007fffb940 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCMS.so
#23 0x000040007ff69390 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#24 0x0000400032d516e4 in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/arena.cpp:757
#25 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#26 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/arena.cpp:758
#27 0x000040007ff6e4ac in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#28 0x00004000313003d0 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#29 0x00004000312fa090 in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#30 0x0000400031286e08 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#31 0x0000400031287570 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#32 0x0000400031825160 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libFWCoreConcurrency.so
#33 0x0000400032d58774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000ec159300, this=0x400033c33d80) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#34 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400033c33d80) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#35 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#36 0x00004000311ff4d4 in edm::FinalWaitingTask::wait() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#37 0x000040003120c698 in edm::EventProcessor::processRuns() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#38 0x000040003120cc50 in edm::EventProcessor::runToCompletion() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02791/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_2_X_2023-06-25-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#39 0x000000000040b9c8 in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#40 0x0000400032d510a8 in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins_b/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-d9b6b79f96fc04849cfc544d9852057d/tbb-v2021.9.0/src/tbb/arena.cpp:688
#41 0x000000000040f74c in main::{lambda()#1}::operator()() const ()
#42 0x0000000000406fc4 in main ()

Current Modules:

Module: MkFitProducer:detachedTripletStepTrackCandidatesMkFit (crashed)
Module: SeedCreatorFromRegionConsecutiveHitsTripletOnlyEDProducer:detachedTripletStepSeeds
Module: HitPairEDProducer:pixelPairStepHitDoubletsB
Module: TrackTfClassifier:lowPtTripletStep

A fatal system signal has occurred: segmentation violation

[makortel]

Another occurrence in CMSSW_13_2_X_2023-06-28-2300 on el8_aarch64_gcc11 https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_aarch64_gcc11/CMSSW_13_2_X_2023-06-28-2300/pyRelValMatrixLogs/run/136.8861_RunEGamma2018Dml1/step3_RunEGamma2018Dml1.log#/

[makortel]

A similar crash occurred in CMSSW_13_3_X_2023-08-08-2300 on el8_aarch64_gcc11 workflow 136.804 step 3. This time there is also a printout from TCMalloc

src/tcmalloc.cc:333] Attempt to free invalid pointer 0x400033a48ae8 

Thread 5 (Thread 0x400081db9c80 (LWP 1396727) "cmsRun"):
#2  0x0000400037823448 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x000040005b690830 in Eigen::internal::gebp_kernel<float, float, long, Eigen::internal::blas_data_mapper<float, long, 0, 0, 1>, 12, 4, false, false>::operator()(Eigen::internal::blas_data_mapper<float, long, 0, 0, 1> const&, float const*, float const*, long, long, long, float, long, long, long, long) [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#5  0x000040005b691d4c in Eigen::internal::TensorContractionKernel<float, float, float, long, Eigen::internal::blas_data_mapper<float, long, 0, 0, 1>, Eigen::internal::TensorContractionInputMapper<float, long, 1, Eigen::TensorEvaluator<Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, Eigen::ThreadPoolDevice>, Eigen::array<long, 1ul>, Eigen::array<long, 1ul>, 4, true, false, 0, Eigen::MakePointer>, Eigen::internal::TensorContractionInputMapper<float, long, 0, Eigen::TensorEvaluator<Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, Eigen::ThreadPoolDevice>, Eigen::array<long, 1ul>, Eigen::array<long, 1ul>, 4, true, false, 0, Eigen::MakePointer> >::invoke(Eigen::internal::blas_data_mapper<float, long, 0, 0, 1> const&, float* const&, float* const&, long, long, long, float, float) [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#6  0x000040005b69cb98 in void Eigen::TensorContractionEvaluatorBase<Eigen::TensorEvaluator<Eigen::TensorContractionOp<Eigen::array<Eigen::IndexPair<long>, 1ul> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, tensorflow::LaunchFusedMatMulOp<Eigen::ThreadPoolDevice, float>::OutputKernelWrapper const> const, Eigen::ThreadPoolDevice> >::evalGemmPartial<true, true, false, 0, true>(float*, long, long, int) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#7  0x000040005b6b812c in Eigen::internal::TensorExecutor<Eigen::TensorAssignOp<Eigen::TensorMap<Eigen::Tensor<float, 2, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorContractionOp<Eigen::array<Eigen::IndexPair<long>, 1ul> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, tensorflow::LaunchFusedMatMulOp<Eigen::ThreadPoolDevice, float>::OutputKernelWrapper const> const> const, Eigen::ThreadPoolDevice, true, (Eigen::internal::TiledEvaluation)0>::run(Eigen::TensorAssignOp<Eigen::TensorMap<Eigen::Tensor<float, 2, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorContractionOp<Eigen::array<Eigen::IndexPair<long>, 1ul> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, tensorflow::LaunchFusedMatMulOp<Eigen::ThreadPoolDevice, float>::OutputKernelWrapper const> const> const&, Eigen::ThreadPoolDevice const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#8  0x000040005b6b84d8 in tensorflow::LaunchFusedMatMulOp<Eigen::ThreadPoolDevice, float>::operator()(tensorflow::OpKernelContext*, tensorflow::Tensor const&, tensorflow::Tensor const&, Eigen::array<Eigen::IndexPair<long>, 1ul> const&, tensorflow::FusedComputationType, tensorflow::FusedComputationArgs const&, tensorflow::Tensor*, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#9  0x000040005b6b8844 in tensorflow::FusedMatMulOp<Eigen::ThreadPoolDevice, float>::Compute(tensorflow::OpKernelContext*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#10 0x0000400078ddb574 in tensorflow::ThreadPoolDevice::Compute(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#11 0x0000400078d722c8 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ProcessInline(tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*, long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#12 0x0000400078d72d64 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::Process(tensorflow::SimplePropagatorState::TaggedNode, long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#13 0x00004000590a7ba4 in tsl::thread::ThreadPool::Schedule(std::function<void ()>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#14 0x000040005e630b5c in std::_Function_handler<void (std::function<void ()>), tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&)::{lambda(std::function<void ()>)#6}>::_M_invoke(std::_Any_data const&, std::function<void ()>&&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#15 0x0000400078d64114 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ScheduleReady(absl::lts_20220623::InlinedVector<tensorflow::SimplePropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::SimplePropagatorState::TaggedNode> >*, tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#16 0x0000400078d66514 in tensorflow::(anonymous namespace)::ExecutorImpl::RunAsync(tensorflow::Executor::Args const&, std::function<void (tsl::Status const&)>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#17 0x000040005e63ec20 in tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#18 0x000040005e640db4 in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#19 0x0000400052e1baf8 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/lib/el8_aarch64_gcc11/libPhysicsToolsTensorFlow.so
#20 0x0000400052e1bcfc in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolInterface*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/lib/el8_aarch64_gcc11/libPhysicsToolsTensorFlow.so
#21 0x000040007de23d90 in (anonymous namespace)::TfDnn::operator()(std::vector<reco::Track, std::allocator<reco::Track> > const&, reco::BeamSpot const&, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const&) const [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerFinalTrackSelectorsPlugins.so
#22 0x000040007de252cc in TrackMVAClassifier<(anonymous namespace)::TfDnn, void>::computeMVA(std::vector<reco::Track, std::allocator<reco::Track> > const&, reco::BeamSpot const&, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const&, std::vector<std::pair<float, bool>, std::allocator<std::pair<float, bool> > >&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerFinalTrackSelectorsPlugins.so
#23 0x000040007dec40c0 in TrackMVAClassifierBase::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerFinalTrackSelectors.so
#24 0x0000400033973a8c in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#25 0x0000400033959ba0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 4 (Thread 0x4000813a9c80 (LWP 1396726) "cmsRun"):
#3  0x0000400037827d58 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x0000400035a76274 in raise () from /lib64/libc.so.6
#6  0x0000400035a60a2c in abort () from /lib64/libc.so.6
#7  0x00004000354b1550 in tcmalloc::Log (mode=mode@entry=tcmalloc::kCrash, filename=filename@entry=0x4000354be820 "src/tcmalloc.cc", line=line@entry=333, a=..., b=..., c=..., d=...) at src/internal_logging.cc:118
#8  0x00004000354ade30 in (anonymous namespace)::InvalidFree (ptr=<optimized out>) at src/internal_logging.h:72
#9  0x000040007c42184c in std::vector<std::vector<mkfit::TrackCand, std::allocator<mkfit::TrackCand> >, std::allocator<std::vector<mkfit::TrackCand, std::allocator<mkfit::TrackCand> > > >::~vector() [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#10 0x000040007c464f70 in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#11 0x000040007c465100 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#12 0x000040007c46758c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#13 0x0000400035288774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x400093475f00, this=0x400036363e00) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#14 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400036363e00) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#15 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#16 0x000040007c469918 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#17 0x000040007c469f6c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#18 0x0000400035288774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x400093476200, this=0x400036363e00) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#19 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400036363e00) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#20 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#21 0x000040007c45d1bc in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#22 0x0000400035288774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000813a8780, this=0x400036363e00) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#23 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400036363e00) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#24 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#25 0x000040007c45d858 in void tbb::detail::d2::parallel_for_each<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}>(__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, __gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1} const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#26 0x000040007c45f4a0 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#27 0x000040007c3eb940 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCMS.so
#28 0x000040007c359730 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#29 0x00004000352816e4 in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/arena.cpp:757
#30 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#31 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/arena.cpp:758
#32 0x000040007c35e89c in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#33 0x000040003395f2b0 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#34 0x0000400033958f70 in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 3 (Thread 0x400080999c80 (LWP 1396725) "cmsRun"):
#2  0x0000400037823448 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x000040007c479c20 in mkfit::helixAtRFromIterativeCCS(Matriplex::Matriplex<float, 6, 1, 8> const&, Matriplex::Matriplex<int, 1, 1, 8> const&, Matriplex::Matriplex<float, 1, 1, 8> const&, Matriplex::Matriplex<float, 6, 1, 8>&, Matriplex::Matriplex<float, 6, 6, 8>&, Matriplex::Matriplex<int, 1, 1, 8>&, int, mkfit::PropagationFlags const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#5  0x000040007c47b6f8 in mkfit::propagateHelixToRMPlex(Matriplex::MatriplexSym<float, 6, 8> const&, Matriplex::Matriplex<float, 6, 1, 8> const&, Matriplex::Matriplex<int, 1, 1, 8> const&, Matriplex::Matriplex<float, 1, 1, 8> const&, Matriplex::MatriplexSym<float, 6, 8>&, Matriplex::Matriplex<float, 6, 1, 8>&, Matriplex::Matriplex<int, 1, 1, 8>&, int, mkfit::PropagationFlags const&, Matriplex::Matriplex<int, 1, 1, 8> const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#6  0x000040007c4593a0 in mkfit::kalmanPropagateAndUpdate(Matriplex::MatriplexSym<float, 6, 8> const&, Matriplex::Matriplex<float, 6, 1, 8> const&, Matriplex::Matriplex<int, 1, 1, 8>&, Matriplex::MatriplexSym<float, 3, 8> const&, Matriplex::Matriplex<float, 3, 1, 8> const&, Matriplex::MatriplexSym<float, 6, 8>&, Matriplex::Matriplex<float, 6, 1, 8>&, Matriplex::Matriplex<int, 1, 1, 8>&, int, mkfit::PropagationFlags const&, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#7  0x000040007c471274 in mkfit::MkFinder::updateWithLoadedHit(int, mkfit::FindingFoos const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#8  0x000040007c464e3c in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#9  0x000040007c465100 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#10 0x000040007c46758c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#11 0x0000400035288774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x400093472300, this=0x400036363e80) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#12 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400036363e80) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#13 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#14 0x000040007c469918 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#15 0x000040007c469f6c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#16 0x0000400035288774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x400093473100, this=0x400036363e80) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#17 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400036363e80) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#18 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#19 0x000040007c45d1bc in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#20 0x0000400035288774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x400080998780, this=0x400036363e80) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#21 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400036363e80) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#22 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#23 0x000040007c45d858 in void tbb::detail::d2::parallel_for_each<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}>(__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, __gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1} const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#24 0x000040007c45f4a0 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#25 0x000040007c3eb940 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCMS.so
#26 0x000040007c359730 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#27 0x00004000352816e4 in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/arena.cpp:757
#28 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#29 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/arena.cpp:758
#30 0x000040007c35e89c in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#31 0x000040003395f2b0 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 1 (Thread 0x40003545d080 (LWP 1392150) "cmsRun"):
#2  0x0000400037823448 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x0000400035934880 in expf32 () from /lib64/libm.so.6
#5  0x000040005b68f244 in tensorflow::LaunchFusedMatMulOp<Eigen::ThreadPoolDevice, float>::operator()(tensorflow::OpKernelContext*, tensorflow::Tensor const&, tensorflow::Tensor const&, Eigen::array<Eigen::IndexPair<long>, 1ul> const&, tensorflow::FusedComputationType, tensorflow::FusedComputationArgs const&, tensorflow::Tensor*, bool)::{lambda(auto:1)#1}::operator()<tensorflow::BiasAddOutputKernel<float, tensorflow::Elu> >(tensorflow::BiasAddOutputKernel<float, tensorflow::Elu>) const::{lambda(Eigen::internal::blas_data_mapper<float, long, 0, 0, 1> const&, Eigen::TensorContractionParams const&, long, long, long, long)#1}::operator()(Eigen::internal::blas_data_mapper<float, long, 0, 0, 1> const&, Eigen::TensorContractionParams const&, long, long, long, long) const [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#6  0x000040005b69cbdc in void Eigen::TensorContractionEvaluatorBase<Eigen::TensorEvaluator<Eigen::TensorContractionOp<Eigen::array<Eigen::IndexPair<long>, 1ul> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, tensorflow::LaunchFusedMatMulOp<Eigen::ThreadPoolDevice, float>::OutputKernelWrapper const> const, Eigen::ThreadPoolDevice> >::evalGemmPartial<true, true, false, 0, true>(float*, long, long, int) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#7  0x000040005b6b812c in Eigen::internal::TensorExecutor<Eigen::TensorAssignOp<Eigen::TensorMap<Eigen::Tensor<float, 2, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorContractionOp<Eigen::array<Eigen::IndexPair<long>, 1ul> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, tensorflow::LaunchFusedMatMulOp<Eigen::ThreadPoolDevice, float>::OutputKernelWrapper const> const> const, Eigen::ThreadPoolDevice, true, (Eigen::internal::TiledEvaluation)0>::run(Eigen::TensorAssignOp<Eigen::TensorMap<Eigen::Tensor<float, 2, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorContractionOp<Eigen::array<Eigen::IndexPair<long>, 1ul> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, Eigen::TensorMap<Eigen::Tensor<float const, 2, 1, long>, 16, Eigen::MakePointer> const, tensorflow::LaunchFusedMatMulOp<Eigen::ThreadPoolDevice, float>::OutputKernelWrapper const> const> const&, Eigen::ThreadPoolDevice const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#8  0x000040005b6b84d8 in tensorflow::LaunchFusedMatMulOp<Eigen::ThreadPoolDevice, float>::operator()(tensorflow::OpKernelContext*, tensorflow::Tensor const&, tensorflow::Tensor const&, Eigen::array<Eigen::IndexPair<long>, 1ul> const&, tensorflow::FusedComputationType, tensorflow::FusedComputationArgs const&, tensorflow::Tensor*, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#9  0x000040005b6b8844 in tensorflow::FusedMatMulOp<Eigen::ThreadPoolDevice, float>::Compute(tensorflow::OpKernelContext*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#10 0x0000400078ddb574 in tensorflow::ThreadPoolDevice::Compute(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#11 0x0000400078d722c8 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ProcessInline(tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*, long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#12 0x0000400078d72d64 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::Process(tensorflow::SimplePropagatorState::TaggedNode, long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#13 0x00004000590a7ba4 in tsl::thread::ThreadPool::Schedule(std::function<void ()>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#14 0x000040005e630b5c in std::_Function_handler<void (std::function<void ()>), tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&)::{lambda(std::function<void ()>)#6}>::_M_invoke(std::_Any_data const&, std::function<void ()>&&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#15 0x0000400078d64114 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ScheduleReady(absl::lts_20220623::InlinedVector<tensorflow::SimplePropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::SimplePropagatorState::TaggedNode> >*, tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#16 0x0000400078d66514 in tensorflow::(anonymous namespace)::ExecutorImpl::RunAsync(tensorflow::Executor::Args const&, std::function<void (tsl::Status const&)>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#17 0x000040005e63ec20 in tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#18 0x000040005e640db4 in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#19 0x0000400052e1baf8 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/lib/el8_aarch64_gcc11/libPhysicsToolsTensorFlow.so
#20 0x0000400052e1bcfc in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolInterface*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw-patch/CMSSW_13_3_X_2023-08-08-2300/lib/el8_aarch64_gcc11/libPhysicsToolsTensorFlow.so
#21 0x000040007c349a04 in MkFitOutputConverter::computeDNNs(std::vector<TrackCandidate, std::allocator<TrackCandidate> > const&, std::vector<TrajectoryStateOnSurface, std::allocator<TrajectoryStateOnSurface> > const&, reco::BeamSpot const*, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const*, tensorflow::Session const*, std::vector<float, std::allocator<float> > const&, bool) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#22 0x000040007c34b760 in MkFitOutputConverter::convertCandidates(MkFitOutputWrapper const&, mkfit::EventOfHits const&, MkFitClusterIndexToHit const&, MkFitClusterIndexToHit const&, edm::View<TrajectorySeed> const&, MagneticField const&, Propagator const&, Propagator const&, TkClonerImpl const&, std::vector<DetLayer const*, std::allocator<DetLayer const*> > const&, std::vector<mkfit::Track, std::allocator<mkfit::Track> > const&, reco::BeamSpot const*, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const*, tensorflow::Session const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#23 0x000040007c34d6e0 in MkFitOutputConverter::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#24 0x000040003395f2b0 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#25 0x0000400033958f70 in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#26 0x00004000338e6638 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#27 0x00004000338e6da0 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02797/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-08-07-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Current Modules:
Module: MkFitProducer:highPtTripletStepTrackCandidatesMkFit (crashed)
Module: TrackTfClassifier:lowPtQuadStep
Module: MkFitProducer:highPtTripletStepTrackCandidatesMkFit
Module: MkFitOutputConverter:highPtTripletStepTrackCandidates

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_aarch64_gcc11/CMSSW_13_3_X_2023-08-08-2300/pyRelValMatrixLogs/run/136.804_RunHLTPhy2017D/step3_RunHLTPhy2017D.log#/

[smuzaffar]

@makortel , we are using couple of years old version of tcmalloc ( https://github.com/gperftools/gperftools/tree/gperftools-2.9.1 ) which is provided via gperftools. There is a new version https://github.com/gperftools/gperftools/tree/gperftools-2.10.80 available ( testing via https://github.com/cms-sw/cmsdist/pull/8635 ) .

[smuzaffar]

ah the newer version of gpreftools still contains two years old tcmalloc ( https://github.com/gperftools/gperftools/tree/gperftools-2.10.80/src/google )

[slava77]

the last crash stack backtrace is different; one of the last points is std::vector<std::vector<mkfit::TrackCand> >::~vector()

[slava77]

the last crash stack backtrace is different; one of the last points is std::vector<std::vector<mkfit::TrackCand> >::~vector()

I don't think there's anything exotic (like concurrency-related) in the mkfit::TrackCand type and the vector is a local variable in void MkBuilder::find_tracks_in_layers https://github.com/cms-sw/cmssw/blob/2e956e433ecee876155e346ca9c9870cab7ec3b7/RecoTracker/MkFitCore/src/MkBuilder.cc#L977

[makortel]

Another one in CMSSW_13_3_X_2023-09-22-2300 on el8_aarch64_gcc11

Thread 5 (Thread 0x400092a09260 (LWP 1628690) "cmsRun"):
#2  0x00004000376c3448 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x000040005c6dafa4 in CellularAutomaton::createAndConnectCells(std::vector<HitDoublets const*, std::allocator<HitDoublets const*> > const&, TrackingRegion const&, CACut const&, CACut const&, float) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerPixelSeeding.so
#5  0x000040005c6d5ea8 in CAHitQuadrupletGenerator::hitNtuplets(IntermediateHitDoublets const&, std::vector<OrderedHitSeeds, std::allocator<OrderedHitSeeds> >&, SeedingLayerSetsHits const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerPixelSeeding.so
#6  0x00004000de051468 in CAHitNtupletEDProducerT<CAHitQuadrupletGenerator>::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/pluginRecoPixelVertexingPixelTripletsPlugins.so
#7  0x0000400030533acc in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#8  0x0000400030519be0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#9  0x00004000304a6598 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#10 0x00004000304a6e00 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 4 (Thread 0x40008f8d9260 (LWP 1628689) "cmsRun"):
#4  0x0000400084130ec0 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ProcessInline(tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*, long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#5  0x0000400084132d64 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::Process(tensorflow::SimplePropagatorState::TaggedNode, long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#6  0x0000400064467ba4 in tsl::thread::ThreadPool::Schedule(std::function<void ()>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#7  0x00004000699f0b6c in std::_Function_handler<void (std::function<void ()>), tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&)::{lambda(std::function<void ()>)#6}>::_M_invoke(std::_Any_data const&, std::function<void ()>&&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#8  0x0000400084124114 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ScheduleReady(absl::lts_20220623::InlinedVector<tensorflow::SimplePropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::SimplePropagatorState::TaggedNode> >*, tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#9  0x0000400084126514 in tensorflow::(anonymous namespace)::ExecutorImpl::RunAsync(tensorflow::Executor::Args const&, std::function<void (tsl::Status const&)>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_framework.so.2
#10 0x00004000699fec30 in tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#11 0x0000400069a00dc4 in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/external/el8_aarch64_gcc11/lib/libtensorflow_cc.so.2
#12 0x000040005e1dbaf8 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libPhysicsToolsTensorFlow.so
#13 0x000040005e1dbcfc in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolInterface*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libPhysicsToolsTensorFlow.so
#14 0x000040008c101ab0 in (anonymous namespace)::TfDnn::operator()(std::vector<reco::Track, std::allocator<reco::Track> > const&, reco::BeamSpot const&, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const&) const [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerFinalTrackSelectorsPlugins.so
#15 0x000040008c102fec in TrackMVAClassifier<(anonymous namespace)::TfDnn, void>::computeMVA(std::vector<reco::Track, std::allocator<reco::Track> > const&, reco::BeamSpot const&, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const&, std::vector<std::pair<float, bool>, std::allocator<std::pair<float, bool> > >&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerFinalTrackSelectorsPlugins.so
#16 0x000040008c1935b8 in TrackMVAClassifierBase::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerFinalTrackSelectors.so
#17 0x0000400030533acc in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#18 0x0000400030519be0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#19 0x00004000304a6598 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#20 0x00004000304a6e00 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 3 (Thread 0x40008eec9260 (LWP 1628688) "cmsRun"):
#2  0x00004000376c3448 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x000040005c6dd048 in CellularAutomaton::findTriplets(std::vector<HitDoublets const*, std::allocator<HitDoublets const*> > const&, std::vector<std::vector<unsigned int, std::allocator<unsigned int> >, std::allocator<std::vector<unsigned int, std::allocator<unsigned int> > > >&, TrackingRegion const&, CACut const&, CACut const&, float) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerPixelSeeding.so
#5  0x000040005c6e01d0 in CAHitTripletGenerator::hitNtuplets(IntermediateHitDoublets const&, std::vector<OrderedHitSeeds, std::allocator<OrderedHitSeeds> >&, SeedingLayerSetsHits const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerPixelSeeding.so
#6  0x00004000de050e68 in CAHitNtupletEDProducerT<CAHitTripletGenerator>::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/pluginRecoPixelVertexingPixelTripletsPlugins.so
#7  0x0000400030533acc in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#8  0x0000400030519be0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#9  0x00004000304a6598 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#10 0x00004000304a6e00 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 1 (Thread 0x40003211d2f0 (LWP 1626775) "cmsRun"):
#3  0x00004000376c7d58 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x000040008a75f300 in mkfit::MkBuilder::find_tracks_handle_missed_layers(mkfit::MkFinder*, mkfit::LayerInfo const&, std::vector<std::vector<mkfit::TrackCand, std::allocator<mkfit::TrackCand> >, std::allocator<std::vector<mkfit::TrackCand, std::allocator<mkfit::TrackCand> > > >&, std::vector<std::pair<int, int>, std::allocator<std::pair<int, int> > > const&, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#6  0x000040008a764d98 in mkfit::MkBuilder::find_tracks_in_layers(mkfit::CandCloner&, mkfit::MkFinder*, mkfit::SteeringParams::IterationType_e, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#7  0x000040008a765100 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}::operator()(tbb::detail::d1::blocked_range<int> const&) const [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#8  0x000040008a76758c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<int>, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}::operator()(int) const::{lambda(tbb::detail::d1::blocked_range<int> const&)#1}, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#9  0x0000400031f78774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000ecfaa800, this=0x400032e53d80) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#10 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400032e53d80) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#11 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#12 0x000040008a769918 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#13 0x000040008a769f6c in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d2::parallel_for_body_wrapper<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#14 0x0000400031f78774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000ecfaae00, this=0x400032e53d80) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#15 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400032e53d80) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#16 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#17 0x000040008a75d1bc in tbb::detail::d2::for_each_root_task<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}, int, std::random_access_iterator_tag>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#18 0x0000400031f78774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0xffffd29c24c0, this=0x400032e53d80) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#19 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400032e53d80) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#20 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#21 0x000040008a75d858 in void tbb::detail::d2::parallel_for_each<__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1}>(__gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, __gnu_cxx::__normal_iterator<int const*, std::vector<int, std::allocator<int> > >, mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e)::{lambda(int)#1} const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#22 0x000040008a75f4a0 in mkfit::MkBuilder::findTracksCloneEngine(mkfit::SteeringParams::IterationType_e) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCore.so
#23 0x000040008a6eb940 in mkfit::run_OneIteration(mkfit::TrackerInfo const&, mkfit::IterationConfig const&, mkfit::EventOfHits const&, std::vector<std::vector<bool, std::allocator<bool> > const*, std::allocator<std::vector<bool, std::allocator<bool> > const*> > const&, mkfit::MkBuilder&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, std::vector<mkfit::Track, std::allocator<mkfit::Track> >&, bool, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libRecoTrackerMkFitCMS.so
#24 0x000040008a6591b0 in tbb::detail::d1::task_arena_function<MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#25 0x0000400031f716e4 in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/arena.cpp:757
#26 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#27 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins_a/workspace/auto-builds/CMSSW_13_3_0_pre2-el8_aarch64_gcc11/build/CMSSW_13_3_0_pre2-build/BUILD/el8_aarch64_gcc11/external/tbb/v2021.9.0-a460e1ec4c4aa5a9523aaf1bd8c1ee76/tbb-v2021.9.0/src/tbb/arena.cpp:758
#28 0x000040008a65e31c in MkFitProducer::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/pluginRecoTrackerMkFitPlugins.so
#29 0x000040003051f2f0 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#30 0x0000400030518fb0 in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#31 0x00004000304a6598 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#32 0x00004000304a6e00 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02803/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-22-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Current Modules:
Module: MkFitProducer:detachedQuadStepTrackCandidatesMkFit (crashed)
Module: CAHitQuadrupletEDProducer:detachedQuadStepHitQuadruplets
Module: TrackTfClassifier:lowPtTripletStep
Module: CAHitTripletEDProducer:lowPtTripletStepHitTriplets

https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_aarch64_gcc11/CMSSW_13_3_X_2023-09-22-2300/pyRelValMatrixLogs/run/140.024_RunBTagMu2022B/step3_RunBTagMu2022B.log#/

[VinInn]

One could try to replace the tbb container with a simpler one (maybe less flexible) I made a prototype for some other application some time ago https://github.com/VinInn/ctest/blob/master/AtomicPool.cpp (tested on arm)

[aandvalenzuela]

Hello,

RelVal 136.865 segfaulted in CMSSW_14_1_X_2024-05-23-2300 IBs for aarch64 with a similar crash on MkFitOutputConverter:

Fri May 24 07:49:50 CEST 2024
Thread 8 (Thread 0x40008bdc9220 (LWP 3923878) "cmsRun"):
#0  0x0000400011c1dc58 in pthread_cond_wait@@GLIBC_2.17 () from /lib64/libpthread.so.0
#1  0x00004000463b085c in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#2  0x00004000463b0f50 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#3  0x00004000463ae40c in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#4  0x0000400050be112c in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_framework.so.2
#5  0x0000400011c178b8 in start_thread () from /lib64/libpthread.so.0
#6  0x0000400011c73afc in thread_start () from /lib64/libc.so.6
Thread 7 (Thread 0x40008bbb9220 (LWP 3923875) "cmsRun"):
#0  0x0000400011c1dc58 in pthread_cond_wait@@GLIBC_2.17 () from /lib64/libpthread.so.0
#1  0x00004000463b085c in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#2  0x00004000463b0f50 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#3  0x00004000463ae40c in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#4  0x0000400050be112c in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_framework.so.2
#5  0x0000400011c178b8 in start_thread () from /lib64/libpthread.so.0
#6  0x0000400011c73afc in thread_start () from /lib64/libc.so.6
Thread 6 (Thread 0x40008b9a9220 (LWP 3923874) "cmsRun"):
#0  0x0000400011c1dc58 in pthread_cond_wait@@GLIBC_2.17 () from /lib64/libpthread.so.0
#1  0x00004000463b085c in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#2  0x00004000463b0f50 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#3  0x00004000463ae40c in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#4  0x0000400050be112c in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_framework.so.2
#5  0x0000400011c178b8 in start_thread () from /lib64/libpthread.so.0
#6  0x0000400011c73afc in thread_start () from /lib64/libc.so.6
Thread 5 (Thread 0x40005fa09220 (LWP 3922356) "cmsRun"):
#0  0x0000400011d1a91c in poll () from /lib64/libc.so.6
#1  0x00004000187c3b5c in full_read.constprop () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#2  0x000040001877cde4 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#3  0x000040001877cf6c in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x000040005569c530 in Trajectory::~Trajectory() [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginTrackingToolsTrackFittersPlugins.so
#6  0x000040005569ca04 in (anonymous namespace)::KFFittingSmoother::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginTrackingToolsTrackFittersPlugins.so
#7  0x00004000556964f8 in (anonymous namespace)::FlexibleKFFittingSmoother::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginTrackingToolsTrackFittersPlugins.so
#8  0x0000400076a9bb60 in TrackProducerAlgorithm<reco::Track>::buildTrack(TrajectoryFitter const*, Propagator const*, std::vector<AlgoProductTraits<reco::Track>::AlgoProduct, std::allocator<AlgoProductTraits<reco::Track>::AlgoProduct> >&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >&, TrajectoryStateOnSurface&, TrajectorySeed const&, float, reco::BeamSpot const&, edm::RefToBase<TrajectorySeed>, int, signed char) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libRecoTrackerTrackProducer.so
#9  0x000040007697d5b0 in TrackProducerAlgorithm<reco::Track>::runWithCandidate(TrackingGeometry const*, MagneticField const*, std::vector<TrackCandidate, std::allocator<TrackCandidate> > const&, TrajectoryFitter const*, Propagator const*, TransientTrackingRecHitBuilder const*, reco::BeamSpot const&, std::vector<AlgoProductTraits<reco::Track>::AlgoProduct, std::allocator<AlgoProductTraits<reco::Track>::AlgoProduct> >&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerTrackProducerPlugins.so
#10 0x000040007697bf28 in TrackProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerTrackProducerPlugins.so
#11 0x000040000fc274e4 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#12 0x000040000fc0e2f0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#13 0x000040000fb9c304 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#14 0x000040000fb9f050 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#15 0x00004000101350dc in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreConcurrency.so
#16 0x0000400011682dfc in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x4000c5282f00, waiter=<synthetic pointer>..., this=0x400012573f00) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#17 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (waiter=<synthetic pointer>..., t=0x0, this=0x400012573f00) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#18 tbb::detail::r1::arena::process (tls=..., this=0x400012573780) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:137
#19 tbb::detail::r1::market::process (this=0x40001259b080, j=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/market.cpp:599
#20 0x000040001168af2c in tbb::detail::r1::rml::private_worker::run (this=0x400012d4c000) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#21 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x400012d4c000) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#22 0x0000400011c178b8 in start_thread () from /lib64/libpthread.so.0
#23 0x0000400011c73afc in thread_start () from /lib64/libc.so.6
Thread 4 (Thread 0x40005c889220 (LWP 3922355) "cmsRun"):
#0  0x0000400011cf4b3c in nanosleep () from /lib64/libc.so.6
#1  0x0000400011cf49e0 in sleep () from /lib64/libc.so.6
#2  0x000040001877a1cc in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x0000400050cf1674 in tensorflow::SimplePropagatorState::PropagateOutputs(tensorflow::SimplePropagatorState::TaggedNode const&, absl::lts_20220623::InlinedVector<tensorflow::Entry, 4ul, std::allocator<tensorflow::Entry> >*, absl::lts_20220623::InlinedVector<tensorflow::SimplePropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::SimplePropagatorState::TaggedNode> >*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_framework.so.2
#5  0x0000400050ce91b4 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ProcessInline(tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*, long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_framework.so.2
#6  0x0000400050ce9cd4 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::Process(tensorflow::SimplePropagatorState::TaggedNode, long) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_framework.so.2
#7  0x00004000463b15c8 in tsl::thread::ThreadPool::Schedule(std::function<void ()>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#8  0x000040004a472e50 in std::_Function_handler<void (std::function<void ()>), tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&)::{lambda(std::function<void ()>)#6}>::_M_invoke(std::_Any_data const&, std::function<void ()>&&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#9  0x0000400050cd9c94 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ScheduleReady(absl::lts_20220623::InlinedVector<tensorflow::SimplePropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::SimplePropagatorState::TaggedNode> >*, tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_framework.so.2
#10 0x0000400050cdd188 in tensorflow::(anonymous namespace)::ExecutorImpl::RunAsync(tensorflow::Executor::Args const&, std::function<void (tsl::Status const&)>) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_framework.so.2
#11 0x000040004a48158c in tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#12 0x000040004a4837d8 in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/external/el8_aarch64_gcc12/lib/libtensorflow_cc.so.2
#13 0x000040003ddcbba8 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libPhysicsToolsTensorFlow.so
#14 0x000040003ddcbda8 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolInterface*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libPhysicsToolsTensorFlow.so
#15 0x00004000570aab24 in MkFitOutputConverter::computeDNNs(std::vector<TrackCandidate, std::allocator<TrackCandidate> > const&, std::vector<TrajectoryStateOnSurface, std::allocator<TrajectoryStateOnSurface> > const&, reco::BeamSpot const*, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const*, tensorflow::Session const*, std::vector<float, std::allocator<float> > const&, bool) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerMkFitPlugins.so
#16 0x00004000570ac570 in MkFitOutputConverter::convertCandidates(MkFitOutputWrapper const&, mkfit::EventOfHits const&, MkFitClusterIndexToHit const&, MkFitClusterIndexToHit const&, edm::View<TrajectorySeed> const&, MagneticField const&, Propagator const&, Propagator const&, MkFitGeometry const&, TkClonerImpl const&, std::vector<DetLayer const*, std::allocator<DetLayer const*> > const&, std::vector<mkfit::Track, std::allocator<mkfit::Track> > const&, reco::BeamSpot const*, std::vector<reco::Vertex, std::allocator<reco::Vertex> > const*, tensorflow::Session const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerMkFitPlugins.so
#17 0x00004000570a6408 in MkFitOutputConverter::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerMkFitPlugins.so
#18 0x000040000fc135c8 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#19 0x000040000fc0d480 in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#20 0x000040000fb9c304 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#21 0x000040000fb9f050 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#22 0x00004000101350dc in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreConcurrency.so
#23 0x0000400011682dfc in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x4000c51c1300, waiter=<synthetic pointer>..., this=0x400012573e00) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#24 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (waiter=<synthetic pointer>..., t=0x0, this=0x400012573e00) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#25 tbb::detail::r1::arena::process (tls=..., this=0x400012573780) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:137
#26 tbb::detail::r1::market::process (this=0x40001259b080, j=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/market.cpp:599
#27 0x000040001168af2c in tbb::detail::r1::rml::private_worker::run (this=0x400012d4c100) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#28 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x400012d4c100) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#29 0x0000400011c178b8 in start_thread () from /lib64/libpthread.so.0
#30 0x0000400011c73afc in thread_start () from /lib64/libc.so.6
Thread 3 (Thread 0x40005be79220 (LWP 3922354) "cmsRun"):
#0  0x0000400011cf4b3c in nanosleep () from /lib64/libc.so.6
#1  0x0000400011cf49e0 in sleep () from /lib64/libc.so.6
#2  0x000040001877a1cc in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x0000400055700ba8 in KFTrajectorySmoother::trajectory(Trajectory const&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libTrackingToolsTrackFitters.so
#5  0x0000400055698d88 in (anonymous namespace)::KFFittingSmoother::smoothingStep(Trajectory&&) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginTrackingToolsTrackFittersPlugins.so
#6  0x000040005569c9f8 in (anonymous namespace)::KFFittingSmoother::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginTrackingToolsTrackFittersPlugins.so
#7  0x00004000556964f8 in (anonymous namespace)::FlexibleKFFittingSmoother::fitOne(TrajectorySeed const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, TrajectoryStateOnSurface const&, TrajectoryFitter::fitType) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginTrackingToolsTrackFittersPlugins.so
#8  0x0000400076a9bb60 in TrackProducerAlgorithm<reco::Track>::buildTrack(TrajectoryFitter const*, Propagator const*, std::vector<AlgoProductTraits<reco::Track>::AlgoProduct, std::allocator<AlgoProductTraits<reco::Track>::AlgoProduct> >&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >&, TrajectoryStateOnSurface&, TrajectorySeed const&, float, reco::BeamSpot const&, edm::RefToBase<TrajectorySeed>, int, signed char) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libRecoTrackerTrackProducer.so
#9  0x000040007697d5b0 in TrackProducerAlgorithm<reco::Track>::runWithCandidate(TrackingGeometry const*, MagneticField const*, std::vector<TrackCandidate, std::allocator<TrackCandidate> > const&, TrajectoryFitter const*, Propagator const*, TransientTrackingRecHitBuilder const*, reco::BeamSpot const&, std::vector<AlgoProductTraits<reco::Track>::AlgoProduct, std::allocator<AlgoProductTraits<reco::Track>::AlgoProduct> >&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerTrackProducerPlugins.so
#10 0x000040007697bf28 in TrackProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerTrackProducerPlugins.so
#11 0x000040000fc274e4 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#12 0x000040000fc0e2f0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#13 0x000040000fb9c304 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#14 0x000040000fb9f050 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#15 0x00004000101350dc in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreConcurrency.so
#16 0x0000400011682dfc in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x4000c5286e00, waiter=<synthetic pointer>..., this=0x400012573e80) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#17 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (waiter=<synthetic pointer>..., t=0x0, this=0x400012573e80) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#18 tbb::detail::r1::arena::process (tls=..., this=0x400012573780) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:137
#19 tbb::detail::r1::market::process (this=0x40001259b080, j=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/market.cpp:599
#20 0x000040001168af2c in tbb::detail::r1::rml::private_worker::run (this=0x400012d4c080) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#21 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x400012d4c080) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#22 0x0000400011c178b8 in start_thread () from /lib64/libpthread.so.0
#23 0x0000400011c73afc in thread_start () from /lib64/libc.so.6
Thread 2 (Thread 0x40001b409220 (LWP 3921987) "cmsRun"):
#0  0x0000400011c22578 in waitpid () from /lib64/libpthread.so.0
#1  0x000040001877a340 in edm::service::cmssw_stacktrace_fork() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#2  0x000040001877cd50 in edm::service::InitRootHandlers::stacktraceHelperThread() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#3  0x00004000119aeb8c in std::execute_native_thread_routine (__p=0x4000140ac6a0) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#4  0x0000400011c178b8 in start_thread () from /lib64/libpthread.so.0
#5  0x0000400011c73afc in thread_start () from /lib64/libc.so.6
Thread 1 (Thread 0x40001182d0b0 (LWP 3906827) "cmsRun"):
#0  0x0000400011cf4b3c in nanosleep () from /lib64/libc.so.6
#1  0x0000400011cf49e0 in sleep () from /lib64/libc.so.6
#2  0x000040001877a1cc in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00004000111a2c00 in rtree_cache_direct_map (key=<optimized out>) at include/jemalloc/internal/rtree.h:367
#5  rtree_leaf_elm_lookup (init_missing=false, dependent=true, key=70372482940928, rtree_ctx=0x400011832ec0, rtree=0x4000111f8a20 <je_arena_emap_global>, tsdn=0x400011832cd0) at include/jemalloc/internal/rtree.h:362
#6  emap_rtree_leaf_elms_lookup (r_elm_b=0xfffff7be5788, r_elm_a=0xfffff7be5780, init_missing=false, dependent=true, edata=0x400011832ec0, rtree_ctx=0x400011832ec0, emap=0x4000111f8a20 <je_arena_emap_global>, tsdn=0x400011832cd0) at src/emap.c:131
#7  je_emap_merge_prepare (tsdn=tsdn@entry=0x400011832cd0, emap=0x4000111f8a20 <je_arena_emap_global>, prepare=prepare@entry=0xfffff7be5780, lead=lead@entry=0x400067019000, trail=trail@entry=0x40006701dc80) at src/emap.c:330
#8  0x00004000111a4388 in extent_merge_impl (ehooks=0xfffff7be5780, holding_core_locks=true, b=0x40006701dc80, a=0x400067019000, pac=0x400067003ac0, tsdn=0x400011832cd0) at src/extent.c:1273
#9  extent_merge_impl (tsdn=tsdn@entry=0x400011832cd0, pac=pac@entry=0x400067003ac0, ehooks=ehooks@entry=0x400067000480, a=a@entry=0x400067019000, b=b@entry=0x40006701dc80, holding_core_locks=true) at src/extent.c:1242
#10 0x00004000111a4620 in extent_coalesce (forward=true, outer=0x40006701dc80, inner=0x400067019000, ecache=<optimized out>, ehooks=0x400067000480, pac=0x400067003ac0, tsdn=0x400011832cd0) at src/extent.c:808
#11 extent_try_coalesce_impl (tsdn=tsdn@entry=0x400011832cd0, pac=pac@entry=0x400067003ac0, ehooks=ehooks@entry=0x400067000480, ecache=ecache@entry=0x40006700c6c8, edata=edata@entry=0x400067019000, coalesced=coalesced@entry=0x0) at src/extent.c:840
#12 0x00004000111a60f0 in extent_try_coalesce (coalesced=0x0, edata=0x400067019000, ecache=0x40006700c6c8, ehooks=0x400067000480, pac=0x400067003ac0, tsdn=0x400011832cd0) at src/extent.c:877
#13 je_extent_record (tsdn=0x400011832cd0, pac=0x400067003ac0, ehooks=0x400067000480, ecache=0x40006700c6c8, edata=0x400067019000) at src/extent.c:929
#14 0x00004000111a615c in extent_maximally_purge (edata=0x400067019000, ehooks=0x400067000480, pac=0x400067003ac0, tsdn=0x400011832cd0) at src/extent.c:893
#15 je_extent_record (tsdn=0x400011832cd0, pac=0x400067003ac0, ehooks=0x400067000480, ecache=0x400067003af8, edata=0x400067019000) at src/extent.c:945
#16 0x00004000111a61c4 in je_ecache_dalloc (tsdn=<optimized out>, pac=<optimized out>, ehooks=<optimized out>, ecache=<optimized out>, edata=<optimized out>) at src/extent.c:147
#17 0x00004000111b10b8 in pac_dalloc_impl (tsdn=0x400011832cd0, self=0x400067003ac0, edata=0x400067019000, deferred_work_generated=0xfffff7be597f) at src/pac.c:277
#18 0x00004000111aea78 in large_dalloc_finish_impl (edata=<optimized out>, arena=0x400067001400, tsdn=0x400011832cd0) at src/large.c:253
#19 je_large_dalloc (tsdn=0x400011832cd0, edata=<optimized out>) at src/large.c:273
#20 0x0000400011170b88 in arena_dalloc_large (slow_path=false, szind=<optimized out>, tcache=<optimized out>, ptr=<optimized out>, tsdn=0x400011832cd0) at include/jemalloc/internal/arena_inlines_b.h:297
#21 arena_sdalloc (slow_path=<optimized out>, caller_alloc_ctx=<optimized out>, tcache=<optimized out>, size=<optimized out>, ptr=<optimized out>, tsdn=<optimized out>) at include/jemalloc/internal/arena_inlines_b.h:421
#22 isdalloct (slow_path=<optimized out>, alloc_ctx=<optimized out>, tcache=<optimized out>, size=<optimized out>, ptr=<optimized out>, tsdn=<optimized out>) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:133
#23 isfree (slow_path=<optimized out>, tcache=<optimized out>, usize=<optimized out>, ptr=<optimized out>, tsd=<optimized out>) at src/jemalloc.c:2982
#24 je_sdallocx_default (ptr=<optimized out>, size=<optimized out>, flags=<optimized out>) at src/jemalloc.c:3924
#25 0x00004000111c1310 in sizedDeleteImpl (size=<optimized out>, ptr=<optimized out>) at src/jemalloc_cpp.cpp:195
#26 operator delete (ptr=<optimized out>, size=<optimized out>) at src/jemalloc_cpp.cpp:200
#27 0x000040003c0392e8 in CAHitTripletGenerator::hitNtuplets(IntermediateHitDoublets const&, std::vector<OrderedHitSeeds, std::allocator<OrderedHitSeeds> >&, SeedingLayerSetsHits const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libRecoTrackerPixelSeeding.so
#28 0x0000400099090cf4 in CAHitNtupletEDProducerT<CAHitTripletGenerator>::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/pluginRecoTrackerPixelSeedingPlugins.so
#29 0x000040000fc274e4 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#30 0x000040000fc0e2f0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#31 0x000040000fb9c304 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#32 0x000040000fb9f050 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#33 0x00004000101350dc in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreConcurrency.so
#34 0x0000400011688774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000c503b100, this=0x400012573d80) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#35 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400012573d80) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#36 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#37 0x000040000fb16e34 in edm::FinalWaitingTask::wait() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#38 0x000040000fb260f4 in edm::EventProcessor::processRuns() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#39 0x000040000fb203a8 in edm::EventProcessor::runToCompletion() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02838/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-23-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#40 0x0000000000406fe4 in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#41 0x00004000116810d8 in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:688
#42 0x0000000000408854 in main::{lambda()#1}::operator()() const ()
#43 0x0000000000404954 in main ()

Current Modules:

Module: TrackProducer:highPtTripletStepTracks (crashed)
Module: MkFitOutputConverter:highPtTripletStepTrackCandidates
Module: TrackProducer:highPtTripletStepTracks
Module: CAHitTripletEDProducer:highPtTripletStepHitTriplets

A fatal system signal has occurred: segmentation violation
timeout: the monitored command dumped core

[cmsbuild]

cms-bot internal usage

[slava77]

for aarch64 with a similar crash on MkFitOutputConverter:

sorry, how is this similar? the stack points to Module: TrackProducer:highPtTripletStepTracks (crashed)