search

cms-sw / cmssw

CMS Offline Software
http://cms-sw.github.io/
Apache License 2.0
1.09k stars 4.33k forks source link

[AARCH64] MixingModule:mix segfaulted in Relval 250402.181 #40942

Open iarspider opened 1 year ago

iarspider commented 1 year ago

Log:

Thread 1 (Thread 0x40003512d030 (LWP 70155) "cmsRun"):
#0  0x00004000355eb5cc in poll () from /lib64/libc.so.6
#1  0x000040003c5cb66c in full_read.constprop () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#2  0x000040003c59bb84 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  0x000040003c5971b8 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x00004000afcc9128 in edm::RootDelayedReader::getProduct_(edm::BranchID const&, edm::EDProductGetter const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginIOPoolInput.so
#6  0x000040003356df4c in edm::DelayedReader::getProduct(edm::BranchID const&, edm::EDProductGetter const*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#7  0x0000400033621cc4 in edm::DelayedReaderInputProductResolver::resolveProduct_(edm::Principal const&, bool, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#8  0x0000400033623bfc in edm::DelayedReaderInputProductResolver::resolveProduct_(edm::Principal const&, bool, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#9  0x0000400033610648 in edm::Principal::findProductByLabel(edm::KindOfType, edm::TypeID const&, edm::InputTag const&, edm::EDConsumerBase const*, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#10 0x0000400033610958 in edm::Principal::getByLabel(edm::KindOfType, edm::TypeID const&, edm::InputTag const&, edm::EDConsumerBase const*, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#11 0x00004000aeccb148 in cms::PileupVertexAccumulator::accumulate(PileUpEventPrincipal const&, edm::EventSetup const&, edm::StreamID const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralPileupInformationPlugins.so
#12 0x00004000afb87ad4 in edm::MixingModule::accumulateEvent(PileUpEventPrincipal const&, edm::EventSetup const&, edm::StreamID const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralMixingModulePlugins.so
#13 0x00004000afb87c30 in edm::MixingModule::pileAllWorkers(edm::EventPrincipal const&, edm::ModuleCallingContext const*, int, int, int&, edm::EventSetup const&, edm::StreamID const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralMixingModulePlugins.so
#14 0x00004000afb89274 in edm::MixingModule::doPileUp(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralMixingModulePlugins.so
#15 0x00004000afbff4e0 in edm::BMixingModule::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libMixingBase.so
#16 0x00004000336946fc in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#17 0x000040003367aaa0 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#18 0x0000400033605f78 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#19 0x00004000336066e0 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#20 0x0000400033b25160 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreConcurrency.so
#21 0x0000400034fd8508 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000bc9dae00, this=0x400035e03d80) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/task_dispatcher.h:322
#22 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400035e03d80) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/task_dispatcher.h:458
#23 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/task_dispatcher.cpp:168
#24 0x000040003357ec84 in edm::FinalWaitingTask::wait() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#25 0x000040003358bcc8 in edm::EventProcessor::processRuns() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#26 0x000040003358c250 in edm::EventProcessor::runToCompletion() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#27 0x000000000040b9c8 in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#28 0x0000400034fd0f5c in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/arena.cpp:694
#29 0x000000000040f73c in main::{lambda()#1}::operator()() const ()
#30 0x0000000000406fc4 in main ()

Full log: link

cmsbuild commented 1 year ago

A new Issue was created by @iarspider .

@Dr15Jones, @perrotta, @dpiparo, @rappoccio, @makortel, @smuzaffar can you please review it and eventually sign/assign? Thanks.

cms-bot commands are listed here

makortel commented 1 year ago

assign core

cmsbuild commented 1 year ago

New categories assigned: core

@Dr15Jones,@smuzaffar,@makortel you have been requested to review this Pull request/Issue and eventually sign? Thanks

makortel commented 1 year ago

Stack traces of all threads

Thread 5 (Thread 0x400081a09270 (LWP 71607) "cmsRun"):
#2  0x000040003c5928b8 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x0000400055500848 in HcalQIECoder::adc(HcalQIEShape const&, float, unsigned int) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libCondFormatsHcalObjects.so
#5  0x00004000552cf50c in void HcalCoderDb::fC2adc_<QIE11DataFrame>(CaloSamples const&, QIE11DataFrame&, int) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libCalibFormatsHcalObjects.so
#6  0x00004000b0a3d0b8 in void HcalElectronicsSim::convert<QIE11DataFrame>(CaloSamples&, QIE11DataFrame&, CLHEP::HepRandomEngine*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libSimCalorimetryHcalSimAlgos.so
#7  0x00004000b0a42fe8 in HcalElectronicsSim::analogToDigital(CLHEP::HepRandomEngine*, CaloSamples&, QIE11DataFrame&, double, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libSimCalorimetryHcalSimAlgos.so
#8  0x00004000b09ee994 in CaloTDigitizer<HcalQIE11DigitizerTraits, CaloTDigitizerQIE1011Run>::run(HcalDataFrameContainer<QIE11DataFrame>&, CLHEP::HepRandomEngine*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libSimCalorimetryHcalSimProducers.so
#9  0x00004000b09f4574 in HcalDigitizer::finalizeEvent(edm::Event&, edm::EventSetup const&, CLHEP::HepRandomEngine*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libSimCalorimetryHcalSimProducers.so
#10 0x00004000b09e7ddc in HcalDigiProducer::finalizeEvent(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libSimCalorimetryHcalSimProducers.so
#11 0x00004000afb84c80 in edm::MixingModule::finalizeEvent(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralMixingModulePlugins.so
#12 0x00004000afbff55c in edm::BMixingModule::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libMixingBase.so
#13 0x00004000336946fc in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 4 (Thread 0x40007e639270 (LWP 71606) "cmsRun"):
#2  0x000040003c5928b8 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x0000400035ae87e0 in ZSTD_compressBlock_lazy2_row () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libzstd.so.1
#5  0x0000400035aa64a0 in ZSTD_compressBlock_internal () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libzstd.so.1
#6  0x0000400035aa7578 in ZSTD_compress_frameChunk () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libzstd.so.1
#7  0x0000400035aaaa60 in ZSTD_compressEnd () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libzstd.so.1
#8  0x00004000347ca9ac in R__zipZSTD () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libCore.so
#9  0x0000400033c67694 in TBasket::WriteBuffer() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libTree.so
#10 0x0000400033c77628 in std::_Function_handler<void (), ROOT::Internal::TBranchIMTHelper::Run<TBranch::WriteBasketImpl(TBasket*, int, ROOT::Internal::TBranchIMTHelper*)::{lambda()#1}>(TBranch::WriteBasketImpl(TBasket*, int, ROOT::Internal::TBranchIMTHelper*)::{lambda()#1} const&)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libTree.so
#11 0x00004000356a87ac in tbb::detail::d1::function_task<std::function<void ()> >::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libImt.so
#12 0x0000400034fd8508 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000bcce7300, this=0x400035e03e00) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/task_dispatcher.h:322
#13 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x400035e03e00) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/task_dispatcher.h:458
#14 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/task_dispatcher.cpp:168
#15 0x00004000356a8474 in ROOT::Experimental::TTaskGroup::Wait() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libImt.so
#16 0x0000400033cef548 in TTree::Fill() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/external/el8_aarch64_gcc11/lib/libTree.so
#17 0x00004000aec44104 in tbb::detail::d1::task_arena_function<edm::RootOutputTree::fillTree()::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libIOPoolOutput.so
#18 0x0000400034fd1554 in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/arena.cpp:767
#19 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#20 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins_b/workspace/build-any-ib/w/BUILD/el8_aarch64_gcc11/external/tbb/v2021.8.0-00a1da2ea6897843151bdc54c2bd1c06/tbb-v2021.8.0/src/tbb/arena.cpp:768
#21 0x00004000aec48e8c in edm::RootOutputTree::fillTree() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libIOPoolOutput.so
#22 0x00004000aec40e54 in edm::RootOutputFile::fillBranches(edm::BranchType const&, edm::OccurrenceForOutput const&, unsigned int, std::vector<edm::StoredProductProvenance, std::allocator<edm::StoredProductProvenance> >*, edm::ProductProvenanceRetriever const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libIOPoolOutput.so
#23 0x00004000aec41874 in edm::RootOutputFile::writeOne(edm::EventForOutput const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libIOPoolOutput.so
#24 0x00004000aec2d76c in edm::PoolOutputModule::write(edm::EventForOutput const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libIOPoolOutput.so
#25 0x00004000335ffa1c in edm::core::OutputModuleCore::doEvent_(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#26 0x0000400033688c88 in edm::one::OutputModuleBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 3 (Thread 0x40007dc29270 (LWP 71605) "cmsRun"):
#2  0x000040003c5928b8 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#3  <signal handler called>
#4  0x00004000b090908c in EcalHitResponse::setEventTime(unsigned long long const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libSimCalorimetryEcalSimAlgos.so
#5  0x00004000b08a4bf8 in EcalDigiProducer::checkCalibrations(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libSimCalorimetryEcalSimProducers.so
#6  0x00004000b08a5ff0 in EcalDigiProducer::initializeEvent(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libSimCalorimetryEcalSimProducers.so
#7  0x00004000afb84c20 in edm::MixingModule::initializeEvent(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralMixingModulePlugins.so
#8  0x00004000afbff524 in edm::BMixingModule::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libMixingBase.so
#9  0x00004000336946fc in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Thread 1 (Thread 0x40003512d030 (LWP 70155) "cmsRun"):
#3  0x000040003c5971b8 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginFWCoreServicesPlugins.so
#4  <signal handler called>
#5  0x00004000afcc9128 in edm::RootDelayedReader::getProduct_(edm::BranchID const&, edm::EDProductGetter const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginIOPoolInput.so
#6  0x000040003356df4c in edm::DelayedReader::getProduct(edm::BranchID const&, edm::EDProductGetter const*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#7  0x0000400033621cc4 in edm::DelayedReaderInputProductResolver::resolveProduct_(edm::Principal const&, bool, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#8  0x0000400033623bfc in edm::DelayedReaderInputProductResolver::resolveProduct_(edm::Principal const&, bool, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#9  0x0000400033610648 in edm::Principal::findProductByLabel(edm::KindOfType, edm::TypeID const&, edm::InputTag const&, edm::EDConsumerBase const*, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#10 0x0000400033610958 in edm::Principal::getByLabel(edm::KindOfType, edm::TypeID const&, edm::InputTag const&, edm::EDConsumerBase const*, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so
#11 0x00004000aeccb148 in cms::PileupVertexAccumulator::accumulate(PileUpEventPrincipal const&, edm::EventSetup const&, edm::StreamID const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralPileupInformationPlugins.so
#12 0x00004000afb87ad4 in edm::MixingModule::accumulateEvent(PileUpEventPrincipal const&, edm::EventSetup const&, edm::StreamID const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralMixingModulePlugins.so
#13 0x00004000afb87c30 in edm::MixingModule::pileAllWorkers(edm::EventPrincipal const&, edm::ModuleCallingContext const*, int, int, int&, edm::EventSetup const&, edm::StreamID const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralMixingModulePlugins.so
#14 0x00004000afb89274 in edm::MixingModule::doPileUp(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/pluginSimGeneralMixingModulePlugins.so
#15 0x00004000afbff4e0 in edm::BMixingModule::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libMixingBase.so
#16 0x00004000336946fc in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02774/el8_aarch64_gcc11/cms/cmssw/CMSSW_13_1_X_2023-03-02-2300/lib/el8_aarch64_gcc11/libFWCoreFramework.so

Current Modules:
Module: MixingModule:mix (crashed)
Module: MixingModule:mix
Module: MixingModule:mix
Module: PoolOutputModule:PREMIXoutput