search

SCIInstitute / ShapeWorks

ShapeWorks
http://sciinstitute.github.io/ShapeWorks/
Other
102 stars 32 forks source link

Crash at large particle split #2117

Closed akenmorris closed 12 months ago

akenmorris commented 1 year ago

In a project with 14 domains and 1024 particles, after the 1024 particle split, shapeworks optimization crashes

split number = 10

Particle count: 1024  1024  1024  1024  1024  1024  1024  1024  1024  1024  1024  1024  1024  1024
zsh: segmentation fault (core dumped)

stack trace:

#0  __memset_avx2_unaligned_erms () at ../sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S:351
#1  0x000055f13aaf50e1 in _mm_store_pd(double*, double __vector(2)) (__A=..., __P=<optimized out>) at /usr/lib/gcc/x86_64-linux-gnu/12/include/emmintrin.h:166
#2  Eigen::internal::pstore<double, double __vector(2)>(double*, double __vector(2) const&) (from=..., to=<optimized out>)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/arch/SSE/PacketMath.h:790
#3  Eigen::internal::pstoret<double, double __vector(2), 16>(double*, double __vector(2) const&) (from=..., to=<optimized out>)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/GenericPacketMath.h:978
#4  Eigen::internal::assign_op<double, double>::assignPacket<16, double __vector(2)>(double*, double __vector(2) const&) const (b=..., a=<optimized out>, this=<optimized out>)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/functors/AssignmentFunctors.h:28
#5  Eigen::internal::generic_dense_assignment_kernel<Eigen::internal::evaluator<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> > >, Eigen::internal::evaluator<Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> > >, Eigen::internal::assign_op<double, double>, 0>::assignPacket<16, 0, double __vector(2)>(long) (index=<optimized out>,
    this=<optimized out>) at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/AssignEvaluator.h:681
#6  Eigen::internal::dense_assignment_loop<Eigen::internal::generic_dense_assignment_kernel<Eigen::internal::evaluator<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> > >, Eigen::internal::evaluator<Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> > >, Eigen::internal::assign_op<double, double>, 0>, 3, 0>::run (
    kernel=<synthetic pointer>...) at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/AssignEvaluator.h:437
#7  Eigen::internal::call_dense_assignment_loop<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> >, Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> >, Eigen::internal::assign_op<double, double> > (func=..., src=..., dst=...) at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/AssignEvaluator.h:785
#8  Eigen::internal::Assignment<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> >, Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> >, Eigen::internal::assign_op<double, double>, Eigen::internal::Dense2Dense, void>::run (func=..., src=..., dst=...)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/AssignEvaluator.h:954
#9  Eigen::internal::call_assignment_no_alias<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> >, Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> >, Eigen::internal::assign_op<double, double> > (func=..., src=..., dst=...) at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/AssignEvaluator.h:890
#10 Eigen::internal::call_assignment<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> >, Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> >, Eigen::internal::assign_op<double, double> >(Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> >&, Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> > const&, Eigen::internal::assign_op<double, double> const&, Eigen::internal::enable_if<!Eigen::internal::evaluator_assume_aliasing<Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> >, Eigen::internal::evaluator_traits<Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> > >::Shape>::value, void*>::type) (func=..., src=..., dst=...) at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/AssignEvaluator.h:858
#11 Eigen::internal::call_assignment<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> >, Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> > > (src=..., dst=...) at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/AssignEvaluator.h:836
#12 Eigen::MatrixBase<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> > >::operator=<Eigen::CwiseNullaryOp<Eigen::internal::scalar_constant_op<double>, Eigen::Matrix<double, -1, -1, 1, -1, -1> > > (other=..., this=0x7fff58aefaa0) at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/Assign.h:66
#13 Eigen::DenseBase<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> > >::setConstant (val=<optimized out>, this=0x7fff58aefaa0)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/CwiseNullaryOp.h:347
#14 Eigen::DenseBase<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> > >::setZero (this=0x7fff58aefaa0)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/CwiseNullaryOp.h:548
#15 Eigen::internal::generic_product_impl<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1> const, 0, Eigen::Stride<0, 0> >, Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1> const, 0, Eigen::Stride<0, 0> >, Eigen::DenseShape, Eigen::DenseShape, 8>::evalTo<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> > > (rhs=..., lhs=..., dst=...)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/products/GeneralMatrixMatrix.h:444
#16 Eigen::internal::Assignment<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> >, Eigen::Product<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1> const, 0, Eigen::Stride<0, 0> >, Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1> const, 0, Eigen::Stride<0, 0> >, 0>, Eigen::internal::assign_op<double, double>, Eigen::internal::Dense2Dense, void>::run (src=..., dst=...)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/ProductEvaluators.h:148
#17 Eigen::internal::call_assignment_no_alias<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> >, Eigen::Product<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1> const, 0, Eigen::Stride<0, 0> >, Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1> const, 0, Eigen::Stride<0, 0> >, 0>, Eigen::internal::assign_op<double, double> > (func=..., src=..., dst=...)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/AssignEvaluator.h:890
#18 Eigen::NoAlias<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1>, 0, Eigen::Stride<0, 0> >, Eigen::MatrixBase>::operator=<Eigen::Product<Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1> const, 0, Eigen::Stride<0, 0> >, Eigen::Map<Eigen::Matrix<double, -1, -1, 1, -1, -1> const, 0, Eigen::Stride<0, 0> >, 0> > (other=..., this=<optimized out>)
    at /home/amorris/miniconda3/envs/shapeworks/include/eigen3/Eigen/src/Core/NoAlias.h:43
#19 Utils::multiply_into<double> (out=..., lhs=..., rhs=...) at /home/amorris/sci/shapeworks/Libs/Utils/Utils.cpp:420
#20 0x000055f13a997fca in shapeworks::CorrespondenceFunction::ComputeUpdates (this=0x55f13d873e10, c=0x55f13d870640) at /usr/include/c++/12/bits/shared_ptr_base.h:1665
#21 0x000055f13aa0464e in shapeworks::DualVectorFunction::BeforeIteration (this=0x55f13d85fa50) at /home/amorris/sci/shapeworks/Libs/Optimize/Function/DualVectorFunction.h:263
#22 0x000055f13a9a31e2 in shapeworks::GradientDescentOptimizer::StartAdaptiveGaussSeidelOptimization (this=0x55f13d838d10) at /home/amorris/sci/shapeworks/Libs/Optimize/GradientDescentOptimizer.cpp:92
#23 0x000055f13a9bc921 in shapeworks::Optimize::Initialize (this=this@entry=0x7fff58af06e0) at /home/amorris/sci/shapeworks/Libs/Optimize/Optimize.cpp:685
#24 0x000055f13a9bd628 in shapeworks::Optimize::Run (this=this@entry=0x7fff58af06e0) at /home/amorris/sci/shapeworks/Libs/Optimize/Optimize.cpp:173
#25 0x000055f1397793e5 in shapeworks::OptimizeCommand::execute (this=<optimized out>, options=..., sharedData=...) at /home/amorris/sci/shapeworks/Applications/shapeworks/Commands.cpp:143
#26 0x000055f1397731b8 in shapeworks::Command::run (this=<optimized out>, sharedData=...) at /home/amorris/sci/shapeworks/Applications/shapeworks/Command.cpp:27
#27 0x000055f13976456a in shapeworks::Executable::run (this=this@entry=0x7fff58af1160, arguments=..., sharedData=...) at /home/amorris/sci/shapeworks/Applications/shapeworks/Executable.cpp:94
#28 0x000055f139767fbd in shapeworks::Executable::run (this=this@entry=0x7fff58af1160, argc=argc@entry=5, argv=argv@entry=0x7fff58af1528) at /home/amorris/sci/shapeworks/Applications/shapeworks/Executable.cpp:128
#29 0x000055f13976243f in main (argc=5, argv=0x7fff58af1528) at /home/amorris/sci/shapeworks/Applications/shapeworks/shapeworks.cpp:114

I believe this is in the Utils::multiply_into call that performs the matrix multiplication for the covariance matrix.