search

HorribleSanity / Raven.jl

MIT License
3 stars 1 forks source link

Random crashes in CI for MacOS and Windows #70

Open lcw opened 1 year ago

lcw commented 1 year ago

I was able to reproduce the random CI crashes that happen in Julia 1.9.3 on macOS 11.7.10 in a virtual machine in qemu. 

using TestEnv
TestEnv.activate()
using Raven
using StableRNGs
for i = 1:1000
    @info "StableRNG($i)"
    rng = StableRNG(i)
    AT = Array
    FT = Float64
    N = (3, 3, 3)
    R = 2
    r = 1
    coarse_grid = Raven.cubeshellgrid(R, r)
    gm = GridManager(LobattoCell{Tuple{N...},FT,AT}(), coarse_grid, min_level = 2)
    indicator = rand(rng, (Raven.AdaptNone, Raven.AdaptRefine), length(gm))
    adapt!(gm, indicator)
    grid = generate(gm);
end

This will result in errors like

[ Info: StableRNG(1)
[ Info: StableRNG(2)
[ Info: StableRNG(3)
[ Info: StableRNG(4)
[ Info: StableRNG(5)
ERROR: AssertionError: kind == 1 + numchildfaces
Stacktrace:
 [1] materializefacemaps(cell::LobattoCell{Tuple{3, 3, 3}, Float64, Array, 3, Tuple{Array{Float64, 3}, Array{Float64, 3}, Array{Float64, 3}}, Vector{StaticArraysCore.SVector{3, Float64}}, Tuple{Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}, Matrix{Float64}}}, Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Matrix{Float64}, Raven.Eye{Float64, 3}}}, Raven.Kron{Tuple{Matrix{Float64}, Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}}}}, Tuple{Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}, Matrix{Float64}}}, Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Matrix{Float64}, Raven.Eye{Float64, 3}}}, Raven.Kron{Tuple{Matrix{Float64}, Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}}}}, Tuple{Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}, Matrix{Float64}}}, Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Matrix{Float64}, Raven.Eye{Float64, 3}}}, Raven.Kron{Tuple{Matrix{Float64}, Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}}}}, LinearAlgebra.Diagonal{Float64, Vector{Float64}}, LinearAlgebra.Diagonal{Float64, Vector{Float64}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}, Tuple{Tuple{Matrix{Float64}, Matrix{Float64}}, Tuple{Matrix{Float64}, Matrix{Float64}}, Tuple{Matrix{Float64}, Matrix{Float64}}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}}, numcells_local::Int32, ctod_degree3_local::SparseArrays.SparseMatrixCSC{Bool, Int64}, dtoc_degree3_local::Array{Int32, 4}, dtoc_degree3_global::Array{Int64, 4}, quadranttolevel::Vector{Int8}, quadranttoglobalid::Vector{Int64})
   @ Raven ~/.julia/dev/Raven/src/lobattocells.jl:1924
 [2] generate(warp::Function, gm::GridManager{LobattoCell{Tuple{3, 3, 3}, Float64, Array, 3, Tuple{Array{Float64, 3}, Array{Float64, 3}, Array{Float64, 3}}, Vector{StaticArraysCore.SVector{3, Float64}}, Tuple{Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}, Matrix{Float64}}}, Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Matrix{Float64}, Raven.Eye{Float64, 3}}}, Raven.Kron{Tuple{Matrix{Float64}, Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}}}}, Tuple{Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}, Matrix{Float64}}}, Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Matrix{Float64}, Raven.Eye{Float64, 3}}}, Raven.Kron{Tuple{Matrix{Float64}, Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}}}}, Tuple{Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}, Matrix{Float64}}}, Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Matrix{Float64}, Raven.Eye{Float64, 3}}}, Raven.Kron{Tuple{Matrix{Float64}, Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}}}}, LinearAlgebra.Diagonal{Float64, Vector{Float64}}, LinearAlgebra.Diagonal{Float64, Vector{Float64}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}, Tuple{Tuple{Matrix{Float64}, Matrix{Float64}}, Tuple{Matrix{Float64}, Matrix{Float64}}, Tuple{Matrix{Float64}, Matrix{Float64}}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}}, Raven.CoarseGrid{P4estTypes.Connectivity{8, Ptr{P4est.LibP4est.p8est_connectivity}}, Vector{StaticArraysCore.SVector{3, Float64}}, Vector{NTuple{8, Int64}}, Raven.var"#cubespherewarp#65", Raven.var"#cubesphereunwarp#66"}, Vector{NTuple{8, Int64}}, Vector{StaticArraysCore.SVector{3, Float64}}, P4estTypes.Pxest{8, Raven.QuadData, Ptr{P4est.LibP4est.p8est}, P4estTypes.Connectivity{8, Ptr{P4est.LibP4est.p8est_connectivity}}}})
   @ Raven ~/.julia/dev/Raven/src/gridmanager.jl:321
 [3] generate(gm::GridManager{LobattoCell{Tuple{3, 3, 3}, Float64, Array, 3, Tuple{Array{Float64, 3}, Array{Float64, 3}, Array{Float64, 3}}, Vector{StaticArraysCore.SVector{3, Float64}}, Tuple{Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}, Matrix{Float64}}}, Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Matrix{Float64}, Raven.Eye{Float64, 3}}}, Raven.Kron{Tuple{Matrix{Float64}, Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}}}}, Tuple{Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}, Matrix{Float64}}}, Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Matrix{Float64}, Raven.Eye{Float64, 3}}}, Raven.Kron{Tuple{Matrix{Float64}, Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}}}}, Tuple{Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}, Matrix{Float64}}}, Raven.Kron{Tuple{Raven.Eye{Float64, 3}, Matrix{Float64}, Raven.Eye{Float64, 3}}}, Raven.Kron{Tuple{Matrix{Float64}, Raven.Eye{Float64, 3}, Raven.Eye{Float64, 3}}}}, LinearAlgebra.Diagonal{Float64, Vector{Float64}}, LinearAlgebra.Diagonal{Float64, Vector{Float64}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}, Tuple{Tuple{Matrix{Float64}, Matrix{Float64}}, Tuple{Matrix{Float64}, Matrix{Float64}}, Tuple{Matrix{Float64}, Matrix{Float64}}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}, Raven.Kron{Tuple{Matrix{Float64}, Matrix{Float64}, Matrix{Float64}}}}, Raven.CoarseGrid{P4estTypes.Connectivity{8, Ptr{P4est.LibP4est.p8est_connectivity}}, Vector{StaticArraysCore.SVector{3, Float64}}, Vector{NTuple{8, Int64}}, Raven.var"#cubespherewarp#65", Raven.var"#cubesphereunwarp#66"}, Vector{NTuple{8, Int64}}, Vector{StaticArraysCore.SVector{3, Float64}}, P4estTypes.Pxest{8, Raven.QuadData, Ptr{P4est.LibP4est.p8est}, P4estTypes.Connectivity{8, Ptr{P4est.LibP4est.p8est_connectivity}}}})
   @ Raven ~/.julia/dev/Raven/src/gridmanager.jl:127
 [4] top-level scope
   @ ./REPL[5]:13

This will not happen at the same iteration each time. This is similar to the error on the windows CI instance.

If we set min_level=0 this will result in errors like

[ Info: StableRNG(1)
[ Info: StableRNG(2)

[843] signal (11.1): Segmentation fault: 11
in expression starting at REPL[5]:1
p8est_ghost_support_lnodes at /Users/try/.julia/artifacts/0fd7c70fa76506379bfdac82a02baf2d2dc01b5d/lib/libp4est.0.dylib (unknown line)
p8est_ghost_support_lnodes at /Users/try/.julia/packages/P4est/PSrSX/src/LibP4est.jl:10943 [inlined]
expand! at /Users/try/.julia/packages/P4estTypes/lc5TS/src/ghost.jl:154 [inlined]
generate at /Users/try/.julia/dev/Raven/src/gridmanager.jl:304
generate at /Users/try/.julia/dev/Raven/src/gridmanager.jl:127
unknown function (ip: 0x1285d6382)
ijl_apply_generic at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
top-level scope at ./REPL[5]:13
jl_toplevel_eval_flex at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
jl_toplevel_eval_flex at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
ijl_toplevel_eval_in at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
eval at ./boot.jl:370 [inlined]
eval_user_input at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-grannysmith-C07ZQ07RJYVY.0/build/default-grannysmith-C07ZQ07RJYVY-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:153
repl_backend_loop at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-grannysmith-C07ZQ07RJYVY.0/build/default-grannysmith-C07ZQ07RJYVY-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:249
#start_repl_backend#46 at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-grannysmith-C07ZQ07RJYVY.0/build/default-grannysmith-C07ZQ07RJYVY-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:234
start_repl_backend at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-grannysmith-C07ZQ07RJYVY.0/build/default-grannysmith-C07ZQ07RJYVY-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:231
ijl_apply_generic at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
#run_repl#59 at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-grannysmith-C07ZQ07RJYVY.0/build/default-grannysmith-C07ZQ07RJYVY-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:379
run_repl at /Users/julia/.julia/scratchspaces/a66863c6-20e8-4ff4-8a62-49f30b1f605e/agent-cache/default-grannysmith-C07ZQ07RJYVY.0/build/default-grannysmith-C07ZQ07RJYVY-0/julialang/julia-release-1-dot-9/usr/share/julia/stdlib/v1.9/REPL/src/REPL.jl:365
jfptr_run_repl_61406 at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/sys.dylib (unknown line)
ijl_apply_generic at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
#1017 at ./client.jl:421
jfptr_YY.1017_52203 at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/sys.dylib (unknown line)
ijl_apply_generic at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
jl_f__call_latest at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
#invokelatest#2 at ./essentials.jl:819 [inlined]
invokelatest at ./essentials.jl:816 [inlined]
run_main_repl at ./client.jl:405
exec_options at ./client.jl:322
_start at ./client.jl:522
jfptr__start_49586 at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/sys.dylib (unknown line)
ijl_apply_generic at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
true_main at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
jl_repl_entrypoint at /Applications/Julia-1.9.app/Contents/Resources/julia/lib/julia/libjulia-internal.1.9.dylib (unknown line)
Allocations: 36138442 (Pool: 36110783; Big: 27659); GC: 49
zsh: segmentation fault  /Applications/Julia-1.9.app/Contents/Resources/julia/bin/julia --project=.

Again, this will not happen at the same iteration each time.

So it seems that there is a bug in the interaction with p4est. I have yet to reproduce this on linux.