search

JuliaIO / JLD2.jl

HDF5-compatible file format in pure Julia
Other
537 stars 84 forks source link

ArgumentError: destination has fewer elements than required #480

Closed lijas closed 11 months ago

lijas commented 11 months ago

I am randomly running in to a problem when I want save one of my structs. It fails about 50% of the time with this error message:

Error encountered while save FileIO.File{FileIO.DataFormat{:JLD2}, String}("matrix_triangulation_(20, 40, 20).jld2").

Fatal error:
ERROR: LoadError: ArgumentError: destination has fewer elements than required
Stacktrace:
  [1] copyto!(dest::Vector{Pair{Int64, Nothing}}, src::Dict{Int64, Nothing})
    @ Base ./abstractarray.jl:947
  [2] _collect
    @ ./array.jl:713 [inlined]
  [3] collect
    @ ./array.jl:707 [inlined]
  [4] wconvert
    @ ~/.julia/packages/JLD2/cHcDY/src/data/specialcased_types.jl:252 [inlined]
  [5] h5convert!
    @ ~/.julia/packages/JLD2/cHcDY/src/data/custom_serialization.jl:34 [inlined]
  [6] macro expansion
    @ ~/.julia/packages/JLD2/cHcDY/src/data/writing_datatypes.jl:237 [inlined]
  [7] h5convert!(out::JLD2.IndirectPointer, #unused#::JLD2.OnDiskRepresentation{(0,), Tuple{Dict{Int64, Nothing}}, Tuple{JLD2.CustomSerialization{Vector{Pair{Int64, Nothing}}, JLD2.RelOffset}}, 8}, file::JLD2.JLDFile{JLD2.MmapIO}, x::Set{Int64}, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/cHcDY/src/data/writing_datatypes.jl:237
  [8] macro expansion
    @ ~/.julia/packages/JLD2/cHcDY/src/data/writing_datatypes.jl:237 [inlined]
  [9] h5convert!(out::JLD2.IndirectPointer, #unused#::JLD2.OnDiskRepresentation{(0, 8, 16, 24, 32), Tuple{BezierGrid{3}, Dict{Int64, QuadratureRule{3, RefCube, Float64}}, Set{Int64}, Vector{ImmersNurbs.GhostInterface}, ImmersNurbs.ImmersoGeometry{3, G} where G<:Ferrite.AbstractGrid{3}}, Tuple{JLD2.RelOffset, JLD2.CustomSerialization{Vector{Pair{Int64, QuadratureRule{3, RefCube, Float64}}}, JLD2.RelOffset}, JLD2.OnDiskRepresentation{(0,), Tuple{Dict{Int64, Nothing}}, Tuple{JLD2.CustomSerialization{Vector{Pair{Int64, Nothing}}, JLD2.RelOffset}}, 8}(), JLD2.RelOffset, JLD2.RelOffset}, 40}, file::JLD2.JLDFile{JLD2.MmapIO}, x::ImmersNurbs.ImmersoNurbsResult{3}, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/cHcDY/src/data/writing_datatypes.jl:237
 [10] write_data(io::JLD2.MmapIO, f::JLD2.JLDFile{JLD2.MmapIO}, data::ImmersNurbs.ImmersoNurbsResult{3}, odr::JLD2.OnDiskRepresentation{(0, 8, 16, 24, 32), Tuple{BezierGrid{3}, Dict{Int64, QuadratureRule{3, RefCube, Float64}}, Set{Int64}, Vector{ImmersNurbs.GhostInterface}, ImmersNurbs.ImmersoGeometry{3, G} where G<:Ferrite.AbstractGrid{3}}, Tuple{JLD2.RelOffset, JLD2.CustomSerialization{Vector{Pair{Int64, QuadratureRule{3, RefCube, Float64}}}, JLD2.RelOffset}, JLD2.OnDiskRepresentation{(0,), Tuple{Dict{Int64, Nothing}}, Tuple{JLD2.CustomSerialization{Vector{Pair{Int64, Nothing}}, JLD2.RelOffset}}, 8}(), JLD2.RelOffset, JLD2.RelOffset}, 40}, #unused#::JLD2.HasReferences, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/cHcDY/src/dataio.jl:96
 [11] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.WriteDataspace{0, Tuple{}}, datatype::JLD2.CommittedDatatype, odr::JLD2.OnDiskRepresentation{(0, 8, 16, 24, 32), Tuple{BezierGrid{3}, Dict{Int64, QuadratureRule{3, RefCube, Float64}}, Set{Int64}, Vector{ImmersNurbs.GhostInterface}, ImmersNurbs.ImmersoGeometry{3, G} where G<:Ferrite.AbstractGrid{3}}, Tuple{JLD2.RelOffset, JLD2.CustomSerialization{Vector{Pair{Int64, QuadratureRule{3, RefCube, Float64}}}, JLD2.RelOffset}, JLD2.OnDiskRepresentation{(0,), Tuple{Dict{Int64, Nothing}}, Tuple{JLD2.CustomSerialization{Vector{Pair{Int64, Nothing}}, JLD2.RelOffset}}, 8}(), JLD2.RelOffset, JLD2.RelOffset}, 40}, data::ImmersNurbs.ImmersoNurbsResult{3}, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/cHcDY/src/datasets.jl:578
 [12] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, x::ImmersNurbs.ImmersoNurbsResult{3}, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/cHcDY/src/datasets.jl:646
 [13] write(g::JLD2.Group{JLD2.JLDFile{JLD2.MmapIO}}, name::String, obj::ImmersNurbs.ImmersoNurbsResult{3}, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}}; compress::Nothing)
    @ JLD2 ~/.julia/packages/JLD2/cHcDY/src/compression.jl:137
 [14] write
    @ ~/.julia/packages/JLD2/cHcDY/src/compression.jl:125 [inlined]
 [15] #write#119
    @ ~/.julia/packages/JLD2/cHcDY/src/compression.jl:121 [inlined]
 [16] write
    @ ~/.julia/packages/JLD2/cHcDY/src/compression.jl:121 [inlined]
 [17] (::JLD2.var"#96#97"{String, ImmersNurbs.ImmersoNurbsResult{3}, Tuple{}})(file::JLD2.JLDFile{JLD2.MmapIO})
    @ JLD2 ~/.julia/packages/JLD2/cHcDY/src/fileio.jl:26
 [18] jldopen(::Function, ::String, ::Vararg{String}; kws::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ JLD2 ~/.julia/packages/JLD2/cHcDY/src/loadsave.jl:4
 [19] jldopen
    @ ~/.julia/packages/JLD2/cHcDY/src/loadsave.jl:1 [inlined]
 [20] #fileio_save#95
    @ ~/.julia/packages/JLD2/cHcDY/src/fileio.jl:24 [inlined]
 [21] fileio_save(::FileIO.File{FileIO.DataFormat{:JLD2}, String}, ::String, ::ImmersNurbs.ImmersoNurbsResult{3})
    @ JLD2 ~/.julia/packages/JLD2/cHcDY/src/fileio.jl:20
 [22] invokelatest(::Any, ::Any, ::Vararg{Any}; kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Base ./essentials.jl:816
 [23] invokelatest(::Any, ::Any, ::Vararg{Any})
    @ Base ./essentials.jl:813
 [24] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::FileIO.Formatted, ::String, ::Vararg{Any}; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ FileIO ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:219
 [25] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::FileIO.Formatted, ::String, ::ImmersNurbs.ImmersoNurbsResult{3})
    @ FileIO ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:196
 [26] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::Symbol, ::String, ::String, ::Vararg{Any}; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ FileIO ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:185
 [27] action
    @ ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:185 [inlined]
 [28] #save#20
    @ ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:129 [inlined]
 [29] save
    @ ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:125 [inlined]
 [30] create_grid()
    @ Main ~/Documents/julia_tmp_codes/setup_immerso_3dcomp/result_depth_check/run_depth1/gen.jl:293
 [31] top-level scope
    @ ~/Documents/julia_tmp_codes/setup_immerso_3dcomp/result_depth_check/run_depth1/gen.jl:340
Stacktrace:
  [1] handle_error(e::ArgumentError, q::Base.PkgId, bt::Vector{Union{Ptr{Nothing}, Base.InterpreterIP}})
    @ FileIO ~/.julia/packages/FileIO/BE7iZ/src/error_handling.jl:61
  [2] handle_exceptions(exceptions::Vector{Tuple{Any, Union{Base.PkgId, Module}, Vector}}, action::String)
    @ FileIO ~/.julia/packages/FileIO/BE7iZ/src/error_handling.jl:56
  [3] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::FileIO.Formatted, ::String, ::Vararg{Any}; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ FileIO ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:228
  [4] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::FileIO.Formatted, ::String, ::ImmersNurbs.ImmersoNurbsResult{3})
    @ FileIO ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:196
  [5] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::Symbol, ::String, ::String, ::Vararg{Any}; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ FileIO ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:185
  [6] action
    @ ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:185 [inlined]
  [7] #save#20
    @ ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:129 [inlined]
  [8] save
    @ ~/.julia/packages/FileIO/BE7iZ/src/loadsave.jl:125 [inlined]
  [9] create_grid()
    @ Main ~/Documents/julia_tmp_codes/setup_immerso_3dcomp/result_depth_check/run_depth1/gen.jl:293
 [10] top-level scope
    @ ~/Documents/julia_tmp_codes/setup_immerso_3dcomp/result_depth_check/run_depth1/gen.jl:340
in expression starting at /home/elias/Documents/julia_tmp_codes/setup_immerso_3dcomp/result_depth_check/run_depth1/gen.jl:340
julia> versioninfo()
Julia Version 1.9.0
Commit 8e630552924 (2023-05-07 11:25 UTC)
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 8 × Intel(R) Core(TM) i7-8665U CPU @ 1.90GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-14.0.6 (ORCJIT, skylake)
  Threads: 4 on 8 virtual cores
Environment:
  JULIA_PARDISO = /home/elias/Documents/julia_tmp_codes/paradiso_test
JLD2 v0.4.33

I tried making a MWE, but was not able to reproduce it. Is there any other information I can provide?. Or can the stacktrace provide enough information.

JonasIsensee commented 11 months ago

From your stacktrace I gather that somewhere in your FEM (?) structures there's a Dict{Int, Nothing} which I find questionable but this should of course work.

The error doesn't actually seem to occur within JLD2 code but rather in the Base.collect call. However, this works just fine for me.

d = Dict{Int, Nothing}(1=>nothing, 2=>nothing)
collect(d)
lijas commented 11 months ago

In the REPL, I was able to "dissect" my data struct, and it seems like it is a Set{Int} that is causing the problem :thinking: So it seems like it is an issue with JLD2?

julia> trimmed_voxels
Set{Int64} with 7915 elements:
  4986
  14215
  7144
  6073
  4576
  12778
  9773
  1956
  15889
  14288
  8690
  5975
  2841
  2876
  12059
  687
  1090
  9802
  15833
  12255
  11280
  11251
  12715
  422
  4888
  15859
  7451
  4030
  551
  11373
  15931
  14832
  ⋮ 

julia> @save "matrix_data.jld2" trimmed_voxels
ERROR: ArgumentError: destination has fewer elements than required
Stacktrace:
  [1] copyto!(dest::Vector{Pair{Int64, Nothing}}, src::Dict{Int64, Nothing})
    @ Base ./abstractarray.jl:947
  [2] _collect
....

Edit: I think You are correct, if I do collect(trimmed_voxels.dict) I get the same error.

lijas commented 11 months ago

I was pushing to a Set{Int} in a threaded loop, but sets are not thread-safe, so the error was somehow related to that .