search

JuliaIO / JLD2.jl

HDF5-compatible file format in pure Julia
Other
547 stars 85 forks source link

StackOverflowError with recursive Dict types #427

Closed dpinol closed 8 months ago

dpinol commented 1 year ago

It looks like recursive struct definitions work fine (I tested with Union{Nothing, Node} and Vector{Node}) but with Dict I get a crash

using JLD2

struct Node
    children::Dict{Int, Node}
end

save_object("kk.jld2", Node(Dict{Int, Node}()))
cc = load_object("kk.jld2")

I have julia 1.8.2 & JLD2 v0.4.25

thanks

julia> cc = load_object("kk.jld2")
ERROR: StackOverflowError:
Stacktrace:
     [1] jltype(f::JLD2.JLDFile{JLD2.MmapIO}, cdt::JLD2.CommittedDatatype)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/data/reconstructing_datatypes.jl:22
     [2] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.ReadDataspace, datatype_class::UInt8, datatype_offset::Int64, layout::JLD2.DataLayout, filters::JLD2.FilterPipeline, header_offset::JLD2.RelOffset, attributes::Nothing)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/datasets.jl:183
     [3] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.ReadDataspace, datatype_class::UInt8, datatype_offset::Int64, layout::JLD2.DataLayout, filters::JLD2.FilterPipeline) (repeats 2 times)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/datasets.jl:179
     [4] read_attr_data(f::JLD2.JLDFile{JLD2.MmapIO}, attr::JLD2.ReadAttribute)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/datasets.jl:136
     [5] jltype(f::JLD2.JLDFile{JLD2.MmapIO}, cdt::JLD2.CommittedDatatype)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/data/reconstructing_datatypes.jl:56
     [6] types_from_refs(f::JLD2.JLDFile{JLD2.MmapIO}, ptr::Ptr{Nothing})
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/data/reconstructing_datatypes.jl:313
     [7] jlconvert(rr::JLD2.ReadRepresentation{DataType, JLD2.OnDiskRepresentation{(0, 16), Tuple{String, Vector{Any}}, Tuple{JLD2.Vlen{String}, JLD2.Vlen{JLD2.RelOffset}}, 32}()}, f::JLD2.JLDFile{JLD2.MmapIO}, ptr::Ptr{Nothing}, header_offset::JLD2.RelOffset)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/data/reconstructing_datatypes.jl:329
     [8] read_scalar(f::JLD2.JLDFile{JLD2.MmapIO}, rr::JLD2.ReadRepresentation{DataType, JLD2.OnDiskRepresentation{(0, 16), Tuple{String, Vector{Any}}, Tuple{JLD2.Vlen{String}, JLD2.Vlen{JLD2.RelOffset}}, 32}()}, header_offset::JLD2.RelOffset)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/dataio.jl:37
     [9] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, rr::Any, read_dataspace::Tuple{JLD2.ReadDataspace, JLD2.RelOffset, JLD2.DataLayout, JLD2.FilterPipeline}, attributes::Nothing)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/datasets.jl:238
    [10] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.ReadDataspace, datatype_class::UInt8, datatype_offset::Int64, layout::JLD2.DataLayout, filters::JLD2.FilterPipeline, header_offset::JLD2.RelOffset, attributes::Nothing)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/datasets.jl:194
    [11] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.ReadDataspace, datatype_class::UInt8, datatype_offset::Int64, layout::JLD2.DataLayout, filters::JLD2.FilterPipeline) (repeats 2 times)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/datasets.jl:179
    [12] read_attr_data(f::JLD2.JLDFile{JLD2.MmapIO}, attr::JLD2.ReadAttribute)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/datasets.jl:136
    [13] jltype(f::JLD2.JLDFile{JLD2.MmapIO}, cdt::JLD2.CommittedDatatype)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/data/reconstructing_datatypes.jl:56
    [14] constructrr(f::JLD2.JLDFile{JLD2.MmapIO}, T::DataType, dt::JLD2.CompoundDatatype, attrs::Vector{JLD2.ReadAttribute}, hard_failure::Bool)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/data/reconstructing_datatypes.jl:170
    [15] constructrr(f::JLD2.JLDFile{JLD2.MmapIO}, T::DataType, dt::JLD2.CompoundDatatype, attrs::Vector{JLD2.ReadAttribute})
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/data/reconstructing_datatypes.jl:133
    [16] jltype(f::JLD2.JLDFile{JLD2.MmapIO}, cdt::JLD2.CommittedDatatype)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/data/reconstructing_datatypes.jl:73
--- the last 11 lines are repeated 1146 more times ---
 [12623] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.ReadDataspace, datatype_class::UInt8, datatype_offset::Int64, layout::JLD2.DataLayout, filters::JLD2.FilterPipeline, header_offset::JLD2.RelOffset, attributes::Vector{JLD2.ReadAttribute})
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/datasets.jl:183
 [12624] load_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, offset::JLD2.RelOffset)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/datasets.jl:125
 [12625] getindex(g::JLD2.Group{JLD2.JLDFile{JLD2.MmapIO}}, name::String)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/groups.jl:109
 [12626] getindex
       @ ~/.julia/packages/JLD2/HnW0g/src/JLD2.jl:451 [inlined]
 [12627] (::JLD2.var"#77#78"{String})(file::JLD2.JLDFile{JLD2.MmapIO})
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/loadsave.jl:214
 [12628] jldopen(::Function, ::String, ::Vararg{String}; kws::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/loadsave.jl:4
 [12629] jldopen
       @ ~/.julia/packages/JLD2/HnW0g/src/loadsave.jl:1 [inlined]
 [12630] load_object(filename::String)
       @ JLD2 ~/.julia/packages/JLD2/HnW0g/src/loadsave.jl:210
JonasIsensee commented 1 year ago

This is a very tricky and tbh rather niche error I tried fixing this, but it is not obvious how to do it.

In the meantime there are two workarounds:

dpinol commented 1 year ago

thank you so much! Actually I wanted to make my structs mutable anyway :-)