search

JuliaIO / JLD2.jl

HDF5-compatible file format in pure Julia
Other
537 stars 84 forks source link

error when serializing expression containing `Int128` #511

Closed lucaferranti closed 7 months ago

lucaferranti commented 7 months ago

An Int128 literal is a macro call in disguise

julia> ex = :(1000000000000000000000)
:(1000000000000000000000)

julia> dump(ex)
Expr
  head: Symbol macrocall
  args: Array{Any}((3,))
    1: GlobalRef
      mod: Module Core
      name: Symbol @int128_str
      binding: Ptr{Nothing} @0x00007f7501d5a138
    2: Nothing nothing
    3: String "1000000000000000000000"

Currently, when trying to save to a jld2 file the expression above, it errors

julia> ex = :(1000000000000000000000);

julia> save_object("tmp.jld2", ex)
ERROR: Type Module does not have a definite size.
Stacktrace:
  [1] sizeof
    @ ./essentials.jl:559 [inlined]
  [2] hasdata(T::DataType, encounteredtypes::Vector{DataType})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:39
  [3] hasdata
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:39 [inlined]
  [4] odr(#unused#::Type{Module})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:592
  [5] objodr
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:119 [inlined]
  [6] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, x::Module, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:645
  [7] write_ref_mutable(f::JLD2.JLDFile{JLD2.MmapIO}, x::Module, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:651
  [8] write_ref
    @ ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:659 [inlined]
  [9] h5convert!
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:298 [inlined]
 [10] macro expansion
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:237 [inlined]
 [11] h5convert!
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:237 [inlined]
 [12] write_data(io::JLD2.MmapIO, f::JLD2.JLDFile{JLD2.MmapIO}, data::GlobalRef, odr::JLD2.OnDiskRepresentation{(0, 8, 24), Tuple{Module, Symbol, Ptr{Nothing}}, Tuple{JLD2.RelOffset, JLD2.Vlen{String}, nothing}, 24}, #unused#::JLD2.HasReferences, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/dataio.jl:96
 [13] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.WriteDataspace{0, Tuple{}}, datatype::JLD2.CommittedDatatype, odr::JLD2.OnDiskRepresentation{(0, 8, 24), Tuple{Module, Symbol, Ptr{Nothing}}, Tuple{JLD2.RelOffset, JLD2.Vlen{String}, nothing}, 24}, data::GlobalRef, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:578
 [14] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, x::GlobalRef, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:646
 [15] write_ref(f::JLD2.JLDFile{JLD2.MmapIO}, x::GlobalRef, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:661
 [16] h5convert!
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:298 [inlined]
 [17] write_data(io::JLD2.MmapIO, f::JLD2.JLDFile{JLD2.MmapIO}, data::Vector{Any}, odr::Type{JLD2.RelOffset}, #unused#::JLD2.HasReferences, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/dataio.jl:163
 [18] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.WriteDataspace{1, Tuple{JLD2.WrittenAttribute{JLD2.WriteDataspace{0, Tuple{}}, JLD2.BasicDatatype, JLD2.RelOffset}}}, datatype::JLD2.BasicDatatype, odr::Type{JLD2.RelOffset}, data::Vector{Any}, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}}, compress::Bool)
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:525
 [19] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, x::Vector{Any}, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}}, compress::Bool)
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/inlineunion.jl:44
 [20] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, x::Vector{Any}, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/inlineunion.jl:36
 [21] write_ref_mutable
    @ ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:651 [inlined]
 [22] write_ref
    @ ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:659 [inlined]
 [23] h5convert!
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:298 [inlined]
 [24] macro expansion
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:237 [inlined]
 [25] h5convert!(out::JLD2.IndirectPointer, #unused#::JLD2.OnDiskRepresentation{(0, 16), Tuple{Symbol, Vector{Any}}, Tuple{JLD2.Vlen{String}, JLD2.RelOffset}, 24}, file::JLD2.JLDFile{JLD2.MmapIO}, x::Expr, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:237
 [26] write_data(io::JLD2.MmapIO, f::JLD2.JLDFile{JLD2.MmapIO}, data::Expr, odr::JLD2.OnDiskRepresentation{(0, 16), Tuple{Symbol, Vector{Any}}, Tuple{JLD2.Vlen{String}, JLD2.RelOffset}, 24}, #unused#::JLD2.HasReferences, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/dataio.jl:96
 [27] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.WriteDataspace{0, Tuple{}}, datatype::JLD2.CommittedDatatype, odr::JLD2.OnDiskRepresentation{(0, 16), Tuple{Symbol, Vector{Any}}, Tuple{JLD2.Vlen{String}, JLD2.RelOffset}, 24}, data::Expr, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:578
 [28] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, x::Expr, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:646
 [29] write(g::JLD2.Group{JLD2.JLDFile{JLD2.MmapIO}}, name::String, obj::Expr, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}}; compress::Nothing)
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/compression.jl:137
 [30] write (repeats 2 times)
    @ ~/.julia/packages/JLD2/u57Vt/src/compression.jl:125 [inlined]
 [31] setindex!
    @ ~/.julia/packages/JLD2/u57Vt/src/groups.jl:125 [inlined]
 [32] setindex!
    @ ~/.julia/packages/JLD2/u57Vt/src/JLD2.jl:484 [inlined]
 [33] (::JLD2.var"#77#78"{Expr})(file::JLD2.JLDFile{JLD2.MmapIO})
    @ JLD2 ~/.

My system version info

julia> versioninfo()
Julia Version 1.9.4
Commit 8e5136fa297 (2023-11-14 08:46 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 12 × 12th Gen Intel(R) Core(TM) i7-1255U
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-14.0.6 (ORCJIT, alderlake)
  Threads: 1 on 12 virtual cores
Environment:
  JULIA_EDITOR = hx
(@v1.9) pkg> st JLD2
Status `~/.julia/environments/v1.9/Project.toml`
  [033835bb] JLD2 v0.4.38
lucaferranti commented 7 months ago

The problematic bit is the Module Core node in the expression tree. The following causes the same error

julia> a = Core
Core

julia> save_object("tmp.jld2", a)
ERROR: Type Module does not have a definite size.
Stacktrace:
  [1] sizeof
    @ ./essentials.jl:559 [inlined]
  [2] hasdata(T::DataType, encounteredtypes::Vector{DataType})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:39
  [3] hasdata
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:39 [inlined]
  [4] odr(#unused#::Type{Module})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:592
  [5] objodr
    @ ~/.julia/packages/JLD2/u57Vt/src/data/writing_datatypes.jl:119 [inlined]
  [6] write_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, x::Module, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/datasets.jl:645
  [7] write(g::JLD2.Group{JLD2.JLDFile{JLD2.MmapIO}}, name::String, obj::Module, wsession::JLD2.JLDWriteSession{Dict{UInt64, JLD2.RelOffset}}; compress::Nothing)
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/compression.jl:137
  [8] write (repeats 2 times)
    @ ~/.julia/packages/JLD2/u57Vt/src/compression.jl:125 [inlined]
  [9] setindex!
    @ ~/.julia/packages/JLD2/u57Vt/src/groups.jl:125 [inlined]
 [10] setindex!
    @ ~/.julia/packages/JLD2/u57Vt/src/JLD2.jl:484 [inlined]
 [11] (::JLD2.var"#77#78"{Module})(file::JLD2.JLDFile{JLD2.MmapIO})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/loadsave.jl:186
 [12] jldopen(::Function, ::String, ::Vararg{String}; kws::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/loadsave.jl:4
 [13] jldopen
    @ ~/.julia/packages/JLD2/u57Vt/src/loadsave.jl:1 [inlined]
 [14] save_object(filename::String, x::Module)
    @ JLD2 ~/.julia/packages/JLD2/u57Vt/src/loadsave.jl:185
 [15] top-level scope
    @ REPL[46]:1
JonasIsensee commented 7 months ago

Thanks for the report!

I suppose one could implement a custom serialization for module (names).

It doesn't make sense for most modules, but Core and Base might be useful.

cstjean commented 7 months ago

It doesn't make sense for most modules,

I'm curious why you say so. Isn't it the same as for functions? You can't save its "true state", but saving as a reference is useful.

JonasIsensee commented 7 months ago

It doesn't make sense for most modules,

I'm curious why you say so. Isn't it the same as for functions? You can't save its "true state", but saving as a reference is useful.

Yes, that is quite similar and I would say the same about functions... Nonetheless, for package-modules this should now work fine! (with v0.4.40)