Draft: explicit datasets

This implements a macro for a declaritive description of header messages and makes the library use them. One nice side effect is nearly free pretty printing of header messages :

Demo header message declaration:

@pseudostruct HM_DATASPACE begin
    version::UInt8 = 2
    dimensionality::UInt8 = length(kw.dimensions)
    flags::UInt8
    (version == 2) && dataspace_type::UInt8
    if version == 1
        dataspace_type::@computed(DS_V1)
        @skip(5)
    end
    dim_offset::@Offset
    dimensions::NTuple{Int(dimensionality), Int64}
    isset(flags,0) && max_dimension_size::NTuple{Int(dimensionality), Int64}
end

It supports

the standard field declaration is name::Type
for construction it may be useful to provide default values name::Type = value
the value may be a function of other values. (must be explicitly given and have syntax kw.<dependency>) e.g. dimensionality::UInt8 = length(kw.dimensions)
fields which don't always exist: (version == 2) && dataspace_type::UInt8 (also if but not else or elseif)
Computed fields - useful if a derived quantity is of interest e.g. dataspace_type::@computed(DS_V1) where DS_V1 is the value dataspace_type should take when it is not encoded in the file
@skip for marking empty / reserved bytes
dim_offset::@Offset returns the offset in the file at this location
types can be computed from other values e.g. dimensions::NTuple{Int(dimensionality), Int64}
there are a few more options..

Demo Header message printing

``` julia> f JLDFile /tmp/jl_Y3lEf5/test.jld (read/write) ├─📂 test_group_1 │ ├─🔢 x1 │ └─🔢 x2 ├─📂 test_group_2 │ └─🔢 x1 ├─📂 test_group_3 │ ├─🔢 x1 │ └─📂 contained_group (1 entry) ├─📂 test_group_4 (2 entries) └─ ⋯ (4 more entries) julia> JLD2.print_header_messages(f, f.root_group_offset) ┌─ Header Message: HM_LINK_INFO │ ┌─ offset: RelOffset(1824) │ │ size: 18 │ └─ flags: 0 │ version: 0 │ flags: 0 │ fractal_heap_address: UNDEFINED_ADDRESS └─ v2_btree_name_index: UNDEFINED_ADDRESS ┌─ Header Message: HM_GROUP_INFO │ ┌─ offset: RelOffset(1846) │ │ size: 2 │ └─ flags: 0 │ version: 0 └─ flags: 0 ┌─ Header Message: HM_LINK_MESSAGE │ ┌─ offset: RelOffset(1852) │ │ size: 24 │ └─ flags: 0 │ version: 1 │ flags: 16 │ link_name_charset: 1 │ link_name_len: 12 │ link_name: test_group_1 └─ target: RelOffset(447) ┌─ Header Message: HM_LINK_MESSAGE │ ┌─ offset: RelOffset(1880) │ │ size: 24 │ └─ flags: 0 │ version: 1 │ flags: 16 │ link_name_charset: 1 │ link_name_len: 12 │ link_name: test_group_2 └─ target: RelOffset(590) ┌─ Header Message: HM_LINK_MESSAGE │ ┌─ offset: RelOffset(1908) │ │ size: 24 │ └─ flags: 0 │ version: 1 │ flags: 16 │ link_name_charset: 1 │ link_name_len: 12 │ link_name: test_group_3 └─ target: RelOffset(888) ┌─ Header Message: HM_LINK_MESSAGE │ ┌─ offset: RelOffset(1936) │ │ size: 24 │ └─ flags: 0 │ version: 1 │ flags: 16 │ link_name_charset: 1 │ link_name_len: 12 │ link_name: test_group_4 └─ target: RelOffset(1193) ┌─ Header Message: HM_LINK_MESSAGE │ ┌─ offset: RelOffset(1964) │ │ size: 23 │ └─ flags: 0 │ version: 1 │ flags: 16 │ link_name_charset: 1 │ link_name_len: 11 │ link_name: empty_group └─ target: RelOffset(1349) ┌─ Header Message: HM_LINK_MESSAGE │ ┌─ offset: RelOffset(1991) │ │ size: 38 │ └─ flags: 0 │ version: 1 │ flags: 16 │ link_name_charset: 1 │ link_name_len: 26 │ link_name: group_with_one_group_child └─ target: RelOffset(1659) ┌─ Header Message: HM_NIL │ ┌─ offset: RelOffset(2033) │ │ size: 16 │ └─ flags: 0 └─ ```

Dataset pretty printing

This can print the written structures and also the encoded julia type name without needing to reconstruct them. This should help when normal loading does not work.

julia> jldsave("test.jld2"; d=rand(ComplexF64, 50,100))

julia> jldopen("test.jld2") do f
           display(JLD2.get_dataset(f, "d"))
       end
┌─ Dataset: "d" at RelOffset(4551)
│  datatype: JLD2.SharedDatatype (committed)
│       committed at: RelOffset(4312)
│       written structure: @NamedTuple{re::Float64, im::Float64}
│       type name: Base.Complex{Float64}
│  dataspace:
│       type: Simple
│       dimensions: (100, 50)
│  layout:
│       class: LcContiguous
└─

julia> struct MyStruct{T}
           a::Int
           b::T
       end

julia> struct B; x::String; end

julia> jldopen("test.jld2", "w") do f
           dset = JLD2.create_dataset(f, "d")
           JLD2.add_attribute(dset, :description, "A description of what this is for.")
           JLD2.write_dataset(dset, MyStruct(1, B("TestString")))
       end
RelOffset(4632)

julia> jldopen("test.jld2", "r") do f
           display(JLD2.get_dataset(f, "d"))
       end
┌─ Dataset: "d" at RelOffset(4632)
│  datatype: JLD2.SharedDatatype (committed)
│       committed at: RelOffset(4427)
│       written structure: @NamedTuple{a::Int64, b::@NamedTuple{x::String}}
│       type name: MyStruct{B}
│               a::Int64
│               b::B
│  dataspace:
│       type: Scalar
│       dimensions: ()
│  layout:
│       class: LcCompact
│  Attributes:
│       description = "A description of what this is for."
└─

Summary of new (experimental) API

get_dataset(f, name)
create_dataset(f, name)
attributes(dset)
add_attribute(dset, symbolname, payload)

ToDo

[ ] Improve compile time
[ ] Improve datasets: API, tests, documentation. This is an experimental feature. Could be done in a future PR

Files	Patch %	Lines
src/committed_datatype_introspection.jl	0.00%	103 Missing :warning:
src/explicit_datasets.jl	63.86%	86 Missing :warning:
src/object_headers.jl	69.92%	40 Missing :warning:
src/macros_utils.jl	89.26%	22 Missing :warning:
src/misc.jl	54.54%	15 Missing :warning:
src/types.jl	60.00%	14 Missing :warning:
src/headermessages.jl	25.00%	6 Missing :warning:
src/datalayouts.jl	93.15%	5 Missing :warning:
src/datatypes.jl	90.32%	3 Missing :warning:
src/fractal_heaps.jl	75.00%	3 Missing :warning:
... and 3 more

JuliaIO / JLD2.jl