search

JuliaDataCubes / YAXArrays.jl

Yet Another XArray-like Julia package
https://juliadatacubes.github.io/YAXArrays.jl/
Other
99 stars 16 forks source link

Reading in variable containing different length strings throws error #299

Open alex-s-gardner opened 1 year ago

alex-s-gardner commented 1 year ago 
julia> mydataset.dem_names[1]
"AST_L1A_00307282000230229_final.zip"
julia> mydataset.dem_names[:]
241-element YAXArray{String,1} with dimensions: 
  Ti Sampled{DateTime} DateTime[2000-07-28T00:00:00, …, 2019-08-18T00:00:00] ForwardOrdered Irregular Points
name: dem_names
Error showing value of type YAXArray{String, 1, YAXArrayBase.NetCDFVariable{String, 1}, Tuple{DimensionalData.Dimensions.Ti{DimensionalData.Dimensions.LookupArrays.Sampled{DateTime, Vector{DateTime}, DimensionalData.Dimensions.LookupArrays.ForwardOrdered, DimensionalData.Dimensions.LookupArrays.Irregular{Tuple{Nothing, Nothing}}, DimensionalData.Dimensions.LookupArrays.Points, DimensionalData.Dimensions.LookupArrays.NoMetadata}}}}:
ERROR: Type String does not have a definite size.
Stacktrace:
  [1] sizeof(x::Type)
    @ Base ./essentials.jl:559
  [2] cubesize(c::YAXArray{String, 1, YAXArrayBase.NetCDFVariable{String, 1}, Tuple{DimensionalData.Dimensions.Ti{DimensionalData.Dimensions.LookupArrays.Sampled{DateTime, Vector{DateTime}, DimensionalData.Dimensions.LookupArrays.ForwardOrdered, DimensionalData.Dimensions.LookupArrays.Irregular{Tuple{Nothing, Nothing}}, DimensionalData.Dimensions.LookupArrays.Points, DimensionalData.Dimensions.LookupArrays.NoMetadata}}}})
    @ YAXArrays.Cubes ~/Documents/GitHub/YAXArrays.jl/src/Cubes/Cubes.jl:496
  [3] show_after(io::IOContext{Base.TTY}, mime::MIME{Symbol("text/plain")}, c::YAXArray{String, 1, YAXArrayBase.NetCDFVariable{String, 1}, Tuple{DimensionalData.Dimensions.Ti{DimensionalData.Dimensions.LookupArrays.Sampled{DateTime, Vector{DateTime}, DimensionalData.Dimensions.LookupArrays.ForwardOrdered, DimensionalData.Dimensions.LookupArrays.Irregular{Tuple{Nothing, Nothing}}, DimensionalData.Dimensions.LookupArrays.Points, DimensionalData.Dimensions.LookupArrays.NoMetadata}}}})
    @ YAXArrays.Cubes ~/Documents/GitHub/YAXArrays.jl/src/Cubes/Cubes.jl:509
  [4] show(io::IOContext{Base.TTY}, mime::MIME{Symbol("text/plain")}, A::YAXArray{String, 1, YAXArrayBase.NetCDFVariable{String, 1}, Tuple{DimensionalData.Dimensions.Ti{DimensionalData.Dimensions.LookupArrays.Sampled{DateTime, Vector{DateTime}, DimensionalData.Dimensions.LookupArrays.ForwardOrdered, DimensionalData.Dimensions.LookupArrays.Irregular{Tuple{Nothing, Nothing}}, DimensionalData.Dimensions.LookupArrays.Points, DimensionalData.Dimensions.LookupArrays.NoMetadata}}}})
    @ DimensionalData ~/.julia/packages/DimensionalData/pS9IE/src/array/show.jl:27
  [5] (::REPL.var"#55#56"{REPL.REPLDisplay{REPL.LineEditREPL}, MIME{Symbol("text/plain")}, Base.RefValue{Any}})(io::Any)
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:276
  [6] with_repl_linfo(f::Any, repl::REPL.LineEditREPL)
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:557
  [7] display(d::REPL.REPLDisplay, mime::MIME{Symbol("text/plain")}, x::Any)
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:262
  [8] display(d::REPL.REPLDisplay, x::Any)
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:281
  [9] display(x::Any)
    @ Base.Multimedia ./multimedia.jl:340
 [10] print_response(errio::IO, response::Any, show_value::Bool, have_color::Bool, specialdisplay::Union{Nothing, AbstractDisplay})
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:0
 [11] (::REPL.var"#57#58"{REPL.LineEditREPL, Pair{Any, Bool}, Bool, Bool})(io::Any)
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:287
 [12] with_repl_linfo(f::Any, repl::REPL.LineEditREPL)
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:557
 [13] print_response(repl::REPL.AbstractREPL, response::Any, show_value::Bool, have_color::Bool)
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:285
 [14] (::REPL.var"#do_respond#80"{Bool, Bool, REPL.var"#93#103"{REPL.LineEditREPL, REPL.REPLHistoryProvider}, REPL.LineEditREPL, REPL.LineEdit.Prompt})(s::REPL.LineEdit.MIState, buf::Any, ok::Bool)
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:899
 [15] (::VSCodeServer.var"#98#101"{REPL.var"#do_respond#80"{Bool, Bool, REPL.var"#93#103"{REPL.LineEditREPL, REPL.REPLHistoryProvider}, REPL.LineEditREPL, REPL.LineEdit.Prompt}})(mi::REPL.LineEdit.MIState, buf::IOBuffer, ok::Bool)
    @ VSCodeServer ~/.vscode/extensions/julialang.language-julia-1.47.2/scripts/packages/VSCodeServer/src/repl.jl:122
 [16] #invokelatest#2
    @ ./essentials.jl:816 [inlined]
 [17] invokelatest
    @ ./essentials.jl:813 [inlined]
 [18] run_interface(terminal::REPL.Terminals.TextTerminal, m::REPL.LineEdit.ModalInterface, s::REPL.LineEdit.MIState)
    @ REPL.LineEdit ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/LineEdit.jl:2647
 [19] run_frontend(repl::REPL.LineEditREPL, backend::REPL.REPLBackendRef)
    @ REPL ~/.julia/juliaup/julia-1.9.0-rc3+0.aarch64.apple.darwin14/share/julia/stdlib/v1.9/REPL/src/REPL.jl:1300
 [20] (::REPL.var"#62#68"{REPL.LineEditREPL, REPL.REPLBackendRef})()
    @ REPL ./task.jl:514
felixcremer commented 1 year ago

It seems as if this is happening in the show method and therefore you could try to suppress the printing with a semicolon at the end. I think, it is not that the Strings are different length but more that the printing expects that the values in the cube have all the same expected size and for Strings this is not defined. But should these dem_names actually be in the data or is it that the dem_names should be used as input for one Dimension?

alex-s-gardner commented 1 year ago

If I suppress printing then I just get a warning:

mydataset.dem_names[:];
┌ Warning: Can not determine size of element type. Using DiskArrays.fallback_element_size[] = 100 bytes
└ @ DiskArrays ~/.julia/packages/DiskArrays/dnhik/src/chunks.jl:287

should these dem_names actually be in the data or is it that the dem_names should be used as input for one Dimension

In this dataset dem_names[1] is the path to the file used to populate all fields at time[1]