JuliaDataCubes / YAXArrays.jl

Yet Another XArray-like Julia package
https://juliadatacubes.github.io/YAXArrays.jl/
Other
103 stars 18 forks source link

Can't specify netcdf output in OutDims #422

Open felixcremer opened 3 months ago

felixcremer commented 3 months ago

When I am trying to set the output of mapCube via the OutDims path and backend combination to a netcdf file I get the following error:

rqatrend(path::AbstractString; thresh=2, outpath=tempname()*".zarr",overwrite=false, kwargs...) = rqatrend(Cube(path); thresh, outpath, overwrite, kwargs...)
function rqatrend(cube; thresh=2, outpath=tempname() * ".zarr", overwrite=false, kwargs...)
    @show outpath
    mapCube(rqatrend, cube, thresh; indims=InDims("Time"), outdims=OutDims(;outtype=Float16, path=outpath, overwrite, kwargs...))
end

julia>     rqa = RQADeforestation.rqatrend(c; thresh=1., outpath=outnc, backend=:netcdf)
outpath = "/home/fcremer/Documents/rqapython/rqatrend.nc"
ERROR: ArgumentError: array must be non-empty
Stacktrace:
  [1] _throw_argerror(s::String)
    @ Base ./array.jl:383
  [2] popfirst!
    @ ./array.jl:1530 [inlined]
  [3] jl2nc(t::DataType)
    @ NetCDF ~/.julia/packages/NetCDF/BeMqn/src/NetCDF.jl:91
  [4] getNCType
    @ ~/.julia/packages/NetCDF/BeMqn/src/NetCDF.jl:101 [inlined]
  [5] NcVar(name::String, dimin::Vector{…}; atts::Dict{…}, t::DataType, compress::Int64, chunksize::Tuple{…})
    @ NetCDF ~/.julia/packages/NetCDF/BeMqn/src/NetCDF.jl:256
  [6] nccreate(::String, ::String, ::String, ::Vararg{…}; atts::Dict{…}, gatts::Dict{…}, compress::Int64, t::DataType, mode::UInt16, chunksize::Tuple{…})
    @ NetCDF ~/.julia/packages/NetCDF/BeMqn/src/NetCDF.jl:1210
  [7] #add_var#128
    @ ~/.julia/packages/YAXArrayBase/R6Frw/src/datasets/netcdf.jl:63 [inlined]
  [8] create_dataset(T::Type, path::String, gatts::Dict{…}, dimnames::Vector{…}, dimvals::Vector{…}, dimattrs::Vector{…}, vartypes::Vector{…}, varnames::Vector{…}, vardims::Vector{…}, varattrs::Vector{…}, varchunks::Vector{…}; kwargs::@Kwargs{})
    @ YAXArrayBase ~/.julia/packages/YAXArrayBase/R6Frw/src/datasets/datasetinterface.jl:62
  [9] create_dataset(T::Type, path::String, gatts::Dict{…}, dimnames::Vector{…}, dimvals::Vector{…}, dimattrs::Vector{…}, vartypes::Vector{…}, varnames::Vector{…}, vardims::Vector{…}, varattrs::Vector{…}, varchunks::Vector{…})
    @ YAXArrayBase ~/.julia/packages/YAXArrayBase/R6Frw/src/datasets/datasetinterface.jl:53
 [10] createdataset(DS::Type, axlist::Vector{…}; path::String, persist::Nothing, T::Type, chunksize::Tuple{…}, chunkoffset::Tuple{…}, overwrite::Bool, properties::Dict{…}, globalproperties::Dict{…}, datasetaxis::String, layername::String, kwargs::@Kwargs{})
    @ YAXArrays.Datasets ~/Documents/open-call-use-case-pangeo-julia/dev/YAXArrays/src/DatasetAPI/Datasets.jl:731
 [11] generateOutCube(::Type{…}, elementtype::Type, oc::YAXArrays.DAT.OutputCube, loopcachesize::Tuple{…}, co::Tuple{…}; kwargs::@Kwargs{…})
    @ YAXArrays.DAT ~/Documents/open-call-use-case-pangeo-julia/dev/YAXArrays/src/DAT/DAT.jl:830
 [12] generateOutCube(oc::YAXArrays.DAT.OutputCube, ispar::Base.RefValue{…}, max_cache::Float64, loopcachesize::Tuple{…}, co::Tuple{…})
    @ YAXArrays.DAT ~/Documents/open-call-use-case-pangeo-julia/dev/YAXArrays/src/DAT/DAT.jl:869
 [13] (::YAXArrays.DAT.var"#121#122"{YAXArrays.DAT.DATConfig{…}, Tuple{…}, Tuple{…}})(c::YAXArrays.DAT.OutputCube)
    @ YAXArrays.DAT ~/.julia/packages/YAXArrays/b5XBB/src/DAT/DAT.jl:864
 [14] #62
    @ ./tuple.jl:627 [inlined]
 [15] BottomRF
    @ ./reduce.jl:86 [inlined]
 [16] afoldl
    @ ./operators.jl:544 [inlined]
 [17] _foldl_impl
    @ ./reduce.jl:68 [inlined]
 [18] foldl_impl
    @ ./reduce.jl:48 [inlined]
 [19] mapfoldl_impl(f::typeof(identity), op::Base.var"#62#63"{…}, nt::Nothing, itr::Tuple{…})
    @ Base ./reduce.jl:44
 [20] mapfoldl(f::Function, op::Function, itr::Tuple{YAXArrays.DAT.OutputCube}; init::Nothing)
    @ Base ./reduce.jl:175
 [21] mapfoldl
    @ ./reduce.jl:175 [inlined]
 [22] foldl
    @ ./reduce.jl:198 [inlined]
 [23] foreach
    @ ./tuple.jl:627 [inlined]
 [24] generateOutCubes(dc::YAXArrays.DAT.DATConfig{1, 1})
    @ YAXArrays.DAT ~/Documents/open-call-use-case-pangeo-julia/dev/YAXArrays/src/DAT/DAT.jl:863
 [25] mapCube(fu::typeof(RQADeforestation.rqatrend), cdata::Tuple{…}, addargs::Float64; max_cache::Float64, indims::InDims, outdims::OutDims, inplace::Bool, ispar::Bool, debug::Bool, include_loopvars::Bool, showprog::Bool, irregular_loopranges::Bool, nthreads::Vector{…}, loopchunksize::Dict{…}, do_gc::Bool, kwargs::@Kwargs{})
    @ YAXArrays.DAT ~/Documents/open-call-use-case-pangeo-julia/dev/YAXArrays/src/DAT/DAT.jl:495
 [26] mapCube
    @ ~/.julia/packages/YAXArrays/b5XBB/src/DAT/DAT.jl:311 [inlined]
 [27] #rqatrend#52
    @ ~/Documents/open-call-use-case-pangeo-julia/src/analysis.jl:60 [inlined]
 [28] top-level scope
    @ REPL[68]:1
Some type information was truncated. Use `show(err)` to see complete types.
felixcremer commented 3 months ago

This problem also appears, when I try to save the resulting YAXArray to netcdf via savecube.

felixcremer commented 3 months ago

The problem seems to come from trying to use Float16 and I think this is actually a NetCDF issue, because it could give a clearer error message rather than this cryptic array needs to be non-empty.

Balinus commented 3 months ago

Indeed, as far as I know, Float16 is not supported:

https://github.com/Unidata/netcdf-c/issues/2359

https://github.com/pydata/xarray/issues/1877