search

JuliaDataCubes / EarthDataLab.jl

Julia interface for Reading from the Earth System Datacube
http://earthsystemdatacube.net
Other
33 stars 14 forks source link

filterTSFFT error "ScaleAxis" unknown #250

Closed noralinscheid closed 3 years ago

noralinscheid commented 3 years ago

In the newer ESDL / YAXArray versions the filterTSFFT fails because "ScaleAxis" is unknown, I assume it should become a CategoricalAxis.. it is now still defined in TSdecomposition.jl as

outdims = OutDims("time",(c,p)->ScaleAxis(["Trend", "Long-Term Variability", "Annual Cycle", "Fast Oscillations"]))

Julia v1.6 ESDL v 0.9.0 YAXArrayBase v0.2.1 YAXArrays v0.1.2

cc = Cube("/scratch/DataCube/v2.0.0/esdc-8d-0.25deg-184x90x90-2.0.0.zarr/")
ctiny = cc[lon=(10,11),lat = (50,51), var="gross",time=2005:2010]
cfft = filterTSFFT(ctiny)

UndefVarError: ScaleAxis not defined

Stacktrace:
  [1] (::ESDL.Proc.var"#24#27")(c::Tuple{YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}}, p::Tuple{Int64, Dict{Int64, Distributed.Future}})
    @ ESDL.Proc /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/ESDL/izxrc/src/TSDecomposition.jl:38
  [2] getOutAxis(desc::YAXArrays.Cubes.Axes.ByFunction, axlist::Vector{CubeAxis}, incubes::Tuple{YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}}, pargs::Tuple{Int64, Dict{Int64, Distributed.Future}}, f::Function)
    @ YAXArrays.Cubes.Axes /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/YAXArrays/FgHc0/src/Cubes/Axes.jl:221
  [3] (::YAXArrays.DAT.var"#32#33"{Vector{CubeAxis}, Tuple{YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}}, Tuple{Int64, Dict{Int64, Distributed.Future}}, typeof(filterTSFFT)})(i::YAXArrays.Cubes.Axes.ByFunction)
    @ YAXArrays.DAT /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/YAXArrays/FgHc0/src/DAT/DAT.jl:131
  [4] map
    @ ./tuple.jl:214 [inlined]
  [5] getOutAxis(desc::Tuple{YAXArrays.Cubes.Axes.ByName, YAXArrays.Cubes.Axes.ByFunction}, inAxes::Vector{CubeAxis}, incubes::Tuple{YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}}, pargs::Tuple{Int64, Dict{Int64, Distributed.Future}}, f::Function)
    @ YAXArrays.DAT /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/YAXArrays/FgHc0/src/DAT/DAT.jl:131
  [6] YAXArrays.DAT.OutputCube(desc::OutDims, inAxes::Vector{CubeAxis}, incubes::Tuple{YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}}, pargs::Tuple{Int64, Dict{Int64, Distributed.Future}}, f::Function)
    @ YAXArrays.DAT /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/YAXArrays/FgHc0/src/DAT/DAT.jl:133
  [7] #36
    @ /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/YAXArrays/FgHc0/src/DAT/DAT.jl:182 [inlined]
  [8] #4
    @ ./generator.jl:36 [inlined]
  [9] iterate
    @ ./generator.jl:47 [inlined]
 [10] collect(itr::Base.Generator{Base.Iterators.Zip{Tuple{UnitRange{Int64}, Tuple{OutDims}}}, Base.var"#4#5"{YAXArrays.DAT.var"#36#39"{Tuple{YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}}, typeof(filterTSFFT), Tuple{Int64, Dict{Int64, Distributed.Future}}, Vector{CubeAxis}}}})
    @ Base ./array.jl:678
 [11] map(::Function, ::UnitRange{Int64}, ::Tuple{OutDims})
    @ Base ./abstractarray.jl:2383
 [12] YAXArrays.DAT.DATConfig(cdata::Tuple{YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}}, indims::InDims, outdims::OutDims, inplace::Bool, max_cache::Float64, fu::Function, ispar::Bool, include_loopvars::Bool, nthreads::Vector{Int64}, addargs::Tuple{Int64, Dict{Int64, Distributed.Future}}, kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ YAXArrays.DAT /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/YAXArrays/FgHc0/src/DAT/DAT.jl:181
 [13] mapCube(::typeof(filterTSFFT), ::Tuple{YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}}, ::Int64, ::Vararg{Any, N} where N; max_cache::Float64, indims::InDims, outdims::OutDims, inplace::Bool, ispar::Bool, debug::Bool, include_loopvars::Bool, showprog::Bool, nthreads::Vector{Int64}, loopchunksize::Dict{Any, Any}, kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ YAXArrays.DAT /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/YAXArrays/FgHc0/src/DAT/DAT.jl:299
 [14] mapCube(::Function, ::YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}, ::Int64, ::Vararg{Any, N} where N; kwargs::Base.Iterators.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:indims, :outdims), Tuple{InDims, OutDims}}})
    @ YAXArrays.DAT /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/YAXArrays/FgHc0/src/DAT/DAT.jl:208
 [15] filterTSFFT(c::YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}}; kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ ESDL.Proc /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/ESDL/izxrc/src/TSDecomposition.jl:49
 [16] filterTSFFT(c::YAXArray{Union{Missing, Float32}, 1, SubArray{Union{Missing, Float32}, 1, Array{Union{Missing, Float32}, 3}, Tuple{Base.OneTo{Int64}, Int64, Int64}, true}, Vector{CubeAxis}})
    @ ESDL.Proc /Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/packages/ESDL/izxrc/src/TSDecomposition.jl:37
 [17] top-level scope
    @ In[14]:1
 [18] eval
    @ ./boot.jl:360 [inlined]
 [19] include_string(mapexpr::typeof(REPL.softscope), mod::Module, code::String, filename::String)
    @ Base ./loading.jl:1094
noralinscheid commented 3 years ago

Fixed the issue above in the fftfix branch, but now another error occurs:

using Pkg

]add https://github.com/esa-esdl/ESDL.jl #fftfix

using ESDL

ds = Cube("/scratch/DataCube/v2.0.0/esdc-8d-0.25deg-184x90x90-2.0.0.zarr/")
csub = ds[lon=(10,11),lat=(50,51),var=["leaf_area_index"],time=2005:2010]
cfft = filterTSFFT(csub)

Error: Error during running loop: TaskFailedException(Task (failed) @0x00007f7a7922c550) TaskFailedException nested task error: MethodError: no method matching iterate(::Distributed.RemoteException)

Status `/Net/Groups/BGI/scratch/nlinsch/julia_atacama_depots/environments/ESDL_test_fftfix/Project.toml`
  [359177bc] ESDL v0.9.0 `https://github.com/esa-esdl/ESDL.jl#fftfix`
  [7a1cc6ca] FFTW v1.4.1
  [91a5bcdd] Plots v1.13.2
  [c21b50f5] YAXArrays v0.1.2
  [10745b16] Statistics
meggart commented 3 years ago

I looks like the plan_fft! function is not detected. This is due to an erroneous import here: https://github.com/esa-esdl/ESDL.jl/blob/bcb6f05b88d245d9a70464c753038ec74779a65d/src/TSDecomposition.jl#L1 i.e. the exclamation mark is missing.

meggart commented 3 years ago

@noralinscheid did my suggestion work for you?

noralinscheid commented 3 years ago

Yes! After also importing Statistics.cov and mean (for linreg function), the decomposition is running again! Than you!!