search

yeesian / ArchGDAL.jl

A high level API for GDAL - Geospatial Data Abstraction Library
https://yeesian.github.io/ArchGDAL.jl/stable/
Other
141 stars 27 forks source link

Can we handle Complex{Int16} raster values? KeyError: key ArchGDAL.GDT_CInt16 not found. #326

Closed rafaqz closed 2 years ago

rafaqz commented 2 years ago

Since I can't transfer the issue here, reposting this from @lupemba over at Rasters.jl https://github.com/rafaqz/Rasters.jl/issues/319

The file is here: https://we.tl/t-pKuLOIMfT5

This is a complex SAR tiff file, also 1GB download.

@yeesian @mathieu17g do you understand this error?

It's not so easy to follow why this definition is missing through the @convert macros. But I imagine this should come back as a Complex{Int16} do we even handle complex numbers?

fn = "s1a-iw1-slc-vv-20220918t074922-20220918t074947-045056-056232-004.tiff"
ArchGDAL.read(fn) do ds
    ArchGDAL.read(ds)
end

ERROR: KeyError: key ArchGDAL.GDT_CInt16 not found
Stacktrace:
  [1] getindex
    @ ./dict.jl:819 [inlined]
  [2] convert
    @ ~/.julia/packages/ArchGDAL/1I2FZ/src/utils.jl:131 [inlined]
  [3] pixeltype(ptr::Ptr{Nothing})
    @ ArchGDAL ~/.julia/packages/ArchGDAL/1I2FZ/src/types.jl:177
  [4] ArchGDAL.IRasterBand(ptr::Ptr{Nothing}; ownedby::ArchGDAL.Dataset)
    @ ArchGDAL ~/.julia/packages/ArchGDAL/1I2FZ/src/types.jl:205
  [5] getband
    @ ~/.julia/packages/ArchGDAL/1I2FZ/src/dataset.jl:697 [inlined]
  [6] (::ArchGDAL.var"#77#78"{ArchGDAL.Dataset})(i::Int64)
    @ ArchGDAL ~/.julia/packages/ArchGDAL/1I2FZ/src/dataset.jl:850
  [7] iterate
    @ ./generator.jl:47 [inlined]
  [8] _collect(c::UnitRange{Int64}, itr::Base.Generator{UnitRange{Int64}, ArchGDAL.var"#77#78"{ArchGDAL.Dataset}}, #unused#::Base.EltypeUnknown, isz::Base.HasShape{1})
    @ Base ./array.jl:807
  [9] collect_similar
    @ ./array.jl:716 [inlined]
 [10] map
    @ ./abstractarray.jl:2933 [inlined]
 [11] pixeltype(ds::ArchGDAL.Dataset)
    @ ArchGDAL ~/.julia/packages/ArchGDAL/1I2FZ/src/dataset.jl:849
 [12] ArchGDAL.RasterDataset(ds::ArchGDAL.Dataset)
    @ ArchGDAL ~/.julia/packages/ArchGDAL/1I2FZ/src/raster/array.jl:32
 [13] #unsafe_readraster#108
    @ ~/.julia/packages/ArchGDAL/1I2FZ/src/raster/array.jl:126 [inlined]
 [14] unsafe_readraster
    @ ~/.julia/packages/ArchGDAL/1I2FZ/src/raster/array.jl:126 [inlined]
 [15] readraster(f::ComposedFunction{typeof(Rasters.cleanreturn), Rasters.var"#51#52"{Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, String}}, args::String; kwargs::B
ase.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ ArchGDAL ~/.julia/packages/ArchGDAL/1I2FZ/src/context.jl:265
 [16] readraster(f::Function, args::String)
    @ ArchGDAL ~/.julia/packages/ArchGDAL/1I2FZ/src/context.jl:264
 [17] _open(f::Function, ::Type{Rasters.GDALfile}, filename::String; write::Bool, kw::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Rasters ~/.julia/dev/Rasters/src/sources/gdal.jl:306
 [18] _open(f::Function, ::Type{Rasters.GDALfile}, filename::String)
    @ Rasters ~/.julia/dev/Rasters/src/sources/gdal.jl:292
 [19] _open(f::Function, filename::String; kw::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Rasters ~/.julia/dev/Rasters/src/convenience.jl:20
 [20] _open(f::Function, filename::String)
    @ Rasters ~/.julia/dev/Rasters/src/convenience.jl:19
 [21] Raster(filename::String; name::Nothing, key::Nothing, kw::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Rasters ~/.julia/dev/Rasters/src/array.jl:259
 [22] Raster(filename::String)
    @ Rasters ~/.julia/dev/Rasters/src/array.jl:258
 [23] top-level scope
    @ REPL[67]:1
yeesian commented 2 years ago

I can reproduce the error (by following comment#1), but I'm not sure how to resolve it yet.

Here's what I have so far:

  | | |_| | | | (_| |  |  Version 1.8.0 (2022-08-17)
 _/ |\__'_|_|_|\__'_|  |  Official https://julialang.org/ release
|__/                   |

julia> using ArchGDAL, GDAL
[ Info: Precompiling ArchGDAL [c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3]
┌ Warning: User provided progress functions are unsupported on this architecture.
└ @ ArchGDAL ~/.julia/dev/ArchGDAL/src/utils.jl:233
┌ Warning: User provided progress functions are unsupported on this architecture.
└ @ ArchGDAL ~/.julia/dev/ArchGDAL/src/utils.jl:233
┌ Warning: User provided progress functions are unsupported on this architecture.
└ @ ArchGDAL ~/.julia/dev/ArchGDAL/src/utils.jl:233
┌ Warning: User provided progress functions are unsupported on this architecture.
└ @ ArchGDAL ~/.julia/dev/ArchGDAL/src/utils.jl:233
┌ Warning: User provided progress functions are unsupported on this architecture.
└ @ ArchGDAL ~/.julia/dev/ArchGDAL/src/utils.jl:233

julia> Base.convert(GDAL.GDALDataType, ArchGDAL.GDT_CInt16)
GDT_CInt16::GDALDataType = 0x00000008

julia> Base.convert(ArchGDAL.GDALDataType, GDAL.GDT_CInt16)
GDT_CInt16::GDALDataType = 8

julia> macroexpand(Main, :(ArchGDAL.@convert(
                  GDALDataType::GDAL.GDALDataType,
                  GDT_Unknown::GDAL.GDT_Unknown,
                  GDT_Byte::GDAL.GDT_Byte,
                  GDT_UInt16::GDAL.GDT_UInt16,
                  GDT_Int16::GDAL.GDT_Int16,
                  GDT_UInt32::GDAL.GDT_UInt32,
                  GDT_Int32::GDAL.GDT_Int32,
                  GDT_Float32::GDAL.GDT_Float32,
                  GDT_Float64::GDAL.GDT_Float64,
                  GDT_CInt16::GDAL.GDT_CInt16,
                  GDT_CInt32::GDAL.GDT_CInt32,
                  GDT_CFloat32::GDAL.GDT_CFloat32,
                  GDT_CFloat64::GDAL.GDT_CFloat64,
                  GDT_TypeCount::GDAL.GDT_TypeCount,
              )))
quote
    const GDALDataType_to_GDAL_GDALDataType_map = (ArchGDAL.Base).ImmutableDict([Pair(GDT_Unknown, GDAL.GDT_Unknown), Pair(GDT_Byte, GDAL.GDT_Byte), Pair(GDT_UInt16, GDAL.GDT_UInt16), Pair(GDT_Int16, GDAL.GDT_Int16), Pair(GDT_UInt32, GDAL.GDT_UInt32), Pair(GDT_Int32, GDAL.GDT_Int32), Pair(GDT_Float32, GDAL.GDT_Float32), Pair(GDT_Float64, GDAL.GDT_Float64), Pair(GDT_CInt16, GDAL.GDT_CInt16), Pair(GDT_CInt32, GDAL.GDT_CInt32), Pair(GDT_CFloat32, GDAL.GDT_CFloat32), Pair(GDT_CFloat64, GDAL.GDT_CFloat64), Pair(GDT_TypeCount, GDAL.GDT_TypeCount)]...)
    function (ArchGDAL.Base).convert(::ArchGDAL.Type{GDAL.GDALDataType}, var"#26#ft"::GDALDataType)
        #= /Users/yeesian/.julia/dev/ArchGDAL/src/utils.jl:130 =#
        #= /Users/yeesian/.julia/dev/ArchGDAL/src/utils.jl:131 =#
        return GDALDataType_to_GDAL_GDALDataType_map[var"#26#ft"]
    end
    const GDAL_GDALDataType_to_GDALDataType_map = (ArchGDAL.Base).ImmutableDict([Pair(GDAL.GDT_Unknown, GDT_Unknown), Pair(GDAL.GDT_Byte, GDT_Byte), Pair(GDAL.GDT_UInt16, GDT_UInt16), Pair(GDAL.GDT_Int16, GDT_Int16), Pair(GDAL.GDT_UInt32, GDT_UInt32), Pair(GDAL.GDT_Int32, GDT_Int32), Pair(GDAL.GDT_Float32, GDT_Float32), Pair(GDAL.GDT_Float64, GDT_Float64), Pair(GDAL.GDT_CInt16, GDT_CInt16), Pair(GDAL.GDT_CInt32, GDT_CInt32), Pair(GDAL.GDT_CFloat32, GDT_CFloat32), Pair(GDAL.GDT_CFloat64, GDT_CFloat64), Pair(GDAL.GDT_TypeCount, GDT_TypeCount)]...)
    function (ArchGDAL.Base).convert(::ArchGDAL.Type{GDALDataType}, var"#28#ft"::GDAL.GDALDataType)
        #= /Users/yeesian/.julia/dev/ArchGDAL/src/utils.jl:163 =#
        #= /Users/yeesian/.julia/dev/ArchGDAL/src/utils.jl:164 =#
        return GDAL_GDALDataType_to_GDALDataType_map[var"#28#ft"]
    end
end

I.e. It seems that the KeyError is being triggered inside the convert macro here: https://github.com/yeesian/ArchGDAL.jl/blob/66066333e7a6c06f3a1086ec1de0b7d4ac3ca79c/src/utils.jl#L122-L131

However,

So I'm still dumbfounded for now.

yeesian commented 2 years ago

Oh I see now, we'll need to define the conversion in https://github.com/yeesian/ArchGDAL.jl/blob/66066333e7a6c06f3a1086ec1de0b7d4ac3ca79c/src/types.jl#L292-L304

E.g. with https://github.com/yeesian/ArchGDAL.jl/commit/66c284e77820bacadc5dae1ba631ef91aba063a0, the example in https://github.com/yeesian/ArchGDAL.jl/issues/326#issue-1386480491 will run (although I can't vouch for the correctness of the output).