slimgroup / JUDI.jl

Julia Devito inversion.
https://slimgroup.github.io/JUDI.jl
MIT License
96 stars 29 forks source link

RTM with `subsampling_factor` with `space_order` cause error #163

Closed kerim371 closed 11 months ago

kerim371 commented 1 year ago

Hi,

The following code snippet lead to an error during RTM. The error only appears when both space_order=16 and subsampling_factor=2 are set. If space_order=8 then it works fine. But I have tried JUDI's example modeling_basic_2D.jl with the same space_order and subsampling_factor parameters and there was no such error. So maybe the fact that I also use OOC geometry and segy_scan have some sense...

container = segy_scan(prestk_dir, prestk_file, ["SourceX", "SourceY", "GroupX", "GroupY", "RecGroupElevation", "SourceSurfaceElevation", "dt"])
d_obs = judiVector(container; segy_depth_key = "RecGroupElevation")

# Set up wavelet and source vector
src_geometry = Geometry(container; key = "source", segy_depth_key = "SourceSurfaceElevation")
q = judiVector(src_geometry, wavelet)

t_sub = 2

# JUDI options
jopt = JUDI.Options(
    space_order=16,
    limit_m = true,
    buffer_size = buffer_size,
    optimal_checkpointing=false,
    IC = "isic",
    subsampling_factor=t_sub)

# Setup operators
Pr = judiProjection(d_obs.geometry)
F = judiModeling(model0; options=jopt)
Ps = judiProjection(q.geometry)
J = judiJacobian(Pr*F*adjoint(Ps), q)

# RTM
rtm = adjoint(J)*d_obs

The error:

┌ Info: JOLI method error for combination:
│   left_name = "(judiJacobian{Float32, :adjoint_born, judiDataSourceModeling{Float32, :forward}}*DataMute{Float32, :reflection})"
│   left_type = JOLI.joLinearOperator{Float32, Float32}
│   right_name = "non-JOLI"
└   right_type = Vector{SegyIO.IBMFloat32} (alias for Array{SegyIO.IBMFloat32, 1})
ERROR: JOLI.joUtilsException("*(jo,vec) not implemented or type mismatch")
Stacktrace:
 [1] jo_method_error(L::JOLI.joLinearOperator{Float32, Float32}, R::Vector{SegyIO.IBMFloat32}, s::String)
   @ JOLI ~/Documents/Colada/r/julia-1.6/.julia/packages/JOLI/SZxFH/src/joUtils.jl:53
 [2] *(A::JOLI.joLinearOperator{Float32, Float32}, v::Vector{SegyIO.IBMFloat32})
   @ JOLI ~/Documents/Colada/r/julia-1.6/.julia/packages/JOLI/SZxFH/src/joAbstractOperator/base_functions.jl:80
 [3] *(J::JOLI.joLinearOperator{Float32, Float32}, x::judiVector{Float32, SeisCon})
   @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/Types/abstract.jl:84
 [4] *(::judiJacobian{Float32, :adjoint_born, judiDataSourceModeling{Float32, :forward}}, ::DataMute{Float32, :reflection}, ::judiVector{Float32, SeisCon})
   @ Base ./operators.jl:560
 [5] top-level scope
   @ /mnt/HDD_2TB/VikingGraben/line12/inversion/scripts/rtm.jl:164

caused by: PyError ($(Expr(:escape, :(ccall(#= /home/kerim/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/pyfncall.jl:43 =# @pysym(:PyObject_Call), PyPtr, (PyPtr, PyPtr, PyPtr), o, pyargsptr, kw))))) <class 'devito.exceptions.InvalidArgument'>
InvalidArgument('OOB detected due to x_M=597')
  File "/home/kerim/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/pysource/interface.py", line 359, in J_adjoint
    return J_adjoint_standard(model, src_coords, wavelet, rec_coords, recin,
  File "/home/kerim/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/pysource/interface.py", line 486, in J_adjoint_standard
    g, Iv, _ = gradient(model, residual, rec_coords, u, space_order=space_order, ic=ic,
  File "/home/kerim/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/pysource/propagators.py", line 127, in gradient
    summary = op(**kw)
  File "/home/kerim/Documents/Colada/r/python-install/lib/python3.9/site-packages/devito/operator/operator.py", line 686, in __call__
    return self.apply(**kwargs)
  File "/home/kerim/Documents/Colada/r/python-install/lib/python3.9/site-packages/devito/operator/operator.py", line 752, in apply
    args = self.arguments(**kwargs)
  File "/home/kerim/Documents/Colada/r/python-install/lib/python3.9/site-packages/devito/operator/operator.py", line 601, in arguments
    args = self._prepare_arguments(**kwargs)
  File "/home/kerim/Documents/Colada/r/python-install/lib/python3.9/site-packages/devito/operator/operator.py", line 552, in _prepare_arguments
    p._arg_check(args, self._dspace[p])
  File "/home/kerim/Documents/Colada/r/python-install/lib/python3.9/site-packages/devito/types/dense.py", line 1424, in _arg_check
    super(TimeFunction, self)._arg_check(args, intervals)
  File "/home/kerim/Documents/Colada/r/python-install/lib/python3.9/site-packages/devito/types/dense.py", line 874, in _arg_check
    i._arg_check(args, s, intervals[i])
  File "/home/kerim/Documents/Colada/r/python-install/lib/python3.9/site-packages/devito/types/dimension.py", line 293, in _arg_check
    raise InvalidArgument("OOB detected due to %s=%d" % (self.max_name,

Stacktrace:
  [1] pyerr_check
    @ ~/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/exception.jl:62 [inlined]
  [2] pyerr_check
    @ ~/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/exception.jl:66 [inlined]
  [3] _handle_error(msg::String)
    @ PyCall ~/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/exception.jl:83
  [4] macro expansion
    @ ~/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/exception.jl:97 [inlined]
  [5] #107
    @ ~/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/pyfncall.jl:43 [inlined]
  [6] disable_sigint
    @ ./c.jl:458 [inlined]
  [7] __pycall!
    @ ~/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/pyfncall.jl:42 [inlined]
  [8] _pycall!(ret::PyCall.PyObject, o::PyCall.PyObject, args::Tuple{PyCall.PyObject, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}}, nargs::Int64, kw::PyCall.PyObject)
    @ PyCall ~/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/pyfncall.jl:29
  [9] _pycall!(ret::PyCall.PyObject, o::PyCall.PyObject, args::Tuple{PyCall.PyObject, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}}, kwargs::Base.Iterators.Pairs{Symbol, Any, NTuple{10, Symbol}, NamedTuple{(:fw, :t_sub, :space_order, :checkpointing, :freq_list, :ic, :is_residual, :dft_sub, :f0, :illum), Tuple{Bool, Int64, Int64, Bool, Nothing, String, Bool, Int64, Float32, Bool}}})
    @ PyCall ~/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/pyfncall.jl:11
 [10] pycall(::PyCall.PyObject, ::Type{Tuple{PyCall.PyArray, PyCall.PyObject, PyCall.PyObject}}, ::PyCall.PyObject, ::Vararg{Any, N} where N; kwargs::Base.Iterators.Pairs{Symbol, Any, NTuple{10, Symbol}, NamedTuple{(:fw, :t_sub, :space_order, :checkpointing, :freq_list, :ic, :is_residual, :dft_sub, :f0, :illum), Tuple{Bool, Int64, Int64, Bool, Nothing, String, Bool, Int64, Float32, Bool}}})
    @ PyCall ~/Documents/Colada/r/julia-1.6/.julia/packages/PyCall/ygXW2/src/pyfncall.jl:80
 [11] (::JUDI.var"#198#199"{Base.Iterators.Pairs{Symbol, Any, NTuple{10, Symbol}, NamedTuple{(:fw, :t_sub, :space_order, :checkpointing, :freq_list, :ic, :is_residual, :dft_sub, :f0, :illum), Tuple{Bool, Int64, Int64, Bool, Nothing, String, Bool, Int64, Float32, Bool}}}, PyCall.PyObject, Tuple{PyCall.PyObject, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}}, DataType})()
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/Modeling/python_interface.jl:52
 [12] (::JUDI.var"#1#2"{JUDI.var"#198#199"{Base.Iterators.Pairs{Symbol, Any, NTuple{10, Symbol}, NamedTuple{(:fw, :t_sub, :space_order, :checkpointing, :freq_list, :ic, :is_residual, :dft_sub, :f0, :illum), Tuple{Bool, Int64, Int64, Bool, Nothing, String, Bool, Int64, Float32, Bool}}}, PyCall.PyObject, Tuple{PyCall.PyObject, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}}, DataType}})()
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/JUDI.jl:68
 [13] lock(f::JUDI.var"#1#2"{JUDI.var"#198#199"{Base.Iterators.Pairs{Symbol, Any, NTuple{10, Symbol}, NamedTuple{(:fw, :t_sub, :space_order, :checkpointing, :freq_list, :ic, :is_residual, :dft_sub, :f0, :illum), Tuple{Bool, Int64, Int64, Bool, Nothing, String, Bool, Int64, Float32, Bool}}}, PyCall.PyObject, Tuple{PyCall.PyObject, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}}, DataType}}, l::ReentrantLock)
    @ Base ./lock.jl:187
 [14] pylock(f::JUDI.var"#198#199"{Base.Iterators.Pairs{Symbol, Any, NTuple{10, Symbol}, NamedTuple{(:fw, :t_sub, :space_order, :checkpointing, :freq_list, :ic, :is_residual, :dft_sub, :f0, :illum), Tuple{Bool, Int64, Int64, Bool, Nothing, String, Bool, Int64, Float32, Bool}}}, PyCall.PyObject, Tuple{PyCall.PyObject, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}, Matrix{Float32}}, DataType})
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/JUDI.jl:65
 [15] wrapcall_grad(::PyCall.PyObject, ::PyCall.PyObject, ::Vararg{Any, N} where N; kw::Base.Iterators.Pairs{Symbol, Any, NTuple{10, Symbol}, NamedTuple{(:fw, :t_sub, :space_order, :checkpointing, :freq_list, :ic, :is_residual, :dft_sub, :f0, :illum), Tuple{Bool, Int64, Int64, Bool, Nothing, String, Bool, Int64, Float32, Bool}}})
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/Modeling/python_interface.jl:51
 [16] devito_interface(modelPy::PyCall.PyObject, srcGeometry::GeometryIC{Float32}, srcData::Matrix{Float32}, recGeometry::GeometryIC{Float32}, recData::Matrix{Float32}, dm::Nothing, options::JUDIOptions, illum::Bool, fw::Bool)
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/Modeling/python_interface.jl:143
 [17] time_modeling(model_full::Model, srcGeometry::GeometryOOC{Float32}, srcData::Matrix{Float32}, recGeometry::GeometryOOC{Float32}, recData::Matrix{Float32}, dm::Nothing, op::Symbol, options::JUDIOptions, fw::Bool)
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/Modeling/time_modeling_serial.jl:39
 [18] propagate(F::judiJacobian{Float32, :adjoint_born, judiDataSourceModeling{Float32, :forward}}, q::judiVector{Float32, Matrix{Float32}})
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/Modeling/propagation.jl:9
 [19] run_and_reduce(func::Function, #unused#::Nothing, nsrc::Int64, arg_func::JUDI.var"#228#229"{judiJacobian{Float32, :adjoint_born, judiDataSourceModeling{Float32, :forward}}, judiVector{Float32, Matrix{Float32}}})
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/Modeling/propagation.jl:32
 [20] multi_src_propagate(F::judiJacobian{Float32, :adjoint_born, judiDataSourceModeling{Float32, :forward}}, q::judiVector{Float32, Matrix{Float32}})
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/Modeling/propagation.jl:67
 [21] *
    @ ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/LinearOperators/operators.jl:173 [inlined]
 [22] (::JOLI.var"#173#177"{judiJacobian{Float32, :adjoint_born, judiDataSourceModeling{Float32, :forward}}, DataMute{Float32, :reflection}})(v1::judiVector{Float32, SeisCon})
    @ JOLI ~/Documents/Colada/r/julia-1.6/.julia/packages/JOLI/SZxFH/src/joAbstractLinearOperator/base_functions.jl:139
 [23] *(J::JOLI.joLinearOperator{Float32, Float32}, x::judiVector{Float32, SeisCon})
    @ JUDI ~/Documents/Colada/r/julia-1.6/.julia/dev/JUDI/src/TimeModeling/Types/abstract.jl:84
 [24] *(::judiJacobian{Float32, :adjoint_born, judiDataSourceModeling{Float32, :forward}}, ::DataMute{Float32, :reflection}, ::judiVector{Float32, SeisCon})
    @ Base ./operators.jl:560
 [25] top-level scope
    @ /mnt/HDD_2TB/VikingGraben/line12/inversion/scripts/rtm.jl:164
mloubout commented 1 year ago

There must be some type of out of bound FD or your gemoetry, would check that receivers and such stay inside the domain. Bit odd it only breaks for this combination of orders and subsampling.

Would you be able to make a simple self contained breaking example

kerim371 commented 1 year ago

I will prepare it in few days

kerim371 commented 1 year ago

Here is the cutted modeling_basic_2D.jl that run into the same error.

As I figured out, the it is caused only when opt = Options(space_order=16, subsampling_factor=2, IC = "isic"). If IC is not set then then there is no error. If IC="isic" and space_order=8 there is no error as well. But when these three are combined together they cause an error.

#' # Modeling and inversion with JUDI
#' ---
#' title: Overview of JUDI modeling and inversion usage
#' author: Mathias Louboutin, Philipp Witte
#' date: April 2022
#' ---

#' This example script is written using [Weave.jl](https://github.com/JunoLab/Weave.jl) and can be converted to different format for documentation and usage
#' This example is converted to a markdown file for the documentation.

#' # Import JUDI, Linear algebra utilities and Plotting
using JUDI, PyPlot, LinearAlgebra

#+ echo = false; results = "hidden"
close("all")

#' # Create a JUDI model structure
#' In JUDI, a `Model` structure contains the grid information (origin, spacing, number of gridpoints)
#' and the physical parameters. The squared slowness is always required as the base physical parameter for propagation. In addition,
#' JUDI supports additional physical representations. First we accept `density` that can either be a direct input `Model(n, d, o, m, rho)` or
#' an optional keyword argument `Model(n,d,o,m;rho=rho)`. Second, we also provide VTI/TTI kernels parametrized by the THomsen parameters that can be input as keyword arguments
#' `Model(n,d,o,m; rho=rho, epsilon=epsilon;delta=delta,theta=theta,phi=phi)`. Because the thomsen parameters are optional the propagator wil lonloy use the ones provided. 
#' For example `Model(n,d,o,m; rho=rho, epsilon=epsilon;delta=delta)` will infer a VTI propagation

#' ## Create discrete parameters
# Set up model structure
n = (120, 100)   # (x,y,z) or (x,z)
d = (10., 10.)
o = (0., 0.)

# Velocity [km/s]
v = ones(Float32,n) .+ 0.5f0
v0 = ones(Float32,n) .+ 0.5f0
v[:,Int(round(end/2)):end] .= 3.5f0
rho = (v0 .+ .5f0) ./ 2

# Slowness squared [s^2/km^2]
m = (1f0 ./ v).^2
m0 = (1f0 ./ v0).^2
dm = vec(m0 - m)

# Setup model structure
nsrc = 2    # number of sources
model = Model(n, d, o, m)
model0 = Model(n, d, o, m0)

#' # Create acquisition geometry
#' In this simple usage example, we create a simple acquisiton by hand. In practice the acquisition geometry will be defined by the dataset
#' beeing inverted. We show in a spearate tutorial how to use [SegyIO.jl](https://github.com/slimgroup/SegyIO.jl) to handle SEGY seismic datasets in JUDI.

#' ## Create source and receivers positions at the surface
# Set up receiver geometry
nxrec = 120
xrec = range(0f0, stop=(n[1]-1)*d[1], length=nxrec)
yrec = 0f0 # WE have to set the y coordiante to zero (or any number) for 2D modeling
zrec = range(d[1], stop=d[1], length=nxrec)

# receiver sampling and recording time
timeD = 1250f0   # receiver recording time [ms]
dtD = 2f0    # receiver sampling interval [ms]

# Set up receiver structure
recGeometry = Geometry(xrec, yrec, zrec; dt=dtD, t=timeD, nsrc=nsrc)

#' The source geometry is a but different. Because we want to create a survey with `nsrc` shot records, we need
#' to convert the vector of sources postions `[s0, s1, ... sn]` into an array of array [[s0], [s1], ...] so that
#' JUDI understands that this is a set of indepednet `nsrc`

xsrc = convertToCell(range(0f0, stop=(n[1]-1)*d[1], length=nsrc))
ysrc = convertToCell(range(0f0, stop=0f0, length=nsrc))
zsrc = convertToCell(range(d[1], stop=d[1], length=nsrc))

# Set up source structure
srcGeometry = Geometry(xsrc, ysrc, zsrc; dt=dtD, t=timeD)

#' # Source judiVector
#' Finally, with the geometry defined, we can create a source wavelet (a simple Ricker wavelet here) a our first `judiVector`
#' In JUDI, a `judiVector` is the core structure that represent a acquisition-geometry based dataset. This structure encapsulate
#' the physical locations (trace coordinates) and corrsponding data trace in a source-based structure. for a given `judiVector` `d` then
#' `d[1]` will be the shot record for the first source, or in the case of the source term, the first source wavelet and its positon.

# setup wavelet
f0 = 0.01f0     # kHz
wavelet = ricker_wavelet(timeD, dtD, f0)
q = judiVector(srcGeometry, wavelet)

#' # Modeling
#' With our survey and subsurface model setup, we can now model and image seismic data. We first define a few options. In this tutorial
#' we will choose to compute gradients/images subsampling the forward wavefield every two time steps `subsampling_factor=2` and we fix the computational
#' time step to be `1ms` wiuth `dt_comp=1.0` know to satisfy the CFL condition for this simple example. In practice, when `dt_comp` isn't provided, JUDI will compute the CFL
#' condition for the propagation.

# Setup options
opt = Options(space_order=16, subsampling_factor=2, IC = "isic")

#' Linear Operators
#' The core idea behind JUDI is to abstract seismic inverse problems in term of linear algebra. In its simplest form, seismic inversion can be formulated as
#' ```math
#' \underset{\mathbf{m}}{\text{argmin}} \ \ \phi(\mathbf{m}) = \frac{1}{2} ||\mathbf{P}_r \mathbf{F}(\mathbf{m}) \mathbf{P}_s^{\top} \mathbf{q} - \mathbf{d} ||_2^2 \\
#' \text{   } \\
#' \nabla_{\mathbf{m}} \phi(\mathbf{m}) = \mathbf{J}(\mathbf{m}, \mathbf{q})^{\top} (\mathbf{P}_r \mathbf{F}(\mathbf{m}) \mathbf{P}_s^{\top} \mathbf{q} - \mathbf{d})
#' ```
#' 
#' where $\mathbf{P}_r$ is the receiver projection (measurment operator) and $\mathbf{P}_s^{\top}$ is the source injection operator (adjoint of measurment at the source location).
#' Therefore, we bastracted these operation to be able to define these operators

# Setup operators
Pr = judiProjection(recGeometry)
F = judiModeling(model; options=opt)
F0 = judiModeling(model0; options=opt)
Ps = judiProjection(srcGeometry)
J = judiJacobian(Pr*F0*adjoint(Ps), q)

#' # Model and image data

#' We first model synthetic data using our defined source and true model 
# Nonlinear modeling
dobs = Pr*F*adjoint(Ps)*q

#' Plot the shot record
fig = figure()
imshow(dobs.data[1], vmin=-1, vmax=1, cmap="PuOr", extent=[xrec[1], xrec[end], timeD/1000, 0], aspect="auto")
xlabel("Receiver position (m)")
ylabel("Time (s)")
title("Synthetic data")
display(fig)

#' Because we have abstracted the linear algebra, we can solve the adjoint wave-equation as well 
#' where the data becomes the source. This adjoint solve will be part of the imaging procedure.
# # Adjoint
qad = Ps*adjoint(F)*adjoint(Pr)*dobs

#' We can easily now test the adjointness of our operator with the standard dot test. Because we
#' intend to conserve our linear algebra abstraction, `judiVector` implements all the necessary linear 
#' algebra functions such as dot product or norm to be used directly.
# <x, F'y>
dot1 = dot(q, qad)
# <F x, y>
dot2 = dot(dobs, dobs)
# Compare
@show dot1, dot2, (dot2 - dot2)/(dot1 + dot2)

#' # Inversion
#' Our main goal is to provide an inversion framework for seismic inversion. To this end, as shown earlier,
#' users can easily define the Jacobian operator and compute an RTM image (i.e FWI gradient) with a simple matrix-vector product.
#' Once again, we provide both the Jacobian and its adjoint and we can compute Born linearized data.

# Linearized modeling J*dm
dD = J*dm
# Adjoint jacobian, RTM image
rtm = adjoint(J)*dD

#' We show the linearized data.
fig = figure()
imshow(dD.data[1], vmin=-1, vmax=1, cmap="PuOr", extent=[xrec[1], xrec[end], timeD/1000, 0], aspect="auto")
xlabel("Receiver position (m)")
ylabel("Time (s)")
title("Linearized data")
display(fig)

#' And the RTM image
fig = figure()
imshow(rtm', vmin=-1e2, vmax=1e2, cmap="Greys", extent=[0, (n[1]-1)*d[1], (n[2]-1)*d[2], 0 ], aspect="auto")
xlabel("Lateral position(m)")
ylabel("Depth (m)")
title("RTM image")
display(fig)