search

SciML / SciMLSensitivity.jl

A component of the DiffEq ecosystem for enabling sensitivity analysis for scientific machine learning (SciML). Optimize-then-discretize, discretize-then-optimize, adjoint methods, and more for ODEs, SDEs, DDEs, DAEs, etc.
https://docs.sciml.ai/SciMLSensitivity/stable/
Other
333 stars 70 forks source link

Discontinuous NeuralODEs fail as soon as FunctionCallingCallbacks are used #809

Closed ThummeTo closed 1 year ago

ThummeTo commented 1 year ago

Dear @ChrisRackauckas and @frankschae,

this is strongly related to issue #564 but I can narrow the error down now, see the following MWE which basically trains a minimal NeuralODE and is using a FunctionCallingCallback that does nothing:

using Flux, DiffEqFlux
using DifferentialEquations
using DiffEqCallbacks
using DiffEqFlux: ODEFunction, basic_tgrad, ODEProblem, ZygoteVJP, InterpolatingAdjoint, solve

numStates = 2
tspan = (0.0f0, 5.0f0)
saveat = 0.0f0:0.1f0:5.0f0

optim = Adam()

x0 = [0.0f0, 0.0f0]

function f(x, p, t)
    return neuralODE.re(p)(x)
end 

function functionCalling(x, t, integrator)
    # @info "Step: $(t)"
end

callback = FunctionCallingCallback(functionCalling;
                                            func_everystep=true,
                                            func_start=true)
### ML
posData = zeros(Float64, length(saveat)) # target position data ("flat line"), not a good training target but this doens't matter here ;-)

# MSE between net output and data
function losssum(prob, neuralODE, sensealg)
    solution = solve(prob, neuralODE.args...; sensealg=sensealg, saveat=saveat, callback=CallbackSet(callback), neuralODE.kwargs...)
    posNet = collect(data[1] for data in solution.u)
    return Flux.Losses.mse(posNet, posData) 
end 

net = Chain(Dense(numStates, numStates, identity),
            Dense(numStates, numStates, identity),
            Dense(numStates, numStates, identity))

sensealgs = [ForwardDiffSensitivity(;convert_tspan=true), InterpolatingAdjoint(autojacvec=ReverseDiffVJP())] 

for sensealg in sensealgs

    print("Sensealg: $(sensealg)\n")

    global neuralODE

    neuralODE = NeuralODE(net, tspan; saveat=saveat)

    dudt_op(x, p, t) = f(x, p, t);
    ff = ODEFunction{false}(dudt_op,tgrad=basic_tgrad);
    prob = ODEProblem{false}(ff, x0, tspan, neuralODE.p);
    p_net = Flux.params(neuralODE);

   Flux.train!(()->losssum(prob, neuralODE, sensealg), p_net, Iterators.repeated((), 1), Adam()); 
end

Fails on current Julia (1.8.5) and most up-to-date-versions of SciML-Ecosystem.

Let me know if I can support. Best regards!

The error is:

ERROR: type TrackedAffect has no field funciter
Stacktrace:
  [1] getproperty
    @ .\Base.jl:38 [inlined]
  [2] functioncalling_initialize(cb::DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}, u::Vector{Float32}, t::Float32, integrator::OrdinaryDiffEq.ODEIntegrator{CompositeAlgorithm{Tuple{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{2, false, GenericLUFactorization{LinearAlgebra.RowMaximum}, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, false, Vector{Float32}, Nothing, Float32, Vector{Float32}, Float32, Float32, Float32, Float32, Vector{Vector{Float32}}, ODESolution{Float32, 2, Vector{Vector{Float32}}, Nothing, Nothing, Vector{Float32}, Vector{Vector{Vector{Float32}}}, ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Tuple{Symbol}, NamedTuple{(:callback,), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}}, SciMLBase.StandardODEProblem}, CompositeAlgorithm{Tuple{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{2, false, GenericLUFactorization{LinearAlgebra.RowMaximum}, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, OrdinaryDiffEq.CompositeInterpolationData{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Vector{Float32}}, Vector{Float32}, Vector{Vector{Vector{Float32}}}, OrdinaryDiffEq.CompositeCache{Tuple{OrdinaryDiffEq.Tsit5ConstantCache, OrdinaryDiffEq.Rosenbrock23ConstantCache{Float32, SciMLBase.TimeDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, 
Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Float32}, Vector{Float32}}, SciMLBase.UDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Float32, Vector{Float32}}, Matrix{Float32}, LinearAlgebra.LU{Float32, Matrix{Float32}, Vector{Int64}}, Nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}}, DiffEqBase.Stats, Vector{Int64}}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, OrdinaryDiffEq.CompositeCache{Tuple{OrdinaryDiffEq.Tsit5ConstantCache, OrdinaryDiffEq.Rosenbrock23ConstantCache{Float32, SciMLBase.TimeDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Float32}, Vector{Float32}}, SciMLBase.UDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Float32, Vector{Float32}}, Matrix{Float32}, LinearAlgebra.LU{Float32, Matrix{Float32}, Vector{Int64}}, Nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, OrdinaryDiffEq.DEOptions{Float32, Float32, Float32, Float32, PIController{Rational{Int64}}, typeof(DiffEqBase.ODE_DEFAULT_NORM), typeof(LinearAlgebra.opnorm), Bool, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, typeof(DiffEqBase.ODE_DEFAULT_ISOUTOFDOMAIN), typeof(DiffEqBase.ODE_DEFAULT_PROG_MESSAGE), typeof(DiffEqBase.ODE_DEFAULT_UNSTABLE_CHECK), DataStructures.BinaryHeap{Float32, DataStructures.FasterForward}, DataStructures.BinaryHeap{Float32, DataStructures.FasterForward}, Nothing, Nothing, Int64, Tuple{}, Tuple{}, Tuple{}}, Vector{Float32}, Float32, Nothing, OrdinaryDiffEq.DefaultInit})
    @ DiffEqCallbacks C:\Users\.julia\packages\DiffEqCallbacks\zjOVB\src\function_caller.jl:40
  [3] initialize!(u::Vector{Float32}, t::Float32, integrator::OrdinaryDiffEq.ODEIntegrator{CompositeAlgorithm{Tuple{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{2, false, GenericLUFactorization{LinearAlgebra.RowMaximum}, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, false, Vector{Float32}, Nothing, Float32, Vector{Float32}, Float32, Float32, Float32, Float32, Vector{Vector{Float32}}, ODESolution{Float32, 2, Vector{Vector{Float32}}, Nothing, Nothing, Vector{Float32}, Vector{Vector{Vector{Float32}}}, ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, 
typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Tuple{Symbol}, NamedTuple{(:callback,), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}}, SciMLBase.StandardODEProblem}, CompositeAlgorithm{Tuple{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{2, false, GenericLUFactorization{LinearAlgebra.RowMaximum}, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, OrdinaryDiffEq.CompositeInterpolationData{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Vector{Float32}}, Vector{Float32}, Vector{Vector{Vector{Float32}}}, OrdinaryDiffEq.CompositeCache{Tuple{OrdinaryDiffEq.Tsit5ConstantCache, OrdinaryDiffEq.Rosenbrock23ConstantCache{Float32, SciMLBase.TimeDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Float32}, Vector{Float32}}, SciMLBase.UDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Float32, Vector{Float32}}, Matrix{Float32}, LinearAlgebra.LU{Float32, Matrix{Float32}, Vector{Int64}}, Nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}}, DiffEqBase.Stats, Vector{Int64}}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, OrdinaryDiffEq.CompositeCache{Tuple{OrdinaryDiffEq.Tsit5ConstantCache, OrdinaryDiffEq.Rosenbrock23ConstantCache{Float32, SciMLBase.TimeDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Float32}, Vector{Float32}}, SciMLBase.UDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Float32, Vector{Float32}}, Matrix{Float32}, LinearAlgebra.LU{Float32, Matrix{Float32}, Vector{Int64}}, Nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, OrdinaryDiffEq.DEOptions{Float32, Float32, Float32, Float32, PIController{Rational{Int64}}, typeof(DiffEqBase.ODE_DEFAULT_NORM), typeof(LinearAlgebra.opnorm), Bool, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), 
DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, typeof(DiffEqBase.ODE_DEFAULT_ISOUTOFDOMAIN), typeof(DiffEqBase.ODE_DEFAULT_PROG_MESSAGE), typeof(DiffEqBase.ODE_DEFAULT_UNSTABLE_CHECK), DataStructures.BinaryHeap{Float32, DataStructures.FasterForward}, DataStructures.BinaryHeap{Float32, DataStructures.FasterForward}, Nothing, Nothing, Int64, Tuple{}, Tuple{}, Tuple{}}, Vector{Float32}, Float32, Nothing, OrdinaryDiffEq.DefaultInit}, any_modified::Bool, c::DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)})
    @ DiffEqBase C:\Users\.julia\packages\DiffEqBase\qPmC2\src\callbacks.jl:18
  [4] initialize!(cb::CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, u::Vector{Float32}, t::Float32, integrator::OrdinaryDiffEq.ODEIntegrator{CompositeAlgorithm{Tuple{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{2, false, GenericLUFactorization{LinearAlgebra.RowMaximum}, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, false, Vector{Float32}, Nothing, Float32, 
Vector{Float32}, Float32, Float32, Float32, Float32, Vector{Vector{Float32}}, ODESolution{Float32, 2, Vector{Vector{Float32}}, Nothing, Nothing, Vector{Float32}, Vector{Vector{Vector{Float32}}}, ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), 
DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Tuple{Symbol}, NamedTuple{(:callback,), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}}, SciMLBase.StandardODEProblem}, CompositeAlgorithm{Tuple{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{2, false, GenericLUFactorization{LinearAlgebra.RowMaximum}, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, OrdinaryDiffEq.CompositeInterpolationData{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Vector{Float32}}, Vector{Float32}, Vector{Vector{Vector{Float32}}}, OrdinaryDiffEq.CompositeCache{Tuple{OrdinaryDiffEq.Tsit5ConstantCache, OrdinaryDiffEq.Rosenbrock23ConstantCache{Float32, SciMLBase.TimeDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Float32}, Vector{Float32}}, SciMLBase.UDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Float32, Vector{Float32}}, Matrix{Float32}, LinearAlgebra.LU{Float32, Matrix{Float32}, Vector{Int64}}, Nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}}, DiffEqBase.Stats, Vector{Int64}}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, OrdinaryDiffEq.CompositeCache{Tuple{OrdinaryDiffEq.Tsit5ConstantCache, OrdinaryDiffEq.Rosenbrock23ConstantCache{Float32, SciMLBase.TimeDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Float32}, Vector{Float32}}, SciMLBase.UDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Float32, Vector{Float32}}, Matrix{Float32}, LinearAlgebra.LU{Float32, Matrix{Float32}, Vector{Int64}}, Nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, OrdinaryDiffEq.DEOptions{Float32, Float32, Float32, Float32, PIController{Rational{Int64}}, typeof(DiffEqBase.ODE_DEFAULT_NORM), typeof(LinearAlgebra.opnorm), Bool, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, typeof(DiffEqBase.ODE_DEFAULT_ISOUTOFDOMAIN), typeof(DiffEqBase.ODE_DEFAULT_PROG_MESSAGE), typeof(DiffEqBase.ODE_DEFAULT_UNSTABLE_CHECK), DataStructures.BinaryHeap{Float32, DataStructures.FasterForward}, DataStructures.BinaryHeap{Float32, 
DataStructures.FasterForward}, Nothing, Nothing, Int64, Tuple{}, Tuple{}, Tuple{}}, Vector{Float32}, Float32, Nothing, OrdinaryDiffEq.DefaultInit})
    @ DiffEqBase C:\Users\.julia\packages\DiffEqBase\qPmC2\src\callbacks.jl:7
  [5] initialize_callbacks!(integrator::OrdinaryDiffEq.ODEIntegrator{CompositeAlgorithm{Tuple{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{2, false, GenericLUFactorization{LinearAlgebra.RowMaximum}, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, false, Vector{Float32}, Nothing, Float32, Vector{Float32}, Float32, Float32, Float32, Float32, Vector{Vector{Float32}}, ODESolution{Float32, 2, Vector{Vector{Float32}}, Nothing, Nothing, Vector{Float32}, Vector{Vector{Vector{Float32}}}, ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Tuple{Symbol}, NamedTuple{(:callback,), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}}, SciMLBase.StandardODEProblem}, CompositeAlgorithm{Tuple{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{2, false, GenericLUFactorization{LinearAlgebra.RowMaximum}, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, OrdinaryDiffEq.CompositeInterpolationData{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Vector{Float32}}, Vector{Float32}, Vector{Vector{Vector{Float32}}}, OrdinaryDiffEq.CompositeCache{Tuple{OrdinaryDiffEq.Tsit5ConstantCache, OrdinaryDiffEq.Rosenbrock23ConstantCache{Float32, SciMLBase.TimeDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), 
Nothing, Nothing}, Vector{Float32}, Vector{Float32}}, SciMLBase.UDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Float32, Vector{Float32}}, Matrix{Float32}, LinearAlgebra.LU{Float32, Matrix{Float32}, Vector{Int64}}, Nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}}, DiffEqBase.Stats, Vector{Int64}}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, OrdinaryDiffEq.CompositeCache{Tuple{OrdinaryDiffEq.Tsit5ConstantCache, OrdinaryDiffEq.Rosenbrock23ConstantCache{Float32, SciMLBase.TimeDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Vector{Float32}, Vector{Float32}}, SciMLBase.UDerivativeWrapper{ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Float32, Vector{Float32}}, Matrix{Float32}, LinearAlgebra.LU{Float32, Matrix{Float32}, Vector{Int64}}, Nothing}}, OrdinaryDiffEq.AutoSwitchCache{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, OrdinaryDiffEq.DEOptions{Float32, Float32, Float32, Float32, PIController{Rational{Int64}}, typeof(DiffEqBase.ODE_DEFAULT_NORM), typeof(LinearAlgebra.opnorm), Bool, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, typeof(DiffEqBase.ODE_DEFAULT_ISOUTOFDOMAIN), typeof(DiffEqBase.ODE_DEFAULT_PROG_MESSAGE), typeof(DiffEqBase.ODE_DEFAULT_UNSTABLE_CHECK), DataStructures.BinaryHeap{Float32, DataStructures.FasterForward}, DataStructures.BinaryHeap{Float32, DataStructures.FasterForward}, Nothing, Nothing, Int64, Tuple{}, Tuple{}, Tuple{}}, Vector{Float32}, Float32, Nothing, OrdinaryDiffEq.DefaultInit}, initialize_save::Bool)
    @ OrdinaryDiffEq C:\Users\.julia\packages\OrdinaryDiffEq\FttEX\src\solve.jl:636
  [6] __init(prob::ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Tuple{Symbol}, NamedTuple{(:callback,), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}}, SciMLBase.StandardODEProblem}, alg::CompositeAlgorithm{Tuple{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{2, false, GenericLUFactorization{LinearAlgebra.RowMaximum}, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}}, AutoSwitch{Tsit5{typeof(OrdinaryDiffEq.trivial_limiter!), typeof(OrdinaryDiffEq.trivial_limiter!), Static.False}, Rosenbrock23{0, false, Nothing, typeof(OrdinaryDiffEq.DEFAULT_PRECS), Val{:forward}, true, nothing}, Rational{Int64}, Int64}}, timeseries_init::Tuple{}, ts_init::Tuple{}, ks_init::Tuple{}, recompile::Type{Val{true}}; saveat::Tuple{}, tstops::Tuple{}, d_discontinuities::Tuple{}, save_idxs::Nothing, save_everystep::Bool, save_on::Bool, save_start::Bool, save_end::Bool, callback::CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, dense::Bool, calck::Bool, dt::Float32, dtmin::Nothing, dtmax::Float32, force_dtmin::Bool, adaptive::Bool, gamma::Rational{Int64}, abstol::Nothing, reltol::Nothing, qmin::Rational{Int64}, qmax::Int64, qsteady_min::Int64, qsteady_max::Int64, beta1::Nothing, beta2::Nothing, qoldinit::Rational{Int64}, controller::Nothing, fullnormalize::Bool, failfactor::Int64, maxiters::Int64, internalnorm::typeof(DiffEqBase.ODE_DEFAULT_NORM), internalopnorm::typeof(LinearAlgebra.opnorm), isoutofdomain::typeof(DiffEqBase.ODE_DEFAULT_ISOUTOFDOMAIN), unstable_check::typeof(DiffEqBase.ODE_DEFAULT_UNSTABLE_CHECK), verbose::Bool, timeseries_errors::Bool, dense_errors::Bool, advance_to_tstop::Bool, stop_at_next_tstop::Bool, initialize_save::Bool, progress::Bool, progress_steps::Int64, progress_name::String, progress_message::typeof(DiffEqBase.ODE_DEFAULT_PROG_MESSAGE), userdata::Nothing, allow_extrapolation::Bool, initialize_integrator::Bool, alias_u0::Bool, alias_du0::Bool, initializealg::OrdinaryDiffEq.DefaultInit, kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol, Symbol}, NamedTuple{(:default_set, :save_noise), Tuple{Bool, Bool}}})
    @ OrdinaryDiffEq C:\Users\.julia\packages\OrdinaryDiffEq\FttEX\src\solve.jl:498
  [7] #__solve#627
    @ C:\Users\.julia\packages\OrdinaryDiffEq\FttEX\src\solve.jl:5 [inlined]
  [8] __solve(::ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Tuple{Symbol}, NamedTuple{(:callback,), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}}, SciMLBase.StandardODEProblem}, ::Nothing; default_set::Bool, kwargs::Base.Pairs{Symbol, Any, NTuple{4, Symbol}, 
NamedTuple{(:callback, :save_noise, :save_start, :save_end), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Bool, Bool, Bool}}})
    @ DifferentialEquations C:\Users\.julia\packages\DifferentialEquations\quY78\src\default_solve.jl:14
  [9] #solve_call#22
    @ C:\Users\.julia\packages\DiffEqBase\qPmC2\src\solve.jl:509 [inlined]
 [10] solve_up(prob::ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Tuple{Symbol}, NamedTuple{(:callback,), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}}, SciMLBase.StandardODEProblem}, sensealg::Nothing, u0::Vector{Float32}, p::Vector{Float32}, args::Nothing; kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:save_noise, :save_start, :save_end), Tuple{Bool, Bool, Bool}}})
    @ DiffEqBase C:\Users\.julia\packages\DiffEqBase\qPmC2\src\solve.jl:940
 [11] solve(prob::ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Tuple{Symbol}, NamedTuple{(:callback,), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", SciMLSensitivity.TrackedAffect{Float32, Vector{Float32}, Vector{Float32}, DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, Nothing, Int64}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}}, SciMLBase.StandardODEProblem}, args::Nothing; sensealg::Nothing, u0::Nothing, p::Nothing, wrap::Val{true}, kwargs::Base.Pairs{Symbol, Bool, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:save_noise, :save_start, :save_end), Tuple{Bool, Bool, Bool}}})
    @ DiffEqBase C:\Users\.julia\packages\DiffEqBase\qPmC2\src\solve.jl:842
 [12] _concrete_solve_adjoint(::ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, SciMLBase.StandardODEProblem}, ::Nothing, ::InterpolatingAdjoint{0, 
true, Val{:central}, ReverseDiffVJP{false}}, ::Vector{Float32}, ::Vector{Float32}, ::SciMLBase.ChainRulesOriginator; save_start::Bool, save_end::Bool, saveat::StepRangeLen{Float32, Float64, Float64, Int64}, save_idxs::Nothing, kwargs::Base.Pairs{Symbol, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}, Tuple{Symbol}, NamedTuple{(:callback,), Tuple{CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}})
    @ SciMLSensitivity C:\Users\.julia\packages\SciMLSensitivity\nMFgq\src\concrete_solve.jl:304
 [13] #_solve_adjoint#53
    @ C:\Users\.julia\packages\DiffEqBase\qPmC2\src\solve.jl:1346 [inlined]
 [14] #rrule#51
    @ C:\Users\.julia\packages\DiffEqBase\qPmC2\src\solve.jl:1301 [inlined]
 [15] chain_rrule_kw
    @ C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\chainrules.jl:235 [inlined]
 [16] macro expansion
    @ C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\interface2.jl:0 [inlined]
 [17] _pullback
    @ C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\interface2.jl:9 [inlined]
 [18] _apply
    @ .\boot.jl:816 [inlined]
 [19] adjoint
    @ C:\Users\.julia\packages\Zygote\TSj5C\src\lib\lib.jl:203 [inlined]
 [20] _pullback
    @ C:\Users\.julia\packages\ZygoteRules\OgCVT\src\adjoint.jl:66 [inlined]
 [21] _pullback
    @ C:\Users\.julia\packages\DiffEqBase\qPmC2\src\solve.jl:842 [inlined]
 [22] _pullback(::Zygote.Context{true}, ::DiffEqBase.var"##solve#27", ::InterpolatingAdjoint{0, true, Val{:central}, ReverseDiffVJP{false}}, ::Nothing, ::Nothing, ::Val{true}, ::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:saveat, :callback), Tuple{StepRangeLen{Float32, Float64, Float64, Int64}, CallbackSet{Tuple{}, Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}}, ::typeof(solve), ::ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, SciMLBase.StandardODEProblem})
    @ Zygote C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\interface2.jl:0
 [23] _apply(::Function, ::Vararg{Any})
    @ Core .\boot.jl:816
 [24] adjoint
    @ C:\Users\.julia\packages\Zygote\TSj5C\src\lib\lib.jl:203 [inlined]
 [25] _pullback
    @ C:\Users\.julia\packages\ZygoteRules\OgCVT\src\adjoint.jl:66 [inlined]
 [26] _pullback
    @ C:\Users\.julia\packages\DiffEqBase\qPmC2\src\solve.jl:832 [inlined]
 [27] _pullback(::Zygote.Context{true}, ::CommonSolve.var"#solve##kw", ::NamedTuple{(:sensealg, :saveat, :callback), Tuple{InterpolatingAdjoint{0, true, Val{:central}, ReverseDiffVJP{false}}, StepRangeLen{Float32, Float64, Float64, Int64}, CallbackSet{Tuple{}, 
Tuple{DiscreteCallback{DiffEqCallbacks.var"#27#28", DiffEqCallbacks.FunctionCallingAffect{typeof(functionCalling), DataStructures.BinaryMinHeap{Float64}, Vector{Float64}}, typeof(DiffEqCallbacks.functioncalling_initialize), typeof(SciMLBase.FINALIZE_DEFAULT)}}}}}, ::typeof(solve), ::ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, SciMLBase.StandardODEProblem})
    @ Zygote C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\interface2.jl:0
 [28] _apply(::Function, ::Vararg{Any})
    @ Core .\boot.jl:816
 [29] adjoint
    @ C:\Users\.julia\packages\Zygote\TSj5C\src\lib\lib.jl:203 [inlined]
 [30] _pullback
    @ C:\Users\.julia\packages\ZygoteRules\OgCVT\src\adjoint.jl:66 [inlined]
 [31] _pullback
    @ c:\Users\Documents\Julia\MWE_trackedaffect.jl:40 [inlined]
 [32] _pullback(::Zygote.Context{true}, ::typeof(losssum), ::ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, SciMLBase.StandardODEProblem}, ::NeuralODE{Chain{Tuple{Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}, Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}, Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}}}, Vector{Float32}, Optimisers.Restructure{Chain{Tuple{Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}, Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}, Dense{typeof(identity), Matrix{Float32}, Vector{Float32}}}}, NamedTuple{(:layers,), Tuple{Tuple{NamedTuple{(:weight, :bias, :σ), Tuple{Int64, Int64, Tuple{}}}, NamedTuple{(:weight, :bias, :σ), Tuple{Int64, Int64, Tuple{}}}, NamedTuple{(:weight, :bias, :σ), Tuple{Int64, Int64, Tuple{}}}}}}}, Tuple{Float32, Float32}, Tuple{}, Base.Pairs{Symbol, StepRangeLen{Float32, Float64, Float64, Int64}, Tuple{Symbol}, NamedTuple{(:saveat,), Tuple{StepRangeLen{Float32, Float64, Float64, Int64}}}}}, ::InterpolatingAdjoint{0, true, Val{:central}, ReverseDiffVJP{false}})
    @ Zygote C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\interface2.jl:0
 [33] _pullback
    @ c:\Users\Documents\Julia\MWE_trackedaffect.jl:65 [inlined]
 [34] _pullback(::Zygote.Context{true}, ::var"#7#9"{InterpolatingAdjoint{0, true, Val{:central}, ReverseDiffVJP{false}}, ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, SciMLBase.StandardODEProblem}})
    @ Zygote C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\interface2.jl:0
 [35] _apply
    @ .\boot.jl:816 [inlined]
 [36] adjoint
    @ C:\Users\.julia\packages\Zygote\TSj5C\src\lib\lib.jl:203 [inlined]
 [37] _pullback
    @ C:\Users\.julia\packages\ZygoteRules\OgCVT\src\adjoint.jl:66 [inlined]
 [38] _pullback
    @ C:\Users\.julia\packages\Flux\Nzh8J\src\optimise\train.jl:143 [inlined]
 [39] _pullback(::Zygote.Context{true}, ::Flux.Optimise.var"#37#40"{var"#7#9"{InterpolatingAdjoint{0, true, Val{:central}, ReverseDiffVJP{false}}, ODEProblem{Vector{Float32}, Tuple{Float32, Float32}, false, Vector{Float32}, ODEFunction{false, SciMLBase.FullSpecialize, var"#dudt_op#8", LinearAlgebra.UniformScaling{Bool}, Nothing, typeof(basic_tgrad), Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, typeof(SciMLBase.DEFAULT_OBSERVED), Nothing, Nothing}, Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, SciMLBase.StandardODEProblem}}, Tuple{}})
    @ Zygote C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\interface2.jl:0
 [40] pullback(f::Function, ps::Params{Zygote.Buffer{Any, Vector{Any}}})
    @ Zygote C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\interface.jl:384
 [41] withgradient(f::Function, args::Params{Zygote.Buffer{Any, Vector{Any}}})
    @ Zygote C:\Users\.julia\packages\Zygote\TSj5C\src\compiler\interface.jl:132
 [42] macro expansion
    @ C:\Users\.julia\packages\Flux\Nzh8J\src\optimise\train.jl:142 [inlined]
 [43] macro expansion
    @ C:\Users\.julia\packages\ProgressLogging\6KXlp\src\ProgressLogging.jl:328 [inlined]
 [44] train!(loss::Function, ps::Params{Zygote.Buffer{Any, Vector{Any}}}, data::Base.Iterators.Take{Base.Iterators.Repeated{Tuple{}}}, opt::Adam; cb::Flux.Optimise.var"#38#41")
    @ Flux.Optimise C:\Users\.julia\packages\Flux\Nzh8J\src\optimise\train.jl:140
 [45] train!(loss::Function, ps::Params{Zygote.Buffer{Any, Vector{Any}}}, data::Base.Iterators.Take{Base.Iterators.Repeated{Tuple{}}}, opt::Adam)
    @ Flux.Optimise C:\Users\.julia\packages\Flux\Nzh8J\src\optimise\train.jl:136
 [46] top-level scope
    @ c:\Users\Documents\Julia\MWE_trackedaffect.jl:65
ChrisRackauckas commented 1 year ago

Thanks! This ended up finding and fixing something that is quite useful across the board.

ThummeTo commented 1 year ago

With pleasure. Thanks also for the quick exchange and integration!

ThummeTo commented 1 year ago

BTW is a patch release planned for the near future?