Updating the RFFSPs Component Time Dimension

[lrennels]

Tracking an issue from a user about updating the RFFSPs component time dimension in case it is helpful for others. Related to

https://github.com/mimiframework/Mimi.jl/issues https://github.com/rffscghg/MimiRFFSPs.jl/issues/31

[lrennels]

I have updated my model to run from 1750 to 2100 with set_dimension!(m, :time, 1750:2100) but get the following indexing error in the Socioeconomic component

ERROR: BoundsError: attempt to access 81×184 view(::Matrix{Union{Missing, Float64}}, 271:351, :) with eltype Union{Missing, Float64} at index [86, 1]
Stacktrace:
  [1] throw_boundserror(A::SubArray{Union{Missing, Float64}, 2, Matrix{Union{Missing, Float64}}, Tuple{UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}, I::Tuple{Int64, Int64})
    @ Base .\abstractarray.jl:703
  [2] checkbounds
    @ .\abstractarray.jl:668 [inlined]
  [3] setindex!
    @ .\subarray.jl:322 [inlined]
  [4] setindex!
    @ C:\Users\XXX\.julia\packages\Mimi\OClro\src\core\time_arrays.jl:354 [inlined]
  [5] fill_socioeconomics!(source_Year::Arrow.Primitive{Union{Missing, Int32}, Vector{Int32}}, source_Country::Arrow.DictEncoded{Union{Missing, String}, Int16, Arrow.List{Union{Missing, String}, Int32, Vector{UInt8}}}, source_Pop::Arrow.Primitive{Union{Missing, Float64}, Vector{Float64}}, source_GDP::Arrow.Primitive{Union{Missing, Float64}, Vector{Float64}}, population::Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 2, 1, SubArray{Union{Missing, Float64}, 2, Matrix{Union{Missing, Float64}}, Tuple{UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, gdp::Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 2, 1, SubArray{Union{Missing, Float64}, 2, Matrix{Union{Missing, Float64}}, Tuple{UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, country_lookup::Dict{String, Int64}, start_year::Int64, end_year::Int64)
    @ MimiRFFSPs C:\Users\XXX\.julia\packages\MimiRFFSPs\KKA1f\src\components\SPs.jl:14
  [6] init_MimiRFFSPs_SPs(p::Mimi.ComponentInstanceParameters{NamedTuple{(:start_year, :end_year, :country_names, :id), Tuple{Mimi.ScalarModelParameter{Int64}, Mimi.ScalarModelParameter{Int64}, Vector{Union{Missing, String}}, Mimi.ScalarModelParameter{Int64}}}}, v::Mimi.ComponentInstanceVariables{NamedTuple{(:population, :population_global, :deathrate, :gdp, :gdp_global, :population1990, :gdp1990, :co2_emissions, :ch4_emissions, :n2o_emissions), Tuple{Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 2, 1, SubArray{Union{Missing, Float64}, 2, Matrix{Union{Missing, Float64}}, Tuple{UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 
1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 2, 1, SubArray{Union{Missing, Float64}, 2, Matrix{Union{Missing, Float64}}, Tuple{UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 2, 1, SubArray{Union{Missing, Float64}, 2, Matrix{Union{Missing, Float64}}, Tuple{UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}, Vector{Float64}, Vector{Float64}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}}}}, d::NamedTuple{(:time, :other_ghg, :ozone_depleting_substances, Symbol("4"), Symbol("2"), :country, :fund_regions, :segments, :ag_mapping_input_regions, :ag_mapping_output_regions, :energy_countries, :domestic_countries, Symbol("3"), :agegroups, :ageyr, :pcgdpquartiles), Tuple{Vector{Mimi.AbstractTimestep}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}}})
    @ MimiRFFSPs C:\Users\XXX\.julia\packages\Mimi\OClro\src\core\defcomp.jl:94
  [7] init(ci::Mimi.LeafComponentInstance{Mimi.ComponentInstanceVariables{NamedTuple{(:population, :population_global, :deathrate, :gdp, :gdp_global, :population1990, :gdp1990, :co2_emissions, :ch4_emissions, :n2o_emissions), Tuple{Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 2, 1, SubArray{Union{Missing, Float64}, 2, Matrix{Union{Missing, Float64}}, Tuple{UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 2, 1, SubArray{Union{Missing, Float64}, 2, Matrix{Union{Missing, Float64}}, 
Tuple{UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 2, 1, SubArray{Union{Missing, Float64}, 2, Matrix{Union{Missing, Float64}}, Tuple{UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}, Vector{Float64}, Vector{Float64}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}, Mimi.TimestepArray{Mimi.FixedTimestep{1750, 1, 2100}, Union{Missing, Float64}, 1, 1, SubArray{Union{Missing, Float64}, 1, Vector{Union{Missing, Float64}}, Tuple{UnitRange{Int64}}, true}}}}}, Mimi.ComponentInstanceParameters{NamedTuple{(:start_year, :end_year, :country_names, :id), Tuple{Mimi.ScalarModelParameter{Int64}, Mimi.ScalarModelParameter{Int64}, Vector{Union{Missing, String}}, Mimi.ScalarModelParameter{Int64}}}}}, dims::NamedTuple{(:time, :other_ghg, :ozone_depleting_substances, Symbol("4"), Symbol("2"), :country, :fund_regions, :segments, :ag_mapping_input_regions, :ag_mapping_output_regions, :energy_countries, :domestic_countries, Symbol("3"), :agegroups, :ageyr, :pcgdpquartiles), Tuple{Vector{Mimi.AbstractTimestep}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}}})
    @ Mimi C:\Users\XXX\.julia\packages\Mimi\OClro\src\core\instances.jl:281
  [8] init(obj::Mimi.ModelInstance, dims::NamedTuple{(:time, :other_ghg, :ozone_depleting_substances, Symbol("4"), Symbol("2"), :country, :fund_regions, :segments, :ag_mapping_input_regions, :ag_mapping_output_regions, :energy_countries, :domestic_countries, Symbol("3"), :agegroups, :ageyr, :pcgdpquartiles), Tuple{Vector{Mimi.AbstractTimestep}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}, Vector{Int64}}})
    @ Mimi C:\Users\XXX\.julia\packages\Mimi\OClro\src\core\instances.jl:288
  [9] run(mi::Mimi.ModelInstance, ntimesteps::Int64, dimkeys::Nothing)
    @ Mimi C:\Users\XXX\.julia\packages\Mimi\OClro\src\core\instances.jl:363
 [10] run(m::Model; ntimesteps::Int64, rebuild::Bool, dim_keys::Nothing)
    @ Mimi C:\Users\XXX\.julia\packages\Mimi\OClro\src\core\model.jl:539
 [11] run(m::Model)
    @ Mimi C:\Users\XXX\.julia\packages\Mimi\OClro\src\core\model.jl:527
 [12] top-level scope
    @ c:\Users\XXX\Documents\migration-policy\mig-policy\code\run_MimiGIVEMigPol.jl:8

[lrennels]

Hmmm I'll look. So what I'm seeing is that the script is trying to access a value in an array of size 81 x 184 (this means it was something allocated with years x countries) and index [86, 1] meaning it's looking for timestep 86 for some odd reason. I got that from [1] ... so now I'm looking down and I see the function fill_socioeconomics! in [5] which is a function held in the the SPs component of the MimiRFFSPs model as shown at the end of line [5]Then looking at line [6] we have the init_MimiRFFSPs_SPs function ... which is where the error must be occurring. Ok so then I look at the very end of [6] and see MimiRFFSPs C:\Users\XXX\.julia\packages\MimiRFFSPs\KKA1f\src\components\SPs.jl:14 so I'm going to go look at line 14 of the SPs component in MimiRFFSPs. Finally I see it is from run_model() in run_MimiGIVEMigPol from line [12].

[lrennels]

OK I think I've got it. So this is the function within init

function fill_socioeconomics!(source_Year, source_Country, source_Pop, source_GDP, population, gdp, country_lookup, start_year, end_year)
    for i in 1:length(source_Year)
        if source_Year[i] >= start_year && source_Year[i] <= end_year
            year_index = TimestepIndex(source_Year[i] - start_year + 1)
            # year_index = TimestepValue(source_Year[i]) # current bug in Mimi
            country_index = country_lookup[source_Country[i]]

            population[year_index, country_index] = source_Pop[i] ./ 1e3 # convert thousands to millions
            gdp[year_index, country_index] = source_GDP[i] ./ 1e3 .* pricelevel_2011_to_2005 # convert millions to billions; convert $2011 to $2005

        end
    end
end

and we have failure to index into population with the year_index because it's looking for year index 86 but we only have 81. So that function get's called in init here

fill_socioeconomics!(t.Year, t.Country, t.Pop, t.GDP, v.population, v.gdp, country_lookup, p.start_year, p.end_year)

[lrennels]

So basically since p.end_yearis a parameter to the component, when you adjusted it in the model, it didn't update it because it's an actual parameter not a dimension. If you're curious, we did this because (annoyingly) the init function doesn't have access to all of the dimension sinformation so we couldn't look it up from there. It's something I want to fix!

[lrennels]

This should be the fix:

set_dimension!(m, :time, model_first, model_last) # in this case you are using model_first = 1750 and model_last = 2100
# update the end years of the MimiRFFSPs component as needed
# NOTE: for now do not try to update the start_year due to concerns about initial conditions -- can discuss if needed through
# NOTE: this manual step is necessary due to a performance improvement in the component that makes start_year a parameter as well as held within the time dimension

if socioeconomics_source == :RFF
        update_param!(m, :Socioeconomic, :end_year, model_last) 
end