Open rburghol opened 1 year ago
Import a ts and transform into step indexed, timestep uniform data frame.
import os
os.chdir('C:/usr/local/home/git/HSPsquared/')
import h5py
import numpy as np
import pandas as pd
from collections import defaultdict
from pandas import DataFrame, read_hdf, HDFStore
from numba.typed import Dict
# now load HSP2 stuff
#import HSP2.utilities # note: we need to import this first, otherwise we get errors with HSP2IO.io and HSPFIO.hdf
from HSP2.utilities import * # need this to access transform, tho maybe more pointed imports would suffice
import HSP2IO
from HSP2IO.hdf import HDF5
from HSP2IO.io import IOManager, SupportsReadTS, Category
fpath = 'C:/Workspace/modeling/cbp/hsp2/river/PM7_4581_4580.h5'
# Load simulation info
hdf5_instance = HDF5(fpath)
io_manager = IOManager(hdf5_instance)
uci_obj = io_manager.read_uci()
siminfo = uci_obj.siminfo
# note, siminfo delt is specific to each module being run (or is potentially varying)
# so it is NOT set at first read of UCI, rather, set by "operation"
# delt = time step in minutes
siminfo['delt'] = 60
siminfo['tindex'] = date_range(siminfo['start'], siminfo['stop'], freq=Minute(siminfo['delt']))[1:]
siminfo['steps'] = len(siminfo['tindex'])
# close the already opened file first if we are re-running
# f.close()
f = h5py.File(fpath,'a') # use mode 'a' which allows read, write, modify
dset = f['TIMESERIES/TS011/table']
# Try this, transform(ts, name, how, siminfo)
# name = 'NA' since this should only be used if we don't give a `how`
# how = 'SAME', which means just resample - I think this is good for a Quantity, maybe not rate?
ts10 = io_manager.read_ts(Category.INPUTS, None, "TS010")
t = transform(ts10, 'NA', 'SAME', siminfo)
Work session 12/26/2023
# use above code to load libs etc
fpath = 'C:/Workspace/modeling/hspf/hsp2/testcbp/PL3_5250_0001.h5'
#f = h5py.File(fpath,'a') # use mode 'a' which allows read, write, modify
hdf5_instance = HDF5(fpath)
io_manager = IOManager(hdf5_instance)
uci_obj = io_manager.read_uci()
siminfo = uci_obj.siminfo
state = {}
state['model_data'] = {}
dc = uci_obj.specactions['ACTIONS'] for ix in dc.index:
speca = dc[ix:(ix+1)]
print("row", ix, "keys:", speca.keys())
# need to add a name attribute
opname = 'SPEC' + 'ACTION' + str(ix)
state['model_data'][opname] = {}
opname = 'SPEC' + 'ACTION' + str(ix)
state['model_data'][opname]['name'] = opname
for ik in speca.keys():
# this extracts the data from the data frame to the model_data dict
state['model_data'][opname][ik] = speca.to_dict()[ik][ix]
Time series writers, i.e, used for logging, may not be included in the domain dependency list, so we need a way of saying "also, run anything that depends on the STATE value of things that are running in the current domain". Example, O2 = withdrawal in cfs, and we may wish to log the original value, before HSP2 decides whether or not there is sufficient water available.
# get the deps for our reach that influence O2 and O3 (withdrawals in our CBP template)
dep_list = model_domain_dependencies(state, '/STATE/JL1_6562_6560/RCHRES_R001', ['O2', 'O3'])
# returns: [44, 45, 48, 49, 43, 47, 4]
# now, find anything that is linked to state for anything WITHIN this dep_list
all_deps = model_input_dependencies(state, dep_list)
develop-specact-ts
https://github.com/HARPgroup/HSPsquared/tree/develop-specact-ts/HSP2EXT SOURCES
andEXT TARGETS
), for the reach in question.HSP2IO/hdf.py
:read_ts()
root
of a given domain, i.e.,RCHRES_R001
orRCHRES_R002
, there is currently no data that contains that information passed intohydr
!get_timeseries()
requires information akin to that in theEXT SOURCES
block lines. Specifically, it uses:MFACTOR
- multiplier (default is 1.0, which means no conversion)TMEMN
- member name (see UCI EXT SOURCES) - only used intransform
to guess conversion ifhow
is FalseTRAN
- the transformation method (passed to transform ashow
)transform(ts, name, how, siminfo)
get_timeseries()
1st Draft
def load_sim_ts(f, state_ix, ts_ix): tsq = f['TIMESERIES/'] # f is the hdf5 file
this code replicates a piece of the function of get_timeseries, and loads the full timeseries into a Dict
the dict will be keyed by a simple integer, and have values at the time step only. The actual time
at that step will be contained in ts_ts
ts_tables = list(tsq.keys()) for i in ts_tables: if i == 'SUMMARY': continue # skip this non-numeric table var_path = '/TIMESERIES/' + i ix = set_state(state_ix, state_paths, var_path, 0.0) ts_ix[ix] = np.asarray(tsq[i]['table']['values'], dtype="float32")
import os import h5py import numpy as np import pandas as pd from pandas.io.pytables import read_hdf from collections import defaultdict from pandas import DataFrame, read_hdf, HDFStore from numba.typed import Dict
now load HSP2 stuff
import HSP2.utilities import HSP2IO from HSP2IO.hdf import HDF5 from HSP2IO.io import IOManager
os.chdir('C:/usr/local/home/git/HSPsquared/') fpath = 'C:/Workspace/modeling/cbp/hsp2/river/PM7_4581_4580.h5'
close the already opened file first if we are re-running
f.close()
f = h5py.File(fpath,'a') # use mode 'a' which allows read, write, modify dset = f['TIMESERIES/TS011/table']
Try this, transform(ts, name, how, siminfo)
name = 'NA' since this should only be used if we don't give a
how
how = 'SAME', means just resample - I think this is good for a Quantity, maybe not rate?
hdf5_instance = HDF5(fpath) io_manager = IOManager(hdf5_instance) uci_obj = io_manager.read_uci() siminfo = uci_obj.siminfo
note, siminfo delt is specific to each module being run (or is potentially varying)
so it is NOT set at first read of UCI, rather, set by "operation"
delt = time step in minutes
siminfo['delt'] = 60
hsp2 adds tindex and computes steps based on start/stop and delt for each sequence
siminfo['tindex'] = date_range(siminfo['start'], siminfo['stop'], freq=Minute(siminfo['delt']))[1:] siminfo['steps'] = len(siminfo['tindex'])
dset from direct grab of the table path does not work (see above)
using method from HSP2IO/hdf.py
dstore = pd.HDFStore(fpath) dset = read_hdf(dstore, '/TIMESERIES/TS011/') t = transform(dset, 'NA', 'SAME', siminfo)
the same can be achieved directly using the hdf5_instance: (note Category is set somewhere in the loaded hsp2 modules)
ts10 = hdf5_instance.read_ts(Category.INPUTS, None, "TS010")
same can be achieved with a direct call to the io_manager:
ts10 = io_manager.read_ts(Category.INPUTS, None, "TS010")