Closed lbianchi-lbl closed 1 year ago
@bknueven Running git bisect
within the same commit range as #164 points to IDAES/idaes-pse@fa67d74:
fa67d746ab95744206cafb85967c67d06164e7ed is the first bad commit
commit fa67d746ab95744206cafb85967c67d06164e7ed
Author: bknueven <30801372+bknueven@users.noreply.github.com>
Date: Mon Nov 7 11:27:05 2022 -0700
ensuring the coordinator honors the initial_status and initial_p_output passed in by user (#1002)
Co-authored-by: Keith Beattie <ksbeattie@lbl.gov>
idaes/apps/grid_integration/coordinator.py | 14 ++++++++++----
idaes/apps/grid_integration/tests/test_coordinator.py | 4 +---
idaes/apps/grid_integration/tests/util.py | 2 +-
3 files changed, 12 insertions(+), 8 deletions(-)
Steps to reproduce:
mkdir /tmp/dir-for-dispatches-167 && cd /tmp/dir-for-dispatches-167
conda create -n test-dispatches-167 --yes python=3.8 && conda activate test-dispatches-167
pip install -e git+https://github.com/gmlc-dispatches/dispatches@edba882#egg=dispatches
pip uninstall idaes-pse --yes && pip install -e git+https://github.com/IDAES/idaes-pse@bf61436#egg=idaes-pse
pip uninstall pyomo --yes && pip install -e git+https://github.com/Pyomo/pyomo@6.4.4#egg=pyomo
cd src/dispatches
# create file dispatches/case_studies/renewables_case/mre_167.py
cd ../idaes-pse
bash -c 'idaes get-extensions && cd ../dispatches/dispatches/case_studies/renewables_case && DISPATCHES_TESTING_MODE=1 python mre_167.py'
I'm reopening this as it looks like the same error is still happening: https://github.com/gmlc-dispatches/dispatches/actions/runs/3717046055/jobs/6304033661#step:5:388
_ dispatches/case_studies/renewables_case/DoubleLoopOptimization.ipynb::Cell 9 _
Notebook cell execution failed
Cell 9: Cell execution caused an exception
Input:
from types import ModuleType
import shutil
class PrescientPluginModule(ModuleType):
def __init__(self, get_configuration, register_plugins):
self.get_configuration = get_configuration
self.register_plugins = register_plugins
plugin_module = PrescientPluginModule(
get_configuration=coordinator.get_configuration,
register_plugins=coordinator.register_plugins,
)
prescient_options['plugin']['doubleloop']['module'] = plugin_module
if output_dir.exists():
shutil.rmtree(output_dir)
Prescient().simulate(**prescient_options)
Traceback:
30 def _get_uc_model(model_data, formulation_list, relax_binaries, **kwargs):
31 formulation = UCFormulation(*formulation_list)
---> 32 return generate_model(model_data, formulation, relax_binaries, **kwargs)
33
34 def create_tight_unit_commitment_model(model_data,
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/egret/model_library/unit_commitment/uc_model_generator.py in generate_model(model_data, uc_formulation, relax_binaries, ptdf_options, PTDF_matrix_dict, slack_type)
72 md = model_data.clone_in_service()
73 scale_ModelData_to_pu(md, inplace=True)
---> 74 return _generate_model( md, *_get_formulation_from_UCFormulation( uc_formulation ), relax_binaries , ptdf_options, PTDF_matrix_dict, slack_type )
75
76 def _generate_model( model_data,
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/egret/model_library/unit_commitment/uc_model_generator.py in _generate_model(model_data, _status_vars, _power_vars, _reserve_vars, _non_dispatchable_vars, _generation_limits, _ramping_limits, _production_costs, _uptime_downtime, _startup_costs, _power_balance, _reserve_requirement, _objective, _relax_binaries, _ptdf_options, _PTDF_matrix_dict, _slack_type)
121 model.relax_binaries = _relax_binaries
122
--> 123 params.load_params(model, model_data, _slack_type)
124 getattr(status_vars, _status_vars)(model)
125 getattr(power_vars, _power_vars)(model)
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/egret/model_library/unit_commitment/uc_utils.py in wrapper(*args, **kwds)
62 warnings.warn(msg)
63 setattr(model, attr, func.__name__)
---> 64 return func(*args, **kwds)
65 return wrapper
66 return actual_decorator
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/egret/model_library/unit_commitment/params.py in load_params(model, model_data, slack_type)
623 within=Binary,
624 initialize=t0_unit_on_rule,
--> 625 mutable=True)
626
627 _add_initial_time_periods_on_off_line(model)
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/pyomo/core/base/block.py in __setattr__(self, name, val)
647 # Pyomo components are added with the add_component method.
648 #
--> 649 self.add_component(name, val)
650 else:
651 #
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/pyomo/core/base/block.py in add_component(self, name, val)
1217 _blockName, str(data))
1218 try:
-> 1219 val.construct(data)
1220 except:
1221 err = sys.exc_info()[1]
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/pyomo/core/base/param.py in construct(self, data)
753 # Step #1: initialize data from rule value
754 #
--> 755 self._construct_from_rule_using_setitem()
756 #
757 # Step #2: allow any user-specified (external) data to override
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/pyomo/core/base/indexed_component.py in _construct_from_rule_using_setitem(self)
718 else:
719 for index in self.index_set():
--> 720 self._setitem_when_not_present(index, rule(block, index))
721 except:
722 err = sys.exc_info()[1]
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/pyomo/core/base/initializer.py in __call__(self, parent, idx)
302 return self._fcn(parent, *idx)
303 else:
--> 304 return self._fcn(parent, idx)
305
306
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/egret/model_library/unit_commitment/params.py in t0_unit_on_rule(m, g)
618
619 def t0_unit_on_rule(m, g):
--> 620 return int(value(m.UnitOnT0State[g]) > 0.)
621
622 model.UnitOnT0 = Param(model.ThermalGenerators,
pyomo/core/expr/numvalue.pyx in pyomo.core.expr.numvalue.value()
pyomo/core/expr/numvalue.pyx in pyomo.core.expr.numvalue.value()
/opt/hostedtoolcache/Python/3.7.15/x64/lib/python3.7/site-packages/pyomo/core/base/param.py in __call__(self, exception)
189 "from a scalar Param or mutable Indexed Param without\n"
190 "\tan initial or default value."
--> 191 % ( self.name, ))
192 else:
193 return None
ValueError: Error evaluating Param value (UnitOnT0State[309_WIND_1]):
The Param value is currently set to an invalid value. This is
typically from a scalar Param or mutable Indexed Param without
an initial or default value.
For reference, pip list
:
Package Version Editable project location
---------------------------------- ----------- ---------------------------------------
addheader 0.2.4
alabaster 0.7.12
anyio 3.6.2
argon2-cffi 21.3.0
argon2-cffi-bindings 21.2.0
attrs 22.1.0
Babel 2.11.0
backcall 0.2.0
backports.shutil-get-terminal-size 1.0.0
beautifulsoup4 4.11.1
bleach 5.0.1
bunch 1.0.1
certifi 2022.12.7
cffi 1.15.1
charset-normalizer 2.1.1
click 8.1.3
colorama 0.4.6
coramin 0.1.1
coverage 6.5.0
cycler 0.11.0
debugpy 1.6.4
decorator 5.1.1
defusedxml 0.7.1
dispatches 1.1.dev0 /home/runner/work/dispatches/dispatches
dispatches-sample-data 22.9.19
distro 1.8.0
docutils 0.17.1
entrypoints 0.4
et-xmlfile 1.1.0
exceptiongroup 1.0.4
fastjsonschema 2.16.2
Flask 2.2.2
Flask-Cors 3.0.10
fonttools 4.38.0
graphviz 0.20.1
gridx-egret 0.5.3
gridx-prescient 2.2.1
idaes-pse 2.0.0b2
idna 3.4
imagesize 1.4.1
importlib-metadata 5.1.0
importlib-resources 5.10.1
iniconfig 1.1.1
ipykernel 6.16.2
ipython 7.34.0
ipython-genutils 0.2.0
ipywidgets 8.0.3
itsdangerous 2.1.2
jedi 0.18.2
Jinja2 3.1.2
jsonschema 4.17.3
jupyter 1.0.0
jupyter_client 7.4.8
jupyter-console 6.4.4
jupyter_core 4.12.0
jupyter-server 1.23.3
jupyterlab-pygments 0.2.2
jupyterlab-widgets 3.0.4
kiwisolver 1.4.4
lxml 4.9.2
MarkupSafe 2.1.1
matplotlib 3.5.3
matplotlib-inline 0.1.6
mistune 2.0.4
mpmath 1.2.1
nbclassic 0.4.8
nbclient 0.7.2
nbconvert 7.2.6
nbformat 5.7.0
nbsphinx 0.8.10
nbval 0.9.6
nest-asyncio 1.5.6
networkx 2.6.3
notebook 6.5.2
notebook_shim 0.2.2
NREL-PySAM 4.0.0
numpy 1.21.6
omlt 0.3.1
openpyxl 3.0.10
packaging 22.0
pandas 1.3.5
pandocfilters 1.5.0
parso 0.8.3
pexpect 4.8.0
pickleshare 0.7.5
Pillow 9.3.0
Pint 0.18
pip 22.3.1
pkgutil_resolve_name 1.3.10
pluggy 1.0.0
ply 3.11
prometheus-client 0.15.0
prompt-toolkit 3.0.36
psutil 5.9.4
ptyprocess 0.7.0
pycparser 2.21
Pygments 2.13.0
Pyomo 6.4.4
pyparsing 3.0.9
pyrsistent 0.19.2
pytest 7.2.0
pytest-cov 4.0.0
python-dateutil 2.8.2
python-dotenv 0.21.0
python-slugify 7.0.0
pytz 2022.6
PyYAML 6.0
pyzmq 24.0.1
qtconsole 5.4.0
QtPy 2.3.0
rbfopt 4.2.5
requests 2.28.1
scipy 1.7.3
seaborn 0.12.1
Send2Trash 1.8.0
setuptools 65.6.3
six 1.16.0
sniffio 1.3.0
snowballstemmer 2.2.0
soupsieve 2.3.2.post1
Sphinx 4.2.0
sphinx-rtd-theme 1.1.1
sphinxcontrib-applehelp 1.0.2
sphinxcontrib-devhelp 1.0.2
sphinxcontrib-htmlhelp 2.0.0
sphinxcontrib-jsmath 1.0.1
sphinxcontrib-qthelp 1.0.3
sphinxcontrib-serializinghtml 1.1.5
sympy 1.10.1
terminado 0.17.1
text-unidecode 1.3
tinycss2 1.2.1
tinydb 4.7.0
tomli 2.0.1
tornado 6.2
traitlets 5.7.1
typing_extensions 4.4.0
urllib3 1.26.13
wcwidth 0.2.5
webencodings 0.5.1
websocket-client 1.4.2
Werkzeug 2.2.2
wheel 0.38.4
widgetsnbextension 4.0.4
xlrd 2.0.1
zipp 3.11.0
Prescient 2.2.1 reverts a change from Prescient 2.2 which patched this for this particular workflow but not in general.
Prescient 2.2.1 has a more robust solution, but at the cost of adding additional Prescient callbacks. These are utilized in #169 (there's also a related PR in IDAES https://github.com/IDAES/idaes-pse/pull/1054, which implements the same change).
We'd ideally be running our Prescient simulations in our tests for at least two days, but that seems to hit the timeout criterion of an individual notebook cell. There's also the issue that I think the renewables double loop notebook is the only one actually running a Prescient simulation in our tests.
This occurred with #164, possibly as a consequence of updating to IDAES 2.0.0b2:
See https://github.com/gmlc-dispatches/dispatches/actions/runs/3678468960/jobs/6221778238#step:5:385 for more details.