search

UCL-SML / Doubly-Stochastic-DGP

Deep Gaussian Processes with Doubly Stochastic Variational Inference
Apache License 2.0
147 stars 47 forks source link

Adam Optimizer: `FailedPreconditionError` #15

Closed imsrgadich closed 6 years ago

imsrgadich commented 6 years ago

Hi,

I was running the notebook demo_regression_UCI and while running

for m, name in zip([m_dgp1, m_dgp2, m_dgp3, m_dgp4, m_dgp5],\
                ['dgp1 (sgp+adam)', 'dgp2', 'dgp3', 'dgp4', 'dgp5']):
    AdamOptimizer(0.01).minimize(m, maxiter=5000)
    lik, rmse = batch_assess(m, assess_sampled, Xs, Ys)
    print(s.format(name, lik, rmse))

I get the following error. 

FailedPreconditionError: Attempting to use uninitialized value beta1_power
     [[Node: beta1_power/read = Identity[T=DT_FLOAT, _class=["loc:@DGP-26d70ccd-89/layers/0/feature/Z/unconstrained"], _device="/job:localhost/replica:0/task:0/device:CPU:0"](beta1_power)]]

Any idea whats causing this? Thank you.

Full error below. 

---------------------------------------------------------------------------
FailedPreconditionError                   Traceback (most recent call last)
/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1326     try:
-> 1327       return fn(*args)
   1328     except errors.OpError as e:

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
   1311       return self._call_tf_sessionrun(
-> 1312           options, feed_dict, fetch_list, target_list, run_metadata)
   1313 

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
   1419             self._session, options, feed_dict, fetch_list, target_list,
-> 1420             status, run_metadata)
   1421 

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py in __exit__(self, type_arg, value_arg, traceback_arg)
    515             compat.as_text(c_api.TF_Message(self.status.status)),
--> 516             c_api.TF_GetCode(self.status.status))
    517     # Delete the underlying status object from memory otherwise it stays alive

FailedPreconditionError: Attempting to use uninitialized value beta1_power
     [[Node: beta1_power/read = Identity[T=DT_FLOAT, _class=["loc:@DGP-26d70ccd-89/layers/0/feature/Z/unconstrained"], _device="/job:localhost/replica:0/task:0/device:CPU:0"](beta1_power)]]

During handling of the above exception, another exception occurred:

FailedPreconditionError                   Traceback (most recent call last)
<ipython-input-7-21c2fc99d2cf> in <module>()
      1 for m, name in zip([m_dgp1, m_dgp2, m_dgp3, m_dgp4, m_dgp5],                ['dgp1 (sgp+adam)', 'dgp2', 'dgp3', 'dgp4', 'dgp5']):
----> 2     AdamOptimizer(0.01).minimize(m, maxiter=5000)
      3     lik, rmse = batch_assess(m, assess_sampled, Xs, Ys)
      4     print(s.format(name, lik, rmse))

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/gpflow/training/tensorflow_optimizer.py in minimize(self, model, session, var_list, feed_dict, maxiter, initialize, anchor, **kwargs)
    113         with session.as_default():
    114             for _ in range(maxiter):
--> 115                 opt()
    116 
    117         if anchor:

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/gpflow/actions.py in __call__(self, context)
    141         try:
    142             self.watcher.start()
--> 143             self.run(context)
    144         finally:
    145             self.watcher.stop()

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/gpflow/actions.py in run(self, context)
    363 
    364     def run(self, context: ActionContext) -> None:
--> 365         context.session.run(self.optimizer_tensor, **self.run_kwargs)
    366 
    367 

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
    903     try:
    904       result = self._run(None, fetches, feed_dict, options_ptr,
--> 905                          run_metadata_ptr)
    906       if run_metadata:
    907         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
   1138     if final_fetches or final_targets or (handle and feed_dict_tensor):
   1139       results = self._do_run(handle, final_targets, final_fetches,
-> 1140                              feed_dict_tensor, options, run_metadata)
   1141     else:
   1142       results = []

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1319     if handle is None:
   1320       return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1321                            run_metadata)
   1322     else:
   1323       return self._do_call(_prun_fn, handle, feeds, fetches)

/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
   1338         except KeyError:
   1339           pass
-> 1340       raise type(e)(node_def, op, message)
   1341 
   1342   def _extend_graph(self):

FailedPreconditionError: Attempting to use uninitialized value beta1_power
     [[Node: beta1_power/read = Identity[T=DT_FLOAT, _class=["loc:@DGP-26d70ccd-89/layers/0/feature/Z/unconstrained"], _device="/job:localhost/replica:0/task:0/device:CPU:0"](beta1_power)]]

Caused by op 'beta1_power/read', defined at:
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/ipykernel/__main__.py", line 3, in <module>
    app.launch_new_instance()
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/traitlets/config/application.py", line 658, in launch_instance
    app.start()
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/ipykernel/kernelapp.py", line 486, in start
    self.io_loop.start()
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tornado/platform/asyncio.py", line 112, in start
    self.asyncio_loop.run_forever()
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/asyncio/base_events.py", line 421, in run_forever
    self._run_once()
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/asyncio/base_events.py", line 1425, in _run_once
    handle._run()
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/asyncio/events.py", line 127, in _run
    self._callback(*self._args)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tornado/platform/asyncio.py", line 102, in _handle_events
    handler_func(fileobj, events)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tornado/stack_context.py", line 276, in null_wrapper
    return fn(*args, **kwargs)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py", line 450, in _handle_events
    self._handle_recv()
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py", line 480, in _handle_recv
    self._run_callback(callback, msg)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py", line 432, in _run_callback
    callback(*args, **kwargs)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tornado/stack_context.py", line 276, in null_wrapper
    return fn(*args, **kwargs)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
    return self.dispatch_shell(stream, msg)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
    handler(stream, idents, msg)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
    user_expressions, allow_stdin)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/ipykernel/ipkernel.py", line 208, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/ipykernel/zmqshell.py", line 537, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2728, in run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2850, in run_ast_nodes
    if self.run_code(code, result):
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/IPython/core/interactiveshell.py", line 2910, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-7-21c2fc99d2cf>", line 2, in <module>
    AdamOptimizer(0.01).minimize(m, maxiter=5000)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/gpflow/training/tensorflow_optimizer.py", line 107, in minimize
    feed_dict=feed_dict, **kwargs)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/gpflow/training/tensorflow_optimizer.py", line 76, in make_optimize_action
    optimizer_tensor = self.make_optimize_tensor(model, session, var_list=var_list, **kwargs)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/gpflow/training/tensorflow_optimizer.py", line 54, in make_optimize_tensor
    minimize = self.optimizer.minimize(objective, var_list=full_var_list, **kwargs)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/training/optimizer.py", line 409, in minimize
    name=name)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/training/optimizer.py", line 552, in apply_gradients
    self._create_slots([_get_variable_for(v) for v in var_list])
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/training/adam.py", line 124, in _create_slots
    colocate_with=first_var)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/training/optimizer.py", line 665, in _create_non_slot_variable
    v = variable_scope.variable(initial_value, name=name, trainable=False)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py", line 2157, in variable
    use_resource=use_resource)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py", line 2147, in <lambda>
    previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py", line 2130, in default_variable_creator
    constraint=constraint)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/ops/variables.py", line 235, in __init__
    constraint=constraint)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/ops/variables.py", line 391, in _init_from_args
    self._snapshot = array_ops.identity(self._variable, name="read")
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py", line 142, in identity
    return gen_array_ops.identity(input, name=name)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/ops/gen_array_ops.py", line 3053, in identity
    "Identity", input=input, name=name)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 3290, in create_op
    op_def=op_def)
  File "/l/gadichs1/conda_envs/deepgps_2_3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1654, in __init__
    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access

FailedPreconditionError (see above for traceback): Attempting to use uninitialized value beta1_power
     [[Node: beta1_power/read = Identity[T=DT_FLOAT, _class=["loc:@DGP-26d70ccd-89/layers/0/feature/Z/unconstrained"], _device="/job:localhost/replica:0/task:0/device:CPU:0"](beta1_power)]]
imsrgadich commented 6 years ago

solved in GPFlow #677 by mrksr.