During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/data1/home/RoseTTAFold/DAN-msa/ErrorPredictorMSA.py", line 222, in
main()
File "/data1/home/RoseTTAFold/DAN-msa/ErrorPredictorMSA.py", line 181, in main
verbose=args.verbose)
File "/data1/home/RoseTTAFold/DAN-msa/pyErrorPred/predict.py", line 84, in predict
lddt, estogram, mask = model.predict2(batch)
File "/data1/home/RoseTTAFold/DAN-msa/pyErrorPred/model.py", line 471, in predict2
return self.sesh.run(operations, feed_dict=feed_dict)
File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 956, in run
run_metadata_ptr)
File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 1180, in _run
feed_dict_tensor, options, run_metadata)
File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 1359, in _do_run
run_metadata)
File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 1384, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: 2 root error(s) found.
(0) Internal: Blas SGEMM launch failed : m=2612736, n=20, k=20
[[node 3d_conv/conv3d/Conv3D (defined at home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]
[[2d_conv/lddt/truediv/_1161]]
(1) Internal: Blas SGEMM launch failed : m=2612736, n=20, k=20
[[node 3d_conv/conv3d/Conv3D (defined at home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]
0 successful operations.
0 derived errors ignored.
Original stack trace for '3d_conv/conv3d/Conv3D':
File "data1/home/RoseTTAFold/DAN-msa/ErrorPredictorMSA.py", line 222, in
main()
File "data1/home/RoseTTAFold/DAN-msa/ErrorPredictorMSA.py", line 181, in main
verbose=args.verbose)
File "data1/home/RoseTTAFold/DAN-msa/pyErrorPred/predict.py", line 77, in predict
verbose=False)
File "data1/home/RoseTTAFold/DAN-msa/pyErrorPred/model.py", line 73, in init
self.ops = self.build()
File "data1/home/RoseTTAFold/DAN-msa/pyErrorPred/model.py", line 117, in build
layers.append(tf.layers.conv3d(grid3d, 20, 1, padding='same', use_bias=False))
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/util/deprecation.py", line 324, in new_func
return func(*args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/layers/convolutional.py", line 632, in conv3d
return layer.apply(inputs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/util/deprecation.py", line 324, in new_func
return func(*args, *kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py", line 1700, in apply
return self.call(inputs, args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/layers/base.py", line 548, in call
outputs = super(Layer, self).call(inputs, *args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py", line 854, in call
outputs = call_fn(cast_inputs, *args, *kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/autograph/impl/api.py", line 234, in wrapper
return converted_call(f, options, args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/autograph/impl/api.py", line 439, in converted_call
return _call_unconverted(f, args, kwargs, options)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/autograph/impl/api.py", line 330, in _call_unconverted
return f(args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/keras/layers/convolutional.py", line 197, in call
outputs = self._convolution_op(inputs, self.kernel)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/ops/nn_ops.py", line 1134, in call
return self.conv_op(inp, filter)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/ops/nn_ops.py", line 639, in call
return self.call(inp, filter)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/ops/nn_ops.py", line 238, in call
name=self.name)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_nn_ops.py", line 1553, in conv3d
dilations=dilations, name=name)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py", line 1748, in init
self._traceback = tf_stack.extract_stack()
Error detail: Running DeepAccNet-msa Traceback (most recent call last): File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 1365, in _do_call return fn(*args) File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 1350, in _run_fn target_list, run_metadata) File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 1443, in _call_tf_sessionrun run_metadata) tensorflow.python.framework.errors_impl.InternalError: 2 root error(s) found. (0) Internal: Blas SGEMM launch failed : m=2612736, n=20, k=20 [[{{node 3d_conv/conv3d/Conv3D}}]] [[2d_conv/lddt/truediv/_1161]] (1) Internal: Blas SGEMM launch failed : m=2612736, n=20, k=20 [[{{node 3d_conv/conv3d/Conv3D}}]] 0 successful operations. 0 derived errors ignored.
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "/data1/home/RoseTTAFold/DAN-msa/ErrorPredictorMSA.py", line 222, in
main()
File "/data1/home/RoseTTAFold/DAN-msa/ErrorPredictorMSA.py", line 181, in main
verbose=args.verbose)
File "/data1/home/RoseTTAFold/DAN-msa/pyErrorPred/predict.py", line 84, in predict
lddt, estogram, mask = model.predict2(batch)
File "/data1/home/RoseTTAFold/DAN-msa/pyErrorPred/model.py", line 471, in predict2
return self.sesh.run(operations, feed_dict=feed_dict)
File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 956, in run
run_metadata_ptr)
File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 1180, in _run
feed_dict_tensor, options, run_metadata)
File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 1359, in _do_run
run_metadata)
File "/home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/client/session.py", line 1384, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: 2 root error(s) found.
(0) Internal: Blas SGEMM launch failed : m=2612736, n=20, k=20
[[node 3d_conv/conv3d/Conv3D (defined at home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]
[[2d_conv/lddt/truediv/_1161]]
(1) Internal: Blas SGEMM launch failed : m=2612736, n=20, k=20
[[node 3d_conv/conv3d/Conv3D (defined at home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]
0 successful operations.
0 derived errors ignored.
Original stack trace for '3d_conv/conv3d/Conv3D': File "data1/home/RoseTTAFold/DAN-msa/ErrorPredictorMSA.py", line 222, in
main()
File "data1/home/RoseTTAFold/DAN-msa/ErrorPredictorMSA.py", line 181, in main
verbose=args.verbose)
File "data1/home/RoseTTAFold/DAN-msa/pyErrorPred/predict.py", line 77, in predict
verbose=False)
File "data1/home/RoseTTAFold/DAN-msa/pyErrorPred/model.py", line 73, in init
self.ops = self.build()
File "data1/home/RoseTTAFold/DAN-msa/pyErrorPred/model.py", line 117, in build
layers.append(tf.layers.conv3d(grid3d, 20, 1, padding='same', use_bias=False))
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/util/deprecation.py", line 324, in new_func
return func(*args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/layers/convolutional.py", line 632, in conv3d
return layer.apply(inputs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/util/deprecation.py", line 324, in new_func
return func(*args, *kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py", line 1700, in apply
return self.call(inputs, args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/layers/base.py", line 548, in call
outputs = super(Layer, self).call(inputs, *args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py", line 854, in call
outputs = call_fn(cast_inputs, *args, *kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/autograph/impl/api.py", line 234, in wrapper
return converted_call(f, options, args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/autograph/impl/api.py", line 439, in converted_call
return _call_unconverted(f, args, kwargs, options)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/autograph/impl/api.py", line 330, in _call_unconverted
return f(args, kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/keras/layers/convolutional.py", line 197, in call
outputs = self._convolution_op(inputs, self.kernel)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/ops/nn_ops.py", line 1134, in call
return self.conv_op(inp, filter)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/ops/nn_ops.py", line 639, in call
return self.call(inp, filter)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/ops/nn_ops.py", line 238, in call
name=self.name)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_nn_ops.py", line 1553, in conv3d
dilations=dilations, name=name)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "home/dpai1/anaconda3/envs/rose37/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py", line 1748, in init
self._traceback = tf_stack.extract_stack()