Model SHAHD trains on CPU , but fails on apple silicon

Here is the error:

Loading jj_SAEHD model...

Choose one or several GPU idxs (separated by comma).

[CPU] : CPU [0] : METAL

[0] Which GPU indexes to choose? : 0

session vars GPU:0 Metal device set to: Apple M2 Pro

systemMemory: 16.00 GB maxCacheSize: 5.33 GB

device config <core.leras.nn.nn.DeviceConfig object at 0x127b64ca0> devices <core.leras.device.Devices object at 0x127b64b80> worst devices [0]:[METAL][0.0/0.0] Press enter in 2 seconds to override model settings.models opt device GPU:0 archi_type liae Initializing models: 0%| | 0/5 [00:00<?, ?it/s] Error: Graph execution error:

Detected at node 'encoder/down1/downs_0/conv1/weight/Initializer/cai' defined at (most recent call last): File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 930, in _bootstrap self._bootstrap_inner() File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 973, in _bootstrap_inner self.run() File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 910, in run self._target(*self._args, *self._kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/mainscripts/Trainer.py", line 46, in trainerThread model = models.import_model(model_class_name)( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/ModelBase.py", line 193, in init self.on_initialize() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/Model_SAEHD/Model.py", line 337, in on_initialize self.src_dst_saveable_weights = self.encoder.get_weights() + self.inter_AB.get_weights() + self.inter_B.get_weights() + self.decoder.get_weights() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 77, in get_weights self.build() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build self._build_sub(v[name],name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub layer.build() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build self._build_sub(v[name],name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 20, in _build_sub self._buildsub(sublayer, f"{name}{i}") File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub layer.build() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build self._build_sub(v[name],name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 33, in _build_sub layer.build_weights() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/layers/Conv2D.py", line 61, in build_weights self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.out_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable ) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/initializers/init.py", line 13, in call return tf.zeros( shape, dtype=dtype, name="cai") Node: 'encoder/down1/downs_0/conv1/weight/Initializer/cai' Detected at node 'encoder/down1/downs_0/conv1/weight/Initializer/cai' defined at (most recent call last): File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 930, in _bootstrap self._bootstrap_inner() File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 973, in _bootstrap_inner self.run() File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 910, in run self._target(self._args, **self._kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/mainscripts/Trainer.py", line 46, in trainerThread model = models.import_model(model_class_name)( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/ModelBase.py", line 193, in init self.on_initialize() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/Model_SAEHD/Model.py", line 337, in on_initialize self.src_dst_saveable_weights = self.encoder.get_weights() + self.inter_AB.get_weights() + self.inter_B.get_weights() + self.decoder.get_weights() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 77, in get_weights self.build() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build self._build_sub(v[name],name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub layer.build() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build self._build_sub(v[name],name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 20, in _build_sub self._buildsub(sublayer, f"{name}{i}") File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub layer.build() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build self._build_sub(v[name],name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 33, in _build_sub layer.build_weights() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/layers/Conv2D.py", line 61, in build_weights self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.out_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable ) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/initializers/init.py", line 13, in call return tf.zeros( shape, dtype=dtype, name="cai") Node: 'encoder/down1/downs_0/conv1/weight/Initializer/cai' Cannot assign a device for operation encoder/down1/downs_0/conv1/weight/Initializer/cai: Could not satisfy explicit device specification '' because the node {{colocation_node encoder/down1/downs_0/conv1/weight/Initializer/cai}} was colocated with a group of nodes that required incompatible device '/device:GPU:0'. All available devices [/job:localhost/replica:0/task:0/device:CPU:0, /job:localhost/replica:0/task:0/device:GPU:0]. Colocation Debug Info: Colocation group had the following types and supported devices: Root Member(assigned_device_nameindex=-1 requested_devicename='/device:GPU:0' assigned_devicename='' resource_devicename='/device:GPU:0' supported_devicetypes=[CPU] possibledevices=[] Identity: GPU CPU Assign: CPU VariableV2: GPU CPU Fill: GPU CPU Const: GPU CPU

Colocation members, user-requested devices, and framework assigned devices, if any: encoder/down1/downs_0/conv1/weight/Initializer/cai/shape_as_tensor (Const) encoder/down1/downs_0/conv1/weight/Initializer/cai/Const (Const) encoder/down1/downs_0/conv1/weight/Initializer/cai (Fill) encoder/down1/downs_0/conv1/weight (VariableV2) /device:GPU:0 encoder/down1/downs_0/conv1/weight/Assign (Assign) /device:GPU:0 encoder/down1/downs_0/conv1/weight/read (Identity) /device:GPU:0 Assign_2 (Assign) /device:GPU:0 Assign_168 (Assign) /device:GPU:0

 [[{{node encoder/down1/downs_0/conv1/weight/Initializer/_cai_}}]]

Original stack trace for 'encoder/down1/downs_0/conv1/weight/Initializer/cai': File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 930, in _bootstrap self._bootstrap_inner() File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 973, in _bootstrap_inner self.run() File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 910, in run self._target(*self._args, self._kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/mainscripts/Trainer.py", line 46, in trainerThread model = models.import_model(model_class_name)( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/ModelBase.py", line 193, in init self.on_initialize() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/Model_SAEHD/Model.py", line 337, in on_initialize self.src_dst_saveable_weights = self.encoder.get_weights() + self.inter_AB.get_weights() + self.inter_B.get_weights() + self.decoder.get_weights() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 77, in get_weights self.build() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build self._build_sub(v[name],name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub layer.build() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build self._build_sub(v[name],name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 20, in _build_sub self._buildsub(sublayer, f"{name}{i}") File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub layer.build() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build self._build_sub(v[name],name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 33, in _build_sub layer.build_weights() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/layers/Conv2D.py", line 61, in build_weights self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.out_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable ) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variable_scope.py", line 1616, in get_variable return get_variable_scope().get_variable( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variable_scope.py", line 1326, in get_variable return var_store.get_variable( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variable_scope.py", line 582, in get_variable return _true_getter( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variable_scope.py", line 535, in _true_getter return self._get_single_variable( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variable_scope.py", line 956, in _get_single_variable v = variables.VariableV1( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/util/traceback_utils.py", line 150, in error_handler return fn(*args, *kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variables.py", line 264, in call return cls._variable_v1_call(args, kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variables.py", line 209, in _variable_v1_call return previous_getter( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variables.py", line 202, in previous_getter = lambda kwargs: default_variable_creator(None, kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variable_scope.py", line 2721, in default_variable_creator return variables.RefVariable( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/util/traceback_utils.py", line 150, in error_handler return fn(*args, kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variables.py", line 268, in call return super(VariableMetaclass, cls).call(*args, *kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variables.py", line 1647, in init self._init_from_args( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variables.py", line 1790, in _init_from_args initial_value = initial_value() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/initializers/init.py", line 13, in call return tf.zeros( shape, dtype=dtype, name="cai") File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/util/traceback_utils.py", line 150, in error_handler return fn(args, kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/util/dispatch.py", line 1082, in op_dispatch_handler return dispatch_target(*args, kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/array_ops.py", line 2972, in wrapped tensor = fun(*args, *kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/array_ops.py", line 3033, in zeros output = fill(shape, constant(zero, dtype=dtype), name=name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/util/traceback_utils.py", line 150, in error_handler return fn(args, kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/util/dispatch.py", line 1082, in op_dispatch_handler return dispatch_target(*args, **kwargs) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/array_ops.py", line 246, in fill result = gen_array_ops.fill(dims, value, name=name) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/gen_arrayops.py", line 3509, in fill , _, _op, _outputs = _op_def_library._apply_op_helper( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/framework/op_def_library.py", line 797, in _apply_op_helper op = g._create_op_internal(op_type_name, inputs, dtypes=None, File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/framework/ops.py", line 3754, in _create_op_internal ret = Operation( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/framework/ops.py", line 2133, in init self._traceback = tf_stack.extract_stack_for_node(self._c_op)

Traceback (most recent call last): File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1377, in _do_call return fn(*args) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1359, in _run_fn self._extend_graph() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1400, in _extend_graph tf_session.ExtendSession(self._session) tensorflow.python.framework.errors_impl.InvalidArgumentError: Cannot assign a device for operation encoder/down1/downs_0/conv1/weight/Initializer/cai: Could not satisfy explicit device specification '' because the node {{colocation_node encoder/down1/downs_0/conv1/weight/Initializer/cai}} was colocated with a group of nodes that required incompatible device '/device:GPU:0'. All available devices [/job:localhost/replica:0/task:0/device:CPU:0, /job:localhost/replica:0/task:0/device:GPU:0]. Colocation Debug Info: Colocation group had the following types and supported devices: Root Member(assigned_device_nameindex=-1 requested_devicename='/device:GPU:0' assigned_devicename='' resource_devicename='/device:GPU:0' supported_devicetypes=[CPU] possibledevices=[] Identity: GPU CPU Assign: CPU VariableV2: GPU CPU Fill: GPU CPU Const: GPU CPU

 [[{{node encoder/down1/downs_0/conv1/weight/Initializer/_cai_}}]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/mainscripts/Trainer.py", line 46, in trainerThread model = models.import_model(model_class_name)( File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/ModelBase.py", line 193, in init self.on_initialize() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/Model_SAEHD/Model.py", line 660, in on_initialize model.init_weights() File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/layers/Saveable.py", line 106, in init_weights nn.init_weights(self.get_weights()) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/ops/init.py", line 48, in init_weights nn.tf_sess.run (ops) File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 967, in run result = self._run(None, fetches, feed_dict, options_ptr, File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1190, in _run results = self._do_run(handle, final_targets, final_fetches, File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1370, in _do_run return self._do_call(_run_fn, feeds, fetches, targets, options, File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/client/session.py", line 1396, in _do_call raise type(e)(node_def, op, message) # pylint: disable=no-value-for-parameter tensorflow.python.framework.errors_impl.InvalidArgumentError: Graph execution error:

 [[{{node encoder/down1/downs_0/conv1/weight/Initializer/_cai_}}]]

chychkan / DeepFaceLab_MacOS

Model SHAHD trains on CPU , but fails on apple silicon #134