TypeError: `generator` yielded an element that did not match the expected structure.

[XWcode11]

errors:

(humatch) [test@localhost Humatch]$ Humatch-classify -H QVQLVQSGAEVNKPGASVKVSCKASGYTFTGYVVHWVRQAPGQRLEWMGWINTGNGDTKYSQKFQGRVSITRDTSANTAYMEVSTLRSEDTAVYYCARDRGGSGDFDYWGQGTLVTVSS -L EIVLTQSPVTLSLSPGERATLSCRASQSVSFYLAWYQQKPGQAPRLLICDASNRATGIPARFSGSGSGTDFTLTISSLEPEDFAVYYCQQRSDWPYTFGQGTKLEIK -s Traceback (most recent call last): File "/home/kl/anaconda3/envs/humatch/bin/Humatch-classify", line 8, in sys.exit(command_line_interface()) File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/Humatch/classify.py", line 158, in command_line_interface predictions_heavy = predict_from_list_of_seq_strs(H_seqs, load_cnn(HEAVY_WEIGHTS, "heavy")) File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/Humatch/classify.py", line 33, in predict_from_list_of_seq_strs return model.predict(test_generator, verbose=CNN_verbose) File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/keras/src/utils/traceback_utils.py", line 122, in error_handler raise e.with_traceback(filtered_tb) from None File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/tensorflow/python/framework/ops.py", line 5983, in raise_from_not_ok_status raise core._status_to_exception(e) from None # pylint: disable=protected-access tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node wrappedIteratorGetNext_output_types_1device/job:localhost/replica:0/task:0/device:CPU:0}} TypeError: generator yielded an element that did not match the expected structure. The expected structure was (tf.float64,), but the yielded element was [[[-0.47 0.24 0.07 ... -0.71 -0.03 -2.33] [-0.74 -0.71 2.04 ... 0.06 -0.46 0.65] [-0.47 0.24 0.07 ... -0.71 -0.03 -2.33] ... [-0.74 -0.71 2.04 ... 0.06 -0.46 0.65] [ 0.81 -1.08 0.16 ... -1.15 -0.97 -0.23] [ 0.81 -1.08 0.16 ... -1.15 -0.97 -0.23]]]. Traceback (most recent call last):

File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/tensorflow/python/data/ops/from_generator_op.py", line 204, in generator_py_func flattened_values = nest.flatten_up_to(output_types, values)

File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/tensorflow/python/data/util/nest.py", line 237, in flatten_up_to return nest_util.flatten_up_to(

File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/tensorflow/python/util/nest_util.py", line 1541, in flatten_up_to return _tf_data_flatten_up_to(shallow_tree, input_tree)

File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/tensorflow/python/util/nest_util.py", line 1570, in _tf_data_flatten_up_to _tf_data_assert_shallow_structure(shallow_tree, input_tree)

File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/tensorflow/python/util/nest_util.py", line 1414, in _tf_data_assert_shallow_structure raise TypeError(

TypeError: If shallow structure is a sequence, input must also be a sequence. Input has type: 'ndarray'.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):

File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/tensorflow/python/ops/script_ops.py", line 270, in call ret = func(*args)

File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/tensorflow/python/autograph/impl/api.py", line 643, in wrapper return func(*args, **kwargs)

File "/home/kl/anaconda3/envs/humatch/lib/python3.9/site-packages/tensorflow/python/data/ops/from_generator_op.py", line 206, in generator_py_func raise TypeError(

TypeError: generator yielded an element that did not match the expected structure. The expected structure was (tf.float64,), but the yielded element was [[[-0.47 0.24 0.07 ... -0.71 -0.03 -2.33] [-0.74 -0.71 2.04 ... 0.06 -0.46 0.65] [-0.47 0.24 0.07 ... -0.71 -0.03 -2.33] ... [-0.74 -0.71 2.04 ... 0.06 -0.46 0.65] [ 0.81 -1.08 0.16 ... -1.15 -0.97 -0.23] [ 0.81 -1.08 0.16 ... -1.15 -0.97 -0.23]]].

 [[{{node PyFunc}}]] [Op:IteratorGetNext] name: 

packages version

flatbuffers-24.3.25 keras-3.6.0 ml-dtypes-0.3.2 tensorboard-2.16.2 tensorflow-2.16.1

thanks!

[XWcode11]

I modified the CustomDataGenerator class in dataset.py, and the issue has been resolved.

class CustomDataGenerator(tf.keras.utils.Sequence):
    '''
    Custom generator is required for sparse data input to keras model
    tf.keras.utils.Sequence allows multiprocessing in safe way (won't train on same batch twice)

    :param seqs: list of aligned sequence strings
    :param batch_size: int, batch size for training
    :param num_cpus: int, number of cpus to use when encoding sequences
    '''
    def __init__(self, seqs, batch_size=16384, num_cpus=None):
        '''
        '''
        super().__init__()
        self.seqs = seqs
        self.batch_size = batch_size
        self.num_cpus = num_cpus

    def __len__(self):
        '''
        Get the number of batches
        '''
        return math.ceil(len(self.seqs) / self.batch_size)

    def __getitem__(self, index):
        '''
        Get Kidera encoded ndarrays, X, for a batch of sequences
        '''
        low_idx = index * self.batch_size
        high_idx = min((index + 1) * self.batch_size, len(self.seqs))
        batch_seqs = self.seqs[low_idx:high_idx]

        # modified
        X = get_X_from_list_of_seq_strs(batch_seqs, self.num_cpus)

        if X.dtype != np.float64:
            X = X.astype(np.float64)

        return (X,)