At `build()` only two last axis are numerics

[markub3327]

Hello,

I created weights at build():

def build(self, input_shape):
        super(PositionalEmbedding, self).build(input_shape)

        print(input_shape)
        self.position = self.add_weight(
            name="position",
            shape=(1, input_shape[1], input_shape[2], self.units),
            initializer=TruncatedNormal(stddev=0.02),
            trainable=True,
        )

The input's shape: (64, 7, 25, 81) # (batches, timesteps, patches, features) input_shape of build(): (None, None, 25, 81) # (batches, timesteps, patches, features)

Error

ValueError: in user code:

    File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/engine/training.py", line 1021, in train_function  *
        return step_function(self, iterator)
    File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/engine/training.py", line 1010, in step_function  **
        outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/engine/training.py", line 1000, in run_step  **
        outputs = model.train_step(data)
    File "/var/folders/7v/fqqcktvs23qc8fwgftjpz_gh0000gn/T/ipykernel_6464/643090246.py", line 97, in train_step
        y_pred, _ = self([inputs, targets_inputs], training=True)
    File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 67, in error_handler
        raise e.with_traceback(filtered_tb) from None

    ValueError: Exception encountered when calling layer "transformer_111" (type Transformer).

    in user code:

        File "/var/folders/7v/fqqcktvs23qc8fwgftjpz_gh0000gn/T/ipykernel_6464/643090246.py", line 52, in call  *
            x_e = self.pos_embs_0(x_e, training=training)
        File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 67, in error_handler  **
            raise e.with_traceback(filtered_tb) from None
        File "/var/folders/7v/fqqcktvs23qc8fwgftjpz_gh0000gn/T/ipykernel_6464/3588400932.py", line 14, in build
            self.position = self.add_weight(

        ValueError: Can't convert Python sequence with mixed types to Tensor.

    Call arguments received:
      • inputs=['tf.Tensor(shape=(None, None, 25, 81), dtype=float32)']
      • training=True

[tilakrayal]

@markub3327 , In order to expedite the trouble-shooting process, could you please provide a complete code and the TensorFlow version you are using.

[markub3327]

@tilakrayal

TF version: 2.8.0 Python: 3.9.7

Code

It's the 4D variant of your Transfromer.

from tensorflow.keras.layers import Dense, Dropout, Layer, Input
from tensorflow.keras.initializers import TruncatedNormal
from tensorflow.keras.models import Model

import tensorflow as tf
import numpy as np

class PositionalEmbedding(Layer):
    def __init__(self, units, dropout_rate, **kwargs):
        super(PositionalEmbedding, self).__init__(**kwargs)

        self.units = units

        self.projection = Dense(units, kernel_initializer=TruncatedNormal(stddev=0.02))
        self.dropout = Dropout(rate=dropout_rate)

    def build(self, input_shape):
        super(PositionalEmbedding, self).build(input_shape)

        print(input_shape, '\n')
        self.position = self.add_weight(
            name="position",
            shape=(1, input_shape[1], input_shape[2], self.units),
            initializer=TruncatedNormal(stddev=0.02),
            trainable=True,
        )

    def call(self, inputs, training):
        x = self.projection(inputs)
        x += self.position

        return self.dropout(x, training=training)

class Transformer(Model):
    def __init__(
        self,
        embed_dim,
        dropout_rate,
        **kwargs
    ):
        super(Transformer, self).__init__(**kwargs)

        # Input
        self.pos_embs = PositionalEmbedding(embed_dim, dropout_rate)

    def compile(self, optimizer, loss):
        super(Transformer, self).compile()
        self.optimizer = optimizer
        self.loss = loss

    def call(self, inputs, training):
        inputs, targets = inputs

        return self.pos_embs(inputs, training=training)

    def train_step(self, inputs):
        inputs, targets = inputs

        print(inputs.shape)
        print(targets.shape, '\n')

        targets_inputs = targets[:, :-1]
        targets_real = targets[:, 1:]

        with tf.GradientTape() as tape:
            y_pred = self([inputs, targets_inputs], training=True)
            loss = self.loss(targets_real, y_pred)

        trainable_vars = self.trainable_variables
        gradients = tape.gradient(loss, trainable_vars)
        self.optimizer.apply_gradients(zip(gradients, trainable_vars))

        return {
            "loss": loss,
        }

def load_dataset(batch_size, window_size):
  x_all = np.ones((1000, 25, 81)) #np.load('./dataset/X_all.npy')
  y_all = np.ones((1000, 25, 1)) #np.load('./dataset/y_all.npy')

  inputs_dataset = tf.keras.preprocessing.timeseries_dataset_from_array(
      x_all, None, sequence_length=window_size, sequence_stride=(window_size // 2), batch_size=batch_size)
  targets_dataset = tf.keras.preprocessing.timeseries_dataset_from_array(
      y_all, None, sequence_length=window_size, sequence_stride=(window_size // 2), batch_size=batch_size)

  return inputs_dataset, targets_dataset

# load dataset
inputs_dataset, targets_dataset = load_dataset(
      batch_size=64,
      window_size=7,
)
dataset = tf.data.Dataset.zip((inputs_dataset, targets_dataset))

for batch in dataset:
    inputs, targets = batch
    print(inputs.shape)
    print(targets.shape, '\n')
    break

sample_transformer = Transformer(
    embed_dim=256, dropout_rate=0.1,
)

sample_transformer.compile(
      loss=tf.keras.losses.mean_squared_error,
      optimizer=tf.keras.optimizers.Adam(),
)

# Train model
sample_transformer.fit(
      dataset,
      epochs=10
)

Result

(64, 7, 25, 81)
(64, 7, 25, 1) 

Epoch 1/10
(None, None, 25, 81)
(None, None, 25, 1) 

(None, None, 25, 81)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/var/folders/7v/fqqcktvs23qc8fwgftjpz_gh0000gn/T/ipykernel_82475/681363551.py in <module>
    111 
    112 # Train model
--> 113 sample_transformer.fit(
    114       dataset,
    115       epochs=10

~/miniforge3/lib/python3.9/site-packages/keras/utils/traceback_utils.py in error_handler(*args, **kwargs)
     65     except Exception as e:  # pylint: disable=broad-except
     66       filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67       raise e.with_traceback(filtered_tb) from None
     68     finally:
     69       del filtered_tb

~/miniforge3/lib/python3.9/site-packages/tensorflow/python/framework/func_graph.py in autograph_handler(*args, **kwargs)
   1145           except Exception as e:  # pylint:disable=broad-except
   1146             if hasattr(e, "ag_error_metadata"):
-> 1147               raise e.ag_error_metadata.to_exception(e)
   1148             else:
   1149               raise

ValueError: in user code:

    File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/engine/training.py", line 1021, in train_function  *
        return step_function(self, iterator)
    File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/engine/training.py", line 1010, in step_function  **
        outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/engine/training.py", line 1000, in run_step  **
        outputs = model.train_step(data)
    File "/var/folders/7v/fqqcktvs23qc8fwgftjpz_gh0000gn/T/ipykernel_82475/681363551.py", line 66, in train_step
        y_pred = self([inputs, targets_inputs], training=True)
    File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 67, in error_handler
        raise e.with_traceback(filtered_tb) from None

    ValueError: Exception encountered when calling layer "transformer_17" (type Transformer).

    in user code:

        File "/var/folders/7v/fqqcktvs23qc8fwgftjpz_gh0000gn/T/ipykernel_82475/681363551.py", line 54, in call  *
            return self.pos_embs(inputs, training=training)
        File "/Users/martin/miniforge3/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 67, in error_handler  **
            raise e.with_traceback(filtered_tb) from None
        File "/var/folders/7v/fqqcktvs23qc8fwgftjpz_gh0000gn/T/ipykernel_82475/681363551.py", line 21, in build
            self.position = self.add_weight(

        ValueError: Can't convert Python sequence with mixed types to Tensor.

    Call arguments received:
      • inputs=['tf.Tensor(shape=(None, None, 25, 81), dtype=float32)', 'tf.Tensor(shape=(None, None, 25, 1), dtype=float32)']
      • training=True

[tilakrayal]

@gadagashwini , I was able to reproduce the issue in tf v2.8 and nightly.Please find the gist here.

[markub3327]

@gadagashwini

Hi, it is becouse print(dataset) gives me: <ZipDataset element_spec=(TensorSpec(shape=(None, None, 25, 81), dtype=tf.float64, name=None), TensorSpec(shape=(None, None, 25, 1), dtype=tf.float64, name=None))> .

The model knows from the dataset object only the last two axis, but I need to it knows the last 3 axis (or all axis without the batch axis 0).

EDIT

The shape must be defined for example: shape=(None, 7, 25, 81) or shape=(None, 7, 25, 1) !!!

[qlzh727]

Took a closer look. I think the issue here is from the dataset, which doesn't populate the shape info for the batch and sequence dim. Keras model/layer was working correctly, since the element spec from the dataset doesn't have the static shape info on those 2 dims.

Probably you want to check how you dataset is constructed, and update that.

[qlzh727]

I don't think there is anything keras team need to address here.

[google-ml-butler[bot]]

Are you satisfied with the resolution of your issue? Yes No

[markub3327]

@qlzh727 Yes, TF Dataset is part of TF. I'll ask at the TF community.

Thanks.

[gowthamkpr]

@qlzh727 Can you PTAL. Linking the issue 55546. Thanks!

[markub3327]

Thanks for re-open.

[markub3327]

@fchollet

This is the solution by @aaudiber.

targets_dataset = tf.keras.preprocessing.timeseries_dataset_from_array(
      y_all, None, sequence_length=window_size, sequence_stride=(window_size // 2), batch_size=batch_size)
# Set the shape for the elements of the dataset.
targets_dataset = targets_dataset.map(lambda x: tf.ensure_shape(x, <expected shape>)

I think the better way is directly fix tf.keras.preprocessing.timeseries_dataset_from_array to pass correct shape without using map and tf.ensure_shape.

---

>>> dataset = tf.keras.preprocessing.timeseries_dataset_from_array(data=tf.random.uniform([10000, 25, 81]), targets=tf.random.uniform([10000, 25, 1]), sequence_length=30, sequence_stride=1, batch_size=64)
>>> print(dataset)
<BatchDataset element_spec=(TensorSpec(shape=(None, None, 25, 81), dtype=tf.float32, name=None), TensorSpec(shape=(None, 25, 1), dtype=tf.float32, name=None))>

[qlzh727]

Ack. I guess we could have a determined value populated for the element spec for the dataset (we know the length of the data and the sequence length). Will try to fix it when I have some free cycle.

In the meantime, feel free to send PR if you have already have a fix for it.

[markub3327]

@qlzh727

Do you think the tf.data.Dataset.window will be a better way to create timeseries dataset from array?

[qlzh727]

@qlzh727

Do you think the tf.data.Dataset.window will be a better way to create timeseries dataset from array?

Good point, we might be able to rewrite the existing implementation with tf.data API in a cleaner way.

[markub3327]

@divyashreepathihalli

Please wait ... I'll create PR.