Transformer encoder part missing dense layer.

[sentinel8b]

Hello, I found there's no embedding(Dense) layer on encoder part of transformer model.

class TransformerEncoder(tf.keras.layers.Layer):
    def __init__(
        self,
        num_layers,
        d_model,
        num_heads,
        dff,
        maximum_position_encoding,
        dropout_rate=0.1,
    ):
        super(TransformerEncoder, self).__init__()

        self.d_model = d_model
        self.num_layers = num_layers
        self.pos_encoding = positional_encoding_1d(
            maximum_position_encoding, self.d_model
        )
        self.enc_layers = [
            TransformerEncoderLayer(d_model, num_heads, dff, dropout_rate)
            for _ in range(num_layers)
        ]
        self.dropout = tf.keras.layers.Dropout(dropout_rate)

    def call(self, x, training, mask=None):
        # adding embedding and position encoding.
        #   – (batch_size, input_seq_len, d_model)
        # -> Here you need embedding! ( I think you just deleted it updating the code)
        x += self.pos_encoding
        x = self.dropout(x, training=training)

        for i in range(self.num_layers):
            x = self.enc_layers[i](x, training, mask)

        #   – (batch_size, input_seq_len, d_model)
        return x

[Kohulan]

@sentinel8b Thanks for the issue it is fixed now