Error when verifying output shape for relative pos emb

[mmz33]

I am getting this issue when running: python3 -m tests.test_nn_conformer

Here is the error backtrace:

EXCEPTION
Traceback (most recent call last):
  File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.7/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    line: return _run_code(code, main_globals, None,
                           "__main__", mod_spec)
    locals:
      _run_code = <global> <function _run_code at 0x1008c9a60>
      code = <local> <code object <module> at 0x100888930, file "/Users/mzeineldeen/Desktop/dev/returnn_common/tests/test_nn_conformer.py", line 3>
      main_globals = <local> {'__name__': '__main__', '__doc__': '\nTest nn.conformer.\n', '__package__': 'returnn_common.tests', '__loader__': <_frozen_importlib_external.SourceFileLoader object at 0x100936240>, '__spec__': ModuleSpec(name='tests.test_nn_conformer', loader=<_frozen_importlib_external.SourceFileLoader object..., len = 10
      mod_spec = <local> ModuleSpec(name='tests.test_nn_conformer', loader=<_frozen_importlib_external.SourceFileLoader object at 0x100936240>, origin='/Users/mzeineldeen/Desktop/dev/returnn_common/tests/test_nn_conformer.py')
  File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.7/lib/python3.7/runpy.py", line 85, in _run_code
    line: exec(code, run_globals)
    locals:
      exec = <builtin> <built-in function exec>
      code = <local> <code object <module> at 0x100888930, file "/Users/mzeineldeen/Desktop/dev/returnn_common/tests/test_nn_conformer.py", line 3>
      run_globals = <local> {'__name__': '__main__', '__doc__': '\nTest nn.conformer.\n', '__package__': 'returnn_common.tests', '__loader__': <_frozen_importlib_external.SourceFileLoader object at 0x100936240>, '__spec__': ModuleSpec(name='tests.test_nn_conformer', loader=<_frozen_importlib_external.SourceFileLoader object..., len = 10
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/tests/test_nn_conformer.py", line 7, in <module>
    line: from . import _setup_test_env  # noqa
    locals:
      from = <not found>
      from.import = <not found>
      _setup_test_env = <not found>
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/tests/_setup_test_env.py", line 307, in <module>
    line: setup()
    locals:
      setup = <local> <function setup at 0x10091e158>
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/tests/_setup_test_env.py", line 134, in setup
    line: _main(main_mod_)
    locals:
      _main = <global> <function _main at 0x10091e6a8>
      main_mod_ = <local> <module 'returnn_common.tests.test_nn_conformer' from '/Users/mzeineldeen/Desktop/dev/returnn_common/tests/test_nn_conformer.py'>
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/tests/_setup_test_env.py", line 298, in _main
    line: value()
    locals:
      value = <local> <function test_nn_conformer at 0x14e7c2ae8>
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/tests/test_nn_conformer.py", line 39, in test_nn_conformer
    line: out, _ = conformer(data, in_spatial_dim=time_dim)
    locals:
      out = <not found>
      _ = <not found>
      conformer = <local> <ConformerEncoder>
      data = <local> <Tensor /'data:data' [B,T|'time'[B],F|F'input'(10)]>
      in_spatial_dim = <not found>
      time_dim = <local> Dim{'time'[B]}
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/nn/conformer.py", line 278, in ConformerEncoder.__call__
    line: x = self.layers(x, axis=out_spatial_dim)
    locals:
      x = <local> <Tensor /'conformer_encoder'/'dropout' [B,T|'conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0'[B],F|F'out'(14)] via 'dropout'>
      self = <local> <ConformerEncoder>
      self.layers = <local> <Sequential>, len = 2
      axis = <not found>
      out_spatial_dim = <local> Dim{'conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0'[B]}
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/nn/container.py", line 80, in Sequential.__call__
    line: inp = module(inp, **kwargs)
    locals:
      inp = <local> <Tensor /'conformer_encoder'/'dropout' [B,T|'conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0'[B],F|F'out'(14)] via 'dropout'>
      module = <local> <ConformerEncoderLayer>
      kwargs = <local> {'axis': Dim{'conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0'[B]}}
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/nn/conformer.py", line 203, in ConformerEncoderLayer.__call__
    line: x_mhsa = self.self_att(x_mhsa_ln, axis=axis)
    locals:
      x_mhsa = <not found>
      self = <local> <ConformerEncoderLayer>
      self.self_att = <local> <RelPosSelfAttention>
      x_mhsa_ln = <local> <Tensor /'conformer_encoder'/'layers'/'0'/'self_att_layer_norm'/'add_0'/'combine'/'add' [B,T|'conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0'[B],F|F'out'(14)] via 'combine'>
      axis = <local> Dim{'conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0'[B]}
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/nn/attention.py", line 183, in RelPosSelfAttention.__call__
    line: pos_emb, pos_emb_spatial_dim = relative_positional_encoding(axis, self.in_dim)
    locals:
      pos_emb = <not found>
      pos_emb_spatial_dim = <not found>
      relative_positional_encoding = <global> <function relative_positional_encoding at 0x14f6c4a60>
      axis = <local> Dim{'conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0'[B]}
      self = <local> <RelPosSelfAttention>
      self.in_dim = <local> Dim{F'out'(14)}
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/nn/attention.py", line 281, in relative_positional_encoding
    line: emb.verify_out_shape({out_spatial_dim, feat_dim})
    locals:
      emb = <local> <Tensor /'conformer_encoder'/'layers'/'0'/'self_att'/'relative_positional_encoding'/'sin'/'_activation'/'sin' [T|'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B],F|F'out'(14)] via 'activ...
      emb.verify_out_shape = <local> <bound method Tensor.verify_out_shape of <Tensor /'conformer_encoder'/'layers'/'0'/'self_att'/'relative_positional_encoding'/'sin'/'_activation'/'sin' [T|'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-s...
      out_spatial_dim = <local> Dim{'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B]}
      feat_dim = <local> Dim{F'out'(14)}
  File "/Users/mzeineldeen/Desktop/dev/returnn_common/nn/base.py", line 237, in Tensor.verify_out_shape
    line: self.data.verify_out_shape(out_shape)
    locals:
      self = <local> <Tensor /'conformer_encoder'/'layers'/'0'/'self_att'/'relative_positional_encoding'/'sin'/'_activation'/'sin' [T|'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B],F|F'out'(14)] via 'activ...
      self.data = <local> Data{'sin_output', [T|'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B],F|F'out'(14)]}
      self.data.verify_out_shape = <local> <bound method Data.verify_out_shape of Data{'sin_output', [T|'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B],F|F'out'(14)]}>
      out_shape = <local> {Dim{F'out'(14)}, Dim{'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B]}}, len = 2
  File "/Users/mzeineldeen/Desktop/dev/returnn/returnn/tf/util/data.py", line 3099, in Data.verify_out_shape
    line: raise VerifyOutShapeException(
            "%s verify_out_shape, dims %s are not specified in out_shape %s" % (self, remaining, out_shape))
    locals:
      VerifyOutShapeException = <global> <class 'returnn.tf.util.data.VerifyOutShapeException'>
      self = <local> Data{'sin_output', [T|'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B],F|F'out'(14)]}
      remaining = <local> {Dim{B}}, len = 1
      out_shape = <local> {Dim{F'out'(14)}, Dim{'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B]}}, len = 2
VerifyOutShapeException: Data{'sin_output', [T|'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B],F|F'out'(14)]} verify_out_shape, dims {Dim{B}} are not specified in out_shape {Dim{F'out'(14)}, Dim{'(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)+-1+(conformer_encoder/conv_subsample_layer/pool2d_0/_pool_nd:out-spatial-dim0)'[B]}}

The batch dim tag is still there in the remaining which should not be the case i guess right?

[albertz]

I also just stumbled upon this. I describe the problem here: https://github.com/rwth-i6/returnn/issues/1153

[albertz]

For now, if you want to continue working on Conformer, you could just comment out the verify_out_shape line. But there might be other errors.

[albertz]

Ok this error should be fixed now. You need to update RETURNN again.