BUG in audio poisoning speech commands

[Uncertain-Quark]

After 0.16.2 release, I am stuck with the following OOM issue. I raised this previously and was fixed in 0.16.1, This is the error log:

Traceback (most recent call last):

File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/runpy.py", line 196, in _run_module_as_main return _run_code(code, main_globals, None, │ │ └ {'name': 'main', 'doc': '\nMain script for running scenarios. Users will run a scenario by calling:\n armory r... │ └ <code object at 0x7f454058fb50, file "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/armory/s... └ <function _run_code at 0x7f4629d19900> File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/runpy.py", line 86, in _run_code exec(code, run_globals) │ └ {'name': 'main', 'doc': '\nMain script for running scenarios. Users will run a scenario by calling:\n armory r... └ <code object at 0x7f454058fb50, file "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/armory/s... File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/armory/scenarios/main.py", line 228, in run_config( └ <function run_config at 0x7f451f411120> File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/armory/scenarios/main.py", line 144, in run_config scenario.evaluate() │ └ <function Scenario.evaluate at 0x7f4511fa9120> └ <armory.scenarios.poison.Poison object at 0x7f462941aef0>

File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/armory/scenarios/scenario.py", line 440, in evaluate self._evaluate() │ └ <function Scenario._evaluate at 0x7f4511fa9090> └ <armory.scenarios.poison.Poison object at 0x7f462941aef0> File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/armory/scenarios/scenario.py", line 429, in _evaluate self.load() │ └ <function Poison.load at 0x7f4511faa050> └ <armory.scenarios.poison.Poison object at 0x7f462941aef0> File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/armory/scenarios/poison.py", line 477, in load self.fit() │ └ <function Poison.fit at 0x7f4511fa9cf0> └ <armory.scenarios.poison.Poison object at 0x7f462941aef0> File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/armory/scenarios/poison.py", line 332, in fit self.model.fit( │ │ └ <function InputFilter.init..make_replacement..replacement_function at 0x7f45480ea710> │ └ art.estimators.classification.tensorflow.TensorFlowV2Classifier(model=<keras.engine.sequential.Sequential object at 0x7f45102... └ <armory.scenarios.poison.Poison object at 0x7f462941aef0> File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/art/estimators/classification/classifier.py", line 73, in replacement_function return fdict[func_name](self, *args, **kwargs) │ │ │ │ └ {'batch_size': 64, 'nb_epochs': 20, 'verbose': False, 'shuffle': True} │ │ │ └ (array([[-4.5776367e-04, -5.4931641e-04, -3.6621094e-04, ..., │ │ │ 4.8828125e-04, 6.4086914e-04, 7.6293945e-04], │ │ │ ... │ │ └ art.estimators.classification.tensorflow.TensorFlowV2Classifier(model=<keras.engine.sequential.Sequential object at 0x7f45102... │ └ 'fit' └ {'module': 'art.estimators.classification.tensorflow', 'qualname': 'TensorFlowV2Classifier', 'doc': '\n This c... File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/art/estimators/classification/tensorflow.py", line 959, in fit x_preprocessed, y_preprocessed = self._apply_preprocessing(x, y, fit=True) │ │ │ └ array([[0., 0., 0., ..., 0., 0., 0.], │ │ │ [0., 0., 0., ..., 0., 0., 0.], │ │ │ [0., 0., 0., ..., 0., 0., 0.], │ │ │ ...,... │ │ └ array([[-4.5776367e-04, -5.4931641e-04, -3.6621094e-04, ..., │ │ 4.8828125e-04, 6.4086914e-04, 7.6293945e-04], │ │ ... │ └ <function TensorFlowV2Estimator._apply_preprocessing at 0x7f4550441630> └ art.estimators.classification.tensorflow.TensorFlowV2Classifier(model=<keras.engine.sequential.Sequential object at 0x7f45102... File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/art/estimators/tensorflow.py", line 192, in _apply_preprocessing x, y = preprocess.forward(x, y) │ │ │ │ └ <tf.Tensor: shape=(85511, 12), dtype=float32, numpy= │ │ │ │ array([[0., 0., 0., ..., 0., 0., 0.], │ │ │ │ [0., 0., 0., ..., 0., 0., 0... │ │ │ └ <tf.Tensor: shape=(85511, 16000), dtype=float32, numpy= │ │ │ array([[-4.5776367e-04, -5.4931641e-04, -3.6621094e-04, ..., │ │ │ ... │ │ └ <function StandardisationMeanStdTensorFlow.forward at 0x7f451047d990> │ └ StandardisationMeanStdTensorFlow(mean=0.0, std=1.0, apply_fit=True, apply_predict=True) └ <tf.Tensor: shape=(85511, 16000), dtype=float32, numpy= array([[-4.5776367e-04, -5.4931641e-04, -3.6621094e-04, ..., ... File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/art/preprocessing/standardisation_mean_std/tensorflow.py", line 78, in forward x_norm = x - self._broadcastable_mean │ │ └ array(0., dtype=float32) │ └ StandardisationMeanStdTensorFlow(mean=0.0, std=1.0, apply_fit=True, apply_predict=True) └ <tf.Tensor: shape=(85511, 16000), dtype=float32, numpy= array([[-4.5776367e-04, -5.4931641e-04, -3.6621094e-04, ..., ... File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/tensorflow/python/util/traceback_utils.py", line 153, in error_handler raise e.with_traceback(filtered_tb) from None File "/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/tensorflow/python/framework/ops.py", line 7215, in raise_from_not_ok_status raise core._status_to_exception(e) from None # pylint: disable=protected-access │ │ └ _NotOkStatusException() │ └ <function _status_to_exception at 0x7f455b28add0> └ <module 'tensorflow.python.eager.core' from '/home/usr/miniconda3/envs/armory_core/lib/python3.10/site-packages/tensorflow...

tensorflow.python.framework.errors_impl.ResourceExhaustedError: {{function_node wrappedSubdevice/job:localhost/replica:0/task:0/device:GPU:0}} failed to allocate memory [Op:Sub]

Originally posted by @Uncertain-Quark in https://github.com/twosixlabs/armory/issues/1761#issuecomment-1377823605

[swsuggs]

@Uncertain-Quark Are you using "fit_generator":true as described in #1761 comment?

    ...
    "scenario": {
        "kwargs": {
            "fit_generator": true
        },
    ...

When I run using a batch size of 64, I don't see the GPU memory usage go above 6 GB.