Blackbox attack of a basic binary TensorFlow classifier with tabular data

[amrasekh]

I am implementing a blackbox attack against the basic binary TensorFlow classifier with tabular data below. Here is the notebook: credit_default.ipynb.zip

It errors out due to a size-incompatibility during the training of the attack object. attackobj.train_attack(). It appears to be related to how shape is defined in ml_privacy_meter.utils.attack_data.AttackData, but I am not able to see how it can be set correctly. Thank you for the help in advance (since this type of classifier is very common, adding it to the library demos may also add value).

TensorFlow version: 2.1.4

`--------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last)

in ----> 1 attackobj.train_attack() ~/username/git/ml_privacy_meter/ml_privacy_meter/attack/meminf.py in train_attack(self) 443 model = self.target_train_model 444 --> 445 pred = model(nm_features) 446 acc = accuracy_score(nm_labels, np.argmax(pred, axis=1)) 447 print('Target model test accuracy', acc) ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs) 820 with base_layer_utils.autocast_context_manager( 821 self._compute_dtype): --> 822 outputs = self.call(cast_inputs, *args, **kwargs) 823 self._handle_activity_regularization(inputs, outputs) 824 self._set_mask_metadata(inputs, outputs, input_masks) ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/sequential.py in call(self, inputs, training, mask) 265 if not self.built: 266 self._init_graph_network(self.inputs, self.outputs, name=self.name) --> 267 return super(Sequential, self).call(inputs, training=training, mask=mask) 268 269 outputs = inputs # handle the corner case where self.layers is empty ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in call(self, inputs, training, mask) 715 return self._run_internal_graph( 716 inputs, training=training, mask=mask, --> 717 convert_kwargs_to_constants=base_layer_utils.call_context().saving) 718 719 def compute_output_shape(self, input_shape): ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in _run_internal_graph(self, inputs, training, mask, convert_kwargs_to_constants) 889 890 # Compute outputs. --> 891 output_tensors = layer(computed_tensors, **kwargs) 892 893 # Update tensor_dict. ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs) 820 with base_layer_utils.autocast_context_manager( 821 self._compute_dtype): --> 822 outputs = self.call(cast_inputs, *args, **kwargs) 823 self._handle_activity_regularization(inputs, outputs) 824 self._set_mask_metadata(inputs, outputs, input_masks) ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/keras/layers/core.py in call(self, inputs) 1129 if rank > 2: 1130 # Broadcasting is required for the inputs. -> 1131 outputs = standard_ops.tensordot(inputs, self.kernel, [[rank - 1], [0]]) 1132 # Reshape the output back to the original ndim of the input. 1133 if not context.executing_eagerly(): ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/ops/math_ops.py in tensordot(a, b, axes, name) 4104 b_reshape, b_free_dims, b_free_dims_static = _tensordot_reshape( 4105 b, b_axes, True) -> 4106 ab_matmul = matmul(a_reshape, b_reshape) 4107 if isinstance(a_free_dims, list) and isinstance(b_free_dims, list): 4108 return array_ops.reshape(ab_matmul, a_free_dims + b_free_dims, name=name) ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/util/dispatch.py in wrapper(*args, **kwargs) 178 """Call target, and fall back on dispatchers if there is a TypeError.""" 179 try: --> 180 return target(*args, **kwargs) 181 except (TypeError, ValueError): 182 # Note: convert_to_eager_tensor currently raises a ValueError, not a ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/ops/math_ops.py in matmul(a, b, transpose_a, transpose_b, adjoint_a, adjoint_b, a_is_sparse, b_is_sparse, name) 2796 else: 2797 return gen_math_ops.mat_mul( -> 2798 a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name) 2799 2800 ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/ops/gen_math_ops.py in mat_mul(a, b, transpose_a, transpose_b, name) 5614 pass # Add nodes to the TensorFlow graph. 5615 except _core._NotOkStatusException as e: -> 5616 _ops.raise_from_not_ok_status(e, name) 5617 # Add nodes to the TensorFlow graph. 5618 if transpose_a is None: ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py in raise_from_not_ok_status(e, name) 6604 message = e.message + (" name: " + name if name is not None else "") 6605 # pylint: disable=protected-access -> 6606 six.raise_from(core._status_to_exception(e.code, message), None) 6607 # pylint: enable=protected-access 6608 ~/username/git/credoai_research/pythonenv3/lib/python3.6/site-packages/six.py in raise_from(value, from_value) InvalidArgumentError: Matrix size-incompatible: In[0]: [44220,1], In[1]: [22,1] [Op:MatMul] name: sequential/layer1/Tensordot/MatMul/ `

[changhongyan123]

@amrasekh , thanks. We have reconstructed the code. The framework is now agnostic to the framework used by the target model. If you still have problems, can you try to use new code to see if you still encounter the same issue?