pngmafia
commented
2 years ago Tried using
input_ids = {"input_ids": decoder_input_ids.cpu(),"past_key_values": None, "attention_mask": None, "token_type_ids": None,"position_ids": None,"head_mask":None, "inputs_embeds": None, "encoder_hidden_states": encoder_hidden_states.cpu()}
# some inference engines don't support int64 tensor as inputs, we convert all input tensors to int32 type
for k, v in input_ids.items():
if not isinstance(v, torch.Tensor):
continue
if v.dtype in [torch.long, torch.int64]:
input_ids[k] = v.type(torch.int32)
convert_to_onnx(
model_pytorch=decoder_model,
output_path="test-gpt2.onnx",
inputs_pytorch=dict(input_ids),
quantization=False,
var_output_seq=True,
output_names=["output"], # we inform ONNX export tool that the output shape will vary with the input shape
)
# model may switch to train mode for some unknown reasons, we force the eval mode.
_ = decoder_model.eval()The values are getting passed to the correct variables but theres an issue with dynamic sizes
Complete Error Stack:
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
Cell In [27], line 50
47 input_names = list(input_ids.keys())
49 with torch.no_grad():
---> 50 torch.onnx.export(
51 decoder_model, # model to optimize
52 args=tuple(input_ids.values()), # tuple of multiple inputs
53 f="test-gpt2.onnx", # output path / file object
54 opset_version=13, # the ONNX version to use, >= 13 supports channel quantized model
55 do_constant_folding=True, # simplify model (replace constant expressions)
56 input_names=input_names, # input names
57 output_names=["output"], # output names
58 dynamic_axes=dynamic_axis, # declare dynamix axis for each input / output
59 training=TrainingMode.EVAL, # always put the model in evaluation mode
60 verbose=False,
61 )
62 proto = onnx.load("test-gpt2.onnx", load_external_data=False)
63 save_onnx(proto=proto, model_path="test-gpt2.onnx")
File /usr/local/lib/python3.8/dist-packages/torch/onnx/__init__.py:350, in export(model, args, f, export_params, verbose, training, input_names, output_names, operator_export_type, opset_version, do_constant_folding, dynamic_axes, keep_initializers_as_inputs, custom_opsets, export_modules_as_functions)
74 r"""
75 Exports a model into ONNX format. If ``model`` is not a
76 :class:`torch.jit.ScriptModule` nor a :class:`torch.jit.ScriptFunction`, this runs
(...)
345 model to the file ``f`` even if this is raised.
346 """
348 from torch.onnx import utils
--> 350 return utils.export(
351 model,
352 args,
353 f,
354 export_params,
355 verbose,
356 training,
357 input_names,
358 output_names,
359 operator_export_type,
360 opset_version,
361 do_constant_folding,
362 dynamic_axes,
363 keep_initializers_as_inputs,
364 custom_opsets,
365 export_modules_as_functions,
366 )
File /usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py:163, in export(model, args, f, export_params, verbose, training, input_names, output_names, operator_export_type, opset_version, do_constant_folding, dynamic_axes, keep_initializers_as_inputs, custom_opsets, export_modules_as_functions)
145 def export(
146 model,
147 args,
(...)
160 export_modules_as_functions=False,
161 ):
--> 163 _export(
164 model,
165 args,
166 f,
167 export_params,
168 verbose,
169 training,
170 input_names,
171 output_names,
172 operator_export_type=operator_export_type,
173 opset_version=opset_version,
174 do_constant_folding=do_constant_folding,
175 dynamic_axes=dynamic_axes,
176 keep_initializers_as_inputs=keep_initializers_as_inputs,
177 custom_opsets=custom_opsets,
178 export_modules_as_functions=export_modules_as_functions,
179 )
File /usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py:1074, in _export(model, args, f, export_params, verbose, training, input_names, output_names, operator_export_type, export_type, opset_version, do_constant_folding, dynamic_axes, keep_initializers_as_inputs, fixed_batch_size, custom_opsets, add_node_names, onnx_shape_inference, export_modules_as_functions)
1071 dynamic_axes = {}
1072 _validate_dynamic_axes(dynamic_axes, model, input_names, output_names)
-> 1074 graph, params_dict, torch_out = _model_to_graph(
1075 model,
1076 args,
1077 verbose,
1078 input_names,
1079 output_names,
1080 operator_export_type,
1081 val_do_constant_folding,
1082 fixed_batch_size=fixed_batch_size,
1083 training=training,
1084 dynamic_axes=dynamic_axes,
1085 )
1087 # TODO: Don't allocate a in-memory string for the protobuf
1088 defer_weight_export = (
1089 export_type is not torch.onnx.ExportTypes.PROTOBUF_FILE
1090 )
File /usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py:731, in _model_to_graph(model, args, verbose, input_names, output_names, operator_export_type, do_constant_folding, _disable_torch_constant_prop, fixed_batch_size, training, dynamic_axes)
728 params_dict = _get_named_param_dict(graph, params)
730 try:
--> 731 graph = _optimize_graph(
732 graph,
733 operator_export_type,
734 _disable_torch_constant_prop=_disable_torch_constant_prop,
735 fixed_batch_size=fixed_batch_size,
736 params_dict=params_dict,
737 dynamic_axes=dynamic_axes,
738 input_names=input_names,
739 module=module,
740 )
741 except Exception as e:
742 torch.onnx.log("Torch IR graph at exception: ", graph)
File /usr/local/lib/python3.8/dist-packages/torch/onnx/utils.py:306, in _optimize_graph(graph, operator_export_type, _disable_torch_constant_prop, fixed_batch_size, params_dict, dynamic_axes, input_names, module)
304 input_names = [] if input_names is None else input_names
305 dynamic_axes = {} if dynamic_axes is None else dynamic_axes
--> 306 _C._jit_pass_onnx_set_dynamic_input_shape(graph, dynamic_axes, input_names)
307 _C._jit_pass_onnx_lint(graph)
308 graph = _C._jit_pass_onnx(graph, operator_export_type)
RuntimeError: Dynamic shape axis should be no more than the shape dimension for sequence
ayoub-louati
For the onnxruntime I plan to send encoder_hidden_states as part of inputs as well
but it seems like the inputs are not carried onto the forward pass in dictionary and that encoder_hidden_states is being set as past_key_values which should have been None(by default), hence the issue
Complete error stack:
Am I missing something? Please help @pommedeterresautee