Error when convert dynamic input onnx to rt

[phamkhactu]

Thanks for great repo.

I want to convert model onnx with dynamic input x=[1,3,-1,-1] to RT(my tensorrt version is 8.5.3.1), but when I run script to convert I get error:

[network.cpp::validate::3089] Error Code 4: Internal Error (x: dynamic input is missing dimensions in profile 0.)

I think this error comes from:

        network.get_input(0).shape = shape # [1,3,-1,-1]

Here my code:

import tensorrt as trt
from onnx import ModelProto

TRT_LOGGER = trt.Logger(trt.Logger.WARNING)
def build_engine(onnx_path, shape = [1,224,224,3]):

    """
    This is the function to create the TensorRT engine
    Args:
        onnx_path : Path to onnx_file. 
        shape : Shape of the input of the ONNX file. 
    """
    with trt.Builder(TRT_LOGGER) as builder, builder.create_network(1) as network, builder.create_builder_config() as config, trt.OnnxParser(network, TRT_LOGGER) as parser:
        config.set_flag(trt.BuilderFlag.FP16)
        builder.max_batch_size = 1

        profile = builder.create_optimization_profile()
        config.max_workspace_size = (1 << 20)
        config.add_optimization_profile(profile)

        with open(onnx_path, 'rb') as model:
            parser.parse(model.read())

        network.get_input(0).shape = shape # [1,3,-1,-1]
        # engine = builder.build_engine(network, config)
        serialized_engine = builder.build_serialized_network(network, config)
        return serialized_engine

def save_engine(serialized_engine, file_name):
    # buf = serialized_engine.serialize()
    with open(file_name, 'wb') as f:
       f.write(serialized_engine)

def load_engine(trt_runtime, plan_path):
   with open(plan_path, 'rb') as f:
       engine_data = f.read()
   engine = trt_runtime.deserialize_cuda_engine(engine_data)
   return engine

def convert_to_rt():

    engine_name = "pp_ocr_det_model_f16.plan"
    onnx_path = "pp_ocr_det_model.onnx" 
    batch_size = 1 

    model = ModelProto()
    with open(onnx_path, "rb") as f:
        model.ParseFromString(f.read())

    d0 = model.graph.input[0].type.tensor_type.shape.dim[1].dim_value
    d1 = model.graph.input[0].type.tensor_type.shape.dim[2].dim_value
    d2 = model.graph.input[0].type.tensor_type.shape.dim[3].dim_value
    shape = [batch_size , d0, d1 ,d2]
    print("shape input model:", shape) #[1, 3, -1, -1]
    engine = build_engine(onnx_path, shape= shape)
    save_engine(engine, engine_name) 

if __name__ == '__main__':
    convert_to_rt()

Thank you very much for your help.

[zerollzeng]

Your code is wrong, you can refer to https://github.com/NVIDIA/TensorRT/blob/1d6bf36034bb07fc88d924ebf5d34d358298e545/samples/python/efficientdet/build_engine.py#L168 or any other TRT samples in the repo.

[phamkhactu]

@zerollzeng Thank you, but when I change code as you suggest:

def build_detec_engine(onnx_path, using_half=True, dynamic_input=True, workspace_size=2, 
                min_shape=(1,3,256,256), opt_shape=(1,3,640,960), max_shape=(1,3,1280,1280)):
    trt.init_libnvinfer_plugins(None, '')
    with trt.Builder(TRT_LOGGER) as builder, builder.create_network(1) as network, trt.OnnxParser(network, TRT_LOGGER) as parser:
        builder.max_batch_size = 1 # always 1 for explicit batch
        config = builder.create_builder_config()
        config.max_workspace_size = GiB(int(workspace_size))
        if using_half:
            config.set_flag(trt.BuilderFlag.FP16)
        # Load the Onnx model and parse it in order to populate the TensorRT network.
        with open(onnx_path, 'rb') as model:
            if not parser.parse(model.read()):
                print ('ERROR: Failed to parse the ONNX file.')
                for error in range(parser.num_errors):
                    print (parser.get_error(error))
                return None

        if dynamic_input:
            profile = builder.create_optimization_profile();
            profile.set_shape("x", min_shape, opt_shape, max_shape) 
            config.add_optimization_profile(profile)
        return builder.build_serialized_network(network, config) 

I get error:

 with trt.Builder(TRT_LOGGER) as builder, builder.create_network(1) as network, trt.OnnxParser(network, TRT_LOGGER) as parser:
TypeError: pybind11::init(): factory function returned nullptr

Tensorrt version is: 8.5.3.1 Can you give me some advance to solve it? Thank you.

[phamkhactu]

I have found that this issue comes from version conflict, some layer can not be converted to tensorrt layer.
Thank you very much.

[yanxinlei]

What version of tensorrt did you use to solve the problem