model inputs and out put problem

[919106840533]

Description

My model has two inputs, one size is （1， 5，256，256）, the other size is （1，3，512，512）, there are eight outputs, I need to use the trt model to achieve reasoning, and get reasoning results. The relevant code I have looked up so far is for one input, and I want to know how to write code that has multiple inputs and outputs for the model.

Mycode

runtime= trt.Runtime(trt.Logger())
inference_time = []

with open('/home/AGX-exte/qry/AGX_EXP/model_oxnn/model.trt','rb') as f,runtime.deserialize_cuda_engine(f.read()) as engine:

context = engine.cteate_excuteion_context()

for i,data in enumerate(test_data):
    
    context.set_binding_shape(0, (1, 5, 256, 256))
    
    context.set_binding_shape(1, (1, 3, 512, 512))

    grd, sat, gt, gt_ori, gt_orientation, angle = (iterm for iterm in data)

    input_data1 = grd

    input_data2 = sat

    #host inpiut
    input_data1_np = input_data1.numpy()
    input_data2_np = input_data2.numpy()
    #host output
    '''
    output size is：
        (1, 262144)
        (1, 1, 512, 512)
        (1, 2, 512, 512)
        (1, 12, 16, 16)
        (1, 12, 32, 32)
        (1, 12, 64, 64)
        (1, 12, 128, 128)
        (1, 12, 256, 256)
    '''
    output1 = np.empty((1, 262144),dtype = np.float32)
    output2 = np.empty((1, 1, 512, 512),dtype = np.float32)
    output3 = np.empty((1, 2, 512, 512),dtype = np.float32)
    output4 = np.empty((1, 12, 16, 16),dtype = np.float32)
    output5 = np.empty((1, 12, 32, 32),dtype = np.float32)
    output6 = np.empty((1, 12, 64, 64),dtype = np.float32)
    output7 = np.empty((1, 12, 128, 128),dtype = np.float32)
    output8 = np.empty((1, 12, 256, 256),dtype = np.float32)
    #alloc input Mem
    input_device_buffer = [cuda.men_alloc(input_data1_np.nbytes),cuda.men_alloc(input_data2_np.nbytes)]

    cuda.memcpy_htod(input_device_buffer[0], input_data1_np)
    cuda.memcpy_htod(input_device_buffer[1], input_data2_np)
    #alloc Mem
    output_device_buffer = [cuda.mem_alloc(output1.nbytes), cuda.mem_alloc(output2.nbytes),cuda.mem_alloc(output3.nbytes),cuda.mem_alloc(output4.nbytes),
                    cuda.mem_alloc(output5.nbytes),cuda.mem_alloc(output6.nbytes),cuda.mem_alloc(output7.nbytes),cuda.mem_alloc(output8.nbytes)
                    ]

    # do inference
    begin = time.time()
    context.execute_v2(bindings=[int(i) for i in input_device_buffer] + [int(j) for j in output_device_buffer])
    end = time.timinference_timee()
    inference_time.append(begin-end)
    cuda.memcpy_dtoh(output1, output_device_buffer[0])
    cuda.memcpy_dtoh(output2, output_device_buffer[1])
    cuda.memcpy_dtoh(output3, output_device_buffer[2])
    cuda.memcpy_dtoh(output4, output_device_buffer[3])
    cuda.memcpy_dtoh(output5, output_device_buffer[4])
    cuda.memcpy_dtoh(output6, output_device_buffer[5])
    cuda.memcpy_dtoh(output7, output_device_buffer[6])
    cuda.memcpy_dtoh(output8, output_device_buffer[7])

inference_time = np.array(inference_time)
np.save('',inference_time)`

Operating System:
linux
Python Version (if applicable):
3.6.9

Relevant Files

Model link:

Steps To Reproduce

Commands or scripts:

Have you tried the latest release?:

Can this model run on other frameworks? For example run ONNX model with ONNXRuntime (polygraphy run <model.onnx> --onnxrt):

[lix19937]

from plan to infer by pytrt

import numpy as np
import tensorrt as trt
from cuda import cudart
import cv2

with open(plan_file, "rb") as f:
  serialized_engine = f.read()
  trt_logger = trt.Logger(trt.Logger.VERBOSE)  # trt.Logger.ERROR
  runtime = trt.Runtime(trt_logger)
  engine = runtime.deserialize_cuda_engine(serialized_engine)
  
  ## total num tensors
  nIO = engine.num_io_tensors
  lTensorName = [engine.get_tensor_name(i) for i in range(nIO)]
  nInput = [engine.get_tensor_mode(lTensorName[i]) for i in range(nIO)].count(trt.TensorIOMode.INPUT)
  
  context = engine.create_execution_context()
  context.set_input_shape(lTensorName[0], [1, 1, nHeight, nWidth])
  for i in range(nIO):
      print("[%2d]%s->" % (i, "Input " if i < nInput else "Output"), 
        engine.get_tensor_dtype(lTensorName[i]), engine.get_tensor_shape(lTensorName[i]), 
        context.get_tensor_shape(lTensorName[i]), lTensorName[i])
  
  bufferH = []
  data = cv2.imread(inferenceImage, cv2.IMREAD_GRAYSCALE).astype(np.float32).reshape(1, 1, nHeight, nWidth)
  bufferH.append(np.ascontiguousarray(data))
  for i in range(nInput, nIO):
      bufferH.append(np.empty(context.get_tensor_shape(lTensorName[i]), dtype=trt.nptype(engine.get_tensor_dtype(lTensorName[i]))))
  bufferD = []
  for i in range(nIO):
      bufferD.append(cudart.cudaMalloc(bufferH[i].nbytes)[1])
  
  for i in range(nInput):
      cudart.cudaMemcpy(bufferD[i], bufferH[i].ctypes.data, bufferH[i].nbytes, cudart.cudaMemcpyKind.cudaMemcpyHostToDevice)
  
  for i in range(nIO):
      context.set_tensor_address(lTensorName[i], int(bufferD[i]))
  
  ## infer 
  context.execute_async_v3(0)
  
  ## get outputs  D2H 
  for i in range(nInput, nIO):
      cudart.cudaMemcpy(bufferH[i].ctypes.data, bufferD[i], bufferH[i].nbytes, cudart.cudaMemcpyKind.cudaMemcpyDeviceToHost)
  
  ## inputs  
  for i in range(nIO):
      print(lTensorName[i])
      print(bufferH[i])
  
  ## free mem  
  for b in bufferD:
      cudart.cudaFree(b)
  
  print("Succeeded running model in TensorRT @lix19937 !")