cuMemcpyHtoDAsync failed: unspecified launch failure

[skybocai]

In the Flask environment, the Five models in one class is passed into the multithreading as a global variable.An error is reported when running for a long time。
log.info
input memcpy_htod_async: cuMemcpyHtoDAsync failed: unspecified launch failure
[TRT] [E] 1: [context.cpp::nvinfer1::rt::CommonContext::setStream::121] Error Code 1: Cudnn (CUDNN_STATUS_MAPPING_ERROR)
memcpy_dtoh_async to cpu: cuMemcpyDtoHAsync failed: unspecified launch failure
cuStreamSynchronize :cuStreamSynchronize failed: unspecified launch failure
the Trtmodel.py
``
import numpy as np
import tensorrt as trt
import os
import pycuda.driver as cuda
import time
import logging as log
import pycuda.autoinit
TRT_LOGGER = trt.Logger(trt.Logger.WARNING)
cuda.init()
trt.init_libnvinfer_plugins(TRT_LOGGER, '')
import threading

Simple helper data class that's a little nicer to use than a 2-tuple.

class HostDeviceMem(object):

def __init__(self, host_mem, device_mem):
    self.host = host_mem
    self.device = device_mem

def __str__(self):
    return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device)

def __repr__(self):
    return self.__str__()

class TrtModel(object):

def __init__(self, model=None):
    self.engine_file = model
    self.engine = None
    self.inputs = None
    self.outputs = None
    self.bindings = None
    self.stream = None
    self.context = None
    self.input_shapes = None
    self.out_shapes = None
    self.max_batch_size = 1
    self.ctx = cuda.Device(0).make_context()

def build(self):
    assert os.path.exists(self.engine_file), "Engine file doesn't exist"
    with open(self.engine_file, 'rb') as f, trt.Runtime(TRT_LOGGER) as runtime:
        self.engine = runtime.deserialize_cuda_engine(f.read())
    self.context = self.engine.create_execution_context()
    self.inputs, self.outputs, self.bindings, self.stream, self.input_shapes, self.out_shapes, \
    self.out_names, self.input_name, self.max_batch_size = self.allocate_buffers(
        self.context.engine)  # 新增input_name输入名称输出
    assert self.inputs is not None
    self.start = cuda.Event()
    self.end = cuda.Event()

def run(self, input, deflatten: bool = True, as_dict=False):
    # self.ctx.push()
    # lazy load implementation
    if self.engine is None:
        self.build()
    input = np.asarray(input)
    batch_size = input.shape[0]
    allocate_place = np.prod(input.shape)
    self.inputs[0].host[:allocate_place] = input.flatten(order='C').astype(np.float32)
    self.context.set_binding_shape(0, input.shape)
    self.ctx.push()
    trt_outputs = self.do_inference(
        self.context, bindings=self.bindings,
        inputs=self.inputs, outputs=self.outputs, stream=self.stream)
    self.ctx.pop()
    # del self.ctx
    #Reshape TRT outputs to original shape instead of flattened array
    if deflatten:
        trt_outputs = [output.reshape(shape) for output, shape in zip(trt_outputs, self.out_shapes)]
    if as_dict:
        return {name: trt_outputs[i] for i, name in enumerate(self.out_names)}

    return [trt_output[:batch_size] for trt_output in trt_outputs]

def do_inference(self, context, bindings, inputs, outputs, stream):
    # self.ctx.push()
    # Transfer input data to the GPU.
    try:
        # self.start.record(stream)
        try:
            [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs]
        except Exception as e:
            print("input memcpy_htod_async", e)
        try:
            # Run inference.
            context.execute_async_v2(bindings=bindings, stream_handle=stream.handle)
        except Exception as e:
            print("执行推理报错", e)
        try:
            # Transfer predictions back from the GPU.
            [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs]
        except Exception as e:
            print("memcpy_dtoh_async to cpu", e)
        # Synchronize the stream
        # self.end.record(stream)
        try:
            stream.synchronize()
        except Exception as e:
            print("synchronize", e)
        # self.end.record(stream)
        # Return only the host outputs.
        return [out.host for out in outputs]
    except Exception as e:
        print("inputs======", inputs[0].host.shape, threading.currentThread().getName())
        print("do_inference", e)

def allocate_buffers(self, engine):
    inputs = []
    outputs = []
    bindings = []
    stream = cuda.Stream()
    out_shapes = []
    input_shapes = []
    out_names = []
    input_names = []
    max_batch_size = engine.get_profile_shape(0, 0)[2][0]
    for binding in engine:
        binding_shape = engine.get_binding_shape(binding)
        # Fix -1 dimension for proper memory allocation for batch_size > 1
        if binding_shape[0] == -1:
            binding_shape = (1,) + binding_shape[1:]
        size = trt.volume(binding_shape) * max_batch_size
        dtype = trt.nptype(engine.get_binding_dtype(binding))
        # Allocate host and device buffers
        host_mem = cuda.pagelocked_empty(size, dtype)
        device_mem = cuda.mem_alloc(host_mem.nbytes)
        # Append the device buffer to device bindings.
        bindings.append(int(device_mem))
        # Append to the appropriate list.
        if engine.binding_is_input(binding):
            inputs.append(HostDeviceMem(host_mem, device_mem))
            input_shapes.append(engine.get_binding_shape(binding))
            input_names.append(binding)  # 2022/9/20 增加输入名称参数
        else:
            outputs.append(HostDeviceMem(host_mem, device_mem))
            # Collect original output shapes and names from engine
            out_shapes.append(engine.get_binding_shape(binding))
            out_names.append(binding)
    return inputs, outputs, bindings, stream, input_shapes, out_shapes, out_names, input_names, max_batch_size

def destory(self):
    self.ctx.pop()

''
the SCRFD.py
''
from .TrtModel import *
import numpy as np
import os.path as osp
import cv2
import cupy as cp

class SCRFD:

def __init__(self, model_file=None, session=None, outputs=None):
    self.model_file = model_file
    self.session = session
    self.taskname = 'detection'
    self.batched = False
    self.outputs = outputs
    if self.session is None:
        assert self.model_file is not None
        assert osp.exists(self.model_file)
        self.session = TrtModel(self.model_file)  # 2022/9/20 将 onnxruntime推理修改为trt推理
    self.center_cache = {}
    self.nms_thresh = 0.4
    self._init_vars()

def _init_vars(self):
    self.session.build()
    input_shape = self.session.input_shapes[0]  # (1,3,640,640)  
    if isinstance(input_shape[2], str):
        self.input_size = None
    else:
        self.input_size = tuple(input_shape[2:4][::-1])  # (640,640)
    input_name = self.session.input_name 
    outputs_shape = self.session.out_shapes 
    if len(outputs_shape[0]) == 3:
        self.batched = True
    output_names = self.session.out_names 
    self.input_name = input_name
    self.output_names = output_names
    self.use_kps = False
    self._num_anchors = 1
    if len(output_names) == 6:
        self.fmc = 3
        self._feat_stride_fpn = [8, 16, 32]
        self._num_anchors = 2
    elif len(output_names) == 9:
        self.fmc = 3
        self._feat_stride_fpn = [8, 16, 32]
        self._num_anchors = 2
        self.use_kps = True
    elif len(output_names) == 10:
        self.fmc = 5
        self._feat_stride_fpn = [8, 16, 32, 64, 128]
        self._num_anchors = 1
    elif len(output_names) == 15:
        self.fmc = 5
        self._feat_stride_fpn = [8, 16, 32, 64, 128]
        self._num_anchors = 1
        self.use_kps = True

def prepare(self, **kwargs):
    log.info("warming up Scrfd TensorRT engine ....")
    nms_thresh = kwargs.get('nms_thresh', None)
    if nms_thresh is not None:
        self.nms_thresh = nms_thresh
    input_size = kwargs.get('input_size', None)
    if input_size is not None:
        if self.input_size is not None:
            print('warning: det_size is already set in scrfd model, ignore')
        else:
            self.input_size = input_size

def distance2bbox(self, points, distance, max_shape=None):
    """Decode distance prediction to bounding box.

    Args:
        points (Tensor): Shape (n, 2), [x, y].
        distance (Tensor): Distance from the given point to 4
            boundaries (left, top, right, bottom).
        max_shape (tuple): Shape of the image.

    Returns:
        Tensor: Decoded bboxes.
    """
    return np.stack([x1, y1, x2, y2], axis=-1)

def distance2kps(self, points, distance, max_shape=None):
    """Decode distance prediction to bounding box.

    Args:
        points (Tensor): Shape (n, 2), [x, y].
        distance (Tensor): Distance from the given point to 4
            boundaries (left, top, right, bottom).
        max_shape (tuple): Shape of the image.

    Returns:
        Tensor: Decoded bboxes.
    """
    return np.stack(preds, axis=-1)

def forward(self, img, thresh):
    try:
        scores_list = []
        bboxes_list = []
        kpss_list = []
        input_size = tuple(img.shape[0:2][::-1])
        blob = cv2.dnn.blobFromImage(img, 1.0 / 128, input_size, (127.5, 127.5, 127.5), swapRB=True)
        net_outs = self.session.run(blob, deflatten=True, as_dict=True)
        net_outs = [net_outs[e] for e in self.outputs]
        input_height = blob.shape[2]
        input_width = blob.shape[3]
        fmc = self.fmc  # fmc=3
        try:
            for idx, stride in enumerate(self._feat_stride_fpn):  # _feat_strie_fpn 8,16,32
                # If model support batch dim, take first output
                if self.batched:
                    scores = net_outs[idx][0]
                    bbox_preds = net_outs[idx + fmc][0]
                    bbox_preds = bbox_preds * stride
                    if self.use_kps:
                        kps_preds = net_outs[idx + fmc * 2][0] * stride
                # If model doesn't support batching take output as is
                else:
                    scores = net_outs[idx]
                    bbox_preds = net_outs[idx + fmc]
                    bbox_preds = bbox_preds * stride
                    if self.use_kps:
                        kps_preds = net_outs[idx + fmc * 2] * stride

                height = input_height // stride
                width = input_width // stride
                K = height * width
                key = (height, width, stride)
                if key in self.center_cache:
                    anchor_centers = self.center_cache[key]
                else:
                    # solution-3:
                    anchor_centers = np.stack(np.mgrid[:height, :width][::-1], axis=-1).astype(np.float32)
                    # print(anchor_centers.shape)

                    anchor_centers = (anchor_centers * stride).reshape((-1, 2))
                    if self._num_anchors > 1:
                        anchor_centers = np.stack([anchor_centers] * self._num_anchors, axis=1).reshape((-1, 2))
                    if len(self.center_cache) < 100:
                        self.center_cache[key] = anchor_centers

                pos_inds = np.where(scores >= thresh)[0]
                bboxes = self.distance2bbox(anchor_centers, bbox_preds)
                pos_scores = scores[pos_inds]
                pos_bboxes = bboxes[pos_inds]
                scores_list.append(pos_scores)
                bboxes_list.append(pos_bboxes)
                if self.use_kps:
                    kpss = self.distance2kps(anchor_centers, kps_preds)
                    # kpss = kps_preds
                    kpss = kpss.reshape((kpss.shape[0], -1, 2))
                    pos_kpss = kpss[pos_inds]
                    kpss_list.append(pos_kpss)
        except Exception as e:
            print("人脸检测失败", e)
        return scores_list, bboxes_list, kpss_list
    except Exception as e:
        print("人脸检测推理失败", e)

def detect(self, img, thresh=0.5, input_size=None, max_num=0, metric='default'):
    ''''''
    scores_list, bboxes_list, kpss_list = self.forward(det_img, thresh)
    ''''''''
    return det, kpss

def nms(self, dets):
   '''''''''''
    return keep

''

Environment (please complete the following information):

• OS: [ windows10]
• CUDA version: [10.2]
• CUDNN version: [8.0.3]
• CUDA driver version: [442.53]
• PyCUDA version: [ 2021.1]
• Python version: [3.7]