confusion about the cuda stability.

[xjock]

import os
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
import cv2
import torch
import torchvision
from vpf import PyNvCodec as nvc
from vpf import PytorchNvCodec as pnvc
print("PyTorch Version {}".format(torch.__version__))
print("Cuda Version {}".format(torch.version.cuda))
print("CUDNN Version {}".format(torch.backends.cudnn.version()))

device = torch.device('cuda:0')
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(
    pretrained=True)
model.float()
model.to(device)
model.eval()

gpuID = 0
encFilePath = '/home/lhcz/sdb/data/bazhou_videos/videos/videos/20210103/6912003160731176192.mp4'
nvDec = nvc.PyNvDecoder(encFilePath, gpuID)
to_rgb = nvc.PySurfaceConverter(nvDec.Width(), nvDec.Height(), nvc.PixelFormat.NV12, nvc.PixelFormat.RGB, gpuID)
to_planar = nvc.PySurfaceConverter(nvDec.Width(), nvDec.Height(), nvc.PixelFormat.RGB, nvc.PixelFormat.RGB_PLANAR,
                                   gpuID)

cnt_frame = 0
while True:
    rawSurface = nvDec.DecodeSingleSurface()
    if (rawSurface.Empty()):
        break
    rgb_byte = to_rgb.Execute(rawSurface)
    rgb_planar = to_planar.Execute(rgb_byte)
    surfPlane = rgb_planar.PlanePtr()
    surface_tensor = pnvc.makefromDevicePtrUint8(surfPlane.GpuMem(), surfPlane.Width(), surfPlane.Height(),
                                                 surfPlane.Pitch(), surfPlane.ElemSize())

    print(surface_tensor)

    print(cnt_frame)
    cnt_frame += 1
    # surface_tensor = surface_tensor.type(dtype=torch.cuda.FloatTensor)
    # do inference here
    img_tensor = (surface_tensor.resize_(1, 3, nvDec.Height(), nvDec.Width()) / 255.)#.float().cuda()
    out = model(img_tensor)
    boxes = out[0]['boxes']
    labels = out[0]['labels']
    scores = out[0]['scores']
    for idx in range(boxes.shape[0]):
        if scores[idx] >= 0.8:
            x1, y1, x2, y2 = boxes[idx][0], boxes[idx][1], boxes[idx][2], boxes[idx][3]
            print(int(x1), int(y1), int(x2), int(y2), float(scores[idx]), str(labels[idx].item()))

The above code runs well with the line print(surface_tensor)

However, if I comment the line print(surface_tensor), the above code return the following error

/root/anaconda3/envs/insightface/bin/python /home/lhcz/sdb/dev/vpf_test3.py PyTorch Version 1.6.0 Cuda Version 10.2 CUDNN Version 7605 Decoding on GPU 0 /root/anaconda3/envs/insightface/lib/python3.6/site-packages/torchvision/ops/boxes.py:101: UserWarning: This overload of nonzero is deprecated: nonzero() Consider using one of the following signatures instead: nonzero(, bool as_tuple) (Triggered internally at /pytorch/torch/csrc/utils/python_arg_parser.cpp:766.) keep = keep.nonzero().squeeze(1) 257 436 438 913 0.997122585773468 1 330 599 636 1080 0.9901776909828186 1 469 714 666 1069 0.9539728760719299 62 169 834 385 1027 0.8340146541595459 62 436 861 557 1043 0.8114858865737915 62 Traceback (most recent call last): File "/home/lhcz/sdb/dev/ethnic_face_cls/vpf_test3.py", line 47, in out = model(img_tensor) File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl result = self.forward(input, kwargs) File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torchvision/models/detection/generalized_rcnn.py", line 95, in forward features = self.backbone(images.tensors) File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl result = self.forward(*input, *kwargs) File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torchvision/models/detection/backbone_utils.py", line 39, in forward x = self.body(x) File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl result = self.forward(input, kwargs) File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torchvision/models/_utils.py", line 63, in forward x = module(x) File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/module.py", line 722, in _call_impl result = self.forward(*input, **kwargs) File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 419, in forward return self._conv_forward(input, self.weight) File "/root/anaconda3/envs/insightface/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 416, in _conv_forward self.padding, self.dilation, self.groups) RuntimeError: cuDNN error: CUDNN_STATUS_MAPPING_ERROR

Process finished with exit code 1

[courtfu]

I met the same error.