cuda_copy_md.c:489 UCX WARN cuPointerSetAttribute error with CUDA VMM API

[MinassZhang]

Describe the bug

Server uses cuda virtual memory management API (cuMemAddressReserve, cuMemCreate..) to create memory region and issues error cuda_copy_md.c:489 UCX WARN cuPointerSetAttribute(0x7f85c0000000, SYNC_MEMOPS) error: operation not supported.

Can RNDV protocl support the memory space allocated by CUDA VMM API?

Steps to Reproduce

• UCX version

  # Library version: 1.17.0
  # Library path: /home/xxx/ucx-1.17.0/install/lib/libucs.so.0
  # API headers version: 1.17.0
  # Git branch '', revision 
  # Configured with: --disable-logging --disable-debug --disable-assertions --disable-params-check --enable-gtest --enable-examples --enable-optimizations --disable-logging --disable-debug --disable-assertions --disable-params-check --without-xpmem --without-java --with-cuda=/usr/local/cuda-11.7 --with-gdrcopy --prefix=/home/xxx/ucx-1.17.0/install

Setup and versions

• OS version: Ubuntu 20.04.6 LTS x86_64
• GPU type: NVIDIA Tesla V100-PCIE-32GB
• Cuda version: CUDA 11.7
• Cuda driver version: 535.129.03

Additional information

• VMM API to create memory region

  cudaError_t vmm_alloc(void **ptr, size_t size, int currentDevice, CUdeviceptr start_add) {
  CUmemAllocationProp prop = {};
  memset(&prop,0,sizeof(prop));
  prop.type          = CU_MEM_ALLOCATION_TYPE_PINNED;
  prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
  prop.location.id   = currentDevice;
  prop.allocFlags.gpuDirectRDMACapable = 1;
  
  size_t granularity = 0;
  CUresult result;
   // Calculates either the minimal or recommended granularity.
  result = cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM);
  if (result != CUDA_SUCCESS) {
      printf("cudaErrorMemoryAllocation error %d\n",result);
      return cudaErrorMemoryAllocation;
  }
  
  size = ((size - 1) / granularity + 1) * granularity;
  
  CUdeviceptr dptr;
  // Allocate an address range reservation.
  result = cuMemAddressReserve(&dptr, size, 0, start_add + granularity, 0);
  if (result != CUDA_SUCCESS) {
      printf("cuMemAddressReserve error %d\n",result);
      return cudaErrorMemoryAllocation;
  }
  printf("dptr = %p\n",dptr);
  
  CUmemGenericAllocationHandle allocationHandle;
  // Create a CUDA memory handle representing a memory allocation of a given size described by the given properties.
  result = cuMemCreate(&allocationHandle, size, &prop, 0);
  if (result != CUDA_SUCCESS) {
      printf("cuMemCreate error %d\n",result);
      return cudaErrorMemoryAllocation;
  }
  
  // Maps an allocation handle to a reserved virtual address range.
  // cuMemMap can only create mappings on VA range reservations that are not currently mapped.
  result = cuMemMap(dptr, size, 0, allocationHandle, 0);
  if (result != CUDA_SUCCESS) {
      printf("cuMemMap error %d\n",result);
      return cudaErrorMemoryAllocation;
  }
  
  CUmemAccessDesc accessDescriptor;
  accessDescriptor.location.id   = prop.location.id;
  accessDescriptor.location.type = prop.location.type;
  accessDescriptor.flags         = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;
  // Set the access flags for each location specified in desc for the given virtual address range.
  // Any new mapping to this virtual address will need to have access granted through cuMemSetAccess, as all mappings start with no accessibility setup.
  result = cuMemSetAccess(dptr, size, &accessDescriptor, 1);
  if (result != CUDA_SUCCESS) {
      printf("cuMemSetAccess error %d\n",result);
      return cudaErrorMemoryAllocation;
  }
  *ptr = (void *)dptr;
  
  return cudaSuccess;
  }

[yosefe]

VMM is currently not supported. It should be fixed by #9867 cc @Akshay-Venkatesh