windows PCI (or least CUDA) locality

Based on http://www.open-mpi.org/community/lists/hwloc-users/2013/11/0926.php

Also useful: http://blogs.technet.com/b/winserverperformance/archive/2008/09/13/getting-system-topology-information-on-windows.aspx

This code should map a CUdevice to a numa node (by enumerating all PCI devices). I have not compiled the the code in this form or tested it as is, but the calls should work fine for mapping any cuda device to the OS enumeration wrt to PCI device location:

{{{ long GetNumaNode(CUdevice dev) { BOOL ret; DWORD lastError; // get the cuda device string char cuDevString[CUDA_DEV_STRING_LEN]; unsigned long cudaBus; unsigned long cudaSubdevice; unsigned long cudaFunction; CUresult status = cuDeviceGetPCIBusId(cuDevString, CUDA_DEV_STRING_LEN, dev); assert(CUDA_SUCCESS == status); if (CUDA_SUCCESS != status) { return 0; } char tmp; char tmp2; char del[] = ":."; // remove domain tmp = strtok_s(cuDevString, del, &tmp2); // get bus tmp = strtok_s(NULL, del, &tmp2); sscanf_s(tmp, "%x", &cudaBus); // get subdevice tmp = strtok_s(NULL, del, &tmp2); sscanf_s(tmp, "%x", &cudaSubdevice); // get function tmp = strtok_s(NULL, del, &tmp2); sscanf_s(tmp, "%x", &cudaFunction); // Use NULL as the first parameter as we need to look at non display devices too HDEVINFO hNvDevInfo = SetupDiGetClassDevs(NULL, NULL, NULL, DIGCF_PRESENT | DIGCF_ALLCLASSES); if(hNvDevInfo == INVALID_HANDLE_VALUE) { assert(!"INVALID_HANDLE_VALUE"); return 0; } // Find the deviceInfoData for each GPU DWORD deviceIndex; for (deviceIndex = 0; ; deviceIndex++) { SP_DEVINFO_DATA deviceInfoData; unsigned long bus; unsigned long subdevice; unsigned long function; deviceInfoData.cbSize = sizeof(SP_DEVINFO_DATA); ret = SetupDiEnumDeviceInfo(hNvDevInfo, deviceIndex, &deviceInfoData); if (!ret) { // MSDN says: // call SetupDiEnumDeviceInfo until there are no more values (the function fails and a call // to GetLastError returns ERROR_NO_MORE_ITEMS). lastError = GetLastError(); assert(lastError == ERROR_NO_MORE_ITEMS); break; }

    char locinfo[256];
    ret = SetupDiGetDeviceRegistryPropertyA(hNvDevInfo, &deviceInfoData, SPDRP_LOCATION_INFORMATION, NULL,
        (PBYTE)locinfo, sizeof(locinfo), NULL);
    if (!ret)
    {
        lastError = GetLastError();
    }
    bool dataSet = false;
    if (strncmp(locinfo, "PCI", 3) == 0) {
        char *busString = strstr(locinfo, "bus");
        if (busString) {
            busString += 3;
            char *deviceString = strstr(locinfo, ",");
            if (deviceString) {
                deviceString[0] = 0;
                bus = atoi(busString);
                deviceString++;
                deviceString = strstr(deviceString, "device");
                if (deviceString) {
                    deviceString+=6;
                    char *functionStr = strstr(deviceString, ",");
                    if (functionStr) {
                        functionStr[0] = 0;
                        subdevice = atoi(deviceString);
                        functionStr++;
                        functionStr = strstr(functionStr, "function");
                        if (functionStr) {
                            functionStr+=8;
                            function = atoi(functionStr);
                            dataSet = true;
                        }
                    }
                }
            }
        }
    }
    if (dataSet &&
        (bus == cudaBus) &&
        (subdevice == cudaSubdevice) &&
        (function == cudaFunction))
    {
        ret = SetupDiGetDeviceRegistryPropertyA(hNvDevInfo, &deviceInfoData, SPDRP_HARDWAREID, NULL,
            (PBYTE)locinfo, sizeof(locinfo), NULL);
        printf("locinfo %s\n", locinfo);
        int data[20];
        data[0] = 0;
        DEVPROPTYPE type;
        DEVPROPKEY key = DEVPKEY_Numa_Proximity_Domain;
        lastError = 0;
        ret =  SetupDiGetDeviceProperty(hNvDevInfo, &deviceInfoData,&key , &type, (PBYTE)&data[0], 20*sizeof(int), NULL,0);
        if (!ret)
        {
            lastError = GetLastError();
        }
        printf("DEVPKEY_Numa_Proximity_Domain %d err %d\n", data[0], lastError);
        key = DEVPKEY_Device_Numa_Node;
        lastError = 0;
        ret =  SetupDiGetDeviceProperty(hNvDevInfo, &deviceInfoData,&key , &type, (PBYTE)&data[0], 20*sizeof(int), NULL,0);
        if (!ret)
        {
            lastError = GetLastError();
        }
        printf("DEVPKEY_Device_Numa_Node %d err %d\n", data[0], lastError);
        return data[0];
    }
}
return -1;

}

open-mpi / hwloc

windows PCI (or least CUDA) locality #108