search

pytorch / torcheval

A library that contains a rich collection of performant PyTorch model metrics, a simple interface to create new metrics, a toolkit to facilitate metric computation in distributed training and tools for PyTorch model evaluations.
https://pytorch.org/torcheval
Other
211 stars 46 forks source link

Torcheval pointing to wrong directory for nvrtc-builtins64_121.dll file. #194

Open 123tsnedd opened 4 months ago

123tsnedd commented 4 months ago

šŸ› Describe the bug

Attempting to use Torcheval r2_score, but isn't finding the file in Conda environment. I had to copy the file from anaconda3/envs/{myenv}/bin to the local GPU computing toolkit/CUDA/V12.4/bin for it to work. Local GPU toolkit was downloaded from NVIDIA

import torch
from torcheval.metrics.functional import r2_score
# code training lope
# prediction == <class 'torch.Tensor'>
#target == <class 'torch.Tensor'>
r2 = r2_score(prediction, target)
# remaining code

traceback

            "outputType": "error",
            "originalError": {
                "output_type": "error",
                "ename": "RuntimeError",
                "evalue": "nvrtc: error: failed to open nvrtc-builtins64_121.dll.\n  Make sure that nvrtc-builtins64_121.dll is installed correctly.\nnvrtc compilation failed: \n\n#define NAN __int_as_float(0x7fffffff)\n#define POS_INFINITY __int_as_float(0x7f800000)\n#define NEG_INFINITY __int_as_float(0xff800000)\n\n\ntemplate<typename T>\n__device__ T maximum(T a, T b) {\n  return isnan(a) ? a : (a > b ? a : b);\n}\n\ntemplate<typename T>\n__device__ T minimum(T a, T b) {\n  return isnan(a) ? a : (a < b ? a : b);\n}\n\nextern \"C\" __global__\nvoid fused_sub_div_neg_add(float* trss_1, float* tsum_squared_obs_1, float* tv_, float* aten_add, float* aten_sub) {\n{\n  float tsum_squared_obs_1_1 = __ldg(tsum_squared_obs_1 + 0ll);\n  float tv__1 = __ldg(tv_ + 0ll);\n  aten_sub[0ll] = tsum_squared_obs_1_1 - tv__1;\n  float v = __ldg(trss_1 + 0ll);\n  aten_add[0ll] = (0.f - v / (tsum_squared_obs_1_1 - tv__1)) + 1.f;\n}\n}\n",
                "traceback": [
                    "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
                    "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
                    "Cell \u001b[1;32mIn[16], line 62\u001b[0m\n\u001b[0;32m     59\u001b[0m \u001b[38;5;66;03m#################################   \u001b[39;00m\n\u001b[0;32m     60\u001b[0m    \u001b[38;5;66;03m#r2 score\u001b[39;00m\n\u001b[0;32m     61\u001b[0m    \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28mtype\u001b[39m(prediction))\n\u001b[1;32m---> 62\u001b[0m    R2 \u001b[38;5;241m=\u001b[39m \u001b[43mr2_score\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprediction\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     63\u001b[0m    train_r2_list\u001b[38;5;241m.\u001b[39mappend(R2) \u001b[38;5;66;03m# all scores recorded\u001b[39;00m\n\u001b[0;32m     64\u001b[0m    mean_train_epoch_r2 \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m R2 \u001b[38;5;66;03m# per batch list\u001b[39;00m\n",
                    "File \u001b[1;32mc:\\Users\\snedd\\anaconda3\\envs\\ptorch\\Lib\\site-packages\\torch\\utils\\_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator.<locals>.decorate_context\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    112\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[0;32m    113\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m    114\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[1;32m--> 115\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
                    "File \u001b[1;32mc:\\Users\\snedd\\anaconda3\\envs\\ptorch\\Lib\\site-packages\\torcheval\\metrics\\functional\\regression\\r2_score.py:79\u001b[0m, in \u001b[0;36mr2_score\u001b[1;34m(input, target, multioutput, num_regressors)\u001b[0m\n\u001b[0;32m     75\u001b[0m _r2_score_param_check(multioutput, num_regressors)\n\u001b[0;32m     76\u001b[0m sum_squared_obs, sum_obs, sum_squared_residual, num_obs \u001b[38;5;241m=\u001b[39m _r2_score_update(\n\u001b[0;32m     77\u001b[0m     \u001b[38;5;28minput\u001b[39m, target\n\u001b[0;32m     78\u001b[0m )\n\u001b[1;32m---> 79\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_r2_score_compute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     80\u001b[0m \u001b[43m    \u001b[49m\u001b[43msum_squared_obs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     81\u001b[0m \u001b[43m    \u001b[49m\u001b[43msum_obs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     82\u001b[0m \u001b[43m    \u001b[49m\u001b[43msum_squared_residual\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     83\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnum_obs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     84\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmultioutput\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     85\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnum_regressors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m     86\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
                    "File \u001b[1;32mc:\\Users\\snedd\\anaconda3\\envs\\ptorch\\Lib\\site-packages\\torcheval\\metrics\\functional\\regression\\r2_score.py:126\u001b[0m, in \u001b[0;36m_r2_score_compute\u001b[1;34m(sum_squared_obs, sum_obs, rss, num_obs, multioutput, num_regressors)\u001b[0m\n\u001b[0;32m    121\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m num_regressors \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m num_obs \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m    122\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    123\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe `num_regressors` must be smaller than n_samples - 1, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    124\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgot num_regressors=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnum_regressors\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, n_samples=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnum_obs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m    125\u001b[0m     )\n\u001b[1;32m--> 126\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_compute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    127\u001b[0m \u001b[43m    \u001b[49m\u001b[43msum_squared_obs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    128\u001b[0m \u001b[43m    \u001b[49m\u001b[43msum_obs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    129\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrss\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    130\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnum_obs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    131\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmultioutput\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    132\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnum_regressors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    133\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
                    "\u001b[1;31mRuntimeError\u001b[0m: nvrtc: error: failed to open nvrtc-builtins64_121.dll.\n  Make sure that nvrtc-builtins64_121.dll is installed correctly.\nnvrtc compilation failed: \n\n#define NAN __int_as_float(0x7fffffff)\n#define POS_INFINITY __int_as_float(0x7f800000)\n#define NEG_INFINITY __int_as_float(0xff800000)\n\n\ntemplate<typename T>\n__device__ T maximum(T a, T b) {\n  return isnan(a) ? a : (a > b ? a : b);\n}\n\ntemplate<typename T>\n__device__ T minimum(T a, T b) {\n  return isnan(a) ? a : (a < b ? a : b);\n}\n\nextern \"C\" __global__\nvoid fused_sub_div_neg_add(float* trss_1, float* tsum_squared_obs_1, float* tv_, float* aten_add, float* aten_sub) {\n{\n  float tsum_squared_obs_1_1 = __ldg(tsum_squared_obs_1 + 0ll);\n  float tv__1 = __ldg(tv_ + 0ll);\n  aten_sub[0ll] = tsum_squared_obs_1_1 - tv__1;\n  float v = __ldg(trss_1 + 0ll);\n  aten_add[0ll] = (0.f - v / (tsum_squared_obs_1_1 - tv__1)) + 1.f;\n}\n}\n"
                ]
            }

Versions

PyTorch version: 2.2.2
Is debug build: False
CUDA used to build PyTorch: 12.1
ROCM used to build PyTorch: N/A

OS: Microsoft Windows 11 Pro
GCC version: Could not collect
Clang version: Could not collect
CMake version: Could not collect
Libc version: N/A

Python version: 3.12.2 | packaged by Anaconda, Inc. | (main, Feb 27 2024, 17:28:07) [MSC v.1916 64 bit (AMD64)] (64-bit runtime)
Python platform: Windows-11-10.0.22631-SP0
Is CUDA available: True
CUDA runtime version: 12.4.131
CUDA_MODULE_LOADING set to: LAZY
GPU models and configuration: GPU 0: NVIDIA GeForce RTX 3070
Nvidia driver version: 552.12
cuDNN version: Could not collect
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

CPU:
Architecture=9
CurrentClockSpeed=3792
DeviceID=CPU0
Family=198
L2CacheSize=2048
L2CacheSpeed=
Manufacturer=GenuineIntel
MaxClockSpeed=3792
Name=Intel(R) Core(TM) i7-10700K CPU @ 3.80GHz
ProcessorType=3
Revision=

Versions of relevant libraries:
[pip3] numpy==1.26.4
[pip3] torch==2.2.2
[pip3] torchaudio==2.2.2
[pip3] torcheval==0.0.7
[pip3] torchvision==0.17.2
[conda] blas                      1.0                         mkl
[conda] mkl                       2023.1.0         h6b88ed4_46358
[conda] mkl-service               2.4.0           py312h2bbff1b_1
[conda] mkl_fft                   1.3.8           py312h2bbff1b_0
[conda] mkl_random                1.2.4           py312h59b6b97_0
[conda] numpy                     1.26.4          py312hfd52020_0
[conda] numpy-base                1.26.4          py312h4dde369_0
[conda] pytorch                   2.2.2           py3.12_cuda12.1_cudnn8_0    pytorch
[conda] pytorch-cuda              12.1                 hde6ce7c_5    pytorch
[conda] pytorch-mutex             1.0                        cuda    pytorch
[conda] torchaudio                2.2.2                    pypi_0    pypi
[conda] torcheval                 0.0.7                    pypi_0    pypi
[conda] torchvision               0.17.2                   pypi_0    pypi