Basic this is failing on datashader, note it only fails when numba is disabled.
import os
os.environ['NUMBA_DISABLE_JIT'] = '1'
from numba import jit
import numpy as np
ngjit = jit(nopython=True, nogil=True)
bin_vec = np.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
dtype=np.uint8)
@ngjit
def _binary_2_int(bin_vec):
"""Convert a binary byte array to an integer"""
res = 0
next_val = 1
width = len(bin_vec)
for i in range(width):
res += next_val*bin_vec[width - i - 1]
next_val <<= 1
return res
_binary_2_int(bin_vec)
OverflowError Traceback (most recent call last)
Cell In[4], line 18
15 next_val <<= 1
16 return res
---> 18 _binary_2_int(bin_vec)
Cell In[4], line 13, in _binary_2_int(bin_vec)
10 width = len(bin_vec)
11 for i in range(width):
12 # res += next_val*int(bin_vec[width - i - 1])
---> 13 res += next_val*bin_vec[width - i - 1]
15 next_val <<= 1
16 return res
OverflowError: Python integer 256 out of bounds for uint8
Basic this is failing on datashader, note it only fails when numba is disabled.