`OverflowError: can't convert negative int to unsigned`

[jmakov]

After converting lazy frame's frequency, an exception is thrown by itables v2.1.3:

lf = ...  # contains ts_event column with UInt64 type
lf.collect()  # displays the data frame nicely
lf.with_columns(polars.from_epoch("ts_event", "ns")).collect()  # problematic

Traceback:

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File ~/mambaforge-pypy3/envs/proj/lib/python3.11/site-packages/itables/datatables_format.py:98, in datatables_rows(df, count, warn_on_unexpected_types, pure_json)
     94 try:
     95     # Pandas DataFrame
     96     data = list(
     97         zip(
---> 98             *(empty_columns + [_format_column(x, pure_json) for _, x in df.items()])
     99         )
    100     )
    101     has_bigints = any(
    102         x.dtype.kind == "i"
    103         and ((x > JS_MAX_SAFE_INTEGER).any() or (x < JS_MIN_SAFE_INTEGER).any())
    104         for _, x in df.items()
    105     )

AttributeError: 'DataFrame' object has no attribute 'items'

During handling of the above exception, another exception occurred:

OverflowError                             Traceback (most recent call last)
File ~/mambaforge-pypy3/envs/proj/lib/python3.11/site-packages/IPython/core/formatters.py:347, in BaseFormatter.__call__(self, obj)
    345     method = get_real_method(obj, self.print_method)
    346     if method is not None:
--> 347         return method()
    348     return None
    349 else:

File ~/mambaforge-pypy3/envs/proj/lib/python3.11/site-packages/itables/javascript.py:305, in _datatables_repr_(df)
    304 def _datatables_repr_(df):
--> 305     return to_html_datatable(df, connected=_CONNECTED)

File ~/mambaforge-pypy3/envs/proj/lib/python3.11/site-packages/itables/javascript.py:433, in to_html_datatable(df, caption, tableId, connected, use_to_html, **kwargs)
    430 # When the header has an extra column, we add
    431 # an extra empty column in the table data #141
    432 column_count = _column_count_in_header(table_header)
--> 433 dt_data = datatables_rows(
    434     df,
    435     column_count,
    436     warn_on_unexpected_types=warn_on_unexpected_types,
    437 )
    439 return html_table_from_template(
    440     table_header,
    441     table_id=tableId,
   (...)
    445     column_filters=column_filters,
    446 )

File ~/mambaforge-pypy3/envs/proj/lib/python3.11/site-packages/itables/datatables_format.py:116, in datatables_rows(df, count, warn_on_unexpected_types, pure_json)
    113     data = list(df.iter_rows())
    114     import polars as pl
--> 116     has_bigints = any(
    117         x.dtype in [pl.Int64, pl.UInt64]
    118         and ((x > JS_MAX_SAFE_INTEGER).any() or (x < JS_MIN_SAFE_INTEGER).any())
    119         for x in (df[col] for col in df.columns)
    120     )
    121     js = json.dumps(data, cls=generate_encoder(False), allow_nan=not pure_json)
    123 if has_bigints:

File ~/mambaforge-pypy3/envs/proj/lib/python3.11/site-packages/itables/datatables_format.py:118, in <genexpr>(.0)
    113     data = list(df.iter_rows())
    114     import polars as pl
    116     has_bigints = any(
    117         x.dtype in [pl.Int64, pl.UInt64]
--> 118         and ((x > JS_MAX_SAFE_INTEGER).any() or (x < JS_MIN_SAFE_INTEGER).any())
    119         for x in (df[col] for col in df.columns)
    120     )
    121     js = json.dumps(data, cls=generate_encoder(False), allow_nan=not pure_json)
    123 if has_bigints:

File ~/mambaforge-pypy3/envs/proj/lib/python3.11/site-packages/polars/series/series.py:800, in Series.__lt__(self, other)
    798 if isinstance(other, pl.Expr):
    799     return F.lit(self).__lt__(other)
--> 800 return self._comp(other, "lt")

File ~/mambaforge-pypy3/envs/proj/lib/python3.11/site-packages/polars/series/series.py:749, in Series._comp(self, other, op)
    746 if f is None:
    747     return NotImplemented
--> 749 return self._from_pyseries(f(other))

OverflowError: can't convert negative int to unsigned

[mwouts]

Hi @jmakov , thanks for reporting this! Apparently that's the comparison of an unsigned integer with the (negative) JS_MIN_SAFE_INTEGER value that fails.

I have a development version ready for you to test, see the linked PR.

[jmakov]

The PR works for me, tnx for the quick response! I'll let the PR close this issue when merged.