filtering interactive dataframe with interactive boolean series / array / list not working

[JanHomann]

Versions

hvplot      0.8.1
holoviews   1.15.1
panel       0.13.1

pandas      1.5.0
numpy       1.22.4
python      3.9.13
jupyterlab  3.4.5

Chrome Browser

Description

Filtering an interactive dataframe using a comparison operator using .loc fails, probably due to issues with the boolean interactive dataframe that is generated as an intermediate step.

Example

import numpy as np
import pandas as pd
import hvplot.pandas
import panel as pn

df = pd.DataFrame(50*np.random.randn(10,50))

# problematic code for interactive dataframe
dfi = df.rename(columns=str).interactive(width=800)          # setting column names to strings currently necessary
threshold = pn.widgets.IntSlider(start=10, end=50, value=20)
threshold                                          # make widget

dfi.loc[:,dfi.mean() > threshold]                 # AttributeError:
dfi.loc[:,(dfi.mean() > threshold).values]        # TypeError: 'numpy.ndarray' object is not callable
dfi.loc[:,(dfi.mean() > threshold).to_list()]     # AttributeError:

# NOTE: (dfi.mean() > threshold).to_list() produces a strange repr

# equivalent code for pandas dataframe (works)
df = df.rename(columns=str)
threshold = 20
df.loc[:,df.mean() > threshold]
df.loc[:,(df.mean() > threshold).values] 
df.loc[:,(df.mean() > threshold).to_list()] 

Stack traceback for all three cases

case 1

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Input In [222], in <cell line: 9>()
      6 threshold = pnw.IntSlider(start=10, end=50, value=20, name='Spike threshold')
      7 threshold
----> 9 dfi.loc[:,(dfi.mean() > threshold)]

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:624, in Interactive.__getitem__(self, other)
    622 def __getitem__(self, other):
    623     other = other._transform if isinstance(other, Interactive) else other
--> 624     return self._apply_operator(operator.getitem, other)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:511, in Interactive._apply_operator(self, operator, reverse, *args, **kwargs)
    509 transform = new._transform
    510 transform = type(transform)(transform, operator, *args, reverse=reverse)
--> 511 return new._clone(transform)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:360, in Interactive._clone(self, transform, plot, loc, center, dmap, copy, max_rows, **kwargs)
    358 else:
    359     kwargs = dict(self._inherit_kwargs, **dict(self._kwargs, **kwargs))
--> 360 return type(self)(self._obj, fn=self._fn, transform=transform, plot=plot, depth=depth,
    361                  loc=loc, center=center, dmap=dmap, _shared_obj=self._shared_obj,
    362                  max_rows=max_rows, **kwargs)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:266, in Interactive.__init__(self, obj, transform, fn, plot, depth, loc, center, dmap, inherit_kwargs, max_rows, method, _shared_obj, _current, **kwargs)
    264     self._current = _current
    265 else:
--> 266     self._current = self._transform.apply(ds, keep_index=True, compute=False)
    267 self._init = True
    268 self.hvplot = _hvplot(self)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/holoviews/util/transform.py:773, in dim.apply(self, dataset, flat, expanded, ranges, all_values, keep_index, compute, strict)
    771     drange = ranges.get(eldim, {})
    772     drange = drange.get('combined', drange)
--> 773     data = self._apply_fn(dataset, data, fn, fn_name, args,
    774                           kwargs, accessor, drange)
    775 drop_index = keep_index_for_compute and not keep_index
    776 compute = not compute_for_compute and compute

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/holoviews/util/transform.py:673, in dim._apply_fn(self, dataset, data, fn, fn_name, args, kwargs, accessor, drange)
    671                 raise e
    672 else:
--> 673     data = fn(*args, **kwargs)
    675 return data

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/pandas/core/indexing.py:1065, in _LocationIndexer.__getitem__(self, key)
   1063 if type(key) is tuple:
   1064     key = tuple(list(x) if is_iterator(x) else x for x in key)
-> 1065     key = tuple(com.apply_if_callable(x, self.obj) for x in key)
   1066     if self._is_scalar_access(key):
   1067         return self.obj._get_value(*key, takeable=self._takeable)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/pandas/core/indexing.py:1065, in <genexpr>(.0)
   1063 if type(key) is tuple:
   1064     key = tuple(list(x) if is_iterator(x) else x for x in key)
-> 1065     key = tuple(com.apply_if_callable(x, self.obj) for x in key)
   1066     if self._is_scalar_access(key):
   1067         return self.obj._get_value(*key, takeable=self._takeable)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/pandas/core/common.py:364, in apply_if_callable(maybe_callable, obj, **kwargs)
    353 """
    354 Evaluate possibly callable input using obj and kwargs if it is callable,
    355 otherwise return as it is.
   (...)
    361 **kwargs
    362 """
    363 if callable(maybe_callable):
--> 364     return maybe_callable(obj, **kwargs)
    366 return maybe_callable

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:480, in Interactive.__call__(self, *args, **kwargs)
    478         return self._clone(*args, **kwargs)
    479     # TODO: When is this error raised?
--> 480     raise AttributeError
    481 elif self._method == 'plot':
    482     # This - {ax: get_ax} - is passed as kwargs to the plot method in
    483     # the dim expression.
    484     kwargs['ax'] = self._get_ax_fn()

AttributeError: 

case 2

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Input In [223], in <cell line: 9>()
      6 threshold = pnw.IntSlider(start=10, end=50, value=20)
      7 threshold
----> 9 dfi.loc[:,(dfi.mean() > threshold).values]

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:624, in Interactive.__getitem__(self, other)
    622 def __getitem__(self, other):
    623     other = other._transform if isinstance(other, Interactive) else other
--> 624     return self._apply_operator(operator.getitem, other)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:511, in Interactive._apply_operator(self, operator, reverse, *args, **kwargs)
    509 transform = new._transform
    510 transform = type(transform)(transform, operator, *args, reverse=reverse)
--> 511 return new._clone(transform)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:360, in Interactive._clone(self, transform, plot, loc, center, dmap, copy, max_rows, **kwargs)
    358 else:
    359     kwargs = dict(self._inherit_kwargs, **dict(self._kwargs, **kwargs))
--> 360 return type(self)(self._obj, fn=self._fn, transform=transform, plot=plot, depth=depth,
    361                  loc=loc, center=center, dmap=dmap, _shared_obj=self._shared_obj,
    362                  max_rows=max_rows, **kwargs)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:266, in Interactive.__init__(self, obj, transform, fn, plot, depth, loc, center, dmap, inherit_kwargs, max_rows, method, _shared_obj, _current, **kwargs)
    264     self._current = _current
    265 else:
--> 266     self._current = self._transform.apply(ds, keep_index=True, compute=False)
    267 self._init = True
    268 self.hvplot = _hvplot(self)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/holoviews/util/transform.py:773, in dim.apply(self, dataset, flat, expanded, ranges, all_values, keep_index, compute, strict)
    771     drange = ranges.get(eldim, {})
    772     drange = drange.get('combined', drange)
--> 773     data = self._apply_fn(dataset, data, fn, fn_name, args,
    774                           kwargs, accessor, drange)
    775 drop_index = keep_index_for_compute and not keep_index
    776 compute = not compute_for_compute and compute

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/holoviews/util/transform.py:673, in dim._apply_fn(self, dataset, data, fn, fn_name, args, kwargs, accessor, drange)
    671                 raise e
    672 else:
--> 673     data = fn(*args, **kwargs)
    675 return data

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/pandas/core/indexing.py:1065, in _LocationIndexer.__getitem__(self, key)
   1063 if type(key) is tuple:
   1064     key = tuple(list(x) if is_iterator(x) else x for x in key)
-> 1065     key = tuple(com.apply_if_callable(x, self.obj) for x in key)
   1066     if self._is_scalar_access(key):
   1067         return self.obj._get_value(*key, takeable=self._takeable)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/pandas/core/indexing.py:1065, in <genexpr>(.0)
   1063 if type(key) is tuple:
   1064     key = tuple(list(x) if is_iterator(x) else x for x in key)
-> 1065     key = tuple(com.apply_if_callable(x, self.obj) for x in key)
   1066     if self._is_scalar_access(key):
   1067         return self.obj._get_value(*key, takeable=self._takeable)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/pandas/core/common.py:364, in apply_if_callable(maybe_callable, obj, **kwargs)
    353 """
    354 Evaluate possibly callable input using obj and kwargs if it is callable,
    355 otherwise return as it is.
   (...)
    361 **kwargs
    362 """
    363 if callable(maybe_callable):
--> 364     return maybe_callable(obj, **kwargs)
    366 return maybe_callable

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:489, in Interactive.__call__(self, *args, **kwargs)
    487     method = type(new._transform)(new._transform, new._method, accessor=True)
    488     kwargs = dict(new._inherit_kwargs, **kwargs)
--> 489     clone = new._clone(method(*args, **kwargs), plot=new._method == 'plot')
    490 finally:
    491     # If an error occurs reset _method anyway so that, e.g. the next
    492     # attempt in a Notebook, is set appropriately.
    493     new._method = None

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:360, in Interactive._clone(self, transform, plot, loc, center, dmap, copy, max_rows, **kwargs)
    358 else:
    359     kwargs = dict(self._inherit_kwargs, **dict(self._kwargs, **kwargs))
--> 360 return type(self)(self._obj, fn=self._fn, transform=transform, plot=plot, depth=depth,
    361                  loc=loc, center=center, dmap=dmap, _shared_obj=self._shared_obj,
    362                  max_rows=max_rows, **kwargs)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:266, in Interactive.__init__(self, obj, transform, fn, plot, depth, loc, center, dmap, inherit_kwargs, max_rows, method, _shared_obj, _current, **kwargs)
    264     self._current = _current
    265 else:
--> 266     self._current = self._transform.apply(ds, keep_index=True, compute=False)
    267 self._init = True
    268 self.hvplot = _hvplot(self)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/holoviews/util/transform.py:773, in dim.apply(self, dataset, flat, expanded, ranges, all_values, keep_index, compute, strict)
    771     drange = ranges.get(eldim, {})
    772     drange = drange.get('combined', drange)
--> 773     data = self._apply_fn(dataset, data, fn, fn_name, args,
    774                           kwargs, accessor, drange)
    775 drop_index = keep_index_for_compute and not keep_index
    776 compute = not compute_for_compute and compute

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/holoviews/util/transform.py:671, in dim._apply_fn(self, dataset, data, fn, fn_name, args, kwargs, accessor, drange)
    669                 data = method(*args, **kwargs)
    670             else:
--> 671                 raise e
    672 else:
    673     data = fn(*args, **kwargs)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/holoviews/util/transform.py:665, in dim._apply_fn(self, dataset, data, fn, fn_name, args, kwargs, accessor, drange)
    663 else:
    664     try:
--> 665         data = method(*args, **kwargs)
    666     except Exception as e:
    667         if 'axis' in kwargs:

TypeError: 'numpy.ndarray' object is not callable

Case 3

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Input In [15], in <cell line: 9>()
      6 threshold = pn.widgets.IntSlider(start=10, end=50, value=20)
      7 threshold
----> 9 dfi.loc[:,(dfi.mean() > threshold).to_list()]

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:624, in Interactive.__getitem__(self, other)
    622 def __getitem__(self, other):
    623     other = other._transform if isinstance(other, Interactive) else other
--> 624     return self._apply_operator(operator.getitem, other)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:511, in Interactive._apply_operator(self, operator, reverse, *args, **kwargs)
    509 transform = new._transform
    510 transform = type(transform)(transform, operator, *args, reverse=reverse)
--> 511 return new._clone(transform)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:360, in Interactive._clone(self, transform, plot, loc, center, dmap, copy, max_rows, **kwargs)
    358 else:
    359     kwargs = dict(self._inherit_kwargs, **dict(self._kwargs, **kwargs))
--> 360 return type(self)(self._obj, fn=self._fn, transform=transform, plot=plot, depth=depth,
    361                  loc=loc, center=center, dmap=dmap, _shared_obj=self._shared_obj,
    362                  max_rows=max_rows, **kwargs)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:266, in Interactive.__init__(self, obj, transform, fn, plot, depth, loc, center, dmap, inherit_kwargs, max_rows, method, _shared_obj, _current, **kwargs)
    264     self._current = _current
    265 else:
--> 266     self._current = self._transform.apply(ds, keep_index=True, compute=False)
    267 self._init = True
    268 self.hvplot = _hvplot(self)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/holoviews/util/transform.py:773, in dim.apply(self, dataset, flat, expanded, ranges, all_values, keep_index, compute, strict)
    771     drange = ranges.get(eldim, {})
    772     drange = drange.get('combined', drange)
--> 773     data = self._apply_fn(dataset, data, fn, fn_name, args,
    774                           kwargs, accessor, drange)
    775 drop_index = keep_index_for_compute and not keep_index
    776 compute = not compute_for_compute and compute

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/holoviews/util/transform.py:673, in dim._apply_fn(self, dataset, data, fn, fn_name, args, kwargs, accessor, drange)
    671                 raise e
    672 else:
--> 673     data = fn(*args, **kwargs)
    675 return data

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/pandas/core/indexing.py:1065, in _LocationIndexer.__getitem__(self, key)
   1063 if type(key) is tuple:
   1064     key = tuple(list(x) if is_iterator(x) else x for x in key)
-> 1065     key = tuple(com.apply_if_callable(x, self.obj) for x in key)
   1066     if self._is_scalar_access(key):
   1067         return self.obj._get_value(*key, takeable=self._takeable)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/pandas/core/indexing.py:1065, in <genexpr>(.0)
   1063 if type(key) is tuple:
   1064     key = tuple(list(x) if is_iterator(x) else x for x in key)
-> 1065     key = tuple(com.apply_if_callable(x, self.obj) for x in key)
   1066     if self._is_scalar_access(key):
   1067         return self.obj._get_value(*key, takeable=self._takeable)

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/pandas/core/common.py:364, in apply_if_callable(maybe_callable, obj, **kwargs)
    353 """
    354 Evaluate possibly callable input using obj and kwargs if it is callable,
    355 otherwise return as it is.
   (...)
    361 **kwargs
    362 """
    363 if callable(maybe_callable):
--> 364     return maybe_callable(obj, **kwargs)
    366 return maybe_callable

File ~/Dropbox/Berry-Vaziri-Collaboration/.venv/lib/python3.9/site-packages/hvplot/interactive.py:480, in Interactive.__call__(self, *args, **kwargs)
    478         return self._clone(*args, **kwargs)
    479     # TODO: When is this error raised?
--> 480     raise AttributeError
    481 elif self._method == 'plot':
    482     # This - {ax: get_ax} - is passed as kwargs to the plot method in
    483     # the dim expression.
    484     kwargs['ax'] = self._get_ax_fn()

AttributeError: 

[JanHomann]

I just wanted to add that this here works:

dfi.loc[dfi.mean(axis=1) > threshold]

But this here doesn't:

dfi.loc[dfi.mean(axis=1) > threshold,:]

Note: In both cases, you should set the starting point of the slider values a bit lower to see something:

threshold = pn.widgets.IntSlider(start=-10, end=50, value=0)

[JanHomann]

@maximlt The interactive DataFrames get rendered now, and they are filtered, but the interactivity in the filtering still does not work. When I move the slider, the interactive DataFrame does not update.

A workaround that I came up with is:

dfi.T.loc[dfi.mean() > threshold].T

This is equivalent to the original code that does not work:

dfi.loc[:,dfi.mean() > threshold]

So you transpose the dataframe before the operation and apply the dynamic filtering on the rows instead of the columns, and transpose it back after the operation.