`datashader=True` caused `subplots=True` to raised `KeyError: 'Variable`

[p4perf4ce]

Bug mentioned in https://github.com/holoviz/hvplot/issues/486 re-emerged. Whenever datashade=True or rasterize=True, KeyError: 'Variable' will be raised.

Environments: datashader==0.14.4 holoviews==1.15.4 hvplot==0.8.3 panel==0.14.4

This bug can be reproduced using example given in this thread or

from hvplot.sample_data import airline_flights, us_crime
us_crime.hvplot(x='Year', y=['Burglary rate', 'Violent Crime rate', 'Robbery rate'],
                value_label='Rate', subplots=True, width=300, height=200, datashade=True)

Error log:

KeyError                                  Traceback (most recent call last)
File /opt/conda/envs/rhodes/lib/python3.10/site-packages/pandas/core/indexes/base.py:3802, in Index.get_loc(self, key, method, tolerance)
   3801 try:
-> 3802     return self._engine.get_loc(casted_key)
   3803 except KeyError as err:

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/pandas/_libs/index.pyx:138, in pandas._libs.index.IndexEngine.get_loc()

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/pandas/_libs/index.pyx:165, in pandas._libs.index.IndexEngine.get_loc()

File pandas/_libs/hashtable_class_helper.pxi:5745, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas/_libs/hashtable_class_helper.pxi:5753, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'Variable'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
File /opt/conda/envs/rhodes/lib/python3.10/site-packages/IPython/core/formatters.py:972, in MimeBundleFormatter.__call__(self, obj, include, exclude)
    969     method = get_real_method(obj, self.print_method)
    971     if method is not None:
--> 972         return method(include=include, exclude=exclude)
    973     return None
    974 else:

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/dimension.py:1290, in Dimensioned._repr_mimebundle_(self, include, exclude)
   1283 def _repr_mimebundle_(self, include=None, exclude=None):
   1284     """
   1285     Resolves the class hierarchy for the class rendering the
   1286     object using any display hooks registered on Store.display
   1287     hooks.  The output of all registered display_hooks is then
   1288     combined and returned.
   1289     """
-> 1290     return Store.render(self)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/options.py:1425, in Store.render(cls, obj)
   1423 data, metadata = {}, {}
   1424 for hook in hooks:
-> 1425     ret = hook(obj)
   1426     if ret is None:
   1427         continue

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/ipython/display_hooks.py:279, in pprint_display(obj)
    277 if not ip.display_formatter.formatters['text/plain'].pprint:
    278     return None
--> 279 return display(obj, raw_output=True)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/ipython/display_hooks.py:250, in display(obj, raw_output, **kwargs)
    248 elif isinstance(obj, (Layout, NdLayout, AdjointLayout)):
    249     with option_state(obj):
--> 250         output = layout_display(obj)
    251 elif isinstance(obj, (HoloMap, DynamicMap)):
    252     with option_state(obj):

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/ipython/display_hooks.py:141, in display_hook.<locals>.wrapped(element)
    139 try:
    140     max_frames = OutputSettings.options['max_frames']
--> 141     mimebundle = fn(element, max_frames=max_frames)
    142     if mimebundle is None:
    143         return {}, {}

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/ipython/display_hooks.py:215, in layout_display(layout, max_frames)
    212     max_frame_warning(max_frames)
    213     return None
--> 215 return render(layout)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/ipython/display_hooks.py:68, in render(obj, **kwargs)
     65 if renderer.fig == 'pdf':
     66     renderer = renderer.instance(fig='png')
---> 68 return renderer.components(obj, **kwargs)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/plotting/renderer.py:394, in Renderer.components(self, obj, fmt, comm, **kwargs)
    391 embed = (not (dynamic or streams or self.widget_mode == 'live') or config.embed)
    393 if embed or config.comms == 'default':
--> 394     return self._render_panel(plot, embed, comm)
    395 return self._render_ipywidget(plot)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/plotting/renderer.py:401, in Renderer._render_panel(self, plot, embed, comm)
    399 doc = Document()
    400 with config.set(embed=embed):
--> 401     model = plot.layout._render_model(doc, comm)
    402 if embed:
    403     return render_model(model, comm)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/panel/viewable.py:509, in Renderable._render_model(self, doc, comm)
    507 if comm is None:
    508     comm = state._comm_manager.get_server_comm()
--> 509 model = self.get_root(doc, comm)
    511 if config.embed:
    512     embed_state(self, model, doc,
    513                 json=config.embed_json,
    514                 json_prefix=config.embed_json_prefix,
    515                 save_path=config.embed_save_path,
    516                 load_path=config.embed_load_path,
    517                 progress=False)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/panel/viewable.py:560, in Renderable.get_root(self, doc, comm, preprocess)
    543 """
    544 Returns the root model and applies pre-processing hooks
    545 
   (...)
    557 Returns the bokeh model corresponding to this panel object
    558 """
    559 doc = init_doc(doc)
--> 560 root = self._get_model(doc, comm=comm)
    561 if preprocess:
    562     self._preprocess(root)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/panel/layout/base.py:146, in Panel._get_model(self, doc, root, parent, comm)
    144 if root is None:
    145     root = model
--> 146 objects = self._get_objects(model, [], doc, root, comm)
    147 props = dict(self._init_params(), objects=objects)
    148 model.update(**self._process_param_change(props))

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/panel/layout/base.py:131, in Panel._get_objects(self, model, old_objects, doc, root, comm)
    129 else:
    130     try:
--> 131         child = pane._get_model(doc, root, model, comm)
    132     except RerenderError:
    133         return self._get_objects(model, current_objects[:i], doc, root, comm)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/panel/pane/holoviews.py:367, in HoloViews._get_model(self, doc, root, parent, comm)
    365     plot = self.object
    366 else:
--> 367     plot = self._render(doc, comm, root)
    369 plot.pane = self
    370 backend = plot.renderer.backend

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/panel/pane/holoviews.py:442, in HoloViews._render(self, doc, comm, root)
    439     if comm:
    440         kwargs['comm'] = comm
--> 442 return renderer.get_plot(self.object, **kwargs)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/plotting/bokeh/renderer.py:70, in BokehRenderer.get_plot(self_or_cls, obj, doc, renderer, **kwargs)
     63 @bothmethod
     64 def get_plot(self_or_cls, obj, doc=None, renderer=None, **kwargs):
     65     """
     66     Given a HoloViews Viewable return a corresponding plot instance.
     67     Allows supplying a document attach the plot to, useful when
     68     combining the bokeh model with another plot.
     69     """
---> 70     plot = super().get_plot(obj, doc, renderer, **kwargs)
     71     if plot.document is None:
     72         plot.document = Document() if self_or_cls.notebook_context else curdoc()

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/plotting/renderer.py:213, in Renderer.get_plot(self_or_cls, obj, doc, renderer, comm, **kwargs)
    210     raise SkipRendering(msg.format(dims=dims))
    212 # Initialize DynamicMaps with first data item
--> 213 initialize_dynamic(obj)
    215 if not renderer:
    216     renderer = self_or_cls

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/plotting/util.py:257, in initialize_dynamic(obj)
    255     continue
    256 if not len(dmap):
--> 257     dmap[dmap._initial_key()]

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/spaces.py:1213, in DynamicMap.__getitem__(self, key)
   1211 # Not a cross product and nothing cached so compute element.
   1212 if cache is not None: return cache
-> 1213 val = self._execute_callback(*tuple_key)
   1214 if data_slice:
   1215     val = self._dataslice(val, data_slice)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/spaces.py:980, in DynamicMap._execute_callback(self, *args)
    977     kwargs['_memoization_hash_'] = hash_items
    979 with dynamicmap_memoization(self.callback, self.streams):
--> 980     retval = self.callback(*args, **kwargs)
    981 return self._style(retval)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/spaces.py:580, in Callable.__call__(self, *args, **kwargs)
    577     args, kwargs = (), dict(pos_kwargs, **kwargs)
    579 try:
--> 580     ret = self.callable(*args, **kwargs)
    581 except KeyError:
    582     # KeyError is caught separately because it is used to signal
    583     # invalid keys on DynamicMap and should not warn
    584     raise

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/util/__init__.py:1011, in Dynamic._dynamic_operation.<locals>.dynamic_operation(*key, **kwargs)
   1009 def dynamic_operation(*key, **kwargs):
   1010     key, obj = resolve(key, kwargs)
-> 1011     return apply(obj, *key, **kwargs)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/util/__init__.py:1003, in Dynamic._dynamic_operation.<locals>.apply(element, *key, **kwargs)
   1001 def apply(element, *key, **kwargs):
   1002     kwargs = dict(util.resolve_dependent_kwargs(self.p.kwargs), **kwargs)
-> 1003     processed = self._process(element, key, kwargs)
   1004     if (self.p.link_dataset and isinstance(element, Dataset) and
   1005         isinstance(processed, Dataset) and processed._dataset is None):
   1006         processed._dataset = element.dataset

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/util/__init__.py:985, in Dynamic._process(self, element, key, kwargs)
    983 elif isinstance(self.p.operation, Operation):
    984     kwargs = {k: v for k, v in kwargs.items() if k in self.p.operation.param}
--> 985     return self.p.operation.process_element(element, key, **kwargs)
    986 else:
    987     return self.p.operation(element, **kwargs)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/operation.py:194, in Operation.process_element(self, element, key, **params)
    191 else:
    192     self.p = param.ParamOverrides(self, params,
    193                                   allow_extra_keywords=self._allow_extra_keywords)
--> 194 return self._apply(element, key)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/operation.py:141, in Operation._apply(self, element, key)
    139     if not in_method:
    140         element._in_method = True
--> 141 ret = self._process(element, key)
    142 if hasattr(element, '_in_method') and not in_method:
    143     element._in_method = in_method

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/operation/datashader.py:1594, in datashade._process(self, element, key)
   1593 def _process(self, element, key=None):
-> 1594     agg = rasterize._process(self, element, key)
   1595     shaded = shade._process(self, agg, key)
   1596     return shaded

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/operation/datashader.py:1573, in rasterize._process(self, element, key)
   1570     op = transform.instance(**{k:v for k,v in extended_kws.items()
   1571                                if k in transform.param})
   1572     op._precomputed = self._precomputed
-> 1573     element = element.map(op, predicate)
   1574     self._precomputed = op._precomputed
   1576 unused_params = list(all_supplied_kws - all_allowed_kws)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/data/__init__.py:199, in PipelineMeta.pipelined.<locals>.pipelined_fn(*args, **kwargs)
    196     inst._in_method = True
    198 try:
--> 199     result = method_fn(*args, **kwargs)
    200     if PipelineMeta.disable:
    201         return result

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/data/__init__.py:1207, in Dataset.map(self, *args, **kwargs)
   1205 @wraps(LabelledData.map)
   1206 def map(self, *args, **kwargs):
-> 1207     return super().map(*args, **kwargs)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/dimension.py:700, in LabelledData.map(self, map_fn, specs, clone)
    698     return deep_mapped
    699 else:
--> 700     return map_fn(self) if applies else self

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/operation.py:214, in Operation.__call__(self, element, **kwargs)
    210         return element.clone([(k, self._apply(el, key=k))
    211                               for k, el in element.items()])
    212     elif ((self._per_element and isinstance(element, Element)) or
    213           (not self._per_element and isinstance(element, ViewableElement))):
--> 214         return self._apply(element)
    215 elif 'streams' not in kwargs:
    216     kwargs['streams'] = self.p.streams

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/core/operation.py:141, in Operation._apply(self, element, key)
    139     if not in_method:
    140         element._in_method = True
--> 141 ret = self._process(element, key)
    142 if hasattr(element, '_in_method') and not in_method:
    143     element._in_method = in_method

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/operation/datashader.py:491, in aggregate._process(self, element, key)
    489     x, y, data, glyph = self._precomputed[element._plot_id]
    490 else:
--> 491     x, y, data, glyph = self.get_agg_data(element, category)
    493 if self.p.precompute:
    494     self._precomputed[element._plot_id] = x, y, data, glyph

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/holoviews/operation/datashader.py:450, in aggregate.get_agg_data(cls, obj, category)
    448 else:
    449     df = paths[0] if paths else pd.DataFrame([], columns=[x.name, y.name])
--> 450 if category and df[category].dtype.name != 'category':
    451     df[category] = df[category].astype('category')
    453 is_custom = isinstance(df, dd.DataFrame) or cuDFInterface.applies(df)

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/pandas/core/frame.py:3807, in DataFrame.__getitem__(self, key)
   3805 if self.columns.nlevels > 1:
   3806     return self._getitem_multilevel(key)
-> 3807 indexer = self.columns.get_loc(key)
   3808 if is_integer(indexer):
   3809     indexer = [indexer]

File /opt/conda/envs/rhodes/lib/python3.10/site-packages/pandas/core/indexes/base.py:3804, in Index.get_loc(self, key, method, tolerance)
   3802     return self._engine.get_loc(casted_key)
   3803 except KeyError as err:
-> 3804     raise KeyError(key) from err
   3805 except TypeError:
   3806     # If we have a listlike key, _check_indexing_error will raise
   3807     #  InvalidIndexError. Otherwise we fall through and re-raise
   3808     #  the TypeError.
   3809     self._check_indexing_error(key)

KeyError: 'Variable'

Originally posted by @p4perf4ce in https://github.com/holoviz/hvplot/issues/486#issuecomment-1484087848

[p4perf4ce]

Downgrade some package to the version mentioned in the original issue seems to fix the problem. Latest version of holoviews and panel can still be used.

datashader==0.13.0 holoviews==1.15.4 hvplot==0.7.3 panel==0.14.4

Edit: datashader==0.14.4 also works

[hoxbro]

A git bisect shows this was introduced in https://github.com/holoviz/hvplot/pull/759.

The case not taken into account is subplots=True.