Issues with using custom matplotlib colormap

[cdeciampa]

I'm having two issues:

1. When using xarray.hvplot.quadmesh, I'm unable to use a a custom LinearSegmentedColormap unless I register it with with matplotlib and call it that way.

2. I'm able to use a custom registered matplotlib colormap in one of my conda environments (running Python 3.10.5), but not the other (running Python 3.8.13). This is the same exact code as successfully plotted above:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/IPython/core/formatters.py:973, in MimeBundleFormatter.__call__(self, obj, include, exclude)
    970     method = get_real_method(obj, self.print_method)
    972     if method is not None:
--> 973         return method(include=include, exclude=exclude)
    974     return None
    975 else:

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/core/dimension.py:1294, in Dimensioned._repr_mimebundle_(self, include, exclude)
   1287 def _repr_mimebundle_(self, include=None, exclude=None):
   1288     """
   1289     Resolves the class hierarchy for the class rendering the
   1290     object using any display hooks registered on Store.display
   1291     hooks.  The output of all registered display_hooks is then
   1292     combined and returned.
   1293     """
-> 1294     return Store.render(self)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/core/options.py:1426, in Store.render(cls, obj)
   1424 data, metadata = {}, {}
   1425 for hook in hooks:
-> 1426     ret = hook(obj)
   1427     if ret is None:
   1428         continue

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/ipython/display_hooks.py:277, in pprint_display(obj)
    275 if not ip.display_formatter.formatters['text/plain'].pprint:
    276     return None
--> 277 return display(obj, raw_output=True)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/ipython/display_hooks.py:253, in display(obj, raw_output, **kwargs)
    251 elif isinstance(obj, (HoloMap, DynamicMap)):
    252     with option_state(obj):
--> 253         output = map_display(obj)
    254 elif isinstance(obj, Plot):
    255     output = render(obj)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/ipython/display_hooks.py:141, in display_hook.<locals>.wrapped(element)
    139 try:
    140     max_frames = OutputSettings.options['max_frames']
--> 141     mimebundle = fn(element, max_frames=max_frames)
    142     if mimebundle is None:
    143         return {}, {}

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/ipython/display_hooks.py:201, in map_display(vmap, max_frames)
    198     max_frame_warning(max_frames)
    199     return None
--> 201 return render(vmap)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/ipython/display_hooks.py:68, in render(obj, **kwargs)
     65 if renderer.fig == 'pdf':
     66     renderer = renderer.instance(fig='png')
---> 68 return renderer.components(obj, **kwargs)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/renderer.py:398, in Renderer.components(self, obj, fmt, comm, **kwargs)
    395 embed = (not (dynamic or streams or self.widget_mode == 'live') or config.embed)
    397 if embed or config.comms == 'default':
--> 398     return self._render_panel(plot, embed, comm)
    399 return self._render_ipywidget(plot)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/renderer.py:405, in Renderer._render_panel(self, plot, embed, comm)
    403 doc = Document()
    404 with config.set(embed=embed):
--> 405     model = plot.layout._render_model(doc, comm)
    406 if embed:
    407     return render_model(model, comm)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/panel/viewable.py:505, in Renderable._render_model(self, doc, comm)
    503 if comm is None:
    504     comm = state._comm_manager.get_server_comm()
--> 505 model = self.get_root(doc, comm)
    507 if config.embed:
    508     embed_state(self, model, doc,
    509                 json=config.embed_json,
    510                 json_prefix=config.embed_json_prefix,
    511                 save_path=config.embed_save_path,
    512                 load_path=config.embed_load_path,
    513                 progress=False)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/panel/viewable.py:556, in Renderable.get_root(self, doc, comm, preprocess)
    539 """
    540 Returns the root model and applies pre-processing hooks
    541 
   (...)
    553 Returns the bokeh model corresponding to this panel object
    554 """
    555 doc = init_doc(doc)
--> 556 root = self._get_model(doc, comm=comm)
    557 if preprocess:
    558     self._preprocess(root)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/panel/layout/base.py:146, in Panel._get_model(self, doc, root, parent, comm)
    144 if root is None:
    145     root = model
--> 146 objects = self._get_objects(model, [], doc, root, comm)
    147 props = dict(self._init_params(), objects=objects)
    148 model.update(**self._process_param_change(props))

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/panel/layout/base.py:131, in Panel._get_objects(self, model, old_objects, doc, root, comm)
    129 else:
    130     try:
--> 131         child = pane._get_model(doc, root, model, comm)
    132     except RerenderError:
    133         return self._get_objects(model, current_objects[:i], doc, root, comm)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/panel/pane/holoviews.py:265, in HoloViews._get_model(self, doc, root, parent, comm)
    263     plot = self.object
    264 else:
--> 265     plot = self._render(doc, comm, root)
    267 plot.pane = self
    268 backend = plot.renderer.backend

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/panel/pane/holoviews.py:342, in HoloViews._render(self, doc, comm, root)
    339     if comm:
    340         kwargs['comm'] = comm
--> 342 return renderer.get_plot(self.object, **kwargs)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/bokeh/renderer.py:70, in BokehRenderer.get_plot(self_or_cls, obj, doc, renderer, **kwargs)
     63 @bothmethod
     64 def get_plot(self_or_cls, obj, doc=None, renderer=None, **kwargs):
     65     """
     66     Given a HoloViews Viewable return a corresponding plot instance.
     67     Allows supplying a document attach the plot to, useful when
     68     combining the bokeh model with another plot.
     69     """
---> 70     plot = super().get_plot(obj, doc, renderer, **kwargs)
     71     if plot.document is None:
     72         plot.document = Document() if self_or_cls.notebook_context else curdoc()

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/renderer.py:240, in Renderer.get_plot(self_or_cls, obj, doc, renderer, comm, **kwargs)
    237     defaults = [kd.default for kd in plot.dimensions]
    238     init_key = tuple(v if d is None else d for v, d in
    239                      zip(plot.keys[0], defaults))
--> 240     plot.update(init_key)
    241 else:
    242     plot = obj

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/plot.py:949, in DimensionedPlot.update(self, key)
    947 if len(self) == 1 and ((key == 0) or (key == self.keys[0])) and not self.drawn:
    948     return self.initialize_plot()
--> 949 item = self.__getitem__(key)
    950 self.traverse(lambda x: setattr(x, '_updated', True))
    951 return item

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/plot.py:432, in DimensionedPlot.__getitem__(self, frame)
    430 if isinstance(frame, int) and frame > len(self):
    431     self.param.warning("Showing last frame available: %d" % len(self))
--> 432 if not self.drawn: self.handles['fig'] = self.initialize_plot()
    433 if not isinstance(frame, tuple):
    434     frame = self.keys[frame]

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/geoviews/plotting/bokeh/plot.py:111, in GeoPlot.initialize_plot(self, ranges, plot, plots, source)
    109 def initialize_plot(self, ranges=None, plot=None, plots=None, source=None):
    110     opts = {} if isinstance(self, HvOverlayPlot) else {'source': source}
--> 111     fig = super(GeoPlot, self).initialize_plot(ranges, plot, plots, **opts)
    112     if self.geographic and self.show_bounds and not self.overlaid:
    113         from . import GeoShapePlot

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/bokeh/element.py:1397, in ElementPlot.initialize_plot(self, ranges, plot, plots, source)
   1394     self.handles['y_range'] = plot.y_range
   1395 self.handles['plot'] = plot
-> 1397 self._init_glyphs(plot, element, ranges, source)
   1398 if not self.overlaid:
   1399     self._update_plot(key, plot, style_element)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/bokeh/element.py:1341, in ElementPlot._init_glyphs(self, plot, element, ranges, source)
   1339 else:
   1340     style = self.style[self.cyclic_index]
-> 1341     data, mapping, style = self.get_data(element, ranges, style)
   1342     current_id = element._plot_id
   1344 with abbreviated_exception():

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/geoviews/plotting/bokeh/plot.py:171, in GeoPlot.get_data(self, element, ranges, style)
    169 if self._project_operation and self.geographic:
    170     element = self._project_operation(element, projection=self.projection)
--> 171 return super(GeoPlot, self).get_data(element, ranges, style)

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/bokeh/raster.py:90, in RasterPlot.get_data(self, element, ranges, style)
     88 mapping = dict(image='image', x='x', y='y', dw='dw', dh='dh')
     89 val_dim = element.vdims[0]
---> 90 style['color_mapper'] = self._get_colormapper(val_dim, element, ranges, style)
     91 if 'alpha' in style:
     92     style['global_alpha'] = style['alpha']

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/bokeh/element.py:1913, in ColorbarPlot._get_colormapper(self, eldim, element, ranges, style, factors, colors, group, name)
   1908     if isinstance(cmap, list) and len(cmap) != ncolors:
   1909         raise ValueError('The number of colors in the colormap '
   1910                          'must match the intervals defined in the '
   1911                          'color_levels, expected %d colors found %d.'
   1912                          % (ncolors, len(cmap)))
-> 1913 palette = process_cmap(cmap, ncolors, categorical=categorical)
   1914 if isinstance(self.color_levels, list):
   1915     palette, (low, high) = color_intervals(palette, self.color_levels, clip=(low, high))

File /storage/work/cnd5285/miniconda/envs/uviz/lib/python3.8/site-packages/holoviews/plotting/util.py:911, in process_cmap(cmap, ncolors, provider, categorical)
    909         palette = colorcet_cmap_to_palette(cmap, ncolors, categorical)
    910     else:
--> 911         raise ValueError("Supplied cmap %s not found among %s colormaps." %
    912                          (cmap,providers_checked))
    913 else:
    914     try:
    915         # Try processing as matplotlib colormap

ValueError: Supplied cmap FLUT CIMSS not found among matplotlib, bokeh, or colorcet colormaps.

Successful environment (Python 3.10.5):

bokeh                     2.4.3           py310hff52083_0    conda-forge
holoviews                 1.14.9             pyhd8ed1ab_0    conda-forge
hvplot                    0.8.1                      py_0    pyviz

Failed environment (Python 3.8.13):

bokeh                     2.4.3              pyhd8ed1ab_3    conda-forge
holoviews                 1.15.1                     py_0    pyviz
hvplot                    0.8.1                      py_0    pyviz

[maximlt]

Hi @cdeciampa ! Could you provide with your issue a minimal reproducible example with code for us to copy/paste and run to try to reproduce the problem you describe? That would help tremendously!

[cdeciampa]

import os
import glob

import numpy as np
import xarray as xr

import matplotlib.pyplot as plt
import matplotlib as mpl
from matplotlib.colors import LinearSegmentedColormap

import hvplot.xarray

parent_28km_dir = "/path/to/28km/model/output/netCDFs"
h3p_files = glob.glob(os.path.join(parent_28km_dir, '*.h3.*.nc'))
h3pr_ds = xr.open_mfdataset([f for f in h3p_files if 'remap' in f])

def T_to_FLUT(T, unit='K'):
    if unit == 'C':
        T += 273.15
    sigma = 5.6693E-8
    olr = sigma*(T**4)

    return olr

# Normalized CIMSS outgoing longwave radiation flux colormap
bw_colors = [(0, '#BCBCBC'), (1, '#000000')]  
bw_cmp = LinearSegmentedColormap.from_list('FLUT bw', bw_colors, N=435)

levels = np.array([T_to_FLUT(temp, 'C') for temp in [-110, -105, -87.5, -80, -70, -60, -50, -35, -27.5, -22.5]])
fracs = levels-T_to_FLUT(-110, 'C')

rainbow_colors = [(0, '#0febff'), # cyan
                  ((fracs[1]/fracs[-1]), '#7f007f'), # purple
                  ((fracs[2]/fracs[-1]), '#e5e4e5'), # white
                  ((fracs[3]/fracs[-1]), '#000000'), # black
                  ((fracs[4]/fracs[-1]), '#ff0000'), # red
                  ((fracs[5]/fracs[-1]), '#FFFF00'), # yellow
                  ((fracs[6]/fracs[-1]), '#00FF00'), # green
                  ((fracs[7]/fracs[-1]), '#000073'), # navy
                  (1, '#00ffff')] # cyan

rainbow_cmp = LinearSegmentedColormap.from_list('FLUT colors', rainbow_colors, N=184)

bws = plt.get_cmap(bw_cmp)
bws_colors = bws(np.linspace(0, 1, 435))
rainbow = plt.get_cmap(rainbow_cmp)
r_colors = rainbow(np.linspace(0, 1, 184))

all_colors = np.vstack((r_colors, bws_colors))
flut_cimss = LinearSegmentedColormap.from_list('FLUT_CIMSS', all_colors)

# Registers as matplotlib colormap
mpl.colormaps.register(flut_cimss, name='FLUT_CIMSS', force=True)

h3pr_ds.FLUT.isel(time=7).hvplot.quadmesh('lon', 'lat', geo=True, rasterize=True, dynamic=False,
                                           clim=(T_to_FLUT(-110, 'C'), T_to_FLUT(55, 'C')),
                                           cmap = 'FLUT_CIMSS',  coastline='10m', frame_width=500)

[maximlt]

Unfortunately @cdeciampa I won't be able to run this code without having the files you use. So either you could share with me one file or you could modify your script to generate data instead of relying on reading files.

[cdeciampa]

I'm sharing two files so you don't need to modify xr.open_mfdataset and should be able to run the code as-is. They're small files, ~4 mb each. test_files.zip

[maximlt]

Thanks for sharing the test files. I'll just note that the glob pattern didn't work with these files and that the dataset has not FLUT variable. Yet I was able to reproduce the second problem reported by replacing FLUT by PRECC. It seems that HoloViews doesn't correctly retrieve the list of Matplotlib colormaps, I'll open an issue on HoloViews.

On the first issue, I realize that cmap isn't documented anywhere on the website, I'm not sure what types it accepts so one would first need to document what it accepts.