First of all, thanks for building this amazing library! I'm reporting a bug that causes a RecursionError to be raised in Jupyter, when printing text containing new lines within a live display context (e.g. progress.track). There has been similar bug reports in the past, but I can reproduce the issue with the latest version of rich.
Here's a minimum reproducible example. Simply run this in Jupyterlab:
from rich.progress import track
for x in track(list(range(5))):
print("first line\nlast line") # the new line is key here
This prints out:
Working... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0% -:--:--
first line
last line
last line
last line
last line
last line
last line
last line
and keeps printing last line until running into a RecursionError:
Expand for truncated traceback
```
RecursionError Traceback (most recent call last)
Cell In[1], line 4
1 from rich.progress import track
3 for x in track(list(range(5))):
----> 4 print("first line\nlast line")
File [~/Library/Python/3.10/lib/python/site-packages/rich/file_proxy.py:43](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/file_proxy.py#line=42), in FileProxy.write(self, text)
41 if lines:
42 console = self.__console
---> 43 with console:
44 output = Text("\n").join(
45 self.__ansi_decoder.decode_line(line) for line in lines
46 )
47 console.print(output)
File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:865](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=864), in Console.__exit__(self, exc_type, exc_value, traceback)
863 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
864 """Exit buffer context."""
--> 865 self._exit_buffer()
File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:823](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=822), in Console._exit_buffer(self)
821 """Leave buffer context, and render content if required."""
822 self._buffer_index -= 1
--> 823 self._check_buffer()
File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:2007](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=2006), in Console._check_buffer(self)
2004 if self.is_jupyter: # pragma: no cover
2005 from .jupyter import display
-> 2007 display(self._buffer, self._render_buffer(self._buffer[:]))
2008 del self._buffer[:]
2009 else:
File [~/Library/Python/3.10/lib/python/site-packages/rich/jupyter.py:91](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/jupyter.py#line=90), in display(segments, text)
88 try:
89 from IPython.display import display as ipython_display
---> 91 ipython_display(jupyter_renderable)
92 except ModuleNotFoundError:
93 # Handle the case where the Console has force_jupyter=True,
94 # but IPython is not installed.
95 pass
File [~/Library/Python/3.10/lib/python/site-packages/IPython/core/display_functions.py:305](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/IPython/core/display_functions.py#line=304), in display(include, exclude, metadata, transient, display_id, raw, clear, *objs, **kwargs)
302 if metadata:
303 # kwarg-specified metadata gets precedence
304 _merge(md_dict, metadata)
--> 305 publish_display_data(data=format_dict, metadata=md_dict, **kwargs)
306 if display_id:
307 return DisplayHandle(display_id)
File [~/Library/Python/3.10/lib/python/site-packages/IPython/core/display_functions.py:93](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/IPython/core/display_functions.py#line=92), in publish_display_data(data, metadata, source, transient, **kwargs)
90 if transient:
91 kwargs['transient'] = transient
---> 93 display_pub.publish(
94 data=data,
95 metadata=metadata,
96 **kwargs
97 )
File [~/Library/Python/3.10/lib/python/site-packages/ipykernel/zmqshell.py:103](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/ipykernel/zmqshell.py#line=102), in ZMQDisplayPublisher.publish(self, data, metadata, transient, update)
81 def publish(
82 self,
83 data,
(...)
86 update=False,
87 ):
88 """Publish a display-data message
89
90 Parameters
(...)
101 If True, send an update_display_data message instead of display_data.
102 """
--> 103 self._flush_streams()
104 if metadata is None:
105 metadata = {}
File [~/Library/Python/3.10/lib/python/site-packages/ipykernel/zmqshell.py:66](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/ipykernel/zmqshell.py#line=65), in ZMQDisplayPublisher._flush_streams(self)
64 def _flush_streams(self):
65 """flush IO Streams prior to display"""
---> 66 sys.stdout.flush()
67 sys.stderr.flush()
File [~/Library/Python/3.10/lib/python/site-packages/rich/file_proxy.py:53](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/file_proxy.py#line=52), in FileProxy.flush(self)
51 output = "".join(self.__buffer)
52 if output:
---> 53 self.__console.print(output)
54 del self.__buffer[:]
File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:1673](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=1672), in Console.print(self, sep, end, style, justify, overflow, no_wrap, emoji, markup, highlight, width, height, crop, soft_wrap, new_line_start, *objects)
1671 crop = False
1672 render_hooks = self._render_hooks[:]
-> 1673 with self:
1674 renderables = self._collect_renderables(
1675 objects,
1676 sep,
(...)
1681 highlight=highlight,
1682 )
1683 for hook in render_hooks:
File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:865](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=864), in Console.__exit__(self, exc_type, exc_value, traceback)
863 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
864 """Exit buffer context."""
--> 865 self._exit_buffer()
... [remaining frames truncated because they repeat]
```
My understanding of what happened is:
The rich progress bar replaces sys.stdout and sys.stderr with rich.file_proxy.FileProxy so printing doesn't mess with the progress bar placement.
FileProxy.write buffers contents and only prints when .flush() is called, or a newline is encountered, so the highlighter works correctly.
FileProxy.write calls console.print, which ends up calling IPython.display, which then ends up calling sys.stdout.flush().
Now, that actually calls FileProxy.flush. if there's a partial line in the proxy's buffer, that gets written out via console.print, and it clears the buffer after printing... You can see where this is going.
I think this is a bug in FileProxy, and there can be two solutions for this. I'm not super familiar with rich so I'm not sure which makes more sense, or if I'm missing some obscure case:
Modify FileProxy.flush so it clears the buffer before calling console.print.
Modify FileProxy.write so it accumulates lines to a local buffer. It then replaces self.__buffer only after calling console.print.
Let me know if you agree with my analysis and if any of these fixes make sense. I'd be happy to put up a PR for the fix.
github-actions[bot]
commented
5 months ago
Describe the bug
First of all, thanks for building this amazing library! I'm reporting a bug that causes a
RecursionErrorto be raised in Jupyter, when printing text containing new lines within a live display context (e.g.progress.track). There has been similar bug reports in the past, but I can reproduce the issue with the latest version of rich.Here's a minimum reproducible example. Simply run this in Jupyterlab:
This prints out:
and keeps printing
last lineuntil running into aRecursionError:Expand for truncated traceback
``` RecursionError Traceback (most recent call last) Cell In[1], line 4 1 from rich.progress import track 3 for x in track(list(range(5))): ----> 4 print("first line\nlast line") File [~/Library/Python/3.10/lib/python/site-packages/rich/file_proxy.py:43](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/file_proxy.py#line=42), in FileProxy.write(self, text) 41 if lines: 42 console = self.__console ---> 43 with console: 44 output = Text("\n").join( 45 self.__ansi_decoder.decode_line(line) for line in lines 46 ) 47 console.print(output) File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:865](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=864), in Console.__exit__(self, exc_type, exc_value, traceback) 863 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 864 """Exit buffer context.""" --> 865 self._exit_buffer() File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:823](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=822), in Console._exit_buffer(self) 821 """Leave buffer context, and render content if required.""" 822 self._buffer_index -= 1 --> 823 self._check_buffer() File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:2007](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=2006), in Console._check_buffer(self) 2004 if self.is_jupyter: # pragma: no cover 2005 from .jupyter import display -> 2007 display(self._buffer, self._render_buffer(self._buffer[:])) 2008 del self._buffer[:] 2009 else: File [~/Library/Python/3.10/lib/python/site-packages/rich/jupyter.py:91](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/jupyter.py#line=90), in display(segments, text) 88 try: 89 from IPython.display import display as ipython_display ---> 91 ipython_display(jupyter_renderable) 92 except ModuleNotFoundError: 93 # Handle the case where the Console has force_jupyter=True, 94 # but IPython is not installed. 95 pass File [~/Library/Python/3.10/lib/python/site-packages/IPython/core/display_functions.py:305](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/IPython/core/display_functions.py#line=304), in display(include, exclude, metadata, transient, display_id, raw, clear, *objs, **kwargs) 302 if metadata: 303 # kwarg-specified metadata gets precedence 304 _merge(md_dict, metadata) --> 305 publish_display_data(data=format_dict, metadata=md_dict, **kwargs) 306 if display_id: 307 return DisplayHandle(display_id) File [~/Library/Python/3.10/lib/python/site-packages/IPython/core/display_functions.py:93](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/IPython/core/display_functions.py#line=92), in publish_display_data(data, metadata, source, transient, **kwargs) 90 if transient: 91 kwargs['transient'] = transient ---> 93 display_pub.publish( 94 data=data, 95 metadata=metadata, 96 **kwargs 97 ) File [~/Library/Python/3.10/lib/python/site-packages/ipykernel/zmqshell.py:103](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/ipykernel/zmqshell.py#line=102), in ZMQDisplayPublisher.publish(self, data, metadata, transient, update) 81 def publish( 82 self, 83 data, (...) 86 update=False, 87 ): 88 """Publish a display-data message 89 90 Parameters (...) 101 If True, send an update_display_data message instead of display_data. 102 """ --> 103 self._flush_streams() 104 if metadata is None: 105 metadata = {} File [~/Library/Python/3.10/lib/python/site-packages/ipykernel/zmqshell.py:66](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/ipykernel/zmqshell.py#line=65), in ZMQDisplayPublisher._flush_streams(self) 64 def _flush_streams(self): 65 """flush IO Streams prior to display""" ---> 66 sys.stdout.flush() 67 sys.stderr.flush() File [~/Library/Python/3.10/lib/python/site-packages/rich/file_proxy.py:53](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/file_proxy.py#line=52), in FileProxy.flush(self) 51 output = "".join(self.__buffer) 52 if output: ---> 53 self.__console.print(output) 54 del self.__buffer[:] File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:1673](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=1672), in Console.print(self, sep, end, style, justify, overflow, no_wrap, emoji, markup, highlight, width, height, crop, soft_wrap, new_line_start, *objects) 1671 crop = False 1672 render_hooks = self._render_hooks[:] -> 1673 with self: 1674 renderables = self._collect_renderables( 1675 objects, 1676 sep, (...) 1681 highlight=highlight, 1682 ) 1683 for hook in render_hooks: File [~/Library/Python/3.10/lib/python/site-packages/rich/console.py:865](http://localhost:9000/lab/tree/~/Library/Python/3.10/lib/python/site-packages/rich/console.py#line=864), in Console.__exit__(self, exc_type, exc_value, traceback) 863 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 864 """Exit buffer context.""" --> 865 self._exit_buffer() ... [remaining frames truncated because they repeat] ```
My understanding of what happened is:
sys.stdoutandsys.stderrwithrich.file_proxy.FileProxyso printing doesn't mess with the progress bar placement.FileProxy.writebuffers contents and only prints when.flush()is called, or a newline is encountered, so the highlighter works correctly.FileProxy.writecallsconsole.print, which ends up callingIPython.display, which then ends up callingsys.stdout.flush().FileProxy.flush. if there's a partial line in the proxy's buffer, that gets written out viaconsole.print, and it clears the buffer after printing... You can see where this is going.I think this is a bug in
FileProxy, and there can be two solutions for this. I'm not super familiar with rich so I'm not sure which makes more sense, or if I'm missing some obscure case:FileProxy.flushso it clears the buffer before callingconsole.print.FileProxy.writeso it accumulates lines to a local buffer. It then replacesself.__bufferonly after callingconsole.print.Let me know if you agree with my analysis and if any of these fixes make sense. I'd be happy to put up a PR for the fix.
Platform
Click to expand
> What platform (Win/Linux/Mac) are you running on? What terminal software are you using? I've tested on two different platforms: macOS Sonoma 14.5, and Debian GNU/Linux 11 (bullseye). The bug only manifests in Jupyter, and I tested on Jupterlab v4.2.2. ``` ╭────────────────────