simonw
commented
1 year ago Getting prompts out of the model was very easy:
gpt_model = GPT4All(self.model.filename())
output = gpt_model.generate(prompt.prompt, streaming=True)
# yield from outputWhat was harder is that the underlying C++ libraries log a LOT of stuff to stdout and stderr, in a way that can't be easily surpressed from Python.
With GPT-4's help I figured out this pattern:
class SuppressOutput:
def __enter__(self):
# Save a copy of the current file descriptors for stdout and stderr
self.stdout_fd = os.dup(1)
self.stderr_fd = os.dup(2)
# Open a file to /dev/null
self.devnull_fd = os.open(os.devnull, os.O_WRONLY)
# Replace stdout and stderr with /dev/null
os.dup2(self.devnull_fd, 1)
os.dup2(self.devnull_fd, 2)
# Writes to sys.stdout and sys.stderr should still work
self.original_stdout = sys.stdout
self.original_stderr = sys.stderr
sys.stdout = os.fdopen(self.stdout_fd, "w")
sys.stderr = os.fdopen(self.stderr_fd, "w")
def __exit__(self, exc_type, exc_val, exc_tb):
# Restore stdout and stderr to their original state
os.dup2(self.stdout_fd, 1)
os.dup2(self.stderr_fd, 2)
# Close the saved copies of the original stdout and stderr file descriptors
os.close(self.stdout_fd)
os.close(self.stderr_fd)
# Close the file descriptor for /dev/null
os.close(self.devnull_fd)
# Restore sys.stdout and sys.stderr
sys.stdout = self.original_stdout
sys.stderr = self.original_stderrUsed like this:
class Response(llm.Response):
def iter_prompt(self, prompt):
with SuppressOutput():
gpt_model = GPT4All(self.model.filename())
output = gpt_model.generate(prompt.prompt, streaming=True)
yield from outputI wanted sys.stdout and sys.stderr to continue working when Python code called them mainly to ensure the download progress bar still displayed if the model needed to be downloaded.
Using https://docs.gpt4all.io/gpt4all_python.html