I experienced stability issues when using the HallucinationAnalyzer and the JediAnalyzer, both of which use Jedi.
The issue seems to be related to the use of pickle in Jedi. Perhaps Jedi uses a cache or subprocesses on their own, which may interfere with our multi-threaded use of Jedi.
I'm currently stress testing Jedi and trying to replicate the error in a controlled manner.
Traceback (most recent call last):
File "/home/psaegert/miniconda3/envs/llmcoder/bin/llmcoder", line 8, in <module>
sys.exit(main())
^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/__main__.py", line 83, in main
_ = Evaluation(args.config).run(store=True, n_repeat=args.n_repeat, verbose=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/eval/evaluate.py", line 110, in run
result = self.predict(config=config, store=store, verbose=verbose)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/eval/evaluate.py", line 148, in predict
results = self.run_llmcoder(config=config, inputs=inputs, verbose=verbose)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/eval/evaluate.py", line 199, in run_llmcoder
_ = llmcoder.complete(input, n=config.get('n_choices'))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 150, in complete
completion = self.step(code, temperature, n)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 471, in step
success = self._add_message("assistant", model=self.model_feedback, temperature=temperature, n=n)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 340, in _add_message
message = self._get_completions(model, temperature, n)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 282, in _get_completions
raise exc
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 277, in _get_completions
analysis_results = future.result()
^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/concurrent/futures/_base.py", line 456, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
raise self._exception
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 419, in run_analyzers
analyzer_results[analyzer_name] = analyzer_instance.analyze(code, completion, context=analyzer_results)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/analyze/JediAnalyzer.py", line 83, in analyze
if "def" not in name.description and "class" not in name.description:
^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/api/classes.py", line 349, in description
typ = self.type
^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/api/classes.py", line 191, in type
for value in self._name.infer():
^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/names.py", line 281, in infer
return tree_name_to_values(
^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/plugins/__init__.py", line 21, in wrapper
return built_functions[public_name](*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/plugins/stdlib.py", line 867, in wrapper
return func(inference_state, context, tree_name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/plugins/django.py", line 177, in wrapper
result = func(inference_state, context, tree_name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/syntax_tree.py", line 751, in tree_name_to_values
types = imports.infer_import(context, tree_name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/cache.py", line 44, in wrapper
rv = function(obj, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 53, in infer_import
_prepare_infer_import(module_context, tree_name)
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 116, in _prepare_infer_import
return from_import_name, tuple(import_path), import_node.level, importer.follow()
^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 296, in follow
return import_module_by_names(
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 376, in import_module_by_names
value_set = ValueSet.from_sets([
^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 377, in <listcomp>
import_module(
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/plugins/__init__.py", line 21, in wrapper
return built_functions[public_name](*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/plugins/flask.py", line 20, in wrapper
return callback(inference_state, import_names, module_context, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/gradual/typeshed.py", line 115, in wrapper
python_value_set = ValueSet.from_sets(
^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/base_value.py", line 430, in from_sets
for set_ in sets:
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/gradual/typeshed.py", line 116, in <genexpr>
func(inference_state, import_names, p, sys_path,)
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 411, in import_module
file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/compiled/subprocess/__init__.py", line 130, in wrapper
result = self._compiled_subprocess.run(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/compiled/subprocess/__init__.py", line 227, in run
return self._send(id(inference_state), function, args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/compiled/subprocess/__init__.py", line 249, in _send
is_exception, traceback, result = pickle_load(self._get_process().stdout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/_compatibility.py", line 12, in pickle_load
return pickle.load(file)
^^^^^^^^^^^^^^^^^
_pickle.UnpicklingError: unpickling stack underflow
I experienced stability issues when using the
HallucinationAnalyzerand theJediAnalyzer, both of which use Jedi.The issue seems to be related to the use of pickle in Jedi. Perhaps Jedi uses a cache or subprocesses on their own, which may interfere with our multi-threaded use of Jedi.
I'm currently stress testing Jedi and trying to replicate the error in a controlled manner.