HallucinationAnalyzer: suggests possible attribute completions for modules of attributes that the LLMcoder hallucinated
SignatureAnalyzer can parse more mypy errors and get signatures for attributes of classes
Stable?
During evaluation, I got this error but could not reproduce it again. Perhaps Jedi cannot be run in parallel reliably?
Traceback (most recent call last):
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/cache.py", line 110, in wrapper
return dct[key]
~~~^^^^^
KeyError: ((), frozenset())
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/psaegert/miniconda3/envs/llmcoder/bin/llmcoder", line 8, in <module>
sys.exit(main())
^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/__main__.py", line 83, in main
_ = Evaluation(args.config).run(store=True, n_repeat=args.n_repeat, verbose=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/eval/evaluate.py", line 110, in run
result = self.predict(config=config, store=store, verbose=verbose)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/eval/evaluate.py", line 148, in predict
results = self.run_llmcoder(config=config, inputs=inputs, verbose=verbose)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/eval/evaluate.py", line 199, in run_llmcoder
_ = llmcoder.complete(input, n=config.get('n_choices'))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 150, in complete
completion = self.step(code, temperature, n)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 471, in step
success = self._add_message("assistant", model=self.model_feedback, temperature=temperature, n=n)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 340, in _add_message
message = self._get_completions(model, temperature, n)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 282, in _get_completions
raise exc
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 277, in _get_completions
analysis_results = future.result()
^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/concurrent/futures/_base.py", line 456, in result
return self.__get_result()
^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
raise self._exception
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/LLMCoder.py", line 419, in run_analyzers
analyzer_results[analyzer_name] = analyzer_instance.analyze(code, completion, context=analyzer_results)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/Projects/23ws-LLMcoder/src/llmcoder/analyze/HallucinationAnalyzer.py", line 76, in analyze
suggested_attributes = script.complete_search(module_of_hallucinated_attribute + '.')
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/api/__init__.py", line 351, in complete_search
return self._search_func(string, complete=True, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/utils.py", line 16, in wrapper
return list(func(*args, **kwargs))
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/api/completion.py", line 650, in search_in_module
names = [v.name for v in n.infer()]
^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/cache.py", line 112, in wrapper
result = method(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/names.py", line 607, in infer
return Importer(m.inference_state, [self.string_name], m, level=self._level).follow()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 296, in follow
return import_module_by_names(
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 376, in import_module_by_names
value_set = ValueSet.from_sets([
^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 377, in <listcomp>
import_module(
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/plugins/__init__.py", line 21, in wrapper
return built_functions[public_name](*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/plugins/flask.py", line 20, in wrapper
return callback(inference_state, import_names, module_context, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/gradual/typeshed.py", line 115, in wrapper
python_value_set = ValueSet.from_sets(
^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/base_value.py", line 430, in from_sets
for set_ in sets:
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/gradual/typeshed.py", line 116, in <genexpr>
func(inference_state, import_names, p, sys_path,)
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/imports.py", line 425, in import_module
file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/compiled/subprocess/__init__.py", line 130, in wrapper
result = self._compiled_subprocess.run(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/compiled/subprocess/__init__.py", line 227, in run
return self._send(id(inference_state), function, args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/inference/compiled/subprocess/__init__.py", line 249, in _send
is_exception, traceback, result = pickle_load(self._get_process().stdout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/psaegert/miniconda3/envs/llmcoder/lib/python3.11/site-packages/jedi/_compatibility.py", line 12, in pickle_load
return pickle.load(file)
^^^^^^^^^^^^^^^^^
_pickle.UnpicklingError: could not find MARK
New Features
HallucinationAnalyzer: suggests possible attribute completions for modules of attributes that the LLMcoder hallucinatedSignatureAnalyzercan parse more mypy errors and get signatures for attributes of classesStable?
During evaluation, I got this error but could not reproduce it again. Perhaps Jedi cannot be run in parallel reliably?