Open jguionnet opened 12 months ago
@jguionnet Yea mb, made a fix for this in the demo that I forgot to commit to the llama-index lib
Quick Fix while I get that merged:
node_parser = HierarchicalNodeParser.from_defaults(
chunk_sizes=[
large_chunk_size,
large_chunk_size // 3,
],
text_splitter_ids=[
large_chunk_size,
large_chunk_size // 3,
],
)
I tried to run it
!/usr/bin/env bash -v
set -o errexit set -o nounset set -o pipefail
assumes you have virtualenv installed and you created a virtualenv named .venv
if [ ! -d ".venv" ]; then python3.11 -m venv ".venv" source .venv/bin/activate pip install -r requirements.txt fi
source .venv/bin/activate cd llama_docs_bot python3 app.py
and got this error (after changing from .markdown_docs_reader import MarkdownDocsReader to from markdown_docs_reader import MarkdownDocsReader
Traceback (most recent call last): File "/Users/jguionnet/workspace/llama_docs_bot/llama_docs_bot/indexing.py", line 57, in get_query_engine_tool storage_context = StorageContext.from_defaults( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/llama_index/storage/storage_context.py", line 75, in from_defaults docstore = docstore or SimpleDocumentStore.from_persist_dir( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/llama_index/storage/docstore/simple_docstore.py", line 57, in from_persist_dir return cls.from_persist_path(persist_path, namespace=namespace, fs=fs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/llama_index/storage/docstore/simple_docstore.py", line 75, in from_persist_path simple_kvstore = SimpleKVStore.from_persist_path(persist_path, fs=fs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/llama_index/storage/kvstore/simple_kvstore.py", line 75, in from_persist_path with fs.open(persist_path, "rb") as f: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/fsspec/spec.py", line 1309, in open f = self._open( ^^^^^^^^^^^ File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/fsspec/implementations/local.py", line 180, in _open return LocalFileOpener(path, mode, fs=self, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/fsspec/implementations/local.py", line 298, in init self._open() File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/fsspec/implementations/local.py", line 303, in _open self.f = open(self.path, mode=self.mode) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ FileNotFoundError: [Errno 2] No such file or directory: '/Users/jguionnet/workspace/llama_docs_bot/llama_docs_bot/data_community/docstore.json'
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "/Users/jguionnet/workspace/llama_docs_bot/llama_docs_bot/app.py", line 29, in
query_engine_tools = [
^
File "/Users/jguionnet/workspace/llama_docs_bot/llama_docs_bot/app.py", line 30, in
get_query_engine_tool(directory, description) for directory, description in docs_directories.items()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/jguionnet/workspace/llama_docs_bot/llama_docs_bot/indexing.py", line 67, in get_query_engine_tool
nodes, leaf_nodes = load_markdown_docs(directory)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/jguionnet/workspace/llama_docs_bot/llama_docs_bot/indexing.py", line 51, in load_markdown_docs
nodes = node_parser.get_nodes_from_documents(documents)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/llama_index/node_parser/hierarchical.py", line 210, in get_nodes_from_documents
nodes_from_doc = self._recursively_get_nodes_from_nodes([doc], 0)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/llama_index/node_parser/hierarchical.py", line 178, in _recursively_get_nodes_from_nodes
sub_sub_nodes = self._recursively_get_nodes_from_nodes(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/llama_index/node_parser/hierarchical.py", line 178, in _recursively_get_nodes_from_nodes
sub_sub_nodes = self._recursively_get_nodes_from_nodes(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/jguionnet/workspace/llama_docs_bot/.venv/lib/python3.11/site-packages/llama_index/node_parser/hierarchical.py", line 159, in _recursively_get_nodes_from_nodes
self.text_splitter_map[self.text_splitter_ids[level]],