dosubot[bot]
commented
3 months ago Hey there, @psureshmagadi17! I'm here to help you out. If you need assistance with bugs, questions, or contributions, feel free to ask.
To ensure that the search_client is correctly propagated to the VectorStoreIndex and avoid the AttributeError: 'NoneType' object has no attribute 'search' when using RetrieverEvaluator, you need to make sure that the search_client is properly initialized and assigned within the AzureAISearchVectorStore class.
Here is the relevant part of the AzureAISearchVectorStore class constructor that handles the initialization of the search_client:
def __init__(
self,
search_or_index_client: Any,
id_field_key: str,
chunk_field_key: str,
embedding_field_key: str,
metadata_string_field_key: str,
doc_id_field_key: str,
filterable_metadata_field_keys: Optional[
Union[
List[str],
Dict[str, str],
Dict[str, Tuple[str, MetadataIndexFieldType]],
]
] = None,
index_name: Optional[str] = None,
index_mapping: Optional[
Callable[[Dict[str, str], Dict[str, Any]], Dict[str, str]]
] = None,
index_management: IndexManagement = IndexManagement.NO_VALIDATION,
embedding_dimensionality: int = 1536,
vector_algorithm_type: str = "exhaustiveKnn",
language_analyzer: str = "en.lucene",
**kwargs: Any,
) -> None:
import_err_msg = (
"`azure-search-documents` package not found, please run "
"`pip install azure-search-documents==11.4.0`"
)
try:
import azure.search.documents # noqa
from azure.search.documents import SearchClient
from azure.search.documents.indexes import SearchIndexClient
except ImportError:
raise ImportError(import_err_msg)
self._index_client: SearchIndexClient = cast(SearchIndexClient, None)
self._search_client: SearchClient = cast(SearchClient, None)
self.embedding_dimensionality = embedding_dimensionality
if search_or_index_client is not None:
if isinstance(search_or_index_client, SearchIndexClient):
self._index_client = cast(SearchIndexClient, search_or_index_client)
if not index_name:
raise ValueError(
"index_name must be supplied if search_or_index_client is of "
"type azure.search.documents.SearchIndexClient"
)
self._search_client = self._index_client.get_search_client(
index_name=index_name
)
elif isinstance(search_or_index_client, SearchClient):
self._search_client = cast(SearchClient, search_or_index_client)
if index_name:
raise ValueError(
"index_name cannot be supplied if search_or_index_client "
"is of type azure.search.documents.SearchClient"
)
if not self._index_client and not self._search_client:
raise ValueError(
"search_or_index_client must be of type "
"azure.search.documents.SearchClient or "
"azure.search.documents.SearchIndexClient"
)
else:
raise ValueError("search_or_index_client not specified")
if (
index_management == IndexManagement.CREATE_IF_NOT_EXISTS
and not self._index_client
):
raise ValueError(
"index_management has value of IndexManagement.CREATE_IF_NOT_EXISTS "
"but search_or_index_client is not of type "
"azure.search.documents.SearchIndexClient"
)
self._index_management = index_management
field_mapping = {
"id": id_field_key,
"chunk": chunk_field_key,
"embedding": embedding_field_key,
"metadata": metadata_string_field_key,
"doc_id": doc_id_field_key,
}
self._field_mapping = field_mapping
self._index_mapping = (
self._default_index_mapping if index_mapping is None else index_mapping
)
self._metadata_to_index_field_map = self._normalise_metadata_to_index_fields(
filterable_metadata_field_keys
)
if self._index_management == IndexManagement.CREATE_IF_NOT_EXISTS:
if index_name:
self._create_index_if_not_exists(index_name)
if self._index_management == IndexManagement.VALIDATE_INDEX:
self._validate_index(index_name)Additionally, you can refer to the test cases provided in the llama-index-legacy/tests/vector_stores/test_azureaisearch.py file to ensure that the search_client is correctly mocked and tested. Here is an example of how to create a mock vector store and test the initialization:
from typing import Any, List, Optional
from unittest.mock import MagicMock
import pytest
from llama_index.legacy.schema import NodeRelationship, RelatedNodeInfo, TextNode
from llama_index.legacy.vector_stores.azureaisearch import (
AzureAISearchVectorStore,
IndexManagement,
)
try:
from azure.search.documents import SearchClient
from azure.search.documents.indexes import SearchIndexClient
azureaisearch_installed = True
except ImportError:
azureaisearch_installed = False
search_client = None # type: ignore
def create_mock_vector_store(
search_client: Any,
index_name: Optional[str] = None,
index_management: IndexManagement = IndexManagement.NO_VALIDATION,
) -> AzureAISearchVectorStore:
return AzureAISearchVectorStore(
search_or_index_client=search_client,
id_field_key="id",
chunk_field_key="content",
embedding_field_key="embedding",
metadata_string_field_key="metadata",
doc_id_field_key="doc_id",
filterable_metadata_field_keys=[], # Added to match the updated constructor
index_name=index_name,
index_management=index_management,
embedding_dimensionality=2, # Assuming a dimensionality of 2 for simplicity
)
def create_sample_documents(n: int) -> List[TextNode]:
nodes: List[TextNode] = []
for i in range(n):
nodes.append(
TextNode(
text=f"test node text {i}",
relationships={
NodeRelationship.SOURCE: RelatedNodeInfo(node_id=f"test doc id {i}")
},
embedding=[0.5, 0.5],
)
)
return nodes
@pytest.mark.skipif(
not azureaisearch_installed, reason="azure-search-documents package not installed"
)
def test_azureaisearch_add_two_batches() -> None:
search_client = MagicMock(spec=SearchClient)
vector_store = create_mock_vector_store(search_client)
nodes = create_sample_documents(11)
ids = vector_store.add(nodes)
call_count = search_client.merge_or_upload_documents.call_count
assert ids is not None
assert len(ids) == 11
assert call_count == 2
@pytest.mark.skipif(
not azureaisearch_installed, reason="azure-search-documents package not installed"
)
def test_azureaisearch_add_one_batch() -> None:
search_client = MagicMock(spec=SearchClient)
vector_store = create_mock_vector_store(search_client)
nodes = create_sample_documents(10)
ids = vector_store.add(nodes)
call_count = search_client.merge_or_upload_documents.call_count
assert ids is not None
assert len(ids) == 10
assert call_count == 1
@pytest.mark.skipif(
not azureaisearch_installed, reason="azure-search-documents package not installed"
)
def test_invalid_index_management_for_searchclient() -> None:
search_client = MagicMock(spec=SearchClient)
# No error
create_mock_vector_store(
search_client, index_management=IndexManagement.VALIDATE_INDEX
)
# Cannot supply index name
# ruff: noqa: E501
with pytest.raises(
ValueError,
match="index_name cannot be supplied if search_or_index_client is of type azure.search.documents.SearchClient",
):
create_mock_vector_store(search_client, index_name="test01")
# SearchClient cannot create an index
with pytest.raises(ValueError):
create_mock_vector_store(
search_client,
index_management=IndexManagement.CREATE_IF_NOT_EXISTS,
)
@pytest.mark.skipif(
not azureaisearch_installed, reason="azure-search-documents package not installed"
)
def test_invalid_index_management_for_searchindexclient() -> None:
search_client = MagicMock(spec=SearchIndexClient)
# Index name must be supplied
with pytest.raises(
ValueError,
match="index_name must be supplied if search_or_index_client is of type azure.search.documents.SearchIndexClient",
):
create_mock_vector_store(
search_client, index_management=IndexManagement.VALIDATE_INDEX
)
# No error when index name is supplied with SearchIndexClient
create_mock_vector_store(
search_client,
index_name="test01",
index_management=IndexManagement.CREATE_IF_NOT_EXISTS,
)Ensure that the search_or_index_client is correctly passed and that the search_client is properly initialized within the constructor. This will prevent the NoneType error when the RetrieverEvaluator tries to use the search_client. Additionally, make sure to handle both synchronous and asynchronous clients as shown in the updated constructor and test cases [1][2][3][4].
To continue talking to Dosu, mention @dosu.
psureshmagadi17
Bug Description
vector_store_address=
vector_store_password=
search_client = SearchClient(
endpoint=vector_store_address,
index_name=INDEX_NAME,
credential=AzureKeyCredential(vector_store_password)
)
Define a query that matches all documents
query = "*"
Set the API version
api_version =
search_mode = "all"
Search for documents with the query
response = search_client.search( search_text=query, search_mode=search_mode )
docs = [i for i in response]
llama_docs = []
for doc in docs: llama_docs.append( llamadoc( id=doc['id'], text= doc['content'], metadata={
"filename": doc['filename'], "file_path": doc['file_path'], "a": doc['a'], "b": doc['b'], "c": doc['c'], "page_number": doc['page_number'], }, ) )
Settings.embed_model = AzureOpenAIEmbedding( model=conf.openai_embedding_model, deployment_name=conf.openai_embedding_model, api_key=conf.openai_api_key, api_base=conf.openai_endpoint, azure_endpoint=conf.openai_endpoint, api_version=conf.openai_version, async_http_client=httpx.AsyncClient(proxies=),
http_client=httpx.Client(proxies=),
)
Settings.llm = AzureOpenAI( engine=OPENAI_GPT_4, model=OPENAI_GPT_4, temperature=0.0, azure_endpoint=conf.openai_endpoint, api_key=conf.openai_api_key, api_version=conf.openai_version, http_client=httpx.Client(proxies=),
)
sample_llama_docs = random.sample(llama_docs, 50)
llama_vector_store = AzureAISearchVectorStore( search_or_index_client=search_client, filterable_metadata_field_keys=['filename', 'a', 'b', 'c'], id_field_key="id", chunk_field_key="content", embedding_field_key="content_vector", doc_id_field_key="id", metadata_string_field_key="metadata" )
storage_context = StorageContext.from_defaults(vector_store=llama_vector_store)
index = VectorStoreIndex.from_documents( [], storage_context=storage_context, embed_model=Settings.embed_model, )
relevant_ids = [i for i in dict(qa_dataset)['relevant_docs'].keys()]
g,s,f=qa_dataset.relevant_docs[relevantids[0]][0].split('')[:3]
a_filter = MetadataFilter(key=a, operator=FilterOperator.EQUAL_TO, value=g) b_filter = MetadataFilter(key=b, operator=FilterOperator.EQUAL_TO, value=s) c_filter = MetadataFilter(key=c, operator=FilterOperator.EQUAL_TO, value=f)
filter_names = [a_filter, b_filter, c_filter] filters = MetadataFilters(filters=filter_names, condition=FilterCondition.AND)
test_llama_retriver = index.as_retriever(similarity_top_k=3, filters=filters)
this works
print(test_llama_retriver.retrieve('Whats the name?')) print("\n\n")
retriever_evaluator = RetrieverEvaluator.from_metric_names( ["mrr", "hit_rate", "precision", "recall"], retriever=test_llama_retriver )
sample_id=relevant_ids[0] sample_query = qa_dataset.queries[sample_id] sample_expected = qa_dataset.relevant_docs[sample_id]
nest_asyncio.apply()
eval_result = retriever_evaluator.evaluate(sample_query,sample_expected)
print(eval_result)
Issue:
eval_result = retriever_evaluator.evaluate(sample_query,sample_expected)triggers "AttributeError: 'NoneType' object has no attribute 'search'After tracing the code, the below function causes the issue -
`async def _acreate_query_result( self, search_query: str, vectors: Optional[List[Any]] ) -> VectorStoreQueryResult: results = await self._search_client.search( search_text=search_query, vector_queries=vectors, top=self._query.similarity_top_k, select=self._select_fields, filter=self._odata_filter, )
`
This is because I need to convert
AzureAISearchVectorStoreto aVectorStoreIndexobject to useas_retrieverand thesearch_clientpassed to instantiateAzureAISearchVectorStoredoes not get propagated to VectorStoreIndexI even tried manually overriding
index._search_client = search_clientbut does not workllama-index versions:
@dosubot
Version
0.10.62
Steps to Reproduce
vector_store_address=
vector_store_password=
search_client = SearchClient(
endpoint=vector_store_address,
index_name=INDEX_NAME,
credential=AzureKeyCredential(vector_store_password)
)
Define a query that matches all documents
query = "*"
Set the API version
api_version =
search_mode = "all"
Search for documents with the query
response = search_client.search( search_text=query, search_mode=search_mode )
docs = [i for i in response]
llama_docs = []
for doc in docs: llama_docs.append( llamadoc( id=doc['id'], text= doc['content'], metadata={
"filename": doc['filename'], "file_path": doc['file_path'], "a": doc['a'], "b": doc['b'], "c": doc['c'], "page_number": doc['page_number'], }, ) )
Settings.embed_model = AzureOpenAIEmbedding( model=conf.openai_embedding_model, deployment_name=conf.openai_embedding_model, api_key=conf.openai_api_key, api_base=conf.openai_endpoint, azure_endpoint=conf.openai_endpoint, api_version=conf.openai_version, async_http_client=httpx.AsyncClient(proxies=),
http_client=httpx.Client(proxies=),
)
Settings.llm = AzureOpenAI( engine=OPENAI_GPT_4, model=OPENAI_GPT_4, temperature=0.0, azure_endpoint=conf.openai_endpoint, api_key=conf.openai_api_key, api_version=conf.openai_version, http_client=httpx.Client(proxies=),
)
sample_llama_docs = random.sample(llama_docs, 50)
llama_vector_store = AzureAISearchVectorStore( search_or_index_client=search_client, filterable_metadata_field_keys=['filename', 'a', 'b', 'c'], id_field_key="id", chunk_field_key="content", embedding_field_key="content_vector", doc_id_field_key="id", metadata_string_field_key="metadata" )
storage_context = StorageContext.from_defaults(vector_store=llama_vector_store)
index = VectorStoreIndex.from_documents( [], storage_context=storage_context, embed_model=Settings.embed_model, )
relevant_ids = [i for i in dict(qa_dataset)['relevant_docs'].keys()]
g,s,f=qa_dataset.relevant_docs[relevantids[0]][0].split('')[:3]
a_filter = MetadataFilter(key=a, operator=FilterOperator.EQUAL_TO, value=g) b_filter = MetadataFilter(key=b, operator=FilterOperator.EQUAL_TO, value=s) c_filter = MetadataFilter(key=c, operator=FilterOperator.EQUAL_TO, value=f)
filter the retriever to respond from relevant document chunks
filter_names = [a_filter, b_filter, c_filter] filters = MetadataFilters(filters=filter_names, condition=FilterCondition.AND)
test_llama_retriver = index.as_retriever(similarity_top_k=3, filters=filters)
this works
print(test_llama_retriver.retrieve('Whats the patients name?')) print("\n\n")
retriever_evaluator = RetrieverEvaluator.from_metric_names( ["mrr", "hit_rate", "precision", "recall"], retriever=test_llama_retriver )
sample_id=relevant_ids[0] sample_query = qa_dataset.queries[sample_id] sample_expected = qa_dataset.relevant_docs[sample_id]
nest_asyncio.apply()
eval_result = retriever_evaluator.evaluate(sample_query,sample_expected)
print(eval_result)
Relevant Logs/Tracbacks
No response