Issue about custom LLM. The example file dosn't work!

[Buzeg]

When I use the official example to use my own LLM, such as GLM-4-Plus, it prompts an error

[liukidar]

Hello there! os.getenv() is a function, so you should do model=os.getenv("...") and similar for the other parameters. Let me know if that fixes the issue.

[Buzeg]

Hello there! os.getenv() is a function, so you should do model=os.getenv("...") and similar for the other parameters. Let me know if that fixes the issue.

Hello! Thanks for your reply! But is remind me OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable I set a .env file os.environ["embed_api_key"] = "****************************" os.environ["OPENAI_API_KEY"] = "*************************" os.environ["base_url"] = "https://open.bigmodel.cn/api/paas/v4" os.environ["llm_model"] = "glm-4-flash" os.environ["embed_model"] = "embedding-3"

And load_dotenv() return TRUE How can I fix this problem! Really appreciate! @liukidar

[lawcompany-SH]

Set .env file with below format. see https://pypi.org/project/python-dotenv/

embed_api_key=*******
OPENAI_API_KEY=******
base_url=https://open.bigmodel.cn/api/paas/v4

[Buzeg]

Set .env file with below format. see https://pypi.org/project/python-dotenv/

embed_api_key=*******
OPENAI_API_KEY=******
base_url=https://open.bigmodel.cn/api/paas/v4

It still return error. .env file

embed_api_key = ***********************
OPENAI_API_KEY = ************************
base_url = https://open.bigmodel.cn/api/paas/v4
llm_model = glm-4-plus
embed_model = embedding-3

And

working_dir = r"D:\Implement\CodeData\fast-graphrag\fast"

grag = GraphRAG(
    working_dir=working_dir,
    domain=DOMAIN,
    example_queries="\n".join(QUERIES),
    entity_types=ENTITY_TYPES,
    config=GraphRAG.Config(
        llm_service=OpenAILLMService(
            model=os.getenv("llm_model"),
            base_url=os.getenv("base_url"),
            api_key=os.getenv("OPENAI_API_KEY")
        ),
        embedding_service=OpenAIEmbeddingService(
            model=os.getenv("embed_model"),
            base_url=os.getenv("base_url"),
            api_key=os.getenv("embed_api_key"),
            embedding_dim=512
        )
    )
)

return

{
    "name": "OpenAIError",
    "message": "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable",
    "stack": "---------------------------------------------------------------------------
OpenAIError                               Traceback (most recent call last)
Cell In[7], line 9
      1 working_dir = r\"D:\\Implement\\CodeData\\fast-graphrag\\fast\"
      3 grag = GraphRAG(
      4     working_dir=working_dir,
      5     domain=DOMAIN,
      6     example_queries=\"\
\".join(QUERIES),
      7     entity_types=ENTITY_TYPES,
      8     config=GraphRAG.Config(
----> 9         llm_service=OpenAILLMService(
     10             model=os.getenv(\"llm_model\"),
     11             base_url=os.getenv(\"base_url\"),
     12             api_key=os.getenv(\"OPENAI_API_KEY\")
     13         ),
     14         embedding_service=OpenAIEmbeddingService(
     15             model=os.getenv(\"embed_model\"),
     16             base_url=os.getenv(\"base_url\"),
     17             api_key=os.getenv(\"embed_api_key\"),
     18             embedding_dim=512
     19         )
     20     )
     21 )

File <string>:6, in __init__(self, model, base_url, api_key)

File d:\\Implement\\Anaconda3\\envs\\fastgr\\Lib\\site-packages\\fast_graphrag\\_llm\\_llm_openai.py:34, in OpenAILLMService.__post_init__(self)
     32 def __post_init__(self):
     33     logger.debug(\"Initialized OpenAILLMService with patched OpenAI client.\")
---> 34     self.llm_async_client: instructor.AsyncInstructor = instructor.from_openai(AsyncOpenAI(api_key=self.api_key))

File d:\\Implement\\Anaconda3\\envs\\fastgr\\Lib\\site-packages\\openai\\_client.py:319, in AsyncOpenAI.__init__(self, api_key, organization, project, base_url, timeout, max_retries, default_headers, default_query, http_client, _strict_response_validation)
    317     api_key = os.environ.get(\"OPENAI_API_KEY\")
    318 if api_key is None:
--> 319     raise OpenAIError(
    320         \"The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\"
    321     )
    322 self.api_key = api_key
    324 if organization is None:

OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable"
}

@ @lawcompany-SH

[liukidar]

Hello, I am unable to replicate your issue, would you mind sharing the full code you're using? you can use generic values for api_keys as real ones are not necessary to initialise the model.

[tmceld]

I think i am hitting the same problem, with Ollama:

from typing import List

from dotenv import load_dotenv

from fast_graphrag import GraphRAG
from fast_graphrag._llm import OpenAIEmbeddingService, OpenAILLMService

load_dotenv()

DOMAIN = "Analyze this story and identify the characters. Focus on how they interact with each other, the locations they explore, and their relationships."

EXAMPLE_QUERIES = [
    "What is the significance of shoes in Caledonian Road?",
    "How does the setting of London contribute to the story's themes?",
    "Describe the chain of events that leads to Bykov's demise.",
    "What does Capmbell's Mother represent in this story?",
]

ENTITY_TYPES = ["Character", "Animal", "Place", "Object", "Activity", "Event"]

api_key = "ollama"
working_dir = "./examples/"
grag = GraphRAG(
    working_dir=working_dir,
    domain=DOMAIN,
    example_queries="\n".join(EXAMPLE_QUERIES),
    entity_types=ENTITY_TYPES,
    config=GraphRAG.Config(
        llm_service=OpenAILLMService(
            model="llama3.2:latest",
            base_url="http://localhost:11434/v1",
            api_key=api_key,
        ),
        embedding_service=OpenAIEmbeddingService(
            model="nomic-embed-text",
            base_url="http://localhost:11434/api/embeddings/",
            api_key=api_key,
            embedding_dim=512,  # the output embedding dim of the chosen model
        ),
    ),
)

with open("./Caledonian Road_ From the award-winning au - Andrew O'Hagan.txt") as f:
    grag.insert(f.read())

print(grag.query("Who is Campbell?").response)

Giving error:

 python test.py
Traceback (most recent call last):
  File "/Users/toast/Developer/ai/fast-graphrag/test.py", line 28, in <module>
    config=GraphRAG.Config(
           ^^^^^^^^^^^^^^^^
  File "<string>", line 10, in __init__
  File "/Users/toast/Developer/ai/fast-graphrag/.venv/lib/python3.11/site-packages/fast_graphrag/__init__.py", line 69, in <lambda>
    DefaultVectorStorageConfig(embedding_dim=DefaultEmbeddingService().embedding_dim)
                                             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<string>", line 8, in __init__
  File "/Users/toast/Developer/ai/fast-graphrag/.venv/lib/python3.11/site-packages/fast_graphrag/_llm/_llm_openai.py", line 115, in __post_init__
    self.embedding_async_client: AsyncOpenAI = AsyncOpenAI(api_key=self.api_key)
                                               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/toast/Developer/ai/fast-graphrag/.venv/lib/python3.11/site-packages/openai/_client.py", line 319, in __init__
    raise OpenAIError(
openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable

[tmceld]

So i got around the above error, by exporting an OPENAI_API_KEY but not sure why i needed to do this.

I now have further errors with embedding, and am wondering given the status of embedding on ollama - has anyone actually got this to work?

[liukidar]

I think i am hitting the same problem, with Ollama:

from typing import List

from dotenv import load_dotenv

from fast_graphrag import GraphRAG
from fast_graphrag._llm import OpenAIEmbeddingService, OpenAILLMService

load_dotenv()

DOMAIN = "Analyze this story and identify the characters. Focus on how they interact with each other, the locations they explore, and their relationships."

EXAMPLE_QUERIES = [
    "What is the significance of shoes in Caledonian Road?",
    "How does the setting of London contribute to the story's themes?",
    "Describe the chain of events that leads to Bykov's demise.",
    "What does Capmbell's Mother represent in this story?",
]

ENTITY_TYPES = ["Character", "Animal", "Place", "Object", "Activity", "Event"]

api_key = "ollama"
working_dir = "./examples/"
grag = GraphRAG(
    working_dir=working_dir,
    domain=DOMAIN,
    example_queries="\n".join(EXAMPLE_QUERIES),
    entity_types=ENTITY_TYPES,
    config=GraphRAG.Config(
        llm_service=OpenAILLMService(
            model="llama3.2:latest",
            base_url="http://localhost:11434/v1",
            api_key=api_key,
        ),
        embedding_service=OpenAIEmbeddingService(
            model="nomic-embed-text",
            base_url="http://localhost:11434/api/embeddings/",
            api_key=api_key,
            embedding_dim=512,  # the output embedding dim of the chosen model
        ),
    ),
)

with open("./Caledonian Road_ From the award-winning au - Andrew O'Hagan.txt") as f:
    grag.insert(f.read())

print(grag.query("Who is Campbell?").response)

Giving error:

 python test.py
Traceback (most recent call last):
  File "/Users/toast/Developer/ai/fast-graphrag/test.py", line 28, in <module>
    config=GraphRAG.Config(
           ^^^^^^^^^^^^^^^^
  File "<string>", line 10, in __init__
  File "/Users/toast/Developer/ai/fast-graphrag/.venv/lib/python3.11/site-packages/fast_graphrag/__init__.py", line 69, in <lambda>
    DefaultVectorStorageConfig(embedding_dim=DefaultEmbeddingService().embedding_dim)
                                             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<string>", line 8, in __init__
  File "/Users/toast/Developer/ai/fast-graphrag/.venv/lib/python3.11/site-packages/fast_graphrag/_llm/_llm_openai.py", line 115, in __post_init__
    self.embedding_async_client: AsyncOpenAI = AsyncOpenAI(api_key=self.api_key)
                                               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/toast/Developer/ai/fast-graphrag/.venv/lib/python3.11/site-packages/openai/_client.py", line 319, in __init__
    raise OpenAIError(
openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable

Just to be sure, are you using the pip version or this cloned repo? It may be a bug that we fixed but didn't push to pypi.

[liukidar]

So i got around the above error, by exporting an OPENAI_API_KEY but not sure why i needed to do this.

I now have further errors with embedding, and am wondering given the status of embedding on ollama - has anyone actually got this to work?

Mmmh, looking here it seems it is supported https://github.com/ollama/ollama/issues/2416 ? But indeed we should clarify this better. In the post they also suggest to look at this: https://github.com/severian42/GraphRAG-Local-UI/blob/main/embedding_proxy.py

[tmceld]

Just to be sure, are you using the pip version or this cloned repo? It may be a bug that we fixed but didn't push to pypi.

Ah, yes i am using the pip installed version

[liukidar]

Let me know if using the repo directly fixes the problem

[jon-torres]

I am experiencing a similar issue with Azure. But it's throwing resource not found. I have installed fast-graphrag with pip.

Linux mint 22 python 3.11

from fast_graphrag import GraphRAG
from fast_graphrag._llm import OpenAILLMService, OpenAIEmbeddingService

DOMAIN = "Analyze this story and identify the characters. Focus on how they interact with each other, the locations they explore, and their relationships."

EXAMPLE_QUERIES = [
    "What is the significance of Christmas Eve in A Christmas Carol?",
    "How does the setting of Victorian London contribute to the story's themes?",
    "Describe the chain of events that leads to Scrooge's transformation.",
    "How does Dickens use the different spirits (Past, Present, and Future) to guide Scrooge?",
    "Why does Dickens choose to divide the story into \"staves\" rather than chapters?"
]

ENTITY_TYPES = ["Character", "Animal", "Place", "Object", "Activity", "Event"]

model = "gpt-4o"
base_url = "https://<resource>.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-08-01-preview"
api_key = "5xxxxxxxx"

working_dir="./book_example"
grag = GraphRAG(
    working_dir=working_dir,
    domain=DOMAIN,
    example_queries="\n".join(EXAMPLE_QUERIES),
    entity_types=ENTITY_TYPES,
    config=GraphRAG.Config(
        llm_service=OpenAILLMService(model=model, base_url=base_url, api_key=api_key),
         embedding_service=OpenAIEmbeddingService(
            model=model,
            base_url=base_url,
            api_key=api_key,
            embedding_dim=512,  # the output embedding dim of the chosen model
        ),
    ),
)

with open("./book.txt") as f:
    grag.insert(f.read())

print(grag.query("Who is Scrooge?").response)

Extracting data:   0%|                                    | 0/1 [00:00<?, ?it/s]Error during information extraction from document: Error code: 404 - {'error': {'code': '404', 'message': 'Resource not found'}}
Extracting data: 100%|████████████████████████████| 1/1 [00:09<00:00,  9.48s/it]
Error during query: Error code: 404 - {'error': {'code': '404', 'message': 'Resource not found'}}
Traceback (most recent call last):
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/instructor/retry.py", line 222, in retry_async
    response: ChatCompletion = await func(*args, **kwargs)
                               ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/openai/resources/chat/completions.py", line 1633, in create
    return await self._post(
           ^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/openai/_base_client.py", line 1838, in post
    return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/openai/_base_client.py", line 1532, in request
    return await self._request(
           ^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/openai/_base_client.py", line 1633, in _request
    raise self._make_status_error_from_response(err.response) from None
openai.NotFoundError: Error code: 404 - {'error': {'code': '404', 'message': 'Resource not found'}}

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/instructor/retry.py", line 217, in retry_async
    async for attempt in max_retries:
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/tenacity/asyncio/__init__.py", line 166, in __anext__
    do = await self.iter(retry_state=self._retry_state)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/tenacity/asyncio/__init__.py", line 153, in iter
    result = await action(retry_state)
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/tenacity/_utils.py", line 99, in inner
    return call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/tenacity/__init__.py", line 419, in exc_check
    raise retry_exc from fut.exception()
tenacity.RetryError: RetryError[<Future at 0x715107d72c50 state=finished raised NotFoundError>]

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/jontorres/Documents/development/loro/fast_graphrag/graphrag_test.py", line 47, in <module>
    print(grag.query("Who is Scrooge?").response)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/fast_graphrag/_graphrag.py", line 150, in query
    return get_event_loop().run_until_complete(_query())
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/fast_graphrag/_graphrag.py", line 146, in _query
    raise e
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/fast_graphrag/_graphrag.py", line 142, in _query
    answer = await self.async_query(query, params)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/fast_graphrag/_graphrag.py", line 168, in async_query
    extracted_entities = await self.information_extraction_service.extract_entities_from_query(
                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/fast_graphrag/_services/_information_extraction.py", line 46, in extract_entities_from_query
    entities, _ = await format_and_send_prompt(
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/fast_graphrag/_llm/_base.py", line 40, in format_and_send_prompt
    return await llm.send_message(prompt=formatted_prompt, response_model=response_model, **args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/fast_graphrag/_utils.py", line 45, in wait_func
    result = await func(*args, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/fast_graphrag/_llm/_llm_openai.py", line 80, in send_message
    llm_response: GTResponseModel = await self.llm_async_client.chat.completions.create(
                                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/instructor/client.py", line 387, in create
    return await self.create_fn(
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/instructor/patch.py", line 161, in new_create_async
    response = await retry_async(
               ^^^^^^^^^^^^^^^^^^
  File "/home/jontorres/anaconda3/envs/loro/lib/python3.11/site-packages/instructor/retry.py", line 248, in retry_async
    raise InstructorRetryException(
instructor.exceptions.InstructorRetryException: Error code: 404 - {'error': {'code': '404', 'message': 'Resource not found'}}