Too many request with llamaindex integration

Yes, it is possible to add the run_config argument to the evaluate function in the src/ragas/integrations/llama_index.py module to control parameters like max_workers and avoid errors such as 'Too Many Requests' or `RateLimitError' from OpenAI.

Here is an example of how you might modify the evaluate function to include the run_config argument:

from ragas.run_config import RunConfig

def evaluate(
    query_engine,
    dataset: dict,
    metrics: list[Metric],
    llm: t.Optional[LlamaindexLLM] = None,
    embeddings: t.Optional[LlamaIndexEmbeddings] = None,
    raise_exceptions: bool = True,
    column_map: t.Optional[t.Dict[str, str]] = None,
    run_config: RunConfig = None,  # Add run_config argument
) -> Result:
    column_map = column_map or {}

    # wrap llms and embeddings
    li_llm = None
    if llm is not None:
        li_llm = LlamaIndexLLMWrapper(llm)
    li_embeddings = None
    if embeddings is not None:
        li_embeddings = LlamaIndexEmbeddingsWrapper(embeddings)

    # validate and transform dataset
    if dataset is None:
        raise ValueError("Provide dataset!")

    if run_config is None:
        run_config = RunConfig()

    exec = Executor(
        desc="Running Query Engine",
        keep_progress_bar=True,
        raise_exceptions=raise_exceptions,
        run_config=run_config,  # Pass run_config to Executor
    )

    # get query
    queries = dataset["question"]
    for i, q in enumerate(queries):
        exec.submit(query_engine.aquery, q, name=f"query-{i}")

    answers: t.List[str] = []
    contexts: t.List[t.List[str]] = []
    try:
        results = exec.results()
        if results == []:
            raise ExceptionInRunner()
    except Exception as e:
        raise e
    else:
        for r in results:
            answers.append(r.response)
            contexts.append([n.node.text for n in r.source_nodes])

    # create HF dataset
    hf_dataset = Dataset.from_dict(
        {
            "question": queries,
            "contexts": contexts,
            "answer": answers,
        }
    )
    if "ground_truth" in dataset:
        hf_dataset = hf_dataset.add_column(
            name="ground_truth",
            column=dataset["ground_truth"],
            new_fingerprint=str(uuid4()),
        )

    results = ragas_evaluate(
        dataset=hf_dataset,
        metrics=metrics,
        llm=li_llm,
        embeddings=li_embeddings,
        raise_exceptions=raise_exceptions,
    )

    return results

In this example, the evaluate function accepts a run_config argument, which is then used to set the configuration for the Executor. This allows you to control parameters like max_workers and handle rate limiting errors ^[1].

To continue talking to Dosu, mention @dosu.

explodinggradients / ragas

Too many request with llamaindex integration #1076