Automatic language adaptation Bugs. Cannot do with german language

[DatTNH23]

[ ] I have checked the documentation and related resources and couldn't resolve my bug.

Describe the bug A clear and concise description of what the bug is.

Ragas version: 0.1.20 Python version: Python 3.12.6

Code to Reproduce from langchain_core.language_models import BaseLanguageModel from langchain_core.outputs.llm_result import LLMResult from langchain_core.outputs.generation import Generation from langchain_core.runnables.config import run_in_executor from langchain_core.embeddings import Embeddings from langchain_core.messages import BaseMessage from langchain_community.document_loaders import DirectoryLoader from langchain_community.embeddings import HuggingFaceEmbeddings from ragas.embeddings import LangchainEmbeddingsWrapper from ragas.testset.generator import TestsetGenerator from ragas.llms.prompt import PromptValue from ragas.testset.evolutions import simple, reasoning,conditional from typing import List, Optional, Any, Sequence import requests import json import traceback import logging

logging.basicConfig(level=logging.INFO) logger = logging.getLogger(name)

def handle_response(response_text):

Split the response into lines

lines = response_text.strip().split('\n')

# Initialize variables to store the accumulated response
accumulated_response = []
is_done = False

for line in lines:
    try:
        # Parse the JSON object from the line
        response_obj = json.loads(line)
    except json.JSONDecodeError:
        continue  # Skip lines that are not valid JSON

    # Append the response piece to the accumulated list
    if not is_done:
        accumulated_response.append(response_obj.get("response", ""))

    # Check if the response is marked as done
    if response_obj.get("done", False):
        is_done = True
        # Append the final piece to the accumulated response
        accumulated_response.append(response_obj.get("response", ""))
        break

# Join all accumulated response pieces into a single string
full_response = ''.join(accumulated_response).strip()

return full_response

class OllamaLLM(BaseLanguageModel): model: str = "llama3"

def generate_prompt(
    self,
    prompts: List[PromptValue],
    stop: Optional[List[str]] = None,
    callbacks: Optional[List[callable]] = None,
    **kwargs,
) -> LLMResult:
    url = "http://localhost:11434/api/generate"
    print("run normal generate_prompt")
    try:
        prompt = prompts[0].to_string()
        print("Prompt------------------------------")
        print(prompt)

        response = requests.post(url, json={"model": self.model, "prompt": prompt}, timeout=120,stream=False)
        response.raise_for_status()

        model_response = handle_response(response.text)
        print("Response------------------------------")
        print(model_response)
        res = Generation(text=model_response)

        return LLMResult(generations=[[res]])
    except requests.exceptions.RequestException as e:
        print(f"Error making request to Ollama API: {e}")
        return ""

async def agenerate_prompt(
    self,
    prompts: List[PromptValue],
    stop: Optional[List[str]] = None,
    callbacks: Optional[List[callable]] = None,
    **kwargs,
) -> LLMResult:
    print("run agenerate_prompt")
    results = self.generate_prompt(prompts, stop=stop, **kwargs)
    print("FINISH agenerate_prompt")
    return results
    #return LLMResult(generations=results)

def predict(
    self, text: str, *, stop: Optional[Sequence[str]] = None, **kwargs: Any
) -> str:
    return ""

def predict_messages(
    self,
    messages: List[BaseMessage],
    *,
    stop: Optional[Sequence[str]] = None,
    **kwargs: Any,
) -> BaseMessage:
    return None
async def apredict(
    self, text: str, *, stop: Optional[Sequence[str]] = None, **kwargs: Any
) -> str:
    return ""

async def apredict_messages(
    self,
    messages: List[BaseMessage],
    *,
    stop: Optional[Sequence[str]] = None,
    **kwargs: Any,
) -> BaseMessage:
    return None
def invoke():
    return None

class EmbeddingOllama(Embeddings):

url = "http://localhost:11434" + "/api/embeddings"
def __init__(self, model = 'bge-m3'): # default
    self.model = model
def embed_documents(self, texts: List[str]) -> List[List[float]]:
    res : List[str] = []
    for i, text in enumerate(texts):
        try:
            response = requests.post(self.url, json={"model": self.model, "prompt": text})
            response.raise_for_status()
            response_json = response.json()

            res.append(response_json["embedding"])
        except requests.exceptions.HTTPError as http_err:
            print(f"HTTP error occurred: {http_err}")  # Handle specific HTTP error
        except requests.exceptions.ConnectionError as conn_err:
            print(f"Connection error occurred: {conn_err}")  # Handle connection issues
        except requests.exceptions.Timeout as timeout_err:
            print(f"Timeout error occurred: {timeout_err}")  # Handle timeout errors
        except requests.exceptions.RequestException as req_err:
            print(f"An error occurred: {req_err}")
    return res

def embed_query(self, text: str) -> List[float]:
    try:
        response = requests.post(self.url, json={"model": self.model, "prompt": text})
        response.raise_for_status()
        response_json = response.json()
        return response_json["embedding"]
    except requests.exceptions.HTTPError as http_err:
        print(f"HTTP error occurred: {http_err}")  # Handle specific HTTP error
    except requests.exceptions.ConnectionError as conn_err:
        print(f"Connection error occurred: {conn_err}")  # Handle connection issues
    except requests.exceptions.Timeout as timeout_err:
        print(f"Timeout error occurred: {timeout_err}")  # Handle timeout errors
    except requests.exceptions.RequestException as req_err:
        print(f"An error occurred: {req_err}")  

async def aembed_documents(self, texts: List[str]) -> List[List[float]]:
    return await run_in_executor(None, self.embed_documents, texts)

async def aembed_query(self, text: str) -> List[float]:
    return await run_in_executor(None, self.embed_query, text)

def main(): model_name = "llama3.1"

embedding_model = "bge-m3"

langchain_llm = OllamaLLM(model=model_name)
langchain_embeddings = HuggingFaceEmbeddings(model_name="BAAI/bge-m3")
#langchain_embeddings = EmbeddingOllama(model=embedding_model)
wrapped_embeddings = LangchainEmbeddingsWrapper(langchain_embeddings)

try:
    loader = DirectoryLoader("./data")
    documents = loader.load()
    for document in documents:
        document.metadata['filename'] = document.metadata['source']
    generator = TestsetGenerator.from_langchain(
        langchain_llm,  # generator LLM
        langchain_llm,  # critic LLM
        wrapped_embeddings
    )
    try:
        language = "deutsch"
        generator.adapt(language, evolutions=[simple, reasoning,conditional])

        distributions = {
            simple: 0.4,
            reasoning: 0.3,
            conditional: 0.3
        }
        testset = generator.generate_with_langchain_docs(
            documents, 
            20, 
            distributions,
            with_debugging_logs=True, 
        )
        df = testset.to_pandas()
        df.to_excel("generate_de_results.xlsx", index=False)
    except Exception as e:
        print(f"Error in generate_with_langchain_docs: {e}")
        print(traceback.format_exc())
except Exception as e:
    print(f"An error occurred: {e}")

if name == "main": main()

Error trace Error in generate_with_langchain_docs: invalid syntax (, line 1) Traceback (most recent call last): File "/Users/nguyenthanhdat/Documents/Projekte/test/generate_ragas.py", line 195, in main generator.adapt(language, evolutions=[simple, reasoning,conditional]) File "/Users/nguyenthanhdat/Documents/Projekte/test/myenv/lib/python3.12/site-packages/ragas/testset/generator.py", line 340, in adapt self.docstore.extractor.adapt(language, cache_dir=cache_dir) File "/Users/nguyenthanhdat/Documents/Projekte/test/myenv/lib/python3.12/site-packages/ragas/testset/extractor.py", line 61, in adapt self.extractor_prompt = self.extractor_prompt.adapt( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/nguyenthanhdat/Documents/Projekte/test/myenv/lib/python3.12/site-packages/ragas/llms/prompt.py", line 246, in adapt example_dict[self.output_key] = ast.literal_eval(dict_str) ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.6/Frameworks/Python.framework/Versions/3.12/lib/python3.12/ast.py", line 66, in literal_eval node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval') ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/opt/homebrew/Cellar/python@3.12/3.12.6/Frameworks/Python.framework/Versions/3.12/lib/python3.12/ast.py", line 52, in parse return compile(source, filename, mode, flags, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1 To translate the values in the given JSON to the target language ^^^^^^^^^ SyntaxError: invalid syntax

Expected behavior I except RAGAS can generate german dataset for me.

Additional context Here I use LLM from Ollama (LLama3.1) and HuggingFace Embedding

[dosubot[bot]]

Hey there, @DatTNH23! I'm here to help you with your bug and answer any questions you may have. Let's tackle this together!

I found a similar open issue: Generate test data for 1 pdf, which is still open and was last updated on September 20, 2024. This issue was referenced by another issue: Runner thread raised exception and testset function is not accessible, last updated on September 24, 2024.

To resolve the syntax error in the generate_with_langchain_docs method when using RAGAS to generate a German dataset with Ollama LLM and HuggingFace Embedding, ensure that the method is properly indented and correctly formatted. Here is the corrected version of the generate_with_langchain_docs method:

def generate_with_langchain_docs(
    self,
    docs: t.Sequence[Document],
    test_size: int,
    distribution: QADistribution,
) -> TestDataset:
    return self.generate(docs, test_size, distribution)

Make sure that the return statement is correctly indented and placed within the method. This should resolve the syntax error ^[1][2].

Additionally, ensure that the language parameter is correctly set and used in the adapt method. The language parameter must be a string and is used to adapt various components to ensure they function correctly in the specified language ^[1][3].

If you continue to face issues, please check the implementation of the adapt method in the ragas/testset/extractor.py file:

def adapt(self, language: str, cache_dir: t.Optional[str] = None) -> None:
    """
    Adapt the extractor to a different language.
    """
    self.extractor_prompt = self.extractor_prompt.adapt(
        language, self.llm, cache_dir
    )

This method adapts the extractor to the specified language ^[4].

Lastly, ensure that the extractor_prompt.adapt method in the ragas/llms/prompt.py file correctly handles the adaptation process:

example_dict = {}
example_dict.update(
    {k: v for k, v in zip(self.input_keys, example[: len(self.input_keys)])}
)
if self.output_type.lower() == "json":
    example_dict[self.output_key] = json_loader._safe_load(example[-1], llm)
    if example_dict[self.output_key] == {}:
        dict_str = example[-1].split("(")[0].strip()
        example_dict[self.output_key] = ast.literal_eval(dict_str)
    else:
        example_dict[self.output_key] = example[-1]
if self.output_type.lower() == "json":
    output = example_dict[self.output_key]
    if isinstance(output, dict):
        assert (
            set(output.keys()) == output_keys[i]
        ), f"Adapted output keys {set(output.keys())=} do not match with the original output keys: {output_keys[i]=}"
    elif isinstance(output, list) and all(
        isinstance(item, dict) for item in output
    ):
        assert all(
            set(item.keys()) in output_keys[i] for item in output
        ), "Adapted output keys do not match with the original output keys"

This ensures that the example_dict is correctly populated and validated ^[5].

To continue talking to Dosu, mention @dosu.

[DatTNH23]

@dosu It doesn't help, those 2 Issues are not relevant to my issue

[dosubot[bot]]

To resolve the syntax error in the generate_with_langchain_docs method when using RAGAS to generate a German dataset with Ollama LLM and HuggingFace Embedding, ensure that the method is properly indented and correctly formatted. Here is the corrected version of the generate_with_langchain_docs method:

def generate_with_langchain_docs(
    self,
    docs: t.Sequence[Document],
    test_size: int,
    distribution: QADistribution,
) -> TestDataset:
    return self.generate(docs, test_size, distribution)

Make sure that the return statement is correctly indented and placed within the method. This should resolve the syntax error ^[1][2].

Additionally, ensure that the language parameter is correctly set and used in the adapt method. The language parameter must be a string and is used to adapt various components to ensure they function correctly in the specified language ^[1].

If you continue to face issues, please check the implementation of the adapt method in the ragas/testset/extractor.py file:

def adapt(self, language: str, cache_dir: t.Optional[str] = None) -> None:
    """
    Adapt the extractor to a different language.
    """
    self.extractor_prompt = self.extractor_prompt.adapt(
        language, self.llm, cache_dir
    )

This method adapts the extractor to the specified language ^[3].

Lastly, ensure that the extractor_prompt.adapt method in the ragas/llms/prompt.py file correctly handles the adaptation process:

example_dict = {}
example_dict.update(
    {k: v for k, v in zip(self.input_keys, example[: len(self.input_keys)])}
)
if self.output_type.lower() == "json":
    example_dict[self.output_key] = json_loader._safe_load(example[-1], llm)
    if example_dict[self.output_key] == {}:
        dict_str = example[-1].split("(")[0].strip()
        example_dict[self.output_key] = ast.literal_eval(dict_str)
    else:
        example_dict[self.output_key] = example[-1]
if self.output_type.lower() == "json":
    output = example_dict[self.output_key]
    if isinstance(output, dict):
        assert (
            set(output.keys()) == output_keys[i]
        ), f"Adapted output keys {set(output.keys())=} do not match with the original output keys: {output_keys[i]=}"
    elif isinstance(output, list) and all(
        isinstance(item, dict) for item in output
    ):
        assert all(
            set(item.keys()) in output_keys[i] for item in output
        ), "Adapted output keys do not match with the original output keys"

This ensures that the example_dict is correctly populated and validated ^[4].

To continue talking to Dosu, mention @dosu.