EjbejaranosAI
commented
1 month ago Hi @Otavio-Parraga,
I identified some errors in the implementation and made a few modifications to the script. However, after running the code, I noticed that all test results are returning 0. I'm unsure if this issue is due to a configuration problem or something else. Any guidance would be appreciated .
from huggingface_hub import login
from transformers import AutoModelForCausalLM, AutoTokenizer
from deepeval.benchmarks.mmlu.mmlu import MMLU
from deepeval.models.base_model import DeepEvalBaseLLM
login(token="your_hf_token")`
class Mistral7B(DeepEvalBaseLLM):
def __init__(self, model, tokenizer):
self.model = model
self.tokenizer = tokenizer
def load_model(self):
# Move the model to the GPU when loading
return self.model.to("cuda")
def generate(self, prompt: str) -> str:
model = self.load_model()
device = "cuda"
model_inputs = self.tokenizer([prompt], return_tensors="pt", padding=True, truncation=True).to(device)
generated_ids = model.generate(**model_inputs, max_new_tokens=100, do_sample=True)
return self.tokenizer.batch_decode(generated_ids)[0]
async def a_generate(self, prompt: str) -> str:
return self.generate(prompt)
def batch_generate(self, prompts):
model = self.load_model()
device = "cuda"
model_inputs = self.tokenizer(prompts, return_tensors="pt", padding=True, truncation=True).to(device)
generated_ids = model.generate(**model_inputs, max_new_tokens=100, do_sample=True)
return self.tokenizer.batch_decode(generated_ids)
def get_model_name(self):
return "Mistral 7B"
def __call__(self, prompt: str) -> str:
return self.generate(prompt)
model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1")
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
#model = AutoModelForCausalLM.from_pretrained("oshizo/japanese-e5-mistral-1.9b")
#tokenizer = AutoTokenizer.from_pretrained("oshizo/japanese-e5-mistral-1.9b")
# Asignar el pad_token como eos_token
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
mistral_7b = Mistral7B(model=model, tokenizer=tokenizer)
print(mistral_7b("Write me a joke"))
benchmark = MMLU()
results = benchmark.evaluate(model=mistral_7b, batch_size=5)
print("Overall Score: ", results)
print("Overall Score:", benchmark.overall_score)
print("Task-specific Scores: ", benchmark.task_scores)
svnv-svsv-jm
Describe the bug After setting the code, I always get an error with the following description:
AttributeError: 'list' object has no attribute 'find'
I tried to run the code on my local server, with some modifications, and thus on Google Colab, just copying and pasting the website example (https://docs.confident-ai.com/docs/guides-using-custom-llms#creating-a-custom-llm)
Running it on server leads to the same error, and with other metrics (such as SummarizationMetric) I get the same error
AttributeError: 'list' object has no attribute 'find'but replacing find by claimTo Reproduce Google Colab with code: https://colab.research.google.com/drive/1JbzWggqaxMKakQSzuVSYXN1J-96ttjOk?usp=sharing
Expected behavior The lib should evaluate the metric.
Screenshots If applicable, add screenshots to help explain your problem.
Additional context The full error stack trace: