Attribute Error: LM object has no attribute 'kwargs'

[Chandrak1907]

I'm trying to use DSPY with Oracle Cloud's Gen AI platform. API is here -- https://docs.oracle.com/en-us/iaas/api/#/en/generative-ai-inference/20231130/GenerateTextResult/GenerateText. Below is the code defining LM in dspy

from dsp import LM
class Oracle_Llama(LM):
def __init__(self,model, api_key):
      self.model = model
      self.api_key = api_key
      self.provider = "default"
      self.history=[]

def basic_request(self, prompt: str, **kwargs):
    endpoint = "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com"
    generative_ai_inference_client = oci.generative_ai_inference.GenerativeAiInferenceClient(config=config, service_endpoint=endpoint, retry_strategy=oci.retry.NoneRetryStrategy(), timeout=(10,240))
    generate_text_detail = oci.generative_ai_inference.models.GenerateTextDetails()
    llm_inference_request = oci.generative_ai_inference.models.LlamaLlmInferenceRequest()
    # llm_inference_request.prompt = "who is the preident of united states"
    llm_inference_request.prompt = prompt
    llm_inference_request.max_tokens = 600
    llm_inference_request.temperature = 1
    llm_inference_request.frequency_penalty = 0
    llm_inference_request.top_p = 0.75

    generate_text_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode(model_id="ocid1.generativeaimodel.oc1.us-chicago-1.modelid")
    generate_text_detail.inference_request = llm_inference_request
    generate_text_detail.compartment_id = compartment_id
    generate_text_response = generative_ai_inference_client.generate_text(generate_text_detail)

    self.history.append({
        "prompt": prompt,
        "response": generate_text_response.data.inference_response.choices[0].text,
        # "kwargs": kwargs
    })
    return generate_text_response.data.inference_response.choices[0].text

def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):
    response = self.request(prompt, **kwargs)
    return response

Sending a prompt to LLM works fine

import dspy
llama2 = Oracle_Llama(model='llama2',api_key='')
llama2("Hi there, How many world wars happened so far?")

However, below code for running chain of thought fails

from dspy import (InputField,OutputField)

class EntityRecognition(dspy.Signature):
    """You are a helpful Entity Recognition assistant. Extract entities from the sentence: Person, Race_Ethnicity, Religion, Skin_Color."""

    sentence = dspy.InputField()
    output_json = dspy.OutputField(desc="dictionary of extracted entities")

class CoT(dspy.Module):
    def __init__(self):
        super().__init__()
        self.prog = dspy.ChainOfThought(EntityRecognition)

    def forward(self, sentence):
        return self.prog(sentence=sentence)

cot_qa = CoT()

def eval_metric(given, prediction, trace=None):
    return prediction.output_json == given.answer

dev_10 = [x.with_inputs('sentence') for x in dataset[:10]]
train_10=[x.with_inputs('sentence') for x in dataset[11:30]]
from dspy.evaluate import Evaluate
evaluate = Evaluate(devset=dev_10, metric=eval_metric, num_threads=1, display_progress=True, display_table=10, temperature=0)
run_dev_eval = evaluate(cot_qa)

And, I get below errror

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
/tmp/ipykernel_18134/4197482493.py in <cell line: 1>()
----> 1 run_dev_eval = evaluate(cot_qa)

~/conda/pytorch21_p39_gpu_v1/lib/python3.9/site-packages/dspy/evaluate/evaluate.py in __call__(self, program, metric, devset, num_threads, display_progress, display_table, return_all_scores, return_outputs)
    207 
    208         if num_threads == 1:
--> 209             reordered_devset, ncorrect, ntotal = self._execute_single_thread(wrapped_program, devset, display_progress)
    210         else:
    211             reordered_devset, ncorrect, ntotal = self._execute_multi_thread(

~/conda/pytorch21_p39_gpu_v1/lib/python3.9/site-packages/dspy/evaluate/evaluate.py in _execute_single_thread(self, wrapped_program, devset, display_progress)
     82         for idx, arg in devset:
     83             with logging_redirect_tqdm():
---> 84                 example_idx, example, prediction, score = wrapped_program(idx, arg)
     85                 reordered_devset.append((example_idx, example, prediction, score))
     86                 ncorrect += score

~/conda/pytorch21_p39_gpu_v1/lib/python3.9/site-packages/dspy/evaluate/evaluate.py in wrapped_program(example_idx, example)
    194                     current_error_count = self.error_count
    195                 if current_error_count >= self.max_errors:
--> 196                     raise e
    197 
    198                 dspy.logger.error(f"Error for example in dev set: \t\t {e}")

~/conda/pytorch21_p39_gpu_v1/lib/python3.9/site-packages/dspy/evaluate/evaluate.py in wrapped_program(example_idx, example)
    176 
    177             try:
--> 178                 prediction = program(**example.inputs())
    179                 score = metric(
    180                     example,

~/conda/pytorch21_p39_gpu_v1/lib/python3.9/site-packages/dspy/primitives/program.py in __call__(self, *args, **kwargs)
     24 
     25     def __call__(self, *args, **kwargs):
---> 26         return self.forward(*args, **kwargs)
     27 
     28     def named_predictors(self):

/tmp/ipykernel_18134/909339388.py in forward(self, sentence)
      5 
      6     def forward(self, sentence):
----> 7         return self.prog(sentence=sentence)

~/conda/pytorch21_p39_gpu_v1/lib/python3.9/site-packages/dspy/primitives/program.py in __call__(self, *args, **kwargs)
     24 
     25     def __call__(self, *args, **kwargs):
---> 26         return self.forward(*args, **kwargs)
     27 
     28     def named_predictors(self):

~/conda/pytorch21_p39_gpu_v1/lib/python3.9/site-packages/dspy/predict/chain_of_thought.py in forward(self, **kwargs)
     34 
     35         signature = kwargs.pop("new_signature", self._predict.extended_signature if self.activated else self.signature)
---> 36         return self._predict(signature=signature, **kwargs)
     37 
     38     @property

~/conda/pytorch21_p39_gpu_v1/lib/python3.9/site-packages/dspy/predict/predict.py in __call__(self, **kwargs)
     89 
     90     def __call__(self, **kwargs):
---> 91         return self.forward(**kwargs)
     92 
     93     def forward(self, **kwargs):

~/conda/pytorch21_p39_gpu_v1/lib/python3.9/site-packages/dspy/predict/predict.py in forward(self, **kwargs)
    106         # If temperature is 0.0 but its n > 1, set temperature to 0.7.
    107         temperature = config.get("temperature")
--> 108         temperature = lm.kwargs["temperature"] if temperature is None else temperature
    109 
    110         num_generations = config.get("n")

AttributeError: 'Oracle_Llama' object has no attribute 'kwargs'

[okhat]

Use dspy.LM and give it a model name in accordance to the LiteLLM model names

[Chandrak1907]

Thank you @okhat , I already did that. Here is the gist file https://gist.github.com/Chandrak1907/bfe6d8dc71f173882bc6caf3412b74d6 with an attempt of the code to reproduce this blog https://drchrislevy.github.io/posts/dspy/dspy.html

[Chandrak1907]

Below is the further exploration that I've done

In '~/miniconda3/envs/py310/lib/python3.10/site-packages/dspy/predict/predict.py', commented lines 107,108 and 109 and hard-coded temperature and num_generations

temperature = 0
num_generations = 1

This results in answer that produces characters from the response

... completions ...  [{'answer': '('}, {'answer': 'C'}, ...]
 ... pred ...  Prediction(
    answer='(',
    completions=Completions(...)
) (1021 completions omitted)

This could be traced to

line 74 in /miniconda3/envs/py311/lib/python3.11/site-packages/dsp/primitives/predict.py:

completions: list[Example] = [template.extract(example, p) for p in completions]

[okhat]

@Chandrak1907 No I'm suggesting dspy.LM, e.g.

model_name = "provider_name/model_name"

lm = dspy.LM(model_name, api_base="....", api_key="....")
dspy.configure(lm=lm)

[gokulsodar]

I did this and it worked for me, inside the init function

self.kwargs = kwargs