Closed sglickman closed 1 month ago
import dspy
from dspy import LM
from azure.ai.inference import ChatCompletionsClient
from azure.core.credentials import AzureKeyCredential
from dotenv import dotenv_values, load_dotenv
load_dotenv()
config=dotenv_values(".env")
azure_endpoint = config['AZURE_LLAMA_31_8B_ENDPOINT']
azure_key = config['AZURE_LLAMA_31_8B_KEY']
class Llama31_8B(LM):
def __init__(self):
self.provider = "default"
self.history = []
self.api_key = azure_key
self.base_url = azure_endpoint
self.client = ChatCompletionsClient(
endpoint=azure_endpoint,
credential=AzureKeyCredential(azure_key)
)
def basic_request(self, prompt: str, **kwargs):
kwargs = {**self.kwargs, **kwargs}
data = {
**kwargs,
"messages": [
{"role": "user", "content": prompt}
],
"max_tokens": 4096,
"temperature": 0.8,
"top_p": 0.1,
"presence_penalty": 0
}
response = self.client.complete(data)
self.history.append({
'prompt': prompt,
'response': response,
'kwargs': kwargs
})
return response
def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):
response = self.request(prompt, **kwargs)
return response
llama = Llama31_8B()
dspy.settings.configure(lm=llama)
predict = dspy.Predict("question -> answer")
prediction = predict(question="What is the capital of France?")
prediction.answer
Thanks @sglickman ! Check this: https://github.com/stanfordnlp/dspy/blob/main/examples/migration.ipynb
Azure AI studio allow certain models to be used as serverless deployments. In this mode, you have a target URI (e.g., https://Meta-Llama-3-1-405B-Instruct-cyy.eastus.models.ai.azure.com ) and a key, and actual consumption looks like this:
Does DSPy support using something like this as the LM? I tried to follow the Custom LM Client directions but received the following error
Full code below -- am I doing something wrong?