AWSMeta incorrectly raises max_tokens error

[d2kagw]

I have a basic example of DSPy where I am hoping to compare different LLM performance. In the example I hope to try running llama3 8b and 70b, however, 70b throws an error re: max_tokens whereas the 8b model (or other LLMs like Claude and GPT) works file.

Here's the 70b that fails:

Here's the 8b that succeeds:

The issue still occurs on 2.9.12

[arnavsinghvi11]

Hi @d2kagw , can you try initializing the models with max_new_tokens as the parameter? Additionally, does this error persist if you replace dspy.Predict with dspy.ChainOfThought?

[d2kagw]

hi @arnavsinghvi11 , here's a copy of the code that recreates the issue for me:

Note that putting max_new_tokens has no impact on the error. Nor does using a ChainOfThought vs. Predict.

# -------------------------
# Setup Environment
# %pip install dspy-ai==2.4.9 boto3

# Check creds have been properly set
import os
def obsfucate(var): 
    return f"{var[:4]}{'*' * (len(var) - 4)}"

def check_env_vars():
    aws_vars = ['OPENAI_API_KEY', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN']
    for var in aws_vars:
        value = os.environ.get(var)
        if value:
            print(f"{var} is set. Value: {obsfucate(value)}")
        else:
            print(f"{var} is not set.")

check_env_vars()

# Check the version of dspy running
from importlib.metadata import version
version("dspy-ai")

# -------------------------
# Check the AWS credentials (this has been a constant issue)
import boto3

# Create a Boto3 session
session = boto3.Session()

# Get the credentials from the session
credentials = session.get_credentials()

# Access the access key ID and secret access key
access_key = credentials.access_key
secret_key = credentials.secret_key

# Print the credentials
print(f"Access Key ID: Value: {obsfucate(access_key)}")
print(f"Secret Access Key: {obsfucate(secret_key)}")

# Create an S3 client
s3 = boto3.client('s3')

# List S3 buckets
response = s3.list_buckets()

# Iterate over the buckets and print their names
for bucket in response['Buckets']:
    print(bucket['Name'])

# -------------------------
# Define LLMs
import dspy

# Use GPT4
openai_gpt = dspy.OpenAI(model='gpt-4', max_tokens=1000, api_key=os.environ.get('OPENAI_API_KEY'))

aws_provider_ue1 = dspy.Bedrock(region_name="us-east-1")
aws_provider_uw2 = dspy.Bedrock(region_name="us-west-2")

aws_llms = {
    "claude_35": dspy.AWSAnthropic(
        aws_provider=aws_provider_ue1,
        model="anthropic.claude-3-5-sonnet-20240620-v1:0",
    ),
    "llama70b":  dspy.AWSMeta(
        aws_provider=aws_provider_uw2,
        model="meta.llama3-70b-instruct-v1:0",
        max_tokens=1000,
    ),
    "llama8b":  dspy.AWSMeta(
        aws_provider=aws_provider_uw2,
        model="meta.llama3-8b-instruct-v1:0",
        max_tokens=1000,
    ),
    "claude_3_haiku": dspy.AWSAnthropic(
        aws_provider=aws_provider_uw2,
        model="anthropic.claude-3-haiku-20240307-v1:0"
    )
}

# -------------------------
# Choose the LLM to use
# lm = openai_gpt
lm = aws_llms['llama70b']

# set the LLM
dspy.settings.configure(lm=lm)

# -------------------------
# A basic LLM test to ensure the notebook is prperly configured
sentence = "it's a charming and often affecting journey."

classify = dspy.Predict('sentence -> sentiment')
print(classify)

sent = classify(sentence=sentence).sentiment

lm.inspect_history(n=1)

if sent == "Positive":
  print('LLM Test: ✅ Looking good')
else:
  raise ValueError(f'LLM Test: ❌ LLM did not respond as expected: Got "{sent}" expected "Positive"')

# -------------------------
# A Signature version
class Emotion(dspy.Signature):
    # Describe the task
    """Classify emotions in a sentence."""

    sentence = dspy.InputField()
    # Adding description to the output field
    sentiment = dspy.OutputField(desc="Possible choices: sadness, joy, love, anger, fear, surprise.")

classify_class_based = dspy.Predict(Emotion)

# Issue prediction
classify_class_based(sentence=sentence).sentiment