search

swarmauri / swarmauri-sdk

https://swarmauri.com
Apache License 2.0
9 stars 21 forks source link

Parameterize Shuttle AI test cases #75

Open cobycloud opened 1 month ago

cobycloud commented 1 month ago

Overview:

We want to use pytest's parameterize decorator function to enable the testing of multiple models of a provider.

For example:

We automatically test the LLM's default.

@pytest.mark.unit
def test_no_system_context():
    API_KEY = os.getenv('AI21STUDIO_API_KEY')
    model = LLM(api_key = API_KEY)
    conversation = Conversation()

    input_data = "Hello"
    human_message = HumanMessage(content=input_data)
    conversation.add_message(human_message)

    model.predict(conversation=conversation)
    prediction = conversation.get_last().content
    assert type(prediction) == str

Here we test for several different parameter names:


@pytest.mark.unit
@pytest.mark.parameterize(name, [
    "shuttle-2-turbo", "shuttle-turbo", "gpt-4o-2024-05-13", "gpt-4-turbo-2024-04-09",
    "gpt-4-0125-preview", "gpt-4-1106-preview", "gpt-4-1106-vision-preview", "gpt-4-0613",
    "gpt-4-bing", "gpt-4-turbo-bing", "gpt-4-32k-0613", "gpt-3.5-turbo-0125",
    "gpt-3.5-turbo-1106", "claude-3-opus-20240229", "claude-3-sonnet-20240229", "claude-3-haiku-20240307",
    "claude-2.1", "claude-2.0", "claude-instant-1.2", "claude-instant-1.1",
    "claude-instant-1.0", "meta-llama-3-70b-instruct", "meta-llama-3-8b-instruct", "llama-3-sonar-large-32k-online",
    "llama-3-sonar-small-32k-online", "llama-3-sonar-large-32k-chat", "llama-3-sonar-small-32k-chat", "blackbox",
    "blackbox-code", "wizardlm-2-8x22b", "wizardlm-2-70b", "dolphin-2.6-mixtral-8x7b",
    "codestral-latest", "mistral-large", "mistral-next", "mistral-medium",
    "mistral-small", "mistral-tiny", "mixtral-8x7b-instruct-v0.1", "mixtral-8x22b-instruct-v0.1",
    "mistral-7b-instruct-v0.2", "mistral-7b-instruct-v0.1", "nous-hermes-2-mixtral-8x7b", "gemini-1.5-pro-latest",
    "gemini-1.0-pro-latest", "gemini-1.0-pro-vision", "lzlv-70b", "figgs-rp", "cinematika-7b"
    ])
def test_no_system_context(name):
    API_KEY = os.getenv('AI21STUDIO_API_KEY')
    model = LLM(api_key = API_KEY)
    conversation = Conversation()

    input_data = "Hello"
    human_message = HumanMessage(content=input_data)
    conversation.add_message(human_message)

    model.predict(conversation=conversation)
    prediction = conversation.get_last().content
    assert type(prediction) == str
cobycloud commented 1 month ago

for more information on pytest parameterization see: https://github.com/swarmauri/swarmauri-sdk/issues/68#issuecomment-2288070408