Tools: Nested Properties

[retendo]

I would like to specify a nested input schema. It seems that is not possible at the moment.

Would you try to support that? I think there's a lot of use cases for this, when you want to let the model create structured data for you.

[retendo]

I use https://github.com/betalgo/openai for working with OpenAI endpoints and would like to at least use the same input schema for my function calls, to be able to compare them more easily.

[retendo]

What I would need is basically the last example on this documentation page (JSON mode): https://docs.anthropic.com/claude/docs/tool-use-examples

[tghamm]

Working on something like this:

public class ImageSchema
{
    [JsonPropertyName("type")]
    public string Type { get; set; } = "object";

    [JsonPropertyName("required")]
    public string[] Required { get; set; }
    [JsonPropertyName("properties")]
    public Properties Properties { get; set; }
}

public class Properties
{
    [JsonPropertyName("key_colors")]
    public KeyColorsProperty KeyColors { get; set; }
    [JsonPropertyName("description")]
    public DescriptionDetail Description { get; set; }
    [JsonPropertyName("estimated_year")]
    public EstimatedYear EstimatedYear { get; set; }
}
public class KeyColorsProperty
{
    [JsonPropertyName("type")]
    public string Type { get; set; } = "array";
    [JsonPropertyName("items")]
    public ItemProperty Items { get; set; }
    [JsonPropertyName("description")]
    public string Description { get; set; } = "Key colors in the image. Limit to less than four.";
}

public class DescriptionDetail
{
    [JsonPropertyName("type")]
    public string Type { get; set; } = "string";
    [JsonPropertyName("description")]
    public string Description { get; set; } = "Key colors in the image. Limit to less than four.";
}

public class EstimatedYear
{
    [JsonPropertyName("type")]
    public string Type { get; set; } = "string";
    [JsonPropertyName("description")]
    public string Description { get; set; } = "Estimated year that the images was taken, if is it a photo. Only set this if the image appears to be non-fictional. Rough estimates are okay!";
}

public class ItemProperty
{
    public string Type { get; set; } = "object";
    public Dictionary<string, ColorProperty> Properties { get; set; }
    public List<string> Required { get; set; } = new List<string> { "r", "g", "b", "name" };
}

public class ColorProperty
{
    [JsonPropertyName("type")]
    public string Type { get; set; }
    [JsonPropertyName("description")]
    public string Description { get; set; }
}

[TestMethod]
public async Task TestClaude3ImageJsonModeMessage()
{
    string resourceName = "Anthropic.SDK.Tests.Red_Apple.jpg";

    Assembly assembly = Assembly.GetExecutingAssembly();

    await using Stream stream = assembly.GetManifestResourceStream(resourceName);
    byte[] imageBytes;
    using (var memoryStream = new MemoryStream())
    {
        await stream.CopyToAsync(memoryStream);
        imageBytes = memoryStream.ToArray();
    }

    string base64String = Convert.ToBase64String(imageBytes);

    var client = new AnthropicClient();

    var messages = new List<Message>();

    messages.Add(new Message()
    {
        Role = RoleType.User,
        Content = new dynamic[]
        {
            new ImageContent()
            {
                Source = new ImageSource()
                {
                    MediaType = "image/jpeg",
                    Data = base64String
                }
            },
            new TextContent()
            {
                Text = "Use `record_summary` to describe this image."
            }
        }
    });

    var imageSchema = new ImageSchema
    {
        Type = "object",
        Required = new string[] { "key_colors", "description"},
        Properties = new Properties()
        {
            KeyColors = new KeyColorsProperty
            {
            Items = new ItemProperty
            {
                Properties = new Dictionary<string, ColorProperty>
                {
                    { "r", new ColorProperty { Type = "number", Description = "red value [0.0, 1.0]" } },
                    { "g", new ColorProperty { Type = "number", Description = "green value [0.0, 1.0]" } },
                    { "b", new ColorProperty { Type = "number", Description = "blue value [0.0, 1.0]" } },
                    { "name", new ColorProperty { Type = "string", Description = "Human-readable color name in snake_case, e.g. 'olive_green' or 'turquoise'" } }
                }
            }
        },
            Description = new DescriptionDetail { Type = "string", Description = "Image description. One to two sentences max." },
            EstimatedYear = new EstimatedYear { Type = "number", Description = "Estimated year that the images was taken, if is it a photo. Only set this if the image appears to be non-fictional. Rough estimates are okay!" }
        }

    };

    JsonSerializerOptions JsonSerializationOptions = new()
    {
        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
        Converters = { new JsonStringEnumConverter() },
        ReferenceHandler = ReferenceHandler.IgnoreCycles,
    };
    string jsonString = JsonSerializer.Serialize(imageSchema, JsonSerializationOptions);
    var tools = new List<Common.Tool>()
    {
        new Common.Tool(new Function("record_summary", "Record summary of an image into well-structured JSON.",
            JsonNode.Parse(jsonString)))
    };

    var parameters = new MessageParameters()
    {
        Messages = messages,
        MaxTokens = 1024,
        Model = AnthropicModels.Claude3Sonnet,
        Stream = false,
        Temperature = 1.0m,
    };
    var res = await client.Messages.GetClaudeMessageAsync(parameters, tools);
}

[tghamm]

Closing as complete with the latest release, feel free to open with any future issues.