Closed retendo closed 4 months ago
I use https://github.com/betalgo/openai for working with OpenAI endpoints and would like to at least use the same input schema for my function calls, to be able to compare them more easily.
What I would need is basically the last example on this documentation page (JSON mode): https://docs.anthropic.com/claude/docs/tool-use-examples
Working on something like this:
public class ImageSchema
{
[JsonPropertyName("type")]
public string Type { get; set; } = "object";
[JsonPropertyName("required")]
public string[] Required { get; set; }
[JsonPropertyName("properties")]
public Properties Properties { get; set; }
}
public class Properties
{
[JsonPropertyName("key_colors")]
public KeyColorsProperty KeyColors { get; set; }
[JsonPropertyName("description")]
public DescriptionDetail Description { get; set; }
[JsonPropertyName("estimated_year")]
public EstimatedYear EstimatedYear { get; set; }
}
public class KeyColorsProperty
{
[JsonPropertyName("type")]
public string Type { get; set; } = "array";
[JsonPropertyName("items")]
public ItemProperty Items { get; set; }
[JsonPropertyName("description")]
public string Description { get; set; } = "Key colors in the image. Limit to less than four.";
}
public class DescriptionDetail
{
[JsonPropertyName("type")]
public string Type { get; set; } = "string";
[JsonPropertyName("description")]
public string Description { get; set; } = "Key colors in the image. Limit to less than four.";
}
public class EstimatedYear
{
[JsonPropertyName("type")]
public string Type { get; set; } = "string";
[JsonPropertyName("description")]
public string Description { get; set; } = "Estimated year that the images was taken, if is it a photo. Only set this if the image appears to be non-fictional. Rough estimates are okay!";
}
public class ItemProperty
{
public string Type { get; set; } = "object";
public Dictionary<string, ColorProperty> Properties { get; set; }
public List<string> Required { get; set; } = new List<string> { "r", "g", "b", "name" };
}
public class ColorProperty
{
[JsonPropertyName("type")]
public string Type { get; set; }
[JsonPropertyName("description")]
public string Description { get; set; }
}
[TestMethod]
public async Task TestClaude3ImageJsonModeMessage()
{
string resourceName = "Anthropic.SDK.Tests.Red_Apple.jpg";
Assembly assembly = Assembly.GetExecutingAssembly();
await using Stream stream = assembly.GetManifestResourceStream(resourceName);
byte[] imageBytes;
using (var memoryStream = new MemoryStream())
{
await stream.CopyToAsync(memoryStream);
imageBytes = memoryStream.ToArray();
}
string base64String = Convert.ToBase64String(imageBytes);
var client = new AnthropicClient();
var messages = new List<Message>();
messages.Add(new Message()
{
Role = RoleType.User,
Content = new dynamic[]
{
new ImageContent()
{
Source = new ImageSource()
{
MediaType = "image/jpeg",
Data = base64String
}
},
new TextContent()
{
Text = "Use `record_summary` to describe this image."
}
}
});
var imageSchema = new ImageSchema
{
Type = "object",
Required = new string[] { "key_colors", "description"},
Properties = new Properties()
{
KeyColors = new KeyColorsProperty
{
Items = new ItemProperty
{
Properties = new Dictionary<string, ColorProperty>
{
{ "r", new ColorProperty { Type = "number", Description = "red value [0.0, 1.0]" } },
{ "g", new ColorProperty { Type = "number", Description = "green value [0.0, 1.0]" } },
{ "b", new ColorProperty { Type = "number", Description = "blue value [0.0, 1.0]" } },
{ "name", new ColorProperty { Type = "string", Description = "Human-readable color name in snake_case, e.g. 'olive_green' or 'turquoise'" } }
}
}
},
Description = new DescriptionDetail { Type = "string", Description = "Image description. One to two sentences max." },
EstimatedYear = new EstimatedYear { Type = "number", Description = "Estimated year that the images was taken, if is it a photo. Only set this if the image appears to be non-fictional. Rough estimates are okay!" }
}
};
JsonSerializerOptions JsonSerializationOptions = new()
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
Converters = { new JsonStringEnumConverter() },
ReferenceHandler = ReferenceHandler.IgnoreCycles,
};
string jsonString = JsonSerializer.Serialize(imageSchema, JsonSerializationOptions);
var tools = new List<Common.Tool>()
{
new Common.Tool(new Function("record_summary", "Record summary of an image into well-structured JSON.",
JsonNode.Parse(jsonString)))
};
var parameters = new MessageParameters()
{
Messages = messages,
MaxTokens = 1024,
Model = AnthropicModels.Claude3Sonnet,
Stream = false,
Temperature = 1.0m,
};
var res = await client.Messages.GetClaudeMessageAsync(parameters, tools);
}
Closing as complete with the latest release, feel free to open with any future issues.
I would like to specify a nested input schema. It seems that is not possible at the moment.
Would you try to support that? I think there's a lot of use cases for this, when you want to let the model create structured data for you.