microsoft / semantic-kernel

Integrate cutting-edge LLM technology quickly and easily into your apps
https://aka.ms/semantic-kernel
MIT License
21.31k stars 3.13k forks source link

.Net: `IFunctionInvocationFilter` unable to differentiate between streaming and non-streaming invocation for `KernelFunctionFromPrompt` #7336

Open crickman opened 1 month ago

crickman commented 1 month ago

Description

IFunctionInvocationFilter is invoked for a KernelFunctionFromPrompt; however, the filter cannot know whether the prompt-function was invoked usign streaming or not. This limits the ability to provide a compatible response, as any KernelFunctionFromPrompt may be invoked in both modalities.

Use Case

Invalid input is detected which results in the desire to block the function execution.

Repro

Run this code from Concepts project:

// Copyright (c) Microsoft. All rights reserved.
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;

namespace Sanity;

public class PromptFunctionFilterBug(ITestOutputHelper output) : BaseTest(output)
{
    private const string Input = "Is 12341871 a prime number?";

    [Fact]
    public async Task KernelInvokeTestAsync()
    {
        Kernel kernel = this.CreateKernelWithChatCompletion();
        await InvokeKernelAsync(kernel);
    }

    [Fact]
    public async Task KernelInvokeWithFunctionFilterTestAsync()
    {
        Kernel kernel = this.CreateKernelWithChatCompletion();
        kernel.FunctionInvocationFilters.Add(new PromptFunctionFilter());
        await InvokeKernelAsync(kernel);
    }

    [Fact]
    public async Task KernelInvokeStreamingTestAsync()
    {
        Kernel kernel = this.CreateKernelWithChatCompletion();
        await InvokeKernelStreamingAsync(kernel);
    }

    [Theory]
    [InlineData(true)]
    [InlineData(false)]
    public async Task KernelInvokeStreamingWithFunctionFilterTestAsync(bool targetStreaming)
    {
        Kernel kernel = this.CreateKernelWithChatCompletion();
        kernel.FunctionInvocationFilters.Add(new PromptFunctionFilter(targetStreaming));
        await InvokeKernelStreamingAsync(kernel);
    }

    private async Task InvokeKernelAsync(Kernel kernel)
    {
        Console.WriteLine($"[TextContent] {AuthorRole.User}: '{Input}'");

        KernelFunction promptFunction = kernel.CreateFunctionFromPrompt(Input);

        ChatMessageContent content = (await kernel.InvokeAsync<ChatMessageContent>(promptFunction))!;

        Console.WriteLine($"[{content.Items.LastOrDefault()?.GetType().Name ?? "(empty)"}] {content.Role} : '{content.Content}'");
    }

    private async Task InvokeKernelStreamingAsync(Kernel kernel)
    {
        Console.WriteLine($"[TextContent] {AuthorRole.User}: '{Input}'");

        KernelFunction promptFunction = kernel.CreateFunctionFromPrompt(Input);

        await foreach (StreamingChatMessageContent content in kernel.InvokeStreamingAsync<StreamingChatMessageContent>(promptFunction))
        {
            Console.WriteLine($"[{content.Items.LastOrDefault()?.GetType().Name ?? "(empty)"}] {content.Role} : '{content.Content}'");
        }
    }

    private sealed class PromptFunctionFilter(bool forStreaming = false) : IFunctionInvocationFilter
    {
        public Task OnFunctionInvocationAsync(FunctionInvocationContext context, Func<FunctionInvocationContext, Task> next)
        {
            if (forStreaming)
            {
                StreamingChatMessageContent[] contents = [new StreamingChatMessageContent(AuthorRole.Assistant, "Intercepted message.")];
                IAsyncEnumerable<StreamingChatMessageContent> contentsAsync = contents.ToAsyncEnumerable();
                context.Result = new FunctionResult(context.Function, contentsAsync);
            }
            else
            {
                context.Result = new FunctionResult(context.Function, new ChatMessageContent(AuthorRole.Assistant, "Intercepted message."));
            }

            return Task.CompletedTask;
        }
    }
}

Expected

The IFunctionInvocationFilter implementation should be provided with sufficient context in order to generate a compatible result.

Platform

markwallace-microsoft commented 2 weeks ago

@crickman How important is this enhancement? Do we need to prioritise in sprint 33?