Python: Add agent function termination sample. Fix openai chat comple…

…tion chat history bug. (#8300)

### Motivation and Context

The Python agent samples were missing a concept sample to show how to
use the auto function invocation filter with a ChatCompletionAgent. It's
similar to how the filter is used with a normal chat completion service;
however, it's still good to show in the context of using agents.

Additionally, during sample creation, it was found that the chat history
message, during non-streaming function calling, was always being added,
even if the model did not return a function to call. This was causing a
duplication of chat history messages as the caller is usually
responsible to add the message to the chat history once it receives it
from the AI connector.

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

This PR:
- Adds the auto function invoke chat completion agent sample
- Fixes the chat history bug by re-ordering when the tool call message
from the model is added to the chat history during a non-streaming
`get_chat_message_contents` call-- it is added after we find that we do
have functions to call. Otherwise, do not add the completions example to
the chat history as the caller is responsible for doing that.
- Fixed for AzureAI Inference Chat Completion, Google VertexAI Chat
Completion, and GoogleAI Chat Completion
- Fixes the auto invoke function chat message content construction bug
for other AI connectors that was fixed for OpenAI in #8098

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [X] The code builds clean without any errors or warnings
- [X] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [X] All unit tests pass, and I have added new tests where possible
- [X] I didn't break anyone 😄

python/samples/concepts/agents/chat_completion_function_termination.py

@@ -0,0 +1,133 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+from typing import Annotated
+
+from semantic_kernel.agents import ChatCompletionAgent
+from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
+from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion
+from semantic_kernel.contents.chat_history import ChatHistory
+from semantic_kernel.contents.chat_message_content import ChatMessageContent
+from semantic_kernel.contents.function_call_content import FunctionCallContent
+from semantic_kernel.contents.function_result_content import FunctionResultContent
+from semantic_kernel.contents.utils.author_role import AuthorRole
+from semantic_kernel.filters.auto_function_invocation.auto_function_invocation_context import (
+    AutoFunctionInvocationContext,
+)
+from semantic_kernel.filters.filter_types import FilterTypes
+from semantic_kernel.functions.kernel_function_decorator import kernel_function
+from semantic_kernel.kernel import Kernel
+
+###################################################################
+# The following sample demonstrates how to configure the auto     #
+# function invocation filter with use of a ChatCompletionAgent.   #
+###################################################################
+
+
+# Define the agent name and instructions
+HOST_NAME = "Host"
+HOST_INSTRUCTIONS = "Answer questions about the menu."
+
+
+# Define the auto function invocation filter that will be used by the kernel
+async def auto_function_invocation_filter(context: AutoFunctionInvocationContext, next):
+    """A filter that will be called for each function call in the response."""
+    # if we don't call next, it will skip this function, and go to the next one
+    await next(context)
+    if context.function.plugin_name == "menu":
+        context.terminate = True
+
+
+# Define a sample plugin for the sample
+class MenuPlugin:
+    """A sample Menu Plugin used for the concept sample."""
+
+    @kernel_function(description="Provides a list of specials from the menu.")
+    def get_specials(self) -> Annotated[str, "Returns the specials from the menu."]:
+        return """
+        Special Soup: Clam Chowder
+        Special Salad: Cobb Salad
+        Special Drink: Chai Tea
+        """
+
+    @kernel_function(description="Provides the price of the requested menu item.")
+    def get_item_price(
+        self, menu_item: Annotated[str, "The name of the menu item."]
+    ) -> Annotated[str, "Returns the price of the menu item."]:
+        return "$9.99"
+
+
+def _create_kernel_with_chat_completionand_filter(service_id: str) -> Kernel:
+    """A helper function to create a kernel with a chat completion service and a filter."""
+    kernel = Kernel()
+    kernel.add_service(AzureChatCompletion(service_id=service_id))
+    kernel.add_filter(FilterTypes.AUTO_FUNCTION_INVOCATION, auto_function_invocation_filter)
+    kernel.add_plugin(plugin=MenuPlugin(), plugin_name="menu")
+    return kernel
+
+
+def _write_content(content: ChatMessageContent) -> None:
+    """Write the content to the console."""
+    last_item_type = type(content.items[-1]).__name__ if content.items else "(empty)"
+    message_content = ""
+    if isinstance(last_item_type, FunctionCallContent):
+        message_content = f"tool request = {content.items[-1].function_name}"
+    elif isinstance(last_item_type, FunctionResultContent):
+        message_content = f"function result = {content.items[-1].result}"
+    else:
+        message_content = str(content.items[-1])
+    print(f"[{last_item_type}] {content.role} : '{message_content}'")
+
+
+# A helper method to invoke the agent with the user input
+async def invoke_agent(agent: ChatCompletionAgent, input: str, chat_history: ChatHistory) -> None:
+    """Invoke the agent with the user input."""
+    chat_history.add_user_message(input)
+    print(f"# {AuthorRole.USER}: '{input}'")
+
+    async for content in agent.invoke(chat_history):
+        if not any(isinstance(item, (FunctionCallContent, FunctionResultContent)) for item in content.items):
+            chat_history.add_message(content)
+        _write_content(content)
+
+
+async def main():
+    service_id = "agent"
+
+    # Create the kernel used by the chat completion agent
+    kernel = _create_kernel_with_chat_completionand_filter(service_id=service_id)
+
+    settings = kernel.get_prompt_execution_settings_from_service_id(service_id=service_id)
+
+    # Configure the function choice behavior to auto invoke kernel functions
+    settings.function_choice_behavior = FunctionChoiceBehavior.Auto()
+
+    # Create the agent
+    agent = ChatCompletionAgent(
+        service_id=service_id,
+        kernel=kernel,
+        name=HOST_NAME,
+        instructions=HOST_INSTRUCTIONS,
+        execution_settings=settings,
+    )
+
+    # Define the chat history
+    chat = ChatHistory()
+
+    # Respond to user input
+    await invoke_agent(agent=agent, input="Hello", chat_history=chat)
+    await invoke_agent(agent=agent, input="What is the special soup?", chat_history=chat)
+    await invoke_agent(agent=agent, input="What is the special drink?", chat_history=chat)
+    await invoke_agent(agent=agent, input="Thank you", chat_history=chat)
+
+    print("================================")
+    print("CHAT HISTORY")
+    print("================================")
+
+    # Print out the chat history to view the different types of messages
+    for message in chat.messages:
+        _write_content(message)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

python/samples/concepts/agents/mixed_chat_files.py

@@ -5,8 +5,8 @@
 
 from semantic_kernel.agents import AgentGroupChat, ChatCompletionAgent
 from semantic_kernel.agents.open_ai import OpenAIAssistantAgent
-from semantic_kernel.agents.strategies.termination.termination_strategy import TerminationStrategy
 from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import AzureChatCompletion
+from semantic_kernel.contents.annotation_content import AnnotationContent
 from semantic_kernel.contents.chat_message_content import ChatMessageContent
 from semantic_kernel.contents.utils.author_role import AuthorRole
 from semantic_kernel.kernel import Kernel
@@ -19,21 +19,12 @@
 #####################################################################
 
 
-class ApprovalTerminationStrategy(TerminationStrategy):
-    """A strategy for determining when an agent should terminate."""
-
-    async def should_agent_terminate(self, agent, history):
-        """Check if the agent should terminate."""
-        return "approved" in history[-1].content.lower()
-
-
 SUMMARY_INSTRUCTIONS = "Summarize the entire conversation for the user in natural language."
 
 
 def _create_kernel_with_chat_completion(service_id: str) -> Kernel:
     kernel = Kernel()
     kernel.add_service(AzureChatCompletion(service_id=service_id))
-    # kernel.add_service(OpenAIChatCompletion(service_id=service_id))
     return kernel
 
 
@@ -47,6 +38,12 @@ async def invoke_agent(
 
     async for content in chat.invoke(agent=agent):
         print(f"# {content.role} - {content.name or '*'}: '{content.content}'")
+        if len(content.items) > 0:
+            for item in content.items:
+                if isinstance(item, AnnotationContent):
+                    print(f"\n`{item.quote}` => {item.file_id}")
+                    response_content = await agent.client.files.content(item.file_id)
+                    print(response_content.text)
 
 
 async def main():

python/semantic_kernel/connectors/ai/azure_ai_inference/services/azure_ai_inference_chat_completion.py

@@ -34,7 +34,10 @@
 from semantic_kernel.connectors.ai.azure_ai_inference.services.azure_ai_inference_base import AzureAIInferenceBase
 from semantic_kernel.connectors.ai.azure_ai_inference.services.utils import MESSAGE_CONVERTERS
 from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
-from semantic_kernel.connectors.ai.function_calling_utils import update_settings_from_function_call_configuration
+from semantic_kernel.connectors.ai.function_calling_utils import (
+    merge_function_results,
+    update_settings_from_function_call_configuration,
+)
 from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
 from semantic_kernel.contents.chat_history import ChatHistory
 from semantic_kernel.contents.chat_message_content import ITEM_TYPES, ChatMessageContent
@@ -151,11 +154,12 @@ async def get_chat_message_contents(
 
         for request_index in range(settings.function_choice_behavior.maximum_auto_invoke_attempts):
             completions = await self._send_chat_request(chat_history, settings)
-            chat_history.add_message(message=completions[0])
-            function_calls = [item for item in chat_history.messages[-1].items if isinstance(item, FunctionCallContent)]
+            function_calls = [item for item in completions[0].items if isinstance(item, FunctionCallContent)]
             if (fc_count := len(function_calls)) == 0:
                 return completions
 
+            chat_history.add_message(message=completions[0])
+
             results = await self._invoke_function_calls(
                 function_calls=function_calls,
                 chat_history=chat_history,
@@ -167,7 +171,7 @@ async def get_chat_message_contents(
             )
 
             if any(result.terminate for result in results if result is not None):
-                return completions
+                return merge_function_results(chat_history.messages[-len(results) :])
         else:
             # do a final call without auto function calling
             return await self._send_chat_request(chat_history, settings)
@@ -317,7 +321,8 @@ async def _get_streaming_chat_message_contents_auto_invoke(
             )
 
             if any(result.terminate for result in results if result is not None):
-                return
+                yield merge_function_results(chat_history.messages[-len(results) :])  # type: ignore
+                break
 
     async def _send_chat_streaming_request(
         self, chat_history: ChatHistory, settings: AzureAIInferenceChatPromptExecutionSettings

python/semantic_kernel/connectors/ai/function_calling_utils.py

@@ -3,6 +3,9 @@
 from collections import OrderedDict
 from typing import TYPE_CHECKING, Any
 
+from semantic_kernel.contents.chat_message_content import ChatMessageContent
+from semantic_kernel.contents.function_result_content import FunctionResultContent
+from semantic_kernel.contents.utils.author_role import AuthorRole
 from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
 
 if TYPE_CHECKING:
@@ -68,3 +71,22 @@ def _combine_filter_dicts(*dicts: dict[str, list[str]]) -> dict:
         combined_filters[key] = list(combined_functions.keys())
 
     return combined_filters
+
+
+def merge_function_results(
+    messages: list[ChatMessageContent],
+) -> list[ChatMessageContent]:
+    """Combine multiple function result content types to one chat message content type.
+
+    This method combines the FunctionResultContent items from separate ChatMessageContent messages,
+    and is used in the event that the `context.terminate = True` condition is met.
+    """
+    items: list[Any] = []
+    for message in messages:
+        items.extend([item for item in message.items if isinstance(item, FunctionResultContent)])
+    return [
+        ChatMessageContent(
+            role=AuthorRole.TOOL,
+            items=items,
+        )
+    ]

python/semantic_kernel/connectors/ai/google/google_ai/services/google_ai_chat_completion.py

@@ -13,6 +13,7 @@
 from google.generativeai.types import AsyncGenerateContentResponse, GenerateContentResponse, GenerationConfig
 from pydantic import ValidationError
 
+from semantic_kernel.connectors.ai.function_calling_utils import merge_function_results
 from semantic_kernel.connectors.ai.google.google_ai.google_ai_prompt_execution_settings import (
     GoogleAIChatPromptExecutionSettings,
 )
@@ -132,11 +133,12 @@ async def get_chat_message_contents(
 
         for request_index in range(settings.function_choice_behavior.maximum_auto_invoke_attempts):
             completions = await self._send_chat_request(chat_history, settings)
-            chat_history.add_message(message=completions[0])
-            function_calls = [item for item in chat_history.messages[-1].items if isinstance(item, FunctionCallContent)]
+            function_calls = [item for item in completions[0].items if isinstance(item, FunctionCallContent)]
             if (fc_count := len(function_calls)) == 0:
                 return completions
 
+            chat_history.add_message(message=completions[0])
+
             results = await invoke_function_calls(
                 function_calls=function_calls,
                 chat_history=chat_history,
@@ -148,7 +150,7 @@ async def get_chat_message_contents(
             )
 
             if any(result.terminate for result in results if result is not None):
-                return completions
+                return merge_function_results(chat_history.messages[-len(results) :])
         else:
             # do a final call without auto function calling
             return await self._send_chat_request(chat_history, settings)
@@ -294,7 +296,8 @@ async def _get_streaming_chat_message_contents_auto_invoke(
             )
 
             if any(result.terminate for result in results if result is not None):
-                return
+                yield merge_function_results(chat_history.messages[-len(results) :])  # type: ignore
+                break
 
     async def _send_chat_streaming_request(
         self,

python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_chat_completion.py

@@ -10,6 +10,7 @@
 from pydantic import ValidationError
 from vertexai.generative_models import Candidate, GenerationResponse, GenerativeModel
 
+from semantic_kernel.connectors.ai.function_calling_utils import merge_function_results
 from semantic_kernel.connectors.ai.google.shared_utils import (
     configure_function_choice_behavior,
     filter_system_message,
@@ -126,11 +127,12 @@ async def get_chat_message_contents(
 
         for request_index in range(settings.function_choice_behavior.maximum_auto_invoke_attempts):
             completions = await self._send_chat_request(chat_history, settings)
-            chat_history.add_message(message=completions[0])
-            function_calls = [item for item in chat_history.messages[-1].items if isinstance(item, FunctionCallContent)]
+            function_calls = [item for item in completions[0].items if isinstance(item, FunctionCallContent)]
             if (fc_count := len(function_calls)) == 0:
                 return completions
 
+            chat_history.add_message(message=completions[0])
+
             results = await invoke_function_calls(
                 function_calls=function_calls,
                 chat_history=chat_history,
@@ -142,7 +144,7 @@ async def get_chat_message_contents(
             )
 
             if any(result.terminate for result in results if result is not None):
-                return completions
+                return merge_function_results(chat_history.messages[-len(results) :])
         else:
             # do a final call without auto function calling
             return await self._send_chat_request(chat_history, settings)
@@ -287,7 +289,8 @@ async def _get_streaming_chat_message_contents_auto_invoke(
             )
 
             if any(result.terminate for result in results if result is not None):
-                return
+                yield merge_function_results(chat_history.messages[-len(results) :])  # type: ignore
+                break
 
     async def _send_chat_streaming_request(
         self,