[Bug?] Incomplete Implementation of Callback Parameter for Agent

[MRziyi]

Problem:

I tried to use the callbacks parameter (callbacks: List[InstanceOf[BaseCallbackHandler]]) to achieve some custom display when creating an agent. In the instance of BaseCallbackHandler, I implemented all the functions as listed on LangChain. However, I encountered the following issue:

• Only some of the callback functions worked (as listed below in the source code).

Expected Behavior:

All implemented callback functions should be triggered appropriately according to the agent's actions.

Actual Behavior:

Only some of the callback functions are triggered.

Steps to Reproduce:

1. Create an instance of BaseCallbackHandler and implement all the functions as documented.
2. Pass the instance to the callbacks parameter when creating an agent.
3. Observe which callback functions are triggered during agent actions.

Source Code:

Custom callback instance

from typing import Any, Dict, List, Union
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.agents import AgentAction, AgentFinish
from langchain_core.outputs.llm_result import LLMResult
from langchain_core.messages.base import BaseMessage
import json

def log_callback(content: str, user: str):
    with open('log_callback.txt', 'a') as f:
        f.write(f'-------{user}---------\n{content}\n\n')

class AgentCallbackHandler(BaseCallbackHandler):
    def __init__(self, agent_name: str) -> None:
        self.agent_name = agent_name

    # Only these four functions are called
    def on_chain_start(self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any) -> None:
        """Print out that we are entering a chain."""
        log_callback("on_chain_start: " + inputs['input'], user="Manager")

    def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
        """Print out that we finished a chain."""
        log_callback("on_chain_end: " + str(outputs), user=self.agent_name)

    def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
        """Run on agent action."""
        log_callback(f'**Action** : {action.tool}\n**Content** : {str(action.tool_input)}', 
                     user=self.agent_name)

    def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> Any:
        """Run on agent end."""
        log_callback("on_agent_finish: " + str(finish.return_values), user=self.agent_name)

    # None of the funcions below works
    def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any) -> Any:
        """Run when LLM starts running."""
        log_callback("on_llm_start: " + str(prompts), user=self.agent_name)

    def on_chat_model_start(self, serialized: Dict[str, Any], messages: List[List[BaseMessage]], **kwargs: Any) -> Any:
        """Run when Chat Model starts running."""
        messages_dict = [[message.dict() for message in message_list] for message_list in messages]
        log_callback("on_chat_model_start: " + json.dumps(messages_dict, indent=4), user=self.agent_name)

    def on_llm_new_token(self, token: str, **kwargs: Any) -> Any:
        """Run on new LLM token. Only available when streaming is enabled."""

    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> Any:
        """Run when LLM ends running."""
        response_dict = response.dict()
        log_callback(f'on_llm_end: {json.dumps(response_dict, indent=4)}', user=self.agent_name)

    def on_llm_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> Any:
        """Run when LLM errors."""
        log_callback("on_llm_error: " + str(error), user=self.agent_name)

    def on_chain_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> Any:
        """Run when chain errors."""
        log_callback("on_chain_error: " + str(error), user=self.agent_name)

    def on_tool_start(self, serialized: Dict[str, Any], input_str: str, **kwargs: Any) -> Any:
        """Run when tool starts running."""
        log_callback("on_tool_start: " + str(input_str) + "\n" + str(serialized), user=self.agent_name)

    def on_tool_end(self, output: str, **kwargs: Any) -> Any:
        """Run when tool ends running."""
        log_callback("on_tool_end: " + str(output), user=self.agent_name)

    def on_tool_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> Any:
        """Run when tool errors."""
        log_callback("on_tool_error: " + str(error), user=self.agent_name)

    def on_text(self, text: str, **kwargs: Any) -> Any:
        """Run on arbitrary text."""
        log_callback("on_text: " + str(text), user=self.agent_name)

Driver

from crewai import Agent, Task, Crew, Process
from crewai_tools import SerperDevTool, ScrapeWebsiteTool
import os
from langchain_community.tools import HumanInputRun
from langchain_openai import ChatOpenAI
from agent_callback_handler import AgentCallbackHandler

llm = ChatOpenAI(model="gpt-4",temperature=0.2)

human_tool = HumanInputRun()
search_tool = SerperDevTool()
scrape_tool = ScrapeWebsiteTool()

# Agents definition
genie_agent = Agent(
  role='Genie from Aladdin\'s Lamp',
  goal='Ask user a question and answer it using the search tool',
  backstory=""""You are the Genie from Aladdin's Lamp. You use your magic powers to grant wishes and answer questions.
You should use Action "human" to ask question, the Content should be like "{"query": "question you want to ask"}" """,
  verbose=True,
  allow_delegation=False,
  tools=[human_tool,search_tool,scrape_tool],
  callbacks=[AgentCallbackHandler("Genie")],
  llm=llm
)

story_writer_agent = Agent(
  role='Fairytale Writer',
  goal='Rewrite interactions into fairy tales',
  backstory='''You are a writer who specialises in rewriting interactions into fairy tales in the style of The Thousand and One Nights''',
  verbose=True,
  allow_delegation=False,
  tools=[search_tool,scrape_tool],
  callbacks=[AgentCallbackHandler("Story Writer")],
  llm=llm
)

# Tasks definition
genie_task = Task(
  description="""
Your task is to act as the Genie from Aladdin's Lamp and you should ask the user a question that he wants to know the answer to. 
Use the search tools to find answers to the user's questions.
When the question have been answered by the user, conclude by responding with a list of questions and answers, followed by "finish" on a new line.
""",
  max_inter=3,
  expected_output="The question and its answer from the user.",
  agent=genie_agent,
  tools=[human_tool,search_tool,scrape_tool]
)

story_writer_task = Task(
  description=f"""
Rewrite the interaction between the Genie and the user into a fairy tale in the style of One Thousand and One Nights.
""",
  max_inter=3,
  agent=story_writer_agent,
  expected_output="A fairy tale based on the interaction between the Genie and the user.",
  context=[genie_task]
)

fairy_tale_crew = Crew(
  agents=[genie_agent, story_writer_agent],
  tasks=[genie_task, story_writer_task],
  process=Process.hierarchical,
  manager_llm=llm,
  verbose=True,
  full_output=True,
)

# Get your crew to work!
response = fairy_tale_crew.kickoff()

print("#########")
print(response)
print("#########")

File written

-------Manager---------
on_chain_start: What would you like to know?

This is the expect criteria for your final answer: Your best answer to your coworker asking you this, accounting for the context shared. 
 you MUST return the actual complete content as the final answer, not a summary.

This is the context you're working with:
As the Genie from Aladdin's Lamp, I have the ability to answer any question you might have.

-------Genie---------
**Action** : human
**Content** : {"query": "What is a topic you would like to know more about?"}

-------Genie---------
**Action** : Search the internet
**Content** : {"search_query": "surface area of earth"}

-------Genie---------
on_agent_finish: {'output': 'The surface area of the Earth is approximately 510 million square kilometers or 197 million square miles.'}

-------Genie---------
on_chain_end: {'output': 'The surface area of the Earth is approximately 510 million square kilometers or 197 million square miles.'}

-------Manager---------
on_chain_start: Rewrite the interaction into a fairy tale

This is the expect criteria for your final answer: Your best answer to your coworker asking you this, accounting for the context shared. 
 you MUST return the actual complete content as the final answer, not a summary.

This is the context you're working with:
The interaction is between a Genie and a user. The Genie asks 'What would you like to know?' and the user responds with 'The surface area of the Earth is approximately 510 million square kilometers or 197 million square miles.' The fairy tale needs to be in the style of One Thousand and One Nights.

-------Story Writer---------
**Action** : Search the internet
**Content** : {"search_query": "One Thousand and One Nights narrative style"} 

-------Story Writer---------
on_agent_finish: {'output': 'In the heart of the vast and endless desert, under the shimmering blanket of a thousand stars, a magical Genie resided in an ancient, ornate lamp. One day, a weary traveler, curious and eager for knowledge, stumbled upon the lamp. As he dusted off the centuries-old grime, a brilliant light burst forth, and the Genie appeared.\n\n"Oh, wise traveler," the Genie boomed, his voice echoing across the dunes, "What knowledge do you seek? Ask, and it shall be granted."\n\nThe traveler, awestruck by the spectacle, gathered his courage and said, "I wish to know the surface area of our Earth."\n\nWith a knowing smile, the Genie replied, "The Earth, our home, is grander than you can imagine. Its surface area is approximately 510 million square kilometers or 197 million square miles."\n\nThe traveler\'s eyes widened in wonder, his mind filled with images of vast oceans, towering mountains, sprawling forests, and endless deserts. He thanked the Genie for this newfound knowledge and continued his journey, carrying the wisdom of the Earth\'s vastness with him.\n\nAnd so, the tale of the Genie and the knowledge-seeking traveler was woven into the tapestry of the desert\'s lore, a testament to the boundless curiosity of mankind and the infinite wisdom of the Genie. As the stars twinkled overhead, the desert whispered this story to the winds, to be carried to the ends of the Earth, just as vast and wide as the knowledge the traveler had sought.'}

-------Story Writer---------
on_chain_end: {'output': 'In the heart of the vast and endless desert, under the shimmering blanket of a thousand stars, a magical Genie resided in an ancient, ornate lamp. One day, a weary traveler, curious and eager for knowledge, stumbled upon the lamp. As he dusted off the centuries-old grime, a brilliant light burst forth, and the Genie appeared.\n\n"Oh, wise traveler," the Genie boomed, his voice echoing across the dunes, "What knowledge do you seek? Ask, and it shall be granted."\n\nThe traveler, awestruck by the spectacle, gathered his courage and said, "I wish to know the surface area of our Earth."\n\nWith a knowing smile, the Genie replied, "The Earth, our home, is grander than you can imagine. Its surface area is approximately 510 million square kilometers or 197 million square miles."\n\nThe traveler\'s eyes widened in wonder, his mind filled with images of vast oceans, towering mountains, sprawling forests, and endless deserts. He thanked the Genie for this newfound knowledge and continued his journey, carrying the wisdom of the Earth\'s vastness with him.\n\nAnd so, the tale of the Genie and the knowledge-seeking traveler was woven into the tapestry of the desert\'s lore, a testament to the boundless curiosity of mankind and the infinite wisdom of the Genie. As the stars twinkled overhead, the desert whispered this story to the winds, to be carried to the ends of the Earth, just as vast and wide as the knowledge the traveler had sought.'}

Terminal output

% python HumanInput.py
 [2024-07-14 19:46:38][DEBUG]: Working Agent: Crew Manager
 [2024-07-14 19:46:38][INFO]: Starting Task: 
Your task is to act as the Genie from Aladdin's Lamp and you should ask the user a question that he wants to know the answer to. 
Use the search tools to find answers to the user's questions.
When the question have been answered by the user, conclude by responding with a list of questions and answers, followed by "finish" on a new line.

> Entering new CrewAgentExecutor chain...
Since I am acting as the Genie from Aladdin's Lamp, I need to ask the user a question that they want to know the answer to. However, I don't have any information about the user's interests or what they might want to know. I need to ask them what they would like to know.

Action: Ask question to coworker
Action Input: {"question": "What would you like to know?", "context": "As the Genie from Aladdin's Lamp, I have the ability to answer any question you might have.", "coworker": "Genie from Aladdin's Lamp"}

> Entering new CrewAgentExecutor chain...
As the Genie, I need to ask the user a question and then use my tools to answer it. 

Action: 
human

Action Input: 
{"query": "What is a topic you would like to know more about?"}

What is a topic you would like to know more about?
What is surface aera of earth

What is surface aera of earth

Thought: 
The user wants to know the surface area of the Earth. I will use the search tool to find this information.

Action: 
Search the internet

Action Input: 
{"search_query": "surface area of earth"} 

Search results: Title: Earth - Wikipedia
Link: https://en.wikipedia.org/wiki/Earth
Snippet: Earth · Circumference. 40075.017 km (24901.461 mi), equatorial 40007.86 km (24859.73 mi), meridional · Surface area. 510072000 km (196940000 sq mi) Land: 148 ...
---
【More Search Result】

Thought: 
The search results provide the information I need. The surface area of the Earth is approximately 510 million square kilometers or 197 million square miles.

Final Answer: 
The surface area of the Earth is approximately 510 million square kilometers or 197 million square miles.

> Finished chain.

The surface area of the Earth is approximately 510 million square kilometers or 197 million square miles.

Thought: The user has answered my question and provided the information that the surface area of the Earth is approximately 510 million square kilometers or 197 million square miles. Now, I need to conclude by responding with a list of questions and answers.

Final Answer: 
1. Question: What would you like to know?
   Answer: The surface area of the Earth is approximately 510 million square kilometers or 197 million square miles.

Finish

> Finished chain.
 [2024-07-14 19:47:30][DEBUG]: [Crew Manager] Task output: 1. Question: What would you like to know?
   Answer: The surface area of the Earth is approximately 510 million square kilometers or 197 million square miles.

Finish
 [2024-07-14 19:47:30][DEBUG]: Working Agent: Crew Manager
 [2024-07-14 19:47:30][INFO]: Starting Task: 
Rewrite the interaction between the Genie and the user into a fairy tale in the style of One Thousand and One Nights.

> Entering new CrewAgentExecutor chain...
The task at hand is to rewrite the interaction between the Genie and the user into a fairy tale in the style of One Thousand and One Nights. The context provided is a question and answer exchange about the surface area of the Earth. As a Crew Manager, I don't perform the tasks myself but delegate them to the right people. In this case, the right person is the Fairytale Writer. I need to provide them with all the necessary context to execute the task.

Action: Delegate work to coworker
Action Input: {"task": "Rewrite the interaction into a fairy tale", "context": "The interaction is between a Genie and a user. The Genie asks 'What would you like to know?' and the user responds with 'The surface area of the Earth is approximately 510 million square kilometers or 197 million square miles.' The fairy tale needs to be in the style of One Thousand and One Nights.", "coworker": "Fairytale Writer"}

> Entering new CrewAgentExecutor chain...
I need to transform this interaction into a fairy tale in the style of One Thousand and One Nights. I'll start by researching the key elements and narrative style of One Thousand and One Nights to ensure my story is authentic. 

Action: 
Search the internet

Action Input: 
{"search_query": "One Thousand and One Nights narrative style"} 

Search results: Title: A Thousand and One Nights: Arabian Story-telling in World Literature
Link: https://blogs.loc.gov/international-collections/2017/10/a-thousand-and-one-nights-arabian-story-telling-in-world-literature/
Snippet: The One Thousand and One Nights, or the Arabian Nights, as it is also known, is constructed as a “frame story” to which all the other tales are ...
---
【More Search Result】

Thought: 
The search results indicate that One Thousand and One Nights is characterized by a frame-narrative technique, where there are as many narrators as there are stories. Each tale begins with an 'appearance of destiny' which manifests itself through an anomaly. I will use this information to craft a fairy tale based on the interaction between the Genie and the user.

Final Answer: 
【omitted specific answer】

> Finished chain.

【omitted specific answer】

Thought: The Fairytale Writer has successfully rewritten the interaction into a fairy tale in the style of One Thousand and One Nights. The story captures the essence of the original interaction while adding the flair and mystique associated with the chosen style. I now have the final answer.

Final Answer: 【omitted specific answer】

> Finished chain.
 [2024-07-14 19:48:09][DEBUG]: [Crew Manager] Task output: 【omitted specific answer】

Additional Information:

• CrewAI version: [0.36.1]
• LangChain version: [0.2.7]
• Python version: [3.11.9]

Please let me know if you need any additional information or if there are any workarounds for this issue. Thank you for your assistance.

[dregules]

Same here, the below are not printing out or doing anything inside my customCallbackHandler. This is not expected behavior, as on_chain_start, on_chain_error, and on_agent_action all work like a charm. Are the rest of methods of langchain's BaseCallbackHandler not implemented?. Please kindly advise.

def on_tool_start(
        self, serialized: Dict[str, Any], input_str: str, **kwargs: Any
    ) -> Any:
        """Run when tool starts running."""
        print(f"\n\n\033[1m> Running {serialized['name']}...\033[0m")  # noqa: T201
        print(f"\n\033[1m> Input: {input_str}\033[0m")  # noqa: T201
        print(input_str)
        print(serialized.keys())

 def on_tool_end(
        self,
        output: str,
        color: Optional[str] = None,
        observation_prefix: Optional[str] = None,
        llm_prefix: Optional[str] = None,
        **kwargs: Any,
    ) -> None:
        """If not the final action, print out observation."""
       # Remove the tool's response from the conversation history
        # context["messages"] = context["messages"][:-1]
        #     return tool_output
        print("<<<<<<<TOOL ENDED OUTPUT")
        print()
        print(output)
        print()
        print("TOOL ENDED OUTPUt>>>>>>>")

[theCyberTech]

Hi thanks for raising this, I have raised this internally. Also notice there is an existing PR for this - https://github.com/crewAIInc/crewAI/pull/333/files#diff-61064eabfbcf89548f1d4f5e2e18616dea6e897d06dc0a00bad1eebd634dc9ca