Checked other resources

[X] I added a very descriptive title to this issue.
[X] I searched the LangChain documentation with the integrated search.
[X] I used the GitHub search to find a similar question and didn't find it.
[X] I am sure that this is a bug in LangChain rather than my code.
[X] The bug is not resolved by updating to the latest stable version of LangChain (or the specific integration package).

-- coding: utf-8 --

"""Commands+RAG with personalization .ipynb

Automatically generated by Colab.

Original file is located at https://colab.research.google.com/drive/1RXFuNJKCcXsqP9oTGTL3oiBuAJYCnnui """

!pip freeze | grep langchain

!python -m langchain_core.sys_info

"""# **Dependencies**"""

!pip install  cohere tiktoken langchain \
              openai==0.28.0 \
              pinecone-client==2.2.4 \
              docarray==0.39.0 \
              pydantic==1.10.8

!pip list

from google.colab import userdata

"""# **VectoreStore from Dataset**"""

import pandas as pd
data = pd.read_json('testDataset.jsonl', lines=True)
data

import pinecone

# get API key from app.pinecone.io and environment from console
pinecone.init(
    api_key=userdata.get('PINECONE_API_KEY'),
    environment="gcp-starter"
)

import time

# pinecone.delete_index("llama-2-rag")

index_name = 'llama-2-rag'

if index_name not in pinecone.list_indexes():
    pinecone.create_index(
        index_name,
        dimension=1536,
        metric='cosine'
    )
    # wait for index to finish initialization
    while not pinecone.describe_index(index_name).status['ready']:
        time.sleep(1)

index = pinecone.Index(index_name)

from langchain.embeddings.openai import OpenAIEmbeddings

embed_model = OpenAIEmbeddings(model="text-embedding-ada-002", openai_api_key=userdata.get("OpenAI_API_KEY"))

from tqdm.auto import tqdm  # for progress bar

# data = dataset.to_pandas()  # this makes it easier to iterate over the dataset

batch_size = 100

for i in tqdm(range(0, len(data), batch_size)):
    i_end = min(len(data), i+batch_size) # 13 100
    # get batch of data
    batch = data.iloc[i:i_end]
    # generate unique ids for each chunk
    ids = [f"{x['chunk-id']}" for i, x in batch.iterrows()]
    # get text to embed
    texts = [x['chunk'] for _, x in batch.iterrows()]
    # embed text
    embeds = embed_model.embed_documents(texts)
    # get metadata to store in Pinecone
    metadata = [
        {'text': x['chunk'],
         'title': x['title']} for i, x in batch.iterrows()
    ]
    # add to Pinecone
    print(ids,embeds, metadata)
    try:
      index.upsert(vectors=zip(ids, embeds, metadata))
      print("Inserted")
    except Exception as e:
      print("got exception" + str(e))

print(index)

index.describe_index_stats()

from langchain.vectorstores import Pinecone

text_field = "text"  # the metadata field that contains our text

# initialize the vector store object
vectorstore = Pinecone(
    index, embed_model.embed_query, text_field
)

"""# **Retreiver in action**

# **Function Definition as Commands..**
"""

from langchain.tools import tool
import requests
from pydantic import BaseModel, Field, constr
#import datetime
from datetime import date,datetime

"""# **User Session Info**"""

name = "Shibly"
room_number = 101

# Define the input schema
class OpenMeteoInput(BaseModel):
    latitude: float = Field(..., description="Latitude of the location to fetch weather data for")
    longitude: float = Field(..., description="Longitude of the location to fetch weather data for")

@tool(args_schema=OpenMeteoInput)
def get_current_temperature(latitude: float, longitude: float,name=name) -> dict:
    """Fetch current Weather for given cities or coordinates. For example: what is the weather of Colombo ?"""

    BASE_URL = "https://api.open-meteo.com/v1/forecast"

    # Parameters for the request
    params = {
        'latitude': latitude,
        'longitude': longitude,
        'hourly': 'temperature_2m',
        'forecast_days': 1,
    }

    # Make the request
    response = requests.get(BASE_URL, params=params)

    if response.status_code == 200:
        results = response.json()
    else:
        raise Exception(f"API Request failed with status code: {response.status_code}")

    current_utc_time = datetime.utcnow()
    time_list = [datetime.fromisoformat(time_str.replace('Z', '+00:00')) for time_str in results['hourly']['time']]
    temperature_list = results['hourly']['temperature_2m']

    closest_time_index = min(range(len(time_list)), key=lambda i: abs(time_list[i] - current_utc_time))
    current_temperature = temperature_list[closest_time_index]

    return f'Yes. {name},The current temperature is {current_temperature}°C'

class book_room_input(BaseModel):
    room_type: str = Field(..., description="Which type of room AC or Non-AC")
    class_type: str = Field(...,description="Which class of room it is. Business class or Economic class")
    check_in_date: date = Field(...,description="The date user will check-in")
    check_out_date: date = Field(...,description="The date user will check-out")
    mobile_no : constr(regex=r'(^(?:\+?88)?01[3-9]\d{8})$') = Field(...,description="Mobile number of the user")

@tool(args_schema=book_room_input)
def book_room(room_type: str, class_type: str, check_in_date: date, check_out_date: date, mobile_no: constr, name=name) -> str:
  """
    Book a room with the specified details.

    Args:
        room_type (str): Which type of room to book (AC or Non-AC).
        class_type (str): Which class of room it is (Business class or Economic class).
        check_in_date (date): The date the user will check-in.
        check_out_date (date): The date the user will check-out.
        mobile_no (str): Mobile number of the user.

    Returns:
        str: A message confirming the room booking.
    """
    # Placeholder logic for booking the room
  return f"Okay, {name} Room has been booked for {room_type} {class_type} class from {check_in_date} to {check_out_date}. Mobile number: {mobile_no}."

class requestFoodFromRestaurant(BaseModel):
    item_name : str = Field(..., description="The food item they want to order from the restaurant")
    # room_number: int = Field(..., description="Room number where the request is made")
    dine_in_type : str = Field(..., description="If the customer wants to eat in the door-step, take parcel or dine in the restaurant. It can have at most 3 values 'dine-in-room', 'dine-in-restaurant', 'parcel'")

@tool(args_schema=requestFoodFromRestaurant)
def order_resturant_item(item_name : str, dine_in_type : str, room_number=room_number, name=name) -> str:
  """
  Order food and bevarages at the hotel restaurant with specified details.

  Args:
    item_name (str) : The food item they want to order from the restaurant
    # room_number (int) : The room number from which the customer placed the order
    dine_in_type (str) : inside hotel room, dine in restaurant or parcel

  Returns
    str: A message for confirmation of food order.
  """

  if dine_in_type == "dine-in-room":
    return f"Your order have been placed. The food will get delivered at room {room_number}. Thank you {name}. Can I help you with anything else?"
  elif dine_in_type == "dine-in-restaurant":
    return f"Your order have been placed. You will be notified once the food is almost ready. Thank you {name}. Can I help you with anything else?"
  else:
    return f"Your order have been placed. The parcel will be ready in 45 minutes. Thank you {name}. Can I help you with anything else?"

class requestBillingChangeRequest(BaseModel):
    complaint: str = Field(..., description="Complain about the bill. It could be that the bill is more than it should be. Or some services are charged more than it was supposed to be")
    # room_number: int = Field(..., description="Which Room number the client is making billing request. If not provided, ask the user. Do not guess.")

@tool(args_schema=requestBillingChangeRequest)
def bill_complain_request(complaint: str , room_number=room_number, name=name) -> str:
  """
  Complaints about billing with specified details.
  Args:
    complaint (str) : Complain about the bill. It could be that the bill is more than it should be. Or some services are charged more than it was supposed to be.
    # room_number (int) : The room number from where the complain is made. Not Default value, should be asked from user.
  Returns
    str: A message for confirmation of the bill complaint.
  """

  return f"We  have received your complain from room: {room_number} and notified accounts department to handle the issue. Thank you {name} for keeping your patience while we resolve. You will be notified from the front-desk once it is resolved"

class SecurityEntity(BaseModel):
  exact_location : str = Field(..., description = "The location where the guest needs the help")
  # room : int = Field(..., description = "The room requested customer is staying")

@tool(args_schema=SecurityEntity)
def emergencySafetyRequest(exact_location : str, room_number=room_number, name=name):
  """
  Emergency Safety calls at a location for help with  speciefied details
  Args:
    location (str) : The location where the guest needs the help.
    # room (int) : The room number from where the complain is made. Not Default value, should be asked from user.
    # name (str) : Name of the guest.
  Returns
    str: A message for confirmation of the assistance.
  """

  return f"Our staff is on the way to {exact_location} to assist you. Don't you worry {name}"

class RecommendationExcursion(BaseModel):
  place_type : str = Field(..., description = "The type of place the customer wants to visit. Example - park, zoo, pool.")

class TransportationRecommendationEntity(BaseModel):
  location : str = Field(..., description = "The place customer wants to go visit")

class RoomRecommendation(BaseModel):
  budget_highest : int = Field(..., description = "Maximum customer can pay per day for a room")

@tool(args_schema = None)
def food_recommedation(name=name) -> str:
  """
  Recommend foods that is suited for the customer according to the weather from the restaurant.
  """
  food = "Shirmp Dumplings"

  return f"Sure {name}, I believe {food} would be best for now."

@tool(args_schema = TransportationRecommendationEntity)
def transportation_recommendation(location : str, name=name) -> str:
  """
  Recommends transportation with specified details
  Args:
    location (str) : The place customer wants to go visit
  Returns
    str: A message with transportation recommendation.
  """

  transport = "Private Car"

  return f"{name}, I recommend to go there by {transport}"

@tool(args_schema = RecommendationExcursion)
def excursion_recommendation(place_type : str, name=name) -> str :
  """
  Suggest nice places to visit nearby with specified details
  Args:
    place_type (str) : The type of place the customer wants to visit. Example - park, zoo, pool. Alsways ask for this value.
  Returns
    str: A message with excursion recommendation.
  """

  if place_type.lower() == "park":
    place = "National Park"
  elif place_type.lower() == "zoo":
    place = "National Zoo"
  else:
    place = "National Tower of Hanoy"
  return f"You can visit {place}. You will definitely enjoy it {name}"

@tool(args_schema = RoomRecommendation)
def room_recommendation(budget_highest : int,name=name) -> str:
  """
  Room recommendation for customer with specified details
  Args:
    budget_highest (int) : Maximum customer can pay per day for a room
  Returns
    str: A message with room suggestions according to budget.
  """

  if budget_highest < 1000:
    room = "Normal Suite"
  else:
    room = "Presidental Suite"

  return f"Sure, {name} Within your budget I suggest you to take the {room}"

@tool(args_schema = None)
def housekeeping_service_request(room_number=room_number) -> str:
    """
    Provides housekeeping service to the hotel room.
    """

    return f"We have notified our housekeeping service. A housekeeper will be at room: {room_number} within a moment."

class RoomMaintenanceRequestInput(BaseModel):
  # room_number : int = Field(..., description = "The room number that needs room maintenance service.")
  issue : str = Field(..., description = "The issue for which it needs maintenance service")

@tool(args_schema = RoomMaintenanceRequestInput)
def request_room_maintenance(issue : str, room_number=room_number,name=name) :
  """
  Resolves room issues regarding hardware like toilteries, furnitures, windows or electric gadgets like FAN, TC, AC etc of hotel room. For example:
  My room AC is not working.

  Args:
    issue (str) : The issue for which it needs maintenance service
  Returns
    str: An acknowdelgement that ensures that someone is sent to the room for fixing.
  """

  return f"Sure {name}, We have sent a staff immediately at {room_number} to fix {issue}"

class MiscellaneousRequestEntity(BaseModel):
  # room_number : int = Field(..., description = "The room number that needs the service")
  request : str = Field(..., description = "The service they want")

@tool(args_schema = MiscellaneousRequestEntity)
def request_miscellaneous(request : str, room_number=room_number,name=name):
  """
  Other requests that can be served by ordirnary staff.
  """

  return f"Sure {name}, A staff is sent at your room {room_number} for the issue sir."

# class ReminderEntity(BaseModel):
#   # room_number : int = Field(..., description = "The room number the request is made from")
#   reminder_message : str = Field(..., description = "The reminder message of the customer")
#   reminder_time : datetime = Field(..., description = "The time to remind at")

# @tool(args_schema = ReminderEntity)
# def request_reminder(reminder_message : str, reminder_time : str, name=name):
#   """
#   Set an alarm for the customer to remind about the message at the mentioned time.

#   Args:
#     # room_number (int) : The room number that needs reminder service
#     reminder_message (str) : The reminder message of the customer
#     reminder_time (str) : The time to remind the customer at.
#   Returns
#     str: An acknowdelgement message for the customer.
#   """

#   return f"Sure {name}, We wil remind you at {reminder_time} about {reminder_message}."

class WakeUpEntity(BaseModel):
  # room_number : int = Field(..., description = "The room number the request is made from")
  wakeup_time : str = Field(..., description = "The time to remind at")

@tool(args_schema = WakeUpEntity)
def request_wakeup(wakeup_time : str, room_number=room_number,name=name):
  """
  Set an alarm for the customer to wake him up.

  Args:
    # room_number (int) : The room number that needs wakeup call
    wakeup_time (str) : The time to remind the customer at
  Returns
    str: An acknowdelgement message for the customer.
  """

  return f"Sure {name}, We wil wake you up at {wakeup_time}"

# @tool(args_schema = None)
# def redirect_to_reception() -> str:
#   """
#   Redirects the call to the hotel reception when a customer only wants to directly
#   interact with a real human
#   """

#   return f"We are transferring the call to the hotel reception. Hold on a bit...."

class ShuttleServiceEntity(BaseModel):
    location : str = Field(..., description = "The location from where the customer will be picked up ")
    time : datetime = Field(..., description = "The time at which the customer will be picked up")

@tool(args_schema = ShuttleServiceEntity)
def shuttle_service_request(location : str, time : datetime, name=name) -> str:
  """
  Books a shuttle service that picks up or drops off customer.

  Args :
    location (str) : The location from where the customer will be picked up
    time (datetime) : The exact time of pickup or drop off
  return :
    str : A message that customer is picked or dropped successfully

  """
  return  f"Okay {name}. We have notified our shuttle service. They will be at {location}"

class StockAvailabilityEntity(BaseModel):
  stock_of : int = Field(..., description = "The object that user wants to know the availibility about")
  stock_date : date = Field(..., description = "The date time user wants to know about the stock")

@tool(args_schema = StockAvailabilityEntity)
def check_stock_availability(stock_of : str, stock_date : date):
  """
  Check for stock in the ware house by the specified information.

   Args :
    stock_of (int) : The room number the request is made from
    stock_date (date) : The date time user wants to know about the stock

  return :
    str : A message of the amount of stock

  """

  amount = 24

  return f"Currently we have {amount} in stock of {stock_of}"

class StatusOfRequest(BaseModel):
  room_number : int = Field(..., description = "The room number the request is made from")
  request_type : str = Field(..., description = "The type of request of the customer")

@tool(args_schema = StatusOfRequest)
def check_status_request(room_number : int, request_type : str):
  """
  Checks the status of the request.

  Args :
    room_number (int) : The room number the request is made from
    request_type (int) : The type of request of the customer

  return :
    str : A message of the status of the room
  """

  status = "processing"

  return f"We have checked about your {request_type}. We are currently {status} the request"

tools = [get_current_temperature,
         book_room,
         order_resturant_item,
         bill_complain_request,
         emergencySafetyRequest,
         room_recommendation,
         excursion_recommendation,
         transportation_recommendation,
         housekeeping_service_request,
         request_room_maintenance,
         request_miscellaneous,
         #request_reminder,
         request_wakeup,
         #redirect_to_reception,
         shuttle_service_request,
         check_stock_availability,
         check_status_request

         ] # Add extra function names here...

len(tools)

from langchain.tools.render import format_tool_to_openai_function
functions = [format_tool_to_openai_function(f) for f in tools]

"""# **Model with binded functions !!**"""

from langchain.chat_models import ChatOpenAI
functions = [format_tool_to_openai_function(f) for f in tools]

model = ChatOpenAI(openai_api_key=userdata.get("OpenAI_API_KEY")).bind(functions=functions)

"""# **Prompt Declaration**"""

from langchain.prompts import ChatPromptTemplate
prompt_model= ChatPromptTemplate.from_messages([
    ("system", "Extract the relevant information, if not explicitly provided do not guess. Extract partial info.  "),
    ("human", "{question}")
])

"""\# **Prompt in support of chat_history**"""

from langchain.prompts import MessagesPlaceholder
prompt = ChatPromptTemplate.from_messages([
    MessagesPlaceholder(variable_name="chat_history"),
    prompt_model,
    MessagesPlaceholder(variable_name="agent_scratchpad"),
])

"""# **Creating Chain using Langchain**"""

from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
from langchain.schema.runnable import RunnableMap
chain = RunnableMap({
    "context": lambda x: vectorstore.similarity_search(x["question"],k=2),
    "agent_scratchpad" : lambda x: x["agent_scratchpad"],
    "chat_history" : lambda x: x["chat_history"],
    "question": lambda x: x["question"],
}) | prompt| model | OpenAIFunctionsAgentOutputParser()

"""# **Adding memory for conversations**"""

from langchain.memory import ConversationBufferMemory
memory = ConversationBufferMemory(return_messages=True,memory_key="chat_history")

"""# **Agent Executor for final response**"""

from langchain.schema.runnable import RunnablePassthrough
from langchain.agents import AgentExecutor
from langchain.agents.format_scratchpad import format_to_openai_functions

agent_chain = RunnablePassthrough.assign(
    agent_scratchpad= lambda x: format_to_openai_functions(x["intermediate_steps"]),
) | chain
# print(chain.first.steps['agent_scratchpad'])
agent_executor = AgentExecutor(agent=agent_chain, tools=tools, verbose=True, memory=memory)
print(agent_executor.agent.aplan[0])

"""# **Satrt Conversation with LLM for Commands & Services**"""

response = agent_executor.invoke({"question": "Hi"})

response = agent_executor.invoke({"question": "What is the weather in Delhi"})["output"]

print(response)

agent_executor.invoke({"question": "What is the weather in Delhi"})

agent_executor.invoke({"question": "How can I book a room from 22th March to 23th March ?"})

agent_executor.invoke({"question": "AC room"})

agent_executor.invoke({"question": "Economic class"})

agent_executor.invoke({"question": "22th February 2024 and 24th Feb 2024"})

agent_executor.invoke({"question": "+8801787440110"})

agent_executor.invoke({"question": "Who is Nasif?"})

agent_executor.invoke({"question": "What is his age ?"})

agent_executor.invoke({"question": "I need to order some food."})

agent_executor.invoke({"question": "I want Ramen"})

agent_executor.invoke({"question": "I want to dine in my room "})

agent_executor.invoke({"question": "I have some problem with my bill. It is higher than it shoud be"})

agent_executor.invoke({"question": "I am in danger. Someone is following me."})

agent_executor.invoke({"question": "I am at third floor's balcony"})

agent_executor.invoke({"question": "I want some room recommendation"})

agent_executor.invoke({"question": "I can pay 300$ "})

agent_executor.invoke({"question": "Suggest me some places"})

agent_executor.invoke({"question": "zoo"})

agent_executor.invoke({"question": "How can I go there?"})

agent_executor.invoke({"question": "I need my room cleaned up"})

agent_executor.invoke({"question": "My AC is not working"})

agent_executor.invoke({"question": "Can you wake me up at 8 in the morning ?"})

agent_executor.invoke({"question": "I would like to talk to the Front Desk  "})

agent_executor.invoke({"question": "Can you remind me about my meeting at 4pm at hotel Sarina tomorrow ?"})

agent_executor.invoke({"question": "Can you send someone to pick up the glasses from my room ?"})

agent_executor.invoke({"question": "Can you send someone to pick up from Rajshahi at 4?"})

Error Message and Stack Trace (if applicable)

No response

Description

I'm trying to use LangChain to develop an agent having the capability of both RAG model for localization and OpenAI Function Calling feature for a single prompt together. But, agent fails to respond accurately while switching form the RAG and Functions back and forth. I'm facing two kind of problems:

Firstly, Despite using the ConversationBufferMemory to store chat history with the AgentExecutor, we are experiencing difficulty in obtaining the desired results.

Secondly, the agent is able to provide responses when questions are asked that have answers within the local document. However, it fails to respond to follow-up questions, even though the chat history is available.

Furthermore, when we use the {context} part in the prompt, particularly with functions that have multiple arguments, the agent attempts to guess the missing arguments although instructed not to. Afterwards, this behavior is inconsistent and unreliable.

We have experimented with various prompt descriptions in an attempt to address this issue. But, we have found that either the RAG model works properly or the Function Calling feature performs adequately, but not both simultaneously.

System Info

System Information

OS: Linux OS Version: #1 SMP PREEMPT_DYNAMIC Sat Nov 18 15:31:17 UTC 2023 Python Version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]

Package Information

langchain_core: 0.1.36 langchain: 0.1.13 langchain_community: 0.0.29 langsmith: 0.1.38 langchain_text_splitters: 0.0.1

Packages not installed (Not Necessarily a Problem)

The following packages were not found:

langgraph langserve

langchain-ai / langchain

RAG and OpenAI Function Invocation is not working together properly for a single prompt ! #19817