search

opensearch-project / ml-commons

ml-commons provides a set of common machine learning algorithms, e.g. k-means, or linear regression, to help developers build ML related features within OpenSearch.
Apache License 2.0
91 stars 129 forks source link

[BUG] 4 problems in memory message API response. #2344

Open ramda1234786 opened 5 months ago

ramda1234786 commented 5 months ago

I am using conversational_flow Agent for RAG, but i am getting 4 problems in memory message API response

When we use this API : /_plugins/_ml/memory/xSDabo4BToKzewPCJxU1/messages

Problem 1 : Get List of messages shows origin as null Problem 2 : The List of messages is not sorted in descending order with this API. This is needed to show latest messages. This becomes a problem to show latest message when used as pagination ?max_results=5&next_token=3

{
    "messages": [
        {
            "memory_id": "xSDabo4BToKzewPCJxU1",
            "message_id": "xiDabo4BToKzewPCthVF",
            "create_time": "2024-03-24T05:06:51.07747317Z",
            "input": "what is the genere of rush movie?",
            "prompt_template": null,
            "response": " The genre of the movie \"Rush\" is action.",
            "origin": null,
            "additional_info": {}
        },
        {
            "memory_id": "xSDabo4BToKzewPCJxU1",
            "message_id": "ySDpbo4BToKzewPCHhXx",
            "create_time": "2024-03-24T05:22:35.377643494Z",
            "input": "what is the earnings of rush movie?",
            "prompt_template": null,
            "response": " The earnings of the movie \"Rush\" are 300000.",
            "origin": null,
            "additional_info": {}
        }
    ]
}

Alternatively with Search Message API : /_plugins/_ml/memory/xSDabo4BToKzewPCJxU1/_search

Problem 3 : We get a extra response with origin as null apart from hybrid search and llm model response

Problem 4 : there is no pagination like scroll_id so that we can retrieve all the messages interactions

Note : Traces of messages shows only 2 traces which is perfect as i am using only 2 steps in agent

Below is the JSON body 

{
  "query": {
    "match_all": {}
  },
  "sort": [
    {
      "create_time": {
        "order": "desc"
      }
    }
  ],
  "size":100
}

Below is the Response

{
    "took": 0,
    "timed_out": false,
    "_shards": {
        "total": 1,
        "successful": 1,
        "skipped": 0,
        "failed": 0
    },
    "hits": {
        "total": {
            "value": 3,
            "relation": "eq"
        },
        "max_score": null,
        "hits": [
            {
                "_index": ".plugins-ml-memory-message",
                "_id": "yCDabo4BToKzewPCtxUo",
                "_version": 1,
                "_seq_no": 11751,
                "_primary_term": 1,
                "_score": null,
                "_source": {
                    "input": "what is the genere of rush movie?",
                    "memory_id": "xSDabo4BToKzewPCJxU1",
                    "trace_number": 2,
                    "create_time": "2024-03-24T05:06:51.304367877Z",
                    "additional_info": {},
                    "response": " The genre of the movie \"Rush\" is action.",
                    "origin": "llm_response",
                    "parent_message_id": "xiDabo4BToKzewPCthVF",
                    "prompt_template": null
                },
                "sort": [
                    1711256811304
                ]
            },
            {
                "_index": ".plugins-ml-memory-message",
                "_id": "xyDabo4BToKzewPCthV6",
                "_version": 1,
                "_seq_no": 11750,
                "_primary_term": 1,
                "_score": null,
                "_source": {
                    "input": "what is the genere of rush movie?",
                    "memory_id": "xSDabo4BToKzewPCJxU1",
                    "trace_number": 1,
                    "create_time": "2024-03-24T05:06:51.130383384Z",
                    "additional_info": {},
                    "response": "{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Rush, year is 2013, budget is 500000, earning is 300000, genere is action.\"},\"_id\":\"tt1979320\",\"_score\":1.0}\n{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Avatar, year is 2009, budget is 1200000, earning is 6000000, genere is Sci-fi.\"},\"_id\":\"tt1979323\",\"_score\":0.055511523}\n{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Dr Strange, year is 2022, budget is 2500000, earning is 5000000, genere is Sci-fi.\"},\"_id\":\"tt1979322\",\"_score\":0.04304528}\n{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Jurrasic, year is 2014, budget is 1500000, earning is 2300000, genere is action.\"},\"_id\":\"tt1979321\",\"_score\":0.004722477}\n",
                    "origin": "hybrid_search",
                    "parent_message_id": "xiDabo4BToKzewPCthVF",
                    "prompt_template": null
                },
                "sort": [
                    1711256811130
                ]
            },
            {
                "_index": ".plugins-ml-memory-message",
                "_id": "xiDabo4BToKzewPCthVF",
                "_version": 2,
                "_seq_no": 11752,
                "_primary_term": 1,
                "_score": null,
                "_source": {
                    "input": "what is the genere of rush movie?",
                    "memory_id": "xSDabo4BToKzewPCJxU1",
                    "trace_number": null,
                    "create_time": "2024-03-24T05:06:51.07747317Z",
                    "additional_info": {},
                    "response": " The genre of the movie \"Rush\" is action.",
                    "origin": null,
                    "parent_message_id": null,
                    "prompt_template": null
                },
                "sort": [
                    1711256811077
                ]
            }
        ]
    }
}
model-collapse commented 5 months ago

Hi, @ramda1234786 can you elaborate more on how to reproduce?

ylwu-amzn commented 5 months ago

Ping the memory feature builder @Zhangxunmt, Xun Can you help take a look ? For the first one, is this expected ? For second one, I feel it's not easy to use if not support descending order, let's improve this part ?

Zhangxunmt commented 5 months ago

@ramda1234786, Thanks for all your feedbacks. Please check the below: 1: Did you create the message through the conversational_flow Agent? If you create the message through the Memory APIs, you need to provide the value for origin field.

  1. The default sorting order of messages is Ascending because typically that's how they are reviewed (from most historical to most recent). We can use the Message search API to list in descending.
  2. Is this the same issue with problem 1? Can you elaborate a bit about the hybrid search here?
  3. Does max_results=5&next_token=3 work for pagination? It will show the max_results messages starting from next_token.
ramda1234786 commented 5 months ago

Hi @Zhangxunmt , thanks for your reply. Below are the responses on 4 questions

Question 1 First i created the memory like below /_plugins/_ml/memory

using the below JSON Body


{
  "name": "Chat conversation"
}

Then used the conversational_flow Agent like below to ask question as i get the memory_id from above API

{ "parameters": { "index":"indexname", "question": "what is the genere of rush movie?", "memory_id":"uyDAbo4BToKzewPCyxU123", "message_history_limit": 10 } }

Can you give me the complete Applicable body to Create Memory via API?

Question 2: Yes i have used Search API to get it on descending then got another problem 3 and problem 4 which i mentioned in my first post

Question 3 : No it is not same issue as Problem 1 , this is different

Below is my agent 

{
  "name": "AI Agent",
  "type": "conversational_flow",
  "description": "AI Agent to convert Conversation via LLMs",
  "memory": {
    "type": "conversation_index"
  },
  "app_type": "rag",
  "tools": [
    {
      "type": "SearchIndexTool",
      "name":"hybrid_search",
      "parameters": {
        "input": "{\"index\": \"${parameters.index}\", \"query\": ${parameters.query} }",
        "query": {
            "_source": "vector_column",
            "size": 30,
            "query": {
                "hybrid": {
                    "queries": [
                        {
                            "match": {
                                "vector_column": {
                                    "query": "${parameters.question}"
                                }
                            }
                        },
                        {
                            "neural": {
                                "vector_column_vector": {
                                    "query_text": "${parameters.question}",
                                    "model_id": "xB_DyY0BToKzewPCzvr8",
                                    "k": 30
                                }
                            }
                        }
                    ]
                }
            }
        }
      }
    },
        {
            "type": "MLModelTool",
            "name": "llm_response",
            "description": "A general tool to answer any question",
            "parameters": {
                "model_id": "eiC2T44BToKzewPCoQyq",
                "inputs": "<s> [INST] You are a helpful chatbot assistant which provides answer based on the context given. Do not give any extra information. \nGenerate a concise and informative answer in less than 100 words for the given question.\nCONTEXT: \n${parameters.hybrid_search.output:-}\n\n${parameters.chat_history:-}\n\nHuman:${parameters.question}\n\n\n\nAssistant:[/INST]"
            }
        }
  ]
}

This is message Response with search POST /_plugins/_ml/memory/xSDabo4BToKzewPCJxU1/_search Below is JSON body

{
  "query": {
    "match_all": {}
  },
  "sort": [
    {
      "create_time": {
        "order": "desc"
      }
    }
  ],
  "size":100
}

Below message you can see i am getting 3 JSON blocks one for

Why i am getting null as origin when i do not have anything as such in agent creation

{
    "took": 0,
    "timed_out": false,
    "_shards": {
        "total": 1,
        "successful": 1,
        "skipped": 0,
        "failed": 0
    },
    "hits": {
        "total": {
            "value": 3,
            "relation": "eq"
        },
        "max_score": null,
        "hits": [
            {
                "_index": ".plugins-ml-memory-message",
                "_id": "yCDabo4BToKzewPCtxUo",
                "_version": 1,
                "_seq_no": 11751,
                "_primary_term": 1,
                "_score": null,
                "_source": {
                    "input": "what is the genere of rush movie?",
                    "memory_id": "xSDabo4BToKzewPCJxU1",
                    "trace_number": 2,
                    "create_time": "2024-03-24T05:06:51.304367877Z",
                    "additional_info": {},
                    "response": " The genre of the movie \"Rush\" is action.",
                    "origin": "llm_response",
                    "parent_message_id": "xiDabo4BToKzewPCthVF",
                    "prompt_template": null
                },
                "sort": [
                    1711256811304
                ]
            },
            {
                "_index": ".plugins-ml-memory-message",
                "_id": "xyDabo4BToKzewPCthV6",
                "_version": 1,
                "_seq_no": 11750,
                "_primary_term": 1,
                "_score": null,
                "_source": {
                    "input": "what is the genere of rush movie?",
                    "memory_id": "xSDabo4BToKzewPCJxU1",
                    "trace_number": 1,
                    "create_time": "2024-03-24T05:06:51.130383384Z",
                    "additional_info": {},
                    "response": "{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Rush, year is 2013, budget is 500000, earning is 300000, genere is action.\"},\"_id\":\"tt1979320\",\"_score\":1.0}\n{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Avatar, year is 2009, budget is 1200000, earning is 6000000, genere is Sci-fi.\"},\"_id\":\"tt1979323\",\"_score\":0.055511523}\n{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Dr Strange, year is 2022, budget is 2500000, earning is 5000000, genere is Sci-fi.\"},\"_id\":\"tt1979322\",\"_score\":0.04304528}\n{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Jurrasic, year is 2014, budget is 1500000, earning is 2300000, genere is action.\"},\"_id\":\"tt1979321\",\"_score\":0.004722477}\n",
                    "origin": "hybrid_search",
                    "parent_message_id": "xiDabo4BToKzewPCthVF",
                    "prompt_template": null
                },
                "sort": [
                    1711256811130
                ]
            },
            {
                "_index": ".plugins-ml-memory-message",
                "_id": "xiDabo4BToKzewPCthVF",
                "_version": 2,
                "_seq_no": 11752,
                "_primary_term": 1,
                "_score": null,
                "_source": {
                    "input": "what is the genere of rush movie?",
                    "memory_id": "xSDabo4BToKzewPCJxU1",
                    "trace_number": null,
                    "create_time": "2024-03-24T05:06:51.07747317Z",
                    "additional_info": {},
                    "response": " The genre of the movie \"Rush\" is action.",
                    "origin": null,
                    "parent_message_id": null,
                    "prompt_template": null
                },
                "sort": [
                    1711256811077
                ]
            }
        ]
    }
}

Question: 4 max_results=5&next_token=3 work this does not work with Search API (_search), it works only with /_plugins/_ml/memory/xSDabo4BToKzewPCJxU1/messages

This is the reason i used descending order in the JSON body as mentioned above but then i do not have scroll id to do pagination

Thanks for reviewing this.

RamDa

ylwu-amzn commented 5 months ago

@ramda1234786 To reply your last comment

Question 1:

Create memory API only requires name, other fields will be filled automatically. https://opensearch.org/docs/latest/ml-commons-plugin/api/memory-apis/create-memory/

Question 3:

The origin field stores the tool name in trace data (trace_number is not null), and it's expected to be null for non-trace data ("trace_number": null).

For example your Agent has a SearchIndexTool with name hybrid_search

      "type": "SearchIndexTool",
      "name":"hybrid_search",

That's why you see some message with "origin": "hybrid_search", in search message response

Check this tutorial https://github.com/opensearch-project/ml-commons/blob/2.x/docs/tutorials/agent_framework/RAG_with_conversational_flow_agent.md

For debugging purposes, each message has its own trace data. To get trace data, call the Get Traces API

Generally you don't need to show trace data. You can use such query to search message with desc order

GET /_plugins/_ml/memory/<memory_id>/_search
{
  "query": {
    "bool": {
      "must_not": [
        {
          "exists": {
            "field": "trace_number"
          }
        }
      ]
    }
  },
  "sort": [
    {
      "create_time": {
        "order": "desc"
      }
    }
  ]
}

Question4:

Have you tried such query?

GET /_plugins/_ml/memory/<memory_id>/_search
{
  "query": {
    "bool": {
      "must_not": [
        {
          "exists": {
            "field": "trace_number"
          }
        }
      ]
    }
  },
  "sort": [
    {
      "create_time": {
        "order": "desc"
      }
    }
  ],
  "from": 10, 
  "size": 10
}
Zhangxunmt commented 5 months ago

@ramda1234786 To summarize your questions, we can make 2 updates to address your concerns.

  1. Add a parameter in the list message API like "/_plugins/_ml/memory/xSDabo4BToKzewPCJxU1/messages?desc=true" to list messages in descending order.
  2. Have the conversational_flow Agent to copy the "origin" value from the last trace into the final message.

In the meantime, can you try using the above search query to get the latest message and pagination. For the "origin" value, you can obtain it from the traces or just check the name of the tools in the Agent.

ramda1234786 commented 5 months ago

"For the "origin" value, you can obtain it from the traces or just check the name of the tools in the Agent."

In the traces, it works well, this one i highlighted in the first message. Concern is that extra"origin":null when this is not part of the agent


Note : Traces of messages shows only 2 traces which is perfect as i am using only 2 steps in agent

If we can address this two problem, it will be helpful and memory API will become more mature. Thanks

ylwu-amzn commented 5 months ago

@Zhangxunmt I think we should put app_type to the parent message origin field. For this case, it will be rag. So user can change agent in future, but if they share the same app_type, they can read the old messages and continue the conversation.

@ramda1234786 What value do you expect to see for the origin field ?

ramda1234786 commented 5 months ago

if you see the above messages, i am getting 3 origin in response

Below message you can see i am getting 3 JSON blocks one for

"origin": "llm_response"
"origin": "hybrid_search"
"origin": null

I am getting extra, which i do not need it in response as i have already got the 

"origin": "llm_response"
"origin": "hybrid_search"

the last extra "origin": null block is not needed as per the agent configuration

{
    "took": 0,
    "timed_out": false,
    "_shards": {
        "total": 1,
        "successful": 1,
        "skipped": 0,
        "failed": 0
    },
    "hits": {
        "total": {
            "value": 3,
            "relation": "eq"
        },
        "max_score": null,
        "hits": [
            {
                "_index": ".plugins-ml-memory-message",
                "_id": "yCDabo4BToKzewPCtxUo",
                "_version": 1,
                "_seq_no": 11751,
                "_primary_term": 1,
                "_score": null,
                "_source": {
                    "input": "what is the genere of rush movie?",
                    "memory_id": "xSDabo4BToKzewPCJxU1",
                    "trace_number": 2,
                    "create_time": "2024-03-24T05:06:51.304367877Z",
                    "additional_info": {},
                    "response": " The genre of the movie \"Rush\" is action.",
                    "origin": "llm_response",
                    "parent_message_id": "xiDabo4BToKzewPCthVF",
                    "prompt_template": null
                },
                "sort": [
                    1711256811304
                ]
            },
            {
                "_index": ".plugins-ml-memory-message",
                "_id": "xyDabo4BToKzewPCthV6",
                "_version": 1,
                "_seq_no": 11750,
                "_primary_term": 1,
                "_score": null,
                "_source": {
                    "input": "what is the genere of rush movie?",
                    "memory_id": "xSDabo4BToKzewPCJxU1",
                    "trace_number": 1,
                    "create_time": "2024-03-24T05:06:51.130383384Z",
                    "additional_info": {},
                    "response": "{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Rush, year is 2013, budget is 500000, earning is 300000, genere is action.\"},\"_id\":\"tt1979320\",\"_score\":1.0}\n{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Avatar, year is 2009, budget is 1200000, earning is 6000000, genere is Sci-fi.\"},\"_id\":\"tt1979323\",\"_score\":0.055511523}\n{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Dr Strange, year is 2022, budget is 2500000, earning is 5000000, genere is Sci-fi.\"},\"_id\":\"tt1979322\",\"_score\":0.04304528}\n{\"_index\":\"movie\",\"_source\":{\"vector_column\":\"The title is Jurrasic, year is 2014, budget is 1500000, earning is 2300000, genere is action.\"},\"_id\":\"tt1979321\",\"_score\":0.004722477}\n",
                    "origin": "hybrid_search",
                    "parent_message_id": "xiDabo4BToKzewPCthVF",
                    "prompt_template": null
                },
                "sort": [
                    1711256811130
                ]
            },
            {
                "_index": ".plugins-ml-memory-message",
                "_id": "xiDabo4BToKzewPCthVF",
                "_version": 2,
                "_seq_no": 11752,
                "_primary_term": 1,
                "_score": null,
                "_source": {
                    "input": "what is the genere of rush movie?",
                    "memory_id": "xSDabo4BToKzewPCJxU1",
                    "trace_number": null,
                    "create_time": "2024-03-24T05:06:51.07747317Z",
                    "additional_info": {},
                    "response": " The genre of the movie \"Rush\" is action.",
                    "origin": null,
                    "parent_message_id": null,
                    "prompt_template": null
                },
                "sort": [
                    1711256811077
                ]
            }
        ]
    }
}