Stop sequence key name for meta is not supported, For meta-model (exp: meta.llama2-13b-chat-v1) in BedrockChat

[Sanzid88888]

Checked other resources

• [X] I added a very descriptive title to this issue.
• [X] I searched the LangChain documentation with the integrated search.
• [X] I used the GitHub search to find a similar question and didn't find it.
• [X] I am sure that this is a bug in LangChain rather than my code.
• [X] The bug is not resolved by updating to the latest stable version of LangChain (or the specific integration package).

Example Code

 def get_language_model(self, streaming: bool = False, temperature: int = 0, callbacks: Callbacks = None) -> BaseLanguageModel:
    """
    Get the language model.

    Args:
        streaming (bool): Use streaming or not.
        temperature (int): Temperature for language model generation.
        callbacks (Callbacks): Callbacks for language model.

    Returns:
        BaseLanguageModel: The language model.
    """
    llm_token_usage_callback = llm_token_usage_callback_var.get()
    callbacks = callbacks or []
    if llm_token_usage_callback and self.include_token_usage_cb:
        callbacks.append(llm_token_usage_callback)
    if logger.is_debug:
        callbacks.append(LLMLogCallbackHandlerAsync())
    if self._chat_model.openai_api_key:
        return ChatOpenAI(openai_api_key=self._chat_model.openai_api_key, temperature=temperature, model=self._chat_model.model, streaming=streaming, callbacks=callbacks)
    if self._chat_model.google_api_key:
        safety_settings = {
            HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
            HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
            HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
            HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE
        }
        return ChatGoogleGenerativeAI(
            google_api_key=self._chat_model.google_api_key, temperature=temperature, model=self._chat_model.model,
            streaming=streaming, callbacks=callbacks, convert_system_message_to_human=True, safety_settings=safety_settings
        )

    client = boto3.client(
        'bedrock-runtime',
        region_name=self._chat_model.bedrock_aws_region_name,
        aws_access_key_id=self._chat_model.bedrock_aws_access_key_id,
        aws_secret_access_key=self._chat_model.bedrock_aws_secret_access_key
    )

    return BedrockChat(client=client, model_id=self._chat_model.model, streaming=streaming, callbacks=callbacks, model_kwargs={"temperature": temperature})

async def _get_agent_executor_async(self) -> AgentExecutor:
    """
    Prepare agent using tools and prompt

    Returns:
        AgentExecutor: AgentExecutor to invoke the Agent.
    """
    self.tools = await self._get_tools()
    llm = LLMSelector(self.model).get_language_model().with_config(RunnableConfig(run_name="Agent"))
    prompt = ChatPromptTemplate.from_messages([
        (EnumChatMessageType.SYSTEM, REACT_AGENT_SYSTEM_TEMPLATE),
        (EnumChatMessageType.HUMAN, REACT_AGENT_USER_TEMPLATE),
    ]).partial(system_context=self.model.system_context or '', human_context=self.model.human_context or '')
    agent = create_react_agent(llm=llm, tools=self.tools, prompt=prompt)  # noqa
    return AgentExecutor(agent=agent, tools=self.tools, handle_parsing_errors=self._handle_parser_exception, max_iterations=3)

Error Message and Stack Trace (if applicable)

def _prepare_input_and_invoke_stream(
    self,
    prompt: Optional[str] = None,
    system: Optional[str] = None,
    messages: Optional[List[Dict]] = None,
    stop: Optional[List[str]] = None,
    run_manager: Optional[CallbackManagerForLLMRun] = None,
    **kwargs: Any,
) -> Iterator[GenerationChunk]:
    _model_kwargs = self.model_kwargs or {}
    provider = self._get_provider()

    if stop:
        if provider not in self.provider_stop_sequence_key_name_map:
            raise ValueError(
                f"Stop sequence key name for {provider} is not supported."
            )

        # stop sequence from _generate() overrides
        # stop sequences in the class attribute
        _model_kwargs[self.provider_stop_sequence_key_name_map.get(provider)] = stop

    if provider == "cohere":
        _model_kwargs["stream"] = True

    params = {**_model_kwargs, **kwargs}

    if self._guardrails_enabled:
        params.update(self._get_guardrails_canonical())

    input_body = LLMInputOutputAdapter.prepare_input(
        provider=provider,
        prompt=prompt,
        system=system,
        messages=messages,
        model_kwargs=params,
    )
    body = json.dumps(input_body)

    request_options = {
        "body": body,
        "modelId": self.model_id,
        "accept": "application/json",
        "contentType": "application/json",
    }

    if self._guardrails_enabled:
        request_options["guardrail"] = "ENABLED"
        if self.guardrails.get("trace"):  # type: ignore[union-attr]
            request_options["trace"] = "ENABLED"

    try:
        response = self.client.invoke_model_with_response_stream(**request_options)

    except Exception as e:
        raise ValueError(f"Error raised by bedrock service: {e}")

    for chunk in LLMInputOutputAdapter.prepare_output_stream(
        provider, response, stop, True if messages else False
    ):
        yield chunk
        # verify and raise callback error if any middleware intervened
        self._get_bedrock_services_signal(chunk.generation_info)  # type: ignore[arg-type]

        if run_manager is not None:
            run_manager.on_llm_new_token(chunk.text, chunk=chunk)

Description

When invoking BedrockChat with the meta-model, an error occurs stating that the "Stop sequence" key name is not supported. This issue arises because the create_react_agent function includes a parameter llm_with_stop which is bound to the stop sequence ["\nObservation"], but the meta-model does not support the use of a stop sequence.

System Info

langchain==0.1.12 langchain-community==0.0.28 langchain-core==0.1.32 langchain-google-genai==0.0.11 langchain-openai==0.0.8 langchain-text-splitters==0.0.1 langchainhub==0.1.15

Platform: Mac Python 3.11.6

[SteveMama]

quick solution that worked for me and got the sql agent up and working without issue: llm.provider_stop_sequence_key_name_map = {'anthropic': 'stop_sequences', 'amazon': 'stopSequences', 'ai21': 'stop_sequences', 'cohere': 'stop_sequences', 'mistral': 'stop','meta': 'stop' }

FYI: I'm working on using langchain create_sql_agent to work with Llama 3 instruct model on bedrock. Implementing this small change has gotten the agent to work without any issues!

[AyushSonuu]

TypeError: Additional kwargs key generation_token_count already exists in left dict and value has unsupported type <class 'int'> it s giving this new issue

quick solution that worked for me and got the sql agent up and working without issue: llm.provider_stop_sequence_key_name_map = {'anthropic': 'stop_sequences', 'amazon': 'stopSequences', 'ai21': 'stop_sequences', 'cohere': 'stop_sequences', 'mistral': 'stop','meta': 'stop' }

FYI: I'm working on using langchain create_sql_agent to work with Llama 3 instruct model on bedrock. Implementing this small change has gotten the agent to work without any issues!

[sameeroz]

@AyushSonuu i got the same error as you, when trying to implement @SteveMama solution