Ideally when we set a field to be configurable, it should be updated accordingly when new configurable values are given by per_req_config_modifier.

However, it seems only model_kwargs ('user': 'some_user') is being correctly passed, while the other configurable variables such as openai_api_base, default_headers, etc. are not correctly passed to the final client.

Below is the example code:

def fetch_config_from_header(config: Dict[str, Any], req: Request) -> Dict[str, Any]:
    """ All supported types: 'name', 'cache', 'verbose', 'callbacks', 'tags', 'metadata', 'custom_get_token_ids', 'callback_manager', 'client', 'async_client', 'model_name', 'temperature', 'model_kwargs', 'openai_api_key', 'openai_api_base', 'openai_organization', 'openai_proxy', 'request_timeout', 'max_retries', 'streaming', 'n', 'max_tokens', 'tiktoken_model_name', 'default_headers', 'default_query', 'http_client', 'http_async_client']"""

    config = config.copy()
    configurable = config.get("configurable", {})

    if "x-model-name" in req.headers:
        configurable["model_name"] = req.headers["x-model-name"]
    else:
        raise HTTPException(401, "No model name provided")

    if "x-api-key" in req.headers:
        configurable["default_headers"] = {
            "Content-Type":"application/json",
            "api-key": req.headers["x-api-key"]
        }
    else:
        raise HTTPException(401, "No API key provided")

    if "x-model-kwargs" in req.headers:
        configurable["model_kwargs"] = json.loads(req.headers["x-model-kwargs"])
    else:
        raise HTTPException(401, "No model arguments provided")

    configurable["openai_api_base"] = f"https://someendpoint.com/{req.headers['x-model-name']}"
    config["configurable"] = configurable
    return config

chat_model = ChatOpenAI(
    model_name = "some_model",
    model_kwargs = {},
    default_headers = {},
    openai_api_key = "placeholder",
    openai_api_base = "placeholder").configurable_fields(
        model_name = ConfigurableField(id="model_name"),
        model_kwargs = ConfigurableField(id="model_kwargs"),
        default_headers = ConfigurableField(id="default_headers"),
        openai_api_base = ConfigurableField(id="openai_api_base"),
    )

chain = prompt_template | chat_model | StrOutputParser()
add_routes(
    app,
    chain.with_types(input_type=InputChat),
    path="/some_chain",
    disabled_endpoints=["playground"],
    per_req_config_modifier=fetch_config_from_header,
)

Some of the related values from certain functions

# returned value of config in fetch_config_from_header()
{'configurable': {'model_name': 'some_model', 'default_headers': {'Content-Type': 'application/json', 'api-key': 'some_api_key'}, 'model_kwargs': {'user': 'some_user'}, 'openai_api_base': 'https://someendpoint.com/some_model'}

# returned value (projected_config) from langserve's api_handler.py _unpack_request_config()
{'configurable': {'model_name': 'some_model', 'default_headers': {'Content-Type': 'application/json', 'api-key': 'some_api_key'}, 'model_kwargs': {'user': 'some_user'}, 'openai_api_base': 'https://someendpoint.com/some_model'}

# values of config and input_ from langserve's api_handler.py APIHandler.invoke()
config: {'configurable': {'model_name': 'some_model', 'default_headers': {'Content-Type': 'application/json', 'api-key': 'some_api_key'}, 'model_kwargs': {'user': 'some_user'}, 'openai_api_base': 'https://someendpoint.com/some_model', 'run_name': '/some_chain', 'metadata': {'__useragent': 'python-requests/2.32.3', '__langserve_version': '0.2.2', '__langserve_endpoint': 'invoke'}, 'run_id': 'xxxxx'}
input_: {'inputs': 'some input string here....'}

# values of input, config and kwargs from langchain_core's runnables.base.py RunnableBindingBase.ainvoke() 
input: {'inputs': 'some input string here....'}
config: {'configurable': {'model_name': 'some_model', 'default_headers': {'Content-Type': 'application/json', 'api-key': 'some_api_key'}, 'model_kwargs': {'user': 'some_user'}, 'openai_api_base': 'https://someendpoint.com/some_model', 'run_name': '/some_chain', 'metadata': {'__useragent': 'python-requests/2.32.3', '__langserve_version': '0.2.2', '__langserve_endpoint': 'invoke'}, 'run_id': 'xxxxx', 'callbacks': [<langserve.callbacks.AsyncEventAggregatorCallback object at 0x7fd8fe6878c0>]}
kwargs: {}

# values of kwargs from langchain_core's language_models.chat_models.py BaseChatModel.agenerate_prompt()
{'tags': [], 'metadata': {'__useragent': 'python-requests/2.32.3', '__langserve_version': '0.2.2', '__langserve_endpoint': 'invoke', 'model_name': 'some_model', 'openai_api_base': 'https://someendpoint.com/some_model', 'run_name': None, 'run_id': None}

# value of params from langchain_openai's chat_models.base.py BaseChatOpenAI._agenerate()
{'model': 'some_model', 'stream': False, 'n': 1, 'temperature': 0.7, 'user': 'some_user'}

# values of cast_to, opts in openai's _base_client.py AsyncAPIClient.post()
cast_to: <class 'openai.types.chat.chat_completion.ChatCompletion'>
opts: method='post' url='/chat/completions' params={} headers=NOT_GIVEN max_retries=NOT_GIVEN timeout=NOT_GIVEN files=None idempotency_key=None post_parser=NOT_GIVEN json_data={'messages': [{'content': 'some_content', 'role': 'system'}], 'model': 'some_model', 'n': 1, 'stream': False, 'temperature': 0.7, 'user': 'some_user'} extra_json=None

Environment

langchain==0.2.5
langchain-community==0.2.5
langchain-core==0.2.9
langchain-experimental==0.0.60
langchain-openai==0.1.9
langchain-text-splitters==0.2.1
langgraph==0.1.1
langserve==0.2.2
langsmith==0.1.82
openai==1.35.3

platform = linux
python version = 3.12.4

langchain-ai / langserve

Langserve configurable does not set/update the ConfigurableField variables that is given by per_req_config_modifier #693

Environment

Related issues:

377

314

652