Upgrade DuckDuckGo Search package

[samuelp-mw]

Context

DuckDuckGo currently cannot be used due to the following error :

Traceback (most recent call last):
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 49, in _get_url
opengpts-backend   |     raise DuckDuckGoSearchException("Ratelimit")
opengpts-backend   | duckduckgo_search.exceptions.DuckDuckGoSearchException: Ratelimit
opengpts-backend   | 
opengpts-backend   | The above exception was the direct cause of the following exception:
opengpts-backend   | 
opengpts-backend   | Traceback (most recent call last):
opengpts-backend   |   File "/backend/app/stream.py", line 52, in to_sse
opengpts-backend   |     async for chunk in messages_stream:
opengpts-backend   |   File "/backend/app/stream.py", line 21, in astream_messages
opengpts-backend   |     async for event in app.astream_events(
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 4711, in astream_events
opengpts-backend   |     async for item in self.bound.astream_events(
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1137, in astream_events
opengpts-backend   |     async for log in _astream_log_implementation(  # type: ignore[misc]
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/tracers/log_stream.py", line 616, in _astream_log_implementation
opengpts-backend   |     await task
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/tracers/log_stream.py", line 570, in consume_astream
opengpts-backend   |     async for chunk in runnable.astream(input, config, **kwargs):
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/configurable.py", line 199, in astream
opengpts-backend   |     async for chunk in runnable.astream(input, config, **kwargs):
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/configurable.py", line 199, in astream
opengpts-backend   |     async for chunk in runnable.astream(input, config, **kwargs):
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 4698, in astream
opengpts-backend   |     async for item in self.bound.astream(
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 4698, in astream
opengpts-backend   |     async for item in self.bound.astream(
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/__init__.py", line 900, in astream
opengpts-backend   |     _panic_or_proceed(done, inflight, step)
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/__init__.py", line 1033, in _panic_or_proceed
opengpts-backend   |     raise exc
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langgraph/pregel/__init__.py", line 1206, in _aconsume
opengpts-backend   |     async for _ in iterator:
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 2900, in astream
opengpts-backend   |     async for chunk in self.atransform(input_aiter(), config, **kwargs):
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 2883, in atransform
opengpts-backend   |     async for chunk in self._atransform_stream_with_config(
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1979, in _atransform_stream_with_config
opengpts-backend   |     chunk: Output = await asyncio.create_task(  # type: ignore[call-arg]
opengpts-backend   |                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/tracers/log_stream.py", line 237, in tap_output_aiter
opengpts-backend   |     async for chunk in output:
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 2853, in _atransform
opengpts-backend   |     async for output in final_pipeline:
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1316, in atransform
opengpts-backend   |     async for chunk in input:
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 4166, in atransform
opengpts-backend   |     async for output in self._atransform_stream_with_config(
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 1979, in _atransform_stream_with_config
opengpts-backend   |     chunk: Output = await asyncio.create_task(  # type: ignore[call-arg]
opengpts-backend   |                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/tracers/log_stream.py", line 237, in tap_output_aiter
opengpts-backend   |     async for chunk in output:
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 4135, in _atransform
opengpts-backend   |     output = await acall_func_with_variable_args(
opengpts-backend   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/backend/app/agent_types/openai_agent.py", line 70, in call_tool
opengpts-backend   |     responses = await tool_executor.abatch(actions)
opengpts-backend   |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 738, in abatch
opengpts-backend   |     return await gather_with_concurrency(configs[0].get("max_concurrency"), *coros)
opengpts-backend   |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/utils.py", line 63, in gather_with_concurrency
opengpts-backend   |     return await asyncio.gather(*coros)
opengpts-backend   |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/base.py", line 735, in ainvoke
opengpts-backend   |     return await self.ainvoke(input, config, **kwargs)
opengpts-backend   |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langgraph/utils.py", line 62, in ainvoke
opengpts-backend   |     ret = await self.afunc(
opengpts-backend   |           ^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langgraph/prebuilt/tool_executor.py", line 66, in _aexecute
opengpts-backend   |     output = await tool.ainvoke(tool_invocation.tool_input, config)
opengpts-backend   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/tools.py", line 258, in ainvoke
opengpts-backend   |     return await self.arun(
opengpts-backend   |            ^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/tools.py", line 513, in arun
opengpts-backend   |     raise e
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/tools.py", line 472, in arun
opengpts-backend   |     else await self._arun(*tool_args, **tool_kwargs)
opengpts-backend   |          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/tools.py", line 324, in _arun
opengpts-backend   |     return await run_in_executor(None, self._run, *args, **kwargs)
opengpts-backend   |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_core/runnables/config.py", line 514, in run_in_executor
opengpts-backend   |     return await asyncio.get_running_loop().run_in_executor(
opengpts-backend   |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run
opengpts-backend   |     result = self.fn(*self.args, **self.kwargs)
opengpts-backend   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_community/tools/ddg_search/tool.py", line 39, in _run
opengpts-backend   |     return self.api_wrapper.run(query)
opengpts-backend   |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_community/utilities/duckduckgo_search.py", line 81, in run
opengpts-backend   |     results = self._ddgs_text(query)
opengpts-backend   |               ^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_community/utilities/duckduckgo_search.py", line 57, in _ddgs_text
opengpts-backend   |     return [r for r in ddgs_gen]
opengpts-backend   |            ^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/langchain_community/utilities/duckduckgo_search.py", line 57, in <listcomp>
opengpts-backend   |     return [r for r in ddgs_gen]
opengpts-backend   |            ^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 95, in text
opengpts-backend   |     for i, result in enumerate(results, start=1):
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 146, in _text_api
opengpts-backend   |     resp = self._get_url("GET", "https://links.duckduckgo.com/d.js", params=payload)
opengpts-backend   |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
opengpts-backend   |   File "/usr/local/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 53, in _get_url
opengpts-backend   |     raise DuckDuckGoSearchException(f"_get_url() {url} {type(ex).__name__}: {ex}") from ex
opengpts-backend   | duckduckgo_search.exceptions.DuckDuckGoSearchException: _get_url() https://links.duckduckgo.com/d.js DuckDuckGoSearchException: Ratelimit

The issue was mentioned in the following issues :

• https://github.com/langchain-ai/opengpts/issues/258
• https://github.com/langchain-ai/opengpts/issues/247

Resolution

Upgrading DuckDuckGo search to the latest version (5.3.0) seems to fix the issue.

[mkorpela]

Reproduced the OG problem:

File "/Users/mikkokorpela/workspace/opengpts/backend/.venv/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 146, in _text_api
    resp = self._get_url("GET", "https://links.duckduckgo.com/d.js", params=payload)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mikkokorpela/workspace/opengpts/backend/.venv/lib/python3.11/site-packages/duckduckgo_search/duckduckgo_search.py", line 53, in _get_url
    raise DuckDuckGoSearchException(f"_get_url() {url} {type(ex).__name__}: {ex}") from ex
duckduckgo_search.exceptions.DuckDuckGoSearchException: _get_url() https://links.duckduckgo.com/d.js DuckDuckGoSearchException: Ratelimit

[mkorpela]

And this PR fixes the issue. @samuelp-mw thank you for the contribution!