I tried to run the rayllm frontend on the deployed rayllm head pod:
serve run rayllm.frontend.app:app --non-blocking
But got:
2024-03-11 16:09:55,259 INFO scripts.py:438 -- Running import path: 'rayllm.frontend.app:app'.
[WARNING 2024-03-11 16:09:57,981] api.py: 372 DeprecationWarning: `route_prefix` in `@serve.deployment` has been deprecated. To specify a route prefix for an application, pass it into `serve.run` instead.
[WARNING 2024-03-11 16:09:59,475] mongo_secrets.py: 35 Failed to retrieve mongo secret, Exception: Unable to locate credentials
No MongoDB logger defined, will default to the CSVLogger
Traceback (most recent call last):
File "/home/ray/anaconda3/bin/serve", line 8, in <module>
sys.exit(cli())
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 1157, in __call__
return self.main(*args, **kwargs)
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 1078, in main
rv = self.invoke(ctx)
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 1688, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 1434, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 783, in invoke
return __callback(*args, **kwargs)
File "/home/ray/anaconda3/lib/python3.9/site-packages/ray/serve/scripts.py", line 440, in run
import_attr(import_path), args_dict
File "/home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/utils.py", line 1182, in import_attr
module = importlib.import_module(module_name)
File "/home/ray/anaconda3/lib/python3.9/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1030, in _gcd_import
File "<frozen importlib._bootstrap>", line 1007, in _find_and_load
File "<frozen importlib._bootstrap>", line 986, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 680, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 850, in exec_module
File "<frozen importlib._bootstrap>", line 228, in _call_with_frames_removed
File "/home/ray/anaconda3/lib/python3.9/site-packages/rayllm/frontend/app.py", line 61, in <module>
ALL_MODELS = sdk.models()
File "/home/ray/anaconda3/lib/python3.9/site-packages/rayllm/sdk.py", line 82, in models
models = get_openai_client().models.list()
File "/home/ray/anaconda3/lib/python3.9/site-packages/rayllm/sdk.py", line 75, in get_openai_client
backend = get_aviary_backend()
File "/home/ray/anaconda3/lib/python3.9/site-packages/rayllm/sdk.py", line 60, in get_aviary_backend
raise URLNotSetException("AVIARY_URL or OPENAI_API_BASE must be set")
rayllm.sdk.URLNotSetException: AVIARY_URL or OPENAI_API_BASE must be set
I don't see any document mentioning the value of AVIARY_URL. I tried with export AVIARY_URL="http://localhost:8000" but got a different error:
2024-03-11 16:12:07,032 INFO scripts.py:438 -- Running import path: 'rayllm.frontend.app:app'.
[WARNING 2024-03-11 16:12:09,659] api.py: 372 DeprecationWarning: `route_prefix` in `@serve.deployment` has been deprecated. To specify a route prefix for an application, pass it into `serve.run` instead.
[WARNING 2024-03-11 16:12:11,116] mongo_secrets.py: 35 Failed to retrieve mongo secret, Exception: Unable to locate credentials
No MongoDB logger defined, will default to the CSVLogger
Connecting to Aviary backend at: http://localhost:8000/v1
[INFO 2024-03-11 16:12:11,258] _base_client.py: 952 Retrying request to /models in 0.834944 seconds
[INFO 2024-03-11 16:12:12,101] _base_client.py: 952 Retrying request to /models in 1.502290 seconds
Traceback (most recent call last):
File "/home/ray/anaconda3/lib/python3.9/site-packages/httpcore/_exceptions.py", line 10, in map_exceptions
yield
File "/home/ray/anaconda3/lib/python3.9/site-packages/httpcore/_sync/http11.py", line 142, in _send_request_headers
event = h11.Request(
File "/home/ray/anaconda3/lib/python3.9/site-packages/h11/_events.py", line 96, in __init__
self, "headers", normalize_and_validate(headers, _parsed=_parsed)
File "/home/ray/anaconda3/lib/python3.9/site-packages/h11/_headers.py", line 164, in normalize_and_validate
validate(_field_value_re, value, "Illegal header value {!r}", value)
File "/home/ray/anaconda3/lib/python3.9/site-packages/h11/_util.py", line 91, in validate
raise LocalProtocolError(msg)
h11._util.LocalProtocolError: Illegal header value b'Bearer '
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/ray/anaconda3/bin/serve", line 8, in <module>
sys.exit(cli())
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 1157, in __call__
return self.main(*args, **kwargs)
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 1078, in main
rv = self.invoke(ctx)
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 1688, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 1434, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/ray/anaconda3/lib/python3.9/site-packages/click/core.py", line 783, in invoke
return __callback(*args, **kwargs)
File "/home/ray/anaconda3/lib/python3.9/site-packages/ray/serve/scripts.py", line 440, in run
import_attr(import_path), args_dict
File "/home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/utils.py", line 1182, in import_attr
module = importlib.import_module(module_name)
File "/home/ray/anaconda3/lib/python3.9/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1030, in _gcd_import
File "<frozen importlib._bootstrap>", line 1007, in _find_and_load
File "<frozen importlib._bootstrap>", line 986, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 680, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 850, in exec_module
File "<frozen importlib._bootstrap>", line 228, in _call_with_frames_removed
File "/home/ray/anaconda3/lib/python3.9/site-packages/rayllm/frontend/app.py", line 61, in <module>
ALL_MODELS = sdk.models()
File "/home/ray/anaconda3/lib/python3.9/site-packages/rayllm/sdk.py", line 82, in models
models = get_openai_client().models.list()
File "/home/ray/anaconda3/lib/python3.9/site-packages/openai/resources/models.py", line 75, in list
return self._get_api_list(
File "/home/ray/anaconda3/lib/python3.9/site-packages/openai/_base_client.py", line 1137, in get_api_list
return self._request_api_list(model, page, opts)
File "/home/ray/anaconda3/lib/python3.9/site-packages/openai/_base_client.py", line 982, in _request_api_list
return self.request(page, options, stream=False)
File "/home/ray/anaconda3/lib/python3.9/site-packages/openai/_base_client.py", line 853, in request
return self._request(
File "/home/ray/anaconda3/lib/python3.9/site-packages/openai/_base_client.py", line 896, in _request
return self._retry_request(
File "/home/ray/anaconda3/lib/python3.9/site-packages/openai/_base_client.py", line 958, in _retry_request
return self._request(
File "/home/ray/anaconda3/lib/python3.9/site-packages/openai/_base_client.py", line 896, in _request
return self._retry_request(
File "/home/ray/anaconda3/lib/python3.9/site-packages/openai/_base_client.py", line 958, in _retry_request
return self._request(
File "/home/ray/anaconda3/lib/python3.9/site-packages/openai/_base_client.py", line 905, in _request
raise APIConnectionError(request=request) from err
openai.APIConnectionError: Connection error.
I tried to run the rayllm frontend on the deployed rayllm head pod:
But got:
I don't see any document mentioning the value of
AVIARY_URL
. I tried withexport AVIARY_URL="http://localhost:8000"
but got a different error: