Open vkehfdl1 opened 17 hours ago
Describe the bug A clear and concise description of what the bug is.
To Reproduce Try to use this code.
client = AsyncOpenAI() initial_raw = Raw(pd.read_parquet(raw_path, engine="pyarrow")) initial_corpus = Corpus(pd.read_parquet(corpus_path, engine="pyarrow"), initial_raw) qa = initial_corpus.sample(random_single_hop, n=qa_size).map( lambda df: df.reset_index(drop=True), ).make_retrieval_gt_contents().batch_apply( factoid_query_gen, # query generation client=client, model_name="gpt-4o-2024-08-06", lang="ko", ) qa = qa.batch_apply( make_basic_gen_gt, # answer generation (basic) client=client, model_name="gpt-4o-2024-08-06", lang="ko", ) qa = qa.batch_apply( make_concise_gen_gt, # answer generation (concise) client=client, model_name="gpt-4o-2024-08-06", lang="ko", ).filter( dontknow_filter_rule_based, # filter unanswerable questions lang="ko", client=client, model_name="gpt-4o-2024-08-06", )
Expected behavior A clear and concise description of what you expected to happen.
Full Error log
╭──────────────────────────────────── Traceback (most recent call last) ────────────────────────────────────╮ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/openai/_base_client. │ │ py:1562 in _request │ │ │ │ 1559 │ │ │ kwargs["auth"] = self.custom_auth │ │ 1560 │ │ │ │ 1561 │ │ try: │ │ ❱ 1562 │ │ │ response = await self._client.send( │ │ 1563 │ │ │ │ request, │ │ 1564 │ │ │ │ stream=stream or self._should_stream_response_body(request=request), │ │ 1565 │ │ │ │ **kwargs, │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpx/_client.py:167 │ │ 4 in send │ │ │ │ 1671 │ │ │ │ 1672 │ │ auth = self._build_request_auth(request, auth) │ │ 1673 │ │ │ │ ❱ 1674 │ │ response = await self._send_handling_auth( │ │ 1675 │ │ │ request, │ │ 1676 │ │ │ auth=auth, │ │ 1677 │ │ │ follow_redirects=follow_redirects, │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpx/_client.py:170 │ │ 2 in _send_handling_auth │ │ │ │ 1699 │ │ │ request = await auth_flow.__anext__() │ │ 1700 │ │ │ │ │ 1701 │ │ │ while True: │ │ ❱ 1702 │ │ │ │ response = await self._send_handling_redirects( │ │ 1703 │ │ │ │ │ request, │ │ 1704 │ │ │ │ │ follow_redirects=follow_redirects, │ │ 1705 │ │ │ │ │ history=history, │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpx/_client.py:173 │ │ 9 in _send_handling_redirects │ │ │ │ 1736 │ │ │ for hook in self._event_hooks["request"]: │ │ 1737 │ │ │ │ await hook(request) │ │ 1738 │ │ │ │ │ ❱ 1739 │ │ │ response = await self._send_single_request(request) │ │ 1740 │ │ │ try: │ │ 1741 │ │ │ │ for hook in self._event_hooks["response"]: │ │ 1742 │ │ │ │ │ await hook(response) │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpx/_client.py:177 │ │ 6 in _send_single_request │ │ │ │ 1773 │ │ │ ) │ │ 1774 │ │ │ │ 1775 │ │ with request_context(request=request): │ │ ❱ 1776 │ │ │ response = await transport.handle_async_request(request) │ │ 1777 │ │ │ │ 1778 │ │ assert isinstance(response.stream, AsyncByteStream) │ │ 1779 │ │ response.request = request │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpx/_transports/de │ │ fault.py:377 in handle_async_request │ │ │ │ 374 │ │ │ extensions=request.extensions, │ │ 375 │ │ ) │ │ 376 │ │ with map_httpcore_exceptions(): │ │ ❱ 377 │ │ │ resp = await self._pool.handle_async_request(req) │ │ 378 │ │ │ │ 379 │ │ assert isinstance(resp.stream, typing.AsyncIterable) │ │ 380 │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpcore/_async/conn │ │ ection_pool.py:216 in handle_async_request │ │ │ │ 213 │ │ │ │ closing = self._assign_requests_to_connections() │ │ 214 │ │ │ │ │ 215 │ │ │ await self._close_connections(closing) │ │ ❱ 216 │ │ │ raise exc from None │ │ 217 │ │ │ │ 218 │ │ # Return the response. Note that in this case we still have to manage │ │ 219 │ │ # the point at which the response is closed. │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpcore/_async/conn │ │ ection_pool.py:196 in handle_async_request │ │ │ │ 193 │ │ │ │ │ │ 194 │ │ │ │ try: │ │ 195 │ │ │ │ │ # Send the request on the assigned connection. │ │ ❱ 196 │ │ │ │ │ response = await connection.handle_async_request( │ │ 197 │ │ │ │ │ │ pool_request.request │ │ 198 │ │ │ │ │ ) │ │ 199 │ │ │ │ except ConnectionNotAvailable: │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpcore/_async/conn │ │ ection.py:101 in handle_async_request │ │ │ │ 98 │ │ │ self._connect_failed = True │ │ 99 │ │ │ raise exc │ │ 100 │ │ │ │ ❱ 101 │ │ return await self._connection.handle_async_request(request) │ │ 102 │ │ │ 103 │ async def _connect(self, request: Request) -> AsyncNetworkStream: │ │ 104 │ │ timeouts = request.extensions.get("timeout", {}) │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpcore/_async/http │ │ 11.py:143 in handle_async_request │ │ │ │ 140 │ │ │ with AsyncShieldCancellation(): │ │ 141 │ │ │ │ async with Trace("response_closed", logger, request) as trace: │ │ 142 │ │ │ │ │ await self._response_closed() │ │ ❱ 143 │ │ │ raise exc │ │ 144 │ │ │ 145 │ # Sending the request... │ │ 146 │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpcore/_async/http │ │ 11.py:113 in handle_async_request │ │ │ │ 110 │ │ │ │ │ reason_phrase, │ │ 111 │ │ │ │ │ headers, │ │ 112 │ │ │ │ │ trailing_data, │ │ ❱ 113 │ │ │ │ ) = await self._receive_response_headers(**kwargs) │ │ 114 │ │ │ │ trace.return_value = ( │ │ 115 │ │ │ │ │ http_version, │ │ 116 │ │ │ │ │ status, │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpcore/_async/http │ │ 11.py:186 in _receive_response_headers │ │ │ │ 183 │ │ timeout = timeouts.get("read", None) │ │ 184 │ │ │ │ 185 │ │ while True: │ │ ❱ 186 │ │ │ event = await self._receive_event(timeout=timeout) │ │ 187 │ │ │ if isinstance(event, h11.Response): │ │ 188 │ │ │ │ break │ │ 189 │ │ │ if ( │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpcore/_async/http │ │ 11.py:224 in _receive_event │ │ │ │ 221 │ │ │ │ event = self._h11_state.next_event() │ │ 222 │ │ │ │ │ 223 │ │ │ if event is h11.NEED_DATA: │ │ ❱ 224 │ │ │ │ data = await self._network_stream.read( │ │ 225 │ │ │ │ │ self.READ_NUM_BYTES, timeout=timeout │ │ 226 │ │ │ │ ) │ │ 227 │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/httpcore/_backends/a │ │ nyio.py:35 in read │ │ │ │ 32 │ │ with map_exceptions(exc_map): │ │ 33 │ │ │ with anyio.fail_after(timeout): │ │ 34 │ │ │ │ try: │ │ ❱ 35 │ │ │ │ │ return await self._stream.receive(max_bytes=max_bytes) │ │ 36 │ │ │ │ except anyio.EndOfStream: # pragma: nocover │ │ 37 │ │ │ │ │ return b"" │ │ 38 │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/anyio/streams/tls.py │ │ :205 in receive │ │ │ │ 202 │ │ await self.transport_stream.aclose() │ │ 203 │ │ │ 204 │ async def receive(self, max_bytes: int = 65536) -> bytes: │ │ ❱ 205 │ │ data = await self._call_sslobject_method(self._ssl_object.read, max_bytes) │ │ 206 │ │ if not data: │ │ 207 │ │ │ raise EndOfStream │ │ 208 │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/anyio/streams/tls.py │ │ :147 in _call_sslobject_method │ │ │ │ 144 │ │ │ │ │ if self._write_bio.pending: │ │ 145 │ │ │ │ │ │ await self.transport_stream.send(self._write_bio.read()) │ │ 146 │ │ │ │ │ │ │ ❱ 147 │ │ │ │ │ data = await self.transport_stream.receive() │ │ 148 │ │ │ │ except EndOfStream: │ │ 149 │ │ │ │ │ self._read_bio.write_eof() │ │ 150 │ │ │ │ except OSError as exc: │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/anyio/_backends/_asy │ │ ncio.py:1198 in receive │ │ │ │ 1195 │ │ │ │ and not self._protocol.is_at_eof │ │ 1196 │ │ │ ): │ │ 1197 │ │ │ │ self._transport.resume_reading() │ │ ❱ 1198 │ │ │ │ await self._protocol.read_event.wait() │ │ 1199 │ │ │ │ self._transport.pause_reading() │ │ 1200 │ │ │ else: │ │ 1201 │ │ │ │ await AsyncIOBackend.checkpoint() │ │ │ │ /Users/jeffrey/.pyenv/versions/3.10.14/lib/python3.10/asyncio/locks.py:211 in wait │ │ │ │ 208 │ │ if self._value: │ │ 209 │ │ │ return True │ │ 210 │ │ │ │ ❱ 211 │ │ fut = self._get_loop().create_future() │ │ 212 │ │ self._waiters.append(fut) │ │ 213 │ │ try: │ │ 214 │ │ │ await fut │ │ │ │ /Users/jeffrey/.pyenv/versions/3.10.14/lib/python3.10/asyncio/mixins.py:30 in _get_loop │ │ │ │ 27 │ │ │ │ if self._loop is None: │ │ 28 │ │ │ │ │ self._loop = loop │ │ 29 │ │ if loop is not self._loop: │ │ ❱ 30 │ │ │ raise RuntimeError(f'{self!r} is bound to a different event loop') │ │ 31 │ │ return loop │ │ 32 │ ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────╯ RuntimeError: <asyncio.locks.Event object at 0x1391e42e0 [unset]> is bound to a different event loop The above exception was the direct cause of the following exception: ╭──────────────────────────────────── Traceback (most recent call last) ────────────────────────────────────╮ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/./make_qa.py:72 in <module> │ │ │ │ 69 │ │ 70 │ │ 71 if __name__ == "__main__": │ │ ❱ 72 │ main() │ │ 73 │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/click/core.py:1157 │ │ in __call__ │ │ │ │ 1154 │ │ │ 1155 │ def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any: │ │ 1156 │ │ """Alias for :meth:`main`.""" │ │ ❱ 1157 │ │ return self.main(*args, **kwargs) │ │ 1158 │ │ 1159 │ │ 1160 class Command(BaseCommand): │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/click/core.py:1078 │ │ in main │ │ │ │ 1075 │ │ try: │ │ 1076 │ │ │ try: │ │ 1077 │ │ │ │ with self.make_context(prog_name, args, **extra) as ctx: │ │ ❱ 1078 │ │ │ │ │ rv = self.invoke(ctx) │ │ 1079 │ │ │ │ │ if not standalone_mode: │ │ 1080 │ │ │ │ │ │ return rv │ │ 1081 │ │ │ │ │ # it's not safe to `ctx.exit(rv)` here! │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/click/core.py:1434 │ │ in invoke │ │ │ │ 1431 │ │ │ echo(style(message, fg="red"), err=True) │ │ 1432 │ │ │ │ 1433 │ │ if self.callback is not None: │ │ ❱ 1434 │ │ │ return ctx.invoke(self.callback, **ctx.params) │ │ 1435 │ │ │ 1436 │ def shell_complete(self, ctx: Context, incomplete: str) -> t.List["CompletionItem"]: │ │ 1437 │ │ """Return a list of completions for the incomplete value. Looks │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/click/core.py:783 in │ │ invoke │ │ │ │ 780 │ │ │ │ 781 │ │ with augment_usage_errors(__self): │ │ 782 │ │ │ with ctx: │ │ ❱ 783 │ │ │ │ return __callback(*args, **kwargs) │ │ 784 │ │ │ 785 │ def forward( │ │ 786 │ │ __self, __cmd: "Command", *args: t.Any, **kwargs: t.Any # noqa: B902 │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/./make_qa.py:50 in main │ │ │ │ 47 │ │ │ model_name="gpt-4o-2024-08-06", │ │ 48 │ │ │ lang="ko", │ │ 49 │ │ ) │ │ ❱ 50 │ qa = qa.batch_apply( │ │ 51 │ │ │ make_basic_gen_gt, # answer generation (basic) │ │ 52 │ │ │ client=client, │ │ 53 │ │ │ model_name="gpt-4o-2024-08-06", │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/autorag/data/qa/sche │ │ ma.py:139 in batch_apply │ │ │ │ 136 │ │ qa_dicts = self.data.to_dict(orient="records") │ │ 137 │ │ loop = get_event_loop() │ │ 138 │ │ tasks = [fn(qa_dict, **kwargs) for qa_dict in qa_dicts] │ │ ❱ 139 │ │ results = loop.run_until_complete(process_batch(tasks, batch_size)) │ │ 140 │ │ return QA(pd.DataFrame(results), self.linked_corpus) │ │ 141 │ │ │ 142 │ def batch_filter( │ │ │ │ /Users/jeffrey/.pyenv/versions/3.10.14/lib/python3.10/asyncio/base_events.py:649 in run_until_complete │ │ │ │ 646 │ │ if not future.done(): │ │ 647 │ │ │ raise RuntimeError('Event loop stopped before Future completed.') │ │ 648 │ │ │ │ ❱ 649 │ │ return future.result() │ │ 650 │ │ │ 651 │ def stop(self): │ │ 652 │ │ """Stop running the event loop. │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/autorag/utils/util.p │ │ y:301 in process_batch │ │ │ │ 298 │ │ │ 299 │ for i in range(0, len(tasks), batch_size): │ │ 300 │ │ batch = tasks[i : i + batch_size] │ │ ❱ 301 │ │ batch_results = await asyncio.gather(*batch) │ │ 302 │ │ results.extend(batch_results) │ │ 303 │ │ │ 304 │ return results │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/autorag/data/qa/gene │ │ ration_gt/openai_gen_gt.py:82 in make_basic_gen_gt │ │ │ │ 79 │ │ Default is "en". │ │ 80 │ :return: The output row of the qa dataframe with added "generation_gt" in it. │ │ 81 │ """ │ │ ❱ 82 │ return await make_gen_gt_openai( │ │ 83 │ │ row, client, GEN_GT_SYSTEM_PROMPT["basic"][lang], model_name │ │ 84 │ ) │ │ 85 │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/autorag/data/qa/gene │ │ ration_gt/openai_gen_gt.py:28 in make_gen_gt_openai │ │ │ │ 25 │ passage_str = "\n".join(retrieval_gt_contents) │ │ 26 │ user_prompt = │ │ f"Text:\n<|text_start|>\n{passage_str}\n<|text_end|>\n\nQuestion:\n{query}\n\nAnswer:" │ │ 27 │ │ │ ❱ 28 │ completion = await client.beta.chat.completions.parse( │ │ 29 │ │ model=model_name, │ │ 30 │ │ messages=[ │ │ 31 │ │ │ {"role": "system", "content": system_prompt}, │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/openai/resources/bet │ │ a/chat/completions.py:332 in parse │ │ │ │ 329 │ │ │ **(extra_headers or {}), │ │ 330 │ │ } │ │ 331 │ │ │ │ ❱ 332 │ │ raw_completion = await self._client.chat.completions.create( │ │ 333 │ │ │ messages=messages, │ │ 334 │ │ │ model=model, │ │ 335 │ │ │ response_format=_type_to_response_format(response_format), │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/openai/resources/cha │ │ t/completions.py:1412 in create │ │ │ │ 1409 │ │ timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, │ │ 1410 │ ) -> ChatCompletion | AsyncStream[ChatCompletionChunk]: │ │ 1411 │ │ validate_response_format(response_format) │ │ ❱ 1412 │ │ return await self._post( │ │ 1413 │ │ │ "/chat/completions", │ │ 1414 │ │ │ body=await async_maybe_transform( │ │ 1415 │ │ │ │ { │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/openai/_base_client. │ │ py:1829 in post │ │ │ │ 1826 │ │ opts = FinalRequestOptions.construct( │ │ 1827 │ │ │ method="post", url=path, json_data=body, files=await │ │ async_to_httpx_files(files), **options │ │ 1828 │ │ ) │ │ ❱ 1829 │ │ return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls) │ │ 1830 │ │ │ 1831 │ async def patch( │ │ 1832 │ │ self, │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/openai/_base_client. │ │ py:1523 in request │ │ │ │ 1520 │ │ else: │ │ 1521 │ │ │ retries_taken = 0 │ │ 1522 │ │ │ │ ❱ 1523 │ │ return await self._request( │ │ 1524 │ │ │ cast_to=cast_to, │ │ 1525 │ │ │ options=options, │ │ 1526 │ │ │ stream=stream, │ │ │ │ /Users/jeffrey/PycharmProjects/AutoRAG-tutorial-ko/venv/lib/python3.10/site-packages/openai/_base_client. │ │ py:1596 in _request │ │ │ │ 1593 │ │ │ │ ) │ │ 1594 │ │ │ │ │ 1595 │ │ │ log.debug("Raising connection error") │ │ ❱ 1596 │ │ │ raise APIConnectionError(request=request) from err │ │ 1597 │ │ │ │ 1598 │ │ log.debug( │ │ 1599 │ │ │ 'HTTP Request: %s %s "%i %s"', request.method, request.url, │ │ response.status_code, response.reason_phrase │ ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────╯ APIConnectionError: Connection error.
Code that bug is happened If applicable, add the code that bug is happened. (Especially, your AutoRAG YAML file or python codes that you wrote)
Desktop (please complete the following information):
Additional context Add any other context about the problem here.
This is happening on the python 3.10 version, but not in python 3.9 version. In python 3.9, it retries the request and do not occur event loop error
Describe the bug A clear and concise description of what the bug is.
To Reproduce Try to use this code.
Expected behavior A clear and concise description of what you expected to happen.
Full Error log
Code that bug is happened If applicable, add the code that bug is happened. (Especially, your AutoRAG YAML file or python codes that you wrote)
Desktop (please complete the following information):
Additional context Add any other context about the problem here.