search

miyakogi / pyppeteer

Headless chrome/chromium automation library (unofficial port of puppeteer)
Other
3.57k stars 371 forks source link

pyppeteer throws error if it has to work in cycle: Protocol Error (Runtime.evaluate): Session closed #267

Open banderlog opened 5 years ago

banderlog commented 5 years ago

subj

python >=3.7

websockets==6.0 and websockets==8.1

Code example:

import pyppeteer
import asyncio

async def baz_main():
    browser = await pyppeteer.launch(headless=False, autoClose=False,
                                     args=['--no-sandbox', '--disable-infobars'])

    pages = await browser.pages()
    page = pages[0]
    await page.goto('http://m.reactor.cc')
    #await baz_do(page)
    #await baz_do(page)
    for i in range(5):
        print(i)
        await baz_do(page)
        await asyncio.sleep(3)

    await browser.close()

async def baz_do(page):
    await asyncio.wait([
        page.waitForSelector('a.next'),
        page.click('a.next'),
        page.waitForNavigation(),
    ])
    print(page.url)

asyncio.run(baz_main())
http://m.reactor.cc/18169
1
http://m.reactor.cc/18168
2
http://m.reactor.cc/18167
3
Task exception was never retrieved
future: <Task finished name='Task-88' coro=<Page.waitForNavigation() done, defined at /home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py:887> exception=TimeoutError('Navigation Timeout Exceeded: 30000 ms exceeded.')>
Traceback (most recent call last):
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py", line 938, in waitForNavigation
    raise error
pyppeteer.errors.TimeoutError: Navigation Timeout Exceeded: 30000 ms exceeded.
http://m.reactor.cc/18166
4
Task exception was never retrieved
future: <Task finished name='Task-108' coro=<Page.waitForNavigation() done, defined at /home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py:887> exception=TimeoutError('Navigation Timeout Exceeded: 30000 ms exceeded.')>
Traceback (most recent call last):
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py", line 938, in waitForNavigation
    raise error
pyppeteer.errors.TimeoutError: Navigation Timeout Exceeded: 30000 ms exceeded.
Task exception was never retrieved
future: <Task finished name='Task-109' coro=<_wrap_awaitable() done, defined at /home/kabakov/miniconda3/envs/py37/lib/python3.8/asyncio/tasks.py:677> exception=NetworkError('Protocol Error (Runtime.callFunctionOn): Session closed. Most likely the page has been closed.')>
Traceback (most recent call last):
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/asyncio/tasks.py", line 684, in _wrap_awaitable
    return (yield from awaitable.__await__())
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 832, in __await__
    result = yield from self.promise
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 854, in rerun
    success = await context.evaluateHandle(
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 109, in evaluateHandle
    _rewriteError(e)
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 239, in _rewriteError
    raise error
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 100, in evaluateHandle
    _obj = await self._client.send('Runtime.callFunctionOn', {
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/connection.py", line 217, in send
    raise NetworkError(
pyppeteer.errors.NetworkError: Protocol Error (Runtime.callFunctionOn): Session closed. Most likely the page has been closed.
Task exception was never retrieved
future: <Task finished name='Task-110' coro=<Page.click() done, defined at /home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py:1477> exception=NetworkError('Protocol Error (Runtime.evaluate): Session closed. Most likely the page has been closed.')>
Traceback (most recent call last):
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py", line 1507, in click
    await frame.click(selector, options, **kwargs)
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 567, in click
    handle = await self.J(selector)
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 302, in querySelector
    document = await self._document()
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 312, in _document
    document = (await context.evaluateHandle('document')).asElement()
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 89, in evaluateHandle
    _rewriteError(e)
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 239, in _rewriteError
    raise error
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 81, in evaluateHandle
    _obj = await self._client.send('Runtime.evaluate', {
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/connection.py", line 217, in send
    raise NetworkError(
pyppeteer.errors.NetworkError: Protocol Error (Runtime.evaluate): Session closed. Most likely the page has been closed.
http://m.reactor.cc/18166
banderlog commented 5 years ago

Same for while and headless browser:

async def baz_main():
    browser = await pyppeteer.launch(headless=True)
    pages = await browser.pages()
    page = pages[0]
    await page.goto('http://m.reactor.cc')

    i = 5
    while i > 0:
        i -= 1
        print(i)
        await baz_do(page)

    await browser.close()

Output:

4
http://m.reactor.cc/18169
3
http://m.reactor.cc/18168
2
http://m.reactor.cc/18167
1
Task exception was never retrieved
future: <Task finished name='Task-86' coro=<Page.waitForNavigation() done, defined at /home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py:887> exception=TimeoutError('Navigation Timeout Exceeded: 30000 ms exceeded.')>
Traceback (most recent call last):
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py", line 938, in waitForNavigation
    raise error
pyppeteer.errors.TimeoutError: Navigation Timeout Exceeded: 30000 ms exceeded.
http://m.reactor.cc/18166
0
Task exception was never retrieved
future: <Task finished name='Task-108' coro=<Page.click() done, defined at /home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py:1477> exception=NetworkError('Protocol Error (Runtime.evaluate): Session closed. Most likely the page has been closed.')>
Traceback (most recent call last):
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py", line 1507, in click
    await frame.click(selector, options, **kwargs)
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 567, in click
    handle = await self.J(selector)
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 302, in querySelector
    document = await self._document()
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 312, in _document
    document = (await context.evaluateHandle('document')).asElement()
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 89, in evaluateHandle
    _rewriteError(e)
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 239, in _rewriteError
    raise error
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 81, in evaluateHandle
    _obj = await self._client.send('Runtime.evaluate', {
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/connection.py", line 217, in send
    raise NetworkError(
pyppeteer.errors.NetworkError: Protocol Error (Runtime.evaluate): Session closed. Most likely the page has been closed.
Task exception was never retrieved
future: <Task finished name='Task-109' coro=<_wrap_awaitable() done, defined at /home/kabakov/miniconda3/envs/py37/lib/python3.8/asyncio/tasks.py:677> exception=NetworkError('Protocol Error (Runtime.callFunctionOn): Session closed. Most likely the page has been closed.')>
Traceback (most recent call last):
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/asyncio/tasks.py", line 684, in _wrap_awaitable
    return (yield from awaitable.__await__())
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 832, in __await__
    result = yield from self.promise
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/frame_manager.py", line 854, in rerun
    success = await context.evaluateHandle(
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 109, in evaluateHandle
    _rewriteError(e)
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 239, in _rewriteError
    raise error
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/execution_context.py", line 100, in evaluateHandle
    _obj = await self._client.send('Runtime.callFunctionOn', {
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/connection.py", line 217, in send
    raise NetworkError(
pyppeteer.errors.NetworkError: Protocol Error (Runtime.callFunctionOn): Session closed. Most likely the page has been closed.
Task exception was never retrieved
future: <Task finished name='Task-107' coro=<Page.waitForNavigation() done, defined at /home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py:887> exception=TimeoutError('Navigation Timeout Exceeded: 30000 ms exceeded.')>
Traceback (most recent call last):
  File "/home/kabakov/miniconda3/envs/py37/lib/python3.8/site-packages/pyppeteer/page.py", line 938, in waitForNavigation
    raise error
pyppeteer.errors.TimeoutError: Navigation Timeout Exceeded: 30000 ms exceeded.
http://m.reactor.cc/18166