wkeeling / selenium-wire

Extends Selenium's Python bindings to give you the ability to inspect requests made by the browser.
MIT License
1.9k stars 251 forks source link

Celery gevent with headless mode causes net::ERR_CONNECTION_CLOSED #670

Open adv-zl opened 1 year ago

adv-zl commented 1 year ago

I have project with celery. Celery starts in gevent threads mode: python3 -m celery -A tasks worker --loglevel=INFO -P gevent

If I run seleniumwire in headless mode and opens https page I receive such error:

[2023-04-15 14:34:12,769: ERROR/MainProcess] Task tasks.eiorjf[f0c84ee0-e9ad-4592-b775-460a5cd6da28] raised unexpected: WebDriverException('unknown error: net::ERR_CONNECTION_CLOSED\n  (Session info: headless chrome=110.0.5481.77)', None, ['0   chromedriver                        0x0000000104680138 chromedriver + 4923704', '1   chromedriver                        0x00000001045f89d3 chromedriver + 4368851', '2   chromedriver                        0x0000000104241787 chromedriver + 472967', '3   chromedriver                        0x0000000104239235 chromedriver + 438837', '4   chromedriver                        0x000000010422a9fc chromedriver + 379388', '5   chromedriver                        0x000000010422c4f6 chromedriver + 386294', '6   chromedriver                        0x000000010422adc0 chromedriver + 380352', '7   chromedriver                        0x0000000104229a11 chromedriver + 375313', '8   chromedriver                        0x000000010422987b chromedriver + 374907', '9   chromedriver                        0x00000001042280c6 chromedriver + 368838', '10  chromedriver                        0x000000010422847d chromedriver + 369789', '11  chromedriver                        0x0000000104243af8 chromedriver + 482040', '12  chromedriver                        0x00000001042ca2a1 chromedriver + 1032865', '13  chromedriver                        0x00000001042aefb2 chromedriver + 921522', '14  chromedriver                        0x00000001042c9c0f chromedriver + 1031183', '15  chromedriver                        0x00000001042aed23 chromedriver + 920867', '16  chromedriver                        0x0000000104275a4b chromedriver + 686667', '17  chromedriver                        0x0000000104277044 chromedriver + 692292', '18  chromedriver                        0x000000010464b8fe chromedriver + 4708606', '19  chromedriver                        0x0000000104650e22 chromedriver + 4730402', '20  chromedriver                        0x00000001046596bf chromedriver + 4765375', '21  chromedriver                        0x0000000104651e80 chromedriver + 4734592', '22  chromedriver                        0x0000000104622975 chromedriver + 4540789', '23  chromedriver                        0x0000000104673e78 chromedriver + 4873848', '24  chromedriver                        0x0000000104673ff5 chromedriver + 4874229', '25  chromedriver                        0x00000001046886de chromedriver + 4957918', '26  libsystem_pthread.dylib             0x00007ff81050d259 _pthread_start + 125', '27  libsystem_pthread.dylib             0x00007ff810508c7b thread_start + 15', ''])
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/celery/app/trace.py", line 451, in trace_task
    R = retval = fun(*args, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/celery/app/trace.py", line 734, in __protected_call__
    return self.run(*args, **kwargs)
  File "/Volumes/untitled/Current/PycharmProjects/fbTestCab/appium/tasks.py", line 34, in eiorjf
    driver.get('http://www.paypal.com')
  File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 430, in get
    self.execute(Command.GET, {'url': url})
  File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 418, in execute
    self.error_handler.check_response(response)
  File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 243, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: net::ERR_CONNECTION_CLOSED
  (Session info: headless chrome=110.0.5481.77)

My tasks.py:


from celery import Celery
from seleniumwire import webdriver as wirewebdriver
from seleniumwire.undetected_chromedriver import ChromeOptions
app = Celery('tasks', broker='redis://localhost:6379/0')
app.autodiscover_tasks()

@app.task
def eiorjf():
    driver_path = '/Users/admin/Downloads/Django/default/fbbe/tasks/chromedriver'
    opts = ChromeOptions()
    opts.binary_location = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
    opts.add_argument("--no-sandbox")
    opts.add_argument("--disable-infobars")
    opts.add_argument("--disable-gpu")
    opts.add_argument('--ignore-certificate-errors-spki-list')
    opts.add_argument('--ignore-ssl-errors')
    opts.add_argument('--disable-dev-shm-usage')
    opts.add_argument('--allow-running-insecure-content')
    opts.add_argument('--allow-insecure-localhost')  # differ on driver version. can ignore.
    opts.add_argument("start-maximized")
    opts.add_experimental_option("excludeSwitches", ["enable-automation"])
    opts.add_experimental_option('useAutomationExtension', False)
    opts.add_argument("--disable-blink-features=AutomationControlled")
    opts.add_experimental_option("prefs", {
        "profile.default_content_setting_values.notifications": 2,
        "profile.managed_default_content_settings.images": 2
    })
    opts.headless = True

    driver = wirewebdriver.Chrome(executable_path=driver_path, options=opts)
    driver.execute_cdp_cmd('Network.setUserAgentOverride', {
        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"})

    driver.get('http://www.facebook.com')
    driver.save_screenshot('ds.png')
    driver.quit()

If I add patch_all on the file start, It help to prevent exception but returns empty page.

from gevent import monkey
monkey.patch_all()

Is it possible to fix this problem? Default selenium doesnt have it