search

go-rod / rod

A Chrome DevTools Protocol driver for web automation and scraping.
https://go-rod.github.io
MIT License
5.25k stars 345 forks source link

navigation failed: net::ERR_INVALID_AUTH_CREDENTIALS #994

Open wAnFen1017 opened 9 months ago

wAnFen1017 commented 9 months ago

Rod Version: v0.114.5

我有一个隧道代理服务，支持http和https。他需要通过 http://user:pwd@ip:port 的形式通过验证，该服务测试过没有问题。

参照文档中 go browser.MustHandleAuth(user, pwd)() 成功设置了代理，并打开网站验证无误。但是同样的逻辑放在浏览器池browsePool中则报错，在创建browserPool之前设置browser.MustHandleAuth会报错 navigation failed: net::ERR_INVALID_AUTH_CREDENTIALS；创建后设置browser.MustHandleAuth则会报错 panic: {-32000 Invalid state for continueInterceptedRequest }。以下是我的代码

// Package main
// @Time      : 2023-12-06 09:47
// @Author    : wAnFen
// @File      : render.go
// @Go        : 1.21.3

package main

import (
    "github.com/go-rod/rod"
    "github.com/go-rod/rod/lib/launcher"
    "github.com/go-rod/rod/lib/proto"
    "log"
    "sync"
)

func RunRod(url string) {
    var rodUrl string
    l := launcher.New().
        Headless(false).
        Leakless(true).
        NoSandbox(true).
        Delete("enable-automation").
        Set("ignore-certificate-errors").
        Set("ignore-certificate-errors-spki-list").
        Set("ignore-ssl-errors").
        Set("disable-blink-features", "AutomationControlled").
        Set("disable-setuid-sandbox").
        Set("disable-gpu").
        Set("disable-dev-shm-usage").
        Set("unlimited-storage").
        Set("disable-accelerated-2d-canvas").
        Set("full-memory-crash-report")

    l = l.Proxy("127.0.0.1:6501")

    path, has := launcher.LookPath()
    if has == true {
        rodUrl = l.Bin(path).MustLaunch()
    } else {
        rodUrl = l.MustLaunch()
    }

    log.Println("StartURL : ", rodUrl)
    browser := rod.New().ControlURL(rodUrl).MustConnect()

    browser.MustIgnoreCertErrors(true)

    user := "user"
    pwd := "password"

    go browser.MustHandleAuth(user, pwd)()

    browsePool := rod.NewBrowserPool(20)
    create := func() *rod.Browser {
        //go browser.MustHandleAuth(user, pwd)()
        return browser.MustIncognito()
    }

    // 单一线程正常运行
    //b := browsePool.Get(create)
    //page, err := b.Page(proto.TargetCreateTarget{URL: url})
    //if err != nil {
    //  log.Println("browser.Page Err : ", err)
    //}
    //println(page.MustElement("html").MustText())

    // 在goroutine中运行报错 navigation failed: net::ERR_INVALID_AUTH_CREDENTIALS
    var wg sync.WaitGroup
    for i := 0; i <= 3; i++ {
        wg.Add(1)
        go func() {
            defer wg.Done()
            b := browsePool.Get(create)
            page, err := b.Page(proto.TargetCreateTarget{URL: url})
            if err != nil {
                log.Println("browser.Page Err : ", err)
            }
            println(page.MustElement("html").MustText())
        }()
    }
    wg.Wait()
}

func main() {
    RunRod("http://api.ipify.org")
}
github-actions[bot] commented 9 months ago

Please fix the format of your markdown:

7 MD040/fenced-code-language Fenced code blocks should have a language specified [Context: "```"]

generated by check-issue

ysmood commented 9 months ago

用法的问题，create 里处理 auth

wAnFen1017 commented 9 months ago

用法的问题，create 里处理 auth

如果改成在create中处理时则会报 panic: {-32000 Invalid state for continueInterceptedRequest }

create := func() *rod.Browser {
        go browser.MustHandleAuth(user, pwd)()
        return browser.MustIncognito()
    }
ysmood commented 9 months ago

如果在外围处理，需要处理每个 auth 事件，browser.MustHandleAuth 只处理碰到的第一个，你看看它的源代码就懂了

wAnFen1017 commented 9 months ago

如果在外围处理，需要处理每个 auth 事件，browser.MustHandleAuth 只处理碰到的第一个，你看看它的源代码就懂了

我在每个goroutine中做处理不可实现吗，或者如果在外围统一处理的话，我查到是使用EachEvent去控制全部活动，我改成这样，依旧报错 panic: {-32000 Invalid state for continueInterceptedRequest }

    go browser.EachEvent(func(e *proto.FetchAuthRequired) {
        fakeAuth := &proto.FetchAuthChallengeResponse{
            Response: proto.FetchAuthChallengeResponseResponseProvideCredentials,
            Username: user,
            Password: pwd,
        }
        err := proto.FetchContinueWithAuth{
            RequestID:             e.RequestID,
            AuthChallengeResponse: fakeAuth,
        }.Call(browser)
        if err!=nil{
            panic(err)
        }
    }, func(e *proto.FetchRequestPaused) {
        err := proto.FetchContinueRequest{
            RequestID: e.RequestID,
        }.Call(browser)
        if err!=nil{
            panic(err)
        }
    })()

    browsePool := rod.NewBrowserPool(20)
    create := func() *rod.Browser {
        return browser.MustIncognito()
    }
        // browser.MustPage()
ysmood commented 9 months ago

那你可能需要自己 debug 一下，应该是用法的问题，这方面 chromium cdp 缺少文档说明，只能猜测用法，让我来和让你来是一样的。

你这里报错的意思是 FetchContinueRequest 用法错误

wAnFen1017 commented 9 months ago

我看了您另外的一个回答 https://github.com/go-rod/rod/issues/654 ，我感觉我和他的问题应该是差不多的，但是我这里使用了BrowserPool，所以我猜测是不是BrowserPool的问题，以下是我的完整代码，此时报错信息是 navigation failed: net::ERR_INVALID_AUTH_CREDENTIALS

// Package main
// @Time      : 2023-12-06 09:47
// @Author    : wAnFen
// @File      : render.go
// @Go        : 1.21.3

package main

import (
    "github.com/go-rod/rod"
    "github.com/go-rod/rod/lib/launcher"
    "github.com/go-rod/rod/lib/proto"
    "log"
    "sync"
)

func RunRod(url string) {
    var rodUrl string
    l := launcher.New().
        Headless(false).
        Leakless(true).
        NoSandbox(true).
        Delete("enable-automation").
        Set("ignore-certificate-errors").
        Set("ignore-certificate-errors-spki-list").
        Set("ignore-ssl-errors").
        Set("disable-blink-features", "AutomationControlled").
        Set("disable-setuid-sandbox").
        Set("disable-gpu").
        Set("disable-dev-shm-usage").
        Set("unlimited-storage").
        Set("disable-accelerated-2d-canvas").
        Set("full-memory-crash-report")

    l = l.Proxy("http://127.0.0.1:6501")

    path, has := launcher.LookPath()
    if has == true {
        rodUrl = l.Bin(path).MustLaunch()
    } else {
        rodUrl = l.MustLaunch()
    }

    log.Println("StartURL : ", rodUrl)
    browser := rod.New().ControlURL(rodUrl).MustConnect()

    browser.MustIgnoreCertErrors(true)

    user := "user"
    pwd := "pwd"

    go browser.EachEvent(func(e *proto.FetchAuthRequired) {
        fakeAuth := &proto.FetchAuthChallengeResponse{
            Response: proto.FetchAuthChallengeResponseResponseProvideCredentials,
            Username: user,
            Password: pwd,
        }
        err := proto.FetchContinueWithAuth{
            RequestID:             e.RequestID,
            AuthChallengeResponse: fakeAuth,
        }.Call(browser)
        if err != nil {
            panic(err)
        }
    }, func(e *proto.FetchRequestPaused) {
        err := proto.FetchContinueRequest{
            RequestID: e.RequestID,
        }.Call(browser)
        if err != nil {
            panic(err)
        }
    })()

    browsePool := rod.NewBrowserPool(20)
    create := func() *rod.Browser {
        return browser.MustIncognito()
    }

    var wg sync.WaitGroup
    for i := 0; i <= 3; i++ {
        wg.Add(1)
        go func() {
            defer wg.Done()
            b := browsePool.Get(create)
            page, err := b.Page(proto.TargetCreateTarget{URL: url})
            if err != nil {
                log.Println("browser.Page Err : ", err)
            }
            println(page.MustElement("html").MustText())
        }()
    }
    wg.Wait()
}

func main() {
    RunRod("http://api.ipify.org")
}
lc commented 7 months ago

I'm encountering this issue as well. Not sure what I'm doing wrong here.

I have a proxy that doesn't return HTTP/1.1 407 Proxy Authentication Required and still passes through. However, my proxy provider allows sticky-sessions when a username:pass is sent.

What would be the correct pattern? I know it's not proto.FetchAuthRequired, but I am struggling to find such a use-case in the docs. I tried using HijackRouter and setting the Proxy-Authorization header for the request, but that forwards the header to the fetched URL rather than for the proxy connection.

This code does not work:

    go browser.EachEvent(func(e *proto.FetchAuthRequired) {
        fakeAuth := &proto.FetchAuthChallengeResponse{
            Response: proto.FetchAuthChallengeResponseResponseProvideCredentials,
            Username: "user",
            Password: "pass",
        }
        err := proto.FetchContinueWithAuth{
            RequestID:             e.RequestID,
            AuthChallengeResponse: fakeAuth,
        }.Call(browser)
        if err != nil {
            panic(err)
        }
    }, func(e *proto.FetchRequestPaused) {
        err := proto.FetchContinueRequest{
            RequestID: e.RequestID,
        }.Call(browser)
        if err != nil {
            panic(err)
        }
    })()
xujinzheng commented 6 months ago

following code is worked

  1. create page
  2. auth handle
browser.Page(proto.TargetCreateTarget{})
go browser.MustHandleAuth(username, password)()
lc commented 6 months ago

Switched my proxy provider and it worked.

wAnFen1017 commented 6 months ago

following code is worked

  1. create page
  2. auth handle
browser.Page(proto.TargetCreateTarget{})
go browser.MustHandleAuth(username, password)()

Thank you for your answer. This is indeed effective under normal requests, but it does not work in browser pools