u := launcher.New().
Set("--no-sandbox").
MustLaunch()
browser := rod.New().ControlURL(u).MustConnect()
defer browser.MustClose()
// We create a pool that will hold at most 3 pages which means the max concurrency is 3
pool := rod.NewPagePool(2)
// Create a page if needed
create := func() *rod.Page {
// We use MustIncognito to isolate pages with each other
return browser.MustIncognito().MustPage()
}
imgList = make(map[string][]string)
var lock sync.Mutex
// Run jobs concurrently
wg := sync.WaitGroup{}
// 遍历列表
for _, item := range list {
wg.Add(1)
go func(item model.SpiderList) {
defer wg.Done()
page := pool.Get(create)
defer pool.Put(page)
var resultsImg []string
var err error
fmt.Println("采集图片:", item.Id)
resultsImg, err = Job(item.Url, page)
if err != nil {
fmt.Println(item.Url)
fmt.Println(err) // context deadline exceeded
return
}
lock.Lock() // 加锁
imgList[strconv.FormatInt(item.Id, 10)] = resultsImg
lock.Unlock() // 解锁
//time.Sleep(time.Second * time.Duration(utils.RandInt(1, 3)))
}(item)
}
wg.Wait()
pool.Cleanup(func(p *rod.Page) { defer p.MustClose() })
//rodUtils.Pause() // pause goroutine
return imgList
func Img(list []model.SpiderList) (imgList map[string][]string) {
}
func Job(url string, page *rod.Page) ([]string, error) {
}
出现context deadline exceeded错误,程序异常挂起,排查下来,page没有释放。 Current version is v0.114.5