Go + Colly/v2

[coloraven]

Give the curl command And output as blow:

package main

import (
    "fmt"
    "log"
    "time"

    "github.com/gocolly/colly/v2"
    "github.com/gocolly/colly/v2/debug"
)

// creat a new collector
var c = colly.NewCollector(
    colly.Async(true), // Enable Async
    colly.AllowURLRevisit(),
    colly.Debugger(&debug.LogDebugger{}),
)

func init() {
    // The headers
    Headers := map[string]string{
        "User-Agent":    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
        "Accept-Custom": "en-US,en;q=0.9",
    }

    // Set proxy
    err := c.SetProxy("http://127.0.0.1:xxx")
    if err != nil {
        log.Fatal("set proxy failed:", err)
    }

    c.Limit(&colly.LimitRule{
        DomainGlob:  "*",
        Parallelism: 5,
        RandomDelay: 5 * time.Second,
    })

    c.OnRequest(func(r *colly.Request) {
        if r.Method == "GET" {
        } else if r.Method == "POST" {
        }

        // Set headers
        for k, v := range Headers {
            r.Headers.Set(k, v)
        }
    })

}
func main() {
    url := "https://httpbin.org/html"
    c.OnResponse(func(r *colly.Response) {
        fmt.Println("Response Text:", string(r.Body))
    })
    err := c.Visit(url)
    if err != nil {
        fmt.Println("Visit error:", err)
    }

    c.Wait()
}