headzoo / surf

Stateful programmatic web browsing in Go.
MIT License
1.48k stars 159 forks source link

Form submit hangs when downloading a large zip file #57

Closed cloudvant closed 7 years ago

cloudvant commented 7 years ago

The following code hangs on f2.Submit(). It works when the download is small but for large downloads it hangs and never comes back.


 func (ct CT) GetArticles() {
    bow := surf.NewBrowser()
    bow.Open("https://clinicaltrials.gov/")

    f, _ := bow.Form("form[action='/ct2/results']")
    f.Input("home-search-query", "cancer")
    f.Submit()
    logger.Info(bow.Title())

    bow.Click("a#save-list-link")
    f2, err := bow.Form("form[action='/ct2/results/download']")
    if err != nil {
        logger.Error(err)
    }
    //f2.Set("down_stds", "all")
    f2.Set("down_typ", "study")
    f2.Set("down_flds", "all")
    f2.Set("down_fmt", "xml")
    err = f2.Submit()
    logger.Info(bow.ResponseHeaders())

    file, e := os.Create("./dt.zip")
    if e != nil {
        logger.Error(e)
    }
    defer file.Close()
    bow.Download(file)
}