goscrape

A Go scraper inspired by Python's Scrapy (http://scrapy.org).

This repository is still a work in progress

Roadmap:

1. Better logging
2. Exporting scraped content to JSON, XML or other formats
3. Web Interface for viewing jobs / creating dynamic scrapers
4. Sitemap spider
5. Tests

Example Code

package main

import (
    "fmt"
    "log"
    "strings"

    "github.com/rakanalh/goscrape"
    "github.com/rakanalh/goscrape/extract"
    "github.com/rakanalh/goscrape/processors"
)

func parseHandler(response *goscrape.Response) goscrape.ParseResult {
    log.Println(response.URL)
    if !strings.Contains(response.URL, "category") {
        return goscrape.ParseResult{}
    }

    xpath, err := extract.NewXPath(response.Content)
    nodes, err := xpath.Extract("//ul[@class=\"list-thumb-right\"]/li/a[position()=1]/@href")
    urls, err := processors.GetLinks(nodes)

    // css, err := extract.NewCss(response.Content)
    // selection, err := css.Extract("ul.list-thumb-right > li > a")
    // urls, err := processors.GetLinks(selection)
    fmt.Println(urls)

    if err != nil {
        panic("Could not parse Xpath")
    }

    log.Println("Processing URL: " + response.URL)

    for i, url := range urls {
        urls[i] = "http://www.testdomain.net" + url
    }

    result := goscrape.ParseResult{
        Urls: urls,
    }
    return result
}

func itemHandler(item goscrape.ParseItem) {
    fmt.Println(item)
}

func main() {
    startUrls := []string{
        "http://www.testdomain.net/category/test_uri",
    }

    configs := goscrape.Configs{
        StartUrls:    startUrls,
        WorkersCount: 3,
    }

    spider := goscrape.NewSpider(&configs, parseHandler, itemHandler)
    spider.Start()
}