crawler.go

package main

import (
    "golang.org/x/net/html"
	"log"
	"net/http"
    "fmt"
    "os"
)


func main() {

worklist := make(chan []string)
var n int // number of pending sends to worklist
// Start with the command-line arguments.
n++
go func() { worklist <- os.Args[1:] }()
// Crawl the web concurrently.
seen := make(map[string]bool)
for ; n > 0; n-- {
    list := <-worklist
    for _, link := range list {
        if !seen[link] {
            seen[link] = true
            n++
            go func(link string) {
                worklist <- crawl(link)
            }(link)
        }
    }
}

}

var tokens = make(chan struct{}, 20)
func crawl(url string) []string {
    fmt.Println(url)
    tokens <- struct{}{} // acquire a token
    list, err := Extract(url)
    <-tokens // release the token
    if err != nil {
        log.Print(err)
    }
    return list
}
    

// Extract makes an HTTP GET request to the specified URL, parses
// the response as HTML, and returns the links in the HTML document.
func Extract(url string) ([]string, error) {
        resp, err := http.Get(url)
        if err != nil {
            return nil, err
        }
        if resp.StatusCode != http.StatusOK {
            resp.Body.Close()
            return nil, fmt.Errorf("getting %s: %s", url, resp.Status)
        }
        doc, err := html.Parse(resp.Body)
        resp.Body.Close()
        if err != nil {
            return nil, fmt.Errorf("parsing %s as HTML: %v", url, err)
        }
        var links []string
        visitNode := func(n *html.Node) {
        	if n.Type == html.ElementNode && n.Data == "a" {
            	for _, a := range n.Attr {
            		if a.Key != "href" {
            			continue
        			}
        			link, err := resp.Request.URL.Parse(a.Val)
        			if err != nil {
        				continue // ignore bad URLs
        			}
        			links = append(links, link.String())
        		}
        	}
        }
        forEachNode(doc, visitNode, nil)
        return links, nil
}

// forEachNode calls the functions pre(x) and post(x) for each node
// x in the tree rooted at n. Both functions are optional.
// pre is called before the children are visited (preorder) and
// post is called after (postorder).
func forEachNode(n *html.Node, pre, post func(n *html.Node)) {
    if pre != nil {
    	pre(n)
    }
    for c := n.FirstChild; c != nil; c = c.NextSibling {
    	forEachNode(c, pre, post)
    }
    if post != nil {
    	post(n)
    }
}