最近的评论

• 评论了博文 go-tour练习解答
  递归爬虫程序 **Web Crawler** 可以用 sync.WaitGroup 代替 end channel ``` package main import ( "fmt" "sync" ) type Fetcher interface { // Fetch returns the body of URL and // a slice of URLs found on that page. Fetch(url string) (body string, urls []string, err error) } // Crawl uses fetcher to recursively crawl // pages starting with url, to a maximum of depth. func Crawl(url string, depth int, fetcher Fetcher) { // This implementation doesn't do either: defer wg.Done() if depth <= 0 { return } if cm.urlExist(url) { return } cm.markUrl(url) body, urls, err := fetcher.Fetch(url) if err != nil { out <- fmt.Sprintln(err) return } out <- fmt.Sprintf("found: %s %q\n", url, body) for _, u := range urls { wg.Add(1) go Crawl(u, depth-1, fetcher) } } type crawledUrlMap struct { urlMap map[string]bool crawledMutex sync.Mutex } var out = make(chan string) var cm = crawledUrlMap{urlMap: make(map[string]bool)} var wg sync.WaitGroup func main() { wg.Add(1) go Crawl("http://golang.org/", 4, fetcher) go func() { wg.Wait() close(out) }() for res := range out { fmt.Println(res) } } func (c crawledUrlMap) urlExist(url string) bool { c.crawledMutex.Lock() _, ok := c.urlMap[url] c.crawledMutex.Unlock() return ok } func (c crawledUrlMap) markUrl(url string) { c.crawledMutex.Lock() c.urlMap[url] = true c.crawledMutex.Unlock() } // fakeFetcher is Fetcher that returns canned results. type fakeFetcher map[string]*fakeResult type fakeResult struct { body string urls []string } func (f fakeFetcher) Fetch(url string) (string, []string, error) { if res, ok := f[url]; ok { return res.body, res.urls, nil } return "", nil, fmt.Errorf("not found: %s", url) } // fetcher is a populated fakeFetcher. var fetcher = fakeFetcher{ "http://golang.org/": &fakeResult{ "The Go Programming Language", []string{ "http://golang.org/pkg/", "http://golang.org/cmd/", }, }, "http://golang.org/pkg/": &fakeResult{ "Packages", []string{ "http://golang.org/", "http://golang.org/cmd/", "http://golang.org/pkg/fmt/", "http://golang.org/pkg/os/", }, }, "http://golang.org/pkg/fmt/": &fakeResult{ "Package fmt", []string{ "http://golang.org/", "http://golang.org/pkg/", }, }, "http://golang.org/pkg/os/": &fakeResult{ "Package os", []string{ "http://golang.org/", "http://golang.org/pkg/", }, }, } ```