使用Golang爬取豆瓣电影top250

package main

import (
    "fmt"
    "io"
    "net/http"
    "os"
    "regexp"
    "strconv"
)

func HttpGet(url string) (result string, err error) {
    fmt.Println("url->", url)

    client := &http.Client{}
    req, err1 := http.NewRequest("GET", url, nil)

    if err1 != nil {
        err = err1
        return
    }

    req.Header.Add("User-Agent", "test")

    resp, err2 := client.Do(req)
    if err2 != nil {
        err = err2
        return
    }

    defer resp.Body.Close()

    buf := make([]byte, 4096)
    for {
        n, err3 := resp.Body.Read(buf)
        if n == 0 {
            break
        }

        if err3 != nil && err3 != io.EOF {
            err = err3
            return
        }
        result += string(buf[:n])
    }

    return
}

func Write2File(idx int, filmName, filmScore, filmRate [][]string) {
    f, err := os.Create("第 " + strconv.Itoa(idx) + "页.txt")
    if err != nil {
        fmt.Println("os.Create err", err)
        return
    }
    defer f.Close()
    n := len(filmName)
    f.WriteString("电影名字\t\t\t\t电影分数\t\t\t\t电影评论数\n")
    for i := 0; i < n; i++ {
        f.WriteString(filmName[i][1] + "\t\t\t\t" + filmScore[i][1] + "\t\t\t\t" + filmRate[i][1] + "\n")
    }
}

func SpiderPage(idx int, page chan int) {
    url := "https://movie.douban.com/top250?start=" + strconv.Itoa((idx-1)*25) + "&filter="
    result, err := HttpGet(url)
    if err != nil {
        fmt.Println("HttpGet err:", err)
        return
    }

    nameExp := regexp.MustCompile(`<img width="100" alt="(.*?)"`)
    filmName := nameExp.FindAllStringSubmatch(result, -1)

    scoreExp := regexp.MustCompile(`<span class="rating_num" property="v:average">(.*?)</span>`)
    filmScore := scoreExp.FindAllStringSubmatch(result, -1)

    rateExp := regexp.MustCompile(`<span>(.*?)人评价</span>`)
    filmRate := rateExp.FindAllStringSubmatch(result, -1)

    Write2File(idx, filmName, filmScore, filmRate)
    // 与主线程同步，写入当前page数
    page <- idx
}

func toWork(start, end int) {
    fmt.Printf("正在爬取%d到%d页...\n", start, end)
    page := make(chan int)
    for i := start; i <= end; i++ {
        go SpiderPage(i, page)
    }

    for i := start; i <= end; i++ {
        fmt.Printf("第%d页读取完毕\n", <-page)
    }
}

func main() {
    var start, end int
    fmt.Print("请输入开始爬取的页 (>=1):")
    fmt.Scan(&start)
    fmt.Print("请输入结束爬取的页 (>=start):")
    fmt.Scan(&end)
    toWork(start, end)
}

程序输出如下，

image.png

有疑问加站长微信联系（非本文作者）