使用golang抓取京东全部商品分类信息

package main

import (
	//	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strings"
)

type Mall struct {
	name string
	cat  []*Catagory
}

type Catagory struct {
	id     int64
	name   string
	link   string
	subCat []*SubCatagory
}

type SubCatagory struct {
	id        int64
	name      string
	link      string
	detailCat []*DetailCatagory
}

type DetailCatagory struct {
	id    int64
	name  string
	link  string
	goods map[string]interface{}
}

var Jd = Mall{name: "Jd"}

var topCatagoryStart = regexp.MustCompile(`[[:space:]]*\<div[[:space:]]+class="category-item m"\>`)
var topCatagoryFetch = regexp.MustCompile(`[[:space:]]*.*\<span\>(?P<topC>.*)\</span\>`)
var topCatagoryEnd = regexp.MustCompile(`[[:space:]]*\</div\>`)
var subCatagoryFetch = regexp.MustCompile(``)

var detailCatagoryStart = regexp.MustCompile(`[[:space:]]*\<dt\>\<a[[:space:]]+href=\"//(?P<cat2link>.*)\"[[:space:]]+target="_blank"\>(?P<cat2name>[\p{Han}]+)\</a\>\</dt\>`)

var detailCatagoryFetch = regexp.MustCompile(`[[:space:]]*.*\<a[[:space:]]+href=\"//(?P<cat2link>.*)\"[[:space:]]+target="_blank"\>(?P<cat2name>[\p{Han}]+)\</a\>`)

func dumpJdCatagory(mall *Mall) {
	fmt.Println(mall.name)
	for _, c := range Jd.cat {
		fmt.Println(c)
		fmt.Printf("Catagory: %s\n", c.name)
		for _, sc := range c.subCat {
			fmt.Printf("SubCatagory: %s, Link: %s\n", sc.name, sc.link)
			for _, dc := range sc.detailCat {
				fmt.Printf("DetailCatagory: %s. Link: %s\n", dc.name, dc.link)
			}
		}
	}
}

func main() {
	resp, err := http.Get("http://www.jd.com/allSort.aspx")
	if err != nil {
		panic(err.Error())
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err.Error())
	}
	//	fmt.Println(string(body))

	file, err := os.Create("jd_list.html")
	if err != nil {
		panic(err.Error())
	}

	_, err = file.Write(body)
	if err != nil {
		panic(err.Error())
	}

	Jd.cat = make([]*Catagory, 0, 100)
	var top = false
	var sub = false
	var detail = false
	var cat *Catagory
	var subCat *SubCatagory
	var detailCat *DetailCatagory
	s := strings.Split(string(body), "\n")
	for _, line := range s {
		if topCatagoryStart.MatchString(line) {
			top = true
			sub = false
			detail = false
		}
		if top == true {
			if topCatagoryFetch.MatchString(line) {
				sub = true
				/*
					fmt.Println(topCatagoryFetch.FindStringSubmatch(line)[1])
						cat = &Catagory{name: topCatagoryFetch.FindStringSubmatch(line)[1]}
						cat.subCat = make([]*SubCatagory, 40, 100)
				*/
				cat = new(Catagory)
				cat.name = topCatagoryFetch.FindStringSubmatch(line)[1]
				cat.subCat = make([]*SubCatagory, 0, 100)
				Jd.cat = append(Jd.cat, cat)
				//fmt.Println("Catagory")
				//fmt.Println(cat)
			}
		}

		if sub == true {
			if detailCatagoryStart.MatchString(line) {
				/*
					fmt.Println(detailCatagoryStart.FindStringSubmatch(line)[1])
					fmt.Println(detailCatagoryStart.FindStringSubmatch(line)[2])
						subCat = &SubCatagory{name: detailCatagoryStart.FindStringSubmatch(line)[2], link: detailCatagoryStart.FindStringSubmatch(line)[1]}
						subCat.detailCat = make([]*DetailCatagory, 50, 100)
				*/
				subCat = new(SubCatagory)
				subCat.name = detailCatagoryStart.FindStringSubmatch(line)[2]
				subCat.link = detailCatagoryStart.FindStringSubmatch(line)[1]
				subCat.detailCat = make([]*DetailCatagory, 0, 100)
				cat.subCat = append(cat.subCat, subCat)
				//fmt.Println("SubCatagory")
				//fmt.Println(subCat)
				detail = true
			}
		}

		if detail == true {
			if detailCatagoryFetch.MatchString(line) {
				/*
					fmt.Println(detailCatagoryFetch.FindStringSubmatch(line)[1])
					fmt.Println(detailCatagoryFetch.FindStringSubmatch(line)[2])
						detailCat = &DetailCatagory{name: detailCatagoryFetch.FindStringSubmatch(line)[2], link: detailCatagoryFetch.FindStringSubmatch(line)[1]}
				*/
				detailCat = new(DetailCatagory)
				detailCat.name = detailCatagoryFetch.FindStringSubmatch(line)[2]
				detailCat.link = detailCatagoryFetch.FindStringSubmatch(line)[1]
				subCat.detailCat = append(subCat.detailCat, detailCat)
				//fmt.Println("DetailCatagory")
				//				fmt.Println(detailCat)

			}
			if topCatagoryEnd.MatchString(line) {
				top = false
				sub = false
				detail = false
			}
		}
	}

	dumpJdCatagory(&Jd)
	//fmt.Println(s)
	//fmt.Printf("%d bytes has been write to jd_list.html", n)

}

有疑问加站长微信联系（非本文作者）