Golang: 利用 channel (管道) 技术多线程下载图片

kanglecjr · · 7330 次点击 · · 开始浏览    
这是一个创建于 的文章,其中的信息可能已经有所发展或是发生改变。

利用 channel (管道) 技术多线程下载图片.

 

package main

import (
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	//"strconv"
	"strings"
	"sync"
)

var urlist = [...]string{"http://stock.591hx.com/article/2014-12-03/0000850005s.shtml"}
var album chan string
var w sync.WaitGroup
var dir string

func main() {
	dir = "tmp_chenjo/"
	err := os.Mkdir(dir, 0777)
	if err != nil {
		isexist := os.IsExist(err)
		log.Println(isexist)
	}
	album = make(chan string, 200)

	for _, v := range urlist {
		w.Add(1)
		go GetAlbum(v)
		w.Wait()
	}
}

func GetAlbum(url string) {
	data := GetUrl(url)
	body := string(data)
	//

part := regexp.MustCompile(``) match := part.FindAllStringSubmatch(body, -1) for _, v := range match { if m, _ := regexp.MatchString(`.*/hnimg/201412/03/.*\.jpg`, v[1]); !m { continue } //println(v[1]) album <- v[1] w.Add(1) go GetItem() } w.Done() } func GetItem() { url := <-album println(url) defer func() { ret := recover() if ret != nil { log.Println(ret) w.Done() } else { w.Done() } }() //data := GetUrl(url) //if len(data) > 10 { //body := string(data) //part := regexp.MustCompile(`bigimgsrc="(.*)"`) //match := part.FindAllStringSubmatch(body, -1) //for _, v := range match { str := strings.Split(url, "/") length := len(str) source := GetUrl(url) name := str[length-1] file, err := os.Create(dir + name) if err != nil { panic(err) } size, err := file.Write(source) defer file.Close() if err != nil { panic(err) } log.Println(size) //} //} } func GetUrl(url string) []byte { ret, err := http.Get(url) if err != nil { log.Println(url) status := map[string]string{} status["status"] = "400" status["url"] = url panic(status) } body := ret.Body data, _ := ioutil.ReadAll(body) return data }

 

 console output:

G:/share/Golang/Golang.exe [G:/share/Golang]

2015/01/18 21:54:29 true

http://stock.591hx.com/images/hnimg/201412/03/64/13418266510200941552.jpg

http://stock.591hx.com/images/hnimg/201412/03/7/10886141709285175583.jpg

http://stock.591hx.com/images/hnimg/201412/03/0/5391574706574741364.jpg

http://stock.591hx.com/images/hnimg/201412/03/1/5405780767459854941.jpg

http://stock.591hx.com/images/hnimg/201412/03/16/13722698761317688276.jpg

http://stock.591hx.com/images/hnimg/201412/03/7/16853951343108680551.jpg

http://stock.591hx.com/images/hnimg/201412/03/50/17680852843413447062.jpg

http://stock.591hx.com/images/hnimg/201412/03/34/14366548421421579970.jpg

http://stock.591hx.com/images/hnimg/201412/03/4/17141924098089490820.jpg

2015/01/18 21:54:30 29968

2015/01/18 21:54:30 28697

2015/01/18 21:54:30 51586

2015/01/18 21:54:30 53031

2015/01/18 21:54:30 31234

2015/01/18 21:54:31 47618

2015/01/18 21:54:31 54649

2015/01/18 21:54:31 40422

2015/01/18 21:54:31 155527

成功: 进程退出代码 0.

 

 


有疑问加站长微信联系(非本文作者)

本文来自:ITEYE博客

感谢作者:kanglecjr

查看原文:Golang: 利用 channel (管道) 技术多线程下载图片

入群交流(和以上内容无关):加入Go大咖交流群,或添加微信:liuxiaoyan-s 备注:入群;或加QQ群:692541889

7330 次点击  
加入收藏 微博
暂无回复
添加一条新回复 (您需要 登录 后才能回复 没有账号 ?)
  • 请尽量让自己的回复能够对别人有帮助
  • 支持 Markdown 格式, **粗体**、~~删除线~~、`单行代码`
  • 支持 @ 本站用户;支持表情(输入 : 提示),见 Emoji cheat sheet
  • 图片支持拖拽、截图粘贴等方式上传