```
package main
import (
"xiyu/models"
"runtime"
"sync"
"github.com/astaxie/beego/orm"
)
var limit = make(chan struct{}, 10)
func getInfos(url,proxy string,cat_id int,goods_thumb string,id int) {
limit <- struct{}{}
defer func() {
<-limit
}()
info :=models.Get_Goods_Info("http://www.ehsy.com/"+url,"not")
if len(info.Goods_name) > 5 {
//insert collect info to mysql
info.Cat_id = cat_id
info.Parent_id = id
info.Url = url
info.Old_thumb = goods_thumb
orm.NewOrm().Insert(&info)
//update the goods url status in mysql
detail_url_model := new(models.GoodsUrl)
detail_url_model.Id = id
detail_url_model.Status = 1
detail_url_model.Update("Status")
}
}
func main() {
NCPU := runtime.NumCPU()
runtime.GOMAXPROCS(NCPU)
models.Init()
goods_url := new(models.GoodsUrl)
var goods_urls []*models.GoodsUrl
count ,_ := goods_url.Query().Filter("status",0).Count() //get total url count
var wg sync.WaitGroup
for i:=0;i<=int(count)/100;i++ {
wg.Add(1)
goods_url.Query().Filter("status",0).Limit(100,i*100).All(&goods_urls,"Cat_id","Url","Id","Img") //get 100 urls
for _,v :=range goods_urls {
go func(url string,proxy string ,cat_id int,img string,id int) {
getInfos(url,proxy,cat_id,img,id) //execute collect action
}(v.Url,"",v.Cat_id,v.Img,v.Id)
}
wg.Done()
}
wg.Wait()
}
```
这段代码运行一段时间后,内存回消耗光,然后停止运行。请问该如何优化呢
我接触go的时间并不长,所以对goroutine的内部实现机理还没去深究。但是我观察的情况是,如果硬件CPU调度是足够的冗余度的时(在你给定的业务条件下),goroutine的收放是会处在一个合理的范围的。但是如果业务密集度大大超出CPU的运算时(尤其是在VPS这类环境下),他就无法很好的释放goroutine了。其实这也是合理的,适当的增强服务器的强度,毕竟业务需求摆在那里。我现在一台go的服务器,顶替了3台php的服务器(负载均衡点还没计算),这样适当增强一下go的服务器,也不为过。
#17
更多评论