goruntine 并发高下报错???

dollarkiller · · 1122 次点击 · 开始浏览    置顶
这是一个创建于 的主题,其中的信息可能已经有所发展或是发生改变。

系统并发3000左右报的错误 并发2000没有这个问题? 时在摸不着头脑?? ``` goroutine 17502 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0xee6b2800) /usr/local/go/src/runtime/time.go:105 +0x157 github.com/dollarkillerx/urllib.(*Urllib).BodyRetry(0xc0098ef290, 0x3, 0xe3f34b, 0x3, 0xc0098ef290) /home/wangy/go/pkg/mod/github.com/dollarkillerx/urllib@v1.13.4/urllib.go:526 +0x103 gitlab.mvalley.com/ir/caesarion/internal/crawler/plugins.(*Base).Crawler(0x18038b0, 0xc005b5ca80, 0x0, 0x0) /home/wangy/mvalley/2021/caesarion/internal/crawler/plugins/base.go:30 +0x111 gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).crawl.func2() /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:86 +0xcf github.com/dollarkillerx/async_utils.(*easyPool).core.func1(0xc0014a1c60, 0xc0017032b0, 0xc006a677a0) /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:60 +0xa5 created by github.com/dollarkillerx/async_utils.(*easyPool).core /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:48 +0xac goroutine 18200 [chan send]: github.com/dollarkillerx/async_utils.(*easyPool).core(0xc0050deaa0) /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:46 +0x5f created by github.com/dollarkillerx/async_utils.NewPoolFunc /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:23 +0xc2 goroutine 26661 [select]: net/http.(*persistConn).writeLoop(0xc007c505a0) /usr/local/go/src/net/http/transport.go:2204 +0x123 created by net/http.(*Transport).dialConn /usr/local/go/src/net/http/transport.go:1575 +0xb23 goroutine 1498 [chan receive]: github.com/dollarkillerx/async_utils.(*easyPool).core(0xc0014a0de0) /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:42 +0xdb created by github.com/dollarkillerx/async_utils.NewPoolFunc /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:23 +0xc2 goroutine 1499 [select]: gitlab.mvalley.com/ir/caesarion/internal/generate.(*Generate).GenerateTask(0x18038b0, 0xc0015d25d0, 0x24, 0xc001702250, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/wangy/mvalley/2021/caesarion/internal/generate/generate.go:30 +0x28f created by gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).Run /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:52 +0xdc goroutine 1500 [chan receive]: gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).crawl(0xc0002d7980, 0xc0016a9d40, 0xc0016a9da0) /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:73 +0xce created by gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).Run /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:55 +0x190 goroutine 1501 [chan receive]: gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).clean(0xc0002d7980, 0xc0016a9da0, 0xc0016a9e00) /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:104 +0xce created by gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).Run /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:58 +0x244 goroutine 1502 [chan receive]: gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).storage(0xc0002d7980, 0xc0016a9e00, 0xc0016a9e60) /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:145 +0xe8 created by gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).Run /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:61 +0x2f8 goroutine 1503 [chan receive]: github.com/dollarkillerx/async_utils.(*easyPool).core(0xc0014a0e40) /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:42 +0xdb created by github.com/dollarkillerx/async_utils.NewPoolFunc /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:23 +0xc2 goroutine 1504 [chan receive]: github.com/dollarkillerx/async_utils.(*easyPool).core(0xc0014a0e60) /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:42 +0xdb created by github.com/dollarkillerx/async_utils.NewPoolFunc /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:23 +0xc2 goroutine 164988 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc043703400) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/internal/transport/transport.go:321 +0xcc google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/internal/transport/transport.go:336 google.golang.org/grpc.(*csAttempt).recvMsg(0xc04b5fec00, 0xdd8200, 0xc04b5eb040, 0x0, 0xc04b5be480, 0x23c) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:912 +0x725 google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc04b5fec00, 0x23c, 0x23c) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:777 +0x46 google.golang.org/grpc.(*clientStream).withRetry(0xc04160fe60, 0xc0022fd9f0, 0xc0022fd9c0, 0xc04b5be480, 0xc0436e3a40) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:635 +0x9c google.golang.org/grpc.(*clientStream).RecvMsg(0xc04160fe60, 0xdd8200, 0xc04b5eb040, 0x0, 0x0) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:776 +0x103 google.golang.org/grpc.invoke(0x1150980, 0xc0000a0038, 0xe638c8, 0x29, 0xde8b20, 0xc04b5cb7a0, 0xdd8200, 0xc04b5eb040, 0xc0002d4e00, 0x0, ...) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/call.go:73 +0x13b google.golang.org/grpc.(*ClientConn).Invoke(0xc0002d4e00, 0x1150980, 0xc0000a0038, 0xe638c8, 0x29, 0xde8b20, 0xc04b5cb7a0, 0xdd8200, 0xc04b5eb040, 0x0, ...) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/call.go:37 +0x1b3 gitlab.mvalley.com/ir/caesarion-statistics/rpc/generate.(*caesarionStatisticsClient).GetNextUrls(0xc000330930, 0x1150980, 0xc0000a0038, 0xc04b5cb7a0, 0x0, 0x0, 0x0, 0x4b9ac6, 0x5fe46176, 0xc009d586ce) /home/wangy/go/pkg/mod/gitlab.mvalley.com/ir/caesarion-statistics@v0.0.7-dev/rpc/generate/caesartion_statistics.pb.go:245 +0xd4 gitlab.mvalley.com/ir/caesarion-statistics/sdk/caesarion_statistics.(*CaesarionStatistics).GetNextUrls(0xc000330940, 0xc007f0e100, 0xd, 0x10, 0xc001702250, 0xc, 0x1, 0x5fe4616b, 0x0, 0x0, ...) /home/wangy/go/pkg/mod/gitlab.mvalley.com/ir/caesarion-statistics@v0.0.7-dev/sdk/caesarion_statistics/sdk.go:39 +0xe9 gitlab.mvalley.com/ir/caesarion/internal/storage/plugins.(*NewBaseV1).Storage(0xc0002db280, 0xc002ea2540, 0x11436c0, 0xc0002db280, 0x0, 0x0, 0x2) /home/wangy/mvalley/2021/caesarion/internal/storage/plugins/new_base.go:90 +0x30b gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).storage.func2() /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:166 +0x14d github.com/dollarkillerx/async_utils.(*easyPool).core.func1(0xc0014a0e40, 0xc0017022d0, 0xc006a3a580) /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:60 +0xa5 created by github.com/dollarkillerx/async_utils.(*easyPool).core /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:48 +0xac goroutine 55417 [IO wait]: internal/poll.runtime_pollWait(0x7fadc675a3c8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc00db3a318, 0x72, 0x3400, 0x34cd, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc00db3a300, 0xc0154aca00, 0x34cd, 0x34cd, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(*netFD).Read(0xc00db3a300, 0xc0154aca00, 0x34cd, 0x34cd, 0x203003, 0x17, 0x34c8) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(*conn).Read(0xc00e2d2d70, 0xc0154aca00, 0x34cd, 0x34cd, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 crypto/tls.(*atLeastReader).Read(0xc00ffe5420, 0xc0154aca00, 0x34cd, 0x34cd, 0x28, 0x8, 0xc00493b8a0) /usr/local/go/src/crypto/tls/conn.go:780 +0x60 bytes.(*Buffer).ReadFrom(0xc0077d0958, 0x113c740, 0xc00ffe5420, 0x409de5, 0xd58120, 0xe11f20) /usr/local/go/src/bytes/buffer.go:204 +0xb4 crypto/tls.(*Conn).readFromUntil(0xc0077d0700, 0x113e0a0, 0xc00e2d2d70, 0x5, 0xc00e2d2d70, 0xdaa000) /usr/local/go/src/crypto/tls/conn.go:802 +0xec crypto/tls.(*Conn).readRecordOrCCS(0xc0077d0700, 0x0, 0x0, 0x3) /usr/local/go/src/crypto/tls/conn.go:609 +0x124 crypto/tls.(*Conn).readRecord(...) /usr/local/go/src/crypto/tls/conn.go:577 crypto/tls.(*Conn).Read(0xc0077d0700, 0xc005deb000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/crypto/tls/conn.go:1255 +0x161 net/http.(*persistConn).Read(0xc000322a20, 0xc005deb000, 0x1000, 0x1000, 0xc005098840, 0xc00493bc20, 0x404d15) /usr/local/go/src/net/http/transport.go:1752 +0x75 bufio.(*Reader).fill(0xc00178a8a0) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(*Reader).Peek(0xc00178a8a0, 0x1, 0x0, 0x0, 0x1, 0xc003d8a700, 0x0) /usr/local/go/src/bufio/bufio.go:138 +0x4f net/http.(*persistConn).readLoop(0xc000322a20) /usr/local/go/src/net/http/transport.go:1905 +0x1d6 created by net/http.(*Transport).dialConn /usr/local/go/src/net/http/transport.go:1574 +0xafe goroutine 162234 [select]: google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00192a400) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/internal/transport/transport.go:321 +0xcc google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/internal/transport/transport.go:336 google.golang.org/grpc.(*csAttempt).recvMsg(0xc00d1a1000, 0xdd8200, 0xc012e7cec0, 0x0, 0xc035c73900, 0x4e0) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:912 +0x725 google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00d1a1000, 0x4e0, 0x4e0) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:777 +0x46 google.golang.org/grpc.(*clientStream).withRetry(0xc01fad0360, 0xc01a1d19f0, 0xc01a1d19c0, 0xc035c73900, 0xc01fba98a0) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:635 +0x9c google.golang.org/grpc.(*clientStream).RecvMsg(0xc01fad0360, 0xdd8200, 0xc012e7cec0, 0x0, 0x0) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:776 +0x103 google.golang.org/grpc.invoke(0x1150980, 0xc0000a0038, 0xe638c8, 0x29, 0xde8b20, 0xc039737980, 0xdd8200, 0xc012e7cec0, 0xc0002d4e00, 0x0, ...) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/call.go:73 +0x13b google.golang.org/grpc.(*ClientConn).Invoke(0xc0002d4e00, 0x1150980, 0xc0000a0038, 0xe638c8, 0x29, 0xde8b20, 0xc039737980, 0xdd8200, 0xc012e7cec0, 0x0, ...) /home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/call.go:37 +0x1b3 gitlab.mvalley.com/ir/caesarion-statistics/rpc/generate.(*caesarionStatisticsClient).GetNextUrls(0xc000330930, 0x1150980, 0xc0000a0038, 0xc039737980, 0x0, 0x0, 0x0, 0x4b9ac6, 0x5fe46175, 0xc02d160f21) /home/wangy/go/pkg/mod/gitlab.mvalley.com/ir/caesarion-statistics@v0.0.7-dev/rpc/generate/caesartion_statistics.pb.go:245 +0xd4 gitlab.mvalley.com/ir/caesarion-statistics/sdk/caesarion_statistics.(*CaesarionStatistics).GetNextUrls(0xc000330940, 0xc00658c000, 0x1e, 0x20, 0xc0028d3940, 0x9, 0x1, 0x5fe4616e, 0x0, 0x0, ...) /home/wangy/go/pkg/mod/gitlab.mvalley.com/ir/caesarion-statistics@v0.0.7-dev/sdk/caesarion_statistics/sdk.go:39 +0xe9 gitlab.mvalley.com/ir/caesarion/internal/storage/plugins.(*NewBaseV1).Storage(0xc0002db280, 0xc0294fe460, 0x11436c0, 0xc0002db280, 0x0, 0x0, 0x2) /home/wangy/mvalley/2021/caesarion/internal/storage/plugins/new_base.go:90 +0x30b gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).storage.func2() /home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:166 +0x14d github.com/dollarkillerx/async_utils.(*easyPool).core.func1(0xc002771660, 0xc0028d39b4, 0xc01cddaca0) /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:60 +0xa5 created by github.com/dollarkillerx/async_utils.(*easyPool).core /home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:48 +0xac goroutine 91433 [IO wait]: internal/poll.runtime_pollWait(0x7fadc6459000, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc006db8998, 0x72, 0x2100, 0x21db, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc006db8980, 0xc00a5ab400, 0x21db, 0x21db, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(*netFD).Read(0xc006db8980, 0xc00a5ab400, 0x21db, 0x21db, 0x203001, 0x0, 0x21ce) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(*conn).Read(0xc00365e9b0, 0xc00a5ab400, 0x21db, 0x21db, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 crypto/tls.(*atLeastReader).Read(0xc007106a60, 0xc00a5ab400, 0x21db, 0x21db, 0x28, 0x8, 0xc019cb48a0) /usr/local/go/src/crypto/tls/conn.go:780 +0x60 bytes.(*Buffer).ReadFrom(0xc00e94ecd8, 0x113c740, 0xc007106a60, 0x409de5, 0xd58120, 0xe11f20) /usr/local/go/src/bytes/buffer.go:204 +0xb4 crypto/tls.(*Conn).readFromUntil(0xc00e94ea80, 0x113e0a0, 0xc00365e9b0, 0x5, 0xc00365e9b0, 0xdaa000) /usr/local/go/src/crypto/tls/conn.go:802 +0xec crypto/tls.(*Conn).readRecordOrCCS(0xc00e94ea80, 0x0, 0x0, 0x3) /usr/local/go/src/crypto/tls/conn.go:609 +0x124 crypto/tls.(*Conn).readRecord(...) /usr/local/go/src/crypto/tls/conn.go:577 crypto/tls.(*Conn).Read(0xc00e94ea80, 0xc00a952000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/crypto/tls/conn.go:1255 +0x161 net/http.(*persistConn).Read(0xc00639d560, 0xc00a952000, 0x1000, 0x1000, 0xc010e8be60, 0xc019cb4c20, 0x404d15) /usr/local/go/src/net/http/transport.go:1752 +0x75 bufio.(*Reader).fill(0xc019f93560) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(*Reader).Peek(0xc019f93560, 0x1, 0x0, 0x0, 0x1, 0xc019cdfb00, 0x0) /usr/local/go/src/bufio/bufio.go:138 +0x4f net/http.(*persistConn).readLoop(0xc00639d560) /usr/local/go/src/net/http/transport.go:1905 +0x1d6 created by net/http.(*Transport).dialConn /usr/local/go/src/net/http/transport.go:1574 +0xafe ```

有疑问加站长微信联系(非本文作者)

入群交流(和以上内容无关):加入Go大咖交流群,或添加微信:liuxiaoyan-s 备注:入群;或加QQ群:692541889

1122 次点击  
加入收藏 微博
暂无回复
添加一条新回复 (您需要 登录 后才能回复 没有账号 ?)
  • 请尽量让自己的回复能够对别人有帮助
  • 支持 Markdown 格式, **粗体**、~~删除线~~、`单行代码`
  • 支持 @ 本站用户;支持表情(输入 : 提示),见 Emoji cheat sheet
  • 图片支持拖拽、截图粘贴等方式上传