系统并发3000左右报的错误 并发2000没有这个问题? 时在摸不着头脑??
```
goroutine 17502 [sleep]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:310
time.Sleep(0xee6b2800)
/usr/local/go/src/runtime/time.go:105 +0x157
github.com/dollarkillerx/urllib.(*Urllib).BodyRetry(0xc0098ef290, 0x3, 0xe3f34b, 0x3, 0xc0098ef290)
/home/wangy/go/pkg/mod/github.com/dollarkillerx/urllib@v1.13.4/urllib.go:526 +0x103
gitlab.mvalley.com/ir/caesarion/internal/crawler/plugins.(*Base).Crawler(0x18038b0, 0xc005b5ca80, 0x0, 0x0)
/home/wangy/mvalley/2021/caesarion/internal/crawler/plugins/base.go:30 +0x111
gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).crawl.func2()
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:86 +0xcf
github.com/dollarkillerx/async_utils.(*easyPool).core.func1(0xc0014a1c60, 0xc0017032b0, 0xc006a677a0)
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:60 +0xa5
created by github.com/dollarkillerx/async_utils.(*easyPool).core
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:48 +0xac
goroutine 18200 [chan send]:
github.com/dollarkillerx/async_utils.(*easyPool).core(0xc0050deaa0)
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:46 +0x5f
created by github.com/dollarkillerx/async_utils.NewPoolFunc
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:23 +0xc2
goroutine 26661 [select]:
net/http.(*persistConn).writeLoop(0xc007c505a0)
/usr/local/go/src/net/http/transport.go:2204 +0x123
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1575 +0xb23
goroutine 1498 [chan receive]:
github.com/dollarkillerx/async_utils.(*easyPool).core(0xc0014a0de0)
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:42 +0xdb
created by github.com/dollarkillerx/async_utils.NewPoolFunc
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:23 +0xc2
goroutine 1499 [select]:
gitlab.mvalley.com/ir/caesarion/internal/generate.(*Generate).GenerateTask(0x18038b0, 0xc0015d25d0, 0x24, 0xc001702250, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/home/wangy/mvalley/2021/caesarion/internal/generate/generate.go:30 +0x28f
created by gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).Run
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:52 +0xdc
goroutine 1500 [chan receive]:
gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).crawl(0xc0002d7980, 0xc0016a9d40, 0xc0016a9da0)
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:73 +0xce
created by gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).Run
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:55 +0x190
goroutine 1501 [chan receive]:
gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).clean(0xc0002d7980, 0xc0016a9da0, 0xc0016a9e00)
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:104 +0xce
created by gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).Run
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:58 +0x244
goroutine 1502 [chan receive]:
gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).storage(0xc0002d7980, 0xc0016a9e00, 0xc0016a9e60)
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:145 +0xe8
created by gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).Run
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:61 +0x2f8
goroutine 1503 [chan receive]:
github.com/dollarkillerx/async_utils.(*easyPool).core(0xc0014a0e40)
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:42 +0xdb
created by github.com/dollarkillerx/async_utils.NewPoolFunc
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:23 +0xc2
goroutine 1504 [chan receive]:
github.com/dollarkillerx/async_utils.(*easyPool).core(0xc0014a0e60)
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:42 +0xdb
created by github.com/dollarkillerx/async_utils.NewPoolFunc
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:23 +0xc2
goroutine 164988 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc043703400)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/internal/transport/transport.go:321 +0xcc
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/internal/transport/transport.go:336
google.golang.org/grpc.(*csAttempt).recvMsg(0xc04b5fec00, 0xdd8200, 0xc04b5eb040, 0x0, 0xc04b5be480, 0x23c)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:912 +0x725
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc04b5fec00, 0x23c, 0x23c)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:777 +0x46
google.golang.org/grpc.(*clientStream).withRetry(0xc04160fe60, 0xc0022fd9f0, 0xc0022fd9c0, 0xc04b5be480, 0xc0436e3a40)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:635 +0x9c
google.golang.org/grpc.(*clientStream).RecvMsg(0xc04160fe60, 0xdd8200, 0xc04b5eb040, 0x0, 0x0)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:776 +0x103
google.golang.org/grpc.invoke(0x1150980, 0xc0000a0038, 0xe638c8, 0x29, 0xde8b20, 0xc04b5cb7a0, 0xdd8200, 0xc04b5eb040, 0xc0002d4e00, 0x0, ...)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/call.go:73 +0x13b
google.golang.org/grpc.(*ClientConn).Invoke(0xc0002d4e00, 0x1150980, 0xc0000a0038, 0xe638c8, 0x29, 0xde8b20, 0xc04b5cb7a0, 0xdd8200, 0xc04b5eb040, 0x0, ...)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/call.go:37 +0x1b3
gitlab.mvalley.com/ir/caesarion-statistics/rpc/generate.(*caesarionStatisticsClient).GetNextUrls(0xc000330930, 0x1150980, 0xc0000a0038, 0xc04b5cb7a0, 0x0, 0x0, 0x0, 0x4b9ac6, 0x5fe46176, 0xc009d586ce)
/home/wangy/go/pkg/mod/gitlab.mvalley.com/ir/caesarion-statistics@v0.0.7-dev/rpc/generate/caesartion_statistics.pb.go:245 +0xd4
gitlab.mvalley.com/ir/caesarion-statistics/sdk/caesarion_statistics.(*CaesarionStatistics).GetNextUrls(0xc000330940, 0xc007f0e100, 0xd, 0x10, 0xc001702250, 0xc, 0x1, 0x5fe4616b, 0x0, 0x0, ...)
/home/wangy/go/pkg/mod/gitlab.mvalley.com/ir/caesarion-statistics@v0.0.7-dev/sdk/caesarion_statistics/sdk.go:39 +0xe9
gitlab.mvalley.com/ir/caesarion/internal/storage/plugins.(*NewBaseV1).Storage(0xc0002db280, 0xc002ea2540, 0x11436c0, 0xc0002db280, 0x0, 0x0, 0x2)
/home/wangy/mvalley/2021/caesarion/internal/storage/plugins/new_base.go:90 +0x30b
gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).storage.func2()
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:166 +0x14d
github.com/dollarkillerx/async_utils.(*easyPool).core.func1(0xc0014a0e40, 0xc0017022d0, 0xc006a3a580)
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:60 +0xa5
created by github.com/dollarkillerx/async_utils.(*easyPool).core
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:48 +0xac
goroutine 55417 [IO wait]:
internal/poll.runtime_pollWait(0x7fadc675a3c8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00db3a318, 0x72, 0x3400, 0x34cd, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00db3a300, 0xc0154aca00, 0x34cd, 0x34cd, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00db3a300, 0xc0154aca00, 0x34cd, 0x34cd, 0x203003, 0x17, 0x34c8)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00e2d2d70, 0xc0154aca00, 0x34cd, 0x34cd, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc00ffe5420, 0xc0154aca00, 0x34cd, 0x34cd, 0x28, 0x8, 0xc00493b8a0)
/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc0077d0958, 0x113c740, 0xc00ffe5420, 0x409de5, 0xd58120, 0xe11f20)
/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc0077d0700, 0x113e0a0, 0xc00e2d2d70, 0x5, 0xc00e2d2d70, 0xdaa000)
/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc0077d0700, 0x0, 0x0, 0x3)
/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc0077d0700, 0xc005deb000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc000322a20, 0xc005deb000, 0x1000, 0x1000, 0xc005098840, 0xc00493bc20, 0x404d15)
/usr/local/go/src/net/http/transport.go:1752 +0x75
bufio.(*Reader).fill(0xc00178a8a0)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00178a8a0, 0x1, 0x0, 0x0, 0x1, 0xc003d8a700, 0x0)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc000322a20)
/usr/local/go/src/net/http/transport.go:1905 +0x1d6
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1574 +0xafe
goroutine 162234 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00192a400)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/internal/transport/transport.go:321 +0xcc
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/internal/transport/transport.go:336
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00d1a1000, 0xdd8200, 0xc012e7cec0, 0x0, 0xc035c73900, 0x4e0)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:912 +0x725
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00d1a1000, 0x4e0, 0x4e0)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:777 +0x46
google.golang.org/grpc.(*clientStream).withRetry(0xc01fad0360, 0xc01a1d19f0, 0xc01a1d19c0, 0xc035c73900, 0xc01fba98a0)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:635 +0x9c
google.golang.org/grpc.(*clientStream).RecvMsg(0xc01fad0360, 0xdd8200, 0xc012e7cec0, 0x0, 0x0)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/stream.go:776 +0x103
google.golang.org/grpc.invoke(0x1150980, 0xc0000a0038, 0xe638c8, 0x29, 0xde8b20, 0xc039737980, 0xdd8200, 0xc012e7cec0, 0xc0002d4e00, 0x0, ...)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/call.go:73 +0x13b
google.golang.org/grpc.(*ClientConn).Invoke(0xc0002d4e00, 0x1150980, 0xc0000a0038, 0xe638c8, 0x29, 0xde8b20, 0xc039737980, 0xdd8200, 0xc012e7cec0, 0x0, ...)
/home/wangy/go/pkg/mod/google.golang.org/grpc@v1.34.0/call.go:37 +0x1b3
gitlab.mvalley.com/ir/caesarion-statistics/rpc/generate.(*caesarionStatisticsClient).GetNextUrls(0xc000330930, 0x1150980, 0xc0000a0038, 0xc039737980, 0x0, 0x0, 0x0, 0x4b9ac6, 0x5fe46175, 0xc02d160f21)
/home/wangy/go/pkg/mod/gitlab.mvalley.com/ir/caesarion-statistics@v0.0.7-dev/rpc/generate/caesartion_statistics.pb.go:245 +0xd4
gitlab.mvalley.com/ir/caesarion-statistics/sdk/caesarion_statistics.(*CaesarionStatistics).GetNextUrls(0xc000330940, 0xc00658c000, 0x1e, 0x20, 0xc0028d3940, 0x9, 0x1, 0x5fe4616e, 0x0, 0x0, ...)
/home/wangy/go/pkg/mod/gitlab.mvalley.com/ir/caesarion-statistics@v0.0.7-dev/sdk/caesarion_statistics/sdk.go:39 +0xe9
gitlab.mvalley.com/ir/caesarion/internal/storage/plugins.(*NewBaseV1).Storage(0xc0002db280, 0xc0294fe460, 0x11436c0, 0xc0002db280, 0x0, 0x0, 0x2)
/home/wangy/mvalley/2021/caesarion/internal/storage/plugins/new_base.go:90 +0x30b
gitlab.mvalley.com/ir/caesarion/internal/dispatch.(*Task).storage.func2()
/home/wangy/mvalley/2021/caesarion/internal/dispatch/task.go:166 +0x14d
github.com/dollarkillerx/async_utils.(*easyPool).core.func1(0xc002771660, 0xc0028d39b4, 0xc01cddaca0)
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:60 +0xa5
created by github.com/dollarkillerx/async_utils.(*easyPool).core
/home/wangy/go/pkg/mod/github.com/dollarkillerx/async_utils@v0.0.2/pool.go:48 +0xac
goroutine 91433 [IO wait]:
internal/poll.runtime_pollWait(0x7fadc6459000, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc006db8998, 0x72, 0x2100, 0x21db, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc006db8980, 0xc00a5ab400, 0x21db, 0x21db, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc006db8980, 0xc00a5ab400, 0x21db, 0x21db, 0x203001, 0x0, 0x21ce)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00365e9b0, 0xc00a5ab400, 0x21db, 0x21db, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc007106a60, 0xc00a5ab400, 0x21db, 0x21db, 0x28, 0x8, 0xc019cb48a0)
/usr/local/go/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc00e94ecd8, 0x113c740, 0xc007106a60, 0x409de5, 0xd58120, 0xe11f20)
/usr/local/go/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc00e94ea80, 0x113e0a0, 0xc00365e9b0, 0x5, 0xc00365e9b0, 0xdaa000)
/usr/local/go/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc00e94ea80, 0x0, 0x0, 0x3)
/usr/local/go/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc00e94ea80, 0xc00a952000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1255 +0x161
net/http.(*persistConn).Read(0xc00639d560, 0xc00a952000, 0x1000, 0x1000, 0xc010e8be60, 0xc019cb4c20, 0x404d15)
/usr/local/go/src/net/http/transport.go:1752 +0x75
bufio.(*Reader).fill(0xc019f93560)
/usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc019f93560, 0x1, 0x0, 0x0, 0x1, 0xc019cdfb00, 0x0)
/usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00639d560)
/usr/local/go/src/net/http/transport.go:1905 +0x1d6
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1574 +0xafe
```
有疑问加站长微信联系(非本文作者)