mozilla-services / heka

DEPRECATED: Data collection and processing made easy.
http://hekad.readthedocs.org/
Other
3.4k stars 531 forks source link

The reason for runtime/cgo: pthread_create failed: Resource temporarily unavailable? #1983

Closed SeeWei1985 closed 7 years ago

SeeWei1985 commented 7 years ago

HI, the output log of my heka is described as follow. Thus, It is related with kafka server ?

runtime/cgo: pthread_create failed: Resource temporarily unavailable SIGABRT: abort PC=0x7f28881f15f7 m=6

goroutine 0 [idle]:

goroutine 7 [syscall]: runtime.notetsleepg(0x11b6138, 0x391a774, 0x0) /work/go/src/runtime/lock_futex.go:205 +0x4e fp=0xc820022738 sp=0xc820022710 runtime.timerproc() /work/go/src/runtime/time.go:209 +0xde fp=0xc8200227c0 sp=0xc820022738 runtime.goexit() /work/go/src/runtime/asm_amd64.s:1998 +0x1 fp=0xc8200227c8 sp=0xc8200227c0 created by runtime.addtimerLocked /work/go/src/runtime/time.go:116 +0x11f

goroutine 1 [chan receive, 36 minutes]: github.com/mozilla-services/heka/pipeline.Run(0xc82000c3c0, 0xc8200f8d90) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/pipeline_runner.go:341 +0x1567 main.main() /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/cmd/hekad/main.go:234 +0x166f

goroutine 17 [syscall, 160 minutes, locked to thread]: runtime.goexit() /work/go/src/runtime/asm_amd64.s:1998 +0x1

goroutine 5 [syscall, 36 minutes]: os/signal.signal_recv(0x7f2889266078) /work/go/src/runtime/sigqueue.go:116 +0x132 os/signal.loop() /work/go/src/os/signal/signal_unix.go:22 +0x18 created by os/signal.init.1 /work/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 19 [select, 2 minutes]: github.com/Shopify/sarama.(_client).backgroundMetadataUpdater(0xc820071e80) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/client.go:553 +0x323 github.com/Shopify/sarama.(_client).(github.com/Shopify/sarama.backgroundMetadataUpdater)-fm() /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/client.go:142 +0x20 github.com/Shopify/sarama.withRecover(0xc8201a8140) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/utils.go:42 +0x3a created by github.com/Shopify/sarama.NewClient /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/client.go:142 +0x75a

goroutine 18 [chan receive, 2 minutes]: github.com/Shopify/sarama.(_Broker).responseReceiver(0xc820182770) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/broker.go:345 +0xdf github.com/Shopify/sarama.(_Broker).(github.com/Shopify/sarama.responseReceiver)-fm() /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/broker.go:98 +0x20 github.com/Shopify/sarama.withRecover(0xc8201a8020) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/utils.go:42 +0x3a created by github.com/Shopify/sarama.(*Broker).Open.func1 /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/broker.go:98 +0x5db

goroutine 9 [chan receive, 160 minutes]: github.com/Shopify/sarama.(_partitionConsumer).dispatcher(0xc8201ae180) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:299 +0x57 github.com/Shopify/sarama.(_partitionConsumer).(github.com/Shopify/sarama.dispatcher)-fm() /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:152 +0x20 github.com/Shopify/sarama.withRecover(0xc82018e910) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/utils.go:42 +0x3a created by github.com/Shopify/sarama.(*consumer).ConsumePartition /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:152 +0x451

goroutine 8 [runnable]: github.com/Shopify/sarama.(_Broker).responseReceiver(0xc8201b2000) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/broker.go:371 +0x6a9 github.com/Shopify/sarama.(_Broker).(github.com/Shopify/sarama.responseReceiver)-fm() /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/broker.go:98 +0x20 github.com/Shopify/sarama.withRecover(0xc82018e860) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/utils.go:42 +0x3a created by github.com/Shopify/sarama.(*Broker).Open.func1 /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/broker.go:98 +0x5db

goroutine 10 [chan receive]: github.com/Shopify/sarama.(_partitionConsumer).responseFeeder(0xc8201ae180) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:410 +0x80 github.com/Shopify/sarama.(_partitionConsumer).(github.com/Shopify/sarama.responseFeeder)-fm() /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:153 +0x20 github.com/Shopify/sarama.withRecover(0xc82018e920) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/utils.go:42 +0x3a created by github.com/Shopify/sarama.(*consumer).ConsumePartition /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:153 +0x4a8

goroutine 11 [select]: github.com/Shopify/sarama.(_brokerConsumer).subscriptionManager(0xc82015f7c0) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:551 +0x3e9 github.com/Shopify/sarama.(_brokerConsumer).(github.com/Shopify/sarama.subscriptionManager)-fm() /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:524 +0x20 github.com/Shopify/sarama.withRecover(0xc82018e930) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/utils.go:42 +0x3a created by github.com/Shopify/sarama.(*consumer).newBrokerConsumer /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:524 +0x201

goroutine 12 [select]: github.com/Shopify/sarama.(_Broker).sendAndReceive(0xc8201b2000, 0x7f288926cf78, 0xc8201d80b0, 0x7f288926cfb8, 0xc8201a0008, 0x0, 0x0) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/broker.go:291 +0x240 github.com/Shopify/sarama.(_Broker).Fetch(0xc8201b2000, 0xc8201d80b0, 0xc820037d54, 0x0, 0x0) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/broker.go:209 +0xc1 github.com/Shopify/sarama.(_brokerConsumer).fetchNewMessages(0xc82015f7c0, 0xc820053e00, 0x0, 0x0) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:675 +0x353 github.com/Shopify/sarama.(_brokerConsumer).subscriptionConsumer(0xc82015f7c0) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:587 +0x3a4 github.com/Shopify/sarama.(_brokerConsumer).(github.com/Shopify/sarama.subscriptionConsumer)-fm() /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:525 +0x20 github.com/Shopify/sarama.withRecover(0xc82018e940) /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/utils.go:42 +0x3a created by github.com/Shopify/sarama.(_consumer).newBrokerConsumer /work/heka/install/heka/build/heka/src/github.com/Shopify/sarama/consumer.go:525 +0x254

goroutine 13 [select, 44 minutes]: github.com/mozilla-services/heka/pipeline.(_foRunner).channelLoop(0xc82010a820, 0x7f288926d7d0, 0xc82007e9a0, 0x7f288926d740, 0xc82000c3c0, 0x0, 0x0, 0x0, 0x0) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/plugin_runners.go:1117 +0x810 github.com/mozilla-services/heka/pipeline.(_foRunner).Starter(0xc82010a820, 0x7f288926d7d0, 0xc82007e9a0, 0x7f288926d740, 0xc82000c3c0, 0xc82018fb00) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/plugin_runners.go:1242 +0xf00 created by github.com/mozilla-services/heka/pipeline.(*foRunner).Start /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/plugin_runners.go:1071 +0xbb4

goroutine 14 [runnable]: fmt.(_pp).fmtString(0xc8201ac000, 0xc17d20, 0x9, 0xc800000073) /work/go/src/fmt/print.go:512 fmt.(_pp).printArg(0xc8201ac000, 0x9e7880, 0xc821012d90, 0x73, 0x0, 0x0) /work/go/src/fmt/print.go:797 +0xd95 fmt.(_pp).doPrintf(0xc8201ac000, 0xc109b0, 0x8, 0xc8201158b0, 0x2, 0x2) /work/go/src/fmt/print.go:1238 +0x1dcd fmt.Sprintf(0xc109b0, 0x8, 0xc8201158b0, 0x2, 0x2, 0x0, 0x0) /work/go/src/fmt/print.go:203 +0x6f github.com/mozilla-services/heka/plugins.(_RstEncoder).writeAttr(0xc82018dc40, 0xc820184000, 0xc17d20, 0x9, 0xc82019c3c0, 0x1d) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/plugins/rst_encoder.go:43 +0x181 github.com/mozilla-services/heka/plugins.(_RstEncoder).Encode(0xc82018dc40, 0xc820070300, 0x0, 0x0, 0x0, 0x0, 0x0) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/plugins/rst_encoder.go:83 +0x1d8 github.com/mozilla-services/heka/pipeline.(_foRunner).Encode(0xc82010a9c0, 0xc820070300, 0x0, 0x0, 0x0, 0x0, 0x0) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/plugin_runners.go:1696 +0xa5 github.com/mozilla-services/heka/plugins.(_LogOutput).Run(0xc82018fa20, 0x7f288926d5e8, 0xc82010a9c0, 0x7f288926d740, 0xc82000c3c0, 0x0, 0x0) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/plugins/log_output.go:52 +0x1fb github.com/mozilla-services/heka/pipeline.(_foRunner).OldStarter(0xc82010a9c0, 0x7f288926d740, 0xc82000c3c0, 0xc82018fb00) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/plugin_runners.go:1502 +0x9a9 created by github.com/mozilla-services/heka/pipeline.(*foRunner).Start /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/plugin_runners.go:1073 +0xc9b

goroutine 15 [runnable, locked to thread]: github.com/mozilla-services/heka/sandbox/lua.lookup_field(0xc82103acf0, 0xc821012dc7, 0x6, 0x0, 0x0, 0x4087e3, 0x99a540, 0xc820043818) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/sandbox/lua/lua_sandbox.go:77 github.com/mozilla-services/heka/sandbox/lua.go_lua_read_message(0xc820155fc0, 0x7f28740022e8, 0x0, 0x0, 0x0, 0x43139d, 0xc8200439a0) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/sandbox/lua/lua_sandbox.go:345 +0x459 github.com/mozilla-services/heka/sandbox/lua._cgoexpwrap_fecf2fd76b90_go_lua_read_message(0xc820155fc0, 0x7f28740022e8, 0x0, 0x0, 0x0, 0x0, 0x0) github.com/mozilla-services/heka/sandbox/lua/_obj/_cgo_gotypes.go:232 +0xa0 github.com/mozilla-services/heka/sandbox/lua._Cfunc_process_message(0x7f2874000c00, 0x7f2800000000) github.com/mozilla-services/heka/sandbox/lua/_obj/_cgo_gotypes.go:173 +0x3d github.com/mozilla-services/heka/sandbox/lua.(_LuaSandbox).ProcessMessage(0xc820155fc0, 0xc820070300, 0x725) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/sandbox/lua/lua_sandbox.go:759 +0x9c github.com/mozilla-services/heka/sandbox/plugins.(_SandboxFilter).Run(0xc820105540, 0x7f288926d300, 0xc82010a680, 0x7f288926d740, 0xc82000c3c0, 0x0, 0x0) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/sandbox/plugins/sandbox_filter.go:259 +0x108e github.com/mozilla-services/heka/pipeline.(_foRunner).OldStarter(0xc82010a680, 0x7f288926d740, 0xc82000c3c0, 0xc82000c450) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/plugin_runners.go:1499 +0x858 created by github.com/mozilla-services/heka/pipeline.(_foRunner).Start /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/plugin_runners.go:1073 +0xc9b

goroutine 21 [chan receive]: github.com/mozilla-services/heka/pipeline.(_MatchRunner).run(0xc8201d4800, 0x3e8) /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/router.go:307 +0xe0 created by github.com/mozilla-services/heka/pipeline.(_MatchRunner).Start /work/heka/install/heka/build/heka/src/github.com/mozilla-services/heka/pipeline/router.go:360 +0x3f