search

confluentinc / confluent-kafka-go

Confluent's Apache Kafka Golang client
Apache License 2.0
4.67k stars 659 forks source link

Async Producer Panic #1190

Open mathiasyeremiaaryadi opened 6 months ago

mathiasyeremiaaryadi commented 6 months ago

Description

My Golang service produces the data to the topic where all the data are fetched from the database. I implement the async producer, sometimes the logic works and sometimes an issue occured. The size of the data to be produced is around 1-3million. The issue seems panicking when the producer start to produce the data. I have no idea what is the error about, need some insight about how to solve it. Here is the Go log error:

fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x22c pc=0x924a75]

runtime stack:
runtime.throw({0xdf482a?, 0x0?})
    /usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0x7f8da7ffada0 sp=0x7f8da7ffad70 pc=0x4a43dd
runtime.sigpanic()
    /usr/local/go/src/runtime/signal_unix.go:821 +0x3e9 fp=0x7f8da7ffae00 sp=0x7f8da7ffada0 pc=0x4bb209

goroutine 20336 [syscall]:
runtime.cgocall(0x88e6e0, 0xc000130390)
    /usr/local/go/src/runtime/cgocall.go:157 +0x5c fp=0xc000130368 sp=0xc000130330 pc=0x471b9c
github.com/confluentinc/confluent-kafka-go/kafka._Cfunc_do_produce(0x7f8dfc01e5a0, 0x7f8db4000d00, 0xffffffff, 0x2, 0x0, 0xc000912e00, 0x693, 0x0, 0xc111704a70, 0x10, ...)
    _cgo_gotypes.go:786 +0x4f fp=0xc000130390 sp=0xc000130368 pc=0x80a46f
github.com/confluentinc/confluent-kafka-go/kafka.(*Producer).produce.func2(0xc00026e3c0?, 0x7f8db4000d00, 0xc0001fc230?, 0x0, 0x47b4ea?, 0xc000130528, 0x30?, 0x30?, 0xc000130558, 0x10, ...)
    /go/src/github.com/app/vendor/github.com/confluentinc/confluent-kafka-go/kafka/producer.go:263 +0x248 fp=0xc000130448 sp=0xc000130390 pc=0x8159c8
github.com/confluentinc/confluent-kafka-go/kafka.(*Producer).produce(0xc000b52000, 0xc0000c4160, 0xc000a08078?, 0x0)
    /go/src/github.com/app/vendor/github.com/confluentinc/confluent-kafka-go/kafka/producer.go:263 +0x4e5 fp=0xc000130600 sp=0xc000130448 pc=0x8154e5
github.com/confluentinc/confluent-kafka-go/kafka.(*Producer).Produce(...)
    /go/src/github.com/app/vendor/github.com/confluentinc/confluent-kafka-go/kafka/producer.go:285
main.Repository.ProduceData({_, _, _, _}, _, {{0xc000952d40, 0x3}, {0xc000a08078, 0x10}, {0xc000952d50, ...}, ...})
    /go/src/github.com/app/repository.go:87 +0xfa fp=0xc000130bd0 sp=0xc000130600 pc=0x8862da
main.Usecase.IngestData({0xc00003c100, {0xc00003c100, 0xc0003289c0, 0xc0000b4280, 0xc000b52000}, 0xc000b52000}, 0x135c360?, {{0xc00002c070, 0x46}, {0xc000032074, ...}, ...})
    /go/src/github.com/app/usecase.go:155 +0x11fb fp=0xc000131e60 sp=0xc000130bd0 pc=0x88837b
main.main.func1()
    /go/src/github.com/app/main.go:37 +0x2bd fp=0xc000131f98 sp=0xc000131e60 pc=0x884d3d
github.com/robfig/cron/v3.FuncJob.Run(0xc0001e5650?)
    /go/src/github.com/app/vendor/github.com/robfig/cron/v3/cron.go:136 +0x1a fp=0xc000131fa8 sp=0xc000131f98 pc=0x84c57a
github.com/robfig/cron/v3.(*Cron).startJob.func1()
    /go/src/github.com/app/vendor/github.com/robfig/cron/v3/cron.go:312 +0x6a fp=0xc000131fe0 sp=0xc000131fa8 pc=0x84ddaa
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000131fe8 sp=0xc000131fe0 pc=0x4d7881
created by github.com/robfig/cron/v3.(*Cron).startJob
    /go/src/github.com/app/vendor/github.com/robfig/cron/v3/cron.go:310 +0xad

goroutine 1 [chan receive, 4320 minutes]:
runtime.gopark(0x0?, 0xc0005d5d70?, 0xc7?, 0xb8?, 0xc0005d5d70?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0005a2d40 sp=0xc0005a2d20 pc=0x4a70f6
runtime.chanrecv(0xc0000b2f00, 0x0, 0x1)
    /usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc0005a2dd0 sp=0xc0005a2d40 pc=0x47495d
runtime.chanrecv1(0xc0000b2f00?, 0xc0005d5e88?)
    /usr/local/go/src/runtime/chan.go:442 +0x18 fp=0xc0005a2df8 sp=0xc0005a2dd0 pc=0x474458
main.main()
    /go/src/github.com/app/main.go:62 +0x3e5 fp=0xc0005a2f80 sp=0xc0005a2df8 pc=0x884a25
runtime.main()
    /usr/local/go/src/runtime/proc.go:250 +0x207 fp=0xc0005a2fe0 sp=0xc0005a2f80 pc=0x4a6cc7
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005a2fe8 sp=0xc0005a2fe0 pc=0x4d7881

goroutine 2 [force gc (idle), 3 minutes]:
runtime.gopark(0x46c916c2e908f1?, 0x4617da86fce835?, 0x0?, 0x0?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00006cfb0 sp=0xc00006cf90 pc=0x4a70f6
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:387
runtime.forcegchelper()
    /usr/local/go/src/runtime/proc.go:305 +0xb0 fp=0xc00006cfe0 sp=0xc00006cfb0 pc=0x4a6f30
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00006cfe8 sp=0xc00006cfe0 pc=0x4d7881
created by runtime.init.6
    /usr/local/go/src/runtime/proc.go:293 +0x25

goroutine 3 [GC sweep wait]:
runtime.gopark(0x135be01?, 0x0?, 0x0?, 0x0?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00006d780 sp=0xc00006d760 pc=0x4a70f6
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:387
runtime.bgsweep(0x0?)
    /usr/local/go/src/runtime/mgcsweep.go:319 +0xde fp=0xc00006d7c8 sp=0xc00006d780 pc=0x491c1e
runtime.gcenable.func1()
    /usr/local/go/src/runtime/mgc.go:178 +0x26 fp=0xc00006d7e0 sp=0xc00006d7c8 pc=0x487086
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00006d7e8 sp=0xc00006d7e0 pc=0x4d7881
created by runtime.gcenable
    /usr/local/go/src/runtime/mgc.go:178 +0x6b

goroutine 18 [GC scavenge wait]:
runtime.gopark(0x46c8e702cc951d?, 0x4e721b?, 0x0?, 0x0?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000068770 sp=0xc000068750 pc=0x4a70f6
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:387
runtime.(*scavengerState).park(0x135c3e0)
    /usr/local/go/src/runtime/mgcscavenge.go:400 +0x53 fp=0xc0000687a0 sp=0xc000068770 pc=0x48faf3
runtime.bgscavenge(0x0?)
    /usr/local/go/src/runtime/mgcscavenge.go:633 +0x65 fp=0xc0000687c8 sp=0xc0000687a0 pc=0x4900e5
runtime.gcenable.func2()
    /usr/local/go/src/runtime/mgc.go:179 +0x26 fp=0xc0000687e0 sp=0xc0000687c8 pc=0x487026
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000687e8 sp=0xc0000687e0 pc=0x4d7881
created by runtime.gcenable
    /usr/local/go/src/runtime/mgc.go:179 +0xaa

goroutine 4 [finalizer wait, 6 minutes]:
runtime.gopark(0x0?, 0xc0000bc2d0?, 0x0?, 0xa0?, 0x1000000010?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc003815e28 sp=0xc003815e08 pc=0x4a70f6
runtime.runfinq()
    /usr/local/go/src/runtime/mfinal.go:193 +0x107 fp=0xc003815fe0 sp=0xc003815e28 pc=0x4860c7
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc003815fe8 sp=0xc003815fe0 pc=0x4d7881
created by runtime.createfing
    /usr/local/go/src/runtime/mfinal.go:163 +0x45

goroutine 5 [select]:
runtime.gopark(0xc000533e90?, 0x10?, 0x80?, 0xa4?, 0xc000533a28?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0005336b8 sp=0xc000533698 pc=0x4a70f6
runtime.selectgo(0xc000533e90, 0xc000533a08, 0x1?, 0x0, 0xc0000ae2a0?, 0x1)
    /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0005337f8 sp=0xc0005336b8 pc=0x4b739e
go.elastic.co/apm.(*Tracer).loop(0xc0001c1680)
    /go/src/github.com/app/vendor/go.elastic.co/apm/tracer.go:975 +0xbef fp=0xc000533fc8 sp=0xc0005337f8 pc=0x7f33cf
go.elastic.co/apm.newTracer.func17()
    /go/src/github.com/app/vendor/go.elastic.co/apm/tracer.go:496 +0x26 fp=0xc000533fe0 sp=0xc000533fc8 pc=0x7f2486
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000533fe8 sp=0xc000533fe0 pc=0x4d7881
created by go.elastic.co/apm.newTracer
    /go/src/github.com/app/vendor/go.elastic.co/apm/tracer.go:496 +0xae5

goroutine 19 [select]:
runtime.gopark(0xc00052ff78?, 0x2?, 0xa5?, 0xef?, 0xc00052fefc?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00052fd68 sp=0xc00052fd48 pc=0x4a70f6
runtime.selectgo(0xc00052ff78, 0xc00052fef8, 0x3fb999999999999a?, 0x0, 0xc00053e000?, 0x1)
    /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00052fea8 sp=0xc00052fd68 pc=0x4b739e
go.elastic.co/apm.(*Tracer).loop.func2()
    /go/src/github.com/app/vendor/go.elastic.co/apm/tracer.go:880 +0x2ad fp=0xc00052ffe0 sp=0xc00052fea8 pc=0x7f5f6d
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00052ffe8 sp=0xc00052ffe0 pc=0x4d7881
created by go.elastic.co/apm.(*Tracer).loop
    /go/src/github.com/app/vendor/go.elastic.co/apm/tracer.go:875 +0x3fd

goroutine 20 [GC worker (idle), 3 minutes]:
runtime.gopark(0x46c916c303ebaf?, 0x3?, 0x7?, 0x8?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000068f50 sp=0xc000068f30 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000068fe0 sp=0xc000068f50 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000068fe8 sp=0xc000068fe0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 6 [GC worker (idle)]:
runtime.gopark(0x46c9186e653568?, 0x3?, 0x38?, 0x39?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00006e750 sp=0xc00006e730 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00006e7e0 sp=0xc00006e750 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00006e7e8 sp=0xc00006e7e0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 34 [GC worker (idle)]:
runtime.gopark(0x1394e40?, 0x1?, 0xcf?, 0xa3?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00011c750 sp=0xc00011c730 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00011c7e0 sp=0xc00011c750 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00011c7e8 sp=0xc00011c7e0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 21 [GC worker (idle)]:
runtime.gopark(0x1394e40?, 0x1?, 0x54?, 0xf?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000069750 sp=0xc000069730 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0000697e0 sp=0xc000069750 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000697e8 sp=0xc0000697e0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 7 [GC worker (idle)]:
runtime.gopark(0x46c9186e654a00?, 0x3?, 0xb?, 0x46?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00006ef50 sp=0xc00006ef30 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00006efe0 sp=0xc00006ef50 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00006efe8 sp=0xc00006efe0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 22 [GC worker (idle)]:
runtime.gopark(0x46c9186e6c8d41?, 0x3?, 0xc0?, 0x3b?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000069f50 sp=0xc000069f30 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000069fe0 sp=0xc000069f50 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000069fe8 sp=0xc000069fe0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 35 [GC worker (idle)]:
runtime.gopark(0x46c9186e7057e3?, 0x1?, 0xc9?, 0xed?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00011cf50 sp=0xc00011cf30 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00011cfe0 sp=0xc00011cf50 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00011cfe8 sp=0xc00011cfe0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 36 [GC worker (idle)]:
runtime.gopark(0x46c9186e6a6ffe?, 0x3?, 0xca?, 0xa2?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00011d750 sp=0xc00011d730 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00011d7e0 sp=0xc00011d750 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00011d7e8 sp=0xc00011d7e0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 37 [GC worker (idle)]:
runtime.gopark(0x46c9186e65beb9?, 0x3?, 0x57?, 0xf4?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00011df50 sp=0xc00011df30 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00011dfe0 sp=0xc00011df50 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00011dfe8 sp=0xc00011dfe0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 8 [GC worker (idle)]:
runtime.gopark(0x46c9186e69cda9?, 0x3?, 0x8b?, 0x47?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00006f750 sp=0xc00006f730 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00006f7e0 sp=0xc00006f750 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00006f7e8 sp=0xc00006f7e0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 23 [GC worker (idle)]:
runtime.gopark(0x46c9186e63ddbd?, 0x3?, 0xcc?, 0x22?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00006a750 sp=0xc00006a730 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00006a7e0 sp=0xc00006a750 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00006a7e8 sp=0xc00006a7e0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 38 [GC worker (idle)]:
runtime.gopark(0x46c916c306ddbc?, 0x3?, 0xda?, 0x73?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00011e750 sp=0xc00011e730 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00011e7e0 sp=0xc00011e750 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00011e7e8 sp=0xc00011e7e0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 9 [GC worker (idle)]:
runtime.gopark(0x46c9186e66060b?, 0x3?, 0x7b?, 0xe8?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00006ff50 sp=0xc00006ff30 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00006ffe0 sp=0xc00006ff50 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00006ffe8 sp=0xc00006ffe0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 24 [GC worker (idle), 4 minutes]:
runtime.gopark(0x46c8d5ce74db84?, 0x3?, 0x20?, 0x84?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00006af50 sp=0xc00006af30 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00006afe0 sp=0xc00006af50 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00006afe8 sp=0xc00006afe0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 39 [GC worker (idle)]:
runtime.gopark(0x46c9186e6f2daf?, 0x1?, 0x6a?, 0xde?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00011ef50 sp=0xc00011ef30 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00011efe0 sp=0xc00011ef50 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00011efe8 sp=0xc00011efe0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 10 [GC worker (idle)]:
runtime.gopark(0x1394e40?, 0x1?, 0x3?, 0xf5?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000118750 sp=0xc000118730 pc=0x4a70f6
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0001187e0 sp=0xc000118750 pc=0x488bf1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001187e8 sp=0xc0001187e0 pc=0x4d7881
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 11 [select]:
runtime.gopark(0xc000531de8?, 0x3?, 0x82?, 0x48?, 0xc000531c5a?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000531a90 sp=0xc000531a70 pc=0x4a70f6
runtime.selectgo(0xc000531de8, 0xc000531c54, 0xc0001c8900?, 0x1, 0x1?, 0x1)
    /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000531bd0 sp=0xc000531a90 pc=0x4b739e
go.elastic.co/apm/transport.(*HTTPTransport).WatchConfig.func1()
    /go/src/github.com/app/vendor/go.elastic.co/apm/transport/http.go:398 +0x1e5 fp=0xc000531fe0 sp=0xc000531bd0 pc=0x7d7ac5
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000531fe8 sp=0xc000531fe0 pc=0x4d7881
created by go.elastic.co/apm/transport.(*HTTPTransport).WatchConfig
    /go/src/github.com/app/vendor/go.elastic.co/apm/transport/http.go:390 +0x12a

goroutine 12 [select, 4320 minutes]:
runtime.gopark(0xc00011ff88?, 0x2?, 0x48?, 0xb6?, 0xc00011ff84?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00011fe10 sp=0xc00011fdf0 pc=0x4a70f6
runtime.selectgo(0xc00011ff88, 0xc00011ff80, 0x0?, 0x0, 0x0?, 0x1)
    /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00011ff50 sp=0xc00011fe10 pc=0x4b739e
database/sql.(*DB).connectionOpener(0xc0003289c0, {0xe9bfa0, 0xc000320140})
    /usr/local/go/src/database/sql/sql.go:1218 +0x8d fp=0xc00011ffb8 sp=0xc00011ff50 pc=0x8048ed
database/sql.OpenDB.func1()
    /usr/local/go/src/database/sql/sql.go:791 +0x2e fp=0xc00011ffe0 sp=0xc00011ffb8 pc=0x80348e
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00011ffe8 sp=0xc00011ffe0 pc=0x4d7881
created by database/sql.OpenDB
    /usr/local/go/src/database/sql/sql.go:791 +0x18d

goroutine 115 [select, 7 minutes]:
runtime.gopark(0xc0005a1f68?, 0x5?, 0xa5?, 0xef?, 0xc0005a1e6e?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0005a1cb0 sp=0xc0005a1c90 pc=0x4a70f6
runtime.selectgo(0xc0005a1f68, 0xc0005a1e64, 0xc0001dc1c0?, 0x0, 0xeddd2f014?, 0x1)
    /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc0005a1df0 sp=0xc0005a1cb0 pc=0x4b739e
github.com/robfig/cron/v3.(*Cron).run(0xc000142140)
    /go/src/github.com/app/vendor/github.com/robfig/cron/v3/cron.go:263 +0xab9 fp=0xc0005a1fc8 sp=0xc0005a1df0 pc=0x84d659
github.com/robfig/cron/v3.(*Cron).Start.func2()
    /go/src/github.com/app/vendor/github.com/robfig/cron/v3/cron.go:222 +0x26 fp=0xc0005a1fe0 sp=0xc0005a1fc8 pc=0x84cb06
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0005a1fe8 sp=0xc0005a1fe0 pc=0x4d7881
created by github.com/robfig/cron/v3.(*Cron).Start
    /go/src/github.com/app/vendor/github.com/robfig/cron/v3/cron.go:222 +0xca

goroutine 55 [select, 4320 minutes, locked to thread]:
runtime.gopark(0xc00011a7a8?, 0x2?, 0x0?, 0xb2?, 0xc00011a7a4?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00011a618 sp=0xc00011a5f8 pc=0x4a70f6
runtime.selectgo(0xc00011a7a8, 0xc00011a7a0, 0x0?, 0x0, 0xc000b52010?, 0x1)
    /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00011a758 sp=0xc00011a618 pc=0x4b739e
runtime.ensureSigM.func1()
    /usr/local/go/src/runtime/signal_unix.go:1000 +0x1b0 fp=0xc00011a7e0 sp=0xc00011a758 pc=0x4cf610
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00011a7e8 sp=0xc00011a7e0 pc=0x4d7881
created by runtime.ensureSigM
    /usr/local/go/src/runtime/signal_unix.go:983 +0xbd

goroutine 101 [syscall, 4320 minutes]:
runtime.notetsleepg(0x4ae345?, 0xc00011be50?)
    /usr/local/go/src/runtime/lock_futex.go:236 +0x34 fp=0xc00011bfa0 sp=0xc00011bf68 pc=0x479fd4
os/signal.signal_recv()
    /usr/local/go/src/runtime/sigqueue.go:152 +0x2f fp=0xc00011bfc0 sp=0xc00011bfa0 pc=0x4d3dcf
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:23 +0x19 fp=0xc00011bfe0 sp=0xc00011bfc0 pc=0x858359
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00011bfe8 sp=0xc00011bfe0 pc=0x4d7881
created by os/signal.Notify.func1.1
    /usr/local/go/src/os/signal/signal.go:151 +0x2a

goroutine 20434 [IO wait]:
runtime.gopark(0x4ab756?, 0xb?, 0x0?, 0x0?, 0x6?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc003817ab0 sp=0xc003817a90 pc=0x4a70f6
runtime.netpollblock(0x512a45?, 0x47122f?, 0x0?)
    /usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc003817ae8 sp=0xc003817ab0 pc=0x49fa57
internal/poll.runtime_pollWait(0x7f8e1d8ae7b8, 0x72)
    /usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc003817b08 sp=0xc003817ae8 pc=0x4d1c89
internal/poll.(*pollDesc).wait(0xc0000dc100?, 0xc1eb398000?, 0x0)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc003817b30 sp=0xc003817b08 pc=0x52abd2
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000dc100, {0xc1eb398000, 0x1000, 0x1000})
    /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc003817bc8 sp=0xc003817b30 pc=0x52b739
net.(*netFD).Read(0xc0000dc100, {0xc1eb398000?, 0x2?, 0xc0001c1180?})
    /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc003817c10 sp=0xc003817bc8 pc=0x5ef1a9
net.(*conn).Read(0xc1eb396000, {0xc1eb398000?, 0x0?, 0x0?})
    /usr/local/go/src/net/net.go:183 +0x45 fp=0xc003817c58 sp=0xc003817c10 pc=0x5facc5
net.(*TCPConn).Read(0x0?, {0xc1eb398000?, 0xc262a683c0?, 0x12?})
    <autogenerated>:1 +0x29 fp=0xc003817c88 sp=0xc003817c58 pc=0x605ae9
net/http.(*persistConn).Read(0xc2587ae000, {0xc1eb398000?, 0x4b8140?, 0xc003817ec8?})
    /usr/local/go/src/net/http/transport.go:1943 +0x4e fp=0xc003817ce8 sp=0xc003817c88 pc=0x6c6bae
bufio.(*Reader).fill(0xc000838ae0)
    /usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc003817d20 sp=0xc003817ce8 pc=0x665dff
bufio.(*Reader).Peek(0xc000838ae0, 0x1)
    /usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc003817d40 sp=0xc003817d20 pc=0x665f5d
net/http.(*persistConn).readLoop(0xc2587ae000)
    /usr/local/go/src/net/http/transport.go:2107 +0x1ac fp=0xc003817fc8 sp=0xc003817d40 pc=0x6c79cc
net/http.(*Transport).dialConn.func5()
    /usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc003817fe0 sp=0xc003817fc8 pc=0x6c6166
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc003817fe8 sp=0xc003817fe0 pc=0x4d7881
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1765 +0x16ea

goroutine 20435 [select]:
runtime.gopark(0xc003814f90?, 0x2?, 0xf8?, 0x4d?, 0xc003814f34?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc003814db0 sp=0xc003814d90 pc=0x4a70f6
runtime.selectgo(0xc003814f90, 0xc003814f30, 0xc0001ac700?, 0x0, 0xc0002fc1b0?, 0x1)
    /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc003814ef0 sp=0xc003814db0 pc=0x4b739e
net/http.(*persistConn).writeLoop(0xc2587ae000)
    /usr/local/go/src/net/http/transport.go:2410 +0xf2 fp=0xc003814fc8 sp=0xc003814ef0 pc=0x6c9692
net/http.(*Transport).dialConn.func6()
    /usr/local/go/src/net/http/transport.go:1766 +0x26 fp=0xc003814fe0 sp=0xc003814fc8 pc=0x6c6106
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc003814fe8 sp=0xc003814fe0 pc=0x4d7881
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1766 +0x173d

How to reproduce

Here are my Go producer code:

func (usecase Usecase) IngestData() *ResponseContainer {
    // Fetched data from database here ...
        listData := usecase.repository.GetData()

    // Produce data to the Kafka topic here ...
    go func() {
        for e := range usecase.kafkaProducer.Events() {
            switch ev := e.(type) {
            case *kafka.Message:
                m := ev
                if m.TopicPartition.Error != nil {
                    usecase.logger.Error("Error: ", m.TopicPartition.Error.Error())
                } else {
                    usecase.logger.Infof("Delivered message at [partition: %d, offset: offset: %v]", m.TopicPartition.Partition, m.TopicPartition.Offset)
                }
            case kafka.Error:
                usecase.logger.Errorf("Error: %+v", ev)
            default:
                usecase.logger.Error("Ignored event")
            }
        }
    }()

    for _, data := range listData {
                // The produce data function serializes and produce to the topic
        err = usecase.repository.ProduceData(tx, data)
        if err != nil {
            return BuildInternalErrorResponse(constants.ErrorCodeKafkaError, constants.ResponseCodeInternalError, constants.ResponseMessageKafkaError, err)
        }
    }

       usecase.kafkaProducer.Flush(10000)
       usecase.kafkaProducer.Close()

    return BuildSuccessResponse(map[string]string{"action": "All message produced"})
}

Checklist

Please provide the following information:

milindl commented 6 months ago

Thanks for filing this issue. Can you tell me more about the size of the data: The size of the data to be produced is around 1-3million -> This is the per-message size in bytes, right, after serialization (as it is passed to the producer)?