search

triggermesh / brokers

TriggerMesh broker implementations: Redis backed and Memory
Apache License 2.0
6 stars 12 forks source link

Redis: Arbitrary panics when starting the broker #139

Open odacremolbap opened 1 year ago

odacremolbap commented 1 year ago

I have seen 3 of these, no some of them informing TLS, some not. The last relevant change I remember is upgrading go-redis from v9.0.0-rc2 to v9.0.2

2023/03/30 13:34:40 Applying default configuration
{"level":"info","ts":1680176080.2827864,"logger":"fswatch","caller":"fs/filewatcher.go:65","msg":"Adding file to watch","file":"/home/pablo/go/src/github.com/triggermesh/brokers/.local/broker-config.yaml"}
{"level":"info","ts":1680176080.2831702,"logger":"ingest","caller":"ingest/ingest.go:105","msg":"Ingest Server UpdateFromConfig ..."}
{"level":"info","ts":1680176080.2832122,"logger":"subs","caller":"subscriptions/manager.go:52","msg":"Updating subscriptions configuration"}
{"level":"info","ts":1680176080.2832682,"logger":"subs","caller":"subscriptions/manager.go:93","msg":"Creating new subscription from trigger configuration","name":"trigger1","trigger":{"filters":[{"exact":{"type":"example.type"}}],"target":{"url":"http://localhost:9099","deliveryOptions":{"retry":1,"backoffPolicy":"linear","backoffDelay":"PT5S"}}}}
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x20 pc=0x83f750]

goroutine 1 [running]:
github.com/redis/go-redis/v9/internal/pool.(*ConnPool).waitTurn(0xc0004e6000, {0x0, 0x0})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/internal/pool/pool.go:284 +0x30
github.com/redis/go-redis/v9/internal/pool.(*ConnPool).Get(0xc0004e6000, {0x0, 0x0})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/internal/pool/pool.go:245 +0x45
github.com/redis/go-redis/v9.(*baseClient)._getConn(0xc0004641e0, {0x0, 0x0})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/redis.go:248 +0x38
github.com/redis/go-redis/v9.(*baseClient).getConn(0xc0004641e0, {0x0?, 0x0?})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/redis.go:236 +0x69
github.com/redis/go-redis/v9.(*baseClient).withConn(0xc0004641e0, {0x0, 0x0}, 0xc0009ae388)
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/redis.go:340 +0x54
github.com/redis/go-redis/v9.(*baseClient)._process(0xc0004641e0, {0x0, 0x0}, {0x1dbb828, 0xc00006ff80}, 0xc000100800?)
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/redis.go:382 +0xe5
github.com/redis/go-redis/v9.(*baseClient).process(0xc0004641e0, {0x0, 0x0}, {0x1dbb828, 0xc00006ff80})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/redis.go:364 +0x78
github.com/redis/go-redis/v9.(*hooksMixin).processHook(...)
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/redis.go:173
github.com/redis/go-redis/v9.(*Client).Process(0xc0009ae4b0?, {0x0?, 0x0?}, {0x1dbb828?, 0xc00006ff80})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/redis.go:639 +0x43
github.com/redis/go-redis/v9.cmdable.XGroupCreateMkStream(0xc00052c360, {0x0?, 0x0}, {0xc0004ff100, 0x10}, {0xc000a81300, 0x10}, {0x1b06c79, 0x1})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.2/commands.go:1953 +0x1f3
github.com/triggermesh/brokers/pkg/backend/impl/redis.(*redis).Subscribe(0xc00031e0c0, {0xc000a812e0, 0x8}, 0xc0000f2be0)
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/backend/impl/redis/redis.go:208 +0x1ac
github.com/triggermesh/brokers/pkg/subscriptions.(*Manager).UpdateFromConfig(0xc0008282d0, 0xc00052cc30)
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/subscriptions/manager.go:99 +0x13b3
github.com/triggermesh/brokers/pkg/config/broker/watcher.(*Watcher).update(0xc000828730, {0xc0000fa600?, 0x4b?, 0xc000542060?})
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/config/broker/watcher/watcher.go:85 +0x346
github.com/triggermesh/brokers/pkg/config/broker/watcher.(*Watcher).Start(0xc000828730, {0x1db1c70, 0xc000146540})
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/config/broker/watcher/watcher.go:63 +0xb8
github.com/triggermesh/brokers/pkg/broker.(*Instance).Start(0xc00080c1c0, {0x1db1d18, 0xc0005175c0})
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/broker/broker.go:255 +0x7a6
github.com/triggermesh/brokers/cmd/redis-broker/cmd.(*StartCmd).Run(0xc0004e09a8, 0xc0004e08c0)
        /home/pablo/go/src/github.com/triggermesh/brokers/cmd/redis-broker/cmd/start.go:33 +0x1d5
reflect.Value.call({0x18c2a80?, 0xc0004e09a8?, 0x18?}, {0x1b077f7, 0x4}, {0xc000338180, 0x1, 0x0?})
        /usr/local/go/src/reflect/value.go:584 +0x8c5
reflect.Value.Call({0x18c2a80?, 0xc0004e09a8?, 0x18cf1a0?}, {0xc000338180?, 0x1825040?, 0xc0004e0a10?})
        /usr/local/go/src/reflect/value.go:368 +0xbc
github.com/alecthomas/kong.callMethod({0x1b072a3, 0x3}, {0x18f9460?, 0xc0004e09a8?, 0x3?}, {0x18c2a80?, 0xc0004e09a8?, 0x0?}, 0x0?)
        /home/pablo/go/pkg/mod/github.com/alecthomas/kong@v0.7.1/callbacks.go:95 +0x549
github.com/alecthomas/kong.(*Context).RunNode(0xc00036b500, 0xc0004f00f0, {0xc0009aff58, 0x1, 0x1})
        /home/pablo/go/pkg/mod/github.com/alecthomas/kong@v0.7.1/context.go:755 +0x5f7
github.com/alecthomas/kong.(*Context).Run(0xc0004e08c0?, {0xc0009aff58?, 0xc000793f48?, 0x9?})
        /home/pablo/go/pkg/mod/github.com/alecthomas/kong@v0.7.1/context.go:780 +0x14e
main.main()
        /home/pablo/go/src/github.com/triggermesh/brokers/cmd/redis-broker/main.go:48 +0x345
exit status 2
odacremolbap commented 1 year ago

Not seen for some time, now we are using redis 9.0.5 which might have fixed the issue. Closing.

odacremolbap commented 1 year ago

Reopening, seen again

2023/07/04 18:27:08 Applying default observability configuration
{"level":"info","ts":1688488028.4520907,"logger":"fswatch","caller":"fs/filewatcher.go:65","msg":"Adding file to watch","file":"/home/pablo/go/src/github.com/triggermesh/brokers/.local/broker-config.yaml"}
{"level":"info","ts":1688488028.4525063,"logger":"ingest","caller":"ingest/ingest.go:129","msg":"Ingest Server UpdateFromConfig ..."}
{"level":"info","ts":1688488028.4525287,"logger":"subs","caller":"subscriptions/manager.go:55","msg":"Updating subscriptions configuration"}
{"level":"info","ts":1688488028.4525695,"logger":"subs","caller":"subscriptions/manager.go:139","msg":"Creating new subscription from trigger configuration","name":"trigger1","trigger":{"filters":[{"exact":{"type":"example.type"}}],"target":{"url":"http://localhost:9099","deliveryOptions":{"retry":1,"backoffPolicy":"linear","backoffDelay":"PT5S"}}}}
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x20 pc=0x8631b0]

goroutine 1 [running]:
github.com/redis/go-redis/v9/internal/pool.(*ConnPool).waitTurn(0xc000679830, {0x0, 0x0})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/internal/pool/pool.go:288 +0x30
github.com/redis/go-redis/v9/internal/pool.(*ConnPool).Get(0xc000679830, {0x0, 0x0})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/internal/pool/pool.go:249 +0x45
github.com/redis/go-redis/v9.(*baseClient)._getConn(0xc00065c220, {0x0, 0x0})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/redis.go:247 +0x38
github.com/redis/go-redis/v9.(*baseClient).getConn(0xc00065c220, {0x0?, 0x0?})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/redis.go:235 +0x69
github.com/redis/go-redis/v9.(*baseClient).withConn(0xc00065c220, {0x0, 0x0}, 0xc000b9e450)
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/redis.go:351 +0x54
github.com/redis/go-redis/v9.(*baseClient)._process(0xc00065c220, {0x0, 0x0}, {0x1ea57a8, 0xc000554e40}, 0xc00059711e?)
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/redis.go:393 +0xeb
github.com/redis/go-redis/v9.(*baseClient).process(0xc00065c220, {0x0, 0x0}, {0x1ea57a8, 0xc000554e40})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/redis.go:375 +0x78
github.com/redis/go-redis/v9.(*hooksMixin).processHook(...)
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/redis.go:172
github.com/redis/go-redis/v9.(*Client).Process(0xc000b9e578?, {0x0?, 0x0?}, {0x1ea57a8?, 0xc000554e40})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/redis.go:650 +0x43
github.com/redis/go-redis/v9.cmdable.XGroupCreateMkStream(0xc0000176d0, {0x0?, 0x0}, {0xc000704ee8, 0x14}, {0xc000597180, 0x10}, {0x1e79250, 0x1})
        /home/pablo/go/pkg/mod/github.com/redis/go-redis/v9@v9.0.5/commands.go:2141 +0x1f3
github.com/triggermesh/brokers/pkg/backend/impl/redis.(*redis).Subscribe(0xc0006b77a0, {0xc000597160, 0x8}, 0x4?, 0xc000813310)
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/backend/impl/redis/redis.go:222 +0x308
github.com/triggermesh/brokers/pkg/subscriptions.(*Manager).createSubscriber(0xc00097b1a0, {0xc000597160, 0x8}, {{0xc00081f540, 0x1, 0x4}, {0xc000813230, 0xc0005d04c0}, 0x0, 0x0})
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/subscriptions/manager.go:144 +0x5f1
github.com/triggermesh/brokers/pkg/subscriptions.(*Manager).UpdateFromConfig(0xc00097b1a0, 0xc0008125f0)
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/subscriptions/manager.go:74 +0x6a5
github.com/triggermesh/brokers/pkg/config/broker/watcher.(*Watcher).update(0xc00037f8b0, {0xc000be0200?, 0x4b?, 0xc0008125d0?})
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/config/broker/watcher/watcher.go:85 +0x346
github.com/triggermesh/brokers/pkg/config/broker/watcher.(*Watcher).Start(0xc00037f8b0, {0x1e9b5a0, 0xc00081a910})
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/config/broker/watcher/watcher.go:63 +0xb8
github.com/triggermesh/brokers/pkg/broker.(*Instance).Start(0xc0009dc930, {0x1e9b648, 0xc000706210})
        /home/pablo/go/src/github.com/triggermesh/brokers/pkg/broker/broker.go:286 +0x765
github.com/triggermesh/brokers/cmd/redis-broker/cmd.(*StartCmd).Run(0xc000a80338, 0xc000a80200)
        /home/pablo/go/src/github.com/triggermesh/brokers/cmd/redis-broker/cmd/start.go:33 +0x1d5
reflect.Value.call({0x19b4cc0?, 0xc000a80338?, 0x451a56?}, {0x1c037c6, 0x4}, {0xc000ba42e8, 0x1, 0x1?})
        /usr/local/go/src/reflect/value.go:586 +0xb0b
reflect.Value.Call({0x19b4cc0?, 0xc000a80338?, 0x19c18e0?}, {0xc000ba42e8?, 0x1bed6c0?, 0x40f7aa?})
        /usr/local/go/src/reflect/value.go:370 +0xbc
github.com/alecthomas/kong.callFunction({0x19b4cc0?, 0xc000a80338?, 0x56057c?}, 0x0?)
        /home/pablo/go/pkg/mod/github.com/alecthomas/kong@v0.8.0/callbacks.go:98 +0x445
github.com/alecthomas/kong.callMethod({0x1c0325e, 0x3}, {0x19ec3a0?, 0xc000a80338?, 0x3?}, {0x19b4cc0?, 0xc000a80338?, 0x0?}, 0x0?)
        /home/pablo/go/pkg/mod/github.com/alecthomas/kong@v0.8.0/callbacks.go:132 +0x76
github.com/alecthomas/kong.(*Context).RunNode(0xc000824e80, 0xc0009ae0f0, {0xc000b9ff58, 0x1, 0x1})
        /home/pablo/go/pkg/mod/github.com/alecthomas/kong@v0.8.0/context.go:762 +0x60f
github.com/alecthomas/kong.(*Context).Run(0xc000a80200?, {0xc000b9ff58?, 0xc0008a9f48?, 0x9?})
        /home/pablo/go/pkg/mod/github.com/alecthomas/kong@v0.8.0/context.go:787 +0x14e
main.main()
        /home/pablo/go/src/github.com/triggermesh/brokers/cmd/redis-broker/main.go:48 +0x305
exit status 2