search

bosun-monitor / bosun

Time Series Alerting Framework
http://bosun.org
MIT License
3.4k stars 495 forks source link

Unrecovered Panic in Bosun #2054

Closed kylebrandt closed 7 years ago

kylebrandt commented 7 years ago

Maybe something with the template error handling changes?

        panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0xdba7f6]

goroutine 7243089 [running]:
text/template.errRecover(0xc476fa1798)
        /usr/local/go/src/text/template/exec.go:140 +0x1ec
panic(0xf5fac0, 0x191dc20)
        /usr/local/go/src/runtime/panic.go:489 +0x2cf
bosun.org/vendor/github.com/jmoiron/jsonq.(*JsonQuery).Array(0x0, 0xc46b5698f0, 0x1, 0x1, 0x4b2e71, 0xefa580, 0xc476fa0930, 0x410e62, 0xc499364730)
        /home/teamcity/buildAgent/work/634de9a12c11d0af/src/bosun.org/vendor/github.com/jmoiron/jsonq/jsonq.go:136 +0x26
bosun.org/vendor/github.com/jmoiron/jsonq.(*JsonQuery).ArrayOfObjects(0x0, 0xc46b5698f0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
        /home/teamcity/buildAgent/work/634de9a12c11d0af/src/bosun.org/vendor/github.com/jmoiron/jsonq/jsonq.go:218 +0x67
reflect.Value.call(0x10220e0, 0xc4b1add498, 0x1693, 0x10cd23b, 0x4, 0xc46f73f480, 0x1, 0x1, 0x18fbd40, 0xf6a1c0, ...)
        /usr/local/go/src/reflect/value.go:434 +0x91f
reflect.Value.Call(0x10220e0, 0xc4b1add498, 0x1693, 0xc46f73f480, 0x1, 0x1, 0x18f5ca0, 0xc452115f80, 0xefa580)
        /usr/local/go/src/reflect/value.go:302 +0xa4
text/template.(*state).evalCall(0xc476fa1718, 0x109c4a0, 0xc47455a4e0, 0x16, 0x10220e0, 0xc4b1add498, 0x1693, 0x18f5dc0, 0xc448833800, 0xc4a7b4be04, ...)
        /usr/local/go/src/text/template/exec.go:668 +0x51f
text/template.(*state).evalField(0xc476fa1718, 0x109c4a0, 0xc47455a4e0, 0x16, 0xc4a7b4be04, 0xe, 0x18f5dc0, 0xc448833800, 0xc4a7b4be40, 0x2, ...)
        /usr/local/go/src/text/template/exec.go:558 +0xe18
text/template.(*state).evalFieldChain(0xc476fa1718, 0x109c4a0, 0xc47455a4e0, 0x16, 0x10220e0, 0xc4b1add498, 0x96, 0x18f5dc0, 0xc448833800, 0xc4a7b4be30, ...)
        /usr/local/go/src/text/template/exec.go:526 +0x22b
text/template.(*state).evalVariableNode(0xc476fa1718, 0x109c4a0, 0xc47455a4e0, 0x16, 0xc448833800, 0xc4a7b4be40, 0x2, 0x2, 0x0, 0x0, ...)
        /usr/local/go/src/text/template/exec.go:514 +0x1ac
text/template.(*state).evalCommand(0xc476fa1718, 0x109c4a0, 0xc47455a4e0, 0x16, 0xc4488337a0, 0x0, 0x0, 0x0, 0x10220e0, 0xc4b1add498, ...)
        /usr/local/go/src/text/template/exec.go:438 +0x900
text/template.(*state).evalPipeline(0xc476fa1718, 0x109c4a0, 0xc47455a4e0, 0x16, 0xc44f577720, 0x0, 0x0, 0x96)
        /usr/local/go/src/text/template/exec.go:406 +0xf2
text/template.(*state).walk(0xc476fa1718, 0x109c4a0, 0xc47455a4e0, 0x16, 0x18f5760, 0xc448833830)
        /usr/local/go/src/text/template/exec.go:232 +0x55a
text/template.(*state).walk(0xc476fa1718, 0x109c4a0, 0xc47455a4e0, 0x16, 0x18f5ac0, 0xc4488335c0)
        /usr/local/go/src/text/template/exec.go:240 +0x139
text/template.(*Template).execute(0xc452115dc0, 0x18e9180, 0xc4ab258d90, 0x109c4a0, 0xc47455a4e0, 0x0, 0x0)
        /usr/local/go/src/text/template/exec.go:195 +0x20a
text/template.(*Template).Execute(0xc452115dc0, 0x18e9180, 0xc4ab258d90, 0x109c4a0, 0xc47455a4e0, 0x1, 0xc4ab258d90)
        /usr/local/go/src/text/template/exec.go:178 +0x53
html/template.(*Template).Execute(0xc448833530, 0x18e9180, 0xc4ab258d90, 0x109c4a0, 0xc47455a4e0, 0x10, 0x7efe8d3e8c18)
        /usr/local/go/src/html/template/template.go:120 +0x8c
bosun.org/cmd/bosun/sched.(*Schedule).ExecuteBody(0xc43a802200, 0xc505ebbb00, 0xc496cfaf70, 0xc48a6a08f0, 0xc451f22100, 0xc49f2cb6c0, 0x3e, 0x3e, 0x0, 0x0, ...)
        /home/teamcity/buildAgent/work/634de9a12c11d0af/src/bosun.org/cmd/bosun/sched/template.go:139 +0x1e3
bosun.org/cmd/bosun/sched.(*Schedule).executeTemplates(0xc43a802200, 0xc48a6a08f0, 0xc47455a000, 0xc4917750c0, 0xc496cfaf70, 0xc505ebbb00)
        /home/teamcity/buildAgent/work/634de9a12c11d0af/src/bosun.org/cmd/bosun/sched/check.go:289 +0x578
bosun.org/cmd/bosun/sched.(*Schedule).runHistory(0xc43a802200, 0xc505ebbb00, 0xc4716e4020, 0x15, 0xc4917750c0, 0xc483904e40, 0xefa500, 0x0, 0x0)
        /home/teamcity/buildAgent/work/634de9a12c11d0af/src/bosun.org/cmd/bosun/sched/check.go:197 +0x7eb
bosun.org/cmd/bosun/sched.(*Schedule).RunHistory(0xc43a802200, 0xc505ebbb00)
        /home/teamcity/buildAgent/work/634de9a12c11d0af/src/bosun.org/cmd/bosun/sched/check.go:87 +0x107
bosun.org/cmd/bosun/sched.(*Schedule).checkAlert(0xc43a802200, 0xc496cfaf70, 0xc471d77640)
        /home/teamcity/buildAgent/work/634de9a12c11d0af/src/bosun.org/cmd/bosun/sched/alertRunner.go:88 +0xee
bosun.org/cmd/bosun/sched.(*Schedule).runAlert(0xc43a802200, 0xc496cfaf70, 0xc48bbdf560)
        /home/teamcity/buildAgent/work/634de9a12c11d0af/src/bosun.org/cmd/bosun/sched/alertRunner.go:73 +0x178
created by bosun.org/cmd/bosun/sched.(*Schedule).Run
        /home/teamcity/buildAgent/work/634de9a12c11d0af/src/bosun.org/cmd/bosun/sched/alertRunner.go:30 +0x1c7
[root@co-tsdb01 log]#
kylebrandt commented 7 years ago

You can now check if nil, but we haven't updated the alert code to do that yet. But regardless the panic should be recovered:

        {{ template "header" . }}
        {{ $jq := .HTTPGetJSON "http://ny-tps.ds.stackexchange.com/top-ips" }}
        {{ $topIPs := $jq.ArrayOfObjects "topIps" }}
        <table>
           <tr>
               <th>IP Address</th>
               <th>Hits in the last {{$jq.Int "spanMinutes"}} minutes</th>
               <th>Country</th>
               <th>City</th>
               <th>Crawler</th>
               <th>Whois ORG (Via ARIN API)</th>
           <tr>
        {{ range $i, $ip := $topIPs }}
           {{ if lt $i 10 }}

           <tr>
kylebrandt commented 7 years ago

I thought all of template execution was wrapped in defer, but just sections of it. Going to wrap all of it (since user input) tomorrow and make sure that protects things.

kylebrandt commented 7 years ago

This type of panic can not be wrapped in a recover, see https://stackoverflow.com/questions/43212593/handling-sigsegv-with-recover

closing a dupe of the newer issue https://github.com/bosun-monitor/bosun/issues/2121