rcrowley / go-metrics

Go port of Coda Hale's Metrics library
Other
3.43k stars 493 forks source link

lock on global rand causes latency spike #278

Open cch123 opened 3 years ago

cch123 commented 3 years ago
// Update samples a new value.
func (s *UniformSample) Update(v int64) {
    s.mutex.Lock()
    defer s.mutex.Unlock()
    s.count++
    if len(s.values) < s.reservoirSize {
        s.values = append(s.values, v)
    } else {
        r := rand.Int63n(s.count).  // -------------------> here
        if r < int64(len(s.values)) {
            s.values[int(r)] = v
        }
    }
}

When latency increase, the goroutine stack:

10760 @ 0x42f81f 0x4401d9 0x4401af 0x43ff4d 0x474df9 0x95709a 0x952d0d 0x17466fc 0x17462c4 0x174c8a8 0x174c88a 0x174e308 0x1755b78 0x1749432 0x17588e8 0xf90a54 0xc9670d 0x45d061
#   0x43ff4c    sync.runtime_SemacquireMutex+0x3c               /Users/xargin/sdk/go1.12.17/src/runtime/sema.go:71
#   0x474df8    sync.(*Mutex).Lock+0x108                    /Users/xargin/sdk/go1.12.17/src/sync/mutex.go:134
#   0x957099    github.com/rcrowley/go-metrics.(*UniformSample).Update+0x39 /Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/sample.go:508
#   0x952d0c    github.com/rcrowley/go-metrics.(*StandardHistogram).Update+0x3c /Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/histogram.go:199
#   0x17466fb   mosn.io/mosn/pkg/proxy.(*downStream).requestMetrics+0x2db   /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:257
#   0x17462c3   mosn.io/mosn/pkg/proxy.(*downStream).cleanStream+0x153      /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:218
#   0x174c8a7   mosn.io/mosn/pkg/proxy.(*downStream).endStream+0x137        /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:179
#   0x174c889   mosn.io/mosn/pkg/proxy.(*downStream).appendData+0x119       /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1001
#   0x174e307   mosn.io/mosn/pkg/proxy.(*downStream).onUpstreamData+0x37    /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1155
#   0x1755b77   mosn.io/mosn/pkg/proxy.(*upstreamRequest).receiveData+0x67  /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/upstream.go:138
#   0x1749431   mosn.io/mosn/pkg/proxy.(*downStream).receive+0x22a1     /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:586
#   0x17588e7   mosn.io/mosn/pkg/proxy.(*downStream).OnReceive.func1+0xd7   /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:366
#   0xf90a53    mosn.io/mosn/pkg/sync.(*workerPool).ScheduleAuto.func1+0x23 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/sync/workerpool.go:156
#   0xc9670c    mosn.io/pkg/utils.GoWithRecover.func1+0x4c          /Users/xargin/mosn/vendor/mosn.io/pkg/utils/goroutine.go:51

1 @ 0x42f81f 0x4401d9 0x4401af 0x43ff4d 0x474df9 0x504a3d 0x5040ba 0x5040ac 0x957187 0x957172 0x952d0d 0x17465d0 0x17462c4 0x174c8a8 0x174c88a 0x174e308 0x1755b78 0x1749432 0x17588e8 0xf90a54 0xc9670d 0x45d061
#   0x43ff4c    sync.runtime_SemacquireMutex+0x3c               /Users/xargin/sdk/go1.12.17/src/runtime/sema.go:71
#   0x474df8    sync.(*Mutex).Lock+0x108                    /Users/xargin/sdk/go1.12.17/src/sync/mutex.go:134
#   0x504a3c    math/rand.(*lockedSource).Int63+0x2c                /Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:380
#   0x5040b9    math/rand.(*Rand).Int63+0x69                    /Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:85
#   0x5040ab    math/rand.(*Rand).Int63n+0x5b                   /Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:117
#   0x957186    math/rand.Int63n+0x126                      /Users/xargin/sdk/go1.12.17/src/math/rand/rand.go:319
#   0x957171    github.com/rcrowley/go-metrics.(*UniformSample).Update+0x111    /Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/sample.go:514
#   0x952d0c    github.com/rcrowley/go-metrics.(*StandardHistogram).Update+0x3c /Users/xargin/mosn/vendor/github.com/rcrowley/go-metrics/histogram.go:199
#   0x17465cf   mosn.io/mosn/pkg/proxy.(*downStream).requestMetrics+0x1af   /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:248
#   0x17462c3   mosn.io/mosn/pkg/proxy.(*downStream).cleanStream+0x153      /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:218
#   0x174c8a7   mosn.io/mosn/pkg/proxy.(*downStream).endStream+0x137        /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:179
#   0x174c889   mosn.io/mosn/pkg/proxy.(*downStream).appendData+0x119       /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1001
#   0x174e307   mosn.io/mosn/pkg/proxy.(*downStream).onUpstreamData+0x37    /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:1155
#   0x1755b77   mosn.io/mosn/pkg/proxy.(*upstreamRequest).receiveData+0x67  /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/upstream.go:138
#   0x1749431   mosn.io/mosn/pkg/proxy.(*downStream).receive+0x22a1     /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:586
#   0x17588e7   mosn.io/mosn/pkg/proxy.(*downStream).OnReceive.func1+0xd7   /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/proxy/downstream.go:366
#   0xf90a53    mosn.io/mosn/pkg/sync.(*workerPool).ScheduleAuto.func1+0x23 /Users/xargin/mosn/vendor/mosn.io/mosn/pkg/sync/workerpool.go:156
#   0xc9670c    mosn.io/pkg/utils.GoWithRecover.func1+0x4c          /Users/xargin/mosn/vendor/mosn.io/pkg/utils/goroutine.go:51