xueqiu / rdr

Apache License 2.0
1.18k stars 303 forks source link

分析9GB的rdb文件时报fatal error: out of memory,本地机器内存32GB #40

Open maoburu opened 3 years ago

maoburu commented 3 years ago

如下为异常详情:

runtime: VirtualAlloc of 4194304 bytes failed with errno=1455
fatal error: out of memory

runtime stack:
runtime.throw(0x7c8633, 0xd)
        /usr/local/go/src/runtime/panic.go:617 +0x64
runtime.sysMap(0x6d800000, 0x400000, 0xd55d18)
        /usr/local/go/src/runtime/mem_windows.go:122 +0x116
runtime.(*mheap).sysAlloc(0xd45940, 0x2000, 0x2, 0x420a29)
        /usr/local/go/src/runtime/malloc.go:633 +0x162
runtime.(*mheap).grow(0xd45940, 0x1, 0x0)
        /usr/local/go/src/runtime/mheap.go:1232 +0x34
runtime.(*mheap).allocSpanLocked(0xd45940, 0x1, 0xd55d28, 0x0)
        /usr/local/go/src/runtime/mheap.go:1150 +0x364
runtime.(*mheap).alloc_m(0xd45940, 0x1, 0x11cb0013, 0x1)
        /usr/local/go/src/runtime/mheap.go:977 +0xd5
runtime.(*mheap).alloc.func1()
        /usr/local/go/src/runtime/mheap.go:1048 +0x3e
runtime.systemstack(0x0)
        /usr/local/go/src/runtime/asm_386.s:396 +0x53
runtime.mstart()
        /usr/local/go/src/runtime/proc.go:1153

goroutine 19 [running]:
runtime.systemstack_switch()
        /usr/local/go/src/runtime/asm_386.s:357 fp=0x11c2acd8 sp=0x11c2acd4 pc=0x44c150
runtime.(*mheap).alloc(0xd45940, 0x1, 0x10013, 0x1)
        /usr/local/go/src/runtime/mheap.go:1047 +0x6c fp=0x11c2acfc sp=0x11c2acd8 pc=0x42080c
runtime.(*mcentral).grow(0xd460f8, 0x0)
        /usr/local/go/src/runtime/mcentral.go:256 +0x74 fp=0x11c2ad20 sp=0x11c2acfc pc=0x4134c4
runtime.(*mcentral).cacheSpan(0xd460f8, 0x38)
        /usr/local/go/src/runtime/mcentral.go:106 +0x340 fp=0x11c2ad5c sp=0x11c2ad20 pc=0x412ff0
runtime.(*mcache).refill(0x11906bc, 0x13)
        /usr/local/go/src/runtime/mcache.go:135 +0x91 fp=0x11c2ad6c sp=0x11c2ad5c pc=0x412a71
runtime.(*mcache).nextFree(0x11906bc, 0x13, 0x11c2adb4, 0x0, 0x7e)
        /usr/local/go/src/runtime/malloc.go:786 +0x7a fp=0x11c2ad88 sp=0x11c2ad6c pc=0x40918a
runtime.mallocgc(0x80, 0x0, 0x0, 0x71d11f)
        /usr/local/go/src/runtime/malloc.go:939 +0x5c3 fp=0x11c2addc sp=0x11c2ad88 pc=0x4098f3
runtime.slicebytetostring(0x0, 0x6d6b1f00, 0x7e, 0x7e, 0x0, 0x0)
        /usr/local/go/src/runtime/string.go:102 +0x86 fp=0x11c2adf0 sp=0x11c2addc pc=0x43cc26
main.(*MemProfiler).ElemLen(0x11c50544, 0x6d6b1f00, 0x7e, 0x7e, 0xa0, 0x0)
        /Users/dongmingxin/go/src/github.com/xueqiu/rdr/memprofiler.go:162 +0x3d fp=0x11c2ae14 sp=0x11c2adf0 pc=0x71d17d
main.(*Decoder).Set(0x11c50540, 0x6d7ed920, 0x2b, 0x2b, 0x6d6b1f00, 0x7e, 0x7e, 0x0, 0x0)
        /Users/dongmingxin/go/src/github.com/xueqiu/rdr/decoder.go:59 +0x187 fp=0x11c2ae68 sp=0x11c2ae14 pc=0x7194c7
github.com/xueqiu/rdr/vendor/github.com/dongmx/rdb.(*decode).readObject(0x11c2af60, 0x6d7ed920, 0x2b, 0x2b, 0x0, 0x0, 0x0, 0x0, 0x0)
        /Users/dongmingxin/go/src/github.com/xueqiu/rdr/vendor/github.com/dongmx/rdb/decoder.go:322 +0x41d fp=0x11c2aef4 sp=0x11c2ae68 pc=0x68300d
github.com/xueqiu/rdr/vendor/github.com/dongmx/rdb.(*decode).decode(0x11c2af60, 0x1000, 0x1000)
        /Users/dongmingxin/go/src/github.com/xueqiu/rdr/vendor/github.com/dongmx/rdb/decoder.go:302 +0xf2 fp=0x11c2af44 sp=0x11c2aef4 pc=0x682772
github.com/xueqiu/rdr/vendor/github.com/dongmx/rdb.Decode(0x854f90, 0x11c47410, 0x85c540, 0x11c50540, 0x11c47410, 0x0)
        /Users/dongmingxin/go/src/github.com/xueqiu/rdr/vendor/github.com/dongmx/rdb/decoder.go:75 +0xaf fp=0x11c2afac sp=0x11c2af44 pc=0x68258f
main.decode(0x11ca8140, 0x11c50540, 0x11c58090, 0xf)
        /Users/dongmingxin/go/src/github.com/xueqiu/rdr/main.go:44 +0xe9 fp=0x11c2afe0 sp=0x11c2afac pc=0x71b0d9
runtime.goexit()
        /usr/local/go/src/runtime/asm_386.s:1321 +0x1 fp=0x11c2afe4 sp=0x11c2afe0 pc=0x44d971
created by main.dump
        /Users/dongmingxin/go/src/github.com/xueqiu/rdr/main.go:103 +0x184

goroutine 1 [running]:
        goroutine running on other thread; stack unavailable
maoburu commented 3 years ago

看了下closed的issue,本来以为是32位客户端的问题,但是更换为64位仍然失败