search

blevesearch / bleve

A modern text/numeric/geo-spatial/vector indexing library for go
Apache License 2.0
10.01k stars 678 forks source link

scorch Indexing Out of memory #1604

Open xlzheng021 opened 3 years ago

xlzheng021 commented 3 years ago

Hi,

I am using the Bleve 1.0.14 with scorch indexer, it seems the indexer is aggressively using the whole memory that I had during the final indexing merge, which caused the "out of memory" issue, is there any way that we could limit the memory usage during the index time? Or could upgrade to Bleve 2.* fix the issue?

Thanks,

The indexer config that we are using is:

index, err := bleve.NewUsing(searchdir, mapping, scorch.Name, scorch.Name, map[string]interface{}{
        "forceSegmentType":    "zap",
        "forceSegmentVersion": 12,
    })

The documentation nubmer we are dealing with is about 55k docs.

fatal error: runtime: out of memory
runtime stack:
runtime.throw(0xcf4b3f, 0x16)
    /linux_root/golang/src/runtime/panic.go:1116 +0x72
runtime.sysMap(0xc2cc000000, 0x9c000000, 0x137e038)
    /linux_root/golang/src/runtime/mem_linux.go:169 +0xc6
runtime.(*mheap).sysAlloc(0x1362a20, 0x98c00000, 0x42be57, 0x1362a28)
    /linux_root/golang/src/runtime/malloc.go:727 +0x1e5
runtime.(*mheap).grow(0x1362a20, 0x4c5c9, 0x0)
    /linux_root/golang/src/runtime/mheap.go:1344 +0x85
runtime.(*mheap).allocSpan(0x1362a20, 0x4c5c9, 0x0, 0x137e048, 0x41667b)
    /linux_root/golang/src/runtime/mheap.go:1160 +0x6b6
runtime.(*mheap).alloc.func1()
    /linux_root/golang/src/runtime/mheap.go:907 +0x65
runtime.(*mheap).alloc(0x1362a20, 0x4c5c9, 0xc0000a0100, 0x42990a)
    /linux_root/golang/src/runtime/mheap.go:901 +0x85
runtime.largeAlloc(0x98b90f68, 0x460001, 0xc2ad314000)
    /linux_root/golang/src/runtime/malloc.go:1177 +0x92
runtime.mallocgc.func1()
    /linux_root/golang/src/runtime/malloc.go:1071 +0x46
runtime.systemstack(0xc0003be000)
    /linux_root/golang/src/runtime/asm_amd64.s:370 +0x66
runtime.mstart()
    /linux_root/golang/src/runtime/proc.go:1116

goroutine 1 [running]:
runtime.systemstack_switch()
    /linux_root/golang/src/runtime/asm_amd64.s:330 fp=0xc00009f5f0 sp=0xc00009f5e8 pc=0x468a60
runtime.mallocgc(0x98b90f68, 0xc82e20, 0x1, 0xc2ad314000)
    /linux_root/golang/src/runtime/malloc.go:1070 +0x938 fp=0xc00009f690 sp=0xc00009f5f0 pc=0x40dcf8
runtime.makeslice(0xc82e20, 0x2ba296b, 0x2ba296b, 0xc2ad314000)
    /linux_root/golang/src/runtime/slice.go:98 +0x6c fp=0xc00009f6c0 sp=0xc00009f690 pc=0x44c82c
github.com/blevesearch/zap/v12.(*interim).prepareDicts(0xc12fdc92c0)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/zap/v12@v12.0.14/new.go:400 +0x496 fp=0xc00009f7b0 sp=0xc00009f6c0 pc=0x8850b6
github.com/blevesearch/zap/v12.(*interim).convert(0xc12fdc92c0, 0xc1373c83c0, 0xc12fdc92c0, 0xc30420, 0xc00009f8d0, 0xc1373c8360, 0xc00009f918, 0xc1377908f0, 0xc00009f978)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/zap/v12@v12.0.14/new.go:251 +0x2e5 fp=0xc00009f890 sp=0xc00009f7b0 pc=0x884505
github.com/blevesearch/zap/v12.(*ZapPlugin).newWithChunkMode(0x137bdd8, 0xc0144ce000, 0x84c2, 0x84c2, 0x401, 0xc00009f9b8, 0x4c3a86, 0x60c834eb, 0x1456966b, 0x2e5fb1519c2d)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/zap/v12@v12.0.14/new.go:74 +0x1ac fp=0xc00009f968 sp=0xc00009f890 pc=0x883b6c
github.com/blevesearch/zap/v12.(*ZapPlugin).New(0x137bdd8, 0xc0144ce000, 0x84c2, 0x84c2, 0xc01277df80, 0x8, 0x7800, 0x9600, 0x12)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/zap/v12@v12.0.14/new.go:49 +0x57 fp=0xc00009f9c8 sp=0xc00009f968 pc=0x883957
github.com/blevesearch/bleve/index/scorch.(*Scorch).Batch(0xc0003be000, 0xc00000c060, 0x0, 0x0)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/scorch/scorch.go:373 +0x868 fp=0xc00009fb98 sp=0xc00009f9c8 pc=0x8fb2e8
github.com/blevesearch/bleve.(*indexImpl).Batch(0xc0003983f0, 0xc00007e060, 0x0, 0x0)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index_impl.go:319 +0x8f fp=0xc00009fbe0 sp=0xc00009fb98 pc=0xa60a2f
golang.dhcp/m/com/company/helpcenter/docsearch/pkg/docindexer.DocIndex(0x7ffea2464acc, 0x1, 0x0, 0x0, 0x7ffea2464ad1, 0x5)
    /company/devel/bat/filer/batfs0806-0/Chelp_web.1677886/build/scm/help-web-mw-docsearch/main/src/main/go/pkg/docindexer/docindexer.go:68 +0x37c fp=0xc00009fd10 sp=0xc00009fbe0 pc=0xb5099c
golang.dhcp/m/com/company/helpcenter/docsearch/pkg/cli.glob..func2(0x132b020, 0xc00010ad40, 0x0, 0x4)
    /company/devel/bat/filer/batfs0806-0/Chelp_web.1677886/build/scm/help-web-mw-docsearch/main/src/main/go/pkg/cli/indexCmd.go:27 +0x1fc fp=0xc00009fd98 sp=0xc00009fd10 pc=0xb6073c
github.com/spf13/cobra.(*Command).execute(0x132b020, 0xc00010acc0, 0x4, 0x4, 0x132b020, 0xc00010acc0)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:856 +0x2c2 fp=0xc00009fe70 sp=0xc00009fd98 pc=0x5c5442
github.com/spf13/cobra.(*Command).ExecuteC(0x132b2a0, 0x0, 0x0, 0x0)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:960 +0x375 fp=0xc00009ff48 sp=0xc00009fe70 pc=0x5c6175
github.com/spf13/cobra.(*Command).Execute(...)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/spf13/cobra@v1.1.3/command.go:897
golang.dhcp/m/com/company/helpcenter/docsearch/pkg/cli.Execute()
    /company/devel/bat/filer/batfs0806-0/Chelp_web.1677886/build/scm/help-web-mw-docsearch/main/src/main/go/pkg/cli/rootCmd.go:21 +0x2d fp=0xc00009ff78 sp=0xc00009ff48 pc=0xb5fd4d
main.main()
    /company/devel/bat/filer/batfs0806-0/Chelp_web.1677886/build/scm/help-web-mw-docsearch/main/src/main/go/cmd/mw-docsearch/main.go:9 +0x25 fp=0xc00009ff88 sp=0xc00009ff78 pc=0xb60b45
runtime.main()
    /linux_root/golang/src/runtime/proc.go:204 +0x209 fp=0xc00009ffe0 sp=0xc00009ff88 pc=0x4389c9
runtime.goexit()
    /linux_root/golang/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc00009ffe8 sp=0xc00009ffe0 pc=0x46a6a1

goroutine 18 [select, 3 minutes]:
github.com/blevesearch/bleve/index.AnalysisWorker(0xc0003587e0, 0xc000358840)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/analysis.go:102 +0x112
created by github.com/blevesearch/bleve/index.NewAnalysisQueue
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/analysis.go:94 +0xc8

goroutine 19 [select, 3 minutes]:
github.com/blevesearch/bleve/index.AnalysisWorker(0xc0003587e0, 0xc000358840)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/analysis.go:102 +0x112
created by github.com/blevesearch/bleve/index.NewAnalysisQueue
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/analysis.go:94 +0xc8

goroutine 20 [select, 3 minutes]:
github.com/blevesearch/bleve/index.AnalysisWorker(0xc0003587e0, 0xc000358840)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/analysis.go:102 +0x112
created by github.com/blevesearch/bleve/index.NewAnalysisQueue
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/analysis.go:94 +0xc8

goroutine 21 [select, 3 minutes]:
github.com/blevesearch/bleve/index.AnalysisWorker(0xc0003587e0, 0xc000358840)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/analysis.go:102 +0x112
created by github.com/blevesearch/bleve/index.NewAnalysisQueue
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/analysis.go:94 +0xc8

goroutine 22 [select, 5 minutes]:
github.com/blevesearch/bleve/index/scorch.(*Scorch).introducerLoop(0xc0003be000)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/scorch/introducer.go:54 +0x2df
created by github.com/blevesearch/bleve/index/scorch.(*Scorch).Open
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/scorch/scorch.go:183 +0x90

goroutine 23 [select, 5 minutes]:
github.com/blevesearch/bleve/index/scorch.(*Scorch).persisterLoop(0xc0003be000)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/scorch/persister.go:216 +0xd9d
created by github.com/blevesearch/bleve/index/scorch.(*Scorch).Open
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/scorch/scorch.go:187 +0xf3

goroutine 24 [select, 5 minutes]:
github.com/blevesearch/bleve/index/scorch.(*Scorch).mergerLoop(0xc0003be000)
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/scorch/merge.go:123 +0x78f
created by github.com/blevesearch/bleve/index/scorch.(*Scorch).Open
    /tmp/Chelp_web_1677886_195627/help-web-mw-docsearch/main/mw-docsearch/go/pkg/mod/github.com/blevesearch/bleve@v1.0.14/index/scorch/scorch.go:189 +0x12c
mschoch commented 3 years ago

So, I'd like to first ask more information about the environment. Specifically, how much ram is available to the process? And is this a 64-bit system?

It would be highly unusual for an index with so few documents to run out of memory, as we regularly test with index built on 5 million documents. That said, memory usage does vary by the type of index you are building. Do you have many fields? Or have large values in those fields? The reason I'm asking is that sometimes people end up indexing large values accidentally (usually some binary data inside a string that gets index as a single, unusually large token)

You asked about Bleve v2, and certainly I do recommend upgrading, but we have not fixed any issues that are obviously related to this behavior.

As for next steps, there is no specific setting to control memory used by the indexing. At Couchbase they will throttle the rate of indexing if the memory usage is high, but this done by the application, not bleve itself. And if the memory is consumed during a merge as you suggest, it likely would not help this case. So, that leaves us with either you being able to share some program and data that reproduce the problem, or digging in yourself to see where and why the memory is used.