search

blugelabs / bluge

indexing library for Go
Apache License 2.0
1.9k stars 124 forks source link

Race condition in InMemory usage #41

Closed voldyman closed 3 years ago

voldyman commented 3 years ago

I encountered this multiple times while trying to load 2k documents into the in memory index.

stack trace


fatal error: concurrent map writes

goroutine 13 [running]:
runtime.throw(0x12467fc, 0x15)
    /usr/local/Cellar/go/1.15.6/libexec/src/runtime/panic.go:1116 +0x72 fp=0xc0009cfa70 sp=0xc0009cfa40 pc=0x1034112
runtime.mapassign_fast64(0x1202ec0, 0xc000074a20, 0x1430, 0xc0002f2900)
    /usr/local/Cellar/go/1.15.6/libexec/src/runtime/map_fast64.go:101 +0x33e fp=0xc0009cfab0 sp=0xc0009cfa70 pc=0x10124de
github.com/blugelabs/bluge/index.(*InMemoryDirectory).Persist(0xc00000e0c8, 0x1243273, 0x4, 0x1430, 0x29db9958, 0xc000f0c0a0, 0xc0002f2900, 0x1282400, 0xc000f0c0a0)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/directory_mem.go:70 +0xd0 fp=0xc0009cfb00 sp=0xc0009cfab0 pc=0x1153f10
github.com/blugelabs/bluge/index.(*Writer).merge(0xc000053200, 0xc000e780a0, 0xa, 0xa, 0xc000f0c050, 0xa, 0xa, 0x1430, 0xa, 0x1418e01, ...)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/merge.go:368 +0x135 fp=0xc0009cfb78 sp=0xc0009cfb00 pc=0x1158af5
github.com/blugelabs/bluge/index.(*Writer).executeMergeTask(0xc000053200, 0xc0002f2a20, 0xc000e7c220, 0xc000f0c000, 0xc000e7c0c0)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/merge.go:144 +0x58e fp=0xc0009cfd80 sp=0xc0009cfb78 pc=0x11576ee
github.com/blugelabs/bluge/index.(*Writer).planMergeAtSnapshot(0xc000053200, 0xc0002f2a20, 0xc000526300, 0xa, 0x4c4b40, 0x4024000000000000, 0xa, 0x7d0, 0x4000000000000000, 0x0, ...)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/merge.go:118 +0x32e fp=0xc0009cfe40 sp=0xc0009cfd80 pc=0x11570ae
github.com/blugelabs/bluge/index.(*Writer).mergerLoop(0xc000053200, 0xc0002f2a20, 0xc000066420)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/merge.go:56 +0x293 fp=0xc0009cffc8 sp=0xc0009cfe40 pc=0x1156a93
runtime.goexit()
    /usr/local/Cellar/go/1.15.6/libexec/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc0009cffd0 sp=0xc0009cffc8 pc=0x1067361
created by github.com/blugelabs/bluge/index.OpenWriter
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:131 +0x8cd

goroutine 1 [runnable]:
github.com/blugelabs/bluge/index.(*Writer).prepareSegment(0xc000053200, 0xc00064cbd0, 0xc000f08020, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:324 +0x345
github.com/blugelabs/bluge/index.(*Writer).Batch(0xc000053200, 0xc00042c000, 0x0, 0x0)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:272 +0x2a6
github.com/blugelabs/bluge.(*Writer).Batch(...)
    /Users/shekher/go/src/github.com/blugelabs/bluge/writer.go:63
main.(*Index).IndexDocuments(0xc0001bc3c0, 0xc00074dd50, 0x1, 0x1, 0xc0003fc000, 0x1280e40)
    /Users/shekher/workplace/shekher/src/Shekher/proj/index.go:31 +0x6b
main.createInMemIndex(0xc00010e200, 0xc0000747b0, 0xc0001dfea8)
    /Users/shekher/workplace/shekher/src/Shekher/proj/main.go:63 +0x426
main.main()
    /Users/shekher/workplace/shekher/src/Shekher/proj/main.go:15 +0x47

goroutine 7 [select]:
github.com/blugelabs/bluge/index.analysisWorker(0xc0002f28a0, 0xc0002f2900)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:549 +0xcf
github.com/blugelabs/bluge/index.OpenWriter.func1()
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:69 +0x45
created by github.com/blugelabs/bluge/index.defaultConfig.func2
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/config.go:205 +0x33

goroutine 8 [select]:
github.com/blugelabs/bluge/index.analysisWorker(0xc0002f28a0, 0xc0002f2900)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:549 +0xcf
github.com/blugelabs/bluge/index.OpenWriter.func1()
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:69 +0x45
created by github.com/blugelabs/bluge/index.defaultConfig.func2
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/config.go:205 +0x33

goroutine 9 [select]:
github.com/blugelabs/bluge/index.analysisWorker(0xc0002f28a0, 0xc0002f2900)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:549 +0xcf
github.com/blugelabs/bluge/index.OpenWriter.func1()
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:69 +0x45
created by github.com/blugelabs/bluge/index.defaultConfig.func2
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/config.go:205 +0x33

goroutine 10 [select]:
github.com/blugelabs/bluge/index.analysisWorker(0xc0002f28a0, 0xc0002f2900)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:549 +0xcf
github.com/blugelabs/bluge/index.OpenWriter.func1()
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:69 +0x45
created by github.com/blugelabs/bluge/index.defaultConfig.func2
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/config.go:205 +0x33

goroutine 11 [runnable]:
github.com/blugelabs/bluge/index.(*Writer).introducePersist(0xc000053200, 0xc000f08130, 0x2820)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/introducer.go:181 +0x69a
github.com/blugelabs/bluge/index.(*Writer).introducerLoop(0xc000053200, 0xc0002f2960, 0xc0002f29c0, 0xc0002f2a20, 0xc0000663c0, 0x2820)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/introducer.go:74 +0x2ef
created by github.com/blugelabs/bluge/index.OpenWriter
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:127 +0x7fd

goroutine 12 [select]:
github.com/blugelabs/bluge/index.(*Writer).prepareIntroducePersist(0xc000053200, 0xc0002f29c0, 0xc0003b66f0, 0x1, 0x1, 0x0, 0x0)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/persister.go:372 +0x3e5
github.com/blugelabs/bluge/index.(*Writer).persistSnapshotDirect(0xc000053200, 0xc0002f29c0, 0xc0003fc200, 0xc0003fc200, 0x5fe51000)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/persister.go:325 +0x3ba
github.com/blugelabs/bluge/index.(*Writer).persistSnapshot(0xc000053200, 0xc0002f2a20, 0xc0002f29c0, 0xc0003fc200, 0x0, 0x0)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/persister.go:233 +0x65
github.com/blugelabs/bluge/index.(*Writer).persisterLoop(0xc000053200, 0xc0002f2a20, 0xc0002f29c0, 0xc0000663c0, 0xc000066420, 0x281e)
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/persister.go:81 +0x4ed
created by github.com/blugelabs/bluge/index.OpenWriter
    /Users/shekher/go/src/github.com/blugelabs/bluge/index/writer.go:129 +0x877