search

go-ego / riot

Go Open Source, Distributed, Simple and efficient Search Engine; Warning: This is V1 and beta version, because of big memory consume, and the V2 will be rewrite all code.
Apache License 2.0
6.11k stars 474 forks source link

bug: race condition on parallel search #82

Open bobheadxi opened 5 years ago

bobheadxi commented 5 years ago 
        t.Run("index "+tt.args.object.Hash, func(t *testing.T) {
            t.Parallel()

            // request index
            if err = e.Index(Document{tt.args.object, "", true}); err != nil {
                t.Errorf("wanted Index error = false, got %v", err)
            }

            // we'll be referring to this hash a few times
            var objHash = tt.args.object.Hash

            // make sure object can be found
            if !e.IsIndexed(objHash) {
                t.Errorf("wanted IsIndexed = true, got false")
            }

            // attempt search
            if res, err := e.Search(Query{
                Text:   tt.args.content,
                Hashes: []string{objHash},
            }); err != nil && len(res) > 0 {
                if res[0].Hash != objHash {
                    t.Errorf("wanted Search to find '%s', but failed", objHash)
                }
            } else {
                t.Errorf("wanted Search to find '%s', but failed", objHash)
            }

the full code is available at https://github.com/RTradeLtd/Lens/blob/engine/riot%23parallel-tests/engine/engine_parallel_test.go

the race condition appears to be in https://github.com/go-ego/gse/blob/master/hmm/hmm_seg.go#L18, where the library attempts to write to a global map.

# .... snip
fatal error: concurrent map writes
fatal error: concurrent map writes

goroutine 438 [running]:
runtime.throw(0x47ddbb0, 0x15)
    /usr/local/Cellar/go/1.11.3/libexec/src/runtime/panic.go:608 +0x72 fp=0xc012007070 sp=0xc012007040 pc=0x402e082
runtime.mapassign(0x46d6480, 0xc0000bfe60, 0x4885433, 0xc00e963cf0)
    /usr/local/Cellar/go/1.11.3/libexec/src/runtime/map.go:651 +0x545 fp=0xc0120070f8 sp=0xc012007070 pc=0x40103b5
github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse/hmm.loadDefEmit()
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse/hmm/prob_emit.go:9 +0xcc fp=0xc012007150 sp=0xc0120070f8 pc=0x4448efc
github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse/hmm.LoadModel(0x0, 0x0, 0x0)
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse/hmm/hmm_seg.go:28 +0x1b4 fp=0xc012007188 sp=0xc012007150 pc=0x4448134
github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse.(*Segmenter).LoadModel(0xc000354398, 0x0, 0x0, 0x0)
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse/segmenter.go:107 +0x3f fp=0xc0120071b0 sp=0xc012007188 pc=0x444f81f
github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse.(*Segmenter).cutDAG(0xc000354398, 0x47d6387, 0xf, 0x0, 0x0, 0x0, 0xc000da0630, 0x3, 0xc000053970)
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse/dag.go:129 +0x6a fp=0xc012007348 sp=0xc0120071b0 pc=0x444bd1a
github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse.(*Segmenter).Cut(0xc000354398, 0x47d6387, 0xf, 0xc012007490, 0x1, 0x1, 0x240, 0xc0012161e0, 0x1800)
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/vendor/github.com/go-ego/gse/segmenter.go:74 +0xe7 fp=0xc012007408 sp=0xc012007348 pc=0x444f207
github.com/RTradeLtd/Lens/vendor/github.com/go-ego/riot.(*Engine).Segment(0xc000354240, 0x47d6387, 0xf, 0xf, 0xc0003846a8, 0xc000384680)
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/vendor/github.com/go-ego/riot/engine.go:487 +0x2eb fp=0xc0120074c0 sp=0xc012007408 pc=0x445c60b
github.com/RTradeLtd/Lens/vendor/github.com/go-ego/riot.(*Engine).Tokens(0xc000354240, 0x47d6387, 0xf, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/vendor/github.com/go-ego/riot/engine.go:510 +0x2fc fp=0xc012007550 sp=0xc0120074c0 pc=0x445c92c
github.com/RTradeLtd/Lens/vendor/github.com/go-ego/riot.(*Engine).Search(0xc000354240, 0x47d6387, 0xf, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/vendor/github.com/go-ego/riot/engine.go:767 +0x9c fp=0xc012007888 sp=0xc012007550 pc=0x445e56c
github.com/RTradeLtd/Lens/engine.(*Engine).Search(0xc01179fb80, 0x47d6387, 0xf, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/engine/engine.go:283 +0x46b fp=0xc012007de0 sp=0xc012007888 pc=0x463349b
github.com/RTradeLtd/Lens/engine.TestEngine_parallel.func1(0xc000148700)
    /Users/robertlin/go/src/github.com/RTradeLtd/Lens/engine/engine_parallel_test.go:76 +0x252 fp=0xc012007fa8 sp=0xc012007de0 pc=0x46362c2
testing.tRunner(0xc000148700, 0xc01128c750)
    /usr/local/Cellar/go/1.11.3/libexec/src/testing/testing.go:827 +0xbf fp=0xc012007fd0 sp=0xc012007fa8 pc=0x40f17ff
runtime.goexit()
    /usr/local/Cellar/go/1.11.3/libexec/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc012007fd8 sp=0xc012007fd0 pc=0x405ded1
created by testing.(*T).Run
    /usr/local/Cellar/go/1.11.3/libexec/src/testing/testing.go:878 +0x353

# ......... snip