search

weaviate / contextionary

Weaviate's own language vectorizer, which allows for semantic context-based searches in Weaviate
https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-contextionary
BSD 3-Clause "New" or "Revised" License
14 stars 2 forks source link

Bug: Concurrent read/write error leads to crash #18

Closed etiennedi closed 4 years ago

etiennedi commented 4 years ago

Issue

Crashes due to possible data race

Logs

fatal error: concurrent map read and map write
goroutine 70 [running]:
runtime.throw(0xc370e8, 0x21)
    /usr/local/go/src/runtime/panic.go:617 +0x72 fp=0xc0001b12c0 sp=0xc0001b1290 pc=0x42e382
runtime.mapaccess2_faststr(0xb49640, 0xc0004ac480, 0xc00022b2c0, 0x1e, 0xc00022b2c0, 0xc0001b1430)
    /usr/local/go/src/runtime/map_faststr.go:116 +0x4a9 fp=0xc0001b1330 sp=0xc0001b12c0 pc=0x414a49
github.com/semi-technologies/contextionary/extensions.(*LookerUpper).Lookup(0xc0003e0200, 0xc00022b2c0, 0x1e, 0x3, 0x20300000000000, 0xc000432017)
    /app/extensions/looker_upper.go:29 +0xa4 fp=0xc0001b13c8 sp=0xc0001b1330 pc=0x8d5c34
main.(*Vectorizer).vectorForWord(0xc0001b8360, 0xc00022b2c0, 0x1e, 0xc1fdd2, 0x1, 0xc00022b2c0)
    /app/server/corpus_vectorizer.go:158 +0x52 fp=0xc0001b1440 sp=0xc0001b13c8 pc=0xa03de2
main.(*Vectorizer).vectorsAndOccurrences(0xc0001b8360, 0xc0000ea000, 0x53, 0x53, 0xaa, 0xaf, 0xb0, 0xb4, 0xb5, 0xb7, ...)
    /app/server/corpus_vectorizer.go:122 +0x164 fp=0xc0001b1828 sp=0xc0001b1440 pc=0xa035f4
main.(*Vectorizer).vectorForWords(0xc0001b8360, 0xc0000ea000, 0x53, 0x53, 0xc0000ea000, 0x53, 0x53)
    /app/server/corpus_vectorizer.go:88 +0x67 fp=0xc0001b18b8 sp=0xc0001b1828 pc=0xa032e7
main.(*Vectorizer).vectorForWordOrWords(0xc0001b8360, 0xc0000ea000, 0x53, 0x53, 0x53, 0x53, 0x8d6700)
    /app/server/corpus_vectorizer.go:76 +0xa8 fp=0xc0001b1900 sp=0xc0001b18b8 pc=0xa03238
main.(*Vectorizer).Corpi(0xc0001b8360, 0xc0000207f0, 0x1, 0x1, 0x235, 0x3f800000, 0x0)
    /app/server/corpus_vectorizer.go:51 +0x251 fp=0xc0001b19e0 sp=0xc0001b1900 pc=0xa02f31
github.com/semi-technologies/contextionary/extensions.(*Storer).Put(0xc0003e0220, 0xd14f00, 0xc0003b83f0, 0xc00022b2a0, 0x11, 0xc000432000, 0x235, 0x3f800000, 0xc00034e000, 0xd1ae00)
    /app/extensions/storer.go:35 +0x1dc fp=0xc0001b1b10 sp=0xc0001b19e0 pc=0x8d613c
main.(*server).AddExtension(0xc0001118f0, 0xd14f00, 0xc0003b83f0, 0xc0003b84b0, 0xc0001118f0, 0xc0003b83f0, 0xc00004fbd0)
    /app/server/api.go:14 +0x7a fp=0xc0001b1b70 sp=0xc0001b1b10 pc=0xa001da
github.com/semi-technologies/contextionary/contextionary._Contextionary_AddExtension_Handler(0xbfc8c0, 0xc0001118f0, 0xd14f00, 0xc0003b83f0, 0xc00034e0a0, 0x0, 0xd14f00, 0xc0003b83f0, 0xc0003a0500, 0x250)
    /app/contextionary/contextionary.pb.go:788 +0x23e fp=0xc0001b1be0 sp=0xc0001b1b70 pc=0x806bbe
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0004dc900, 0xd1ad40, 0xc00047af00, 0xc00019c300, 0xc0004ac720, 0x11f7400, 0x0, 0x0, 0x0)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/server.go:971 +0x470 fp=0xc0001b1e78 sp=0xc0001b1be0 pc=0x7edbd0
google.golang.org/grpc.(*Server).handleStream(0xc0004dc900, 0xd1ad40, 0xc00047af00, 0xc00019c300, 0x0)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/server.go:1250 +0xd25 fp=0xc0001b1f80 sp=0xc0001b1e78 pc=0x7f1a45
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0003d0280, 0xc0004dc900, 0xd1ad40, 0xc00047af00, 0xc00019c300)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/server.go:690 +0x9f fp=0xc0001b1fb8 sp=0xc0001b1f80 pc=0x7fd24f
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc0001b1fc0 sp=0xc0001b1fb8 pc=0x459ff1
created by google.golang.org/grpc.(*Server).serveStreams.func1
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/server.go:688 +0xa1
goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7fb85545c010, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc000418398, 0x72, 0x0, 0x0, 0xc23b95)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000418380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1ba
net.(*netFD).accept(0xc000418380, 0xc0000a0001, 0x0, 0x0)
    /usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc000010058, 0xc00049ddf0, 0xc00049ddf8, 0x18)
    /usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc000010058, 0xc553a8, 0xc0004dc900, 0xd1a6e0, 0xc0000a0048)
    /usr/local/go/src/net/tcpsock.go:260 +0x48
google.golang.org/grpc.(*Server).Serve(0xc0004dc900, 0xd12e40, 0xc000010058, 0x0, 0x0)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/server.go:561 +0x1e9
main.main()
    /app/server/server.go:32 +0x2c8
goroutine 23 [select]:
github.com/coreos/etcd/clientv3.(*healthBalancer).updateNotifyLoop(0xc00019c100)
    /go/pkg/mod/github.com/coreos/etcd@v3.3.11+incompatible/clientv3/health_balancer.go:339 +0x33f
created by github.com/coreos/etcd/clientv3.newHealthBalancer
    /go/pkg/mod/github.com/coreos/etcd@v3.3.11+incompatible/clientv3/health_balancer.go:128 +0x2ed
goroutine 24 [select]:
github.com/coreos/etcd/clientv3.(*healthBalancer).updateUnhealthy(0xc00019c100)
    /go/pkg/mod/github.com/coreos/etcd@v3.3.11+incompatible/clientv3/health_balancer.go:236 +0xcf
github.com/coreos/etcd/clientv3.newHealthBalancer.func1(0xc00019c100)
    /go/pkg/mod/github.com/coreos/etcd@v3.3.11+incompatible/clientv3/health_balancer.go:132 +0x5a
created by github.com/coreos/etcd/clientv3.newHealthBalancer
    /go/pkg/mod/github.com/coreos/etcd@v3.3.11+incompatible/clientv3/health_balancer.go:130 +0x32d
goroutine 25 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0000a4e00)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/balancer_conn_wrappers.go:122 +0x110
created by google.golang.org/grpc.newCCBalancerWrapper
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/balancer_conn_wrappers.go:113 +0x14f
goroutine 26 [chan receive]:
google.golang.org/grpc.(*balancerWrapper).lbWatcher(0xc000111880)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/balancer_v1_wrapper.go:122 +0xb6
created by google.golang.org/grpc.(*balancerWrapperBuilder).Build
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/balancer_v1_wrapper.go:60 +0x29d
goroutine 50 [select]:
github.com/coreos/etcd/clientv3.(*watchGrpcStream).serveSubstream(0xc0000948f0, 0xc000248160, 0xc0004d45a0)
    /go/pkg/mod/github.com/coreos/etcd@v3.3.11+incompatible/clientv3/watch.go:611 +0x2a0
created by github.com/coreos/etcd/clientv3.(*watchGrpcStream).run
    /go/pkg/mod/github.com/coreos/etcd@v3.3.11+incompatible/clientv3/watch.go:452 +0xf6f
goroutine 35 [chan receive]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc0004d6000)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/clientconn.go:1066 +0x9bb
created by google.golang.org/grpc.(*addrConn).connect
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/clientconn.go:684 +0xb6
goroutine 7 [IO wait]:
internal/poll.runtime_pollWait(0x7fb85545bf40, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc000022698, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000022680, 0xc000122000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc000022680, 0xc000122000, 0x8000, 0x8000, 0x0, 0x800010601, 0x0)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000010008, 0xc000122000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc0001b8180, 0xc0001a2038, 0x9, 0x9, 0xc0004bc700, 0x7fb859eb9460, 0xc000001c80)
    /usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0xd04d60, 0xc0001b8180, 0xc0001a2038, 0x9, 0x9, 0x9, 0x7281d5, 0xc0000262cc, 0xc0004eee38)
    /usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0001a2038, 0x9, 0x9, 0xd04d60, 0xc0001b8180, 0x0, 0xc000000000, 0xd355c645d, 0x1242aa0)
    /go/pkg/mod/golang.org/x/net@v0.0.0-20181220203305-927f97764cc3/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001a2000, 0xc0000262c0, 0xc0000262c0, 0x0, 0x0)
    /go/pkg/mod/golang.org/x/net@v0.0.0-20181220203305-927f97764cc3/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0001a4000)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/internal/transport/http2_client.go:1223 +0x168
created by google.golang.org/grpc/internal/transport.newHTTP2Client
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/internal/transport/http2_client.go:286 +0xd15
goroutine 8 [runnable]:
runtime.Gosched(...)
    /usr/local/go/src/runtime/proc.go:266
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0001b81e0, 0x0, 0x0)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/internal/transport/controlbuf.go:471 +0x188
google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc0001a4000)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/internal/transport/http2_client.go:330 +0x7b
created by google.golang.org/grpc/internal/transport.newHTTP2Client
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/internal/transport/http2_client.go:328 +0xeb2
goroutine 9 [chan receive]:
main.(*server).watchForSchemaChanges(0xc0001118f0)
    /app/server/contextionary.go:99 +0x169
created by main.(*server).init
    /app/server/contextionary.go:37 +0x1bc
goroutine 10 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000352000, 0x8, 0xc000416160)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/internal/transport/transport.go:267 +0xcc
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/internal/transport/transport.go:278
google.golang.org/grpc.(*csAttempt).recvMsg(0xc000416160, 0xbe39c0, 0xc0004ac210, 0x0, 0xc0001b7200, 0x0)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/stream.go:847 +0x70a
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc000416160, 0x0, 0x0)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/stream.go:712 +0x46
google.golang.org/grpc.(*clientStream).withRetry(0xc0000b4240, 0xc0001b7208, 0xc0001b71f8, 0xc0004b62a0, 0x0)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/stream.go:568 +0x7c
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000b4240, 0xbe39c0, 0xc0004ac210, 0x0, 0x0)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/stream.go:711 +0x100
google.golang.org/grpc.invoke(0xd14e80, 0xc00009c010, 0xc2db8f, 0x16, 0xbfed80, 0xc0001362a0, 0xbe39c0, 0xc0004ac210, 0xc0001a0300, 0x12426c0, ...)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/call.go:73 +0x141
google.golang.org/grpc.(*ClientConn).Invoke(0xc0001a0300, 0xd14e80, 0xc00009c010, 0xc2db8f, 0x16, 0xbfed80, 0xc0001362a0, 0xbe39c0, 0xc0004ac210, 0x12426c0, ...)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/call.go:37 +0x1b4
google.golang.org/grpc.Invoke(...)
    /go/pkg/mod/google.golang.org/grpc@v1.19.0/call.go:60
github.com/coreos/etcd/etcdserver/etcdserverpb.(*kVClient).Range(0xc0000a16c8, 0xd14e80, 0xc00009c010, 0xc0001362a0, 0x12426c0, 0x3, 0x3, 0xc0001b7478, 0x9d0f5d, 0xc00019c180)
    /go/pkg/mod/github.com/coreos/etcd@v3.3.11+incompatible/etcdserver...