search

usnistgov / ndn-dpdk

NDN-DPDK: High-Speed Named Data Networking Forwarder
https://www.nist.gov/publications/ndn-dpdk-ndn-forwarding-100-gbps-commodity-hardware
Other
131 stars 26 forks source link

fatal error: concurrent map writes #50

Closed Pesa closed 2 years ago

Pesa commented 2 years ago 
{"level":"info","ts":1638902706.2380912,"logger":"eal","msg":"vdev initialized","name":"net_memifK00000000000329c3","args":"socket-abstract=no,mac=F2:6D:65:6D:69:66,id=0,role=server,bsize=9000,rsize=12,socket=/run/ndn/ndnc-memif-147545-1638902706234705349.sock","socket":"any"}
{"level":"info","ts":1638902706.2380955,"logger":"eal","msg":"vdev uninitialized","name":"net_memifK00000000000329af"}
{"level":"info","ts":1638902706.238122,"logger":"ethdev","msg":"stopped and detached","id":9,"name":"net_memifK00000000000329af"}
{"level":"info","ts":1638902706.238125,"logger":"ethface","msg":"port opened","port":6}
fatal error: concurrent map writes

goroutine 51 [running, locked to thread]:
runtime.throw({0xaf6c7d, 0xa1e980})
        /usr/local/go/src/runtime/panic.go:1198 +0x71 fp=0xc0000d6c20 sp=0xc0000d6bf0 pc=0x4431d1
runtime.mapdelete(0xa0f380, 0xc00025aae0, 0xffff004400000000)
        /usr/local/go/src/runtime/map.go:793 +0x14c fp=0xc0000d6c88 sp=0xc0000d6c20 pc=0x41e56c
github.com/usnistgov/ndn-dpdk/iface/ethface.(*Port).Close(0xc000276070)
        /root/ndn-dpdk/iface/ethface/port.go:118 +0x399 fp=0xc0000d72d8 sp=0xc0000d6c88 pc=0x8b2579
go4.org/must.Close(...)
        /root/go/pkg/mod/go4.org@v0.0.0-20201209231011-d4a079459e60/must/must.go:25
github.com/usnistgov/ndn-dpdk/iface/ethface.New.func5({0xc00019f9e0, 0x9dfd80})
        /root/ndn-dpdk/iface/ethface/face.go:85 +0x37 fp=0xc0000d72f8 sp=0xc0000d72d8 pc=0x8b5677
github.com/usnistgov/ndn-dpdk/iface.(*face).close(0xc0005a1770)
        /root/ndn-dpdk/iface/face.go:286 +0x147 fp=0xc0000d7368 sp=0xc0000d72f8 pc=0x7d9987
github.com/usnistgov/ndn-dpdk/iface.(*face).Close.func1()
        /root/ndn-dpdk/iface/face.go:269 +0x26 fp=0xc0000d7388 sp=0xc0000d7368 pc=0x7d9806
runtime.call16(0xc0004b4660, 0xc00000e978, 0x0, 0x0, 0x0, 0x0, 0xc0000d78b8)
        /usr/local/go/src/runtime/asm_amd64.s:625 +0x49 fp=0xc0000d73a8 sp=0xc0000d7388 pc=0x471729
runtime.reflectcall(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        <autogenerated>:1 +0x3c fp=0xc0000d73e8 sp=0xc0000d73a8 pc=0x475a9c
reflect.Value.call({0x9d37a0, 0xc00000e978, 0xc0000d7a10}, {0xad98f2, 0x4}, {0x0, 0x0, 0xc0000d7a48})
        /usr/local/go/src/reflect/value.go:543 +0x814 fp=0xc0000d79d8 sp=0xc0000d73e8 pc=0x4bb414
reflect.Value.Call({0x9d37a0, 0xc00000e978, 0xc000348ff0}, {0x0, 0x0, 0x0})
        /usr/local/go/src/reflect/value.go:339 +0xc5 fp=0xc0000d7a50 sp=0xc0000d79d8 pc=0x4ba9c5
github.com/usnistgov/ndn-dpdk/core/cptr.Call.func1()
        /root/ndn-dpdk/core/cptr/function.go:132 +0xc5 fp=0xc0000d7ab8 sp=0xc0000d7a50 pc=0x79df85
github.com/usnistgov/ndn-dpdk/core/cptr.ZeroFunctionType.Void.func1()
        /root/ndn-dpdk/core/cptr/function.go:108 +0x1b fp=0xc0000d7ac8 sp=0xc0000d7ab8 pc=0x79dd7b
github.com/usnistgov/ndn-dpdk/core/cptr.go_functionGo0_once(0xc0004a61a0)
        /root/ndn-dpdk/core/cptr/function.go:182 +0x8d fp=0xc0000d7b18 sp=0xc0000d7ac8 pc=0x79e24d
_cgoexp_ab3994a58111_go_functionGo0_once(0x7fbde9ff67f4)
        _cgo_gotypes.go:409 +0x25 fp=0xc0000d7b30 sp=0xc0000d7b18 pc=0x79e5a5
runtime.cgocallbackg1(0x79e580, 0xc0000d7ce8, 0x0)
        /usr/local/go/src/runtime/cgocall.go:306 +0x29a fp=0xc0000d7c00 sp=0xc0000d7b30 pc=0x41311a
runtime.cgocallbackg(0xc0004a61a0, 0x300000002, 0xc0004a61a0)
        /usr/local/go/src/runtime/cgocall.go:232 +0x109 fp=0xc0000d7c90 sp=0xc0000d7c00 pc=0x412de9
runtime.cgocallbackg(0x79e580, 0x7fbde9ff67f4, 0x0)
        <autogenerated>:1 +0x2f fp=0xc0000d7cb8 sp=0xc0000d7c90 pc=0x4755cf
runtime.cgocallback(0x412c65, 0x936210, 0xc0004d5d48)
        /usr/local/go/src/runtime/asm_amd64.s:915 +0xb4 fp=0xc0000d7ce0 sp=0xc0000d7cb8 pc=0x4731f4
runtime.systemstack_switch()
        /usr/local/go/src/runtime/asm_amd64.s:350 fp=0xc0000d7ce8 sp=0xc0000d7ce0 pc=0x4711e0
runtime.cgocall(0x936210, 0xc0000d7d48)
        /usr/local/go/src/runtime/cgocall.go:166 +0x85 fp=0xc0000d7d20 sp=0xc0000d7ce8 pc=0x412c65
github.com/usnistgov/ndn-dpdk/dpdk/spdkenv._Cfunc_c_SpdkThread_Run(0x17fe66900)
        _cgo_gotypes.go:266 +0x48 fp=0xc0000d7d48 sp=0xc0000d7d20 pc=0x883b28
github.com/usnistgov/ndn-dpdk/dpdk/spdkenv.(*Thread).main.func1(0xc000145f38)
        /root/ndn-dpdk/dpdk/spdkenv/thread.go:59 +0x3b fp=0xc0000d7d80 sp=0xc0000d7d48 pc=0x884e9b
github.com/usnistgov/ndn-dpdk/dpdk/spdkenv.(*Thread).main(0xc000132a50)
        /root/ndn-dpdk/dpdk/spdkenv/thread.go:59 +0x19 fp=0xc0000d7d98 sp=0xc0000d7d80 pc=0x884e39
github.com/usnistgov/ndn-dpdk/dpdk/spdkenv.InitMainThread(0xc0002f4480)
        /root/ndn-dpdk/dpdk/spdkenv/init.go:69 +0x51e fp=0xc0000d7f88 sp=0xc0000d7d98 pc=0x8847fe
github.com/usnistgov/ndn-dpdk/dpdk/ealinit.Init.func1.1()
        /root/ndn-dpdk/dpdk/ealinit/ealinit.go:57 +0xcb fp=0xc0000d7fe0 sp=0xc0000d7f88 pc=0x88734b
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc0000d7fe8 sp=0xc0000d7fe0 pc=0x473401
created by github.com/usnistgov/ndn-dpdk/dpdk/ealinit.Init.func1
        /root/ndn-dpdk/dpdk/ealinit/ealinit.go:46 +0xbd

goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7fbdf0147ab8, 0x72)
        /usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00026c800, 0xc00003c000, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00026c800)
        /usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc00026c800)
        /usr/local/go/src/net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00025c738)
        /usr/local/go/src/net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00025c738)
        /usr/local/go/src/net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc000326000, {0xbbfbb0, 0xc00025c738})
        /usr/local/go/src/net/http/server.go:3001 +0x394
net/http.(*Server).ListenAndServe(0xc000326000)
        /usr/local/go/src/net/http/server.go:2930 +0x7d
net/http.ListenAndServe(...)
        /usr/local/go/src/net/http/server.go:3184
main.glob..func1(0xc000264980)
        /root/ndn-dpdk/cmd/ndndpdk-svc/main.go:161 +0x265
github.com/urfave/cli/v2.(*App).RunContext(0xfb0420, {0xbc6ff0, 0xc000120000}, {0xc00011a180, 0x3, 0x3})
        /root/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:322 +0x7a8
github.com/urfave/cli/v2.(*App).Run(...)
        /root/go/pkg/mod/github.com/urfave/cli/v2@v2.3.0/app.go:224
main.main()
        /root/ndn-dpdk/cmd/ndndpdk-svc/main.go:178 +0x62b

goroutine 34 [chan receive, 801 minutes]:
main.glob..func1.1()
        /root/ndn-dpdk/cmd/ndndpdk-svc/main.go:153 +0x85
created by main.glob..func1
        /root/ndn-dpdk/cmd/ndndpdk-svc/main.go:150 +0x116

goroutine 21 [syscall, 801 minutes]:
os/signal.signal_recv()
        /usr/local/go/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
        /usr/local/go/src/os/signal/signal.go:151 +0x2c

goroutine 50 [syscall]:
syscall.Syscall(0x0, 0x8, 0xc0001fe000, 0x1000)
        /usr/local/go/src/syscall/asm_linux_amd64.s:20 +0x5
syscall.read(0xc0004bc000, {0xc0001fe000, 0x7fbdf1305700, 0x203001})
        /usr/local/go/src/syscall/zsyscall_linux_amd64.go:687 +0x4d
syscall.Read(...)
        /usr/local/go/src/syscall/syscall_unix.go:189
internal/poll.ignoringEINTRIO(...)
        /usr/local/go/src/internal/poll/fd_unix.go:582
internal/poll.(*FD).Read(0xc0004bc000, {0xc0001fe000, 0x1000, 0x1000})
        /usr/local/go/src/internal/poll/fd_unix.go:163 +0x285
os.(*File).read(...)
        /usr/local/go/src/os/file_posix.go:32
os.(*File).Read(0xc0004ba000, {0xc0001fe000, 0x11, 0xc0006f7d10})
        /usr/local/go/src/os/file.go:119 +0x5e
bufio.(*Reader).fill(0xc0006f7f78)
        /usr/local/go/src/bufio/bufio.go:101 +0x103
bufio.(*Reader).ReadSlice(0xc0006f7f78, 0xb8)
        /usr/local/go/src/bufio/bufio.go:360 +0x2f
bufio.(*Reader).collectFragments(0x3, 0x40)
        /usr/local/go/src/bufio/bufio.go:435 +0x74
bufio.(*Reader).ReadBytes(0xc00016e230, 0x69)
        /usr/local/go/src/bufio/bufio.go:463 +0x1d
github.com/usnistgov/ndn-dpdk/dpdk/ealinit.processLogStream()
        /root/ndn-dpdk/dpdk/ealinit/logging.go:128 +0x157
created by github.com/usnistgov/ndn-dpdk/dpdk/ealinit.initLogStream
        /root/ndn-dpdk/dpdk/ealinit/logging.go:111 +0x2e7

goroutine 53 [IO wait, 801 minutes]:
internal/poll.runtime_pollWait(0x7fbdf01478e8, 0x72)
        /usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc0004cc300, 0xc000346800, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0004cc300, {0xc000346800, 0x200, 0x200})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0004cc300, {0xc000346800, 0x7fbe17e2ba68, 0x200})
        /usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0004ba070, {0xc000346800, 0x9e0000, 0x1f30000000001})
        /usr/local/go/src/net/net.go:183 +0x45
encoding/json.(*Decoder).refill(0xc0005ba000)
        /usr/local/go/src/encoding/json/stream.go:165 +0x17f
encoding/json.(*Decoder).readValue(0xc0005ba000)
        /usr/local/go/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc0005ba000, {0xa19960, 0xc0004bc1a0})
        /usr/local/go/src/encoding/json/stream.go:63 +0x78
github.com/powerman/rpc-codec/jsonrpc2.(*clientCodec).ReadResponseHeader(0xc0004bc180, 0xc0003ff680)
        /root/go/pkg/mod/github.com/powerman/rpc-codec@v1.2.2/jsonrpc2/client.go:174 +0x38
net/rpc.(*Client).input(0xc0004bc1e0)
        /usr/local/go/src/net/rpc/client.go:109 +0xba
created by net/rpc.NewClientWithCodec
        /usr/local/go/src/net/rpc/client.go:206 +0xc7

goroutine 82 [chan receive, 801 minutes]:
github.com/usnistgov/ndn-dpdk/container/fib/fibreplica.(*Table).freeLoop(0xc0001bdad0)
        /root/ndn-dpdk/container/fib/fibreplica/table.go:93 +0xba
created by github.com/usnistgov/ndn-dpdk/container/fib/fibreplica.New
        /root/ndn-dpdk/container/fib/fibreplica/table.go:141 +0x26f

goroutine 124491 [select]:
github.com/graphql-go/graphql.Execute({{0xc0002a4b70, {0x100a950, 0x3, 0x3}, 0xc00015e840, 0xc00015e8f0, 0xc00015e9a0, 0xc0002a5590, 0x0, {0x0, ...}}, ...})
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:85 +0x2e5
github.com/graphql-go/graphql.Do({{0xc0002a4b70, {0x100a950, 0x3, 0x3}, 0xc00015e840, 0xc00015e8f0, 0xc00015e9a0, 0xc0002a5590, 0x0, {0x0, ...}}, ...})
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/graphql.go:108 +0x6fc
github.com/bhoriuchi/graphql-go-tools/handler.(*Handler).ContextHandler(0xc0003225c0, {0xbc6fb8, 0xc00042d780}, {0xbbfd90, 0xc000492a80}, 0xc0001a3600)
        /root/go/pkg/mod/github.com/bhoriuchi/graphql-go-tools@v0.3.2/handler/handler.go:144 +0x218
github.com/bhoriuchi/graphql-go-tools/handler.(*Handler).ServeHTTP(0xc00042c600, {0xbbfd90, 0xc000492a80}, 0xc000492a80)
        /root/go/pkg/mod/github.com/bhoriuchi/graphql-go-tools@v0.3.2/handler/handler.go:195 +0x54
net/http.(*ServeMux).ServeHTTP(0x0, {0xbbfd90, 0xc000492a80}, 0xc0001a3600)
        /usr/local/go/src/net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xbbe140}, {0xbbfd90, 0xc000492a80}, 0xc0001a3600)
        /usr/local/go/src/net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc0002e6aa0, {0xbc7060, 0xc0002a5950})
        /usr/local/go/src/net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
        /usr/local/go/src/net/http/server.go:3033 +0x4e8

goroutine 124461 [IO wait]:
internal/poll.runtime_pollWait(0x7fbdf0147800, 0x72)
        /usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc00026dd80, 0xc0005a1ea1, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00026dd80, {0xc0005a1ea1, 0x1, 0x1})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00026dd80, {0xc0005a1ea1, 0xa21180, 0xc0000f3f48})
        /usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000138778, {0xc0005a1ea1, 0xc000443010, 0xc0005a6be0})
        /usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).backgroundRead(0xc0005a1e90)
        /usr/local/go/src/net/http/server.go:672 +0x3f
created by net/http.(*connReader).startBackgroundRead
        /usr/local/go/src/net/http/server.go:668 +0xcf

goroutine 124492 [IO wait]:
internal/poll.runtime_pollWait(0x7fbdf0147718, 0x72)
        /usr/local/go/src/runtime/netpoll.go:234 +0x89
internal/poll.(*pollDesc).wait(0xc000298d80, 0xc000851a51, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000298d80, {0xc000851a51, 0x1, 0x1})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000298d80, {0xc000851a51, 0xa21180, 0xc0003abf48})
        /usr/local/go/src/net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0004ba4a8, {0xc000851a51, 0xc00065b990, 0xc0000fda40})
        /usr/local/go/src/net/net.go:183 +0x45
net/http.(*connReader).backgroundRead(0xc000851a40)
        /usr/local/go/src/net/http/server.go:672 +0x3f
created by net/http.(*connReader).startBackgroundRead
        /usr/local/go/src/net/http/server.go:668 +0xcf

goroutine 124462 [runnable]:
github.com/usnistgov/ndn-dpdk/dpdk/spdkenv.(*Thread).Post(0xc000145f20, {0xbbfa90, 0xc0001388e0})
        /root/ndn-dpdk/dpdk/spdkenv/thread.go:63 +0x7a
github.com/usnistgov/ndn-dpdk/dpdk/eal.PostMain({0xbbfa90, 0xc0001388e0})
        /root/ndn-dpdk/dpdk/eal/eal.go:78 +0x3b
github.com/usnistgov/ndn-dpdk/core/cptr.Call(0xb1fee0, {0x9d37a0, 0xc0002d3e20})
        /root/ndn-dpdk/core/cptr/function.go:131 +0xc2
github.com/usnistgov/ndn-dpdk/dpdk/eal.CallMain({0x9d37a0, 0xc0002d3e20})
        /root/ndn-dpdk/dpdk/eal/eal.go:88 +0x65
github.com/usnistgov/ndn-dpdk/iface.New({{0x0, 0x0, 0x0, 0x2336}, {0x0}, 0x300, 0xc00082f800, 0xc00040b650, 0xc000118fe0, 0xc000119000, ...})
        /root/ndn-dpdk/iface/face.go:162 +0x1d4
github.com/usnistgov/ndn-dpdk/iface/ethface.New(0xc000318e00, {0xbcc9d8, 0xc0005a7ae0})
        /root/ndn-dpdk/iface/ethface/face.go:37 +0x4d8
github.com/usnistgov/ndn-dpdk/iface/ethface.MemifLocator.CreateFace({{{0x400, 0x400}, {0xadda9f, 0x6}, {0xc000526340, 0x33}, 0x0, 0x0, 0x2328, 0x1000}})
        /root/ndn-dpdk/iface/ethface/memif.go:59 +0x21a
github.com/usnistgov/ndn-dpdk/iface.init.0.func7({{0xa122c0, 0x0}, 0xc00082f440, {{0xc000618400, 0xa}, {0xc0001388c8, 0x1, 0x1}, 0xc00040b518, {0xbc7d80, ...}, ...}, ...})
        /root/ndn-dpdk/iface/gql.go:133 +0xad
github.com/graphql-go/graphql.resolveField(0xc000344840, 0xc00015e8f0, {0xa122c0, 0x0}, {0xc0001388c8, 0x1, 0x1}, 0xc00040b518)
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:649 +0x695
github.com/graphql-go/graphql.executeFieldsSerially({0xc000344840, 0xc00015e8f0, {0xa122c0, 0x0}, 0xc00082f380, 0x0})
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:262 +0x1fa
github.com/graphql-go/graphql.executeOperation({0xc000344840, {0xa122c0, 0x0}, {0xbcc5d8, 0xc000318cb0}})
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:187 +0x2bb
github.com/graphql-go/graphql.Execute.func2()
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:78 +0x397
created by github.com/graphql-go/graphql.Execute
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:52 +0x265

goroutine 124460 [select]:
github.com/graphql-go/graphql.Execute({{0xc0002a4b70, {0x100a950, 0x3, 0x3}, 0xc00015e840, 0xc00015e8f0, 0xc00015e9a0, 0xc0002a5590, 0x0, {0x0, ...}}, ...})
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:85 +0x2e5
github.com/graphql-go/graphql.Do({{0xc0002a4b70, {0x100a950, 0x3, 0x3}, 0xc00015e840, 0xc00015e8f0, 0xc00015e9a0, 0xc0002a5590, 0x0, {0x0, ...}}, ...})
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/graphql.go:108 +0x6fc
github.com/bhoriuchi/graphql-go-tools/handler.(*Handler).ContextHandler(0xc0003225c0, {0xbc6fb8, 0xc0004c7400}, {0xbbfd90, 0xc0001c0700}, 0xc00026a900)
        /root/go/pkg/mod/github.com/bhoriuchi/graphql-go-tools@v0.3.2/handler/handler.go:144 +0x218
github.com/bhoriuchi/graphql-go-tools/handler.(*Handler).ServeHTTP(0x0, {0xbbfd90, 0xc0001c0700}, 0xc0001c0700)
        /root/go/pkg/mod/github.com/bhoriuchi/graphql-go-tools@v0.3.2/handler/handler.go:195 +0x54
net/http.(*ServeMux).ServeHTTP(0x0, {0xbbfd90, 0xc0001c0700}, 0xc00026a900)
        /usr/local/go/src/net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xbbe140}, {0xbbfd90, 0xc0001c0700}, 0xc00026a900)
        /usr/local/go/src/net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc0005a2320, {0xbc7060, 0xc0002a5950})
        /usr/local/go/src/net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
        /usr/local/go/src/net/http/server.go:3033 +0x4e8

goroutine 124493 [chan receive]:
github.com/usnistgov/ndn-dpdk/core/cptr.Call(0xb1fee0, {0x9d37a0, 0xc00000e978})
        /root/ndn-dpdk/core/cptr/function.go:139 +0xd7
github.com/usnistgov/ndn-dpdk/dpdk/eal.CallMain({0x9d37a0, 0xc00000e978})
        /root/ndn-dpdk/dpdk/eal/eal.go:88 +0x65
github.com/usnistgov/ndn-dpdk/iface.(*face).Close(0xc0005a1770)
        /root/ndn-dpdk/iface/face.go:269 +0x8d
github.com/usnistgov/ndn-dpdk/iface.init.0.func2({0xa834c0, 0xc00027c780})
        /root/ndn-dpdk/iface/gql.go:67 +0x3b
github.com/usnistgov/ndn-dpdk/core/gqlserver.init.1.func2({{0xa122c0, 0x0}, 0xc000348f60, {{0xc00084c952, 0x6}, {0xc0004ba5f0, 0x1, 0x1}, 0xc00000e960, {0xbc7d80, ...}, ...}, ...})
        /root/ndn-dpdk/core/gqlserver/node.go:244 +0x63
github.com/graphql-go/graphql.resolveField(0xc0001acfc0, 0xc00015e8f0, {0xa122c0, 0x0}, {0xc0004ba5f0, 0x1, 0x1}, 0xc00000e960)
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:649 +0x695
github.com/graphql-go/graphql.executeFieldsSerially({0xc0001acfc0, 0xc00015e8f0, {0xa122c0, 0x0}, 0xc000348ea0, 0x0})
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:262 +0x1fa
github.com/graphql-go/graphql.executeOperation({0xc0001acfc0, {0xa122c0, 0x0}, {0xbcc5d8, 0xc0002b0230}})
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:187 +0x2bb
github.com/graphql-go/graphql.Execute.func2()
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:78 +0x397
created by github.com/graphql-go/graphql.Execute
        /root/go/pkg/mod/github.com/graphql-go/graphql@v0.8.0/executor.go:52 +0x265
yoursunny commented 2 years ago

What NDN-DPDK version or commit SHA are you running? You can see the NDN-DPDK service version with ndndpdk-ctrl show-version command or the first line of service logs.

fatal error: concurrent map writes

In Go, it is not allowed to write to a map from multiple goroutines, and it is not allowed to read from a map when another goroutine is writing to it. Violation of this assumption causes a race condition, which triggers a panic if Go ever finds out. The map causing this error is most likely portByEthDev (I can only confirm after you provide version information).

The log snippet shows a memif face is being closed, and another is being opened:

It seems that there are some code paths that access the map without posting the task to the main thread.

Pesa commented 2 years ago 
{"level":"info","ts":1638833899.0720825,"logger":"main","msg":"NDN-DPDK service starting","version":"v0.0.0-20211025195105-7686941f2f7f","uid":0,"linux":"5.4.0-91-generic","dpdk":"DPDK 21.08.0","spdk":"SPDK v21.07"}
yoursunny commented 2 years ago

v0.0.0-20211025195105-7686941f2f7f

This is really old version. However, the bug is most likely still in the current codebase. I need to check those race conditions.

Pesa commented 2 years ago

I'm seriously concerned about calling a 7-week-old version a "really old version"...

yoursunny commented 2 years ago

It is old in the sense that a major refactoring has occurred in this package.

yoursunny commented 2 years ago

In 253fb71aae89800074a1c5528f0e5e86754f2243, portByEthDev map is replaced by portByEthDevID array, so that this particular crash would not happen again. However, I believe there's still race condition in Port.faces map and possibly other fields of Port type, so that this issue shall remain open.

yoursunny commented 2 years ago

Port.faces race condition is fixed in a0fec5fde7d9c47765f2a8217e372c4534758f4a through sync.Mutex.