search

silverton-io / buz

Serverless multi-protocol + multi-destination event collection system.
https://buz.dev/
Apache License 2.0
186 stars 21 forks source link

Replace jsonschema validator #518

Closed jakthom closed 1 year ago

jakthom commented 1 year ago

Buz should be running https://github.com/santhosh-tekuri/jsonschema instead of https://github.com/qri-io/jsonschema

jakthom commented 1 year ago

At higher volumes the existing lib blows up at higher "single-node" volumes w/ a concurrent map writes error:

{"level":"debug","time":"2023-04-29T10:18:18-04:00","message":"🟡 found cache key io.silverton/buz/example/gettingStarted/v1.0.json"}
fatal error: concurrent map writes

goroutine 4987 [running]:
github.com/qri-io/jsonschema.(*SchemaRegistry).Register(...)
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/schema_registry.go:73
github.com/qri-io/jsonschema.(*Schema).Register(0x1400cccc060, {0x0, 0x0}, 0x0?)
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/schema.go:83 +0x1f8
github.com/qri-io/jsonschema.(*Schema).ValidateKeyword(0x1400cccc060, {0x1049f1e30, 0x140000b6000}, 0x1400c3d7c00, {0x1046c9040, 0x1400ccef170})
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/schema.go:280 +0x7c
github.com/qri-io/jsonschema.(*Schema).Validate(0x1400cccc060, {0x1049f1e30, 0x140000b6000}, {0x1046c9040, 0x1400ccef170})
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/schema.go:260 +0x2fc
github.com/qri-io/jsonschema.(*Schema).ValidateBytes(0x1400c3ddc00?, {0x1049f1e30, 0x140000b6000}, {0x1400c3f41e0, 0x4c, 0x50})
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/schema.go:330 +0xd0
github.com/silverton-io/buz/pkg/validator.validatePayload({0x1400c3f41e0, 0x4c, 0x50}, {0x1400c3ddc00, 0x3e2, 0x3e2})
    /Users/jacobthomas/code/silverton/buz/pkg/validator/payload.go:25 +0x140
github.com/silverton-io/buz/pkg/validator.Validate({{0xa0, 0x3a, 0xed, 0x36, 0x63, 0x92, 0x4f, 0xa3, 0x94, 0x1b, ...}, ...}, ...)
    /Users/jacobthomas/code/silverton/buz/pkg/validator/validator.go:68 +0x1d8
github.com/silverton-io/buz/pkg/annotator.Annotate({0x1400ccea000?, 0x22, 0x103f14844?}, 0x7?)
    /Users/jacobthomas/code/silverton/buz/pkg/annotator/annotator.go:40 +0x10c
github.com/silverton-io/buz/pkg/manifold.(*ChannelManifold).Enqueue(0x1400087ed20, {0x1400ccea000?, 0x1400043f570?, 0x102a0a0cc?})
    /Users/jacobthomas/code/silverton/buz/pkg/manifold/channelManifold.go:63 +0x38
github.com/silverton-io/buz/pkg/protocol/cloudevents.(*CloudeventsInput).Handler.func1(0x14007ab2918?)
    /Users/jacobthomas/code/silverton/buz/pkg/protocol/cloudevents/input.go:37 +0xdc
github.com/gin-gonic/gin.(*Context).Next(...)
    /Users/jacobthomas/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/context.go:173
github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0x1400050aea0, 0x140072a0f00)
    /Users/jacobthomas/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/gin.go:616 +0x51c
github.com/gin-gonic/gin.(*Engine).ServeHTTP(0x1400050aea0, {0x1049f0880?, 0x1400c39d420}, 0x1400ccb9500)
    /Users/jacobthomas/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/gin.go:572 +0x1e4
net/http.serverHandler.ServeHTTP({0x1049ebe40?}, {0x1049f0880, 0x1400c39d420}, 0x1400ccb9500)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:2936 +0x2c0
net/http.(*conn).serve(0x1400c3e5e60, {0x1049f1ea0, 0x140008797d0})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:1995 +0x518
created by net/http.(*Server).Serve
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:3089 +0x4e8

goroutine 1 [chan receive, 1 minutes]:
main.(*App).standardMode(0x140000cd780)
    /Users/jacobthomas/code/silverton/buz/cmd/buz/app.go:239 +0x188
main.(*App).Run(0x140000cd780)
    /Users/jacobthomas/code/silverton/buz/cmd/buz/app.go:263 +0x94
main.main()
    /Users/jacobthomas/code/silverton/buz/cmd/buz/main.go:10 +0x34

goroutine 21 [chan receive]:
github.com/ClickHouse/clickhouse-go.init.0.func1()
    /Users/jacobthomas/go/pkg/mod/github.com/!click!house/clickhouse-go@v1.5.4/bootstrap.go:48 +0x3c
created by github.com/ClickHouse/clickhouse-go.init.0
    /Users/jacobthomas/go/pkg/mod/github.com/!click!house/clickhouse-go@v1.5.4/bootstrap.go:45 +0x44

goroutine 22 [select]:
go.opencensus.io/stats/view.(*worker).start(0x14000603c00)
    /Users/jacobthomas/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:292 +0x88
created by go.opencensus.io/stats/view.init.0
    /Users/jacobthomas/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0xa0

goroutine 23 [select]:
github.com/silverton-io/buz/pkg/backend/backendutils.StartSinkWorker.func1(0x140000a2a80, 0x1400009c1c0, {0x1049f4c98, 0x1400044a1c0})
    /Users/jacobthomas/code/silverton/buz/pkg/backend/backendutils/sink.go:62 +0x78
created by github.com/silverton-io/buz/pkg/backend/backendutils.StartSinkWorker
    /Users/jacobthomas/code/silverton/buz/pkg/backend/backendutils/sink.go:60 +0xb8

goroutine 24 [select]:
github.com/silverton-io/buz/pkg/backend/backendutils.StartSinkWorker.func1(0x140000a2ae0, 0x1400009c2a0, {0x1049f4938, 0x1400044a230})
    /Users/jacobthomas/code/silverton/buz/pkg/backend/backendutils/sink.go:62 +0x78
created by github.com/silverton-io/buz/pkg/backend/backendutils.StartSinkWorker
    /Users/jacobthomas/code/silverton/buz/pkg/backend/backendutils/sink.go:60 +0xb8

goroutine 25 [select]:
github.com/silverton-io/buz/pkg/manifold.(*ChannelManifold).Initialize.func1(0x140000a2b40, 0x1400009c310)
    /Users/jacobthomas/code/silverton/buz/pkg/manifold/channelManifold.go:35 +0x78
created by github.com/silverton-io/buz/pkg/manifold.(*ChannelManifold).Initialize
    /Users/jacobthomas/code/silverton/buz/pkg/manifold/channelManifold.go:33 +0x1a8

goroutine 30 [syscall, 1 minutes]:
os/signal.signal_recv()
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/runtime/sigqueue.go:149 +0x2c
os/signal.loop()
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/os/signal/signal_unix.go:23 +0x1c
created by os/signal.Notify.func1.1
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/os/signal/signal.go:151 +0x28

goroutine 40 [IO wait]:
internal/poll.runtime_pollWait(0x12dc34f08, 0x72)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/runtime/netpoll.go:306 +0xa0
internal/poll.(*pollDesc).wait(0x1400048a000?, 0x14000306a00?, 0x0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:84 +0x28
internal/poll.(*pollDesc).waitRead(...)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0x1400048a000, {0x14000306a00, 0x1500, 0x1500})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_unix.go:167 +0x200
net.(*netFD).Read(0x1400048a000, {0x14000306a00?, 0x14000446658?, 0x0?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/fd_posix.go:55 +0x28
net.(*conn).Read(0x14000880050, {0x14000306a00?, 0x140005929c0?, 0x300000002?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/net.go:183 +0x34
crypto/tls.(*atLeastReader).Read(0x14009672828, {0x14000306a00?, 0x14009672828?, 0x0?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/crypto/tls/conn.go:788 +0x40
bytes.(*Buffer).ReadFrom(0x140003cf410, {0x1049d8b00, 0x14009672828})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/bytes/buffer.go:202 +0x90
crypto/tls.(*Conn).readFromUntil(0x140003cf180, {0x12d7abfc8?, 0x14000880050}, 0x102b1cf08?)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/crypto/tls/conn.go:810 +0xd0
crypto/tls.(*Conn).readRecordOrCCS(0x140003cf180, 0x0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/crypto/tls/conn.go:617 +0xd4
crypto/tls.(*Conn).readRecord(...)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/crypto/tls/conn.go:583
crypto/tls.(*Conn).Read(0x140003cf180, {0x140004d5000, 0x1000, 0x1049f7eb8?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/crypto/tls/conn.go:1316 +0x170
bufio.(*Reader).Read(0x140004d2480, {0x1400046c820, 0x9, 0x140003cf180?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/bufio/bufio.go:237 +0x1d8
io.ReadAtLeast({0x1049d73e0, 0x140004d2480}, {0x1400046c820, 0x9, 0x9}, 0x9)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/io/io.go:332 +0xa0
io.ReadFull(...)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/io/io.go:351
net/http.http2readFrameHeader({0x1400046c820?, 0x9?, 0x140000cda80?}, {0x1049d73e0?, 0x140004d2480?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/h2_bundle.go:1567 +0x58
net/http.(*http2Framer).ReadFrame(0x1400046c7e0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/h2_bundle.go:1831 +0x80
net/http.(*http2clientConnReadLoop).run(0x14000cb9f88)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/h2_bundle.go:9187 +0xf8
net/http.(*http2ClientConn).readLoop(0x140004b2180)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/h2_bundle.go:9082 +0x5c
created by net/http.(*http2Transport).newClientConn
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/h2_bundle.go:7779 +0xa18

goroutine 50 [chan receive]:
github.com/silverton-io/buz/pkg/tele.heartbeat({0x14000496300, {0x14000054f00, 0xa2ea06b126a9, 0xdf8475800, 0x1049d3bc0, {0x1045bd8e0, 0x14000496300}, 0x0, 0x0, 0x1}}, ...)
    /Users/jacobthomas/code/silverton/buz/pkg/tele/tele.go:46 +0x6c
created by github.com/silverton-io/buz/pkg/tele.Metry
    /Users/jacobthomas/code/silverton/buz/pkg/tele/tele.go:113 +0x270

goroutine 51 [IO wait]:
internal/poll.runtime_pollWait(0x12dc34e18, 0x72)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/runtime/netpoll.go:306 +0xa0
internal/poll.(*pollDesc).wait(0x14000110100?, 0x0?, 0x0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:84 +0x28
internal/poll.(*pollDesc).waitRead(...)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0x14000110100)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_unix.go:614 +0x250
net.(*netFD).accept(0x14000110100)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/fd_unix.go:172 +0x28
net.(*TCPListener).accept(0x140004bc4b0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/tcpsock_posix.go:148 +0x28
net.(*TCPListener).Accept(0x140004bc4b0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/tcpsock.go:297 +0x2c
net/http.(*Server).Serve(0x1400060c0f0, {0x1049f0670, 0x140004bc4b0})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:3059 +0x2cc
net/http.(*Server).ListenAndServe(0x1400060c0f0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:2988 +0x84
main.(*App).standardMode.func1()
    /Users/jacobthomas/code/silverton/buz/cmd/buz/app.go:232 +0x5c
created by main.(*App).standardMode
    /Users/jacobthomas/code/silverton/buz/cmd/buz/app.go:230 +0x128

goroutine 4986 [runnable]:
github.com/qri-io/jsonschema.(*AdditionalProperties).ValidateKeyword(0x1400cad3b60, {0x1049f1e30, 0x140000b6000}, 0x1400cbe0300, {0x1046c9040?, 0x1400cbe2630})
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/keywords_object.go:324 +0x5b0
github.com/qri-io/jsonschema.(*Schema).validateSchemakeywords(...)
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/schema.go:318
github.com/qri-io/jsonschema.(*Schema).ValidateKeyword(0x1400cad39e0, {0x1049f1e30, 0x140000b6000}, 0x1400cbe0300, {0x1046c9040, 0x1400cbe2630})
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/schema.go:311 +0x358
github.com/qri-io/jsonschema.(*Schema).Validate(0x1400cad39e0, {0x1049f1e30, 0x140000b6000}, {0x1046c9040, 0x1400cbe2630})
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/schema.go:260 +0x2fc
github.com/qri-io/jsonschema.(*Schema).ValidateBytes(0x1400cbe8000?, {0x1049f1e30, 0x140000b6000}, {0x1400caf4960, 0x4c, 0x50})
    /Users/jacobthomas/go/pkg/mod/github.com/qri-io/jsonschema@v0.2.1/schema.go:330 +0xd0
github.com/silverton-io/buz/pkg/validator.validatePayload({0x1400caf4960, 0x4c, 0x50}, {0x1400cbe8000, 0x3e2, 0x3e2})
    /Users/jacobthomas/code/silverton/buz/pkg/validator/payload.go:25 +0x140
github.com/silverton-io/buz/pkg/validator.Validate({{0xe9, 0xbd, 0xb, 0x38, 0xce, 0x52, 0x40, 0x10, 0x8d, 0x78, ...}, ...}, ...)
    /Users/jacobthomas/code/silverton/buz/pkg/validator/validator.go:68 +0x1d8
github.com/silverton-io/buz/pkg/annotator.Annotate({0x1400cbda000?, 0x22, 0x103f14844?}, 0x7?)
    /Users/jacobthomas/code/silverton/buz/pkg/annotator/annotator.go:40 +0x10c
github.com/silverton-io/buz/pkg/manifold.(*ChannelManifold).Enqueue(0x1400087ed20, {0x1400cbda000?, 0x1400043f570?, 0x102a0a0cc?})
    /Users/jacobthomas/code/silverton/buz/pkg/manifold/channelManifold.go:63 +0x38
github.com/silverton-io/buz/pkg/protocol/cloudevents.(*CloudeventsInput).Handler.func1(0x14000490918?)
    /Users/jacobthomas/code/silverton/buz/pkg/protocol/cloudevents/input.go:37 +0xdc
github.com/gin-gonic/gin.(*Context).Next(...)
    /Users/jacobthomas/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/context.go:173
github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0x1400050aea0, 0x1400b97eb00)
    /Users/jacobthomas/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/gin.go:616 +0x51c
github.com/gin-gonic/gin.(*Engine).ServeHTTP(0x1400050aea0, {0x1049f0880?, 0x1400cba9ce0}, 0x1400cbbad00)
    /Users/jacobthomas/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/gin.go:572 +0x1e4
net/http.serverHandler.ServeHTTP({0x1049ebe40?}, {0x1049f0880, 0x1400cba9ce0}, 0x1400cbbad00)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:2936 +0x2c0
net/http.(*conn).serve(0x1400c3e5dd0, {0x1049f1ea0, 0x140008797d0})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:1995 +0x518
created by net/http.(*Server).Serve
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:3089 +0x4e8

goroutine 5068 [IO wait]:
internal/poll.runtime_pollWait(0x12dc34d28, 0x72)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/runtime/netpoll.go:306 +0xa0
internal/poll.(*pollDesc).wait(0x1400c3d6980?, 0x1400cb91ae1?, 0x0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:84 +0x28
internal/poll.(*pollDesc).waitRead(...)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0x1400c3d6980, {0x1400cb91ae1, 0x1, 0x1})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_unix.go:167 +0x200
net.(*netFD).Read(0x1400c3d6980, {0x1400cb91ae1?, 0x104724060?, 0x104743160?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/fd_posix.go:55 +0x28
net.(*conn).Read(0x1400c243498, {0x1400cb91ae1?, 0x6e6f6974706901?, 0x1400cad4c80?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/net.go:183 +0x34
net/http.(*connReader).backgroundRead(0x1400cb91ad0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:674 +0x44
created by net/http.(*connReader).startBackgroundRead
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:670 +0xc8

goroutine 4988 [IO wait]:
internal/poll.runtime_pollWait(0x12dc34c38, 0x72)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/runtime/netpoll.go:306 +0xa0
internal/poll.(*pollDesc).wait(0x1400c3d6a00?, 0x1400ccb6641?, 0x0)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:84 +0x28
internal/poll.(*pollDesc).waitRead(...)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0x1400c3d6a00, {0x1400ccb6641, 0x1, 0x1})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_unix.go:167 +0x200
net.(*netFD).Read(0x1400c3d6a00, {0x1400ccb6641?, 0x104724060?, 0x104743160?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/fd_posix.go:55 +0x28
net.(*conn).Read(0x1400c2434a0, {0x1400ccb6641?, 0x1?, 0x1400c22ca50?})
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/net.go:183 +0x34
net/http.(*connReader).backgroundRead(0x1400ccb6630)
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:674 +0x44
created by net/http.(*connReader).startBackgroundRead
    /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:670 +0xc8
exit status 2
make[1]: *** [debug] Error 1

Seems like I'm not alone w/ this one:

https://github.com/qri-io/jsonschema/issues/76 https://github.com/qri-io/jsonschema/issues/80