search

loggie-io / loggie

A lightweight, cloud-native data transfer agent and aggregator
https://loggie-io.github.io/docs-en/
Apache License 2.0
1.26k stars 165 forks source link

fatal error: concurrent map iteration and map write #181

Open iamyeka opened 2 years ago

iamyeka commented 2 years ago

What version of Loggie?

docker image built by main branch

Expected Behavior

not panic

Actual Behavior

fatal error: concurrent map iteration and map write

goroutine 168 [running]:
runtime.throw(0x21ac606, 0x26)
    /usr/local/go/src/runtime/panic.go:1117 +0x72 fp=0xc000569578 sp=0xc000569548 pc=0x440b52
runtime.mapiternext(0xc00094d3e0)
    /usr/local/go/src/runtime/map.go:858 +0x54c fp=0xc0005695f8 sp=0xc000569578 pc=0x4196cc
runtime.mapiterinit(0x1e752c0, 0xc0005f2450, 0xc00094d3e0)
    /usr/local/go/src/runtime/map.go:848 +0x1c5 fp=0xc000569618 sp=0xc0005695f8 pc=0x419085
reflect.mapiterinit(0x1e752c0, 0xc0005f2450, 0x8)
    /usr/local/go/src/runtime/map.go:1340 +0x54 fp=0xc000569648 sp=0xc000569618 pc=0x4741d4
github.com/modern-go/reflect2.(*UnsafeMapType).UnsafeIterate(...)
    /go/src/loggie.io/loggie/vendor/github.com/modern-go/reflect2/go_below_118.go:17
github.com/json-iterator/go.(*mapEncoder).Encode(0xc00065aa80, 0xc0005b95d0, 0xc000b14000)
    /go/src/loggie.io/loggie/vendor/github.com/json-iterator/go/reflect_map.go:257 +0x1c5 fp=0xc000569700 sp=0xc000569648 pc=0x94fca5
github.com/json-iterator/go.(*onePtrEncoder).Encode(0xc000848990, 0xc0005f2450, 0xc000b14000)
    /go/src/loggie.io/loggie/vendor/github.com/json-iterator/go/reflect.go:219 +0x68 fp=0xc000569728 sp=0xc000569700 pc=0x946d48
github.com/json-iterator/go.(*Stream).WriteVal(0xc000b14000, 0x1e752c0, 0xc0005f2450)
    /go/src/loggie.io/loggie/vendor/github.com/json-iterator/go/reflect.go:98 +0x150 fp=0xc000569790 sp=0xc000569728 pc=0x945b90
github.com/json-iterator/go.(*dynamicEncoder).Encode(0xc000848970, 0xc000dfb168, 0xc000b14000)
    /go/src/loggie.io/loggie/vendor/github.com/json-iterator/go/reflect_dynamic.go:15 +0x65 fp=0xc0005697c0 sp=0xc000569790 pc=0x948dc5
github.com/json-iterator/go.(*mapEncoder).Encode(0xc00065aa80, 0xc0005b95c8, 0xc000b14000)
    /go/src/loggie.io/loggie/vendor/github.com/json-iterator/go/reflect_map.go:269 +0x371 fp=0xc000569878 sp=0xc0005697c0 pc=0x94fe51
github.com/json-iterator/go.(*onePtrEncoder).Encode(0xc000848990, 0xc000448000, 0xc000b14000)
    /go/src/loggie.io/loggie/vendor/github.com/json-iterator/go/reflect.go:219 +0x68 fp=0xc0005698a0 sp=0xc000569878 pc=0x946d48
github.com/json-iterator/go.(*Stream).WriteVal(0xc000b14000, 0x1e752c0, 0xc000448000)
    /go/src/loggie.io/loggie/vendor/github.com/json-iterator/go/reflect.go:98 +0x150 fp=0xc000569908 sp=0xc0005698a0 pc=0x945b90
github.com/json-iterator/go.(*frozenConfig).Marshal(0xc00047b040, 0x1e752c0, 0xc000448000, 0x0, 0x0, 0x0, 0x0, 0x0)
    /go/src/loggie.io/loggie/vendor/github.com/json-iterator/go/config.go:299 +0xaf fp=0xc000569978 sp=0xc000569908 pc=0x93ad4f
github.com/loggie-io/loggie/pkg/sink/codec/json.(*Json).Encode(0xc0003a6168, 0x24caaa0, 0xc000446300, 0xc000db7650, 0x17, 0x0, 0x0, 0x0)
    /go/src/loggie.io/loggie/pkg/sink/codec/json/json.go:79 +0x15d fp=0xc0005699f8 sp=0xc000569978 pc=0x175cc9d
github.com/loggie-io/loggie/pkg/sink/kafka.(*Sink).Consume(0xc0008c03c0, 0x24afe40, 0xc000d04180, 0x33196e0, 0x0)
    /go/src/loggie.io/loggie/pkg/sink/kafka/sink.go:134 +0x169 fp=0xc000569b88 sp=0xc0005699f8 pc=0x1a9e7c9
github.com/loggie-io/loggie/pkg/core/sink.(*SubscribeInvoker).Invoke(0x3317968, 0x24afe40, 0xc000d04180, 0x7eff4c045e10, 0xc0008c03c0, 0x203000, 0x0)
    /go/src/loggie.io/loggie/pkg/core/sink/invoke.go:42 +0x45 fp=0xc000569bc0 sp=0xc000569b88 pc=0x7f8a45
github.com/loggie-io/loggie/pkg/interceptor/retry.(*Interceptor).Intercept(0xc0003d7b20, 0x247d580, 0x3317968, 0x24afe40, 0xc000d04180, 0x7eff4c045e10, 0xc0008c03c0, 0xaa, 0x0)
    /go/src/loggie.io/loggie/pkg/interceptor/retry/interceptor.go:132 +0xea fp=0xc000569cb8 sp=0xc000569bc0 pc=0x164024a
github.com/loggie-io/loggie/pkg/pipeline.buildSinkInvokerChain.func1(0x24afe40, 0xc000d04180, 0x7eff4c045e10, 0xc0008c03c0, 0x1, 0x0)
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:508 +0x69 fp=0xc000569d10 sp=0xc000569cb8 pc=0x808ac9
github.com/loggie-io/loggie/pkg/core/sink.(*AbstractInvoker).Invoke(0xc0003a6210, 0x24afe40, 0xc000d04180, 0x7eff4c045e10, 0xc0008c03c0, 0x2, 0x1010323a280)
    /go/src/loggie.io/loggie/pkg/core/sink/invoke.go:35 +0x51 fp=0xc000569d50 sp=0xc000569d10 pc=0x7f89d1
github.com/loggie-io/loggie/pkg/interceptor/metric.(*Interceptor).Intercept(0xc0004483f0, 0x247d560, 0xc0003a6210, 0x24afe40, 0xc000d04180, 0x7eff4c045e10, 0xc0008c03c0, 0x0, 0xc00020f6c0)
    /go/src/loggie.io/loggie/pkg/interceptor/metric/interceptor.go:79 +0x63 fp=0xc000569da8 sp=0xc000569d50 pc=0x163f163
github.com/loggie-io/loggie/pkg/pipeline.buildSinkInvokerChain.func1(0x24afe40, 0xc000d04180, 0x7eff4c045e10, 0xc0008c03c0, 0xc000583588, 0xc00070eeec)
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:508 +0x69 fp=0xc000569e00 sp=0xc000569da8 pc=0x808ac9
github.com/loggie-io/loggie/pkg/core/sink.(*AbstractInvoker).Invoke(0xc0003a6228, 0x24afe40, 0xc000d04180, 0x7eff4c045e10, 0xc0008c03c0, 0x2, 0x2)
    /go/src/loggie.io/loggie/pkg/core/sink/invoke.go:35 +0x51 fp=0xc000569e40 sp=0xc000569e00 pc=0x7f89d1
github.com/loggie-io/loggie/pkg/pipeline.(*Pipeline).startSinkConsumer.func1(0x24afe40, 0xc000d04180, 0x0, 0x0)
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:444 +0x55 fp=0xc000569e88 sp=0xc000569e40 pc=0x808895
github.com/loggie-io/loggie/pkg/pipeline.(*Pipeline).sinkInvokeLoop(0xc00039b400, 0x0, 0x7eff4c045e10, 0xc0008c03c0, 0x7eff4c045c48, 0xc0004dea10, 0xc0003f2680, 0x2, 0x2, 0xc00018c1e0)
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:480 +0x1f8 fp=0xc000569f90 sp=0xc000569e88 pc=0x805898
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc000569f98 sp=0xc000569f90 pc=0x47a961
created by github.com/loggie-io/loggie/pkg/pipeline.(*Pipeline).startSinkConsumer
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:460 +0x666

more panic appears just like the one above:

fatal error: concurrent map read and map write

goroutine 61 [running]:
runtime.throw(0x21a3b9c, 0x21)
    /usr/local/go/src/runtime/panic.go:1117 +0x72 fp=0xc0008b9738 sp=0xc0008b9708 pc=0x440b52
runtime.mapaccess2_faststr(0x1e752c0, 0xc0005784b0, 0xc000674b69, 0x5, 0xc00101fb88, 0xc0008b9801)
    /usr/local/go/src/runtime/map_faststr.go:116 +0x4a5 fp=0xc0008b97a8 sp=0xc0008b9738 pc=0x41ce05
github.com/loggie-io/loggie/pkg/util/runtime.(*Object).Get(0xc001020010, 0xc000674b69, 0x5, 0xc001020010)
    /go/src/loggie.io/loggie/pkg/util/runtime/object.go:47 +0x7f fp=0xc0008b97f8 sp=0xc0008b97a8 pc=0x1749adf
github.com/loggie-io/loggie/pkg/util/runtime.(*Object).GetPaths(0xc0008b99d8, 0xc000c581a0, 0x2, 0x2, 0x0)
    /go/src/loggie.io/loggie/pkg/util/runtime/object.go:64 +0x6f fp=0xc0008b9838 sp=0xc0008b97f8 pc=0x1749c8f
github.com/loggie-io/loggie/pkg/util/runtime.(*Object).GetPath(0xc0008b99d8, 0xc000674b62, 0xc, 0xc0008b98d0)
    /go/src/loggie.io/loggie/pkg/util/runtime/object.go:58 +0x91 fp=0xc0008b9890 sp=0xc0008b9838 pc=0x1749bf1
github.com/loggie-io/loggie/pkg/util/runtime.getNew(0xc0008b99d8, 0xc000674b62, 0xc, 0x2130b00, 0xc000800000, 0xc0008b9968, 0x93ae4f)
    /go/src/loggie.io/loggie/pkg/util/runtime/select.go:80 +0x56 fp=0xc0008b98d0 sp=0xc0008b9890 pc=0x174b1b6
github.com/loggie-io/loggie/pkg/util/runtime.PatternFormat(0xc0008b99d8, 0xc000674b60, 0xf, 0xc00040e4b0, 0x1, 0xa, 0xc000a31500, 0x17, 0x0, 0x0)
    /go/src/loggie.io/loggie/pkg/util/runtime/select.go:57 +0xfe fp=0xc0008b9988 sp=0xc0008b98d0 pc=0x174ad3e
github.com/loggie-io/loggie/pkg/sink/kafka.(*Sink).selectTopic(0xc000676c80, 0x24caaa0, 0xc0006cc6f0, 0xc001578f00, 0x2fb, 0x2fb, 0x0)
    /go/src/loggie.io/loggie/pkg/sink/kafka/sink.go:159 +0x99 fp=0xc0008b99f8 sp=0xc0008b9988 pc=0x1a9ee99
github.com/loggie-io/loggie/pkg/sink/kafka.(*Sink).Consume(0xc000676c80, 0x24afe40, 0xc000d6a000, 0x33196e0, 0x0)
    /go/src/loggie.io/loggie/pkg/sink/kafka/sink.go:128 +0x106 fp=0xc0008b9b88 sp=0xc0008b99f8 pc=0x1a9e766
github.com/loggie-io/loggie/pkg/core/sink.(*SubscribeInvoker).Invoke(0x3317968, 0x24afe40, 0xc000d6a000, 0x7f0904fb1750, 0xc000676c80, 0x203000, 0x0)
    /go/src/loggie.io/loggie/pkg/core/sink/invoke.go:42 +0x45 fp=0xc0008b9bc0 sp=0xc0008b9b88 pc=0x7f8a45
github.com/loggie-io/loggie/pkg/interceptor/retry.(*Interceptor).Intercept(0xc0001f36c0, 0x247d580, 0x3317968, 0x24afe40, 0xc000d6a000, 0x7f0904fb1750, 0xc000676c80, 0xaa, 0x0)
    /go/src/loggie.io/loggie/pkg/interceptor/retry/interceptor.go:132 +0xea fp=0xc0008b9cb8 sp=0xc0008b9bc0 pc=0x164024a
github.com/loggie-io/loggie/pkg/pipeline.buildSinkInvokerChain.func1(0x24afe40, 0xc000d6a000, 0x7f0904fb1750, 0xc000676c80, 0x1, 0x0)
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:508 +0x69 fp=0xc0008b9d10 sp=0xc0008b9cb8 pc=0x808ac9
github.com/loggie-io/loggie/pkg/core/sink.(*AbstractInvoker).Invoke(0xc0002141a0, 0x24afe40, 0xc000d6a000, 0x7f0904fb1750, 0xc000676c80, 0x2, 0x1010323a280)
    /go/src/loggie.io/loggie/pkg/core/sink/invoke.go:35 +0x51 fp=0xc0008b9d50 sp=0xc0008b9d10 pc=0x7f89d1
github.com/loggie-io/loggie/pkg/interceptor/metric.(*Interceptor).Intercept(0xc0006d73e0, 0x247d560, 0xc0002141a0, 0x24afe40, 0xc000d6a000, 0x7f0904fb1750, 0xc000676c80, 0x0, 0xc0004c41c0)
    /go/src/loggie.io/loggie/pkg/interceptor/metric/interceptor.go:79 +0x63 fp=0xc0008b9da8 sp=0xc0008b9d50 pc=0x163f163
github.com/loggie-io/loggie/pkg/pipeline.buildSinkInvokerChain.func1(0x24afe40, 0xc000d6a000, 0x7f0904fb1750, 0xc000676c80, 0xc000682c88, 0xc00076beec)
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:508 +0x69 fp=0xc0008b9e00 sp=0xc0008b9da8 pc=0x808ac9
github.com/loggie-io/loggie/pkg/core/sink.(*AbstractInvoker).Invoke(0xc0002141b8, 0x24afe40, 0xc000d6a000, 0x7f0904fb1750, 0xc000676c80, 0x2, 0x2)
    /go/src/loggie.io/loggie/pkg/core/sink/invoke.go:35 +0x51 fp=0xc0008b9e40 sp=0xc0008b9e00 pc=0x7f89d1
github.com/loggie-io/loggie/pkg/pipeline.(*Pipeline).startSinkConsumer.func1(0x24afe40, 0xc000d6a000, 0x0, 0x0)
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:444 +0x55 fp=0xc0008b9e88 sp=0xc0008b9e40 pc=0x808895
github.com/loggie-io/loggie/pkg/pipeline.(*Pipeline).sinkInvokeLoop(0xc00031f540, 0x0, 0x7f0904fb1750, 0xc000676c80, 0x7f0904fb1588, 0xc0002ffe30, 0xc000868500, 0x2, 0x2, 0xc000716140)
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:480 +0x1f8 fp=0xc0008b9f90 sp=0xc0008b9e88 pc=0x805898
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc0008b9f98 sp=0xc0008b9f90 pc=0x47a961
created by github.com/loggie-io/loggie/pkg/pipeline.(*Pipeline).startSinkConsumer
    /go/src/loggie.io/loggie/pkg/pipeline/pipeline.go:460 +0x666

goroutine 1 [chan receive]:
main.main()
    /go/src/loggie.io/loggie/cmd/loggie/main.go:130 +0x7e7

goroutine 6 [chan receive]:
k8s.io/klog/v2.(*loggingT).flushDaemon(0x32e5d20)
    /go/src/loggie.io/loggie/vendor/k8s.io/klog/v2/klog.go:1169 +0x8b
created by k8s.io/klog/v2.init.0
    /go/src/loggie.io/loggie/vendor/k8s.io/klog/v2/klog.go:420 +0xdf

goroutine 131 [chan receive]:

Steps to Reproduce the Problem

  1. create a normal logconfig collecting stdout log from pods 
    apiVersion: v1
items:
- apiVersion: loggie.io/v1beta1
kind: LogConfig
metadata:
name: horizon
namespace: horizon
spec:
pipeline:
  sources: |
    - type: file
      name: mylog
      fields:
        logType: horizon
        topic: filebeat-perftest-temp1
      paths:
      - stdout
selector:
  labelSelector:
    cloudnative.music.netease.com/cluster: horizon-backend-core
  type: pod
kind: List
metadata:
resourceVersion: ""
selfLink: ""
  2. problem happens when a normal restart of the pods
ethfoo commented 2 years ago

find another:

fatal error: concurrent map iteration and map write

goroutine 346 [running]:
runtime.throw(0x21ac606, 0x26)
        /usr/local/go/src/runtime/panic.go:1117 +0x72 fp=0xc000a76e48 sp=0xc000a76e18 pc=0x440b52
runtime.mapiternext(0xc000b069c0)
        /usr/local/go/src/runtime/map.go:858 +0x54c fp=0xc000a76ec8 sp=0xc000a76e48 pc=0x4196cc
reflect.mapiternext(0xc000b069c0)
        /usr/local/go/src/runtime/map.go:1346 +0x2b fp=0xc000a76ee0 sp=0xc000a76ec8 pc=0x47422b
reflect.Value.MapKeys(0x1e752c0, 0xc0003c5cc8, 0x195, 0xc0003c5cc8, 0x195, 0x1e752c0)
        /usr/local/go/src/reflect/value.go:1227 +0x10c fp=0xc000a76f70 sp=0xc000a76ee0 pc=0x4abe0c
github.com/google/go-cmp/cmp.(*state).compareMap(0xc0001d6280, 0x24e57e8, 0x1e752c0, 0x1e752c0, 0xc0003c5cc8, 0x195, 0x1e752c0, 0xc0006c60a8, 0x195)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:522 +0x27d fp=0xc000a77098 sp=0xc000a76f70 pc=0x8503fd
github.com/google/go-cmp/cmp.(*state).compareAny(0xc0001d6280, 0x24badc0, 0xc000a8fb00)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:289 +0xc46 fp=0xc000a77280 sp=0xc000a77098 pc=0x84d146
github.com/google/go-cmp/cmp.(*state).compareStruct(0xc0001d6280, 0x24e57e8, 0x2006000, 0x2006000, 0xc0003c5c80, 0x199, 0x2006000, 0xc0006c6060, 0x199)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:410 +0x685 fp=0xc000a77418 sp=0xc000a77280 pc=0x84f105
github.com/google/go-cmp/cmp.(*state).compareAny(0xc0001d6280, 0x24bad90, 0xc000b06180)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:285 +0xe38 fp=0xc000a77600 sp=0xc000a77418 pc=0x84d338
github.com/google/go-cmp/cmp.(*state).statelessCompare(0xc0001d6280, 0x24bad90, 0xc000b06180, 0x1d9c1a0, 0x1)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:228 +0x97 fp=0xc000a77658 sp=0xc000a77600 pc=0x84c477
github.com/google/go-cmp/cmp.(*state).compareSlice.func2(0x0, 0x0, 0x0, 0x3317968)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:476 +0x85 fp=0xc000a77698 sp=0xc000a77658 pc=0x857fc5
github.com/google/go-cmp/cmp/internal/diff.Difference(0x1, 0x1, 0xc0008af948, 0x1, 0x1, 0xc00094a180)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/internal/diff/diff.go:241 +0x271 fp=0xc000a777d8 sp=0xc000a77698 pc=0x8479b1
github.com/google/go-cmp/cmp.(*state).compareSlice(0xc0001d6280, 0x24e57e8, 0x1d4a1c0, 0x1d4a1c0, 0xc000b04828, 0x97, 0x1d4a1c0, 0xc000b04840, 0x97)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:475 +0x734 fp=0xc000a779a0 sp=0xc000a777d8 pc=0x84fab4
github.com/google/go-cmp/cmp.(*state).compareAny(0xc0001d6280, 0x24afd80, 0xc000c3b080)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:287 +0x945 fp=0xc000a77b88 sp=0xc000a779a0 pc=0x84ce45
github.com/google/go-cmp/cmp.Equal(0x1d4a1c0, 0xc000b04828, 0x1d4a1c0, 0xc000b04840, 0xc000b08040, 0x1, 0x1, 0x7fd6dc33c658)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:93 +0x9d fp=0xc000a77bd0 sp=0xc000a77b88 pc=0x84b55d
github.com/loggie-io/loggie/pkg/core/reloader.diffConfig.func1(0xc0003c5c80, 0x1, 0x1, 0xc0006c6060, 0x1, 0x1, 0x0)
        /go/src/loggie.io/loggie/pkg/core/reloader/reload.go:126 +0x125 fp=0xc000a77c40 sp=0xc000a77bd0 pc=0x8604c5
runtime.call64(0xc000563d70, 0x2249e80, 0xc000c3af40, 0x3000000038)
        /usr/local/go/src/runtime/asm_amd64.s:552 +0x3e fp=0xc000a77c90 sp=0xc000a77c40 pc=0x47903e
reflect.Value.call(0x1e312e0, 0x2249e80, 0x13, 0x2176f76, 0x4, 0xc000563d40, 0x2, 0x2, 0x100000000000000, 0x0, ...)
        /usr/local/go/src/reflect/value.go:476 +0x8e7 fp=0xc000a77e98 sp=0xc000a77c90 pc=0x4a8887
reflect.Value.Call(0x1e312e0, 0x2249e80, 0x13, 0xc000563d40, 0x2, 0x2, 0x0, 0x0, 0x0)
        /usr/local/go/src/reflect/value.go:337 +0xb9 fp=0xc000a77f18 sp=0xc000a77e98 pc=0x4a7d59
github.com/google/go-cmp/cmp.detectRaces(0xc000cb5bc0, 0x1e312e0, 0x2249e80, 0x13, 0xc000563d40, 0x2, 0x2)
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:368 +0xcc fp=0xc000a77fa8 sp=0xc000a77f18 pc=0x84e9ec
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc000a77fb0 sp=0xc000a77fa8 pc=0x47a961
created by github.com/google/go-cmp/cmp.(*state).callTTBFunc
        /go/src/loggie.io/loggie/vendor/github.com/google/go-cmp/cmp/compare.go:353 +0x26e