日志组件异常

[livehl]

容器组：kuboard-loki-0 异常概况： 使用nfs，以及seaweedfs分布式存储，均出现无法flush数据报错的异常。具体日志如下： panic: page 8 already freed goroutine 1122 [running]: go.etcd.io/bbolt.(freelist).free(0xc004710300, 0xe, 0x7f25d5892000) /src/loki/vendor/go.etcd.io/bbolt/freelist.go:175 +0x2c8 go.etcd.io/bbolt.(node).spill(0xc000204930) /src/loki/vendor/go.etcd.io/bbolt/node.go:359 +0x216 go.etcd.io/bbolt.(node).spill(0xc000204770) /src/loki/vendor/go.etcd.io/bbolt/node.go:346 +0xaa go.etcd.io/bbolt.(Bucket).spill(0xc000a445c0) /src/loki/vendor/go.etcd.io/bbolt/bucket.go:570 +0x32f go.etcd.io/bbolt.(Bucket).spill(0xc001d76398) /src/loki/vendor/go.etcd.io/bbolt/bucket.go:537 +0xea go.etcd.io/bbolt.(Tx).Commit(0xc001d76380) /src/loki/vendor/go.etcd.io/bbolt/tx.go:160 +0xe7 go.etcd.io/bbolt.(DB).Update(0xc000678058, 0xc004a6f3a8) /src/loki/vendor/go.etcd.io/bbolt/db.go:748 +0xe5 github.com/grafana/loki/pkg/storage/chunk/local.(BoltIndexClient).WriteToDB(0xc000678000, {0xc0046ab490, 0xa}, 0xc003c5b4a8, {0xc0011b6b70, 0xc0011b6bd0}) /src/loki/pkg/storage/chunk/local/boltdb_index_client.go:175 +0x52 github.com/grafana/loki/pkg/storage/stores/shipper/uploads.(Table).write(0xc000678000, {0x26f86e8, 0xc0011b6cc0}, {0x0, 0x0, 0x39b03e0}, {0xc0011b6b70, 0xc0011b6bd0}) /src/loki/pkg/storage/stores/shipper/uploads/table.go:258 +0x143 github.com/grafana/loki/pkg/storage/stores/shipper/uploads.(Table).Write(0xc000626aa0, {0x26f86e8, 0xc0011b6cc0}, {0xc0011b6b70, 0xc0011b6bd0}) /src/loki/pkg/storage/stores/shipper/uploads/table.go:240 +0x6c github.com/grafana/loki/pkg/storage/stores/shipper/uploads.(TableManager).BatchWrite(0x630f665e, {0x26f86e8, 0xc0011b6cc0}, {0x26c8360, 0xc00000f920}) /src/loki/pkg/storage/stores/shipper/uploads/table_manager.go:134 +0x13e github.com/grafana/loki/pkg/storage/stores/shipper.(Shipper).BatchWrite.func1({0x26f86e8, 0xc0011b6cc0}) /src/loki/pkg/storage/stores/shipper/shipper_index_client.go:219 +0x3f github.com/weaveworks/common/instrument.CollectedRequest({0x26f86e8, 0xc0028c2bd0}, {0x223d5bb, 0x5}, {0x26d2520, 0xc00000f930}, 0x27044d8, 0xc003c5b6e0) /src/loki/vendor/github.com/weaveworks/common/instrument/instrument.go:167 +0x274 github.com/grafana/loki/pkg/storage/stores/shipper.(Shipper).BatchWrite(0xc00057cb00, {0x26f86e8, 0xc0028c2bd0}, {0x26c8360, 0xc00000f920}) /src/loki/pkg/storage/stores/shipper/shipper_index_client.go:218 +0xec github.com/grafana/loki/pkg/storage/chunk.(seriesStore).PutOne(0xc000233500, {0x26f86b0, 0xc00211b0e0}, 0x0, 0x0, {0x831b40b675432c9f, {0xc000d0005c, 0x4}, 0x182e88f70d8, 0x182f32e8ea1, ...}) /src/loki/pkg/storage/chunk/series_store.go:466 +0x3f9 github.com/grafana/loki/pkg/storage/chunk.compositeStore.Put.func1({0x26f86b0, 0xc00211b0e0}, 0x1, 0x0, {0x2735d40, 0xc000233500}) /src/loki/pkg/storage/chunk/composite_store.go:101 +0x86 github.com/grafana/loki/pkg/storage/chunk.compositeStore.forStores({{0x0, 0x0}, {0xc000bb2000, 0x1, 0x1}}, {0x26f86b0, 0xc00211b0e0}, {0xc000d0005c, 0x4}, 0x182e88f70d8, ...) /src/loki/pkg/storage/chunk/composite_store.go:254 +0x37a github.com/grafana/loki/pkg/storage/chunk.compositeStore.Put({{0x0, 0x0}, {0xc000bb2000, 0x1, 0x1}}, {0x26f86b0, 0xc00211b0e0}, {0xc002134180, 0x1, 0x1}) /src/loki/pkg/storage/chunk/composite_store.go:100 +0x178 github.com/grafana/loki/pkg/ingester.(Ingester).flushChunks(0xc000144c00, {0x26f86b0, 0xc00211b0e0}, 0xe, {0xc002130100, 0x8, 0x8}, {0xc00000f900, 0x1, 0x1}, ...) /src/loki/pkg/ingester/flush.go:379 +0x2b7 github.com/grafana/loki/pkg/ingester.(Ingester).flushUserSeries(0xc000144c00, {0xc000d0005c, 0x4}, 0x2, 0x0) /src/loki/pkg/ingester/flush.go:247 +0x1ae github.com/grafana/loki/pkg/ingester.(Ingester).flushLoop(0xc000144c00, 0xf) /src/loki/pkg/ingester/flush.go:219 +0x2d1 created by github.com/grafana/loki/pkg/ingester.(Ingester).InitFlushQueues /src/loki/pkg/ingester/flush.go:119 +0x45