huoshenyingmu
commented
1 year ago task配置： "task": { "name": "test_dynamic_schema", "kafkaClient": "sarama", "topic": "user_opt_log", "earliest": true, "consumerGroup": "group",
"parser": "json",
"tableName": "user_opt_log_lt",
"dims": [
{
"name": "timestamp",
"type": "DateTime"
},
{
"name": "name",
"type": "String"
},
{
"name": "value",
"type": "Float32",
"sourcename": "val"
}
],
"autoSchema" : true,
"excludeColumns": [],
{"level":"error","ts":"2022-09-26T15:56:45.372+0800","msg":"flush batch failed","task":"test_dynamic_schema","try":0,"error":"driver.Batch.Send: code: 16, message: No such column loginType in table data_warehouse.user_opt_log_lt_local (33bae6fc-18b4-4d18-a863-1cbcbcf767ee)","errorVerbose":"driver.Batch.Send\n> github.com/housepower/clickhouse_sinker/output.writeRows\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse_util.go:80\n> github.com/housepower/clickhouse_sinker/output.(ClickHouse).write\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse.go:160\n> github.com/housepower/clickhouse_sinker/output.(ClickHouse).loopWrite\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse.go:179\n> github.com/housepower/clickhouse_sinker/output.(ClickHouse).Send.func1\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse.go:104\n> github.com/housepower/clickhouse_sinker/util.(WorkerPool).wokerFunc\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/util/workerpool.go:59\n> runtime.goexit\t/usr/local/go/src/runtime/asm_amd64.s:1571\ncode: 16, message: No such column loginType in table data_warehouse.user_opt_log_lt_local (33bae6fc-18b4-4d18-a863-1cbcbcf767ee)\n","stacktrace":"github.com/housepower/clickhouse_sinker/output.(ClickHouse).loopWrite\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse.go:194\ngithub.com/housepower/clickhouse_sinker/output.(ClickHouse).Send.func1\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse.go:104\ngithub.com/housepower/clickhouse_sinker/util.(WorkerPool).wokerFunc\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/util/workerpool.go:59"} {"level":"error","ts":"2022-09-26T15:56:45.372+0800","msg":"this is an exception from clickhouse-server","replica":"172.21.2.210:9000","exception":{"Code":16,"Name":"DB::Exception","Message":"No such column loginType in table data_warehouse.user_opt_log_lt_local (33bae6fc-18b4-4d18-a863-1cbcbcf767ee)","StackTrace":"0. DB::Exception::Exception(std::1::basic_string<char, std::__1::char_traits, std:: 1::allocator > const&, int, bool) @ 0xb6fa67a in /usr/bin/clickhouse\n1. DB::InterpreterInsertQuery::getSampleBlock(std::1::vector<std::1::basic_string<char, std::1::char_traits, std::1::allocator >, std:: 1::allocator<std::__1::basic_string<char, std:: 1::char_traits, std::1::allocator > > > const&, std:: 1::shared_ptr const&, std::1::shared_ptr const&) const @ 0x160e139d in /usr/bin/clickhouse\n2. DB::InterpreterInsertQuery::getSampleBlock(DB::ASTInsertQuery const&, std::__1::shared_ptr const&, std:: 1::shared_ptr const&) const @ 0x160e0c00 in /usr/bin/clickhouse\n3. DB::InterpreterInsertQuery::execute() @ 0x160e2a5f in /usr/bin/clickhouse\n4. DB::DistributedSink::writeToLocal(DB::Block const&, unsigned long) @ 0x1695536e in /usr/bin/clickhouse\n5. DB::DistributedSink::writeAsyncImpl(DB::Block const&, unsigned long) @ 0x169521a3 in /usr/bin/clickhouse\n6. DB::DistributedSink::writeSplitAsync(DB::Block const&) @ 0x169527f2 in /usr/bin/clickhouse\n7. DB::DistributedSink::consume(DB::Chunk) @ 0x16950ad7 in /usr/bin/clickhouse\n8. DB::SinkToStorage::onConsume(DB::Chunk) @ 0x17288c9d in /usr/bin/clickhouse\n9. ? @ 0x1720a397 in /usr/bin/clickhouse\n10. ? @ 0x1720a149 in /usr/bin/clickhouse\n11. DB::ExceptionKeepingTransform::work() @ 0x172099ff in /usr/bin/clickhouse\n12. DB::ExecutionThreadContext::executeTask() @ 0x17051548 in /usr/bin/clickhouse\n13. DB::PipelineExecutor::executeStepImpl(unsigned long, std::__1::atomic ) @ 0x17044bbe in /usr/bin/clickhouse\n14. DB::PipelineExecutor::executeStep(std::__1::atomic) @ 0x170443c0 in /usr/bin/clickhouse\n15. DB::TCPHandler::processInsertQuery() @ 0x16fea77c in /usr/bin/clickhouse\n16. DB::TCPHandler::runImpl() @ 0x16fe3620 in /usr/bin/clickhouse\n17. DB::TCPHandler::run() @ 0x16ff39d9 in /usr/bin/clickhouse\n18. Poco::Net::TCPServerConnection::start() @ 0x1b3e44af in /usr/bin/clickhouse\n19. Poco::Net::TCPServerDispatcher::run() @ 0x1b3e6901 in /usr/bin/clickhouse\n20. Poco::PooledThread::run() @ 0x1b5ad349 in /usr/bin/clickhouse\n21. Poco::ThreadImpl::runnableEntry(void) @ 0x1b5aa6a0 in /usr/bin/clickhouse\n22. start_thread @ 0x7ea5 in /usr/lib64/libpthread-2.17.so\n23. clone @ 0xfe9fd in /usr/lib64/libc-2.17.so\n","Nested":null},"stacktrace":"github.com/housepower/clickhouse_sinker/output.shouldReconnect\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse_util.go:20\ngithub.com/housepower/clickhouse_sinker/output.(ClickHouse).loopWrite\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse.go:197\ngithub.com/housepower/clickhouse_sinker/output.(ClickHouse).Send.func1\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse.go:104\ngithub.com/housepower/clickhouse_sinker/util.(WorkerPool).wokerFunc\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/util/workerpool.go:59"}
{"level":"fatal","ts":"2022-09-26T15:56:45.372+0800","msg":"ClickHouse.loopWrite failed","task":"test_dynamic_schema","stacktrace":"github.com/housepower/clickhouse_sinker/output.(ClickHouse).loopWrite\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse.go:201\ngithub.com/housepower/clickhouse_sinker/output.(ClickHouse).Send.func1\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/output/clickhouse.go:104\ngithub.com/housepower/clickhouse_sinker/util.(WorkerPool).wokerFunc\n\t/home/zhichyu/go/src/github.com/housepower/clickhouse_sinker/util/workerpool.go:59"}