search

pingcap / tiflash

The analytical engine for TiDB and TiDB Cloud. Try free: https://tidbcloud.com/free-trial
https://docs.pingcap.com/tidb/stable/tiflash-overview
Apache License 2.0
948 stars 410 forks source link

unstable test "S3DMFile.WithFileCache" #9662

Open JaySon-Huang opened 5 days ago

JaySon-Huang commented 5 days ago

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

2. What did you expect to see? (Required)

3. What did you see instead (Required)

It should be caused by the background task is not finished, but the logger is already invalid.

https://do.pingcap.net/jenkins/blue/organizations/jenkins/pingcap%2Ftiflash%2Fpull_unit_test/detail/pull_unit_test/2158/pipeline/ 

[2024/11/20 16:53:07.427 +08:00] [DEBUG] [FileCache.cpp:775] ["downloading count 1 => s3_key s17/data/t_100/dmf_1/meta start"] [source=FileCache] [thread_id=1]
[2024/11/20 16:53:07.431 +08:00] [DEBUG] [FileCache.cpp:775] ["downloading count 2 => s3_key s17/data/t_100/dmf_1/v1.meta start"] [source=FileCache] [thread_id=1]
[2024/11/20 16:53:07.433 +08:00] [DEBUG] [TiFlashTestEnv.cpp:288] ["Created bucket mockbucket"] [source="bucket=mockbucket root=tiflash_ut/"] [thread_id=1]
[2024/11/20 16:53:07.433 +08:00] [WARN] [TiFlashTestEnv.cpp:336] ["DeleteBucket: :"] [source="bucket=mockbucket root=tiflash_ut/"] [thread_id=1]
[       OK ] DMFileMetaVersion/S3DMFile.WithFileCache/0 (34 ms)
[----------] 1 test from DMFileMetaVersion/S3DMFile (34 ms total)

[----------] Global test environment tear-down
[==========] 1 test from 1 test case ran. (34 ms total)
[  PASSED  ] 1 test.
Received signal Segmentation fault
[2024/11/20 16:53:07.436 +08:00] [INFO] [SegmentReader.cpp:60] ["Pop fail, stop=true"] [thread_id=5]
[2024/11/20 16:53:07.436 +08:00] [INFO] [SegmentReader.cpp:60] ["Pop fail, stop=true"] [thread_id=4]
[2024/11/20 16:53:07.437 +08:00] [DEBUG] [SegmentReader.cpp:45] [Stopped] [thread_id=1]
[2024/11/20 16:53:07.437 +08:00] [DEBUG] [SegmentReader.cpp:45] [Stopped] [thread_id=1]
[2024/11/20 16:53:07.437 +08:00] [INFO] [SegmentReader.cpp:60] ["Pop fail, stop=true"] [thread_id=3]
[2024/11/20 16:53:07.437 +08:00] [INFO] [SegmentReader.cpp:60] ["Pop fail, stop=true"] [thread_id=2]
[2024/11/20 16:53:07.437 +08:00] [DEBUG] [SegmentReader.cpp:45] [Stopped] [thread_id=1]
[2024/11/20 16:53:07.438 +08:00] [DEBUG] [SegmentReader.cpp:45] [Stopped] [thread_id=1]

      0x121f712e    StackTrace::StackTrace() [gtests_dbms+304050478]
       0x85b3c04    fault_signal_handler(int) [gtests_dbms+140196868]
  0x7f70401595b0    <unknown symbol> [libc.so.6+320944]
       0x4c25f43    DB::Logger::is(int) const [gtests_dbms+79847235]
      0x1216d262    DB::tryLogCurrentException(std::__1::shared_ptr<DB::Logger> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&) [gtests_dbms+303485538]
      0x10a62ac2    DB::FileCache::download(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&) [gtests_dbms+279325378]
      0x10a65d33    DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1::operator()() [gtests_dbms+279338291]
      0x10a65d05    decltype(std::declval<DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1&>()()) std::__1::__invoke[abi:ue170006]<DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1&>(DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1&) [gtests_dbms+279338245]
      0x10a65cc5    void std::__1::__invoke_void_return_wrapper<void, true>::__call[abi:ue170006]<DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1&>(DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1&) [gtests_dbms+279338181]
      0x10a65c9d    std::__1::__function::__alloc_func<DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1, std::__1::allocator<DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1>, void ()>::operator()[abi:ue170006]() [gtests_dbms+279338141]
      0x10a650b9    std::__1::__function::__func<DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1, std::__1::allocator<DB::FileCache::bgDownload(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::FileSegment>&)::$_1>, void ()>::operator()() [gtests_dbms+279335097]
       0x51bf072    std::__1::__function::__value_func<void ()>::operator()[abi:ue170006]() const [gtests_dbms+85717106]
       0x51af155    std::__1::function<void ()>::operator()() const [gtests_dbms+85651797]
      0x12284f08    DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::worker(std::__1::__list_iterator<DB::ThreadFromGlobalPoolImpl<false>, void*>) [gtests_dbms+304631560]
      0x1228f514    void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()::operator()() const [gtests_dbms+304674068]
      0x1228f4e5    decltype(std::declval<void>()()) std::__1::__invoke[abi:ue170006]<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()&>(void&&) [gtests_dbms+304674021]
      0x1228f4c9    decltype(auto) std::__1::__apply_tuple_impl[abi:ue170006]<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()&, std::__1::tuple<>&>(void&&, std::__1::tuple<>&, std::__1::__tuple_indices<>) [gtests_dbms+304673993]
      0x1228f38d    decltype(auto) std::__1::apply[abi:ue170006]<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()&, std::__1::tuple<>&>(void&&, std::__1::tuple<>&) [gtests_dbms+304673677]
      0x1228f2a3    DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'()::operator()() [gtests_dbms+304673443]
      0x1228f235    decltype(std::declval<void>()()) std::__1::__invoke[abi:ue170006]<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'()&>(void&&) [gtests_dbms+304673333]
      0x1228f1f5    void std::__1::__invoke_void_return_wrapper<void, true>::__call[abi:ue170006]<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'()&>(DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'()&) [gtests_dbms+304673269]
      0x1228f1cd    std::__1::__function::__alloc_func<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), std::__1::allocator<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'()>, void ()>::operator()[abi:ue170006]() [gtests_dbms+304673229]
      0x1228e5b9    std::__1::__function::__func<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), std::__1::allocator<DB::ThreadFromGlobalPoolImpl<false>::ThreadFromGlobalPoolImpl<void DB::ThreadPoolImpl<DB::ThreadFromGlobalPoolImpl<false>>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'()>, void ()>::operator()() [gtests_dbms+304670137]
       0x51bf072    std::__1::__function::__value_func<void ()>::operator()[abi:ue170006]() const [gtests_dbms+85717106]
       0x51af155    std::__1::function<void ()>::operator()() const [gtests_dbms+85651797]
      0x12282e88    DB::ThreadPoolImpl<std::__1::thread>::worker(std::__1::__list_iterator<std::__1::thread, void*>) [gtests_dbms+304623240]
      0x12288804    void DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()::operator()() const [gtests_dbms+304646148]
      0x122887b5    decltype(std::declval<void>()()) std::__1::__invoke[abi:ue170006]<void DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(void&&) [gtests_dbms+304646069]
      0x1228878d    void std::__1::__thread_execute[abi:ue170006]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>(std::__1::tuple<void, void DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>&, std::__1::__tuple_indices<>) [gtests_dbms+304646029]
      0x122885b2    void* std::__1::__thread_proxy[abi:ue170006]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, void DB::ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, long, std::__1::optional<unsigned long>, bool)::'lambda0'()>>(void*) [gtests_dbms+304645554]
  0x7f70407011ca    start_thread [libpthread.so.0+33226]
  0x7f70401448d3    __clone [libc.so.6+235731]
[2259/2972] DMFileMetaVersion/S3DMFile.WithFileCache/0 returned/aborted with exit code -11 (4422 ms)

4. What is your TiFlash version? (Required)