CI Failure (Assertion in feature table) in `ControlCharacterNag.test_validate_nag_message`

vbotbuildovich commented 2 months ago

https://buildkite.com/redpanda/redpanda/builds/48164

Module: rptest.tests.control_character_flag_test
Class: ControlCharacterNag
Method: test_validate_nag_message
Arguments: {
    "initial_version": [
        22,
        2,
        9
    ]
}

test_id:    ControlCharacterNag.test_validate_nag_message
status:     FAIL
run time:   149.882 seconds

TimeoutError('Redpanda service docker-rp-21 failed to start within 20 sec')
Traceback (most recent call last):
  File "/opt/.ducktape-venv/lib/python3.10/site-packages/ducktape/tests/runner_client.py", line 184, in _do_run
    data = self.run_test()
  File "/opt/.ducktape-venv/lib/python3.10/site-packages/ducktape/tests/runner_client.py", line 276, in run_test
    return self.test_context.function(self.test)
  File "/opt/.ducktape-venv/lib/python3.10/site-packages/ducktape/mark/_mark.py", line 535, in wrapper
    return functools.partial(f, *args, **kwargs)(*w_args, **w_kwargs)
  File "/root/tests/rptest/services/cluster.py", line 103, in wrapped
    r = f(self, *args, **kwargs)
  File "/root/tests/rptest/tests/control_character_flag_test.py", line 150, in test_validate_nag_message
    self._perform_update(initial_version, RedpandaInstaller.HEAD)
  File "/root/tests/rptest/tests/control_character_flag_test.py", line 51, in _perform_update
    self.redpanda.restart_nodes(self.redpanda.nodes)
  File "/root/tests/rptest/services/redpanda.py", line 1263, in restart_nodes
    list(
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 621, in result_iterator
    yield _result_or_cancel(fs.pop())
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 319, in _result_or_cancel
    return fut.result(timeout)
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 458, in result
    return self.__get_result()
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
    raise self._exception
  File "/usr/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/root/tests/rptest/services/redpanda.py", line 1265, in <lambda>
    lambda n: self.start_node(
  File "/root/tests/rptest/services/redpanda.py", line 2844, in start_node
    self.start_service(node, start_rp)
  File "/root/tests/rptest/services/redpanda.py", line 2920, in start_service
    start()
  File "/root/tests/rptest/services/redpanda.py", line 2835, in start_rp
    wait_until(
ducktape.errors.TimeoutError: Redpanda service docker-rp-21 failed to start within 20 sec

JIRA Link: CORE-2631

vbotbuildovich commented 2 months ago

*https://buildkite.com/redpanda/redpanda/builds/48200

michael-redpanda commented 2 months ago

Actual error in startup of Redpanda:

ERROR 2024-04-23 15:48:41,167 [shard 0] assert - Assert failure: (/var/lib/buildkite-agent/builds/buildkite-amd64-builders-i-0e1b920f37ef21713-1/redpanda/redpanda/src/v/features/feature_table.cc:655) 'false' Attempted to upgrade from incompatible logical version 5 to logical version 10!
ERROR 2024-04-23 15:48:41,167 [shard 0] assert - Backtrace below:
0x7243233 0x5f9901f 0x216d043 0x2167bf5 0x216d9f9 0x217a2e8 0x70807ab
   --------
   seastar::continuation<seastar::internal::promise_base_with_type<int>, seastar::futurize<std::__1::invoke_result<application::run(int, char**)::$_0::operator()() const::'lambda'()>::type>::type seastar::async<application::run(int, char**)::$_0::operator()() const::'lambda'()>(seastar::thread_attributes, application::run(int, char**)::$_0::operator()() const::'lambda'()&&)::'lambda0'(), seastar::future<int> seastar::future<void>::then_impl_nrvo<seastar::futurize<std::__1::invoke_result<application::run(int, char**)::$_0::operator()() const::'lambda'()>::type>::type seastar::async<application::run(int, char**)::$_0::operator()() const::'lambda'()>(seastar::thread_attributes, application::run(int, char**)::$_0::operator()() const::'lambda'()&&)::'lambda0'(), seastar::future<int>>(application::run(int, char**)::$_0::operator()() const::'lambda'()&&)::'lambda'(seastar::internal::promise_base_with_type<int>&&, seastar::futurize<std::__1::invoke_result<application::run(int, char**)::$_0::operator()() const::'lambda'()>::type>::type seastar::async<application::run(int, char**)::$_0::operator()() const::'lambda'()>(seastar::thread_attributes, application::run(int, char**)::$_0::operator()() const::'lambda'()&&)::'lambda0'()&, seastar::future_state<seastar::internal::monostate>&&), void>
   --------
   seastar::continuation<seastar::internal::promise_base_with_type<int>, seastar::future<int>::finally_body<seastar::futurize<std::__1::invoke_result<application::run(int, char**)::$_0::operator()() const::'lambda'()>::type>::type seastar::async<application::run(int, char**)::$_0::operator()() const::'lambda'()>(seastar::thread_attributes, application::run(int, char**)::$_0::operator()() const::'lambda'()&&)::'lambda1'(), false>, seastar::futurize<application::run(int, char**)::$_0::operator()() const::'lambda'()>::type seastar::future<int>::then_wrapped_nrvo<seastar::future<int>, seastar::future<int>::finally_body<seastar::futurize<std::__1::invoke_result<application::run(int, char**)::$_0::operator()() const::'lambda'()>::type>::type seastar::async<application::run(int, char**)::$_0::operator()() const::'lambda'()>(seastar::thread_attributes, application::run(int, char**)::$_0::operator()() const::'lambda'()&&)::'lambda1'(), false>>(seastar::future<int>::finally_body<seastar::futurize<std::__1::invoke_result<application::run(int, char**)::$_0::operator()() const::'lambda'()>::type>::type seastar::async<application::run(int, char**)::$_0::operator()() const::'lambda'()>(seastar::thread_attributes, application::run(int, char**)::$_0::operator()() const::'lambda'()&&)::'lambda1'(), false>&&)::'lambda'(seastar::internal::promise_base_with_type<int>&&, seastar::future<int>::finally_body<seastar::futurize<std::__1::invoke_result<application::run(int, char**)::$_0::operator()() const::'lambda'()>::type>::type seastar::async<application::run(int, char**)::$_0::operator()() const::'lambda'()>(seastar::thread_attributes, application::run(int, char**)::$_0::operator()() const::'lambda'()&&)::'lambda1'(), false>&, seastar::future_state<int>&&), int>
   --------
   seastar::continuation<seastar::internal::promise_base_with_type<int>, seastar::future<int>::finally_body<seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&)::$_0::operator()()::'lambda0'(), false>, seastar::futurize<seastar::future<int>>::type seastar::future<int>::then_wrapped_nrvo<seastar::future<int>, seastar::future<int>::finally_body<seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&)::$_0::operator()()::'lambda0'(), false>>(seastar::future<int>::finally_body<seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&)::$_0::operator()()::'lambda0'(), false>&&)::'lambda'(seastar::internal::promise_base_with_type<int>&&, seastar::future<int>::finally_body<seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&)::$_0::operator()()::'lambda0'(), false>&, seastar::future_state<int>&&), int>
   --------
   seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&)::$_0::operator()()::'lambda'(int), seastar::future<void> seastar::future<int>::then_impl_nrvo<seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&)::$_0::operator()()::'lambda'(int), seastar::future<void>>(seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&)::$_0::operator()()::'lambda'(int)&&)::'lambda'(seastar::internal::promise_base_with_type<void>&&, seastar::app_template::run(int, char**, std::__1::function<seastar::future<int> ()>&&)::$_0::operator()()::'lambda'(int)&, seastar::future_state<int>&&), int>
   --------
   seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::future<void>::or_terminate()::'lambda'(auto&&), seastar::futurize<auto>::type seastar::future<void>::then_wrapped_nrvo<void, seastar::future<void>::or_terminate()::'lambda'(auto&&)>(seastar::future<void>::or_terminate()::'lambda'(auto&&)&&)::'lambda'(seastar::internal::promise_base_with_type<void>&&, seastar::future<void>::or_terminate()::'lambda'(auto&&)&, seastar::future_state<seastar::internal::monostate>&&), void>

michael-redpanda commented 2 months ago

Definite test issue. @michael-redpanda to investigate