Panic at 'Storage root must match that calculated'

[johnwstanford]

Environment: Ubuntu 21.04, Rust 1.58 (stable)

My node was synched with the Polkadot relay chain about a week ago. I stopped running my node for several days, did a git pull to get to commit 88d66ac6, did cargo run --release, and got the following output. I don't know exactly which commit I was on before, but it was less than a month old.

2022-03-09 22:49:12 ✨ Imported #9363260 (0xa71e…5327)    
Hash: given=892b595c7c9bf45cc108ae4f815dbd27a229c2c7e4457ccc26f9ec2fc418f5bd, expected=1b16e5eebde14b080d7d6b15bf7d052feae10bd38af3975eeea106f580d574e8

====================

Version: 0.9.17-88d66ac669-x86_64-linux-gnu

   0: sp_panic_handler::set::{{closure}}
   1: std::panicking::rust_panic_with_hook
             at /rustc/db9d1b20bba1968c1ec1fc49616d4742c1725b4b/library/std/src/panicking.rs:610:17
   2: std::panicking::begin_panic_handler::{{closure}}
             at /rustc/db9d1b20bba1968c1ec1fc49616d4742c1725b4b/library/std/src/panicking.rs:500:13
   3: std::sys_common::backtrace::__rust_end_short_backtrace
             at /rustc/db9d1b20bba1968c1ec1fc49616d4742c1725b4b/library/std/src/sys_common/backtrace.rs:139:18
   4: rust_begin_unwind
             at /rustc/db9d1b20bba1968c1ec1fc49616d4742c1725b4b/library/std/src/panicking.rs:498:5
   5: core::panicking::panic_fmt
             at /rustc/db9d1b20bba1968c1ec1fc49616d4742c1725b4b/library/core/src/panicking.rs:107:14
   6: frame_executive::Executive<System,Block,Context,UnsignedValidator,AllPalletsWithSystem,COnRuntimeUpgrade>::final_checks
   7: tracing::span::Span::in_scope
   8: frame_executive::Executive<System,Block,Context,UnsignedValidator,AllPalletsWithSystem,COnRuntimeUpgrade>::execute_block
   9: <polkadot_runtime::Runtime as sp_api::runtime_decl_for_Core::Core<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u32,sp_runtime::traits::BlakeTwo256>,sp_runtime::generic::unchecked_extrinsic::UncheckedExtrinsic<sp_runtime::multiaddress::MultiAddress<<<sp_runtime::MultiSignature as sp_runtime::traits::Verify>::Signer as sp_runtime::traits::IdentifyAccount>::AccountId,()>,polkadot_runtime::Call,sp_runtime::MultiSignature,(frame_system::extensions::check_non_zero_sender::CheckNonZeroSender<polkadot_runtime::Runtime>,frame_system::extensions::check_spec_version::CheckSpecVersion<polkadot_runtime::Runtime>,frame_system::extensions::check_tx_version::CheckTxVersion<polkadot_runtime::Runtime>,frame_system::extensions::check_genesis::CheckGenesis<polkadot_runtime::Runtime>,frame_system::extensions::check_mortality::CheckMortality<polkadot_runtime::Runtime>,frame_system::extensions::check_nonce::CheckNonce<polkadot_runtime::Runtime>,frame_system::extensions::check_weight::CheckWeight<polkadot_runtime::Runtime>,pallet_transaction_payment::ChargeTransactionPayment<polkadot_runtime::Runtime>,polkadot_runtime_common::claims::PrevalidateAttests<polkadot_runtime::Runtime>)>>>>::execute_block
  10: std::panicking::try
  11: std::thread::local::LocalKey<T>::with
  12: sc_executor::native_executor::WasmExecutor<H>::with_instance::{{closure}}
  13: sc_executor::wasm_runtime::RuntimeCache::with_instance
  14: <sc_executor::native_executor::NativeElseWasmExecutor<D> as sp_core::traits::CodeExecutor>::call
  15: sp_state_machine::execution::StateMachine<B,H,Exec>::execute_aux
  16: sp_state_machine::execution::StateMachine<B,H,Exec>::execute_using_consensus_failure_handler
  17: <sc_service::client::call_executor::LocalCallExecutor<Block,B,E> as sc_client_api::call_executor::CallExecutor<Block>>::contextual_call
  18: <sc_service::client::client::Client<B,E,Block,RA> as sp_api::CallApiAt<Block>>::call_api_at
  19: <polkadot_runtime::RuntimeApiImpl<__SR_API_BLOCK__,RuntimeApiImplCall> as sp_api::Core<__SR_API_BLOCK__>>::Core_execute_block_runtime_api_impl
  20: sp_api::Core::execute_block_with_context
  21: sc_service::client::client::Client<B,E,Block,RA>::prepare_block_storage_changes
  22: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
  23: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
  24: <sc_consensus_babe::BabeBlockImport<Block,Client,Inner> as sc_consensus::block_import::BlockImport<Block>>::import_block::{{closure}}
  25: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
  26: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
  27: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
  28: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
  29: <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll
  30: <sc_service::task_manager::prometheus_future::PrometheusFuture<T> as core::future::future::Future>::poll
  31: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
  32: tokio::park::thread::CachedParkThread::block_on
  33: tokio::runtime::handle::Handle::block_on
  34: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
  35: tokio::runtime::task::harness::Harness<T,S>::poll
  36: tokio::runtime::blocking::pool::Inner::run
  37: std::sys_common::backtrace::__rust_begin_short_backtrace
  38: core::ops::function::FnOnce::call_once{{vtable.shim}}
  39: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/db9d1b20bba1968c1ec1fc49616d4742c1725b4b/library/alloc/src/boxed.rs:1694:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/db9d1b20bba1968c1ec1fc49616d4742c1725b4b/library/alloc/src/boxed.rs:1694:9
      std::sys::unix::thread::Thread::new::thread_start
             at /rustc/db9d1b20bba1968c1ec1fc49616d4742c1725b4b/library/std/src/sys/unix/thread.rs:106:17
  40: start_thread
             at ./nptl/./nptl/pthread_create.c:473:8
  41: clone

Thread 'tokio-runtime-worker' panicked at 'Storage root must match that calculated.', /home/john/.cargo/git/checkouts/substrate-7e08433d4c370a21/32a4fe0/frame/executive/src/lib.rs:488

This is a bug. Please report it at:

    https://github.com/paritytech/polkadot/issues/new

[rphmeier]

cc @bkchr

could you try running with --state-cache-size=0? There is a known issue

[bkchr]

This is no state cache issue. The problem is that the mentioned commit is not the latest release, just some commit from master. Then they use the native runtime and that fails.

@johnwstanford either you always run the releases or you run you node with --execution wasm.

[johnwstanford]

Yeah, if I do cargo run --release -- --execution wasm, it works. Thanks.