Closed zjma closed 1 week ago
🚨 3 jobs on the last run were significantly faster/slower than expected
Job | Duration | vs 7d avg | Delta |
---|---|---|---|
rust-images / rust-all | 17m | 13m | |
run-tests-main-branch | 5m | 4m | |
rust-move-tests | 13m | 11m |
framework_upgrade
on aptos-release-v1-12
==> 143dc1e8a121a3681658d685d654afc6cd04d2a3
framework_upgrade
failure on aptos-release-v1-12
==> 143dc1e8a121a3681658d685d654afc6cd04d2a3
Compatibility test results for aptos-release-v1-12 ==> 143dc1e8a121a3681658d685d654afc6cd04d2a3 (PR)
Upgrade the nodes to version: 143dc1e8a121a3681658d685d654afc6cd04d2a3
Test Failed: Tried executing 10 txns, request counters: "success 0, failed submit [0, 10, 10, 10, 10, 10, 0, 10, 10, 0, 0, 0, 10, 0, 10, 10, 0, 10], failed wait [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10], by client: [(0, 60, 60): http://aptos-node-1-validator.forge-framework-upgrade-pr-13330.svc:8080/v1/]\n[(0, 0, 30): http://aptos-node-2-validator.forge-framework-upgrade-pr-13330.svc:8080/v1/]\n[(0, 0, 40): http://aptos-node-3-validator.forge-framework-upgrade-pr-13330.svc:8080/v1/]\n[(0, 50, 50): http://aptos-node-0-validator.forge-framework-upgrade-pr-13330.svc:8080/v1/]"
Caused by:
Unknown error Ledger on endpoint (http://aptos-node-0-validator.forge-framework-upgrade-pr-13330.svc:8080/v1/) is more than 60s behind current time, timing out waiting for the transaction. Warning, transaction (452495ef) might still succeed.
Stack backtrace:
0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.79/src/error.rs:565:25
1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/result.rs:1963:27
2: aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter::submit_check_and_retry::{{closure}}
at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:127:28
3: <futures_util::future::maybe_done::MaybeDone<Fut> as core::future::future::Future>::poll
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/maybe_done.rs:95:38
4: <futures_util::future::join_all::JoinAll<F> as core::future::future::Future>::poll
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/join_all.rs:143:24
5: <aptos_transaction_emitter_lib::emitter::transaction_executor::RestApiReliableTransactionSubmitter as aptos_transaction_generator_lib::ReliableTransactionSubmitter>::execute_transactions_with_counter::{{closure}}
at ./crates/transaction-emitter-lib/src/emitter/transaction_executor.rs:309:10
6: <core::pin::Pin<P> as core::future::future::Future>::poll
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/future/future.rs:125:9
7: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_seed_accounts::{{closure}}
at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:434:18
8: aptos_transaction_emitter_lib::emitter::account_minter::AccountMinter::create_and_fund_accounts::{{closure}}
at ./crates/transaction-emitter-lib/src/emitter/account_minter.rs:326:14
9: aptos_transaction_emitter_lib::emitter::create_accounts::{{closure}}
at ./crates/transaction-emitter-lib/src/emitter/mod.rs:1195:14
10: aptos_transaction_emitter_lib::emitter::TxnEmitter::start_job::{{closure}}
at ./crates/transaction-emitter-lib/src/emitter/mod.rs:717:10
11: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for_impl::{{closure}}
at ./crates/transaction-emitter-lib/src/emitter/mod.rs:827:14
12: aptos_transaction_emitter_lib::emitter::TxnEmitter::emit_txn_for::{{closure}}
at ./crates/transaction-emitter-lib/src/emitter/mod.rs:859:14
13: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.35.1/src/runtime/park.rs:282:63
14: tokio::runtime::coop::with_budget
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.35.1/src/runtime/coop.rs:107:5
15: tokio::runtime::coop::budget
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.35.1/src/runtime/coop.rs:73:5
16: tokio::runtime::park::CachedParkThread::block_on
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.35.1/src/runtime/park.rs:282:31
17: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.35.1/src/runtime/context/blocking.rs:66:9
18: tokio::runtime::scheduler::multi_thread::MultiThread::block_on::{{closure}}
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.35.1/src/runtime/scheduler/multi_thread/mod.rs:87:13
19: tokio::runtime::context::runtime::enter_runtime
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.35.1/src/runtime/context/runtime.rs:65:16
20: tokio::runtime::scheduler::multi_thread::MultiThread::block_on
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.35.1/src/runtime/scheduler/multi_thread/mod.rs:86:9
21: tokio::runtime::runtime::Runtime::block_on
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.35.1/src/runtime/runtime.rs:350:50
22: aptos_testcases::generate_traffic
at ./testsuite/testcases/src/lib.rs:105:17
23: <aptos_testcases::framework_upgrade::FrameworkUpgrade as aptos_forge::interface::network::NetworkTest>::run
at ./testsuite/testcases/src/framework_upgrade.rs:67:24
24: aptos_forge::runner::Forge<F>::run::{{closure}}
at ./testsuite/forge/src/runner.rs:598:42
25: aptos_forge::runner::run_test
at ./testsuite/forge/src/runner.rs:666:11
26: aptos_forge::runner::Forge<F>::run
at ./testsuite/forge/src/runner.rs:598:30
27: forge::run_forge
at ./testsuite/forge-cli/src/main.rs:427:11
28: forge::main
at ./testsuite/forge-cli/src/main.rs:353:21
29: core::ops::function::FnOnce::call_once
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/ops/function.rs:250:5
30: std::sys_common::backtrace::__rust_begin_short_backtrace
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:154:18
31: std::rt::lang_start::{{closure}}
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:167:18
32: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/ops/function.rs:284:13
33: std::panicking::try::do_call
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:552:40
34: std::panicking::try
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:516:19
35: std::panic::catch_unwind
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panic.rs:142:14
36: std::rt::lang_start_internal::{{closure}}
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:148:48
37: std::panicking::try::do_call
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:552:40
38: std::panicking::try
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:516:19
39: std::panic::catch_unwind
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panic.rs:142:14
40: std::rt::lang_start_internal
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:148:20
41: std::rt::lang_start
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:166:17
42: __libc_start_main
43: _start
Trailing Log Lines:
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:516:19
39: std::panic::catch_unwind
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panic.rs:142:14
40: std::rt::lang_start_internal
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:148:20
41: std::rt::lang_start
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:166:17
42: __libc_start_main
43: _start
Swarm logs can be found here: See fgi output for more information.
{"level":"INFO","source":{"package":"aptos_forge","file":"testsuite/forge/src/backend/k8s/cluster_helper.rs:292"},"thread_name":"main","hostname":"forge-framework-upgrade-pr-13330-1715989535-aptos-release-v1-12","timestamp":"2024-05-17T23:54:28.293852Z","message":"Deleting namespace forge-framework-upgrade-pr-13330: Some(NamespaceStatus { conditions: None, phase: Some(\"Terminating\") })"}
{"level":"INFO","source":{"package":"aptos_forge","file":"testsuite/forge/src/backend/k8s/cluster_helper.rs:400"},"thread_name":"main","hostname":"forge-framework-upgrade-pr-13330-1715989535-aptos-release-v1-12","timestamp":"2024-05-17T23:54:28.293871Z","message":"aptos-node resources for Forge removed in namespace: forge-framework-upgrade-pr-13330"}
Failed to run tests:
Tests Failed
failures:
framework_upgrade::framework-upgrade
test result: FAILED. 0 passed; 1 failed; 0 filtered out
Error: Tests Failed
Stack backtrace:
0: anyhow::error::<impl anyhow::Error>::msg
at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.79/src/error.rs:83:36
1: aptos_forge::runner::Forge<F>::run
at ./testsuite/forge/src/runner.rs:618:13
2: forge::run_forge
at ./testsuite/forge-cli/src/main.rs:427:11
3: forge::main
at ./testsuite/forge-cli/src/main.rs:353:21
4: core::ops::function::FnOnce::call_once
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/ops/function.rs:250:5
5: std::sys_common::backtrace::__rust_begin_short_backtrace
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/sys_common/backtrace.rs:154:18
6: std::rt::lang_start::{{closure}}
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:167:18
7: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/core/src/ops/function.rs:284:13
8: std::panicking::try::do_call
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:552:40
9: std::panicking::try
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:516:19
10: std::panic::catch_unwind
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panic.rs:142:14
11: std::rt::lang_start_internal::{{closure}}
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:148:48
12: std::panicking::try::do_call
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:552:40
13: std::panicking::try
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panicking.rs:516:19
14: std::panic::catch_unwind
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/panic.rs:142:14
15: std::rt::lang_start_internal
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:148:20
16: std::rt::lang_start
at /rustc/82e1608dfa6e0b5569232559e3d385fea5a93112/library/std/src/rt.rs:166:17
17: __libc_start_main
18: _start
Debugging output:
NAME READY STATUS RESTARTS AGE
aptos-node-0-validator-0 1/1 Running 0 5m33s
aptos-node-1-validator-0 1/1 Running 0 4m43s
aptos-node-2-validator-0 1/1 Running 0 7m54s
aptos-node-3-validator-0 1/1 Running 0 7m54s
genesis-aptos-genesis-eforge187-n4q8c 0/1 Completed 0 8m27s
Description
Type of Change
Which Components or Systems Does This Change Impact?
How Has This Been Tested?
Key Areas to Review
Checklist