search

zeta-chain / node

ZetaChain’s blockchain node and an observer validator client
https://zetachain.com
MIT License
167 stars 109 forks source link

Zeta Testnet node stops processing blocks #2902

Closed jajaislanina closed 2 months ago

jajaislanina commented 2 months ago

Which environment are you setting a node for?

Testnet

Does your machine match the technical requirements?

Yes

Have you completed the setup?

Yes

Which method are you using?

Snapshot

Did you build your binary from source or download it from the Releases page?

Downloaded from Releases page

Please, share the contents of config.toml

    proxy_app = "tcp://127.0.0.1:26658"

    # IMPORTANT! Replace the placeholder below with your actual moniker
    moniker = "my-node"

    fast_sync = true
    db_backend = "pebbledb"
    db_dir = "data"
    log_level = "info"
    log_format = "json"
    genesis_file = "config/genesis.json"
    priv_validator_key_file = "config/priv_validator_key.json"
    priv_validator_state_file = "data/priv_validator_state.json"
    priv_validator_laddr = ""
    node_key_file = "config/node_key.json"
    abci = "socket"
    filter_peers = false

    [rpc]
    laddr = "tcp://0.0.0.0:26657"
    cors_allowed_origins = []
    cors_allowed_methods = ["HEAD", "GET", "POST", ]
    cors_allowed_headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time", ]
    grpc_laddr = ""
    grpc_max_open_connections = 900
    unsafe = false
    max_open_connections = 900
    max_subscription_clients = 100
    max_subscriptions_per_client = 5
    experimental_subscription_buffer_size = 200
    experimental_websocket_write_buffer_size = 200
    experimental_close_on_slow_client = false
    timeout_broadcast_tx_commit = "10s"
    max_body_bytes = 1000000
    max_header_bytes = 1048576
    tls_cert_file = ""
    tls_key_file = ""
    pprof_laddr = "localhost:6060"

    [p2p]
    laddr = "tcp://0.0.0.0:26656"

    # IMPORTANT! Replace the placeholder below with your actual external IP address
    external_address = "{YOUR_EXTERNAL_IP_ADDRESS_HERE}:26656"

    seeds = "5aaa51a3b9465a32f7f6c9df1d46d4bfcc16aecb@34.30.34.119:26656,ade4d8bc8cbe014af6ebdf3cb7b1e9ad36f412c0@testnet-seeds.polkachu.com:22556,"
    persistent_peers = ""
    upnp = false
    addr_book_file = "config/addrbook.json"
    addr_book_strict = true
    max_num_inbound_peers = 100
    max_num_outbound_peers = 100
    unconditional_peer_ids = ""
    persistent_peers_max_dial_period = "0s"
    flush_throttle_timeout = "100ms"
    max_packet_msg_payload_size = 1024
    send_rate = 5120000
    recv_rate = 5120000
    pex = true
    seed_mode = false
    private_peer_ids = ""
    allow_duplicate_ip = false
    handshake_timeout = "20s"
    dial_timeout = "3s"

    [mempool]
    version = "v0"
    recheck = true
    broadcast = true
    wal_dir = ""

    size = 5000
    max_txs_bytes = 1073741824
    cache_size = 10000
    keep-invalid-txs-in-cache = false
    max_tx_bytes = 1048576
    max_batch_bytes = 0
    ttl-duration = "0s"
    ttl-num-blocks = 0

    [statesync]
    enable = false
    rpc_servers = ""
    trust_height = 0
    trust_hash = ""
    trust_period = "168h0m0s"
    discovery_time = "15s"
    temp_dir = ""
    chunk_request_timeout = "10s"
    chunk_fetchers = "4"

    [fastsync]
    version = "v0"

    [consensus]
    wal_file = "data/cs.wal/wal"
    timeout_propose = "3s"
    timeout_propose_delta = "500ms"
    timeout_prevote = "1s"
    timeout_prevote_delta = "500ms"
    timeout_precommit = "1s"
    timeout_precommit_delta = "500ms"
    timeout_commit = "5s"
    double_sign_check_height = 0
    skip_timeout_commit = false
    create_empty_blocks = true
    create_empty_blocks_interval = "0s"
    peer_gossip_sleep_duration = "100ms"
    peer_query_maj23_sleep_duration = "2s"

    [storage]
    discard_abci_responses = false

    [tx_index]
    indexer = "kv"
    psql-conn = ""

    [instrumentation]
    prometheus = true
    prometheus_listen_addr = ":26660"
    max_open_connections = 3
    namespace = "tendermint"

Please, share the contents of app.toml


    minimum-gas-prices = "1.0azeta"
    pruning = "nothing"
    pruning-keep-recent = "100"
    pruning-keep-every = "0"
    pruning-interval = "10"
    halt-height = 0
    halt-time = 0
    min-retain-blocks = 0
    inter-block-cache = true
    index-events = []
    iavl-cache-size = 781250
    iavl-disable-fastnode = true

    [telemetry]
    service-name = "tss"
    enabled = true
    enable-hostname = true
    enable-hostname-label = true
    enable-service-label = false
    prometheus-retention-time = 10
    global-labels = [ ]

    [api]
    enable = true
    swagger = true
    address = "tcp://0.0.0.0:1317"
    max-open-connections = 1000
    rpc-read-timeout = 10
    rpc-write-timeout = 0
    rpc-max-body-bytes = 1000000
    enabled-unsafe-cors = false

    [rosetta]
    enable = false
    address = ":8080"
    blockchain = "app"
    network = "network"
    retries = 3
    offline = false

    [grpc]
    enable = true
    address = "0.0.0.0:9090"

    [grpc-web]
    enable = true
    address = "0.0.0.0:9091"
    enable-unsafe-cors = false

    [state-sync]
    snapshot-interval = 0
    snapshot-keep-recent = 2

    [evm]
    tracer = ""
    max-tx-gas-wanted = 0

    [json-rpc]
    enable = true
    address = "0.0.0.0:8545"
    ws-address = "0.0.0.0:8546"
    api = "eth,net,web3,txpool,personal,debug"
    gas-cap = 25000000
    evm-timeout = "5s"
    txfee-cap = 1
    filter-cap = 200
    feehistory-cap = 100
    logs-cap = 10000
    block-range-cap = 10000
    http-timeout = "30s"
    http-idle-timeout = "2m0s"
    allow-unprotected-txs = false
    max-open-connections = 0
    enable-indexer = false

    [tls]
    certificate-path = ""

Additional details

Panic before the node stops

panic: self Delegation for operator not found

goroutine 1530 [running]:
github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.BlockValidatorUpdates({{0x46f4e00, 0xc001880a80}, {0x4737960, 0xc001394590}, {0x471fc00, 0xc00163a870}, {0x7d0ef4a44858, 0xc001610f20}, {0x4736a80, 0xc0043fe4c8}, ...}, ...)
    /go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.47.10/x/staking/keeper/val_state_change.go:31 +0xe0b
github.com/cosmos/cosmos-sdk/x/staking.EndBlocker({{0x471b3e0, 0x6686280}, {0x4737570, 0xc1b34bf500}, {{0xb, 0x0}, {0xc1acd6b300, 0xd}, 0x681ed2, {0x19c4e4ce, ...}, ...}, ...}, ...)
    /go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.47.10/x/staking/abci.go:26 +0x118
github.com/cosmos/cosmos-sdk/x/staking.AppModule.EndBlock(...)
    /go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.47.10/x/staking/module.go:192
github.com/cosmos/cosmos-sdk/types/module.(*Manager).EndBlock(_, {{0x471b3e0, 0x6686280}, {0x4737570, 0xc1b34bf500}, {{0xb, 0x0}, {0xc1acd6b300, 0xd}, 0x681ed2, ...}, ...}, ...)
    /go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.47.10/types/module/module.go:583 +0x1bf
github.com/zeta-chain/zetacore/app.(*App).EndBlocker(...)
    /go/delivery/zeta-node/app/app.go:873
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).EndBlock(0xc0044d8000, {0xb37cad80?})
    /go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.47.10/baseapp/abci.go:223 +0x125
github.com/cometbft/cometbft/abci/client.(*localClient).EndBlockSync(0xc00493e7e0, {0xc1b373c800?})
    /go/pkg/mod/github.com/cometbft/cometbft@v0.37.4/abci/client/local_client.go:291 +0xd9
github.com/cometbft/cometbft/proxy.(*appConnConsensus).EndBlockSync(0xc044dd62b8, {0xc1a8cd5200?})
    /go/pkg/mod/github.com/cometbft/cometbft@v0.37.4/proxy/app_conn.go:111 +0x14c
github.com/cometbft/cometbft/state.execBlockOnProxyApp({0x471b450, 0xc08cefdd90}, {0x4733860, 0xc044dd62b8}, 0xc1ad35c1e0, {0x4738188, 0xc08cf17998}, 0x1)
    /go/pkg/mod/github.com/cometbft/cometbft@v0.37.4/state/execution.go:383 +0x6b0
github.com/cometbft/cometbft/state.(*BlockExecutor).ApplyBlock(_, {{{0xb, 0x0}, {0xc08cef4798, 0x7}}, {0xc08cef47a0, 0xd}, 0x1, 0x681ed1, {{0xc1c02ff9e0, ...}, ...}, ...}, ...)
    /go/pkg/mod/github.com/cometbft/cometbft@v0.37.4/state/execution.go:197 +0x18e
github.com/cometbft/cometbft/blocksync.(*Reactor).poolRoutine(0xc000eb9340, 0x0)
    /go/pkg/mod/github.com/cometbft/cometbft@v0.37.4/blocksync/reactor.go:405 +0xb33
created by github.com/cometbft/cometbft/blocksync.(*Reactor).OnStart in goroutine 1
    /go/pkg/mod/github.com/cometbft/cometbft@v0.37.4/blocksync/reactor.go:125 +0x6e
lumtis commented 2 months ago

https://github.com/zeta-chain/node/tree/v20.0.2 must be used for testnet, it contains a fix that has been recently released.

jajaislanina commented 2 months ago

https://github.com/zeta-chain/node/tree/v20.0.2 must be used for testnet, it contains a fix that has been recently released.

Since docker image is not available on docker hub i assume i have to build it? Was there an announcement about this mandatory upgrade published somewhere? How do i keep track of these things without wasting time in the future?

gartnera commented 2 months ago

Typically mandatory consensus breaking upgrades are published as on chain governance proposals. But those only work if the chain is currently producing blocks.

There was public discussion and announcements on discord: https://discord.com/invite/zetachain