zeta-chain / node

ZetaChain’s blockchain node and an observer validator client
https://zetachain.com
MIT License
163 stars 105 forks source link

v19.0.0 Upgrade Failures #2702

Closed gartnera closed 4 weeks ago

gartnera commented 1 month ago

Megathread for v19.0.0 upgrade failures on athens.

Please attach:

Please share any additional debugging you have done.

Update:

Root cause is https://github.com/cosmos/cosmos-sdk/commit/fd7cdd6b7ab6b29be2abd4ad263d49b876ef851f

We've previously used -X github.com/cosmos/cosmos-sdk/types.DBBackend=pebbledb to force the app.toml app-db-backend to be pebbledb. That was removed during the v0.47 upgrade. If the variable is unset now, it will try to open the app db as a leveldb.

Looking at the validator configs in https://github.com/zeta-chain/node/issues/2702, all had app-db-backend set to empty string so they were using goleveldb as the config.toml db-backend and pebbledb as app.toml app-db-backend.

itrocket-team commented 1 month ago

uname -a Linux Ubuntu-2204-jammy-amd64-base 6.5.0-44-generic #44~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Tue Jun 18 14:36:16 UTC 2 x86_64 x86_64 x86_64 GNU/Linux

/etc/os-release

PRETTY_NAME="Ubuntu 22.04.4 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.4 LTS (Jammy Jellyfish)"
VERSION_CODENAME=jammy
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=jammy

config.toml

# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

# NOTE: Any path below can be absolute (e.g. "/var/myawesomeapp/data") or
# relative to the home directory (e.g. "data"). The home directory is
# "$HOME/.cometbft" by default, but could be changed via $CMTHOME env variable
# or --home cmd flag.

#######################################################################
###                   Main Base Config Options                      ###
#######################################################################

# TCP or UNIX socket address of the ABCI application,
# or the name of an ABCI application compiled in with the CometBFT binary
proxy_app = "tcp://127.0.0.1:14658"

# A custom human readable name for this node
moniker = "itrocket"

# If this node is many blocks behind the tip of the chain, FastSync
# allows them to catchup quickly by downloading blocks in parallel
# and verifying their commits
fast_sync = true

# Database backend: goleveldb | cleveldb | boltdb | rocksdb | badgerdb
# * goleveldb (github.com/syndtr/goleveldb - most popular implementation)
#   - pure go
#   - stable
# * cleveldb (uses levigo wrapper)
#   - fast
#   - requires gcc
#   - use cleveldb build tag (go build -tags cleveldb)
# * boltdb (uses etcd's fork of bolt - github.com/etcd-io/bbolt)
#   - EXPERIMENTAL
#   - may be faster is some use-cases (random reads - indexer)
#   - use boltdb build tag (go build -tags boltdb)
# * rocksdb (uses github.com/tecbot/gorocksdb)
#   - EXPERIMENTAL
#   - requires gcc
#   - use rocksdb build tag (go build -tags rocksdb)
# * badgerdb (uses github.com/dgraph-io/badger)
#   - EXPERIMENTAL
#   - use badgerdb build tag (go build -tags badgerdb)
db_backend = "goleveldb"

# Database directory
db_dir = "data"

# Output level for logging, including package level options
log_level = "info"

# Output format: 'plain' (colored text) or 'json'
log_format = "plain"

##### additional base config options #####

# Path to the JSON file containing the initial validator set and other meta data
genesis_file = "config/genesis.json"

# Path to the JSON file containing the private key to use as a validator in the consensus protocol
priv_validator_key_file = "config/priv_validator_key.json"

# Path to the JSON file containing the last sign state of a validator
priv_validator_state_file = "data/priv_validator_state.json"

# TCP or UNIX socket address for CometBFT to listen on for
# connections from an external PrivValidator process
priv_validator_laddr = ""

# Path to the JSON file containing the private key to use for node authentication in the p2p protocol
node_key_file = "config/node_key.json"

# Mechanism to connect to the ABCI application: socket | grpc
abci = "socket"

# If true, query the ABCI app on connecting to a new peer
# so the app can decide if we should keep the connection or not
filter_peers = false

#######################################################################
###                 Advanced Configuration Options                  ###
#######################################################################

#######################################################
###       RPC Server Configuration Options          ###
#######################################################
[rpc]

# TCP or UNIX socket address for the RPC server to listen on
laddr = "tcp://127.0.0.1:14657"

# A list of origins a cross-domain request can be executed from
# Default value '[]' disables cors support
# Use '["*"]' to allow any origin
cors_allowed_origins = []

# A list of methods the client is allowed to use with cross-domain requests
cors_allowed_methods = ["HEAD", "GET", "POST", ]

# A list of non simple headers the client is allowed to use with cross-domain requests
cors_allowed_headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time", ]

# TCP or UNIX socket address for the gRPC server to listen on
# NOTE: This server only supports /broadcast_tx_commit
grpc_laddr = ""

# Maximum number of simultaneous connections.
# Does not include RPC (HTTP&WebSocket) connections. See max_open_connections
# If you want to accept a larger number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
# 1024 - 40 - 10 - 50 = 924 = ~900
grpc_max_open_connections = 900

# Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool
unsafe = false

# Maximum number of simultaneous connections (including WebSocket).
# Does not include gRPC connections. See grpc_max_open_connections
# If you want to accept a larger number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
# 1024 - 40 - 10 - 50 = 924 = ~900
max_open_connections = 900

# Maximum number of unique clientIDs that can /subscribe
# If you're using /broadcast_tx_commit, set to the estimated maximum number
# of broadcast_tx_commit calls per block.
max_subscription_clients = 100

# Maximum number of unique queries a given client can /subscribe to
# If you're using GRPC (or Local RPC client) and /broadcast_tx_commit, set to
# the estimated # maximum number of broadcast_tx_commit calls per block.
max_subscriptions_per_client = 5

# Experimental parameter to specify the maximum number of events a node will
# buffer, per subscription, before returning an error and closing the
# subscription. Must be set to at least 100, but higher values will accommodate
# higher event throughput rates (and will use more memory).
experimental_subscription_buffer_size = 200

# Experimental parameter to specify the maximum number of RPC responses that
# can be buffered per WebSocket client. If clients cannot read from the
# WebSocket endpoint fast enough, they will be disconnected, so increasing this
# parameter may reduce the chances of them being disconnected (but will cause
# the node to use more memory).
#
# Must be at least the same as "experimental_subscription_buffer_size",
# otherwise connections could be dropped unnecessarily. This value should
# ideally be somewhat higher than "experimental_subscription_buffer_size" to
# accommodate non-subscription-related RPC responses.
experimental_websocket_write_buffer_size = 200

# If a WebSocket client cannot read fast enough, at present we may
# silently drop events instead of generating an error or disconnecting the
# client.
#
# Enabling this experimental parameter will cause the WebSocket connection to
# be closed instead if it cannot read fast enough, allowing for greater
# predictability in subscription behaviour.
experimental_close_on_slow_client = false

# How long to wait for a tx to be committed during /broadcast_tx_commit.
# WARNING: Using a value larger than 10s will result in increasing the
# global HTTP write timeout, which applies to all connections and endpoints.
# See https://github.com/tendermint/tendermint/issues/3435
timeout_broadcast_tx_commit = "10s"

# Maximum size of request body, in bytes
max_body_bytes = 1000000

# Maximum size of request header, in bytes
max_header_bytes = 1048576

# The path to a file containing certificate that is used to create the HTTPS server.
# Might be either absolute path or path related to CometBFT's config directory.
# If the certificate is signed by a certificate authority,
# the certFile should be the concatenation of the server's certificate, any intermediates,
# and the CA's certificate.
# NOTE: both tls_cert_file and tls_key_file must be present for CometBFT to create HTTPS server.
# Otherwise, HTTP server is run.
tls_cert_file = ""

# The path to a file containing matching private key that is used to create the HTTPS server.
# Might be either absolute path or path related to CometBFT's config directory.
# NOTE: both tls-cert-file and tls-key-file must be present for CometBFT to create HTTPS server.
# Otherwise, HTTP server is run.
tls_key_file = ""

# pprof listen address (https://golang.org/pkg/net/http/pprof)
pprof_laddr = "localhost:14060"

#######################################################
###           P2P Configuration Options             ###
#######################################################
[p2p]

# Address to listen for incoming connections
laddr = "tcp://0.0.0.0:14656"

# Address to advertise to peers for them to dial
# If empty, will use the same port as the laddr,
# and will introspect on the listener or use UPnP
# to figure out the address. ip and port are required
# example: 159.89.10.97:14656
external_address = "65.109.50.163:14656"

# Comma separated list of seed nodes to connect to
seeds = ""

# Comma separated list of nodes to keep persistent connections to
persistent_peers = "d21b103628b0d5d824bbe81b809d8dc457bd2059@zetachain-testnet-peer.itrocket.net:16656,af58c82b5f4d2268e0b8ca9150190e438c07d90d@34.239.99.239:26656,5c6e5285981dd07732d78f732975cceeefbf424a@65.21.69.95:26656,57693a9bce3ffb5d6023a161ac9f744ac09a2329@162.19.240.28:26656,e3a9810a22a12c04ef1663f8747274e4ef1bdf58@51.159.145.74:26656,2228068135e4c9308efccca474a86535bd34c106@142.132.202.87:26656,a918d08544b5f4e0a9eb20bf91f343eb71b6d5ee@164.90.181.99:26656,644ed724398a2022f150916754d3f4d9ad6b83ef@185.202.223.111:29656,853c46d580fe0673aba2b72b4b93b9d156b882fb@52.42.64.63:26656,2352e5f3bad70d13ebae1876966d6a10c219e819@95.216.244.70:26656,9e180f06bc2fa505efed28081e45bed035254f90@57.129.28.219:26656"

# UPNP port forwarding
upnp = false

# Path to address book
addr_book_file = "config/addrbook.json"

# Set true for strict address routability rules
# Set false for private or local networks
addr_book_strict = true

# Maximum number of inbound peers
max_num_inbound_peers = 40

# Maximum number of outbound peers to connect to, excluding persistent peers
max_num_outbound_peers = 10

# List of node IDs, to which a connection will be (re)established ignoring any existing limits
unconditional_peer_ids = ""

# Maximum pause when redialing a persistent peer (if zero, exponential backoff is used)
persistent_peers_max_dial_period = "0s"

# Time to wait before flushing messages out on the connection
flush_throttle_timeout = "100ms"

# Maximum size of a message packet payload, in bytes
max_packet_msg_payload_size = 1024

# Rate at which packets can be sent, in bytes/second
send_rate = 5120000

# Rate at which packets can be received, in bytes/second
recv_rate = 5120000

# Set true to enable the peer-exchange reactor
pex = true

# Seed mode, in which node constantly crawls the network and looks for
# peers. If another node asks it for addresses, it responds and disconnects.
#
# Does not work if the peer-exchange reactor is disabled.
seed_mode = false

# Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
private_peer_ids = ""

# Toggle to disable guard against peers connecting from the same ip.
allow_duplicate_ip = false

# Peer connection configuration.
handshake_timeout = "20s"
dial_timeout = "3s"

#######################################################
###          Mempool Configuration Option          ###
#######################################################
[mempool]

# Mempool version to use:
#   1) "v0" - (default) FIFO mempool.
#   2) "v1" - prioritized mempool.
version = "v1"

# Recheck (default: true) defines whether CometBFT should recheck the
# validity for all remaining transaction in the mempool after a block.
# Since a block affects the application state, some transactions in the
# mempool may become invalid. If this does not apply to your application,
# you can disable rechecking.
recheck = true
broadcast = true
wal_dir = ""

# Maximum number of transactions in the mempool
size = 2000

# Limit the total size of all txs in the mempool.
# This only accounts for raw transactions (e.g. given 1MB transactions and
# max_txs_bytes=5MB, mempool will only accept 5 transactions).
max_txs_bytes = 1073741824

# Size of the cache (used to filter transactions we saw earlier) in transactions
cache_size = 10000

# Do not remove invalid transactions from the cache (default: false)
# Set to true if it's not possible for any invalid transaction to become valid
# again in the future.
keep-invalid-txs-in-cache = false

# Maximum size of a single transaction.
# NOTE: the max size of a tx transmitted over the network is {max_tx_bytes}.
max_tx_bytes = 1048576

# Maximum size of a batch of transactions to send to a peer
# Including space needed by encoding (one varint per transaction).
# XXX: Unused due to https://github.com/tendermint/tendermint/issues/5796
max_batch_bytes = 0

# ttl-duration, if non-zero, defines the maximum amount of time a transaction
# can exist for in the mempool.
#
# Note, if ttl-num-blocks is also defined, a transaction will be removed if it
# has existed in the mempool at least ttl-num-blocks number of blocks or if it's
# insertion time into the mempool is beyond ttl-duration.
ttl-duration = "0s"

# ttl-num-blocks, if non-zero, defines the maximum number of blocks a transaction
# can exist for in the mempool.
#
# Note, if ttl-duration is also defined, a transaction will be removed if it
# has existed in the mempool at least ttl-num-blocks number of blocks or if
# it's insertion time into the mempool is beyond ttl-duration.
ttl-num-blocks = 0

#######################################################
###         State Sync Configuration Options        ###
#######################################################
[statesync]
# State sync rapidly bootstraps a new node by discovering, fetching, and restoring a state machine
# snapshot from peers instead of fetching and replaying historical blocks. Requires some peers in
# the network to take and serve state machine snapshots. State sync is not attempted if the node
# has any local state (LastBlockHeight > 0). The node will have a truncated block history,
# starting from the height of the snapshot.
enable = false

# RPC servers (comma-separated) for light client verification of the synced state machine and
# retrieval of state data for node bootstrapping. Also needs a trusted height and corresponding
# header hash obtained from a trusted source, and a period during which validators can be trusted.
#
# For Cosmos SDK-based chains, trust_period should usually be about 2/3 of the unbonding time (~2
# weeks) during which they can be financially punished (slashed) for misbehavior.
rpc_servers = "https://zetachain-testnet-rpc.itrocket.net:443,https://zetachain-testnet-rpc.itrocket.net:443"
trust_height = 5402087
trust_hash = "8124EF3732664182E7DBE016BC7B7CF15EBE2E2C0B771E59C18623FE0B11BBDF"
trust_period = "168h0m0s"

# Time to spend discovering snapshots before initiating a restore.
discovery_time = "15s"

# Temporary directory for state sync snapshot chunks, defaults to the OS tempdir (typically /tmp).
# Will create a new, randomly named directory within, and remove it when done.
temp_dir = ""

# The timeout duration before re-requesting a chunk, possibly from a different
# peer (default: 1 minute).
chunk_request_timeout = "10s"

# The number of concurrent chunk fetchers to run (default: 1).
chunk_fetchers = "4"

#######################################################
###       Fast Sync Configuration Connections       ###
#######################################################
[fastsync]

# Fast Sync version to use:
#   1) "v0" (default) - the legacy fast sync implementation
#   2) "v1" - refactor of v0 version for better testability
#   2) "v2" - complete redesign of v0, optimized for testability & readability
version = "v0"

#######################################################
###         Consensus Configuration Options         ###
#######################################################
[consensus]

wal_file = "data/cs.wal/wal"

# How long we wait for a proposal block before prevoting nil
timeout_propose = "3s"
# How much timeout_propose increases with each round
timeout_propose_delta = "500ms"
# How long we wait after receiving +2/3 prevotes for “anything” (ie. not a single block or nil)
timeout_prevote = "1s"
# How much the timeout_prevote increases with each round
timeout_prevote_delta = "500ms"
# How long we wait after receiving +2/3 precommits for “anything” (ie. not a single block or nil)
timeout_precommit = "1s"
# How much the timeout_precommit increases with each round
timeout_precommit_delta = "500ms"
# How long we wait after committing a block, before starting on the new
# height (this gives us a chance to receive some more precommits, even
# though we already have +2/3).
timeout_commit = "5s"

# How many blocks to look back to check existence of the node's consensus votes before joining consensus
# When non-zero, the node will panic upon restart
# if the same consensus key was used to sign {double_sign_check_height} last blocks.
# So, validators should stop the state machine, wait for some blocks, and then restart the state machine to avoid panic.
double_sign_check_height = 0

# Make progress as soon as we have all the precommits (as if TimeoutCommit = 0)
skip_timeout_commit = false

# EmptyBlocks mode and possible interval between empty blocks
create_empty_blocks = true
create_empty_blocks_interval = "0s"

# Reactor sleep duration parameters
peer_gossip_sleep_duration = "100ms"
peer_query_maj23_sleep_duration = "2s"

#######################################################
###         Storage Configuration Options           ###
#######################################################
[storage]

# Set to true to discard ABCI responses from the state store, which can save a
# considerable amount of disk space. Set to false to ensure ABCI responses are
# persisted. ABCI responses are required for /block_results RPC queries, and to
# reindex events in the command-line tool.
discard_abci_responses = false

#######################################################
###   Transaction Indexer Configuration Options     ###
#######################################################
[tx_index]

# What indexer to use for transactions
#
# The application will set which txs to index. In some cases a node operator will be able
# to decide which txs to index based on configuration set in the application.
#
# Options:
#   1) "null"
#   2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
#       - When "kv" is chosen "tx.height" and "tx.hash" will always be indexed.
#   3) "psql" - the indexer services backed by PostgreSQL.
# When "kv" or "psql" is chosen "tx.height" and "tx.hash" will always be indexed.
indexer = "null"

# The PostgreSQL connection configuration, the connection format:
#   postgresql://<user>:<password>@<host>:<port>/<db>?<opts>
psql-conn = ""

#######################################################
###       Instrumentation Configuration Options     ###
#######################################################
[instrumentation]

# When true, Prometheus metrics are served under /metrics on
# PrometheusListenAddr.
# Check out the documentation for the list of available metrics.
prometheus = false

# Address to listen for Prometheus collector(s) connections
prometheus_listen_addr = ":14660"

# Maximum number of simultaneous connections.
# If you want to accept a larger number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
max_open_connections = 3

# Instrumentation namespace
namespace = "tendermint"

.zetacored/config/app.toml

# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

###############################################################################
###                           Base Configuration                            ###
###############################################################################

# The minimum gas prices a validator is willing to accept for processing a
# transaction. A transaction's fees must meet the minimum of any denomination
# specified in this config (e.g. 0.25token1;0.0001token2).
minimum-gas-prices = "0.0azeta"

# default: the last 362880 states are kept, pruning at 10 block intervals
# nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node)
# everything: 2 latest states will be kept; pruning at 10 block intervals.
# custom: allow pruning options to be manually specified through 'pruning-keep-recent', and 'pruning-interval'
pruning = "nothing"

# These are applied if and only if the pruning strategy is custom.
pruning-keep-recent = "100"
pruning-interval = "50"

# HaltHeight contains a non-zero block height at which a node will gracefully
# halt and shutdown that can be used to assist upgrades and testing.
#
# Note: Commitment of state will be attempted on the corresponding block.
halt-height = 0

# HaltTime contains a non-zero minimum block time (in Unix seconds) at which
# a node will gracefully halt and shutdown that can be used to assist upgrades
# and testing.
#
# Note: Commitment of state will be attempted on the corresponding block.
halt-time = 0

# MinRetainBlocks defines the minimum block height offset from the current
# block being committed, such that all blocks past this offset are pruned
# from Tendermint. It is used as part of the process of determining the
# ResponseCommit.RetainHeight value during ABCI Commit. A value of 0 indicates
# that no blocks should be pruned.
#
# This configuration value is only responsible for pruning Tendermint blocks.
# It has no bearing on application state pruning which is determined by the
# "pruning-*" configurations.
#
# Note: Tendermint block pruning is dependant on this parameter in conunction
# with the unbonding (safety threshold) period, state pruning and state sync
# snapshot parameters to determine the correct minimum value of
# ResponseCommit.RetainHeight.
min-retain-blocks = 0

# InterBlockCache enables inter-block caching.
inter-block-cache = true

# IndexEvents defines the set of events in the form {eventType}.{attributeKey},
# which informs Tendermint what to index. If empty, all events will be indexed.
#
# Example:
# ["message.sender", "message.recipient"]
index-events = []

# IavlCacheSize set the size of the iavl tree cache. 
# Default cache size is 50mb.
iavl-cache-size = 781250

# IavlDisableFastNode enables or disables the fast node feature of IAVL. 
# Default is false.
iavl-disable-fastnode = false

# EXPERIMENTAL: IAVLLazyLoading enable/disable the lazy loading of iavl store.
# Default is false.
iavl-lazy-loading = false

# AppDBBackend defines the database backend type to use for the application and snapshots DBs.
# An empty string indicates that a fallback will be used.
# First fallback is the deprecated compile-time types.DBBackend value.
# Second fallback (if the types.DBBackend also isn't set), is the db-backend value set in Tendermint's config.toml.
app-db-backend = ""

###############################################################################
###                         Telemetry Configuration                         ###
###############################################################################

[telemetry]

# Prefixed with keys to separate services.
service-name = ""

# Enabled enables the application telemetry functionality. When enabled,
# an in-memory sink is also enabled by default. Operators may also enabled
# other sinks such as Prometheus.
enabled = false

# Enable prefixing gauge values with hostname.
enable-hostname = false

# Enable adding hostname to labels.
enable-hostname-label = false

# Enable adding service to labels.
enable-service-label = false

# PrometheusRetentionTime, when positive, enables a Prometheus metrics sink.
prometheus-retention-time = 0

# GlobalLabels defines a global set of name/value label tuples applied to all
# metrics emitted using the wrapper functions defined in telemetry package.
#
# Example:
# [["chain_id", "cosmoshub-1"]]
global-labels = [
]

###############################################################################
###                           API Configuration                             ###
###############################################################################

[api]

# Enable defines if the API server should be enabled.
enable = false

# Swagger defines if swagger documentation should automatically be registered.
swagger = false

# Address defines the API server to listen on.
address = "tcp://0.0.0.0:14317"

# MaxOpenConnections defines the number of maximum open connections.
max-open-connections = 1000

# RPCReadTimeout defines the Tendermint RPC read timeout (in seconds).
rpc-read-timeout = 10

# RPCWriteTimeout defines the Tendermint RPC write timeout (in seconds).
rpc-write-timeout = 0

# RPCMaxBodyBytes defines the Tendermint maximum response body (in bytes).
rpc-max-body-bytes = 1000000

# EnableUnsafeCORS defines if CORS should be enabled (unsafe - use it at your own risk).
enabled-unsafe-cors = false

###############################################################################
###                           Rosetta Configuration                         ###
###############################################################################

[rosetta]

# Enable defines if the Rosetta API server should be enabled.
enable = false

# Address defines the Rosetta API server to listen on.
address = ":14080"

# Network defines the name of the blockchain that will be returned by Rosetta.
blockchain = "app"

# Network defines the name of the network that will be returned by Rosetta.
network = "network"

# Retries defines the number of retries when connecting to the node before failing.
retries = 3

# Offline defines if Rosetta server should run in offline mode.
offline = false

# EnableDefaultSuggestedFee defines if the server should suggest fee by default.
# If 'construction/medata' is called without gas limit and gas price,
# suggested fee based on gas-to-suggest and denom-to-suggest will be given.
enable-fee-suggestion = false

# GasToSuggest defines gas limit when calculating the fee
gas-to-suggest = 200000

# DenomToSuggest defines the defult denom for fee suggestion.
# Price must be in minimum-gas-prices.
denom-to-suggest = "uatom"

###############################################################################
###                           gRPC Configuration                            ###
###############################################################################

[grpc]

# Enable defines if the gRPC server should be enabled.
enable = true

# Address defines the gRPC server address to bind to.
address = "0.0.0.0:14090"

# MaxRecvMsgSize defines the max message size in bytes the server can receive.
# The default value is 10MB.
max-recv-msg-size = "10485760"

# MaxSendMsgSize defines the max message size in bytes the server can send.
# The default value is math.MaxInt32.
max-send-msg-size = "2147483647"

###############################################################################
###                        gRPC Web Configuration                           ###
###############################################################################

[grpc-web]

# GRPCWebEnable defines if the gRPC-web should be enabled.
# NOTE: gRPC must also be enabled, otherwise, this configuration is a no-op.
enable = true

# Address defines the gRPC-web server address to bind to.
address = "0.0.0.0:14091"

# EnableUnsafeCORS defines if CORS should be enabled (unsafe - use it at your own risk).
enable-unsafe-cors = false

###############################################################################
###                        State Sync Configuration                         ###
###############################################################################

# State sync snapshots allow other nodes to rapidly join the network without replaying historical
# blocks, instead downloading and applying a snapshot of the application state at a given height.
[state-sync]

# snapshot-interval specifies the block interval at which local state sync snapshots are
# taken (0 to disable).
snapshot-interval = 0

# snapshot-keep-recent specifies the number of recent snapshots to keep and serve (0 to keep all).
snapshot-keep-recent = 2

###############################################################################
###                         Store / State Streaming                         ###
###############################################################################

[store]
streamers = []

[streamers]
[streamers.file]
keys = ["*", ]
write_dir = ""
prefix = ""

# output-metadata specifies if output the metadata file which includes the abci request/responses 
# during processing the block.
output-metadata = "true"

# stop-node-on-error specifies if propagate the file streamer errors to consensus state machine.
stop-node-on-error = "true"

# fsync specifies if call fsync after writing the files.
fsync = "false"

###############################################################################
###                             EVM Configuration                           ###
###############################################################################

[evm]

# Tracer defines the 'vm.Tracer' type that the EVM will use when the node is run in
# debug mode. To enable tracing use the '--evm.tracer' flag when starting your node.
# Valid types are: json|struct|access_list|markdown
tracer = ""

# MaxTxGasWanted defines the gas wanted for each eth tx returned in ante handler in check tx mode.
max-tx-gas-wanted = 0

###############################################################################
###                           JSON RPC Configuration                        ###
###############################################################################

[json-rpc]

# Enable defines if the gRPC server should be enabled.
enable = true

# Address defines the EVM RPC HTTP server address to bind to.
address = "127.0.0.1:14545"

# Address defines the EVM WebSocket server address to bind to.
ws-address = "127.0.0.1:14546"

# API defines a list of JSON-RPC namespaces that should be enabled
# Example: "eth,txpool,personal,net,debug,web3"
api = "eth,net,web3"

# GasCap sets a cap on gas that can be used in eth_call/estimateGas (0=infinite). Default: 25,000,000.
gas-cap = 25000000

# EVMTimeout is the global timeout for eth_call. Default: 5s.
evm-timeout = "5s"

# TxFeeCap is the global tx-fee cap for send transaction. Default: 1eth.
txfee-cap = 1

# FilterCap sets the global cap for total number of filters that can be created
filter-cap = 200

# FeeHistoryCap sets the global cap for total number of blocks that can be fetched
feehistory-cap = 100

# LogsCap defines the max number of results can be returned from single 'eth_getLogs' query.
logs-cap = 10000

# BlockRangeCap defines the max block range allowed for 'eth_getLogs' query.
block-range-cap = 10000

# HTTPTimeout is the read/write timeout of http json-rpc server.
http-timeout = "30s"

# HTTPIdleTimeout is the idle timeout of http json-rpc server.
http-idle-timeout = "2m0s"

# AllowUnprotectedTxs restricts unprotected (non EIP155 signed) transactions to be submitted via
# the node's RPC when the global parameter is disabled.
allow-unprotected-txs = false

# MaxOpenConnections sets the maximum number of simultaneous connections
# for the server listener.
max-open-connections = 0

# EnableIndexer enables the custom transaction indexer for the EVM (ethereum transactions).
enable-indexer = true

# MetricsAddress defines the EVM Metrics server address to bind to. Pass --metrics in CLI to enable
# Prometheus metrics path: /debug/metrics/prometheus
metrics-address = "127.0.0.1:14065"

# Upgrade height for fix of revert gas refund logic when transaction reverted.
fix-revert-gas-refund-height = 0

###############################################################################
###                             TLS Configuration                           ###
###############################################################################

[tls]

# Certificate path defines the cert.pem file path for the TLS configuration.
certificate-path = ""

# Key path defines the key.pem file path for the TLS configuration.
key-path = ""

zetacored logs

Aug 14 19:34:16 Ubuntu-2204-jammy-amd64-base systemd[1]: Started Zetachain node.
Aug 14 19:34:16 Ubuntu-2204-jammy-amd64-base zetacored[877106]: {"level":"info","module":"server","time":"2024-08-14T19:34:16+02:00","message":"Unlocking keyring"}
Aug 14 19:34:16 Ubuntu-2204-jammy-amd64-base zetacored[877106]: {"level":"info","module":"server","time":"2024-08-14T19:34:16+02:00","message":"starting ABCI with Tendermint"}
Aug 14 19:34:29 Ubuntu-2204-jammy-amd64-base zetacored[877106]: fatal error: runtime: out of memory
Aug 14 19:34:29 Ubuntu-2204-jammy-amd64-base zetacored[877106]: runtime stack:
Aug 14 19:34:29 Ubuntu-2204-jammy-amd64-base zetacored[877106]: runtime.throw({0x3407ca3?, 0x2104b3?})
Aug 14 19:34:29 Ubuntu-2204-jammy-amd64-base zetacored[877106]:         /usr/local/go/src/runtime/panic.go:1023 +0x5c fp=0x7663a4ff8c18 sp=0x7663a4ff8be8 pc=0x441dbc
Aug 14 19:34:29 Ubuntu-2204-jammy-amd64-base zetacored[877106]: runtime.sysMapOS(0xc561800000, 0x34d66800000)
Aug 14 19:34:29 Ubuntu-2204-jammy-amd64-base zetacored[877106]:         /usr/local/go/src/runtime/mem_linux.go:167 +0x11b fp=0x7663a4ff8c58 sp=0x7663a4ff8c18 pc=0x41f89b
Hacker-web-Vi commented 1 month ago

Using privided prebuilt binary

https://github.com/zeta-chain/node/releases/download/v19.0.0/zetacored-linux-amd64?checksum=sha256:52521de09e1da1d2e6a3e056b101d77b32e76994d0b0d211cac9f8800671aeb7

sha256sum $(which zetacored)
52521de09e1da1d2e6a3e056b101d77b32e76994d0b0d211cac9f8800671aeb7  /home/zetachain-testnet-validator/go/bin/zetacored

uname -a

Linux trusted-point-1 5.15.0-107-generic #117-Ubuntu SMP Fri Apr 26 12:26:49 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux

cat /etc/os-release

PRETTY_NAME="Ubuntu 22.04.4 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.4 LTS (Jammy Jellyfish)"
VERSION_CODENAME=jammy
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=jammy

config.toml

# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

# NOTE: Any path below can be absolute (e.g. "/var/myawesomeapp/data") or
# relative to the home directory (e.g. "data"). The home directory is
# "$HOME/.cometbft" by default, but could be changed via $CMTHOME env variable
# or --home cmd flag.

#######################################################################
###                   Main Base Config Options                      ###
#######################################################################

# TCP or UNIX socket address of the ABCI application,
# or the name of an ABCI application compiled in with the CometBFT binary
proxy_app = "tcp://127.0.0.1:22509"

# A custom human readable name for this node
moniker = "TrustedPoint"

# If this node is many blocks behind the tip of the chain, FastSync
# allows them to catchup quickly by downloading blocks in parallel
# and verifying their commits
fast_sync = true

# Database backend: goleveldb | cleveldb | boltdb | rocksdb | badgerdb
# * goleveldb (github.com/syndtr/goleveldb - most popular implementation)
#   - pure go
#   - stable
# * cleveldb (uses levigo wrapper)
#   - fast
#   - requires gcc
#   - use cleveldb build tag (go build -tags cleveldb)
# * boltdb (uses etcd's fork of bolt - github.com/etcd-io/bbolt)
#   - EXPERIMENTAL
#   - may be faster is some use-cases (random reads - indexer)
#   - use boltdb build tag (go build -tags boltdb)
# * rocksdb (uses github.com/tecbot/gorocksdb)
#   - EXPERIMENTAL
#   - requires gcc
#   - use rocksdb build tag (go build -tags rocksdb)
# * badgerdb (uses github.com/dgraph-io/badger)
#   - EXPERIMENTAL
#   - use badgerdb build tag (go build -tags badgerdb)
db_backend = "goleveldb"

# Database directory
db_dir = "data"

# Output level for logging, including package level options
log_level = "info"

# Output format: 'plain' (colored text) or 'json'
log_format = "plain"

##### additional base config options #####

# Path to the JSON file containing the initial validator set and other meta data
genesis_file = "config/genesis.json"

# Path to the JSON file containing the private key to use as a validator in the consensus protocol
priv_validator_key_file = "config/priv_validator_key.json"

# Path to the JSON file containing the last sign state of a validator
priv_validator_state_file = "data/priv_validator_state.json"

# TCP or UNIX socket address for CometBFT to listen on for
# connections from an external PrivValidator process
priv_validator_laddr = ""

# Path to the JSON file containing the private key to use for node authentication in the p2p protocol
node_key_file = "config/node_key.json"

# Mechanism to connect to the ABCI application: socket | grpc
abci = "socket"

# If true, query the ABCI app on connecting to a new peer
# so the app can decide if we should keep the connection or not
filter_peers = false

#######################################################################
###                 Advanced Configuration Options                  ###
#######################################################################

#######################################################
###       RPC Server Configuration Options          ###
#######################################################
[rpc]

# TCP or UNIX socket address for the RPC server to listen on
laddr = "tcp://127.0.0.1:22512"

# A list of origins a cross-domain request can be executed from
# Default value '[]' disables cors support
# Use '["*"]' to allow any origin
cors_allowed_origins = []

# A list of methods the client is allowed to use with cross-domain requests
cors_allowed_methods = ["HEAD", "GET", "POST", ]

# A list of non simple headers the client is allowed to use with cross-domain requests
cors_allowed_headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time", ]

# TCP or UNIX socket address for the gRPC server to listen on
# NOTE: This server only supports /broadcast_tx_commit
grpc_laddr = ""

# Maximum number of simultaneous connections.
# Does not include RPC (HTTP&WebSocket) connections. See max_open_connections
# If you want to accept a larger number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
# 1024 - 40 - 10 - 50 = 924 = ~900
grpc_max_open_connections = 900

# Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool
unsafe = false

# Maximum number of simultaneous connections (including WebSocket).
# Does not include gRPC connections. See grpc_max_open_connections
# If you want to accept a larger number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
# 1024 - 40 - 10 - 50 = 924 = ~900
max_open_connections = 900

# Maximum number of unique clientIDs that can /subscribe
# If you're using /broadcast_tx_commit, set to the estimated maximum number
# of broadcast_tx_commit calls per block.
max_subscription_clients = 100

# Maximum number of unique queries a given client can /subscribe to
# If you're using GRPC (or Local RPC client) and /broadcast_tx_commit, set to
# the estimated # maximum number of broadcast_tx_commit calls per block.
max_subscriptions_per_client = 5

# Experimental parameter to specify the maximum number of events a node will
# buffer, per subscription, before returning an error and closing the
# subscription. Must be set to at least 100, but higher values will accommodate
# higher event throughput rates (and will use more memory).
experimental_subscription_buffer_size = 200

# Experimental parameter to specify the maximum number of RPC responses that
# can be buffered per WebSocket client. If clients cannot read from the
# WebSocket endpoint fast enough, they will be disconnected, so increasing this
# parameter may reduce the chances of them being disconnected (but will cause
# the node to use more memory).
#
# Must be at least the same as "experimental_subscription_buffer_size",
# otherwise connections could be dropped unnecessarily. This value should
# ideally be somewhat higher than "experimental_subscription_buffer_size" to
# accommodate non-subscription-related RPC responses.
experimental_websocket_write_buffer_size = 200

# If a WebSocket client cannot read fast enough, at present we may
# silently drop events instead of generating an error or disconnecting the
# client.
#
# Enabling this experimental parameter will cause the WebSocket connection to
# be closed instead if it cannot read fast enough, allowing for greater
# predictability in subscription behaviour.
experimental_close_on_slow_client = false

# How long to wait for a tx to be committed during /broadcast_tx_commit.
# WARNING: Using a value larger than 10s will result in increasing the
# global HTTP write timeout, which applies to all connections and endpoints.
# See https://github.com/tendermint/tendermint/issues/3435
timeout_broadcast_tx_commit = "10s"

# Maximum size of request body, in bytes
max_body_bytes = 1000000

# Maximum size of request header, in bytes
max_header_bytes = 1048576

# The path to a file containing certificate that is used to create the HTTPS server.
# Might be either absolute path or path related to CometBFT's config directory.
# If the certificate is signed by a certificate authority,
# the certFile should be the concatenation of the server's certificate, any intermediates,
# and the CA's certificate.
# NOTE: both tls_cert_file and tls_key_file must be present for CometBFT to create HTTPS server.
# Otherwise, HTTP server is run.
tls_cert_file = ""

# The path to a file containing matching private key that is used to create the HTTPS server.
# Might be either absolute path or path related to CometBFT's config directory.
# NOTE: both tls-cert-file and tls-key-file must be present for CometBFT to create HTTPS server.
# Otherwise, HTTP server is run.
tls_key_file = ""

# pprof listen address (https://golang.org/pkg/net/http/pprof)
pprof_laddr = "localhost:6460"

#######################################################
###           P2P Configuration Options             ###
#######################################################
[p2p]

# Address to listen for incoming connections
laddr = "tcp://0.0.0.0:22513"

# Address to advertise to peers for them to dial
# If empty, will use the same port as the laddr,
# and will introspect on the listener or use UPnP
# to figure out the address. ip and port are required
# example: 159.89.10.97:26656
external_address = ""

# Comma separated list of seed nodes to connect to
seeds = "3f472746f46493309650e5a033076689996c8881@zetachain-testnet.rpc.kjnodes.com:16059,d5519e378247dfb61dfe90652d1fe3e2b3005a5b@65.109.68.190:16056,f96cf94d5eabaf554c9294e731b3994b4c532f3a@46.4.5.45:22556"

# Comma separated list of nodes to keep persistent connections to
persistent_peers = ""

# UPNP port forwarding
upnp = false

# Path to address book
addr_book_file = "config/addrbook.json"

# Set true for strict address routability rules
# Set false for private or local networks
addr_book_strict = true

# Maximum number of inbound peers
max_num_inbound_peers = 40

# Maximum number of outbound peers to connect to, excluding persistent peers
max_num_outbound_peers = 10

# List of node IDs, to which a connection will be (re)established ignoring any existing limits
unconditional_peer_ids = ""

# Maximum pause when redialing a persistent peer (if zero, exponential backoff is used)
persistent_peers_max_dial_period = "0s"

# Time to wait before flushing messages out on the connection
flush_throttle_timeout = "100ms"

# Maximum size of a message packet payload, in bytes
max_packet_msg_payload_size = 1024

# Rate at which packets can be sent, in bytes/second
send_rate = 5120000

# Rate at which packets can be received, in bytes/second
recv_rate = 5120000

# Set true to enable the peer-exchange reactor
pex = true

# Seed mode, in which node constantly crawls the network and looks for
# peers. If another node asks it for addresses, it responds and disconnects.
#
# Does not work if the peer-exchange reactor is disabled.
seed_mode = false

# Comma separated list of peer IDs to keep private (will not be gossiped to other peers)
private_peer_ids = ""

# Toggle to disable guard against peers connecting from the same ip.
allow_duplicate_ip = false

# Peer connection configuration.
handshake_timeout = "20s"
dial_timeout = "3s"

#######################################################
###          Mempool Configuration Option          ###
#######################################################
[mempool]

# Mempool version to use:
#   1) "v0" - (default) FIFO mempool.
#   2) "v1" - prioritized mempool.
version = "v1"

# Recheck (default: true) defines whether CometBFT should recheck the
# validity for all remaining transaction in the mempool after a block.
# Since a block affects the application state, some transactions in the
# mempool may become invalid. If this does not apply to your application,
# you can disable rechecking.
recheck = true
broadcast = true
wal_dir = ""

# Maximum number of transactions in the mempool
size = 5000

# Limit the total size of all txs in the mempool.
# This only accounts for raw transactions (e.g. given 1MB transactions and
# max_txs_bytes=5MB, mempool will only accept 5 transactions).
max_txs_bytes = 1073741824

# Size of the cache (used to filter transactions we saw earlier) in transactions
cache_size = 10000

# Do not remove invalid transactions from the cache (default: false)
# Set to true if it's not possible for any invalid transaction to become valid
# again in the future.
keep-invalid-txs-in-cache = false

# Maximum size of a single transaction.
# NOTE: the max size of a tx transmitted over the network is {max_tx_bytes}.
max_tx_bytes = 1048576

# Maximum size of a batch of transactions to send to a peer
# Including space needed by encoding (one varint per transaction).
# XXX: Unused due to https://github.com/tendermint/tendermint/issues/5796
max_batch_bytes = 0

# ttl-duration, if non-zero, defines the maximum amount of time a transaction
# can exist for in the mempool.
#
# Note, if ttl-num-blocks is also defined, a transaction will be removed if it
# has existed in the mempool at least ttl-num-blocks number of blocks or if it's
# insertion time into the mempool is beyond ttl-duration.
ttl-duration = "0s"

# ttl-num-blocks, if non-zero, defines the maximum number of blocks a transaction
# can exist for in the mempool.
#
# Note, if ttl-duration is also defined, a transaction will be removed if it
# has existed in the mempool at least ttl-num-blocks number of blocks or if
# it's insertion time into the mempool is beyond ttl-duration.
ttl-num-blocks = 0

#######################################################
###         State Sync Configuration Options        ###
#######################################################
[statesync]
# State sync rapidly bootstraps a new node by discovering, fetching, and restoring a state machine
# snapshot from peers instead of fetching and replaying historical blocks. Requires some peers in
# the network to take and serve state machine snapshots. State sync is not attempted if the node
# has any local state (LastBlockHeight > 0). The node will have a truncated block history,
# starting from the height of the snapshot.
enable = false

# RPC servers (comma-separated) for light client verification of the synced state machine and
# retrieval of state data for node bootstrapping. Also needs a trusted height and corresponding
# header hash obtained from a trusted source, and a period during which validators can be trusted.
#
# For Cosmos SDK-based chains, trust_period should usually be about 2/3 of the unbonding time (~2
# weeks) during which they can be financially punished (slashed) for misbehavior.
rpc_servers = ""
trust_height = 0
trust_hash = ""
trust_period = "168h0m0s"

# Time to spend discovering snapshots before initiating a restore.
discovery_time = "15s"

# Temporary directory for state sync snapshot chunks, defaults to the OS tempdir (typically /tmp).
# Will create a new, randomly named directory within, and remove it when done.
temp_dir = ""

# The timeout duration before re-requesting a chunk, possibly from a different
# peer (default: 1 minute).
chunk_request_timeout = "10s"

# The number of concurrent chunk fetchers to run (default: 1).
chunk_fetchers = "4"

#######################################################
###       Fast Sync Configuration Connections       ###
#######################################################
[fastsync]

# Fast Sync version to use:
#   1) "v0" (default) - the legacy fast sync implementation
#   2) "v1" - refactor of v0 version for better testability
#   2) "v2" - complete redesign of v0, optimized for testability & readability
version = "v0"

#######################################################
###         Consensus Configuration Options         ###
#######################################################
[consensus]

wal_file = "data/cs.wal/wal"

# How long we wait for a proposal block before prevoting nil
timeout_propose = "3s"
# How much timeout_propose increases with each round
timeout_propose_delta = "500ms"
# How long we wait after receiving +2/3 prevotes for “anything” (ie. not a single block or nil)
timeout_prevote = "1s"
# How much the timeout_prevote increases with each round
timeout_prevote_delta = "500ms"
# How long we wait after receiving +2/3 precommits for “anything” (ie. not a single block or nil)
timeout_precommit = "1s"
# How much the timeout_precommit increases with each round
timeout_precommit_delta = "500ms"
# How long we wait after committing a block, before starting on the new
# height (this gives us a chance to receive some more precommits, even
# though we already have +2/3).
timeout_commit = "5s"

# How many blocks to look back to check existence of the node's consensus votes before joining consensus
# When non-zero, the node will panic upon restart
# if the same consensus key was used to sign {double_sign_check_height} last blocks.
# So, validators should stop the state machine, wait for some blocks, and then restart the state machine to avoid panic.
double_sign_check_height = 0

# Make progress as soon as we have all the precommits (as if TimeoutCommit = 0)
skip_timeout_commit = false

# EmptyBlocks mode and possible interval between empty blocks
create_empty_blocks = true
create_empty_blocks_interval = "0s"

# Reactor sleep duration parameters
peer_gossip_sleep_duration = "100ms"
peer_query_maj23_sleep_duration = "2s"

#######################################################
###         Storage Configuration Options           ###
#######################################################
[storage]

# Set to true to discard ABCI responses from the state store, which can save a
# considerable amount of disk space. Set to false to ensure ABCI responses are
# persisted. ABCI responses are required for /block_results RPC queries, and to
# reindex events in the command-line tool.
discard_abci_responses = false

#######################################################
###   Transaction Indexer Configuration Options     ###
#######################################################
[tx_index]

# What indexer to use for transactions
#
# The application will set which txs to index. In some cases a node operator will be able
# to decide which txs to index based on configuration set in the application.
#
# Options:
#   1) "null"
#   2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
#       - When "kv" is chosen "tx.height" and "tx.hash" will always be indexed.
#   3) "psql" - the indexer services backed by PostgreSQL.
# When "kv" or "psql" is chosen "tx.height" and "tx.hash" will always be indexed.
indexer = "null"

# The PostgreSQL connection configuration, the connection format:
#   postgresql://<user>:<password>@<host>:<port>/<db>?<opts>
psql-conn = ""

#######################################################
###       Instrumentation Configuration Options     ###
#######################################################
[instrumentation]

# When true, Prometheus metrics are served under /metrics on
# PrometheusListenAddr.
# Check out the documentation for the list of available metrics.
prometheus = false

# Address to listen for Prometheus collector(s) connections
prometheus_listen_addr = ":26660"

# Maximum number of simultaneous connections.
# If you want to accept a larger number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
max_open_connections = 3

# Instrumentation namespace
namespace = "cometbft"

app.toml

# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

###############################################################################
###                           Base Configuration                            ###
###############################################################################

# The minimum gas prices a validator is willing to accept for processing a
# transaction. A transaction's fees must meet the minimum of any denomination
# specified in this config (e.g. 0.25token1;0.0001token2).
minimum-gas-prices = "20000000000azeta"

# default: the last 362880 states are kept, pruning at 10 block intervals
# nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node)
# everything: 2 latest states will be kept; pruning at 10 block intervals.
# custom: allow pruning options to be manually specified through 'pruning-keep-recent', and 'pruning-interval'
pruning = "custom"

# These are applied if and only if the pruning strategy is custom.
pruning-keep-recent = "100"
pruning-interval = "17"

# HaltHeight contains a non-zero block height at which a node will gracefully
# halt and shutdown that can be used to assist upgrades and testing.
#
# Note: Commitment of state will be attempted on the corresponding block.
halt-height = 0

# HaltTime contains a non-zero minimum block time (in Unix seconds) at which
# a node will gracefully halt and shutdown that can be used to assist upgrades
# and testing.
#
# Note: Commitment of state will be attempted on the corresponding block.
halt-time = 0

# MinRetainBlocks defines the minimum block height offset from the current
# block being committed, such that all blocks past this offset are pruned
# from Tendermint. It is used as part of the process of determining the
# ResponseCommit.RetainHeight value during ABCI Commit. A value of 0 indicates
# that no blocks should be pruned.
#
# This configuration value is only responsible for pruning Tendermint blocks.
# It has no bearing on application state pruning which is determined by the
# "pruning-*" configurations.
#
# Note: Tendermint block pruning is dependant on this parameter in conunction
# with the unbonding (safety threshold) period, state pruning and state sync
# snapshot parameters to determine the correct minimum value of
# ResponseCommit.RetainHeight.
min-retain-blocks = 0

# InterBlockCache enables inter-block caching.
inter-block-cache = true

# IndexEvents defines the set of events in the form {eventType}.{attributeKey},
# which informs Tendermint what to index. If empty, all events will be indexed.
#
# Example:
# ["message.sender", "message.recipient"]
index-events = []

# IavlCacheSize set the size of the iavl tree cache. 
# Default cache size is 50mb.
iavl-cache-size = 781250

# IavlDisableFastNode enables or disables the fast node feature of IAVL. 
# Default is false.
iavl-disable-fastnode = false

# EXPERIMENTAL: IAVLLazyLoading enable/disable the lazy loading of iavl store.
# Default is false.
iavl-lazy-loading = false

# AppDBBackend defines the database backend type to use for the application and snapshots DBs.
# An empty string indicates that a fallback will be used.
# First fallback is the deprecated compile-time types.DBBackend value.
# Second fallback (if the types.DBBackend also isn't set), is the db-backend value set in Tendermint's config.toml.
app-db-backend = ""

###############################################################################
###                         Telemetry Configuration                         ###
###############################################################################

[telemetry]

# Prefixed with keys to separate services.
service-name = ""

# Enabled enables the application telemetry functionality. When enabled,
# an in-memory sink is also enabled by default. Operators may also enabled
# other sinks such as Prometheus.
enabled = false

# Enable prefixing gauge values with hostname.
enable-hostname = false

# Enable adding hostname to labels.
enable-hostname-label = false

# Enable adding service to labels.
enable-service-label = false

# PrometheusRetentionTime, when positive, enables a Prometheus metrics sink.
prometheus-retention-time = 0

# GlobalLabels defines a global set of name/value label tuples applied to all
# metrics emitted using the wrapper functions defined in telemetry package.
#
# Example:
# [["chain_id", "cosmoshub-1"]]
global-labels = [
]

###############################################################################
###                           API Configuration                             ###
###############################################################################

[api]

# Enable defines if the API server should be enabled.
enable = false

# Swagger defines if swagger documentation should automatically be registered.
swagger = false

# Address defines the API server to listen on.
address = "tcp://0.0.0.0:1317"

# MaxOpenConnections defines the number of maximum open connections.
max-open-connections = 1000

# RPCReadTimeout defines the Tendermint RPC read timeout (in seconds).
rpc-read-timeout = 10

# RPCWriteTimeout defines the Tendermint RPC write timeout (in seconds).
rpc-write-timeout = 0

# RPCMaxBodyBytes defines the Tendermint maximum response body (in bytes).
rpc-max-body-bytes = 1000000

# EnableUnsafeCORS defines if CORS should be enabled (unsafe - use it at your own risk).
enabled-unsafe-cors = false

###############################################################################
###                           Rosetta Configuration                         ###
###############################################################################

[rosetta]

# Enable defines if the Rosetta API server should be enabled.
enable = false

# Address defines the Rosetta API server to listen on.
address = ":8080"

# Network defines the name of the blockchain that will be returned by Rosetta.
blockchain = "app"

# Network defines the name of the network that will be returned by Rosetta.
network = "network"

# Retries defines the number of retries when connecting to the node before failing.
retries = 3

# Offline defines if Rosetta server should run in offline mode.
offline = false

# EnableDefaultSuggestedFee defines if the server should suggest fee by default.
# If 'construction/medata' is called without gas limit and gas price,
# suggested fee based on gas-to-suggest and denom-to-suggest will be given.
enable-fee-suggestion = false

# GasToSuggest defines gas limit when calculating the fee
gas-to-suggest = 200000

# DenomToSuggest defines the defult denom for fee suggestion.
# Price must be in minimum-gas-prices.
denom-to-suggest = "uatom"

###############################################################################
###                           gRPC Configuration                            ###
###############################################################################

[grpc]

# Enable defines if the gRPC server should be enabled.
enable = false

# Address defines the gRPC server address to bind to.
address = "0.0.0.0:9090"

# MaxRecvMsgSize defines the max message size in bytes the server can receive.
# The default value is 10MB.
max-recv-msg-size = "10485760"

# MaxSendMsgSize defines the max message size in bytes the server can send.
# The default value is math.MaxInt32.
max-send-msg-size = "2147483647"

###############################################################################
###                        gRPC Web Configuration                           ###
###############################################################################

[grpc-web]

# GRPCWebEnable defines if the gRPC-web should be enabled.
# NOTE: gRPC must also be enabled, otherwise, this configuration is a no-op.
enable = false

# Address defines the gRPC-web server address to bind to.
address = "0.0.0.0:9091"

# EnableUnsafeCORS defines if CORS should be enabled (unsafe - use it at your own risk).
enable-unsafe-cors = false

###############################################################################
###                        State Sync Configuration                         ###
###############################################################################

# State sync snapshots allow other nodes to rapidly join the network without replaying historical
# blocks, instead downloading and applying a snapshot of the application state at a given height.
[state-sync]

# snapshot-interval specifies the block interval at which local state sync snapshots are
# taken (0 to disable).
snapshot-interval = 0

# snapshot-keep-recent specifies the number of recent snapshots to keep and serve (0 to keep all).
snapshot-keep-recent = 2

###############################################################################
###                         Store / State Streaming                         ###
###############################################################################

[store]
streamers = []

[streamers]
[streamers.file]
keys = ["*", ]
write_dir = ""
prefix = ""

# output-metadata specifies if output the metadata file which includes the abci request/responses 
# during processing the block.
output-metadata = "true"

# stop-node-on-error specifies if propagate the file streamer errors to consensus state machine.
stop-node-on-error = "true"

# fsync specifies if call fsync after writing the files.
fsync = "false"

###############################################################################
###                             EVM Configuration                           ###
###############################################################################

[evm]

# Tracer defines the 'vm.Tracer' type that the EVM will use when the node is run in
# debug mode. To enable tracing use the '--evm.tracer' flag when starting your node.
# Valid types are: json|struct|access_list|markdown
tracer = ""

# MaxTxGasWanted defines the gas wanted for each eth tx returned in ante handler in check tx mode.
max-tx-gas-wanted = 0

###############################################################################
###                           JSON RPC Configuration                        ###
###############################################################################

[json-rpc]

# Enable defines if the gRPC server should be enabled.
enable = false

# Address defines the EVM RPC HTTP server address to bind to.
address = "127.0.0.1:8545"

# Address defines the EVM WebSocket server address to bind to.
ws-address = "127.0.0.1:8546"

# API defines a list of JSON-RPC namespaces that should be enabled
# Example: "eth,txpool,personal,net,debug,web3"
api = "eth,net,web3"

# GasCap sets a cap on gas that can be used in eth_call/estimateGas (0=infinite). Default: 25,000,000.
gas-cap = 25000000

# EVMTimeout is the global timeout for eth_call. Default: 5s.
evm-timeout = "5s"

# TxFeeCap is the global tx-fee cap for send transaction. Default: 1eth.
txfee-cap = 1

# FilterCap sets the global cap for total number of filters that can be created
filter-cap = 200

# FeeHistoryCap sets the global cap for total number of blocks that can be fetched
feehistory-cap = 100

# LogsCap defines the max number of results can be returned from single 'eth_getLogs' query.
logs-cap = 10000

# BlockRangeCap defines the max block range allowed for 'eth_getLogs' query.
block-range-cap = 10000

# HTTPTimeout is the read/write timeout of http json-rpc server.
http-timeout = "30s"

# HTTPIdleTimeout is the idle timeout of http json-rpc server.
http-idle-timeout = "2m0s"

# AllowUnprotectedTxs restricts unprotected (non EIP155 signed) transactions to be submitted via
# the node's RPC when the global parameter is disabled.
allow-unprotected-txs = false

# MaxOpenConnections sets the maximum number of simultaneous connections
# for the server listener.
max-open-connections = 0

# EnableIndexer enables the custom transaction indexer for the EVM (ethereum transactions).
enable-indexer = false

# MetricsAddress defines the EVM Metrics server address to bind to. Pass --metrics in CLI to enable
# Prometheus metrics path: /debug/metrics/prometheus
metrics-address = "127.0.0.1:6065"

# Upgrade height for fix of revert gas refund logic when transaction reverted.
fix-revert-gas-refund-height = 0

###############################################################################
###                             TLS Configuration                           ###
###############################################################################

[tls]

# Certificate path defines the cert.pem file path for the TLS configuration.
certificate-path = ""

# Key path defines the key.pem file path for the TLS configuration.
key-path = ""

zetacored logs

Started Zetachain testnet validator.
6:04PM INF Unlocking keyring module=server
6:04PM INF starting ABCI with Tendermint module=server
panic: runtime error: makeslice: len out of range
goroutine 1 [running]:
github.com/syndtr/goleveldb/leveldb.(*sessionRecord).readBytes(0xc000ca3500, {0x33e25e6, 0x8}, {0x7f930c2ddb78, 0xc000bb8040})
        /root/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20210819022825-2ae1ddf74ef7/leveldb/session_record.go:236 +0xa5
github.com/syndtr/goleveldb/leveldb.(*sessionRecord).decode(0xc000ca3500, {0x461aba0, 0xc000bb8040})
        /root/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20210819022825-2ae1ddf74ef7/leveldb/session_record.go:274 +0x2a5
github.com/syndtr/goleveldb/leveldb.(*session).recover(0xc0016d0960)
        /root/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20210819022825-2ae1ddf74ef7/leveldb/session.go:170 +0x4cb
github.com/syndtr/goleveldb/leveldb.Open({0x46655a0?, 0xc000bde4d0?}, 0x553500?)
        /root/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20210819022825-2ae1ddf74ef7/leveldb/db.go:189 +0x95
github.com/syndtr/goleveldb/leveldb.OpenFile({0xc0017bd940?, 0x33e7c79?}, 0x0)
        /root/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20210819022825-2ae1ddf74ef7/leveldb/db.go:225 +0x50
github.com/cometbft/cometbft-db.NewGoLevelDBWithOpts({0x33e7c79?, 0xc00159d798?}, {0xc0017bd900?, 0x31?}, 0x0)
        /root/go/pkg/mod/github.com/notional-labs/cometbft-db@v0.0.0-20230321185329-6dc7c0ca6345/goleveldb.go:32 +0x7c
github.com/cometbft/cometbft-db.NewGoLevelDB(...)
        /root/go/pkg/mod/github.com/notional-labs/cometbft-db@v0.0.0-20230321185329-6dc7c0ca6345/goleveldb.go:27
github.com/cometbft/cometbft-db.init.0.func1({0x33e7c79?, 0xc00095fe00?}, {0xc0017bd900?, 0x9?})
        /root/go/pkg/mod/github.com/notional-labs/cometbft-db@v0.0.0-20230321185329-6dc7c0ca6345/goleveldb.go:15 +0x1f
github.com/cometbft/cometbft-db.NewDB({0x33e7c79, 0xb}, {0xc001b226c0, 0x9}, {0xc0017bd900, 0x31})
        /root/go/pkg/mod/github.com/notional-labs/cometbft-db@v0.0.0-20230321185329-6dc7c0ca6345/db.go:69 +0x10d
github.com/zeta-chain/zetacore/server.openDB({0x4613ae0?, 0xc0016f36c0?}, {0xc000fb55f0?, 0xc00159d960?}, {0xc001b226c0, 0x9})
        /go/src/github.com/zeta-chain/node/server/start.go:690 +0x8c
github.com/zeta-chain/zetacore/server.startInProcess(_, {{0x0, 0x0, 0x0}, {0x4669920, 0xc00192a3c0}, 0x0, {0xc00186cd30, 0xd}, {0x467d6c0, ...}, ...}, ...)
        /go/src/github.com/zeta-chain/node/server/start.go:365 +0x407
github.com/zeta-chain/zetacore/server.StartCmd.func2(0xc00173b208, {0x6562ae0?, 0x0?, 0x0?})
        /go/src/github.com/zeta-chain/node/server/start.go:159 +0x305
github.com/spf13/cobra.(*Command).execute(0xc00173b208, {0x6562ae0, 0x0, 0x0})
        /root/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:940 +0x882
github.com/spf13/cobra.(*Command).ExecuteC(0xc0016d6908)
        /root/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:1068 +0x3a5
github.com/spf13/cobra.(*Command).Execute(...)
        /root/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:992
github.com/spf13/cobra.(*Command).ExecuteContext(...)
        /root/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:985
github.com/cosmos/cosmos-sdk/server/cmd.Execute(0xc0016d6908, {0x0, 0x0}, {0xc000fb55f0, 0x2c})
        /root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.47.10/server/cmd/execute.go:33 +0x16e
main.main()
        /go/src/github.com/zeta-chain/node/cmd/zetacored/main.go:20 +0x2f
zetachain-testnet-validator.service: Main process exited, code=exited, status=2/INVALIDARGUMENT
zetachain-testnet-validator.service: Failed with result 'exit-code'.
zetachain-testnet-validator.service: Consumed 5.163s CPU time.
vchong commented 1 month ago

Are you using the release binaries or building from source?

from source

zetacored version --long
...
build_tags: ""
commit: 811fa58b656a9b52588d4364b2da3f374d24672e
cosmos_sdk_version: v0.47.10
go: go version go1.22.6 linux/amd64
name: zetacore
server_name: <appd>
version: 19.0.0

uname -a command output

Linux [redacted] 5.4.0-153-generic #170-Ubuntu SMP Fri Jun 16 13:43:31 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
Linux [redacted] 5.4.0-156-generic #173-Ubuntu SMP Tue Jul 11 07:25:22 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux

/etc/os-release

NAME="Ubuntu"
VERSION="20.04.6 LTS (Focal Fossa)"
ID=ubuntu
ID_LIKE=debian
PRETTY_NAME="Ubuntu 20.04.6 LTS"
VERSION_ID="20.04"
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
VERSION_CODENAME=focal
UBUNTU_CODENAME=focal

.zetacored/config/config.toml


# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml

NOTE: Any path below can be absolute (e.g. "/var/myawesomeapp/data") or

relative to the home directory (e.g. "data"). The home directory is

"$HOME/.tendermint" by default, but could be changed via $TMHOME env variable

or --home cmd flag.

#######################################################################

Main Base Config Options

#######################################################################

TCP or UNIX socket address of the ABCI application,

or the name of an ABCI application compiled in with the Tendermint binary

proxy_app = "tcp://127.0.0.1:26658"

A custom human readable name for this node

moniker = "[redacted] "

If this node is many blocks behind the tip of the chain, FastSync

allows them to catchup quickly by downloading blocks in parallel

and verifying their commits

fast_sync = true

Database backend: goleveldb | cleveldb | boltdb | rocksdb | badgerdb

* goleveldb (github.com/syndtr/goleveldb - most popular implementation)

- pure go

- stable

* cleveldb (uses levigo wrapper)

- fast

- requires gcc

- use cleveldb build tag (go build -tags cleveldb)

* boltdb (uses etcd's fork of bolt - github.com/etcd-io/bbolt)

- EXPERIMENTAL

- may be faster is some use-cases (random reads - indexer)

- use boltdb build tag (go build -tags boltdb)

* rocksdb (uses github.com/tecbot/gorocksdb)

- EXPERIMENTAL

- requires gcc

- use rocksdb build tag (go build -tags rocksdb)

* badgerdb (uses github.com/dgraph-io/badger)

- EXPERIMENTAL

- use badgerdb build tag (go build -tags badgerdb)

db_backend = "goleveldb"

Database directory

db_dir = "data"

Output level for logging, including package level options

log_level = "info"

Output format: 'plain' (colored text) or 'json'

log_format = "plain"

additional base config options

Path to the JSON file containing the initial validator set and other meta data

genesis_file = "config/genesis.json"

Path to the JSON file containing the private key to use as a validator in the consensus protocol

priv_validator_key_file = "config/priv_validator_key.json"

Path to the JSON file containing the last sign state of a validator

priv_validator_state_file = "data/priv_validator_state.json"

TCP or UNIX socket address for Tendermint to listen on for

connections from an external PrivValidator process

priv_validator_laddr = ""

Path to the JSON file containing the private key to use for node authentication in the p2p protocol

node_key_file = "config/node_key.json"

Mechanism to connect to the ABCI application: socket | grpc

abci = "socket"

If true, query the ABCI app on connecting to a new peer

so the app can decide if we should keep the connection or not

filter_peers = false

#######################################################################

Advanced Configuration Options

#######################################################################

#######################################################

RPC Server Configuration Options

####################################################### [rpc]

TCP or UNIX socket address for the RPC server to listen on

laddr = "tcp://0.0.0.0:26657"

A list of origins a cross-domain request can be executed from

Default value '[]' disables cors support

Use '["*"]' to allow any origin

cors_allowed_origins = []

A list of methods the client is allowed to use with cross-domain requests

cors_allowed_methods = ["HEAD", "GET", "POST", ]

A list of non simple headers the client is allowed to use with cross-domain requests

cors_allowed_headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time", ]

TCP or UNIX socket address for the gRPC server to listen on

NOTE: This server only supports /broadcast_tx_commit

grpc_laddr = ""

Maximum number of simultaneous connections.

Does not include RPC (HTTP&WebSocket) connections. See max_open_connections

If you want to accept a larger number than the default, make sure

you increase your OS limits.

0 - unlimited.

Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}

1024 - 40 - 10 - 50 = 924 = ~900

grpc_max_open_connections = 900

Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool

unsafe = false

Maximum number of simultaneous connections (including WebSocket).

Does not include gRPC connections. See grpc_max_open_connections

If you want to accept a larger number than the default, make sure

you increase your OS limits.

0 - unlimited.

Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}

1024 - 40 - 10 - 50 = 924 = ~900

max_open_connections = 900

Maximum number of unique clientIDs that can /subscribe

If you're using /broadcast_tx_commit, set to the estimated maximum number

of broadcast_tx_commit calls per block.

max_subscription_clients = 100

Maximum number of unique queries a given client can /subscribe to

If you're using GRPC (or Local RPC client) and /broadcast_tx_commit, set to

the estimated # maximum number of broadcast_tx_commit calls per block.

max_subscriptions_per_client = 5

Experimental parameter to specify the maximum number of events a node will

buffer, per subscription, before returning an error and closing the

subscription. Must be set to at least 100, but higher values will accommodate

higher event throughput rates (and will use more memory).

experimental_subscription_buffer_size = 200

Experimental parameter to specify the maximum number of RPC responses that

can be buffered per WebSocket client. If clients cannot read from the

WebSocket endpoint fast enough, they will be disconnected, so increasing this

parameter may reduce the chances of them being disconnected (but will cause

the node to use more memory).

#

Must be at least the same as "experimental_subscription_buffer_size",

otherwise connections could be dropped unnecessarily. This value should

ideally be somewhat higher than "experimental_subscription_buffer_size" to

accommodate non-subscription-related RPC responses.

experimental_websocket_write_buffer_size = 200

If a WebSocket client cannot read fast enough, at present we may

silently drop events instead of generating an error or disconnecting the

client.

#

Enabling this experimental parameter will cause the WebSocket connection to

be closed instead if it cannot read fast enough, allowing for greater

predictability in subscription behaviour.

experimental_close_on_slow_client = false

How long to wait for a tx to be committed during /broadcast_tx_commit.

WARNING: Using a value larger than 10s will result in increasing the

global HTTP write timeout, which applies to all connections and endpoints.

See https://github.com/tendermint/tendermint/issues/3435

timeout_broadcast_tx_commit = "10s"

Maximum size of request body, in bytes

max_body_bytes = 1000000

Maximum size of request header, in bytes

max_header_bytes = 1048576

The path to a file containing certificate that is used to create the HTTPS server.

Might be either absolute path or path related to Tendermint's config directory.

If the certificate is signed by a certificate authority,

the certFile should be the concatenation of the server's certificate, any intermediates,

and the CA's certificate.

NOTE: both tls_cert_file and tls_key_file must be present for Tendermint to create HTTPS server.

Otherwise, HTTP server is run.

tls_cert_file = ""

The path to a file containing matching private key that is used to create the HTTPS server.

Might be either absolute path or path related to Tendermint's config directory.

NOTE: both tls-cert-file and tls-key-file must be present for Tendermint to create HTTPS server.

Otherwise, HTTP server is run.

tls_key_file = ""

pprof listen address (https://golang.org/pkg/net/http/pprof)

pprof_laddr = "localhost:6060"

#######################################################

P2P Configuration Options

####################################################### [p2p]

Address to listen for incoming connections

laddr = "tcp://0.0.0.0:26656"

Address to advertise to peers for them to dial

If empty, will use the same port as the laddr,

and will introspect on the listener or use UPnP

to figure out the address. ip and port are required

example: 159.89.10.97:26656

external_address = "tcp://[redacted] "

Comma separated list of seed nodes to connect to

seeds = "[redacted] "

Comma separated list of nodes to keep persistent connections to

persistent_peers = "[redacted] "

UPNP port forwarding

upnp = false

Path to address book

addr_book_file = "config/addrbook.json"

Set true for strict address routability rules

Set false for private or local networks

addr_book_strict = true

Maximum number of inbound peers

max_num_inbound_peers = 20

Maximum number of outbound peers to connect to, excluding persistent peers

max_num_outbound_peers = 10

List of node IDs, to which a connection will be (re)established ignoring any existing limits

unconditional_peer_ids = "[redacted] "

Maximum pause when redialing a persistent peer (if zero, exponential backoff is used)

persistent_peers_max_dial_period = "0s"

Time to wait before flushing messages out on the connection

flush_throttle_timeout = "100ms"

Maximum size of a message packet payload, in bytes

max_packet_msg_payload_size = 1024

Rate at which packets can be sent, in bytes/second

send_rate = 5120000

Rate at which packets can be received, in bytes/second

recv_rate = 5120000

Set true to enable the peer-exchange reactor

pex = true

Seed mode, in which node constantly crawls the network and looks for

peers. If another node asks it for addresses, it responds and disconnects.

#

Does not work if the peer-exchange reactor is disabled.

seed_mode = false

Comma separated list of peer IDs to keep private (will not be gossiped to other peers)

private_peer_ids = ""

Toggle to disable guard against peers connecting from the same ip.

allow_duplicate_ip = false

Peer connection configuration.

handshake_timeout = "20s" dial_timeout = "3s"

#######################################################

Mempool Configuration Option

####################################################### [mempool]

Mempool version to use:

1) "v0" - (default) FIFO mempool.

2) "v1" - prioritized mempool.

version = "v0"

recheck = true broadcast = true wal_dir = ""

Maximum number of transactions in the mempool

size = 5000

Limit the total size of all txs in the mempool.

This only accounts for raw transactions (e.g. given 1MB transactions and

max_txs_bytes=5MB, mempool will only accept 5 transactions).

max_txs_bytes = 1073741824

Size of the cache (used to filter transactions we saw earlier) in transactions

cache_size = 10000

Do not remove invalid transactions from the cache (default: false)

Set to true if it's not possible for any invalid transaction to become valid

again in the future.

keep-invalid-txs-in-cache = false

Maximum size of a single transaction.

NOTE: the max size of a tx transmitted over the network is {max_tx_bytes}.

max_tx_bytes = 1048576

Maximum size of a batch of transactions to send to a peer

Including space needed by encoding (one varint per transaction).

XXX: Unused due to https://github.com/tendermint/tendermint/issues/5796

max_batch_bytes = 0

ttl-duration, if non-zero, defines the maximum amount of time a transaction

can exist for in the mempool.

#

Note, if ttl-num-blocks is also defined, a transaction will be removed if it

has existed in the mempool at least ttl-num-blocks number of blocks or if it's

insertion time into the mempool is beyond ttl-duration.

ttl-duration = "0s"

ttl-num-blocks, if non-zero, defines the maximum number of blocks a transaction

can exist for in the mempool.

#

Note, if ttl-duration is also defined, a transaction will be removed if it

has existed in the mempool at least ttl-num-blocks number of blocks or if

it's insertion time into the mempool is beyond ttl-duration.

ttl-num-blocks = 0

#######################################################

State Sync Configuration Options

####################################################### [statesync]

State sync rapidly bootstraps a new node by discovering, fetching, and restoring a state machine

snapshot from peers instead of fetching and replaying historical blocks. Requires some peers in

the network to take and serve state machine snapshots. State sync is not attempted if the node

has any local state (LastBlockHeight > 0). The node will have a truncated block history,

starting from the height of the snapshot.

enable = false

RPC servers (comma-separated) for light client verification of the synced state machine and

retrieval of state data for node bootstrapping. Also needs a trusted height and corresponding

header hash obtained from a trusted source, and a period during which validators can be trusted.

#

For Cosmos SDK-based chains, trust_period should usually be about 2/3 of the unbonding time (~2

weeks) during which they can be financially punished (slashed) for misbehavior.

rpc_servers = "" trust_height = 0 trust_hash = "" trust_period = "168h0m0s"

Time to spend discovering snapshots before initiating a restore.

discovery_time = "15s"

Temporary directory for state sync snapshot chunks, defaults to the OS tempdir (typically /tmp).

Will create a new, randomly named directory within, and remove it when done.

temp_dir = ""

The timeout duration before re-requesting a chunk, possibly from a different

peer (default: 1 minute).

chunk_request_timeout = "10s"

The number of concurrent chunk fetchers to run (default: 1).

chunk_fetchers = "4"

#######################################################

Fast Sync Configuration Connections

####################################################### [fastsync]

Fast Sync version to use:

1) "v0" (default) - the legacy fast sync implementation

2) "v1" - refactor of v0 version for better testability

2) "v2" - complete redesign of v0, optimized for testability & readability

version = "v0"

#######################################################

Consensus Configuration Options

####################################################### [consensus]

wal_file = "data/cs.wal/wal"

How long we wait for a proposal block before prevoting nil

timeout_propose = "3s"

How much timeout_propose increases with each round

timeout_propose_delta = "500ms"

How long we wait after receiving +2/3 prevotes for “anything” (ie. not a single block or nil)

timeout_prevote = "1s"

How much the timeout_prevote increases with each round

timeout_prevote_delta = "500ms"

How long we wait after receiving +2/3 precommits for “anything” (ie. not a single block or nil)

timeout_precommit = "1s"

How much the timeout_precommit increases with each round

timeout_precommit_delta = "500ms"

How long we wait after committing a block, before starting on the new

height (this gives us a chance to receive some more precommits, even

though we already have +2/3).

timeout_commit = "5s"

How many blocks to look back to check existence of the node's consensus votes before joining consensus

When non-zero, the node will panic upon restart

if the same consensus key was used to sign {double_sign_check_height} last blocks.

So, validators should stop the state machine, wait for some blocks, and then restart the state machine to avoid panic.

double_sign_check_height = 0

Make progress as soon as we have all the precommits (as if TimeoutCommit = 0)

skip_timeout_commit = false

EmptyBlocks mode and possible interval between empty blocks

create_empty_blocks = true create_empty_blocks_interval = "0s"

Reactor sleep duration parameters

peer_gossip_sleep_duration = "100ms" peer_query_maj23_sleep_duration = "2s"

#######################################################

Storage Configuration Options

####################################################### [storage]

Set to true to discard ABCI responses from the state store, which can save a

considerable amount of disk space. Set to false to ensure ABCI responses are

persisted. ABCI responses are required for /block_results RPC queries, and to

reindex events in the command-line tool.

discard_abci_responses = false

#######################################################

Transaction Indexer Configuration Options

####################################################### [tx_index]

What indexer to use for transactions

#

The application will set which txs to index. In some cases a node operator will be able

to decide which txs to index based on configuration set in the application.

#

Options:

1) "null"

2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).

- When "kv" is chosen "tx.height" and "tx.hash" will always be indexed.

3) "psql" - the indexer services backed by PostgreSQL.

When "kv" or "psql" is chosen "tx.height" and "tx.hash" will always be indexed.

indexer = "kv"

The PostgreSQL connection configuration, the connection format:

postgresql://user>:<password>@<host:/?

psql-conn = ""

#######################################################

Instrumentation Configuration Options

####################################################### [instrumentation]

When true, Prometheus metrics are served under /metrics on

PrometheusListenAddr.

Check out the documentation for the list of available metrics.

prometheus = true

Address to listen for Prometheus collector(s) connections

prometheus_listen_addr = ":26660"

Maximum number of simultaneous connections.

If you want to accept a larger number than the default, make sure

you increase your OS limits.

0 - unlimited.

max_open_connections = 3

Instrumentation namespace

namespace = "cometbft"


> `.zetacored/config/app.toml`

This is a TOML config file.

For more information, see https://github.com/toml-lang/toml

###############################################################################

Base Configuration

###############################################################################

The minimum gas prices a validator is willing to accept for processing a

transaction. A transaction's fees must meet the minimum of any denomination

specified in this config (e.g. 0.25token1;0.0001token2).

minimum-gas-prices = "0.1azeta"

default: the last 100 states are kept in addition to every 500th state; pruning at 10 block intervals

nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node)

everything: all saved states will be deleted, storing only the current and previous state; pruning at 10 block intervals

custom: allow pruning options to be manually specified through 'pruning-keep-recent', 'pruning-keep-every', and 'pruning-interval'

pruning = "default"

These are applied if and only if the pruning strategy is custom.

pruning-keep-recent = "0" pruning-keep-every = "0" pruning-interval = "0"

HaltHeight contains a non-zero block height at which a node will gracefully

halt and shutdown that can be used to assist upgrades and testing.

#

Note: Commitment of state will be attempted on the corresponding block.

halt-height = 0

HaltTime contains a non-zero minimum block time (in Unix seconds) at which

a node will gracefully halt and shutdown that can be used to assist upgrades

and testing.

#

Note: Commitment of state will be attempted on the corresponding block.

halt-time = 0

MinRetainBlocks defines the minimum block height offset from the current

block being committed, such that all blocks past this offset are pruned

from Tendermint. It is used as part of the process of determining the

ResponseCommit.RetainHeight value during ABCI Commit. A value of 0 indicates

that no blocks should be pruned.

#

This configuration value is only responsible for pruning Tendermint blocks.

It has no bearing on application state pruning which is determined by the

"pruning-*" configurations.

#

Note: Tendermint block pruning is dependant on this parameter in conunction

with the unbonding (safety threshold) period, state pruning and state sync

snapshot parameters to determine the correct minimum value of

ResponseCommit.RetainHeight.

min-retain-blocks = 0

InterBlockCache enables inter-block caching.

inter-block-cache = true

IndexEvents defines the set of events in the form {eventType}.{attributeKey},

which informs Tendermint what to index. If empty, all events will be indexed.

#

Example:

["message.sender", "message.recipient"]

index-events = []

IavlCacheSize set the size of the iavl tree cache.

Default cache size is 50mb.

iavl-cache-size = 781250

IAVLDisableFastNode enables or disables the fast node feature of IAVL.

Default is true.

iavl-disable-fastnode = true

###############################################################################

Telemetry Configuration

###############################################################################

[telemetry]

Prefixed with keys to separate services.

service-name = ""

Enabled enables the application telemetry functionality. When enabled,

an in-memory sink is also enabled by default. Operators may also enabled

other sinks such as Prometheus.

enabled = false

Enable prefixing gauge values with hostname.

enable-hostname = false

Enable adding hostname to labels.

enable-hostname-label = false

Enable adding service to labels.

enable-service-label = false

PrometheusRetentionTime, when positive, enables a Prometheus metrics sink.

prometheus-retention-time = 0

GlobalLabels defines a global set of name/value label tuples applied to all

metrics emitted using the wrapper functions defined in telemetry package.

#

Example:

[["chain_id", "cosmoshub-1"]]

global-labels = [ ]

###############################################################################

API Configuration

###############################################################################

[api]

Enable defines if the API server should be enabled.

enable = true

Swagger defines if swagger documentation should automatically be registered.

swagger = false

Address defines the API server to listen on.

address = "tcp://0.0.0.0:1317"

MaxOpenConnections defines the number of maximum open connections.

max-open-connections = 1000

RPCReadTimeout defines the Tendermint RPC read timeout (in seconds).

rpc-read-timeout = 10

RPCWriteTimeout defines the Tendermint RPC write timeout (in seconds).

rpc-write-timeout = 0

RPCMaxBodyBytes defines the Tendermint maximum response body (in bytes).

rpc-max-body-bytes = 1000000

EnableUnsafeCORS defines if CORS should be enabled (unsafe - use it at your own risk).

enabled-unsafe-cors = false

###############################################################################

Rosetta Configuration

###############################################################################

[rosetta]

Enable defines if the Rosetta API server should be enabled.

enable = false

Address defines the Rosetta API server to listen on.

address = ":8080"

Network defines the name of the blockchain that will be returned by Rosetta.

blockchain = "app"

Network defines the name of the network that will be returned by Rosetta.

network = "network"

Retries defines the number of retries when connecting to the node before failing.

retries = 3

Offline defines if Rosetta server should run in offline mode.

offline = false

###############################################################################

gRPC Configuration

###############################################################################

[grpc]

Enable defines if the gRPC server should be enabled.

enable = true

Address defines the gRPC server address to bind to.

address = "0.0.0.0:9090"

###############################################################################

gRPC Web Configuration

###############################################################################

[grpc-web]

GRPCWebEnable defines if the gRPC-web should be enabled.

NOTE: gRPC must also be enabled, otherwise, this configuration is a no-op.

enable = true

Address defines the gRPC-web server address to bind to.

address = "0.0.0.0:9091"

EnableUnsafeCORS defines if CORS should be enabled (unsafe - use it at your own risk).

enable-unsafe-cors = false

###############################################################################

State Sync Configuration

###############################################################################

State sync snapshots allow other nodes to rapidly join the network without replaying historical

blocks, instead downloading and applying a snapshot of the application state at a given height.

[state-sync]

snapshot-interval specifies the block interval at which local state sync snapshots are

taken (0 to disable). Must be a multiple of pruning-keep-every.

snapshot-interval = 0

snapshot-keep-recent specifies the number of recent snapshots to keep and serve (0 to keep all).

snapshot-keep-recent = 2

###############################################################################

EVM Configuration

###############################################################################

[evm]

Tracer defines the 'vm.Tracer' type that the EVM will use when the node is run in

debug mode. To enable tracing use the '--evm.tracer' flag when starting your node.

Valid types are: json|struct|access_list|markdown

tracer = ""

MaxTxGasWanted defines the gas wanted for each eth tx returned in ante handler in check tx mode.

max-tx-gas-wanted = 0

###############################################################################

JSON RPC Configuration

###############################################################################

[json-rpc]

Enable defines if the gRPC server should be enabled.

enable = true

Address defines the EVM RPC HTTP server address to bind to.

address = "0.0.0.0:8545"

Address defines the EVM WebSocket server address to bind to.

ws-address = "0.0.0.0:8546"

API defines a list of JSON-RPC namespaces that should be enabled

Example: "eth,txpool,personal,net,debug,web3"

api = "eth,net,web3,txpool,personal,debug"

GasCap sets a cap on gas that can be used in eth_call/estimateGas (0=infinite). Default: 25,000,000.

gas-cap = 25000000

EVMTimeout is the global timeout for eth_call. Default: 5s.

evm-timeout = "5s"

TxFeeCap is the global tx-fee cap for send transaction. Default: 1eth.

txfee-cap = 1

FilterCap sets the global cap for total number of filters that can be created

filter-cap = 200

FeeHistoryCap sets the global cap for total number of blocks that can be fetched

feehistory-cap = 100

LogsCap defines the max number of results can be returned from single 'eth_getLogs' query.

logs-cap = 10000

BlockRangeCap defines the max block range allowed for 'eth_getLogs' query.

block-range-cap = 10000

HTTPTimeout is the read/write timeout of http json-rpc server.

http-timeout = "30s"

HTTPIdleTimeout is the idle timeout of http json-rpc server.

http-idle-timeout = "2m0s"

AllowUnprotectedTxs restricts unprotected (non EIP155 signed) transactions to be submitted via

the node's RPC when the global parameter is disabled.

allow-unprotected-txs = false

MaxOpenConnections sets the maximum number of simultaneous connections

for the server listener.

max-open-connections = 0

EnableIndexer enables the custom transaction indexer for the EVM (ethereum transactions).

enable-indexer = false

###############################################################################

TLS Configuration

###############################################################################

[tls]

Certificate path defines the cert.pem file path for the TLS configuration.

certificate-path = ""

Key path defines the key.pem file path for the TLS configuration.

key-path = ""


> zetacored logs

sentry node

3:07PM INF Unlocking keyring module=server 3:07PM INF starting ABCI with Tendermint module=server fatal error: runtime: out of memory

runtime stack: runtime.throw({0x3407d43?, 0x210823?}) /usr/local/go/src/runtime/panic.go:1023 +0x5c fp=0x7fda037fdcb8 sp=0x7fda037fdc88 pc=0x441dbc runtime.sysMapOS(0xc010800000, 0x3607ac00000) /usr/local/go/src/runtime/mem_linux.go:167 +0x11b fp=0x7fda037fdcf8 sp=0x7fda037fdcb8 pc=0x41f89b runtime.sysMap(0xc010800000, 0x3607ac00000, 0x6517d48?) /usr/local/go/src/runtime/mem.go:155 +0x34 fp=0x7fda037fdd18 sp=0x7fda037fdcf8 pc=0x41f314 runtime.(mheap).grow(0x6507b40, 0x1b03d4c7?) /usr/local/go/src/runtime/mheap.go:1534 +0x236 fp=0x7fda037fdd88 sp=0x7fda037fdd18 pc=0x4326d6 runtime.(mheap).allocSpan(0x6507b40, 0x1b03d4c7, 0x0, 0x1) /usr/local/go/src/runtime/mheap.go:1246 +0x1b0 fp=0x7fda037fde28 sp=0x7fda037fdd88 pc=0x431db0 runtime.(*mheap).alloc.func1() /usr/local/go/src/runtime/mheap.go:964 +0x5c fp=0x7fda037fde70 sp=0x7fda037fde28 pc=0x43185c runtime.systemstack(0x800000) ...

validator node

3:07PM INF Unlocking keyring module=server 3:07PM INF starting ABCI with Tendermint module=server 3:07PM ERR failed to open DB error="failed to initialize database: EOF" module=server Error: failed to initialize database: EOF Usage: zetacored start [flags]

Flags: --abci string specify abci transport (socket | grpc) (default "socket") ...



> additional debugging

Tried up to 256G swap but still the same logs as above.
gartnera commented 1 month ago

Are you using the release binaries or building from source?

release binaries

uname -a

Linux alexg-test 6.5.0-1025-gcp #27~22.04.1-Ubuntu SMP Tue Jul 16 23:03:39 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux

/etc/os-release

PRETTY_NAME="Ubuntu 22.04.4 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.4 LTS (Jammy Jellyfish)"
VERSION_CODENAME=jammy
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=jammy

config.toml

proxy_app = "tcp://127.0.0.1:26658"

# IMPORTANT! Replace the placeholder below with your actual moniker
moniker = "cf53cacfe8df098f1f280636a1008aed"

fast_sync = true
db_backend = "goleveldb"
db_dir = "data"
log_level = "info"
log_format = "plain"
genesis_file = "config/genesis.json"
priv_validator_key_file = "config/priv_validator_key.json"
priv_validator_state_file = "data/priv_validator_state.json"
priv_validator_laddr = ""
node_key_file = "config/node_key.json"
abci = "socket"
filter_peers = false

[rpc]
laddr = "tcp://0.0.0.0:26657"
cors_allowed_origins = []
cors_allowed_methods = ["HEAD", "GET", "POST", ]
cors_allowed_headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time", ]
grpc_laddr = ""
grpc_max_open_connections = 900
unsafe = false
max_open_connections = 900
max_subscription_clients = 100
max_subscriptions_per_client = 5
experimental_subscription_buffer_size = 200
experimental_websocket_write_buffer_size = 200
experimental_close_on_slow_client = false
timeout_broadcast_tx_commit = "10s"
max_body_bytes = 1000000
max_header_bytes = 1048576
tls_cert_file = ""
tls_key_file = ""
pprof_laddr = "localhost:6060"

[p2p]
laddr = "tcp://0.0.0.0:26656"

# IMPORTANT! Replace the placeholder below with your actual external IP address
external_address = "34.72.222.69:26656"

seeds = "5aaa51a3b9465a32f7f6c9df1d46d4bfcc16aecb@34.30.34.119:26656,ade4d8bc8cbe014af6ebdf3cb7b1e9ad36f412c0@testnet-seeds.polkachu.com:22556"
persistent_peers = ""
upnp = false
addr_book_file = "config/addrbook.json"
addr_book_strict = true
max_num_inbound_peers = 40
max_num_outbound_peers = 10
unconditional_peer_ids = ""
persistent_peers_max_dial_period = "0s"
flush_throttle_timeout = "100ms"
max_packet_msg_payload_size = 1024
send_rate = 5120000
recv_rate = 5120000
pex = true
seed_mode = false
private_peer_ids = ""
allow_duplicate_ip = false
handshake_timeout = "20s"
dial_timeout = "3s"

[mempool]
version = "v0"
recheck = true
broadcast = true
wal_dir = ""

size = 5000
max_txs_bytes = 1073741824
cache_size = 10000
keep-invalid-txs-in-cache = false
max_tx_bytes = 1048576
max_batch_bytes = 0
ttl-duration = "0s"
ttl-num-blocks = 0

[statesync]
enable = false
rpc_servers = ""
trust_height = 0
trust_hash = ""
trust_period = "168h0m0s"
discovery_time = "15s"
temp_dir = ""
chunk_request_timeout = "10s"
chunk_fetchers = "4"

[fastsync]
version = "v0"

[consensus]
wal_file = "data/cs.wal/wal"
timeout_propose = "3s"
timeout_propose_delta = "500ms"
timeout_prevote = "1s"
timeout_prevote_delta = "500ms"
timeout_precommit = "1s"
timeout_precommit_delta = "500ms"
timeout_commit = "5s"
double_sign_check_height = 0
skip_timeout_commit = false
create_empty_blocks = true
create_empty_blocks_interval = "0s"
peer_gossip_sleep_duration = "100ms"
peer_query_maj23_sleep_duration = "2s"

[storage]
discard_abci_responses = false

[tx_index]
indexer = "kv"
psql-conn = ""

[instrumentation]
prometheus = true
prometheus_listen_addr = ":26660"
max_open_connections = 3
namespace = "tendermint"

app.toml

minimum-gas-prices = "1.0azeta"
pruning = "default"
pruning-keep-recent = "100"
pruning-keep-every = "0"
pruning-interval = "10"
halt-height = 0
halt-time = 0
min-retain-blocks = 0
inter-block-cache = true
index-events = []
iavl-cache-size = 781250
iavl-disable-fastnode = true

[telemetry]
service-name = "tss"
enabled = true
enable-hostname = true
enable-hostname-label = true
enable-service-label = false
prometheus-retention-time = 10
global-labels = [ ]

[api]
enable = true
swagger = true
address = "tcp://0.0.0.0:1317"
max-open-connections = 1000
rpc-read-timeout = 10
rpc-write-timeout = 0
rpc-max-body-bytes = 1000000
enabled-unsafe-cors = false

[rosetta]
enable = false
address = ":8080"
blockchain = "app"
network = "network"
retries = 3
offline = false

[grpc]
enable = true
address = "0.0.0.0:9090"

[grpc-web]
enable = true
address = "0.0.0.0:9091"
enable-unsafe-cors = false

[state-sync]
snapshot-interval = 0
snapshot-keep-recent = 2

[evm]
tracer = ""
max-tx-gas-wanted = 0

[json-rpc]
enable = true
address = "0.0.0.0:8545"
ws-address = "0.0.0.0:8546"
api = "eth,net,web3,txpool,personal,debug"
gas-cap = 25000000
evm-timeout = "5s"
txfee-cap = 1
filter-cap = 200
feehistory-cap = 100
logs-cap = 10000
block-range-cap = 10000
http-timeout = "30s"
http-idle-timeout = "2m0s"
allow-unprotected-txs = false
max-open-connections = 0
enable-indexer = false

[tls]
certificate-path = ""

logs

zetachain-goleveldb@alexg-test:~$ ./zetacored.v19 start --moniker cf53cacfe8df098f1f280636a1008aed
6:49PM INF Unlocking keyring module=server
6:49PM INF starting ABCI with Tendermint module=server
6:49PM ERR failed to open DB error="failed to initialize database: leveldb: manifest corrupted (field 'comparer'): mismatch: want 'leveldb.BytewiseComparator', got '\x04���\x03��\x1e2s/k:bank/n�9\x14��J.U\n�ٿ�$,�B-���5���F�)�\x1fÚ\x00�N��\x02\x00\x002s/k:bank/n�II>���A�
#\x1cnný\x1a\x16�o\x7f6�\f�\x05���GCT�\x00' [file=MANIFEST-7519372]" module=server
Error: failed to initialize database: leveldb: manifest corrupted (field 'comparer'): mismatch: want 'leveldb.BytewiseComparator', got '2s/k:bank/n9J.U
ٿ$,B-5F)ÚN2s/k:bank/nII>A#nnýo6
                               GCT' [file=MANIFEST-7519372]
Usage:
  zetacored start [flags]