lidofinance / alerting-forta

Forta Detection bots for Lido
https://app.forta.network/bots?search=0x6aea36c4a9cf8ac053c07e662fa27e8bdf047121&sort=recent
GNU General Public License v3.0
34 stars 7 forks source link

feat: added metrics and alerts #563

Closed sergeyWh1te closed 3 weeks ago

sergeyWh1te commented 1 month ago

Added metrics and alerts:

# HELP ethereum_steth_build_info Build information
# TYPE ethereum_steth_build_info gauge
ethereum_steth_build_info{commitHash="unknown",instance="forta"} 1

# HELP ethereum_steth_health_status Bot health status
# TYPE ethereum_steth_health_status gauge
ethereum_steth_health_status{instance="forta"} 1

# HELP ethereum_steth_etherjs_request_total Total number of requests via ether.js library
# TYPE ethereum_steth_etherjs_request_total counter
ethereum_steth_etherjs_request_total{method="getBlockNumber",status="ok",instance="forta"} 1
ethereum_steth_etherjs_request_total{method="getHistory",status="ok",instance="forta"} 2
ethereum_steth_etherjs_request_total{method="etherscanProvider.getBalance",status="ok",instance="forta"} 1
ethereum_steth_etherjs_request_total{method="gate_seal_is_expired",status="ok",instance="forta"} 33
ethereum_steth_etherjs_request_total{method="vebo_hasRole",status="ok",instance="forta"} 33
ethereum_steth_etherjs_request_total{method="wd_queue_hasRole",status="ok",instance="forta"} 33
ethereum_steth_etherjs_request_total{method="wq_queue_is_bunker_mode_active",status="ok",instance="forta"} 1
ethereum_steth_etherjs_request_total{method="wq_queue_get_last_request_id",status="ok",instance="forta"} 33
ethereum_steth_etherjs_request_total{method="getWithdrawalStatus",status="ok",instance="forta"} 50
ethereum_steth_etherjs_request_total{method="getBufferedEther",status="ok",instance="forta"} 96
ethereum_steth_etherjs_request_total{method="getBalanceByBlockHash",status="ok",instance="forta"} 64
ethereum_steth_etherjs_request_total{method="gate_seal_get_expiry_timestamp",status="ok",instance="forta"} 32
ethereum_steth_etherjs_request_total{method="lido_get_eth_distributed_events",status="ok",instance="forta"} 32
ethereum_steth_etherjs_request_total{method="getBalance",status="ok",instance="forta"} 96
ethereum_steth_etherjs_request_total{method="getStakeLimitFullInfo",status="ok",instance="forta"} 1

# HELP ethereum_steth_block_timestamp The last agent iteration
# TYPE ethereum_steth_block_timestamp gauge
ethereum_steth_block_timestamp{method="handleBlock",instance="forta"} 1718294198792
ethereum_steth_block_timestamp{method="handleTx",instance="forta"} 1718294199035

# HELP ethereum_steth_last_block_number The last agent block number
# TYPE ethereum_steth_last_block_number gauge
ethereum_steth_last_block_number{instance="forta"} 20083936

# HELP ethereum_steth_network_errors_total Total number of network errors
# TYPE ethereum_steth_network_errors_total counter
ethereum_steth_network_errors_total{instance="forta"} 0

# HELP ethereum_steth_processed_iterations_total Total number of finding iterations
# TYPE ethereum_steth_processed_iterations_total counter
ethereum_steth_processed_iterations_total{method="handleTx",status="ok",instance="forta"} 5473
ethereum_steth_processed_iterations_total{method="handleBlock",status="ok",instance="forta"} 32

# HELP ethereum_steth_processed_handler_total Speed of handleBlock, handleTransaction
# TYPE ethereum_steth_processed_handler_total summary
ethereum_steth_processed_handler_total{quantile="0.01",method="handleTx",instance="forta"} 0.00027647087999999997
ethereum_steth_processed_handler_total{quantile="0.05",method="handleTx",instance="forta"} 0.00029739178703703704
ethereum_steth_processed_handler_total{quantile="0.5",method="handleTx",instance="forta"} 0.00041845614155844156
ethereum_steth_processed_handler_total{quantile="0.9",method="handleTx",instance="forta"} 0.0009950723000619194
ethereum_steth_processed_handler_total{quantile="0.95",method="handleTx",instance="forta"} 0.0013122745440215912
ethereum_steth_processed_handler_total{quantile="0.99",method="handleTx",instance="forta"} 0.0026317937753475777
ethereum_steth_processed_handler_total{quantile="0.999",method="handleTx",instance="forta"} 0.018416479755809612
ethereum_steth_processed_handler_total_sum{method="handleTx",instance="forta"} 3.7793900639999993
ethereum_steth_processed_handler_total_count{method="handleTx",instance="forta"} 5473
ethereum_steth_processed_handler_total{quantile="0.01",method="handleBlock",instance="forta"} 1.9408876670000001
ethereum_steth_processed_handler_total{quantile="0.05",method="handleBlock",instance="forta"} 1.9792696173
ethereum_steth_processed_handler_total{quantile="0.5",method="handleBlock",instance="forta"} 2.42412975
ethereum_steth_processed_handler_total{quantile="0.9",method="handleBlock",instance="forta"} 5.4716183418
ethereum_steth_processed_handler_total{quantile="0.95",method="handleBlock",instance="forta"} 5.8565598714
ethereum_steth_processed_handler_total{quantile="0.99",method="handleBlock",instance="forta"} 6.717290833
ethereum_steth_processed_handler_total{quantile="0.999",method="handleBlock",instance="forta"} 6.717290833
ethereum_steth_processed_handler_total_sum{method="handleBlock",instance="forta"} 92.997575914
ethereum_steth_processed_handler_total_count{method="handleBlock",instance="forta"} 32
Vinnie161 commented 1 month ago

It's ok, but please add a label rpcUrl to ethereum_steth_etherjs_request_total.

n0guest commented 1 month ago

LGTM in general, but I have one request. If we're talking about generic metrics like ethereum_steth_network_errors_total we should be able to understand which service/vendor/API cause these errors. Hence, we need a sub-dimension here as a label with a name of remote party.