search

helium / blockchain-etl

Blockchain follower that follows and stores the Helium blockchain
Apache License 2.0
64 stars 37 forks source link

ETL crash at height 455154 #232

Closed FTwOoO closed 3 years ago

FTwOoO commented 3 years ago

the ETL crash, restart, 10 minutes+, then crash again i run the ETL with https://snapshots.helium.wtf/blockchain-797990.db.tgz startup 2 weeks ago, DB table blocks rows is increasing, now stop at height 455154 2 days ago

crash.log:

root@iZj6c1yri0h6ip9mkkjj00Z:~# tail -f -n 100  ./etl_data/log/crash.log
    messages: []
    links: [<0.1230.0>,<0.1263.0>]
    dictionary: []
    trap_exit: false
    status: running
    heap_size: 1199557
    stack_size: 27
    reductions: 1085078
  neighbours:
2021-08-24 01:33:42 =SUPERVISOR REPORT====
     Supervisor: {local,be_sup}
     Context:    child_terminated
     Reason:     {{badmatch,{error,height_too_old}},[{blockchain_follower,'-handle_info/2-fun-0-',5,[{file,"blockchain_follower.erl"},{line,107}]},{lists,foldl,3,[{file,"lists.erl"},{line,1263}]},{blockchain_follower,handle_info,2,[{file,"blockchain_follower.erl"},{line,118}]},{blockchain_follower,handle_info,2,[{file,"blockchain_follower.erl"},{line,74}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,637}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,711}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}
     Offender:   [{pid,<0.1401.0>},{id,db_follower},{mfargs,{blockchain_follower,start_link,[[{follower_module,{be_db_follower,[{base_dir,"/var/data"}]}}]]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2021-08-24 01:33:42 =SUPERVISOR REPORT====
     Supervisor: {local,be_sup}
     Context:    shutdown
     Reason:     reached_max_restart_intensity
     Offender:   [{pid,<0.1401.0>},{id,db_follower},{mfargs,{blockchain_follower,start_link,[[{follower_module,{be_db_follower,[{base_dir,"/var/data"}]}}]]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2021-08-24 01:33:42 =ERROR REPORT====
** State machine <0.12200.0> terminating
** When server state  = {connected,{data,#Ref<0.1913172373.2502164481.186907>,blockchain_swarm,inbound,"gossip",[],<0.1246.0>,{undefined,undefined},undefined,{backoff,1000,10000,1000,normal,undefined,undefined},undefined,undefined,{backoff,5000,20000,5000,normal,undefined,undefined},undefined,<0.12140.0>,"gossip/1.0.2"}}
** Reason for termination = exit:killed
** Callback modules = [libp2p_group_worker]
** Callback mode = state_functions
** Stacktrace =
**  [{gen_statem,loop_receive,3,[{file,"gen_statem.erl"},{line,993}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]
2021-08-24 01:33:42 =CRASH REPORT====
  crasher:
    initial call: libp2p_group_worker:init/1
    pid: <0.12200.0>
    registered_name: []
    exception exit: {killed,[{gen_statem,loop_receive,3,[{file,"gen_statem.erl"},{line,993}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}
    ancestors: [<0.1245.0>,libp2p_group_gossip_sup_blockchain_swarm,libp2p_swarm_sup_blockchain_swarm,blockchain_swarm,blockchain_sup,be_sup,<0.1229.0>]
    message_queue_len: 0
    messages: []
    links: []
    dictionary: [{'__lager_metadata',[{target,{undefined,undefined}},{kind,inbound},{group_id,"gossip"}]}]
    trap_exit: true
    status: running
    heap_size: 987
    stack_size: 27
    reductions: 731
  neighbours:
2021-08-24 01:33:42 =ERROR REPORT====
** State machine <0.12195.0> terminating
** When server state  = {established,{state,<0.1436.0>,{"/ip4/172.16.0.194/tcp/44158","/ip4/72.134.120.205/tcp/44158"},<0.12213.0>,undefined,blockchain_swarm,11,open,262144,{recv_state,262144,32,<<>>,<<>>,undefined,undefined},{send_state,239429,undefined,undefined},60000,#Ref<0.1913172373.2502164482.171600>}}
** Reason for termination = exit:killed
** Callback modules = [libp2p_yamux_stream]
** Callback mode = handle_event_function
** Stacktrace =
**  [{gen_statem,loop_receive,3,[{file,"gen_statem.erl"},{line,993}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]
2021-08-24 01:33:42 =CRASH REPORT====
  crasher:
    initial call: libp2p_yamux_stream:init/1
    pid: <0.12195.0>
    registered_name: []
    exception exit: {killed,[{gen_statem,loop_receive,3,[{file,"gen_statem.erl"},{line,993}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}
    ancestors: [<0.1437.0>,<0.1436.0>,libp2p_swarm_session_sup_blockchain_swarm,libp2p_swarm_sup_blockchain_swarm,blockchain_swarm,blockchain_sup,be_sup,<0.1229.0>]
    message_queue_len: 0
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 1598
    stack_size: 27
    reductions: 1282
  neighbours:
2021-08-24 01:33:42 =ERROR REPORT====
** Generic server <0.10019.0> terminating 
** Last message in was {'EXIT',<0.10005.0>,killed}
** When Server state == {state,{<0.10019.0>,libp2p_simple_sup},one_for_one,{[7],#{7 => {child,<0.11672.0>,7,{libp2p_yamux_stream,receive_stream,undefined},temporary,1000,worker,[libp2p_yamux_stream]}}},undefined,1,5,[],0,libp2p_simple_sup,[]}
** Reason for termination ==
** killed
2021-08-24 01:33:42 =CRASH REPORT====
  crasher:
    initial call: supervisor:libp2p_simple_sup/1
    pid: <0.10019.0>
    registered_name: []
    exception exit: {killed,[{gen_server,decode_msg,9,[{file,"gen_server.erl"},{line,432}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]}
    ancestors: [<0.10005.0>,<0.1273.0>,<0.1272.0>,libp2p_swarm_listener_sup_blockchain_swarm,libp2p_swarm_sup_blockchain_swarm,blockchain_swarm,blockchain_sup,be_sup,<0.1229.0>]
    message_queue_len: 0
    messages: []
    links: []
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 987
    stack_size: 27
    reductions: 2716
  neighbours:
2021-08-24 01:34:50 =SUPERVISOR REPORT====
     Supervisor: {<0.1517.0>,libp2p_simple_sup}
     Context:    shutdown_error
     Reason:     noproc
     Offender:   [{pid,<0.1521.0>},{id,1},{mfargs,{libp2p_yamux_stream,open_stream,undefined}},{restart_type,temporary},{shutdown,1000},{child_type,worker}]
madninja commented 3 years ago

It looks like your ledger ran away from your database height. You will have to load a snapshot manually to set the ledger back to lower than the stored database height. There is a snapshot available at https://snapshots.helium.wtf/snap-425521

FTwOoO commented 3 years ago

how do it load this snapshot ?
i run these commands and something wrong: 

/opt/etl # blockchain_etl  snapshot info /var/data/snap-425521
RPC to 'blockchain_etl@127.0.0.1' failed: {'EXIT',
                                           {{badmatch,
                                             {error,bad_snapshot_binary}},
                                            [{blockchain_cli_snapshot,
                                              snapshot_info,3,
                                              [{file,
                                                "blockchain_cli_snapshot.erl"},
                                               {line,187}]},
                                             {clique_command,run,1,
                                              [{file,"clique_command.erl"},
                                               {line,87}]},
                                             {blockchain_console,command,1,
                                              [{file,"blockchain_console.erl"},
                                               {line,14}]},
                                             {rpc,
                                              '-handle_call_call/6-fun-0-',5,
                                              [{file,"rpc.erl"},
                                               {line,197}]}]}}
/opt/etl # blockchain_etl  snapshot load /var/data/snap-425521
RPC to 'blockchain_etl@127.0.0.1' failed: {'EXIT',
                                           {{badmatch,
                                             {error,bad_snapshot_binary}},
                                            [{blockchain_cli_snapshot,
                                              snapshot_load,1,
                                              [{file,
                                                "blockchain_cli_snapshot.erl"},
                                               {line,117}]},
                                             {blockchain_cli_snapshot,
                                              snapshot_load,3,
                                              [{file,
                                                "blockchain_cli_snapshot.erl"},
                                               {line,109}]},
                                             {clique_command,run,1,
                                              [{file,"clique_command.erl"},
                                               {line,87}]},
                                             {blockchain_console,command,1,
                                              [{file,"blockchain_console.erl"},
                                               {line,14}]},
                                             {rpc,
                                              '-handle_call_call/6-fun-0-',5,
                                              [{file,"rpc.erl"},
                                               {line,197}]}]}}