Closed Vantone closed 4 months ago
hubble-1 | {"message":" █████████████████████████████████████░░░ 92% | Getting snapshot | 2870/3100 | ETA: 6m 0s","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"\n","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"\n","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":60,\"time\":1716953352287,\"pid\":29,\"hostname\":\"2021b693128a\",\"err\":{\"type\":\"Error\",\"message\":\"Failed to start hub: failed to sync snapshot - Error extracting snapshot\",\"stack\":\"Error: Failed to start hub\n at file:///home/node/app/apps/hubble/src/cli.ts:697:14\n at /home/node/app/node_modules/neverthrow/dist/index.cjs.js:113:102\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at Command. (file:///home/node/app/apps/hubble/src/cli.ts:695:25)\ncaused by: HubError: failed to sync snapshot - Error extracting snapshot\n at Hub.attemptCatchupSyncWithSnapshot (file:///home/node/app/apps/hubble/src/hubble.ts:923:20)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at Hub.start (file:///home/node/app/apps/hubble/src/hubble.ts:584:27)\n at Command. (file:///home/node/app/apps/hubble/src/cli.ts:695:25)\"},\"msg\":\"Failed to start hub\"}\n","timestamp":"2024-05-29T03:29:12.327Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":60,\"time\":1716953352327,\"pid\":29,\"hostname\":\"2021b693128a\",\"reason\":\"Hub Startup failed\",\"msg\":\"shutting down hub\"}\n","timestamp":"2024-05-29T03:29:12.327Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"\n","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"\n","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"} hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/rocks.hub._default"} hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/rocks.hub._default/trieDb"} hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/farcaster"} hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/rocks.hub._default/trieDb"} hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Opened database","component":"RustRocksDB","path":".rocks/rocks.hub._default/trieDb"} hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/farcaster"} hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Flushing rust log buffer. Writing logs directly from Rust","flush_next":true} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353356,\"pid\":29,\"hostname\":\"2021b693128a\",\"server\":{\"address\":\"statsd\",\"port\":8125,\"family\":4},\"msg\":\"Statsd server specified. Statsd enabled\"}\n","timestamp":"2024-05-29T03:29:13.356Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353356,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"StorageRegistry: 0x00000000fcce7f938e7ae6d3c335bd6a1a7c593d, KeyRegistry: 0x00000000Fc1237824fb747aBDE0FF18990E59b7e, IdRegistry: 0x00000000Fc6c5F01Fc30151999387Bb99A9f489b\"}\n","timestamp":"2024-05-29T03:29:13.356Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353356,\"pid\":29,\"hostname\":\"2021b693128a\",\"totalPruneSize\":10080,\"msg\":\"total default storage limit size\"}\n","timestamp":"2024-05-29T03:29:13.356Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353356,\"pid\":29,\"hostname\":\"2021b693128a\",\"rpcRateLimit\":20000,\"msg\":\"RPC rate limit enabled\"}\n","timestamp":"2024-05-29T03:29:13.356Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"component\":\"utils/p2p\",\"ip\":\"60.190.239.210\",\"msg\":\"Fetched public IP\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"attempting catchup sync with snapshot\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"delta\":378066817,\"limit\":25000000,\"current_item_count\":0,\"msg\":\"catchup sync using snapshot\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"beginning snapshot sync in 30000ms - THIS WILL RESET THE DATABASE\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"db_location\":\".rocks/rocks.hub._default\",\"file_count\":0,\"overwrite\":true,\"msg\":\"DB is empty or overwrite is true, fetching snapshot from S3\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"latestSnapshotKeyBase\":\"snapshots/MAINNET/DB_SCHEMA_11/snapshot-2024-05-28-1716862500.tar.gz\",\"msg\":\"found latest S3 snapshot\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"numChunks\":31,\"msg\":\"Getting snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":1,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":2,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":3,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":4,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":5,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":6,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":7,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":8,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":9,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":10,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":11,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":12,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":13,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":14,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":15,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":16,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":17,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":18,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":19,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":20,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":21,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":22,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":23,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":24,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":25,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":26,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":27,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":28,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":29,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":50,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"error\":{\"code\":\"ECONNRESET\"},\"msg\":\"Error extracting snapshot\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":50,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"error\":{\"errCode\":\"unavailable\",\"presentable\":false,\"name\":\"HubError\"},\"msg\":\"failed to sync snapshot, falling back to diff sync\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":50,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"failed to catchup sync using snapshot\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopping Hubble...\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped http API server\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"gRPC server force shutdown succeeded\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped watching contract events\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped watching contract events\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped watching contract events\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped watching block numbers\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped fname events provider\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"L2EventsProvider stopped\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Sync engine stopped\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped libp2p...\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"stopping engine\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"engine stopped\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Hubble stopped, exiting normally\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"} hubble-1 | 2024-05-29T03:29:13: PM2 log: App name:hubble id:0 disconnected hubble-1 | 2024-05-29T03:29:13: PM2 log: App [hubble:0] exited with code [1] via signal [SIGINT] hubble-1 | 2024-05-29T03:29:13: PM2 log: 0 application online, retry = 3 hubble-1 | 2024-05-29T03:29:15: PM2 log: 0 application online, retry = 2 hubble-1 | 2024-05-29T03:29:17: PM2 log: 0 application online, retry = 1 hubble-1 | 2024-05-29T03:29:19: PM2 log: 0 application online, retry = 0 hubble-1 | 2024-05-29T03:29:19: PM2 log: Stopping app:hubble id:0 hubble-1 | 2024-05-29T03:29:19: PM2 log: pid=29 msg=process killed hubble-1 | 2024-05-29T03:29:19: PM2 log: PM2 successfully stopped hubble-1 exited with code 0
hubble-1 | {"message":" █████████████████████████████████████░░░ 92% | Getting snapshot | 2870/3100 | ETA: 6m 0s","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"\n","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"\n","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"} hubble-1 | {"message":"{\"level\":60,\"time\":1716953352287,\"pid\":29,\"hostname\":\"2021b693128a\",\"err\":{\"type\":\"Error\",\"message\":\"Failed to start hub: failed to sync snapshot - Error extracting snapshot\",\"stack\":\"Error: Failed to start hub\n at file:///home/node/app/apps/hubble/src/cli.ts:697:14\n at /home/node/app/node_modules/neverthrow/dist/index.cjs.js:113:102\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at Command. (file:///home/node/app/apps/hubble/src/cli.ts:695:25)\ncaused by: HubError: failed to sync snapshot - Error extracting snapshot\n at Hub.attemptCatchupSyncWithSnapshot (file:///home/node/app/apps/hubble/src/hubble.ts:923:20)\n at processTicksAndRejections (node:internal/process/task_queues:95:5)\n at Hub.start (file:///home/node/app/apps/hubble/src/hubble.ts:584:27)\n at Command. (file:///home/node/app/apps/hubble/src/cli.ts:695:25)\"},\"msg\":\"Failed to start hub\"}\n","timestamp":"2024-05-29T03:29:12.327Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":60,\"time\":1716953352327,\"pid\":29,\"hostname\":\"2021b693128a\",\"reason\":\"Hub Startup failed\",\"msg\":\"shutting down hub\"}\n","timestamp":"2024-05-29T03:29:12.327Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"\n","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"\n","timestamp":"2024-05-29T03:29:12.286Z","type":"err","process_id":0,"app_name":"hubble"}
hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/rocks.hub._default"}
hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/rocks.hub._default/trieDb"}
hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/farcaster"}
hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/rocks.hub._default/trieDb"}
hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Opened database","component":"RustRocksDB","path":".rocks/rocks.hub._default/trieDb"}
hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Creating new RocksDB","component":"RustRocksDB","path":".rocks/farcaster"}
hubble-1 | {"hostname":"2021b693128a","pid":29,"time":1716941546286,"level":30,"msg":"Flushing rust log buffer. Writing logs directly from Rust","flush_next":true}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353356,\"pid\":29,\"hostname\":\"2021b693128a\",\"server\":{\"address\":\"statsd\",\"port\":8125,\"family\":4},\"msg\":\"Statsd server specified. Statsd enabled\"}\n","timestamp":"2024-05-29T03:29:13.356Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353356,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"StorageRegistry: 0x00000000fcce7f938e7ae6d3c335bd6a1a7c593d, KeyRegistry: 0x00000000Fc1237824fb747aBDE0FF18990E59b7e, IdRegistry: 0x00000000Fc6c5F01Fc30151999387Bb99A9f489b\"}\n","timestamp":"2024-05-29T03:29:13.356Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353356,\"pid\":29,\"hostname\":\"2021b693128a\",\"totalPruneSize\":10080,\"msg\":\"total default storage limit size\"}\n","timestamp":"2024-05-29T03:29:13.356Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353356,\"pid\":29,\"hostname\":\"2021b693128a\",\"rpcRateLimit\":20000,\"msg\":\"RPC rate limit enabled\"}\n","timestamp":"2024-05-29T03:29:13.356Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"component\":\"utils/p2p\",\"ip\":\"60.190.239.210\",\"msg\":\"Fetched public IP\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"attempting catchup sync with snapshot\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"delta\":378066817,\"limit\":25000000,\"current_item_count\":0,\"msg\":\"catchup sync using snapshot\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"beginning snapshot sync in 30000ms - THIS WILL RESET THE DATABASE\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"db_location\":\".rocks/rocks.hub._default\",\"file_count\":0,\"overwrite\":true,\"msg\":\"DB is empty or overwrite is true, fetching snapshot from S3\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"latestSnapshotKeyBase\":\"snapshots/MAINNET/DB_SCHEMA_11/snapshot-2024-05-28-1716862500.tar.gz\",\"msg\":\"found latest S3 snapshot\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"numChunks\":31,\"msg\":\"Getting snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353357,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":1,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.357Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":2,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":3,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":4,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":5,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":6,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":7,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":8,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":9,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":10,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":11,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":12,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":13,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.358Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353358,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":14,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":15,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":16,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":17,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":18,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":19,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":20,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":21,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":22,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":23,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":24,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":25,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":26,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":27,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":28,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353359,\"pid\":29,\"hostname\":\"2021b693128a\",\"chunkCount\":29,\"totalChunks\":31,\"msg\":\"Downloading snapshot chunks...\"}\n","timestamp":"2024-05-29T03:29:13.359Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":50,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"error\":{\"code\":\"ECONNRESET\"},\"msg\":\"Error extracting snapshot\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":50,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"error\":{\"errCode\":\"unavailable\",\"presentable\":false,\"name\":\"HubError\"},\"msg\":\"failed to sync snapshot, falling back to diff sync\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":50,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"failed to catchup sync using snapshot\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopping Hubble...\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped http API server\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"gRPC server force shutdown succeeded\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped watching contract events\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped watching contract events\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped watching contract events\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353360,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped watching block numbers\"}\n","timestamp":"2024-05-29T03:29:13.360Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped fname events provider\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"L2EventsProvider stopped\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Sync engine stopped\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Stopped libp2p...\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"stopping engine\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"engine stopped\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | {"message":"{\"level\":30,\"time\":1716953353361,\"pid\":29,\"hostname\":\"2021b693128a\",\"msg\":\"Hubble stopped, exiting normally\"}\n","timestamp":"2024-05-29T03:29:13.361Z","type":"out","process_id":0,"app_name":"hubble"}
hubble-1 | 2024-05-29T03:29:13: PM2 log: App name:hubble id:0 disconnected
hubble-1 | 2024-05-29T03:29:13: PM2 log: App [hubble:0] exited with code [1] via signal [SIGINT]
hubble-1 | 2024-05-29T03:29:13: PM2 log: 0 application online, retry = 3
hubble-1 | 2024-05-29T03:29:15: PM2 log: 0 application online, retry = 2
hubble-1 | 2024-05-29T03:29:17: PM2 log: 0 application online, retry = 1
hubble-1 | 2024-05-29T03:29:19: PM2 log: 0 application online, retry = 0
hubble-1 | 2024-05-29T03:29:19: PM2 log: Stopping app:hubble id:0
hubble-1 | 2024-05-29T03:29:19: PM2 log: pid=29 msg=process killed
hubble-1 | 2024-05-29T03:29:19: PM2 log: PM2 successfully stopped
hubble-1 exited with code 0