search

eosrio / hyperion-history-api

Scalable Full History API Solution for Antelope (former EOSIO) based blockchains
https://hyperion.docs.eosrio.io
Other
126 stars 72 forks source link

Hyperion Not Indexing Data #65

Closed NatPDeveloper closed 5 months ago

NatPDeveloper commented 3 years ago

I've got my hyperion connected to an RPC/SHIP node endpoint, but it is not indexing any data and when I query the API, it does not return any transactions such as account creation. Not sure what I'm doing wrong or if there's a gap in the documentation or perhaps a setting I am misunderstanding. Any help would be appreciated : )

node -v
v16.10.0

ecosystem.config.js:

const {addApiServer, addIndexer} = require("./definitions/ecosystem_settings");

module.exports = {
    apps: [
        addIndexer('CHAIN_NAME'), // Index chain name
        addApiServer('CHAIN_NAME', 1) // API chain name, API threads number
    ]
};

connections.json:

{
    "amqp": {
      "host": "127.0.0.1:5672",
      "api": "127.0.0.1:15672",
      "user": "hyperion",
      "pass": "hyperion",
      "vhost": "hyperion"
    },
    "elasticsearch": {
      "host": "127.0.0.1:9200",
      "ingest_nodes": [
        "127.0.0.1:9200"
      ],
      "user": "hyperion",
      "pass": "hyperion"
    },
    "redis": {
      "host": "127.0.0.1",
      "port": "6379"
    },
    "chains": {
      "CHAIN_NAME": {
        "name": "Testnet",
        "chain_id": "CHAIN_ID",
        "http": "http://IP_HERE:8888",
        "ship": "ws://IP_HERE:8887",
        "WS_ROUTER_PORT": 7001
      }
    }
  }

/chains/chain_name.config.json:

{
    "api": {
      "chain_name": "Testnet",
      "server_addr": "0.0.0.0",
      "server_port": 7000,
      "server_name": "0.0.0.0:7000",
      "provider_name": "Example Provider",
      "provider_url": "https://example.com",
      "chain_api": "",
      "push_api": "",
      "chain_logo_url": "",
      "enable_caching": true,
      "cache_life": 1,
      "limits": {
        "get_actions": 1000,
        "get_voters": 100,
        "get_links": 1000,
        "get_deltas": 1000,
        "get_trx_actions": 200
      },
      "access_log": false,
      "enable_explorer": true,
      "chain_api_error_log": false,
      "custom_core_token": "",
      "enable_export_action": false,
      "disable_tx_cache": false,
      "tx_cache_expiration_sec": 3600
    },
    "settings": {
      "preview": false,
      "chain": "CHAIN_NAME",
      "eosio_alias": "eosio",
      "parser": "2.1",
      "ignore_snapshot": "true",
      "auto_stop": 0,
      "index_version": "v1",
      "debug": false,
      "bp_logs": false,
      "bp_monitoring": false,
      "ipc_debug_rate": 60000,
      "allow_custom_abi": true,
      "rate_monitoring": true,
      "max_ws_payload_kb": 256,
      "ds_profiling": false,
      "auto_mode_switch": false,
      "hot_warm_policy": false,
      "custom_policy": "",
      "bypass_index_map": false,
  "index_partition_size": 10000000

    },
    "blacklists": {
      "actions": [],
      "deltas": []
    },
    "whitelists": {
      "actions": [],
      "deltas": [],
      "max_depth": 10,
      "root_only": false
    },
    "scaling": {
      "readers": 1,
      "ds_queues": 1,
      "ds_threads": 1,
      "ds_pool_size": 1,
      "indexing_queues": 1,
      "ad_idx_queues": 1,
      "max_autoscale": 4,
      "batch_size": 5000,
      "resume_trigger": 5000,
      "auto_scale_trigger": 20000,
      "block_queue_limit": 10000,
      "max_queue_limit": 100000,
      "routing_mode": "heatmap",
      "polling_interval": 10000
    },
    "indexer": {
      "enabled": true,
      "start_on": 1,
      "stop_on": 0,
      "rewrite": false,
      "purge_queues": true,
      "live_reader": true,
      "live_only_mode": false,
      "abi_scan_mode": false, # ran intially with true
      "fetch_block": true,
      "fetch_traces": true,
      "disable_reading": false,
      "disable_indexing": false,
      "process_deltas": true,
      "disable_delta_rm": false
    },
    "features": {
      "streaming": {
        "enable": true,
        "traces": true,
        "deltas": true
      },
      "tables": {
        "proposals": true,
        "accounts": true,
        "voters": true
      },
      "index_deltas": true,
      "index_transfer_memo": true,
      "index_all_deltas": true,
      "deferred_trx": false,
      "failed_trx": false,
      "resource_limits": false,
      "resource_usage": false
    },
    "prefetch": {
      "read": 50,
      "block": 100,
      "index": 500
    }
  }

Example call /v2/history/get_actions?account=eosio&skip=0&limit=100&sort=desc:

{
  "query_time_ms": 1.296,
  "cached": false,
  "lib": 0,
  "total": {
    "value": 0,
    "relation": "eq"
  },
  "actions": []
}

Health query:

{
  "version": "3.3.4-rc7",
  "version_hash": "18ef675b8804a0bf2257f6553e95b6d1a6282e61",
  "host": "0.0.0.0:7000",
  "health": [
    {
      "service": "RabbitMq",
      "status": "OK",
      "time": 1633148861841
    },
    {
      "service": "NodeosRPC",
      "status": "OK",
      "service_data": {
        "head_block_num": 65788,
        "head_block_time": "2021-10-02T04:19:13.000",
        "time_offset": 508842,
        "last_irreversible_block": 65787,
        "chain_id": "CHAIN_ID"
      },
      "time": 1633148861842
    },
    {
      "service": "Elasticsearch",
      "status": "OK",
      "service_data": {
        "last_indexed_block": 65749,
        "total_indexed_blocks": 65749,
        "active_shards": "100.0%"
      },
      "time": 1633148861845
    }
  ],
  "features": {
    "streaming": {
      "enable": true,
      "traces": true,
      "deltas": true
    },
    "tables": {
      "proposals": true,
      "accounts": true,
      "voters": true
    },
    "index_deltas": true,
    "index_transfer_memo": true,
    "index_all_deltas": true,
    "deferred_trx": false,
    "failed_trx": false,
    "resource_limits": false,
    "resource_usage": false
  },
  "query_time_ms": 8.618
}

Log output:

0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] --------- Hyperion Indexer 3.3.4-rc7 ---------
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Using parser version 2.1
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Chain: CHAIN_NAME
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] 
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: ---------------
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32:  INDEXING MODE 
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: ---------------
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Purging all CHAIN_NAME queues!
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Elasticsearch: 7.15.0 | Lucene: 8.9.0
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Ingest client ready at http://127.0.0.1:9200/
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Painless Update Script loaded!
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Action Mapping added for @voteproducer
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Updating index templates for CHAIN_NAME...
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] 14 index templates updated
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Last indexed block (deltas): 1
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master]  |>> First Block: 1
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master]  >>| Last  Block: 62106
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Setting parallel reader [1] from block 1 to 5001
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Setting live reader at head = 62106
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Loading indices...
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Delta streaming enabled!
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] Action trace streaming enabled!
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] 📣️  Deserialization errors are being logged in:
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [/home/ubuntu/hyperion-history-api/logs/CHAIN_NAME/deserialization_errors.log]
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:32: [31489 - 00_master] 15 workers launched
1|CHAIN_NAME-api      | 2021-10-02T03:48:32: [31505 - 00_master] Chain API URL: "http://IP:8888" | Push API URL: "undefined"
1|CHAIN_NAME-api      | 2021-10-02T03:48:32: Importing stream module
1|CHAIN_NAME-api      | 2021-10-02T03:48:32: [31505 - 00_master] Websocket manager loaded!
1|CHAIN_NAME-api      | 2021-10-02T03:48:32: [31505 - 00_master] starting relay - http://127.0.0.1:7001
1|CHAIN_NAME-api      | 2021-10-02T03:48:33: [31505 - 00_master] Last commit hash on this branch is: 18ef675b8804a0bf2257f6553e95b6d1a6282e61
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: [31531 - 01_reader] Websocket connected!
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: [31489 - 00_master] received ship abi for distribution
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: [31531 - 01_reader] 
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: | SHIP Status Report
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: | Init block: 0
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: | Head block: 0
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: [31614 - 15_delta_updater] Launched delta updater, consuming from CHAIN_NAME:delta_rm
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: [31532 - 02_continuous_reader] Websocket connected!
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: [31532 - 02_continuous_reader] 
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: | SHIP Status Report
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: | Init block: 0
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:33: | Head block: 0
1|CHAIN_NAME-api      | 2021-10-02T03:48:33: [31505 - 00_master] CHAIN_NAME hyperion api ready and listening on port 7000
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:34: [ROUTER] New relay connected with ID = KSTwdFmk-VM1vK44AAAB
1|CHAIN_NAME-api      | 2021-10-02T03:48:34: [31505 - 00_master] Relay Connected!

0|CHAIN_NAME-indexer  | 2021-10-02T03:48:37: [31489 - 00_master] W:15 | R:6002.2 | C:2887.2 | A:0 | D:0 | I:2674.8 | 14425/30000/62105 | syncs in a few seconds (23.2% 48.3%)
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:42: [31489 - 00_master] W:15 | R:6423 | C:3962.8 | A:0 | D:0 | I:3921 | 34229/62105/62105 | syncs in a few seconds (55.1% 100.0%)
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:47: [31489 - 00_master] W:15 | R:2.2 | C:4832.4 | A:0 | D:0 | I:4831.6 | 58380/62105/62105 | syncs in a few seconds (94.0% 100.0%)
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:52: [31489 - 00_master] W:15 | R:1.8 | C:746.8 | A:0 | D:0 | I:1001.8
0|CHAIN_NAME-indexer  | 2021-10-02T03:48:57: [31489 - 00_master] W:15 | R:2 | C:2 | A:0 | D:0 | I:2
0|CHAIN_NAME-indexer  | 2021-10-02T03:49:02: [31489 - 00_master] W:15 | R:2 | C:2 | A:0 | D:0 | I:2
0|CHAIN_NAME-indexer  | 2021-10-02T03:49:07: [31489 - 00_master] W:15 | R:2 | C:2 | A:0 | D:0 | I:2
0|CHAIN_NAME-indexer  | 2021-10-02T03:49:12: [31489 - 00_master] W:15 | R:2 | C:2 | A:0 | D:0 | I:2
0|CHAIN_NAME-indexer  | 2021-10-02T03:49:17: [31489 - 00_master] W:15 | R:2.2 | C:2.2 | A:0 | D:0 | I:2
0|CHAIN_NAME-indexer  | 2021-10-02T03:49:22: [31489 - 00_master] W:15 | R:1.8 | C:1.8 | A:0 | D:0 | I:2
dacom-dark-sun commented 2 years ago

I have the same problem. Kibana shows that indexes was created, but no any data inside. And no any action parsed by the indexer log. What reason can be?

image

RabbitMQ show that queues is works. But i dont know, this is normal work or not.
image

@NatPDeveloper how do you resolve this problem?

NatPDeveloper commented 2 years ago

I have the same problem. Kibana shows that indexes was created, but no any data inside. And no any action parsed by the indexer log. What reason can be?

image

RabbitMQ show that queues is works. But i dont know, this is normal work or not. image

@NatPDeveloper how do you resolve this problem?

If I remember correctly it was a configuration issue. I believe I documented it in the hyperion api channel. https://t.me/EOSHyperion. If you can't find it, feel free to dm on telegram @natpd

igorls commented 5 months ago

please reopen if relevant on the latest version and include more information on the issue