Closed ChristopherWalz closed 3 years ago
Trying to restart the node results in this error:
-- Logs begin at Sat 2021-05-22 10:55:39 CEST, end at Sun 2021-05-30 12:07:34 CEST. --
May 30 12:06:13 iota hornet[119196]: "whiteflagparentssolidtimeout": "2s"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "gossip": {
May 30 12:06:13 iota hornet[119196]: "streamreadtimeout": "1m0s",
May 30 12:06:13 iota hornet[119196]: "streamwritetimeout": "10s"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "logger": {
May 30 12:06:13 iota hornet[119196]: "disablecaller": true,
May 30 12:06:13 iota hornet[119196]: "encoding": "console",
May 30 12:06:13 iota hornet[119196]: "level": "info",
May 30 12:06:13 iota hornet[119196]: "outputpaths": [
May 30 12:06:13 iota hornet[119196]: "stdout"
May 30 12:06:13 iota hornet[119196]: ]
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "migrator": {
May 30 12:06:13 iota hornet[119196]: "querycooldownperiod": "5s",
May 30 12:06:13 iota hornet[119196]: "receiptmaxentries": 110,
May 30 12:06:13 iota hornet[119196]: "statefilepath": "migrator.state"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "mqtt": {
May 30 12:06:13 iota hornet[119196]: "bindaddress": "localhost:1883",
May 30 12:06:13 iota hornet[119196]: "workercount": 100,
May 30 12:06:13 iota hornet[119196]: "wsport": 1888
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "node": {
May 30 12:06:13 iota hornet[119196]: "alias": "HORNET node",
May 30 12:06:13 iota hornet[119196]: "disableplugins": [],
May 30 12:06:13 iota hornet[119196]: "enableplugins": [
May 30 12:06:13 iota hornet[119196]: "Spammer"
May 30 12:06:13 iota hornet[119196]: ],
May 30 12:06:13 iota hornet[119196]: "profile": "auto"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "p2p": {
May 30 12:06:13 iota hornet[119196]: "bindmultiaddresses": [
May 30 12:06:13 iota hornet[119196]: "/ip4/0.0.0.0/tcp/15600"
May 30 12:06:13 iota hornet[119196]: ],
May 30 12:06:13 iota hornet[119196]: "connectionmanager": {
May 30 12:06:13 iota hornet[119196]: "highwatermark": 10,
May 30 12:06:13 iota hornet[119196]: "lowwatermark": 5
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "gossipunknownpeerslimit": 4,
May 30 12:06:13 iota hornet[119196]: "peerstore": {
May 30 12:06:13 iota hornet[119196]: "path": "./p2pstore"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "reconnectinterval": "30s"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "p2pdisc": {
May 30 12:06:13 iota hornet[119196]: "advertiseinterval": "30s",
May 30 12:06:13 iota hornet[119196]: "maxdiscoveredpeerconns": 4,
May 30 12:06:13 iota hornet[119196]: "rendezvouspoint": "between-two-vertices",
May 30 12:06:13 iota hornet[119196]: "routingtablerefreshperiod": "60s"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "pow": {
May 30 12:06:13 iota hornet[119196]: "refreshtipsinterval": "5s"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "profiling": {
May 30 12:06:13 iota hornet[119196]: "bindaddress": "localhost:6060"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "prometheus": {
May 30 12:06:13 iota hornet[119196]: "bindaddress": "localhost:9311",
May 30 12:06:13 iota hornet[119196]: "cachesmetrics": true,
May 30 12:06:13 iota hornet[119196]: "coordinatormetrics": true,
May 30 12:06:13 iota hornet[119196]: "databasemetrics": true,
May 30 12:06:13 iota hornet[119196]: "debugmetrics": false,
May 30 12:06:13 iota hornet[119196]: "fileservicediscovery": {
May 30 12:06:13 iota hornet[119196]: "enabled": false,
May 30 12:06:13 iota hornet[119196]: "path": "target.json",
May 30 12:06:13 iota hornet[119196]: "target": "localhost:9311"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "gometrics": false,
May 30 12:06:13 iota hornet[119196]: "gossipmetrics": true,
May 30 12:06:13 iota hornet[119196]: "migrationmetrics": true,
May 30 12:06:13 iota hornet[119196]: "nodemetrics": true,
May 30 12:06:13 iota hornet[119196]: "processmetrics": false,
May 30 12:06:13 iota hornet[119196]: "promhttpmetrics": false,
May 30 12:06:13 iota hornet[119196]: "restapimetrics": true
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "protocol": {
May 30 12:06:13 iota hornet[119196]: "bech32hrp": "iota",
May 30 12:06:13 iota hornet[119196]: "milestonepublickeycount": 2,
May 30 12:06:13 iota hornet[119196]: "minpowscore": 4000,
May 30 12:06:13 iota hornet[119196]: "networkid": "chrysalis-mainnet",
May 30 12:06:13 iota hornet[119196]: "publickeyranges": [
May 30 12:06:13 iota hornet[119196]: {
May 30 12:06:13 iota hornet[119196]: "end": 777600,
May 30 12:06:13 iota hornet[119196]: "key": "a9b46fe743df783dedd00c954612428b34241f5913cf249d75bed3aafd65e4cd",
May 30 12:06:13 iota hornet[119196]: "start": 0
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: {
May 30 12:06:13 iota hornet[119196]: "end": 1555200,
May 30 12:06:13 iota hornet[119196]: "key": "365fb85e7568b9b32f7359d6cbafa9814472ad0ecbad32d77beaf5dd9e84c6ba",
May 30 12:06:13 iota hornet[119196]: "start": 0
May 30 12:06:13 iota hornet[119196]: }
May 30 12:06:13 iota hornet[119196]: ]
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "pruning": {
May 30 12:06:13 iota hornet[119196]: "delay": 40000,
May 30 12:06:13 iota hornet[119196]: "enabled": true,
May 30 12:06:13 iota hornet[119196]: "milestones": {
May 30 12:06:13 iota hornet[119196]: "enabled": false,
May 30 12:06:13 iota hornet[119196]: "maxmilestonestokeep": 60480
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "prunereceipts": false,
May 30 12:06:13 iota hornet[119196]: "size": {
May 30 12:06:13 iota hornet[119196]: "cooldowntime": "5m0s",
May 30 12:06:13 iota hornet[119196]: "enabled": true,
May 30 12:06:13 iota hornet[119196]: "targetsize": "30GB",
May 30 12:06:13 iota hornet[119196]: "thresholdpercentage": "10"
May 30 12:06:13 iota hornet[119196]: }
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "receipts": {
May 30 12:06:13 iota hornet[119196]: "backup": {
May 30 12:06:13 iota hornet[119196]: "enabled": false,
May 30 12:06:13 iota hornet[119196]: "folder": "receipts"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "validator": {
May 30 12:06:13 iota hornet[119196]: "api": {
May 30 12:06:13 iota hornet[119196]: "address": "http://localhost:14266",
May 30 12:06:13 iota hornet[119196]: "timeout": "5s"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "coordinator": {
May 30 12:06:13 iota hornet[119196]: "address": "UDYXTZBE9GZGPM9SSQV9LTZNDLJIZMPUVVXYXFYVBLIEUHLSEWFTKZZLXYRHHWVQV9MNNX9KZC9D9UZWZ",
May 30 12:06:13 iota hornet[119196]: "merkletreedepth": 24
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "ignoresofterrors": false,
May 30 12:06:13 iota hornet[119196]: "validate": false
May 30 12:06:13 iota hornet[119196]: }
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "requests": {
May 30 12:06:13 iota hornet[119196]: "discardolderthan": "15s",
May 30 12:06:13 iota hornet[119196]: "pendingreenqueueinterval": "5s"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "restapi": {
May 30 12:06:13 iota hornet[119196]: "bindaddress": "0.0.0.0:14265",
May 30 12:06:13 iota hornet[119196]: "excludehealthcheckfromauth": false,
May 30 12:06:13 iota hornet[119196]: "jwtauth": {
May 30 12:06:13 iota hornet[119196]: "enabled": false
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "limits": {
May 30 12:06:13 iota hornet[119196]: "bodylength": "1M",
May 30 12:06:13 iota hornet[119196]: "maxresults": 1000
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "permittedroutes": [
May 30 12:06:13 iota hornet[119196]: "/health",
May 30 12:06:13 iota hornet[119196]: "/mqtt",
May 30 12:06:13 iota hornet[119196]: "/api/v1/info",
May 30 12:06:13 iota hornet[119196]: "/api/v1/tips",
May 30 12:06:13 iota hornet[119196]: "/api/v1/messages/:messageID",
May 30 12:06:13 iota hornet[119196]: "/api/v1/messages/:messageID/metadata",
May 30 12:06:13 iota hornet[119196]: "/api/v1/messages/:messageID/raw",
May 30 12:06:13 iota hornet[119196]: "/api/v1/messages/:messageID/children",
May 30 12:06:13 iota hornet[119196]: "/api/v1/messages",
May 30 12:06:13 iota hornet[119196]: "/api/v1/transactions/:transactionID/included-message",
May 30 12:06:13 iota hornet[119196]: "/api/v1/milestones/:milestoneIndex",
May 30 12:06:13 iota hornet[119196]: "/api/v1/milestones/:milestoneIndex/utxo-changes",
May 30 12:06:13 iota hornet[119196]: "/api/v1/outputs/:outputID",
May 30 12:06:13 iota hornet[119196]: "/api/v1/addresses/:address",
May 30 12:06:13 iota hornet[119196]: "/api/v1/addresses/:address/outputs",
May 30 12:06:13 iota hornet[119196]: "/api/v1/addresses/ed25519/:address",
May 30 12:06:13 iota hornet[119196]: "/api/v1/addresses/ed25519/:address/outputs",
May 30 12:06:13 iota hornet[119196]: "/api/v1/treasury"
May 30 12:06:13 iota hornet[119196]: ],
May 30 12:06:13 iota hornet[119196]: "powenabled": true,
May 30 12:06:13 iota hornet[119196]: "powworkercount": 1,
May 30 12:06:13 iota hornet[119196]: "whitelistedaddresses": [
May 30 12:06:13 iota hornet[119196]: "127.0.0.1",
May 30 12:06:13 iota hornet[119196]: "::1"
May 30 12:06:13 iota hornet[119196]: ]
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "snapshots": {
May 30 12:06:13 iota hornet[119196]: "deltapath": "snapshots/mainnet/delta_snapshot.bin",
May 30 12:06:13 iota hornet[119196]: "deltasizethresholdpercentage": 50,
May 30 12:06:13 iota hornet[119196]: "depth": 50,
May 30 12:06:13 iota hornet[119196]: "downloadurls": [
May 30 12:06:13 iota hornet[119196]: {
May 30 12:06:13 iota hornet[119196]: "delta": "https://chrysalis-dbfiles.iota.org/snapshots/hornet/latest-delta_snapshot.bin",
May 30 12:06:13 iota hornet[119196]: "full": "https://chrysalis-dbfiles.iota.org/snapshots/hornet/latest-full_snapshot.bin"
May 30 12:06:13 iota hornet[119196]: }
May 30 12:06:13 iota hornet[119196]: ],
May 30 12:06:13 iota hornet[119196]: "fullpath": "snapshots/mainnet/full_snapshot.bin",
May 30 12:06:13 iota hornet[119196]: "interval": 200
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "spammer": {
May 30 12:06:13 iota hornet[119196]: "autostart": true,
May 30 12:06:13 iota hornet[119196]: "cpumaxusage": 0.8,
May 30 12:06:13 iota hornet[119196]: "index": "www.iota.wtf SPAM",
May 30 12:06:13 iota hornet[119196]: "indexsemilazy": "HORNET Spammer Semi-Lazy",
May 30 12:06:13 iota hornet[119196]: "message": "www.iota.wtf SPAM",
May 30 12:06:13 iota hornet[119196]: "mpsratelimit": 5,
May 30 12:06:13 iota hornet[119196]: "workers": 0
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "tangle": {
May 30 12:06:13 iota hornet[119196]: "milestonetimeout": "30s"
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "tipsel": {
May 30 12:06:13 iota hornet[119196]: "belowmaxdepth": 15,
May 30 12:06:13 iota hornet[119196]: "maxdeltamsgoldestconerootindextocmi": 13,
May 30 12:06:13 iota hornet[119196]: "maxdeltamsgyoungestconerootindextocmi": 8,
May 30 12:06:13 iota hornet[119196]: "nonlazy": {
May 30 12:06:13 iota hornet[119196]: "maxchildren": 30,
May 30 12:06:13 iota hornet[119196]: "maxreferencedtipage": "3s",
May 30 12:06:13 iota hornet[119196]: "retentionrulestipslimit": 100,
May 30 12:06:13 iota hornet[119196]: "spammertipsthreshold": 0
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "semilazy": {
May 30 12:06:13 iota hornet[119196]: "maxchildren": 2,
May 30 12:06:13 iota hornet[119196]: "maxreferencedtipage": "3s",
May 30 12:06:13 iota hornet[119196]: "retentionrulestipslimit": 20,
May 30 12:06:13 iota hornet[119196]: "spammertipsthreshold": 30
May 30 12:06:13 iota hornet[119196]: }
May 30 12:06:13 iota hornet[119196]: },
May 30 12:06:13 iota hornet[119196]: "warpsync": {
May 30 12:06:13 iota hornet[119196]: "advancementrange": 150
May 30 12:06:13 iota hornet[119196]: }
May 30 12:06:13 iota hornet[119196]: }
May 30 12:06:13 iota hornet[119196]: The following plugins are enabled: SpammerSnapshotInfo:
May 30 12:06:13 iota hornet[119196]: NetworkID: 1454675179895816119
May 30 12:06:13 iota hornet[119196]: SnapshotIndex: 271574
May 30 12:06:13 iota hornet[119196]: EntryPointIndex: 231645
May 30 12:06:13 iota hornet[119196]: PruningIndex: 231645
May 30 12:06:13 iota hornet[119196]: Timestamp: 2021-05-30 02:55:09 +0200 CEST
May 30 12:06:13 iota hornet[119196]: ██╗ ██╗ ██████╗ ██████╗ ███╗ ██╗███████╗████████╗
May 30 12:06:13 iota hornet[119196]: ██║ ██║██╔═══██╗██╔══██╗████╗ ██║██╔════╝╚══██╔══╝
May 30 12:06:13 iota hornet[119196]: ███████║██║ ██║██████╔╝██╔██╗ ██║█████╗ ██║
May 30 12:06:13 iota hornet[119196]: ██╔══██║██║ ██║██╔══██╗██║╚██╗██║██╔══╝ ██║
May 30 12:06:13 iota hornet[119196]: ██║ ██║╚██████╔╝██║ ██║██║ ╚████║███████╗ ██║
May 30 12:06:13 iota hornet[119196]: ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═══╝╚══════╝ ╚═╝
May 30 12:06:13 iota hornet[119196]: v1.0.2
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO App Loading plugins ...
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO Profile Profile mode 'auto', Using profile '2gb'
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO Node Loading core plugin: Profile ... done
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO Node Loading core plugin: ProtoCfg ... done
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO Node Loading core plugin: Graceful Shutdown ... done
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO Node Loading core plugin: Database ... done
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO Node Loading core plugin: PoW ... done
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO P2P never share your ./p2pstore folder as it contains your node's private key!
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO P2P retrieved existing peer identity from p2pstore/key.pub
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO P2P peer configured, ID: 12D3KooWD53oPFPVxqJf3ZC4XrKuipiRMsGvJHs4xpugF649zqfd
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO Node Loading core plugin: P2P ... done
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 INFO Node Loading core plugin: Gossip ... done
May 30 12:06:13 iota hornet[119196]: 2021-05-30T12:06:13+02:00 PANIC Tangle
May 30 12:06:13 iota hornet[119196]: HORNET was not shut down properly, the database may be corrupted.
May 30 12:06:13 iota hornet[119196]: Please restart HORNET with one of the following flags or enable "db.autoRevalidation" in the config.
May 30 12:06:13 iota hornet[119196]: --revalidate: starts the database revalidation (might take a long time)
May 30 12:06:13 iota hornet[119196]: --deleteDatabase: deletes the database
May 30 12:06:13 iota hornet[119196]: --deleteAll: deletes the database and the snapshot files
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/core/tangle.configure
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/core/tangle/core.go:146
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.(*Node).configure.func1
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:212
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.forEachCorePlugin
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:320
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.(*Node).ForEachCorePlugin
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:328
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.(*Node).configure
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:210
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.New
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:49
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.Run
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:62
May 30 12:06:13 iota hornet[119196]: main.main
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/main.go:33
May 30 12:06:13 iota hornet[119196]: runtime.main
May 30 12:06:13 iota hornet[119196]: /usr/local/go/src/runtime/proc.go:225
May 30 12:06:13 iota hornet[119196]: panic:
May 30 12:06:13 iota hornet[119196]: HORNET was not shut down properly, the database may be corrupted.
May 30 12:06:13 iota hornet[119196]: Please restart HORNET with one of the following flags or enable "db.autoRevalidation" in the config.
May 30 12:06:13 iota hornet[119196]: --revalidate: starts the database revalidation (might take a long time)
May 30 12:06:13 iota hornet[119196]: --deleteDatabase: deletes the database
May 30 12:06:13 iota hornet[119196]: --deleteAll: deletes the database and the snapshot files
May 30 12:06:13 iota hornet[119196]: goroutine 1 [running]:
May 30 12:06:13 iota hornet[119196]: go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc0008fa540, 0x0, 0x0, 0x0)
May 30 12:06:13 iota hornet[119196]: /github/home/go/pkg/mod/go.uber.org/zap@v1.16.0/zapcore/entry.go:234 +0x58d
May 30 12:06:13 iota hornet[119196]: go.uber.org/zap.(*SugaredLogger).log(0xc000010590, 0x1f36904, 0x0, 0x0, 0xc002465d40, 0x1, 0x1, 0x0, 0x0, 0x0)
May 30 12:06:13 iota hornet[119196]: /github/home/go/pkg/mod/go.uber.org/zap@v1.16.0/sugar.go:234 +0xf6
May 30 12:06:13 iota hornet[119196]: go.uber.org/zap.(*SugaredLogger).Panic(...)
May 30 12:06:13 iota hornet[119196]: /github/home/go/pkg/mod/go.uber.org/zap@v1.16.0/sugar.go:123
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/core/tangle.configure()
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/core/tangle/core.go:146 +0x415
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.(*Node).configure.func1(0xc00008c7d0, 0x1b70f01)
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:212 +0xdd
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.forEachCorePlugin(0xc00003e400, 0x9, 0x10, 0xc002465e40)
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:320 +0x51
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.(*Node).ForEachCorePlugin(...)
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:328
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.(*Node).configure(0xc000362380)
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:210 +0x7c
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.New(0xc00091ff60, 0x3, 0x3, 0x1c58060)
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:49 +0x248
May 30 12:06:13 iota hornet[119196]: github.com/gohornet/hornet/pkg/node.Run(0xc00091ff60, 0x3, 0x3, 0xc0008cb780)
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/pkg/node/node.go:62 +0x3f
May 30 12:06:13 iota hornet[119196]: main.main()
May 30 12:06:13 iota hornet[119196]: /__w/hornet/hornet/main.go:33 +0x334
May 30 12:06:13 iota systemd[1]: hornet.service: Main process exited, code=exited, status=2/INVALIDARGUMENT
May 30 12:06:13 iota systemd[1]: hornet.service: Failed with result 'exit-code'.
May 30 12:06:14 iota systemd[1]: hornet.service: Scheduled restart job, restart counter is at 5.
May 30 12:06:14 iota systemd[1]: Stopped HORNET.
May 30 12:06:14 iota systemd[1]: hornet.service: Start request repeated too quickly.
May 30 12:06:14 iota systemd[1]: hornet.service: Failed with result 'exit-code'.
May 30 12:06:14 iota systemd[1]: Failed to start HORNET.
It seemed to be out of memory according to /var/log/kern.log
:
May 30 08:31:47 iota kernel: [670199.682257] sshd invoked oom-killer: gfp_mask=0x100cca(GFP_HIGHUSER_MOVABLE), order=0, oom_score_adj=0
May 30 08:31:47 iota kernel: [670199.682262] CPU: 0 PID: 115447 Comm: sshd Not tainted 5.4.0-72-generic #80-Ubuntu
May 30 08:31:47 iota kernel: [670199.682264] Hardware name: Hetzner vServer, BIOS 20171111 11/11/2017
May 30 08:31:47 iota kernel: [670199.682265] Call Trace:
May 30 08:31:47 iota kernel: [670199.682276] dump_stack+0x6d/0x8b
May 30 08:31:47 iota kernel: [670199.682281] dump_header+0x4f/0x1eb
May 30 08:31:47 iota kernel: [670199.682283] oom_kill_process.cold+0xb/0x10
May 30 08:31:47 iota kernel: [670199.682287] out_of_memory.part.0+0x1df/0x3d0
May 30 08:31:47 iota kernel: [670199.682289] out_of_memory+0x6d/0xd0
May 30 08:31:47 iota kernel: [670199.682293] __alloc_pages_slowpath+0xd5e/0xe50
May 30 08:31:47 iota kernel: [670199.682296] __alloc_pages_nodemask+0x2d0/0x320
May 30 08:31:47 iota kernel: [670199.682298] alloc_pages_current+0x87/0xe0
May 30 08:31:47 iota kernel: [670199.682302] __page_cache_alloc+0x72/0x90
May 30 08:31:47 iota kernel: [670199.682304] pagecache_get_page+0xbf/0x300
May 30 08:31:47 iota kernel: [670199.682306] filemap_fault+0x6b2/0xa50
May 30 08:31:47 iota kernel: [670199.682309] ? unlock_page_memcg+0x12/0x20
May 30 08:31:47 iota kernel: [670199.682312] ? page_add_file_rmap+0xff/0x1a0
May 30 08:31:47 iota kernel: [670199.682315] ? xas_load+0xd/0x80
May 30 08:31:47 iota kernel: [670199.682316] ? xas_find+0x17f/0x1c0
May 30 08:31:47 iota kernel: [670199.682318] ? filemap_map_pages+0x24c/0x380
May 30 08:31:47 iota kernel: [670199.682321] ext4_filemap_fault+0x32/0x50
May 30 08:31:47 iota kernel: [670199.682324] __do_fault+0x3c/0x130
May 30 08:31:47 iota kernel: [670199.682326] do_fault+0x24b/0x640
May 30 08:31:47 iota kernel: [670199.682328] ? __switch_to_asm+0x40/0x70
May 30 08:31:47 iota kernel: [670199.682330] __handle_mm_fault+0x4c5/0x7a0
May 30 08:31:47 iota kernel: [670199.682332] handle_mm_fault+0xca/0x200
May 30 08:31:47 iota kernel: [670199.682335] do_user_addr_fault+0x1f9/0x450
May 30 08:31:47 iota kernel: [670199.682337] __do_page_fault+0x58/0x90
May 30 08:31:47 iota kernel: [670199.682341] ? exit_to_usermode_loop+0x8f/0x160
May 30 08:31:47 iota kernel: [670199.682343] do_page_fault+0x2c/0xe0
May 30 08:31:47 iota kernel: [670199.682346] do_async_page_fault+0x39/0x70
May 30 08:31:47 iota kernel: [670199.682348] async_page_fault+0x34/0x40
May 30 08:31:47 iota kernel: [670199.682351] RIP: 0033:0x7f7be6e3ef48
May 30 08:31:47 iota kernel: [670199.682358] Code: Bad RIP value.
May 30 08:31:47 iota kernel: [670199.682359] RSP: 002b:00007fff6fe21100 EFLAGS: 00010287
May 30 08:31:47 iota kernel: [670199.682361] RAX: 0000000000000009 RBX: 00007f7be6a0ed70 RCX: 00007f7be66bfe58
May 30 08:31:47 iota kernel: [670199.682362] RDX: 0000000000000002 RSI: 00007f7be6e55560 RDI: 00007f7be66c8265
May 30 08:31:47 iota kernel: [670199.682363] RBP: 00007fff6fe21200 R08: 00007f7be66bfe58 R09: 00007f7be66bfe58
May 30 08:31:47 iota kernel: [670199.682364] R10: 00007f7be66f7090 R11: 00007f7be689f000 R12: 0000000000000006
May 30 08:31:47 iota kernel: [670199.682365] R13: 0000001500000006 R14: 00007f7be6a0d888 R15: 00007f7be6a2e500
May 30 08:31:47 iota kernel: [670199.682366] Mem-Info:
May 30 08:31:47 iota kernel: [670199.682371] active_anon:447896 inactive_anon:71 isolated_anon:0
May 30 08:31:47 iota kernel: [670199.682371] active_file:205 inactive_file:206 isolated_file:0
May 30 08:31:47 iota kernel: [670199.682371] unevictable:0 dirty:3 writeback:0 unstable:0
May 30 08:31:47 iota kernel: [670199.682371] slab_reclaimable:17811 slab_unreclaimable:10669
May 30 08:31:47 iota kernel: [670199.682371] mapped:389 shmem:173 pagetables:1714 bounce:0
May 30 08:31:47 iota kernel: [670199.682371] free:13075 free_pcp:241 free_cma:0
May 30 08:31:47 iota kernel: [670199.682374] Node 0 active_anon:1791584kB inactive_anon:284kB active_file:820kB inactive_file:824kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:1556kB dirty:12kB writeback:0kB shmem:692kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 159744kB writeback_tmp:0kB unstable:0kB all_unreclaimable? no
May 30 08:31:47 iota kernel: [670199.682375] Node 0 DMA free:7872kB min:364kB low:452kB high:540kB active_anon:1516kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15908kB mlocked:0kB kernel_stack:0kB pagetables:4kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB
May 30 08:31:47 iota kernel: [670199.682378] lowmem_reserve[]: 0 1889 1889 1889 1889
May 30 08:31:47 iota kernel: [670199.682380] Node 0 DMA32 free:44428kB min:44688kB low:55860kB high:67032kB active_anon:1790068kB inactive_anon:284kB active_file:916kB inactive_file:1028kB unevictable:0kB writepending:12kB present:2031472kB managed:1970104kB mlocked:0kB kernel_stack:2032kB pagetables:6852kB bounce:0kB free_pcp:964kB local_pcp:964kB free_cma:0kB
May 30 08:31:47 iota kernel: [670199.682384] lowmem_reserve[]: 0 0 0 0 0
May 30 08:31:47 iota kernel: [670199.682385] Node 0 DMA: 4*4kB (E) 22*8kB (UME) 6*16kB (UE) 17*32kB (UME) 2*64kB (ME) 4*128kB (UM) 3*256kB (UME) 3*512kB (UM) 2*1024kB (ME) 1*2048kB (M) 0*4096kB = 7872kB
May 30 08:31:47 iota kernel: [670199.682393] Node 0 DMA32: 627*4kB (UMEH) 1364*8kB (UMEH) 615*16kB (UMEH) 313*32kB (UMEH) 54*64kB (UMEH) 14*128kB (UME) 9*256kB (ME) 2*512kB (ME) 3*1024kB (M) 0*2048kB 0*4096kB = 44924kB
May 30 08:31:47 iota kernel: [670199.682401] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB
May 30 08:31:47 iota kernel: [670199.682403] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
May 30 08:31:47 iota kernel: [670199.682403] 603 total pagecache pages
May 30 08:31:47 iota kernel: [670199.682405] 0 pages in swap cache
May 30 08:31:47 iota kernel: [670199.682406] Swap cache stats: add 0, delete 0, find 0/0
May 30 08:31:47 iota kernel: [670199.682406] Free swap = 0kB
May 30 08:31:47 iota kernel: [670199.682407] Total swap = 0kB
May 30 08:31:47 iota kernel: [670199.682407] 511866 pages RAM
May 30 08:31:47 iota kernel: [670199.682408] 0 pages HighMem/MovableOnly
May 30 08:31:47 iota kernel: [670199.682409] 15363 pages reserved
May 30 08:31:47 iota kernel: [670199.682409] 0 pages cma reserved
May 30 08:31:47 iota kernel: [670199.682409] 0 pages hwpoisoned
May 30 08:31:47 iota kernel: [670199.682410] Tasks state (memory values in pages):
May 30 08:31:47 iota kernel: [670199.682411] [ pid ] uid tgid total_vm rss pgtables_bytes swapents oom_score_adj name
May 30 08:31:47 iota kernel: [670199.682415] [ 265] 0 265 92745 282 782336 0 -250 systemd-journal
May 30 08:31:47 iota kernel: [670199.682417] [ 299] 0 299 4665 257 61440 0 -1000 systemd-udevd
May 30 08:31:47 iota kernel: [670199.682420] [ 376] 102 376 22557 210 77824 0 0 systemd-timesyn
May 30 08:31:47 iota kernel: [670199.682421] [ 388] 100 388 6649 235 81920 0 0 systemd-network
May 30 08:31:47 iota kernel: [670199.682423] [ 390] 101 390 5973 1016 86016 0 0 systemd-resolve
May 30 08:31:47 iota kernel: [670199.682425] [ 451] 0 451 58883 231 94208 0 0 accounts-daemon
May 30 08:31:47 iota kernel: [670199.682427] [ 456] 0 456 1703 68 53248 0 0 cron
May 30 08:31:47 iota kernel: [670199.682429] [ 457] 103 457 1892 194 57344 0 -900 dbus-daemon
May 30 08:31:47 iota kernel: [670199.682430] [ 463] 0 463 20457 81 57344 0 0 irqbalance
May 30 08:31:47 iota kernel: [670199.682432] [ 469] 0 469 7267 1915 94208 0 0 networkd-dispat
May 30 08:31:47 iota kernel: [670199.682433] [ 471] 0 471 1611 56 49152 0 0 qemu-ga
May 30 08:31:47 iota kernel: [670199.682435] [ 472] 104 472 56087 451 86016 0 0 rsyslogd
May 30 08:31:47 iota kernel: [670199.682437] [ 474] 0 474 4202 237 77824 0 0 systemd-logind
May 30 08:31:47 iota kernel: [670199.682439] [ 476] 0 476 948 45 45056 0 0 atd
May 30 08:31:47 iota kernel: [670199.682440] [ 478] 0 478 3044 232 61440 0 -1000 sshd
May 30 08:31:47 iota kernel: [670199.682442] [ 494] 0 494 1400 30 45056 0 0 agetty
May 30 08:31:47 iota kernel: [670199.682443] [ 497] 0 497 1457 27 45056 0 0 agetty
May 30 08:31:47 iota kernel: [670199.682445] [ 499] 0 499 58179 211 81920 0 0 polkitd
May 30 08:31:47 iota kernel: [670199.682446] [ 503] 0 503 26938 1918 110592 0 0 unattended-upgr
May 30 08:31:47 iota kernel: [670199.682448] [ 55743] 998 55743 900494 437065 4902912 0 0 hornet
May 30 08:31:47 iota kernel: [670199.682450] [ 70785] 0 70785 14493 394 98304 0 0 nginx
May 30 08:31:47 iota kernel: [670199.682452] [ 70788] 33 70788 14553 503 98304 0 0 nginx
May 30 08:31:47 iota kernel: [670199.682453] [ 70789] 33 70789 14619 569 98304 0 0 nginx
May 30 08:31:47 iota kernel: [670199.682455] [ 115443] 0 115443 3302 290 69632 0 0 sshd
May 30 08:31:47 iota kernel: [670199.682456] [ 115444] 0 115444 3302 287 61440 0 0 sshd
May 30 08:31:47 iota kernel: [670199.682458] [ 115445] 108 115445 3044 228 65536 0 0 sshd
May 30 08:31:47 iota kernel: [670199.682459] [ 115446] 108 115446 3044 226 61440 0 0 sshd
May 30 08:31:47 iota kernel: [670199.682461] [ 115447] 0 115447 2963 102 61440 0 0 sshd
May 30 08:31:47 iota kernel: [670199.682463] [ 115448] 0 115448 2963 80 57344 0 0 sshd
May 30 08:31:47 iota kernel: [670199.682464] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),cpuset=/,mems_allowed=0,global_oom,task_memcg=/system.slice/hornet.service,task=hornet,pid=55743,uid=998
May 30 08:31:47 iota kernel: [670199.682524] Out of memory: Killed process 55743 (hornet) total-vm:3601976kB, anon-rss:1748260kB, file-rss:0kB, shmem-rss:0kB, UID:998 pgtables:4788kB oom_score_adj:0
May 30 08:31:48 iota kernel: [670199.899679] oom_reaper: reaped process 55743 (hornet), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB
I guess 2 GB RAM is too little for the hornet node, since https://hornet.docs.iota.org/getting_started/getting_started.html states you need 8 GB minimum. Were the minimum requirements increased lately? I remembered on twitter someone said 2 or 4 GB should be enough.
That seems to be an issue with the coordinate being down at May 30 03:09:52
(https://status.iota.org/)
Usually the milestones were validated, but that suddenly stopped.
May 30 03:09:41 iota hornet[55743]: 2021-05-30T03:09:41+02:00 INFO Tangle Valid milestone detected! Index: 271661
May 30 03:09:41 iota hornet[55743]: 2021-05-30T03:09:41+02:00 INFO Tangle Run solidity check for Milestone (271661)...
May 30 03:09:41 iota hornet[55743]: 2021-05-30T03:09:41+02:00 INFO Tangle Solidifier finished: msgs: 1, collect: 0s, solidity 0s, propagation: 0s, total: 0s
May 30 03:09:41 iota hornet[55743]: 2021-05-30T03:09:41+02:00 INFO Tangle Milestone confirmed (271661): txsReferenced: 636, txsValue: 0, txsZeroValue: 636, txsConflicting: 0, collect: 11ms, total: 63ms
May 30 03:09:41 iota hornet[55743]: 2021-05-30T03:09:41+02:00 INFO Tangle New confirmed milestone: 271661, 61.00 MPS, 63.60 RMPS, 104.26% ref.rate
May 30 03:09:42 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00160/00054/00177, Tips (non-/semi-lazy): 100/0
May 30 03:09:43 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00182/00062/00186, Tips (non-/semi-lazy): 100/0
May 30 03:09:44 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00172/00067/00201, Tips (non-/semi-lazy): 100/0
May 30 03:09:45 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00167/00062/00186, Tips (non-/semi-lazy): 102/0
May 30 03:09:46 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00185/00063/00194, Tips (non-/semi-lazy): 100/0
May 30 03:09:47 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00161/00056/00168, Tips (non-/semi-lazy): 100/0
May 30 03:09:48 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00198/00070/00210, Tips (non-/semi-lazy): 100/0
May 30 03:09:49 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00177/00064/00192, Tips (non-/semi-lazy): 100/0
May 30 03:09:50 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00165/00059/00182, Tips (non-/semi-lazy): 100/0
May 30 03:09:51 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271661/271661, MPS (in/new/out): 00165/00054/00162, Tips (non-/semi-lazy): 100/0
May 30 03:09:51 iota hornet[55743]: 2021-05-30T03:09:51+02:00 INFO Tangle Valid milestone detected! Index: 271662
May 30 03:09:51 iota hornet[55743]: 2021-05-30T03:09:51+02:00 INFO Tangle Run solidity check for Milestone (271662)...
May 30 03:09:51 iota hornet[55743]: 2021-05-30T03:09:51+02:00 INFO Tangle Solidifier finished: msgs: 1, collect: 0s, solidity 0s, propagation: 0s, total: 0s
May 30 03:09:51 iota hornet[55743]: 2021-05-30T03:09:51+02:00 INFO Tangle Milestone confirmed (271662): txsReferenced: 580, txsValue: 0, txsZeroValue: 580, txsConflicting: 0, collect: 13ms, total: 83ms
May 30 03:09:51 iota hornet[55743]: 2021-05-30T03:09:51+02:00 INFO Tangle New confirmed milestone: 271662, 63.30 MPS, 58.00 RMPS, 91.63% ref.rate
May 30 03:09:52 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00182/00063/00194, Tips (non-/semi-lazy): 100/0
May 30 03:09:53 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00180/00062/00189, Tips (non-/semi-lazy): 101/0
May 30 03:09:54 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00173/00060/00180, Tips (non-/semi-lazy): 100/0
May 30 03:09:55 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00147/00056/00178, Tips (non-/semi-lazy): 101/0
May 30 03:09:56 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00175/00060/00180, Tips (non-/semi-lazy): 100/0
May 30 03:09:57 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00172/00068/00204, Tips (non-/semi-lazy): 100/0
May 30 03:09:58 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00191/00074/00222, Tips (non-/semi-lazy): 100/0
May 30 03:09:59 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00161/00053/00164, Tips (non-/semi-lazy): 100/0
May 30 03:10:00 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00181/00061/00180, Tips (non-/semi-lazy): 101/0
May 30 03:10:01 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00174/00058/00177, Tips (non-/semi-lazy): 100/0
May 30 03:10:02 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00185/00063/00189, Tips (non-/semi-lazy): 100/0
May 30 03:10:03 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00196/00071/00218, Tips (non-/semi-lazy): 101/0
May 30 03:10:04 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00152/00051/00153, Tips (non-/semi-lazy): 100/0
May 30 03:10:05 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00178/00068/00204, Tips (non-/semi-lazy): 100/0
May 30 03:10:06 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00166/00066/00198, Tips (non-/semi-lazy): 101/0
May 30 03:10:07 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00190/00065/00195, Tips (non-/semi-lazy): 100/0
May 30 03:10:08 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00195/00068/00204, Tips (non-/semi-lazy): 100/0
May 30 03:10:09 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00210/00068/00204, Tips (non-/semi-lazy): 100/0
May 30 03:10:10 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00261/00066/00200, Tips (non-/semi-lazy): 100/0
May 30 03:10:11 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00195/00066/00202, Tips (non-/semi-lazy): 101/0
May 30 03:10:12 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00178/00060/00180, Tips (non-/semi-lazy): 100/0
May 30 03:10:13 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00165/00059/00182, Tips (non-/semi-lazy): 100/0
May 30 03:10:14 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00174/00057/00171, Tips (non-/semi-lazy): 100/0
May 30 03:10:15 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00143/00058/00174, Tips (non-/semi-lazy): 100/0
May 30 03:10:16 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00175/00066/00198, Tips (non-/semi-lazy): 100/0
May 30 03:10:17 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00176/00062/00183, Tips (non-/semi-lazy): 100/0
May 30 03:10:18 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00175/00061/00186, Tips (non-/semi-lazy): 101/0
May 30 03:10:19 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00171/00055/00165, Tips (non-/semi-lazy): 100/0
May 30 03:10:20 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00195/00066/00198, Tips (non-/semi-lazy): 100/0
May 30 03:10:21 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00207/00069/00213, Tips (non-/semi-lazy): 100/0
May 30 03:10:22 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00199/00067/00201, Tips (non-/semi-lazy): 100/0
May 30 03:10:23 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00168/00062/00187, Tips (non-/semi-lazy): 105/0
May 30 03:10:24 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00204/00076/00228, Tips (non-/semi-lazy): 100/0
May 30 03:10:25 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00178/00062/00186, Tips (non-/semi-lazy): 100/0
May 30 03:10:26 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00166/00058/00179, Tips (non-/semi-lazy): 101/0
May 30 03:10:27 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00165/00058/00171, Tips (non-/semi-lazy): 100/0
May 30 03:10:28 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00179/00060/00183, Tips (non-/semi-lazy): 100/0
May 30 03:10:29 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00170/00056/00173, Tips (non-/semi-lazy): 100/0
May 30 03:10:30 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00186/00065/00200, Tips (non-/semi-lazy): 100/0
May 30 03:10:31 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00188/00063/00189, Tips (non-/semi-lazy): 100/0
May 30 03:10:32 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00175/00062/00186, Tips (non-/semi-lazy): 100/0
May 30 03:10:33 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00184/00059/00182, Tips (non-/semi-lazy): 101/0
May 30 03:10:34 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00201/00066/00198, Tips (non-/semi-lazy): 100/0
May 30 03:10:35 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00210/00069/00207, Tips (non-/semi-lazy): 101/0
May 30 03:10:36 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00154/00052/00156, Tips (non-/semi-lazy): 101/0
May 30 03:10:37 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00176/00059/00177, Tips (non-/semi-lazy): 100/0
May 30 03:10:38 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00200/00068/00204, Tips (non-/semi-lazy): 100/0
May 30 03:10:39 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00198/00067/00206, Tips (non-/semi-lazy): 100/0
May 30 03:10:40 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00203/00069/00213, Tips (non-/semi-lazy): 100/0
May 30 03:10:41 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00196/00067/00201, Tips (non-/semi-lazy): 100/0
May 30 03:10:42 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00153/00055/00165, Tips (non-/semi-lazy): 100/0
May 30 03:10:43 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00179/00058/00174, Tips (non-/semi-lazy): 100/0
May 30 03:10:44 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00182/00065/00195, Tips (non-/semi-lazy): 101/0
May 30 03:10:45 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00188/00067/00201, Tips (non-/semi-lazy): 100/0
May 30 03:10:46 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00166/00059/00177, Tips (non-/semi-lazy): 100/0
May 30 03:10:47 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00167/00055/00165, Tips (non-/semi-lazy): 101/0
May 30 03:10:48 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00177/00060/00180, Tips (non-/semi-lazy): 101/0
May 30 03:10:49 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00170/00056/00168, Tips (non-/semi-lazy): 100/0
May 30 03:10:50 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00179/00060/00180, Tips (non-/semi-lazy): 100/0
May 30 03:10:51 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00199/00070/00210, Tips (non-/semi-lazy): 100/0
May 30 03:10:52 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00194/00065/00205, Tips (non-/semi-lazy): 101/0
May 30 03:10:53 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00214/00074/00222, Tips (non-/semi-lazy): 101/0
May 30 03:10:54 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00190/00061/00183, Tips (non-/semi-lazy): 100/0
May 30 03:10:55 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00135/00046/00143, Tips (non-/semi-lazy): 100/0
May 30 03:10:56 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00249/00082/00251, Tips (non-/semi-lazy): 100/0
May 30 03:10:57 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00191/00064/00197, Tips (non-/semi-lazy): 100/0
May 30 03:10:58 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00174/00058/00174, Tips (non-/semi-lazy): 100/0
May 30 03:10:59 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00154/00050/00150, Tips (non-/semi-lazy): 100/0
May 30 03:11:00 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00184/00062/00186, Tips (non-/semi-lazy): 100/0
May 30 03:11:01 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00181/00061/00183, Tips (non-/semi-lazy): 100/0
May 30 03:11:02 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00201/00066/00203, Tips (non-/semi-lazy): 100/0
May 30 03:11:03 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00164/00057/00171, Tips (non-/semi-lazy): 100/0
May 30 03:11:04 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00193/00062/00186, Tips (non-/semi-lazy): 100/0
May 30 03:11:05 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00201/00067/00201, Tips (non-/semi-lazy): 100/0
May 30 03:11:06 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00178/00062/00186, Tips (non-/semi-lazy): 100/0
May 30 03:11:07 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00213/00068/00209, Tips (non-/semi-lazy): 100/0
May 30 03:11:08 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00188/00063/00189, Tips (non-/semi-lazy): 100/0
May 30 03:11:09 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00189/00061/00183, Tips (non-/semi-lazy): 100/0
May 30 03:11:10 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00203/00072/00216, Tips (non-/semi-lazy): 100/0
May 30 03:11:11 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00173/00054/00162, Tips (non-/semi-lazy): 100/0
May 30 03:11:12 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00212/00073/00219, Tips (non-/semi-lazy): 100/0
May 30 03:11:13 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00167/00059/00177, Tips (non-/semi-lazy): 102/0
May 30 03:11:14 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00196/00058/00174, Tips (non-/semi-lazy): 100/0
May 30 03:11:15 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00168/00057/00171, Tips (non-/semi-lazy): 100/0
May 30 03:11:16 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00199/00065/00195, Tips (non-/semi-lazy): 100/0
May 30 03:11:17 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00184/00063/00194, Tips (non-/semi-lazy): 102/0
May 30 03:11:18 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00177/00060/00180, Tips (non-/semi-lazy): 101/0
May 30 03:11:19 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00203/00066/00198, Tips (non-/semi-lazy): 100/0
May 30 03:11:20 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00190/00070/00215, Tips (non-/semi-lazy): 100/0
May 30 03:11:21 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00163/00058/00174, Tips (non-/semi-lazy): 100/0
May 30 03:11:22 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00186/00066/00198, Tips (non-/semi-lazy): 100/0
May 30 03:11:23 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00143/00055/00165, Tips (non-/semi-lazy): 100/0
May 30 03:11:24 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00190/00065/00195, Tips (non-/semi-lazy): 100/0
May 30 03:11:25 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00156/00062/00186, Tips (non-/semi-lazy): 101/0
May 30 03:11:26 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00152/00059/00177, Tips (non-/semi-lazy): 100/0
May 30 03:11:27 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00147/00052/00156, Tips (non-/semi-lazy): 100/0
May 30 03:11:28 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00165/00072/00216, Tips (non-/semi-lazy): 100/0
May 30 03:11:29 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00188/00067/00201, Tips (non-/semi-lazy): 100/0
May 30 03:11:30 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00168/00064/00192, Tips (non-/semi-lazy): 101/0
May 30 03:11:31 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00165/00062/00191, Tips (non-/semi-lazy): 100/0
May 30 03:11:32 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00195/00068/00204, Tips (non-/semi-lazy): 100/0
May 30 03:11:33 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00182/00061/00183, Tips (non-/semi-lazy): 100/0
May 30 03:11:34 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00179/00064/00192, Tips (non-/semi-lazy): 100/0
May 30 03:11:35 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00166/00061/00183, Tips (non-/semi-lazy): 100/0
May 30 03:11:36 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00131/00065/00200, Tips (non-/semi-lazy): 100/0
May 30 03:11:37 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00163/00066/00208, Tips (non-/semi-lazy): 100/0
May 30 03:11:38 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00185/00073/00216, Tips (non-/semi-lazy): 100/0
May 30 03:11:39 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00164/00068/00207, Tips (non-/semi-lazy): 100/0
May 30 03:11:40 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00164/00063/00196, Tips (non-/semi-lazy): 101/0
May 30 03:11:41 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00192/00061/00183, Tips (non-/semi-lazy): 101/0
May 30 03:11:42 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00169/00061/00186, Tips (non-/semi-lazy): 101/0
May 30 03:11:43 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00182/00057/00176, Tips (non-/semi-lazy): 100/0
May 30 03:11:44 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00180/00074/00222, Tips (non-/semi-lazy): 100/0
May 30 03:11:45 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00168/00059/00174, Tips (non-/semi-lazy): 100/0
May 30 03:11:46 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00182/00060/00188, Tips (non-/semi-lazy): 100/0
May 30 03:11:47 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00154/00054/00162, Tips (non-/semi-lazy): 101/0
May 30 03:11:48 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00163/00059/00177, Tips (non-/semi-lazy): 100/0
May 30 03:11:49 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00206/00070/00215, Tips (non-/semi-lazy): 100/0
May 30 03:11:50 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00193/00063/00189, Tips (non-/semi-lazy): 100/0
May 30 03:11:51 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00166/00058/00180, Tips (non-/semi-lazy): 101/0
May 30 03:11:52 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00180/00061/00183, Tips (non-/semi-lazy): 100/0
May 30 03:11:53 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00211/00071/00213, Tips (non-/semi-lazy): 100/0
May 30 03:11:54 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00206/00069/00207, Tips (non-/semi-lazy): 100/0
May 30 03:11:55 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00193/00070/00215, Tips (non-/semi-lazy): 100/0
May 30 03:11:56 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00194/00070/00211, Tips (non-/semi-lazy): 100/0
May 30 03:11:57 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00192/00067/00201, Tips (non-/semi-lazy): 101/0
May 30 03:11:58 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00173/00071/00213, Tips (non-/semi-lazy): 100/0
May 30 03:11:59 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00161/00057/00171, Tips (non-/semi-lazy): 100/0
May 30 03:12:00 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00171/00062/00191, Tips (non-/semi-lazy): 100/0
May 30 03:12:01 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00199/00079/00237, Tips (non-/semi-lazy): 101/0
May 30 03:12:02 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00159/00060/00180, Tips (non-/semi-lazy): 101/0
May 30 03:12:03 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00162/00059/00177, Tips (non-/semi-lazy): 101/0
May 30 03:12:04 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00177/00068/00204, Tips (non-/semi-lazy): 100/0
May 30 03:12:05 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00156/00063/00189, Tips (non-/semi-lazy): 100/0
May 30 03:12:06 iota hornet[55743]: req(qu/pe/proc/lat): 00000/00000/00000/0000ms, reqQMs: 0, processor: 00000, CMI/LMI: 271662/271662, MPS (in/new/out): 00165/00066/00203, Tips (non-/semi-lazy): 100/0
``´
For me it looks like this part could be the culprit:
May 30 08:31:48 iota hornet[115470]: HORNET was not shut down properly, the database may be corrupted.
May 30 08:31:48 iota hornet[115470]: Please restart HORNET with one of the following flags or enable "db.autoRevalidation" in the config.
May 30 08:31:48 iota hornet[115470]: --revalidate: starts the database revalidation (might take a long time)
May 30 08:31:48 iota hornet[115470]: --deleteDatabase: deletes the database
May 30 08:31:48 iota hornet[115470]: --deleteAll: deletes the database and the snapshot files
Maybe try to delete the db snapshot and rerun it or use one of the config mentioned above
Got same issue. Even deleted the database and enabled the "revalidate" option in the config file. Hornet runs fine at start but eventually crashes and never starts up back again
Crash log: https://pastebin.com/N4dv2xce
The network failure, or better the COO downtime caused the nodes to use more RAM, because they needed to walk longer cones in the tangle. Maybe 2GB was not enough in that case.
I guess this can be closed for now.
Describe the bug Hornet crashed in the morning and was not running when I checked just now. There is still enough space on the device.
Expected behavior No crash.
Environment information:
Additional context