search

Joystream / joystream

Joystream Monorepo
http://www.joystream.org
GNU General Public License v3.0
1.42k stars 115 forks source link

[Colossus] storage node crashed on upload #5003

Closed yasiryagi closed 10 months ago

yasiryagi commented 10 months ago

Description: Storage node unhandled crash during an object upload, while it is accepted

{
  "data": {
    "storageDataObjects": [
      {
        "id": "1454542",
        "size": "20022",
        "isAccepted": true,
        "videoMedia": null,
        "videoThumbnail": {
          "id": "709318"
        }
      }
    ]
  }
}

Product: Colossus Version: 3.7.2 Log:

{
  "_index": "storage-node-adovrn",
  "_id": "sATJeIwBh1Mrve8iRUku",
  "_version": 1,
  "_score": 0,
  "_ignored": [
    "fields.meta.message.keyword",
    "fields.meta.error.stack.keyword",
    "fields.meta.stack.keyword"
  ],
  "_source": {
    "@timestamp": "2023-12-17T17:18:23.213Z",
    "message": "POST /api/v1/files: Error 500: Timeout",
    "severity": "error",
    "fields": {
      "meta": {
        "error": {
          "stack": "Error\n    at new module.exports.TimeoutError (/joystream/node_modules/promise-timeout/index.js:44:16)\n    at module.exports.timeout (/joystream/node_modules/promise-timeout/index.js:20:15)\n    at extrinsicWrapper (/joystream/storage-node/lib/services/runtime/extrinsics.js:207:45)\n    at acceptPendingDataObjects (/joystream/storage-node/lib/services/runtime/extrinsics.js:115:18)\n    at uploadFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:99:57)",
          "message": "Timeout"
        },
        "level": "error",
        "message": "uncaughtException: Timeout\nError\n    at new module.exports.TimeoutError (/joystream/node_modules/promise-timeout/index.js:44:16)\n    at module.exports.timeout (/joystream/node_modules/promise-timeout/index.js:20:15)\n    at extrinsicWrapper (/joystream/storage-node/lib/services/runtime/extrinsics.js:207:45)\n    at acceptPendingDataObjects (/joystream/storage-node/lib/services/runtime/extrinsics.js:115:18)\n    at uploadFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:99:57)",
        "stack": "Error\n    at new module.exports.TimeoutError (/joystream/node_modules/promise-timeout/index.js:44:16)\n    at module.exports.timeout (/joystream/node_modules/promise-timeout/index.js:20:15)\n    at extrinsicWrapper (/joystream/storage-node/lib/services/runtime/extrinsics.js:207:45)\n    at acceptPendingDataObjects (/joystream/storage-node/lib/services/runtime/extrinsics.js:115:18)\n    at uploadFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:99:57)",
        "exception": true,
        "date": "Sun Dec 17 2023 17:18:23 GMT+0000 (Coordinated Universal Time)",
        "process": {
          "pid": 6,
          "uid": 0,
          "gid": 0,
          "cwd": "/joystream/storage-node",
          "execPath": "/usr/local/bin/node",
          "version": "v18.18.2",
          "argv": [
            "/usr/local/bin/node",
            "/joystream/storage-node/bin/run",
            "server",
            "--worker=12",
            "--port=3333",
            "--uploads=/data",
            "--sync",
            "--syncInterval=1",
            "--queryNodeEndpoint=https://adovrn-joystream.yyagi.cloud/graphql",
            "--apiUrl=ws://joystream-node:9944/",
            "--keyFile=/keystore/storage-role-key.json",
            "--password=************",
            "--elasticSearchEndpoint=https://elastic.joyutils.org/",
            "--elasticSearchIndex=storage-node-adovrn",
            "--elasticSearchPassword=***********",
            "--elasticSearchUser=storage-adovrn",
            "--logFilePath=/logs"
          ],
          "memoryUsage": {
            "rss": 2431647744,
            "heapTotal": 1814204416,
            "heapUsed": 1754514264,
            "external": 39393467,
            "arrayBuffers": 1572190
          }
        },
        "os": {
          "loadavg": [
            9.66,
            10.08,
            10.18
          ],
          "uptime": 4578491.05
        },
        "trace": [
          {
            "column": 16,
            "file": "/joystream/node_modules/promise-timeout/index.js",
            "function": "new module.exports.TimeoutError",
            "line": 44,
            "method": "TimeoutError",
            "native": false
          },
          {
            "column": 15,
            "file": "/joystream/node_modules/promise-timeout/index.js",
            "function": "module.exports.timeout",
            "line": 20,
            "method": "timeout",
            "native": false
          },
          {
            "column": 45,
            "file": "/joystream/storage-node/lib/services/runtime/extrinsics.js",
            "function": "extrinsicWrapper",
            "line": 207,
            "method": null,
            "native": false
          },
          {
            "column": 18,
            "file": "/joystream/storage-node/lib/services/runtime/extrinsics.js",
            "function": "acceptPendingDataObjects",
            "line": 115,
            "method": null,
            "native": false
          },
          {
            "column": 57,
            "file": "/joystream/storage-node/lib/services/webApi/controllers/filesApi.js",
            "function": "uploadFile",
            "line": 99,
            "method": null,
            "native": false
          }
        ],
        "req": {
          "url": "/api/v1/files?dataObjectId=1454542&storageBucketId=12&bagId=dynamic%3Achannel%3A29895",
          "headers": {
            "host": "adovrn-joystream.yyagi.cloud",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
            "content-length": "20204",
            "accept": "application/json, text/plain, */*",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "en-US,en;q=0.9",
            "content-type": "multipart/form-data; boundary=----WebKitFormBoundary5owYlVQWBq2ZuZs5",
            "origin": "https://gleev.xyz",
            "referer": "https://gleev.xyz/",
            "sec-ch-ua": "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Windows\"",
            "sec-fetch-dest": "empty",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "cross-site",
            "x-forwarded-for": "99.247.226.235",
            "x-forwarded-host": "adovrn-joystream.yyagi.cloud",
            "x-forwarded-proto": "https"
          },
          "method": "POST",
          "httpVersion": "1.1",
          "originalUrl": "/api/v1/files?dataObjectId=1454542&storageBucketId=12&bagId=dynamic%3Achannel%3A29895",
          "query": {
            "dataObjectId": "1454542",
            "storageBucketId": "12",
            "bagId": "dynamic:channel:29895"
          }
        }
      }
    },
    "source": "StorageProvider_12"
  },
  "fields": {
    "fields.meta.req.originalUrl.keyword": [
      "/api/v1/files?dataObjectId=1454542&storageBucketId=12&bagId=dynamic%3Achannel%3A29895"
    ],
    "fields.meta.trace.native": [
      false,
      false,
      false,
      false,
      false
    ],
    "fields.meta.req.headers.x-forwarded-proto": [
      "https"
    ],
    "fields.meta.process.cwd": [
      "/joystream/storage-node"
    ],
    "fields.meta.process.version": [
      "v18.18.2"
    ],
    "fields.meta.process.memoryUsage.heapUsed": [
      1754514264
    ],
    "fields.meta.req.headers.x-forwarded-host": [
      "adovrn-joystream.yyagi.cloud"
    ],
    "fields.meta.level.keyword": [
      "error"
    ],
    "fields.meta.req.headers.sec-fetch-dest.keyword": [
      "empty"
    ],
    "fields.meta.error.message.keyword": [
      "Timeout"
    ],
    "fields.meta.req.headers.accept.keyword": [
      "application/json, text/plain, */*"
    ],
    "fields.meta.process.memoryUsage.heapTotal": [
      1814204416
    ],
    "fields.meta.process.execPath.keyword": [
      "/usr/local/bin/node"
    ],
    "fields.meta.req.headers.x-forwarded-for.keyword": [
      "99.247.226.235"
    ],
    "fields.meta.process.pid": [
      6
    ],
    "fields.meta.error.message": [
      "Timeout"
    ],
    "fields.meta.req.headers.user-agent.keyword": [
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    ],
    "fields.meta.req.headers.content-length": [
      "20204"
    ],
    "fields.meta.req.query.storageBucketId": [
      "12"
    ],
    "fields.meta.os.uptime": [
      4578491
    ],
    "fields.meta.trace.method": [
      "TimeoutError",
      "timeout"
    ],
    "fields.meta.req.headers.user-agent": [
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    ],
    "fields.meta.level": [
      "error"
    ],
    "fields.meta.req.headers.sec-ch-ua-mobile.keyword": [
      "?0"
    ],
    "fields.meta.exception": [
      true
    ],
    "fields.meta.trace.method.keyword": [
      "TimeoutError",
      "timeout"
    ],
    "fields.meta.req.headers.content-type.keyword": [
      "multipart/form-data; boundary=----WebKitFormBoundary5owYlVQWBq2ZuZs5"
    ],
    "fields.meta.req.headers.sec-ch-ua": [
      "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\""
    ],
    "fields.meta.trace.column": [
      16,
      15,
      45,
      18,
      57
    ],
    "fields.meta.trace.function": [
      "new module.exports.TimeoutError",
      "module.exports.timeout",
      "extrinsicWrapper",
      "acceptPendingDataObjects",
      "uploadFile"
    ],
    "fields.meta.req.headers.origin": [
      "https://gleev.xyz"
    ],
    "fields.meta.req.headers.origin.keyword": [
      "https://gleev.xyz"
    ],
    "fields.meta.req.headers.content-length.keyword": [
      "20204"
    ],
    "fields.meta.os.loadavg": [
      9.66,
      10.08,
      10.18
    ],
    "fields.meta.req.headers.accept-language.keyword": [
      "en-US,en;q=0.9"
    ],
    "fields.meta.req.headers.x-forwarded-host.keyword": [
      "adovrn-joystream.yyagi.cloud"
    ],
    "fields.meta.process.version.keyword": [
      "v18.18.2"
    ],
    "fields.meta.trace.function.keyword": [
      "new module.exports.TimeoutError",
      "module.exports.timeout",
      "extrinsicWrapper",
      "acceptPendingDataObjects",
      "uploadFile"
    ],
    "@timestamp": [
      "2023-12-17T17:18:23.213Z"
    ],
    "fields.meta.trace.file.keyword": [
      "/joystream/node_modules/promise-timeout/index.js",
      "/joystream/node_modules/promise-timeout/index.js",
      "/joystream/storage-node/lib/services/runtime/extrinsics.js",
      "/joystream/storage-node/lib/services/runtime/extrinsics.js",
      "/joystream/storage-node/lib/services/webApi/controllers/filesApi.js"
    ],
    "fields.meta.req.headers.x-forwarded-proto.keyword": [
      "https"
    ],
    "fields.meta.trace.line": [
      44,
      20,
      207,
      115,
      99
    ],
    "fields.meta.req.headers.host": [
      "adovrn-joystream.yyagi.cloud"
    ],
    "fields.meta.process.gid": [
      0
    ],
    "fields.meta.req.headers.sec-fetch-site.keyword": [
      "cross-site"
    ],
    "fields.meta.process.cwd.keyword": [
      "/joystream/storage-node"
    ],
    "fields.meta.req.headers.sec-fetch-mode.keyword": [
      "cors"
    ],
    "fields.meta.process.argv": [
      "/usr/local/bin/node",
      "/joystream/storage-node/bin/run",
      "server",
      "--worker=12",
      "--port=3333",
      "--uploads=/data",
      "--sync",
      "--syncInterval=1",
      "--queryNodeEndpoint=https://adovrn-joystream.yyagi.cloud/graphql",
      "--apiUrl=ws://joystream-node:9944/",
      "--keyFile=/keystore/storage-role-key.json",
      "--password=*****",
      "--elasticSearchEndpoint=https://elastic.joyutils.org/",
      "--elasticSearchIndex=storage-node-adovrn",
      "--elasticSearchPassword=********",
      "--elasticSearchUser=storage-adovrn",
      "--logFilePath=/logs"
    ],
    "fields.meta.req.headers.accept-encoding": [
      "gzip, deflate, br"
    ],
    "fields.meta.error.stack": [
      "Error\n    at new module.exports.TimeoutError (/joystream/node_modules/promise-timeout/index.js:44:16)\n    at module.exports.timeout (/joystream/node_modules/promise-timeout/index.js:20:15)\n    at extrinsicWrapper (/joystream/storage-node/lib/services/runtime/extrinsics.js:207:45)\n    at acceptPendingDataObjects (/joystream/storage-node/lib/services/runtime/extrinsics.js:115:18)\n    at uploadFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:99:57)"
    ],
    "fields.meta.req.headers.accept": [
      "application/json, text/plain, */*"
    ],
    "fields.meta.req.query.dataObjectId": [
      "1454542"
    ],
    "fields.meta.process.memoryUsage.arrayBuffers": [
      1572190
    ],
    "fields.meta.req.headers.sec-ch-ua.keyword": [
      "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\""
    ],
    "source": [
      "StorageProvider_12"
    ],
    "fields.meta.req.method.keyword": [
      "POST"
    ],
    "severity.keyword": [
      "error"
    ],
    "fields.meta.req.headers.accept-language": [
      "en-US,en;q=0.9"
    ],
    "fields.meta.req.url.keyword": [
      "/api/v1/files?dataObjectId=1454542&storageBucketId=12&bagId=dynamic%3Achannel%3A29895"
    ],
    "fields.meta.req.headers.referer": [
      "https://gleev.xyz/"
    ],
    "fields.meta.req.query.storageBucketId.keyword": [
      "12"
    ],
    "fields.meta.req.headers.sec-ch-ua-platform": [
      "\"Windows\""
    ],
    "fields.meta.req.httpVersion.keyword": [
      "1.1"
    ],
    "fields.meta.date.keyword": [
      "Sun Dec 17 2023 17:18:23 GMT+0000 (Coordinated Universal Time)"
    ],
    "fields.meta.req.url": [
      "/api/v1/files?dataObjectId=1454542&storageBucketId=12&bagId=dynamic%3Achannel%3A29895"
    ],
    "fields.meta.req.originalUrl": [
      "/api/v1/files?dataObjectId=1454542&storageBucketId=12&bagId=dynamic%3Achannel%3A29895"
    ],
    "fields.meta.message": [
      "uncaughtException: Timeout\nError\n    at new module.exports.TimeoutError (/joystream/node_modules/promise-timeout/index.js:44:16)\n    at module.exports.timeout (/joystream/node_modules/promise-timeout/index.js:20:15)\n    at extrinsicWrapper (/joystream/storage-node/lib/services/runtime/extrinsics.js:207:45)\n    at acceptPendingDataObjects (/joystream/storage-node/lib/services/runtime/extrinsics.js:115:18)\n    at uploadFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:99:57)"
    ],
    "fields.meta.req.headers.sec-fetch-site": [
      "cross-site"
    ],
    "fields.meta.req.httpVersion": [
      "1.1"
    ],
    "fields.meta.req.method": [
      "POST"
    ],
    "message.keyword": [
      "POST /api/v1/files: Error 500: Timeout"
    ],
    "fields.meta.req.query.dataObjectId.keyword": [
      "1454542"
    ],
    "fields.meta.date": [
      "Sun Dec 17 2023 17:18:23 GMT+0000 (Coordinated Universal Time)"
    ],
    "fields.meta.req.query.bagId.keyword": [
      "dynamic:channel:29895"
    ],
    "fields.meta.process.argv.keyword": [
      "/usr/local/bin/node",
      "/joystream/storage-node/bin/run",
      "server",
      "--worker=12",
      "--port=3333",
      "--uploads=/data",
      "--sync",
      "--syncInterval=1",
      "--queryNodeEndpoint=https://adovrn-joystream.yyagi.cloud/graphql",
      "--apiUrl=ws://joystream-node:9944/",
      "--keyFile=/keystore/storage-role-key.json",
      "--password=***********",
      "--elasticSearchEndpoint=https://elastic.joyutils.org/",
      "--elasticSearchIndex=storage-node-adovrn",
      "--elasticSearchPassword=***************",
      "--elasticSearchUser=storage-adovrn",
      "--logFilePath=/logs"
    ],
    "fields.meta.process.memoryUsage.rss": [
      2431647744
    ],
    "fields.meta.req.headers.host.keyword": [
      "adovrn-joystream.yyagi.cloud"
    ],
    "fields.meta.req.headers.x-forwarded-for": [
      "99.247.226.235"
    ],
    "fields.meta.req.headers.referer.keyword": [
      "https://gleev.xyz/"
    ],
    "source.keyword": [
      "StorageProvider_12"
    ],
    "fields.meta.req.headers.sec-ch-ua-mobile": [
      "?0"
    ],
    "severity": [
      "error"
    ],
    "fields.meta.process.uid": [
      0
    ],
    "message": [
      "POST /api/v1/files: Error 500: Timeout"
    ],
    "fields.meta.req.headers.content-type": [
      "multipart/form-data; boundary=----WebKitFormBoundary5owYlVQWBq2ZuZs5"
    ],
    "fields.meta.process.execPath": [
      "/usr/local/bin/node"
    ],
    "fields.meta.req.headers.sec-fetch-mode": [
      "cors"
    ],
    "fields.meta.req.headers.sec-ch-ua-platform.keyword": [
      "\"Windows\""
    ],
    "fields.meta.trace.file": [
      "/joystream/node_modules/promise-timeout/index.js",
      "/joystream/node_modules/promise-timeout/index.js",
      "/joystream/storage-node/lib/services/runtime/extrinsics.js",
      "/joystream/storage-node/lib/services/runtime/extrinsics.js",
      "/joystream/storage-node/lib/services/webApi/controllers/filesApi.js"
    ],
    "fields.meta.req.headers.accept-encoding.keyword": [
      "gzip, deflate, br"
    ],
    "fields.meta.req.headers.sec-fetch-dest": [
      "empty"
    ],
    "fields.meta.process.memoryUsage.external": [
      39393467
    ],
    "fields.meta.req.query.bagId": [
      "dynamic:channel:29895"
    ],
    "fields.meta.stack": [
      "Error\n    at new module.exports.TimeoutError (/joystream/node_modules/promise-timeout/index.js:44:16)\n    at module.exports.timeout (/joystream/node_modules/promise-timeout/index.js:20:15)\n    at extrinsicWrapper (/joystream/storage-node/lib/services/runtime/extrinsics.js:207:45)\n    at acceptPendingDataObjects (/joystream/storage-node/lib/services/runtime/extrinsics.js:115:18)\n    at uploadFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:99:57)"
    ]
  },
  "ignored_field_values": {
    "fields.meta.message.keyword": [
      "uncaughtException: Timeout\nError\n    at new module.exports.TimeoutError (/joystream/node_modules/promise-timeout/index.js:44:16)\n    at module.exports.timeout (/joystream/node_modules/promise-timeout/index.js:20:15)\n    at extrinsicWrapper (/joystream/storage-node/lib/services/runtime/extrinsics.js:207:45)\n    at acceptPendingDataObjects (/joystream/storage-node/lib/services/runtime/extrinsics.js:115:18)\n    at uploadFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:99:57)"
    ],
    "fields.meta.stack.keyword": [
      "Error\n    at new module.exports.TimeoutError (/joystream/node_modules/promise-timeout/index.js:44:16)\n    at module.exports.timeout (/joystream/node_modules/promise-timeout/index.js:20:15)\n    at extrinsicWrapper (/joystream/storage-node/lib/services/runtime/extrinsics.js:207:45)\n    at acceptPendingDataObjects (/joystream/storage-node/lib/services/runtime/extrinsics.js:115:18)\n    at uploadFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:99:57)"
    ],
    "fields.meta.error.stack.keyword": [
      "Error\n    at new module.exports.TimeoutError (/joystream/node_modules/promise-timeout/index.js:44:16)\n    at module.exports.timeout (/joystream/node_modules/promise-timeout/index.js:20:15)\n    at extrinsicWrapper (/joystream/storage-node/lib/services/runtime/extrinsics.js:207:45)\n    at acceptPendingDataObjects (/joystream/storage-node/lib/services/runtime/extrinsics.js:115:18)\n    at uploadFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:99:57)"
    ]
  }
}
yasiryagi commented 10 months ago

Another crash type (same object)

{
  "_index": "storage-node-itor",
  "_id": "57PJeIwB5UunnGeSSuiI",
  "_version": 1,
  "_score": 0,
  "_ignored": [
    "fields.meta.message.keyword"
  ],
  "_source": {
    "severity": "error",
    "geo": {
      "continent_name": "North America",
      "region_iso_code": "CA-ON",
      "city_name": "Woodbridge",
      "country_iso_code": "CA",
      "country_name": "Canada",
      "region_name": "Ontario",
      "location": {
        "lon": -79.5565,
        "lat": 43.8419
      }
    },
    "@timestamp": "2023-12-17T17:18:26.445Z",
    "source": "StorageProvider_8",
    "message": "POST /api/v1/files: Error 400: Data object 1454542 has already been accepted by storage node",
    "fields": {
      "meta": {
        "exception": true,
        "date": "Sun Dec 17 2023 17:18:26 GMT+0000 (Coordinated Universal Time)",
        "stack": "Error: Data object 1454542 has already been accepted by storage node\n    at validateUploadFileParams (/joystream/storage-node/lib/services/webApi/app.js:203:15)\n    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)",
        "process": {
          "uid": 0,
          "cwd": "/joystream/storage-node",
          "execPath": "/usr/local/bin/node",
          "gid": 0,
          "memoryUsage": {
            "heapUsed": 338321408,
            "external": 61825547,
            "rss": 857182208,
            "heapTotal": 390377472,
            "arrayBuffers": 24004270
          },
          "pid": 8,
          "version": "v18.18.2",
          "argv": [
            "/usr/local/bin/node",
            "/joystream/storage-node/bin/run",
            "server",
            "--worker=8",
            "--port=3333",
            "--uploads=/data",
            "--sync",
            "--syncInterval=1",
            "--queryNodeEndpoint=http://graphql-server:8081/graphql",
            "--apiUrl=ws://joystream-node:9944/",
            "--keyFile=/keystore/storage-role-key.json",
            "--password=**********",
            "--elasticSearchEndpoint=https://elastic.joyutils.org/",
            "--elasticSearchIndex=storage-node-itor",
            "--elasticSearchPassword=***********",
            "--elasticSearchUser=storage-itor",
            "--logFilePath=/logs"
          ]
        },
        "trace": [
          {
            "file": "/joystream/storage-node/lib/services/webApi/app.js",
            "method": null,
            "native": false,
            "line": 203,
            "function": "validateUploadFileParams",
            "column": 15
          },
          {
            "file": "node:internal/process/task_queues",
            "method": "processTicksAndRejections",
            "native": false,
            "line": 95,
            "function": "process.processTicksAndRejections",
            "column": 5
          }
        ],
        "os": {
          "loadavg": [
            5.03,
            2.87,
            2.7
          ],
          "uptime": 3366431.63
        },
        "level": "error",
        "error": {
          "httpStatusCode": 400
        },
        "message": "uncaughtException: Data object 1454542 has already been accepted by storage node\nError: Data object 1454542 has already been accepted by storage node\n    at validateUploadFileParams (/joystream/storage-node/lib/services/webApi/app.js:203:15)\n    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)",
        "req": {
          "headers": {
            "sec-fetch-mode": "cors",
            "content-length": "20204",
            "referer": "https://gleev.xyz/",
            "sec-fetch-site": "cross-site",
            "accept-language": "en-US,en;q=0.9",
            "x-forwarded-proto": "https",
            "origin": "https://gleev.xyz",
            "x-forwarded-for": "99.247.226.235",
            "accept": "application/json, text/plain, */*",
            "sec-ch-ua": "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\"",
            "sec-ch-ua-mobile": "?0",
            "x-forwarded-host": "itor.space",
            "sec-ch-ua-platform": "\"Windows\"",
            "host": "itor.space",
            "content-type": "multipart/form-data; boundary=----WebKitFormBoundaryjyAQCv4p8e7VZxUd",
            "accept-encoding": "gzip, deflate, br",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
            "sec-fetch-dest": "empty"
          },
          "httpVersion": "1.1",
          "method": "POST",
          "query": {
            "bagId": "dynamic:channel:29895",
            "dataObjectId": "1454542",
            "storageBucketId": "1"
          },
          "originalUrl": "/api/v1/files?dataObjectId=1454542&storageBucketId=1&bagId=dynamic%3Achannel%3A29895",
          "url": "/api/v1/files?dataObjectId=1454542&storageBucketId=1&bagId=dynamic%3Achannel%3A29895"
        }
      }
    }
  },
  "fields": {
    "fields.meta.req.originalUrl.keyword": [
      "/api/v1/files?dataObjectId=1454542&storageBucketId=1&bagId=dynamic%3Achannel%3A29895"
    ],
    "fields.meta.trace.native": [
      false,
      false
    ],
    "fields.meta.req.headers.x-forwarded-proto": [
      "https"
    ],
    "fields.meta.process.cwd": [
      "/joystream/storage-node"
    ],
    "fields.meta.process.version": [
      "v18.18.2"
    ],
    "fields.meta.process.memoryUsage.heapUsed": [
      338321408
    ],
    "fields.meta.req.headers.x-forwarded-host": [
      "itor.space"
    ],
    "fields.meta.level.keyword": [
      "error"
    ],
    "fields.meta.req.headers.sec-fetch-dest.keyword": [
      "empty"
    ],
    "geo.region_name.keyword": [
      "Ontario"
    ],
    "geo.country_iso_code.keyword": [
      "CA"
    ],
    "fields.meta.req.headers.accept.keyword": [
      "application/json, text/plain, */*"
    ],
    "geo.region_iso_code.keyword": [
      "CA-ON"
    ],
    "fields.meta.process.memoryUsage.heapTotal": [
      390377472
    ],
    "fields.meta.process.execPath.keyword": [
      "/usr/local/bin/node"
    ],
    "fields.meta.req.headers.x-forwarded-for.keyword": [
      "99.247.226.235"
    ],
    "fields.meta.process.pid": [
      8
    ],
    "geo.country_name.keyword": [
      "Canada"
    ],
    "fields.meta.req.headers.user-agent.keyword": [
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    ],
    "fields.meta.req.headers.content-length": [
      "20204"
    ],
    "fields.meta.req.query.storageBucketId": [
      "1"
    ],
    "fields.meta.os.uptime": [
      3366431.8
    ],
    "fields.meta.trace.method": [
      "processTicksAndRejections"
    ],
    "fields.meta.req.headers.user-agent": [
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    ],
    "fields.meta.level": [
      "error"
    ],
    "geo.country_name": [
      "Canada"
    ],
    "fields.meta.req.headers.sec-ch-ua-mobile.keyword": [
      "?0"
    ],
    "fields.meta.exception": [
      true
    ],
    "fields.meta.trace.method.keyword": [
      "processTicksAndRejections"
    ],
    "fields.meta.req.headers.content-type.keyword": [
      "multipart/form-data; boundary=----WebKitFormBoundaryjyAQCv4p8e7VZxUd"
    ],
    "fields.meta.req.headers.sec-ch-ua": [
      "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\""
    ],
    "fields.meta.trace.column": [
      15,
      5
    ],
    "fields.meta.trace.function": [
      "validateUploadFileParams",
      "process.processTicksAndRejections"
    ],
    "fields.meta.req.headers.origin.keyword": [
      "https://gleev.xyz"
    ],
    "fields.meta.req.headers.origin": [
      "https://gleev.xyz"
    ],
    "fields.meta.req.headers.content-length.keyword": [
      "20204"
    ],
    "fields.meta.error.httpStatusCode": [
      400
    ],
    "fields.meta.os.loadavg": [
      5.03,
      2.87,
      2.7
    ],
    "fields.meta.req.headers.accept-language.keyword": [
      "en-US,en;q=0.9"
    ],
    "fields.meta.req.headers.x-forwarded-host.keyword": [
      "itor.space"
    ],
    "fields.meta.process.version.keyword": [
      "v18.18.2"
    ],
    "geo.city_name.keyword": [
      "Woodbridge"
    ],
    "fields.meta.trace.function.keyword": [
      "validateUploadFileParams",
      "process.processTicksAndRejections"
    ],
    "@timestamp": [
      "2023-12-17T17:18:26.445Z"
    ],
    "fields.meta.req.headers.x-forwarded-proto.keyword": [
      "https"
    ],
    "fields.meta.trace.file.keyword": [
      "/joystream/storage-node/lib/services/webApi/app.js",
      "node:internal/process/task_queues"
    ],
    "fields.meta.trace.line": [
      203,
      95
    ],
    "fields.meta.req.headers.host": [
      "itor.space"
    ],
    "fields.meta.process.gid": [
      0
    ],
    "fields.meta.req.headers.sec-fetch-site.keyword": [
      "cross-site"
    ],
    "fields.meta.process.cwd.keyword": [
      "/joystream/storage-node"
    ],
    "fields.meta.req.headers.sec-fetch-mode.keyword": [
      "cors"
    ],
    "fields.meta.process.argv": [
      "/usr/local/bin/node",
      "/joystream/storage-node/bin/run",
      "server",
      "--worker=8",
      "--port=3333",
      "--uploads=/data",
      "--sync",
      "--syncInterval=1",
      "--queryNodeEndpoint=http://graphql-server:8081/graphql",
      "--apiUrl=ws://joystream-node:9944/",
      "--keyFile=/keystore/storage-role-key.json",
      "--password=**********",
      "--elasticSearchEndpoint=https://elastic.joyutils.org/",
      "--elasticSearchIndex=storage-node-itor",
      "--elasticSearchPassword=***********",
      "--elasticSearchUser=storage-itor",
      "--logFilePath=/logs"
    ],
    "fields.meta.req.headers.accept-encoding": [
      "gzip, deflate, br"
    ],
    "fields.meta.req.headers.accept": [
      "application/json, text/plain, */*"
    ],
    "fields.meta.req.query.dataObjectId": [
      "1454542"
    ],
    "fields.meta.process.memoryUsage.arrayBuffers": [
      24004270
    ],
    "fields.meta.req.headers.sec-ch-ua.keyword": [
      "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\""
    ],
    "source": [
      "StorageProvider_8"
    ],
    "fields.meta.req.method.keyword": [
      "POST"
    ],
    "geo.country_iso_code": [
      "CA"
    ],
    "severity.keyword": [
      "error"
    ],
    "fields.meta.req.headers.accept-language": [
      "en-US,en;q=0.9"
    ],
    "geo.location": [
      {
        "coordinates": [
          -79.5565,
          43.8419
        ],
        "type": "Point"
      }
    ],
    "fields.meta.req.headers.referer": [
      "https://gleev.xyz/"
    ],
    "fields.meta.req.url.keyword": [
      "/api/v1/files?dataObjectId=1454542&storageBucketId=1&bagId=dynamic%3Achannel%3A29895"
    ],
    "fields.meta.req.query.storageBucketId.keyword": [
      "1"
    ],
    "fields.meta.req.headers.sec-ch-ua-platform": [
      "\"Windows\""
    ],
    "fields.meta.req.httpVersion.keyword": [
      "1.1"
    ],
    "fields.meta.date.keyword": [
      "Sun Dec 17 2023 17:18:26 GMT+0000 (Coordinated Universal Time)"
    ],
    "fields.meta.req.url": [
      "/api/v1/files?dataObjectId=1454542&storageBucketId=1&bagId=dynamic%3Achannel%3A29895"
    ],
    "fields.meta.req.originalUrl": [
      "/api/v1/files?dataObjectId=1454542&storageBucketId=1&bagId=dynamic%3Achannel%3A29895"
    ],
    "fields.meta.message": [
      "uncaughtException: Data object 1454542 has already been accepted by storage node\nError: Data object 1454542 has already been accepted by storage node\n    at validateUploadFileParams (/joystream/storage-node/lib/services/webApi/app.js:203:15)\n    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)"
    ],
    "fields.meta.req.headers.sec-fetch-site": [
      "cross-site"
    ],
    "fields.meta.req.httpVersion": [
      "1.1"
    ],
    "fields.meta.req.method": [
      "POST"
    ],
    "geo.city_name": [
      "Woodbridge"
    ],
    "message.keyword": [
      "POST /api/v1/files: Error 400: Data object 1454542 has already been accepted by storage node"
    ],
    "fields.meta.stack.keyword": [
      "Error: Data object 1454542 has already been accepted by storage node\n    at validateUploadFileParams (/joystream/storage-node/lib/services/webApi/app.js:203:15)\n    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)"
    ],
    "geo.continent_name.keyword": [
      "North America"
    ],
    "fields.meta.req.query.dataObjectId.keyword": [
      "1454542"
    ],
    "fields.meta.date": [
      "Sun Dec 17 2023 17:18:26 GMT+0000 (Coordinated Universal Time)"
    ],
    "fields.meta.req.query.bagId.keyword": [
      "dynamic:channel:29895"
    ],
    "fields.meta.process.argv.keyword": [
      "/usr/local/bin/node",
      "/joystream/storage-node/bin/run",
      "server",
      "--worker=8",
      "--port=3333",
      "--uploads=/data",
      "--sync",
      "--syncInterval=1",
      "--queryNodeEndpoint=http://graphql-server:8081/graphql",
      "--apiUrl=ws://joystream-node:9944/",
      "--keyFile=/keystore/storage-role-key.json",
      "--password=*************",
      "--elasticSearchEndpoint=https://elastic.joyutils.org/",
      "--elasticSearchIndex=storage-node-itor",
      "--elasticSearchPassword=************",
      "--elasticSearchUser=storage-itor",
      "--logFilePath=/logs"
    ],
    "fields.meta.process.memoryUsage.rss": [
      857182208
    ],
    "geo.region_iso_code": [
      "CA-ON"
    ],
    "fields.meta.req.headers.host.keyword": [
      "itor.space"
    ],
    "fields.meta.req.headers.x-forwarded-for": [
      "99.247.226.235"
    ],
    "fields.meta.req.headers.referer.keyword": [
      "https://gleev.xyz/"
    ],
    "source.keyword": [
      "StorageProvider_8"
    ],
    "fields.meta.req.headers.sec-ch-ua-mobile": [
      "?0"
    ],
    "severity": [
      "error"
    ],
    "geo.region_name": [
      "Ontario"
    ],
    "geo.continent_name": [
      "North America"
    ],
    "fields.meta.process.uid": [
      0
    ],
    "message": [
      "POST /api/v1/files: Error 400: Data object 1454542 has already been accepted by storage node"
    ],
    "fields.meta.req.headers.content-type": [
      "multipart/form-data; boundary=----WebKitFormBoundaryjyAQCv4p8e7VZxUd"
    ],
    "fields.meta.process.execPath": [
      "/usr/local/bin/node"
    ],
    "fields.meta.req.headers.sec-fetch-mode": [
      "cors"
    ],
    "fields.meta.req.headers.sec-ch-ua-platform.keyword": [
      "\"Windows\""
    ],
    "fields.meta.trace.file": [
      "/joystream/storage-node/lib/services/webApi/app.js",
      "node:internal/process/task_queues"
    ],
    "fields.meta.req.headers.accept-encoding.keyword": [
      "gzip, deflate, br"
    ],
    "fields.meta.req.headers.sec-fetch-dest": [
      "empty"
    ],
    "fields.meta.process.memoryUsage.external": [
      61825547
    ],
    "fields.meta.req.query.bagId": [
      "dynamic:channel:29895"
    ],
    "fields.meta.stack": [
      "Error: Data object 1454542 has already been accepted by storage node\n    at validateUploadFileParams (/joystream/storage-node/lib/services/webApi/app.js:203:15)\n    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)"
    ]
  },
  "ignored_field_values": {
    "fields.meta.message.keyword": [
      "uncaughtException: Data object 1454542 has already been accepted by storage node\nError: Data object 1454542 has already been accepted by storage node\n    at validateUploadFileParams (/joystream/storage-node/lib/services/webApi/app.js:203:15)\n    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)"
    ]
  }
}
yasiryagi commented 10 months ago

Another crash type (different object)

{
  "_index": "storage-node-adovrn",
  "_id": "3gTKeIwBh1Mrve8iuI3W",
  "_version": 1,
  "_score": 0,
  "_ignored": [
    "fields.meta.message.keyword",
    "fields.meta.stack.keyword"
  ],
  "_source": {
    "@timestamp": "2023-12-17T17:19:59.282Z",
    "message": "GET /api/v1/files/1454477: Error 404: ENOENT: no such file or directory, open '/data/1454477'",
    "severity": "error",
    "fields": {
      "meta": {
        "error": {
          "errno": -2,
          "syscall": "open",
          "code": "ENOENT",
          "path": "/data/1454477"
        },
        "level": "error",
        "message": "uncaughtException: ENOENT: no such file or directory, open '/data/1454477'\nError: ENOENT: no such file or directory, open '/data/1454477'\n    at Object.openSync (node:fs:603:3)\n    at module.exports.sync (/joystream/node_modules/with-open-file/index.js:19:17)\n    at module.exports.sync (/joystream/node_modules/read-chunk/index.js:30:33)\n    at getFileInfo (/joystream/storage-node/lib/services/helpers/fileInfo.js:25:41)\n    at getFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:31:59)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at next (/joystream/node_modules/express/lib/router/route.js:137:13)\n    at Route.dispatch (/joystream/node_modules/express/lib/router/route.js:112:3)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at /joystream/node_modules/express/lib/router/index.js:281:22",
        "stack": "Error: ENOENT: no such file or directory, open '/data/1454477'\n    at Object.openSync (node:fs:603:3)\n    at module.exports.sync (/joystream/node_modules/with-open-file/index.js:19:17)\n    at module.exports.sync (/joystream/node_modules/read-chunk/index.js:30:33)\n    at getFileInfo (/joystream/storage-node/lib/services/helpers/fileInfo.js:25:41)\n    at getFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:31:59)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at next (/joystream/node_modules/express/lib/router/route.js:137:13)\n    at Route.dispatch (/joystream/node_modules/express/lib/router/route.js:112:3)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at /joystream/node_modules/express/lib/router/index.js:281:22",
        "exception": true,
        "date": "Sun Dec 17 2023 17:19:59 GMT+0000 (Coordinated Universal Time)",
        "process": {
          "pid": 6,
          "uid": 0,
          "gid": 0,
          "cwd": "/joystream/storage-node",
          "execPath": "/usr/local/bin/node",
          "version": "v18.18.2",
          "argv": [
            "/usr/local/bin/node",
            "/joystream/storage-node/bin/run",
            "server",
            "--worker=12",
            "--port=3333",
            "--uploads=/data",
            "--sync",
            "--syncInterval=1",
            "--queryNodeEndpoint=https://adovrn-joystream.yyagi.cloud/graphql",
            "--apiUrl=ws://joystream-node:9944/",
            "--keyFile=/keystore/storage-role-key.json",
            "--password=^^^^^^",
            "--elasticSearchEndpoint=https://elastic.joyutils.org/",
            "--elasticSearchIndex=storage-node-adovrn",
            "--elasticSearchPassword=********",
            "--elasticSearchUser=storage-adovrn",
            "--logFilePath=/logs"
          ],
          "memoryUsage": {
            "rss": 1465745408,
            "heapTotal": 848318464,
            "heapUsed": 744243304,
            "external": 123854872,
            "arrayBuffers": 86033595
          }
        },
        "os": {
          "loadavg": [
            9.18,
            9.85,
            10.09
          ],
          "uptime": 4578587.12
        },
        "trace": [
          {
            "column": 3,
            "file": "node:fs",
            "function": "Object.openSync",
            "line": 603,
            "method": "openSync",
            "native": false
          },
          {
            "column": 17,
            "file": "/joystream/node_modules/with-open-file/index.js",
            "function": "module.exports.sync",
            "line": 19,
            "method": "sync",
            "native": false
          },
          {
            "column": 33,
            "file": "/joystream/node_modules/read-chunk/index.js",
            "function": "module.exports.sync",
            "line": 30,
            "method": "sync",
            "native": false
          },
          {
            "column": 41,
            "file": "/joystream/storage-node/lib/services/helpers/fileInfo.js",
            "function": "getFileInfo",
            "line": 25,
            "method": null,
            "native": false
          },
          {
            "column": 59,
            "file": "/joystream/storage-node/lib/services/webApi/controllers/filesApi.js",
            "function": "getFile",
            "line": 31,
            "method": null,
            "native": false
          },
          {
            "column": 5,
            "file": "/joystream/node_modules/express/lib/router/layer.js",
            "function": "Layer.handle [as handle_request]",
            "line": 95,
            "method": "handle [as handle_request]",
            "native": false
          },
          {
            "column": 13,
            "file": "/joystream/node_modules/express/lib/router/route.js",
            "function": "next",
            "line": 137,
            "method": null,
            "native": false
          },
          {
            "column": 3,
            "file": "/joystream/node_modules/express/lib/router/route.js",
            "function": "Route.dispatch",
            "line": 112,
            "method": "dispatch",
            "native": false
          },
          {
            "column": 5,
            "file": "/joystream/node_modules/express/lib/router/layer.js",
            "function": "Layer.handle [as handle_request]",
            "line": 95,
            "method": "handle [as handle_request]",
            "native": false
          },
          {
            "column": 22,
            "file": "/joystream/node_modules/express/lib/router/index.js",
            "function": null,
            "line": 281,
            "method": null,
            "native": false
          }
        ],
        "req": {
          "url": "/api/v1/files/1454477",
          "headers": {
            "host": "adovrn-joystream.yyagi.cloud",
            "accept-encoding": "gzip, deflate",
            "x-forwarded-for": "141.101.196.34",
            "x-forwarded-host": "adovrn-joystream.yyagi.cloud",
            "x-forwarded-proto": "https"
          },
          "method": "GET",
          "httpVersion": "1.1",
          "originalUrl": "/api/v1/files/1454477",
          "query": {}
        }
      }
    },
    "source": "StorageProvider_12"
  },
  "fields": {
    "fields.meta.req.originalUrl.keyword": [
      "/api/v1/files/1454477"
    ],
    "fields.meta.process.memoryUsage.arrayBuffers": [
      86033595
    ],
    "fields.meta.trace.native": [
      false,
      false,
      false,
      false,
      false,
      false,
      false,
      false,
      false,
      false
    ],
    "fields.meta.req.headers.x-forwarded-proto": [
      "https"
    ],
    "fields.meta.process.cwd": [
      "/joystream/storage-node"
    ],
    "source": [
      "StorageProvider_12"
    ],
    "fields.meta.error.syscall": [
      "open"
    ],
    "fields.meta.process.version": [
      "v18.18.2"
    ],
    "fields.meta.req.method.keyword": [
      "GET"
    ],
    "severity.keyword": [
      "error"
    ],
    "fields.meta.process.memoryUsage.heapUsed": [
      744243304
    ],
    "fields.meta.req.headers.x-forwarded-host": [
      "adovrn-joystream.yyagi.cloud"
    ],
    "fields.meta.req.url.keyword": [
      "/api/v1/files/1454477"
    ],
    "fields.meta.level.keyword": [
      "error"
    ],
    "fields.meta.date.keyword": [
      "Sun Dec 17 2023 17:19:59 GMT+0000 (Coordinated Universal Time)"
    ],
    "fields.meta.req.httpVersion.keyword": [
      "1.1"
    ],
    "fields.meta.error.code": [
      "ENOENT"
    ],
    "fields.meta.req.originalUrl": [
      "/api/v1/files/1454477"
    ],
    "fields.meta.req.url": [
      "/api/v1/files/1454477"
    ],
    "fields.meta.process.memoryUsage.heapTotal": [
      848318464
    ],
    "fields.meta.req.headers.x-forwarded-for.keyword": [
      "141.101.196.34"
    ],
    "fields.meta.process.execPath.keyword": [
      "/usr/local/bin/node"
    ],
    "fields.meta.error.path.keyword": [
      "/data/1454477"
    ],
    "fields.meta.process.pid": [
      6
    ],
    "fields.meta.message": [
      "uncaughtException: ENOENT: no such file or directory, open '/data/1454477'\nError: ENOENT: no such file or directory, open '/data/1454477'\n    at Object.openSync (node:fs:603:3)\n    at module.exports.sync (/joystream/node_modules/with-open-file/index.js:19:17)\n    at module.exports.sync (/joystream/node_modules/read-chunk/index.js:30:33)\n    at getFileInfo (/joystream/storage-node/lib/services/helpers/fileInfo.js:25:41)\n    at getFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:31:59)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at next (/joystream/node_modules/express/lib/router/route.js:137:13)\n    at Route.dispatch (/joystream/node_modules/express/lib/router/route.js:112:3)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at /joystream/node_modules/express/lib/router/index.js:281:22"
    ],
    "fields.meta.error.code.keyword": [
      "ENOENT"
    ],
    "fields.meta.req.httpVersion": [
      "1.1"
    ],
    "fields.meta.req.method": [
      "GET"
    ],
    "message.keyword": [
      "GET /api/v1/files/1454477: Error 404: ENOENT: no such file or directory, open '/data/1454477'"
    ],
    "fields.meta.date": [
      "Sun Dec 17 2023 17:19:59 GMT+0000 (Coordinated Universal Time)"
    ],
    "fields.meta.os.uptime": [
      4578587
    ],
    "fields.meta.error.errno": [
      -2
    ],
    "fields.meta.process.argv.keyword": [
      "/usr/local/bin/node",
      "/joystream/storage-node/bin/run",
      "server",
      "--worker=12",
      "--port=3333",
      "--uploads=/data",
      "--sync",
      "--syncInterval=1",
      "--queryNodeEndpoint=https://adovrn-joystream.yyagi.cloud/graphql",
      "--apiUrl=ws://joystream-node:9944/",
      "--keyFile=/keystore/storage-role-key.json",
      "--password=^^^^^^^",
      "--elasticSearchEndpoint=https://elastic.joyutils.org/",
      "--elasticSearchIndex=storage-node-adovrn",
      "--elasticSearchPassword=^^^^^^^",
      "--elasticSearchUser=storage-adovrn",
      "--logFilePath=/logs"
    ],
    "fields.meta.trace.method": [
      "openSync",
      "sync",
      "sync",
      "handle [as handle_request]",
      "dispatch",
      "handle [as handle_request]"
    ],
    "fields.meta.process.memoryUsage.rss": [
      1465745408
    ],
    "fields.meta.req.headers.host.keyword": [
      "adovrn-joystream.yyagi.cloud"
    ],
    "fields.meta.level": [
      "error"
    ],
    "fields.meta.exception": [
      true
    ],
    "fields.meta.req.headers.x-forwarded-for": [
      "141.101.196.34"
    ],
    "fields.meta.trace.method.keyword": [
      "openSync",
      "sync",
      "sync",
      "handle [as handle_request]",
      "dispatch",
      "handle [as handle_request]"
    ],
    "fields.meta.error.path": [
      "/data/1454477"
    ],
    "fields.meta.trace.column": [
      3,
      17,
      33,
      41,
      59,
      5,
      13,
      3,
      5,
      22
    ],
    "source.keyword": [
      "StorageProvider_12"
    ],
    "fields.meta.trace.function": [
      "Object.openSync",
      "module.exports.sync",
      "module.exports.sync",
      "getFileInfo",
      "getFile",
      "Layer.handle [as handle_request]",
      "next",
      "Route.dispatch",
      "Layer.handle [as handle_request]"
    ],
    "severity": [
      "error"
    ],
    "fields.meta.os.loadavg": [
      9.18,
      9.85,
      10.09
    ],
    "fields.meta.process.uid": [
      0
    ],
    "message": [
      "GET /api/v1/files/1454477: Error 404: ENOENT: no such file or directory, open '/data/1454477'"
    ],
    "fields.meta.error.syscall.keyword": [
      "open"
    ],
    "fields.meta.req.headers.x-forwarded-host.keyword": [
      "adovrn-joystream.yyagi.cloud"
    ],
    "fields.meta.process.version.keyword": [
      "v18.18.2"
    ],
    "fields.meta.trace.function.keyword": [
      "Object.openSync",
      "module.exports.sync",
      "module.exports.sync",
      "getFileInfo",
      "getFile",
      "Layer.handle [as handle_request]",
      "next",
      "Route.dispatch",
      "Layer.handle [as handle_request]"
    ],
    "@timestamp": [
      "2023-12-17T17:19:59.282Z"
    ],
    "fields.meta.process.execPath": [
      "/usr/local/bin/node"
    ],
    "fields.meta.req.headers.x-forwarded-proto.keyword": [
      "https"
    ],
    "fields.meta.trace.file.keyword": [
      "node:fs",
      "/joystream/node_modules/with-open-file/index.js",
      "/joystream/node_modules/read-chunk/index.js",
      "/joystream/storage-node/lib/services/helpers/fileInfo.js",
      "/joystream/storage-node/lib/services/webApi/controllers/filesApi.js",
      "/joystream/node_modules/express/lib/router/layer.js",
      "/joystream/node_modules/express/lib/router/route.js",
      "/joystream/node_modules/express/lib/router/route.js",
      "/joystream/node_modules/express/lib/router/layer.js",
      "/joystream/node_modules/express/lib/router/index.js"
    ],
    "fields.meta.trace.line": [
      603,
      19,
      30,
      25,
      31,
      95,
      137,
      112,
      95,
      281
    ],
    "fields.meta.trace.file": [
      "node:fs",
      "/joystream/node_modules/with-open-file/index.js",
      "/joystream/node_modules/read-chunk/index.js",
      "/joystream/storage-node/lib/services/helpers/fileInfo.js",
      "/joystream/storage-node/lib/services/webApi/controllers/filesApi.js",
      "/joystream/node_modules/express/lib/router/layer.js",
      "/joystream/node_modules/express/lib/router/route.js",
      "/joystream/node_modules/express/lib/router/route.js",
      "/joystream/node_modules/express/lib/router/layer.js",
      "/joystream/node_modules/express/lib/router/index.js"
    ],
    "fields.meta.req.headers.host": [
      "adovrn-joystream.yyagi.cloud"
    ],
    "fields.meta.process.gid": [
      0
    ],
    "fields.meta.req.headers.accept-encoding.keyword": [
      "gzip, deflate"
    ],
    "fields.meta.process.cwd.keyword": [
      "/joystream/storage-node"
    ],
    "fields.meta.process.argv": [
      "/usr/local/bin/node",
      "/joystream/storage-node/bin/run",
      "server",
      "--worker=12",
      "--port=3333",
      "--uploads=/data",
      "--sync",
      "--syncInterval=1",
      "--queryNodeEndpoint=https://adovrn-joystream.yyagi.cloud/graphql",
      "--apiUrl=ws://joystream-node:9944/",
      "--keyFile=/keystore/storage-role-key.json",
      "--password=^^^^^^^",
      "--elasticSearchEndpoint=https://elastic.joyutils.org/",
      "--elasticSearchIndex=storage-node-adovrn",
      "--elasticSearchPassword=********",
      "--elasticSearchUser=storage-adovrn",
      "--logFilePath=/logs"
    ],
    "fields.meta.req.headers.accept-encoding": [
      "gzip, deflate"
    ],
    "fields.meta.process.memoryUsage.external": [
      123854872
    ],
    "fields.meta.stack": [
      "Error: ENOENT: no such file or directory, open '/data/1454477'\n    at Object.openSync (node:fs:603:3)\n    at module.exports.sync (/joystream/node_modules/with-open-file/index.js:19:17)\n    at module.exports.sync (/joystream/node_modules/read-chunk/index.js:30:33)\n    at getFileInfo (/joystream/storage-node/lib/services/helpers/fileInfo.js:25:41)\n    at getFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:31:59)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at next (/joystream/node_modules/express/lib/router/route.js:137:13)\n    at Route.dispatch (/joystream/node_modules/express/lib/router/route.js:112:3)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at /joystream/node_modules/express/lib/router/index.js:281:22"
    ]
  },
  "ignored_field_values": {
    "fields.meta.message.keyword": [
      "uncaughtException: ENOENT: no such file or directory, open '/data/1454477'\nError: ENOENT: no such file or directory, open '/data/1454477'\n    at Object.openSync (node:fs:603:3)\n    at module.exports.sync (/joystream/node_modules/with-open-file/index.js:19:17)\n    at module.exports.sync (/joystream/node_modules/read-chunk/index.js:30:33)\n    at getFileInfo (/joystream/storage-node/lib/services/helpers/fileInfo.js:25:41)\n    at getFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:31:59)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at next (/joystream/node_modules/express/lib/router/route.js:137:13)\n    at Route.dispatch (/joystream/node_modules/express/lib/router/route.js:112:3)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at /joystream/node_modules/express/lib/router/index.js:281:22"
    ],
    "fields.meta.stack.keyword": [
      "Error: ENOENT: no such file or directory, open '/data/1454477'\n    at Object.openSync (node:fs:603:3)\n    at module.exports.sync (/joystream/node_modules/with-open-file/index.js:19:17)\n    at module.exports.sync (/joystream/node_modules/read-chunk/index.js:30:33)\n    at getFileInfo (/joystream/storage-node/lib/services/helpers/fileInfo.js:25:41)\n    at getFile (/joystream/storage-node/lib/services/webApi/controllers/filesApi.js:31:59)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at next (/joystream/node_modules/express/lib/router/route.js:137:13)\n    at Route.dispatch (/joystream/node_modules/express/lib/router/route.js:112:3)\n    at Layer.handle [as handle_request] (/joystream/node_modules/express/lib/router/layer.js:95:5)\n    at /joystream/node_modules/express/lib/router/index.js:281:22"
    ]
  }
}
kdembler commented 10 months ago

closed by https://github.com/Joystream/joystream/pull/5016