Open dominictory opened 2 months ago
@dominictory do you have a specific repro case? And I'm not sure what's the issue here: is that the platform hanging for a while and sending you this message? Or the fact that you see interceptors calls that are unclear?
@dominictory do you have a specific repro case? And I'm not sure what's the issue here: is that the platform hanging for a while and sending you this message? Or the fact that you see interceptors calls that are unclear?
Honestly not sure, past few versions I've experienced a very unstable platform where sometimes the page I want takes a while to load, and then just gives me 'an unknown error has occurred' in the UI. Having inspected the logs when this happens, I get the above errors. I've noticed similar issue mentioned on Slack fairly recently but I can't work out exactly what the issue is.
Seeing the same thing in a clustered deployment, lots of container restarts.
UI is almost unusable post 6.1.X, dashboard load time is >12 seconds
NAME READY STATUS RESTARTS AGE
opencti-elastic-es-leaders-0 1/1 Running 0 52m
opencti-elastic-es-leaders-1 1/1 Running 0 54m
opencti-elastic-es-leaders-2 1/1 Running 0 56m
opencti-elastic-es-data-0 1/1 Running 0 58m
opencti-elastic-es-data-1 1/1 Running 0 60m
opencti-elastic-es-data-2 1/1 Running 0 62m
opencti-kibana-kb-654f956c44-z7dms 1/1 Running 0 47m
opencti-minio-6896b4f4dc-ntl4g 1/1 Running 0 47m
opencti-opencti-api-54b696d688-dnfj5 2/2 Running 1 (42m ago) 47m
opencti-opencti-api-54b696d688-h9vm2 2/2 Running 1 (2m33s ago) 47m
opencti-opencti-api-54b696d688-j65kv 2/2 Running 0 47m
opencti-opencti-api-54b696d688-x62gz 2/2 Running 1 (10m ago) 47m
opencti-opencti-api-54b696d688-xhxh8 2/2 Running 1 (4m29s ago) 47m
opencti-opencti-web-64c7ffc568-2w69h 2/2 Running 1 (6m15s ago) 45m
opencti-opencti-web-64c7ffc568-4lzng 2/2 Running 1 (73s ago) 46m
opencti-opencti-web-64c7ffc568-fnfhv 2/2 Running 1 (12m ago) 47m
opencti-opencti-web-64c7ffc568-kx8lq 2/2 Running 1 (8m22s ago) 47m
opencti-opencti-web-64c7ffc568-nkh9n 2/2 Running 1 (44m ago) 47m
opencti-opencti-worker-675c866964-7vsg6 1/1 Running 0 47m
opencti-opencti-worker-675c866964-kng68 1/1 Running 0 47m
opencti-opencti-worker-675c866964-lbh8x 1/1 Running 1 (46m ago) 47m
opencti-opencti-worker-675c866964-mvv9g 1/1 Running 0 47m
opencti-opencti-worker-675c866964-xj6hd 1/1 Running 0 47m
opencti-rabbitmq-server-0 1/1 Running 0 60m
opencti-rabbitmq-server-1 1/1 Running 0 62m
opencti-rabbitmq-server-2 1/1 Running 0 63m
opencti-redis-master-0 2/2 Running 1 (54m ago) 63m
{
"category": "APP",
"errors": [
{
"attributes": {
"genre": "TECHNICAL",
"http_status": 500,
"promise": {},
"reason": {
"code": "ERR_HTTP_HEADERS_SENT"
}
},
"message": "Engine unhandled rejection",
"name": "UNKNOWN_ERROR",
"stack": "UNKNOWN_ERROR: Engine unhandled rejection\n at error (/opt/opencti/build/src/config/errors.js:8:10)\n at UnknownError (/opt/opencti/build/src/config/errors.js:82:47)\n at process.<anonymous> (/opt/opencti/build/src/boot.js:60:16)\n at process.emit (node:events:519:28)\n at process.emit (/opt/opencti/build/node_modules/source-map-support/source-map-support.js:516:21)\n at emit (node:internal/process/promises:150:20)\n at processPromiseRejections (node:internal/process/promises:284:27)\n at processTicksAndRejections (node:internal/process/task_queues:96:32)"
}
],
"level": "error",
"message": "Engine unhandled rejection",
"source": "backend",
"timestamp": "2024-06-17T20:21:27.867Z",
"version": "6.1.6"
}
{
"category": "APP",
"errors": [
{
"attributes": {
"genre": "TECHNICAL",
"http_status": 500,
"referer": "https://intel/dashboard/data/import"
},
"message": "Http call interceptor fail",
"name": "UNKNOWN_ERROR",
"stack": "UNKNOWN_ERROR: Http call interceptor fail\n at error (/opt/opencti/build/src/config/errors.js:8:10)\n at UnknownError (/opt/opencti/build/src/config/errors.js:82:47)\n at fn (/opt/opencti/build/src/http/httpPlatform.js:455:18)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:71:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)"
},
{
"message": "stream is not readable",
"name": "InternalServerError",
"stack": "InternalServerError: stream is not readable\n at readStream (/opt/opencti/build/node_modules/raw-body/index.js:185:17)\n at getBody (/opt/opencti/build/node_modules/raw-body/index.js:116:12)\n at read (/opt/opencti/build/node_modules/body-parser/lib/read.js:79:3)\n at fn (/opt/opencti/build/node_modules/body-parser/lib/types/json.js:138:5)\n at lle.handle [as handle_request] (/opt/opencti/build/node_modules/express/lib/router/layer.js:95:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:328:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at cors (/opt/opencti/build/node_modules/cors/lib/index.js:188:7)\n at cb (/opt/opencti/build/node_modules/cors/lib/index.js:224:17)\n at originCallback (/opt/opencti/build/node_modules/cors/lib/index.js:214:15)\n at cb (/opt/opencti/build/node_modules/cors/lib/index.js:219:13)\n at optionsCallback (/opt/opencti/build/node_modules/cors/lib/index.js:199:9)\n at fn (/opt/opencti/build/node_modules/cors/lib/index.js:204:7)\n at lle.handle [as handle_request] (/opt/opencti/build/node_modules/express/lib/router/layer.js:95:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:328:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at Function.handle (/opt/opencti/build/node_modules/express/lib/router/index.js:175:3)\n at router (/opt/opencti/build/node_modules/express/lib/router/index.js:47:12)"
}
],
"level": "error",
"message": "Http call interceptor fail",
"source": "backend",
"timestamp": "2024-06-17T20:21:55.993Z",
"version": "6.1.6"
}
{
"category": "APP",
"errors": [
{
"attributes": {
"genre": "TECHNICAL",
"http_status": 500,
"referer": "https://intel/dashboard/data/import"
},
"message": "Http call interceptor fail",
"name": "UNKNOWN_ERROR",
"stack": "UNKNOWN_ERROR: Http call interceptor fail\n at error (/opt/opencti/build/src/config/errors.js:8:10)\n at UnknownError (/opt/opencti/build/src/config/errors.js:82:47)\n at fn (/opt/opencti/build/src/http/httpPlatform.js:455:18)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:71:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at lle.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)"
},
{
"message": "stream is not readable",
"name": "InternalServerError",
"stack": "InternalServerError: stream is not readable\n at readStream (/opt/opencti/build/node_modules/raw-body/index.js:185:17)\n at getBody (/opt/opencti/build/node_modules/raw-body/index.js:116:12)\n at read (/opt/opencti/build/node_modules/body-parser/lib/read.js:79:3)\n at fn (/opt/opencti/build/node_modules/body-parser/lib/types/json.js:138:5)\n at lle.handle [as handle_request] (/opt/opencti/build/node_modules/express/lib/router/layer.js:95:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:328:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at cors (/opt/opencti/build/node_modules/cors/lib/index.js:188:7)\n at cb (/opt/opencti/build/node_modules/cors/lib/index.js:224:17)\n at originCallback (/opt/opencti/build/node_modules/cors/lib/index.js:214:15)\n at cb (/opt/opencti/build/node_modules/cors/lib/index.js:219:13)\n at optionsCallback (/opt/opencti/build/node_modules/cors/lib/index.js:199:9)\n at fn (/opt/opencti/build/node_modules/cors/lib/index.js:204:7)\n at lle.handle [as handle_request] (/opt/opencti/build/node_modules/express/lib/router/layer.js:95:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:328:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at Function.handle (/opt/opencti/build/node_modules/express/lib/router/index.js:175:3)\n at router (/opt/opencti/build/node_modules/express/lib/router/index.js:47:12)"
}
],
"level": "error",
"message": "Http call interceptor fail",
"source": "backend",
"timestamp": "2024-06-17T20:22:02.012Z",
"version": "6.1.6"
}
Do you still see this happening?
Do you still see this happening?
I do, it seems to be every few hours roughly, although the platform is not unusable:
{"category":"APP","errors":[{"attributes":{"genre":"TECHNICAL","http_status":500},"message":"Http call interceptor fail","name":"UNKNOWN_ERROR","stack":"UNKNOWN_ERROR: Http call interceptor fail\n at error (/opt/opencti/build/src/config/errors.js:8:10)\n at UnknownError (/opt/opencti/build/src/config/errors.js:82:47)\n at fn (/opt/opencti/build/src/http/httpPlatform.js:463:18)\n at $le.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:71:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at $le.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at $le.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at $le.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)"},{"message":"request aborted","name":"BadRequestError","stack":"BadRequestError: request aborted\n at IncomingMessage.onAborted (/opt/opencti/build/node_modules/raw-body/index.js:245:10)\n at IncomingMessage.emit (node:events:519:28)\n at IncomingMessage._destroy (node:_http_incoming:224:10)\n at _destroy (node:internal/streams/destroy:121:10)\n at IncomingMessage.destroy (node:internal/streams/destroy:83:5)\n at abortIncoming (node:_http_server:797:9)\n at socketOnClose (node:_http_server:791:3)\n at TLSSocket.emit (node:events:531:35)\n at node:net:339:12\n at Socket.done (node:_tls_wrap:657:7)\n at Object.onceWrapper (node:events:634:26)\n at Socket.emit (node:events:531:35)\n at TCP.<anonymous> (node:net:339:12)"}],"level":"error","message":"Http call interceptor fail","source":"backend","timestamp":"2024-08-19T05:49:59.817Z","version":"6.2.9"}
Can you check the CPU usage of the platform? Do you have 100% cpu usage? We identify a problem of usage when too many sessions are created, can you check the number of open sessions you have in the platform ?
Thanks
Can you check the CPU usage of the platform? Do you have 100% cpu usage? We identify a problem of usage when too many sessions are created, can you check the number of open sessions you have in the platform ?
Thanks
Total CPU usage is currently around ~50%. Platform showing >98%, sometimes over 100% usage. Where should I look for total open sessions? There's quite a few for the users going back a few days. The platform is stable but every now and then it does produce these errors, I suspect when many bundles are being processed, or maybe there's some other issue.
Hello Found out the same issue at the moment
{"category":"APP","codeStack":"vQe@https://[...]/static/js/front-43R7MMGU.js:91:66684
render@https://[...]/static/js/front-43R7MMGU.js:91:67547\nN8i</a.render@https://[...]/static/js/front-43R7MMGU.js:49:38490...
I was trying to make better firewall rules, but forgot the rabbitmq service...
Found out this error (seconds after the first one)
"message":"connect ETIMEDOUT 172.31.16.223:15671","operation":"Unspecified","query_attributes":[[{"arguments":[],"name":"rabbitMQMetrics"}]],"size":2,"source":"backend","time":129561,"timestamp":"2024-09-03T15:59:37.133Z","type":"READ_ERROR"
Changed the RabbitMQ management port firewall rule, and now my opencti is up !
@dominictory, look at your connections with rabbitmq (both RABBITMQPORT, RABBITMQPORT_MANAGEMENT)
Description
Whilst resource appears fine for my stack, CTI regularly hangs, and the platform often presents me with 'an unknown error has occurred'. I am seeing a lot of HTTP call interceptor fail errors in the logs, as below:
ory":"APP","errors":[{"attributes":{"genre":"TECHNICAL","http_status":500},"message":"Http call interceptor fail","name":"UNKNOWN_ERROR","stack":"UNKNOWN_ERROR: Http call interceptor fail\n at error (/opt/opencti/build/src/config/errors.js:8:10)\n at UnknownError (/opt/opencti/build/src/config/errors.js:82:47)\n at fn (/opt/opencti/build/src/http/httpPlatform.js:455:18)\n at ule.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:71:5)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at ule.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at ule.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)\n at done (/opt/opencti/build/node_modules/express/lib/router/index.js:286:9)\n at Function.process_params (/opt/opencti/build/node_modules/express/lib/router/index.js:346:12)\n at next (/opt/opencti/build/node_modules/express/lib/router/index.js:280:10)\n at ule.handle_error (/opt/opencti/build/node_modules/express/lib/router/layer.js:67:12)\n at trim_prefix (/opt/opencti/build/node_modules/express/lib/router/index.js:326:13)"},{"message":"request aborted","name":"BadRequestError","stack":"BadRequestError: request aborted\n at IncomingMessage.onAborted (/opt/opencti/build/node_modules/raw-body/index.js:245:10)\n at IncomingMessage.emit (node:events:519:28)\n at IncomingMessage._destroy (node:_http_incoming:224:10)\n at _destroy (node:internal/streams/destroy:121:10)\n at IncomingMessage.destroy (node:internal/streams/destroy:83:5)\n at abortIncoming (node:_http_server:797:9)\n at socketOnClose (node:_http_server:791:3)\n at TLSSocket.emit (node:events:531:35)\n at node:net:338:12\n at Socket.done (node:_tls_wrap:657:7)\n at Object.onceWrapper (node:events:634:26)\n at Socket.emit (node:events:531:35)\n at TCP.<anonymous> (node:net:338:12)"}],"level":"error","message":"Http call interceptor fail","source":"backend","timestamp":"2024-06-13T09:30:10.272Z","version":"6.1.10"}
Another error I'm seeing that I've not seen before is as below:
Environment
OpenCTI 6.1.10 Ubuntu 22.04 LTS Docker
Additional information
Also seeing a lot of these now:
(node:7) MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 abort listeners added to [EventEmitter]. Use emitter.setMaxListeners() to increase limit
Other notable errors in RabbitMQ: