Open alutman-innablr opened 7 months ago
lyricnz
commented
1 month ago We are experiencing this, but the default-limit of 10 items in the stack trace is preventing us from seeing which application-code is hitting the limit. Will experiment with increasing this using $NODE_OPTIONS of stack-trace-limit=10000
lyricnz
commented
1 month ago Happened again last night. Here's the full error/stack-tracve
{
"message": "Rate exceeded",
"nativeError": {
"name": "Throttling",
"$fault": "client",
"$metadata": {
"httpStatusCode": 400,
"requestId": "b44b5728-5b1c-43e3-99af-f84f9e68ac06",
"attempts": 5,
"totalRetryDelay": 4961
},
"Type": "Sender",
"Code": "Throttling",
"message": "Rate exceeded"
},
"name": "Throttling",
"stack": [
{
"fullFilePath": "webpack://revolver/./node_modules/@smithy/smithy-client/dist-es/default-error-handler.js?:11:22",
"fileName": "default-error-handler.js?",
"fileNameWithLine": "default-error-handler.js?:11",
"fileColumn": "22",
"fileLine": "11",
"filePath": "//revolver/./node_modules/@smithy/smithy-client/dist-es/default-error-handler.js?",
"filePathWithLine": "/revolver/node_modules/@smithy/smithy-client/dist-es/default-error-handler.js?:11",
"method": "throwDefaultError"
},
{
"fullFilePath": "webpack://revolver/./node_modules/@smithy/smithy-client/dist-es/default-error-handler.js?:20:9",
"fileName": "default-error-handler.js?",
"fileNameWithLine": "default-error-handler.js?:20",
"fileColumn": "9",
"fileLine": "20",
"filePath": "//revolver/./node_modules/@smithy/smithy-client/dist-es/default-error-handler.js?",
"filePathWithLine": "/revolver/node_modules/@smithy/smithy-client/dist-es/default-error-handler.js?:20",
"method": "eval"
},
{
"fullFilePath": "webpack://revolver/./node_modules/@aws-sdk/client-auto-scaling/dist-es/protocols/Aws_query.js?:1604:20",
"fileName": "Aws_query.js?",
"fileNameWithLine": "Aws_query.js?:1604",
"fileColumn": "20",
"fileLine": "1604",
"filePath": "//revolver/./node_modules/@aws-sdk/client-auto-scaling/dist-es/protocols/Aws_query.js?",
"filePathWithLine": "/revolver/node_modules/@aws-sdk/client-auto-scaling/dist-es/protocols/Aws_query.js?:1604",
"method": "de_CommandError"
},
{
"fullFilePath": "node:internal/process/task_queues:95:5",
"fileName": "task_queues",
"fileNameWithLine": "task_queues:95",
"fileColumn": "5",
"fileLine": "95",
"filePath": "internal/process/task_queues",
"filePathWithLine": "internal/process/task_queues:95",
"method": "process.processTicksAndRejections"
},
{
"fullFilePath": "webpack://revolver/./node_modules/@smithy/middleware-serde/dist-es/deserializerMiddleware.js?:8:24",
"fileName": "deserializerMiddleware.js?",
"fileNameWithLine": "deserializerMiddleware.js?:8",
"fileColumn": "24",
"fileLine": "8",
"filePath": "//revolver/./node_modules/@smithy/middleware-serde/dist-es/deserializerMiddleware.js?",
"filePathWithLine": "/revolver/node_modules/@smithy/middleware-serde/dist-es/deserializerMiddleware.js?:8",
"method": "async eval"
},
{
"fullFilePath": "webpack://revolver/./node_modules/@smithy/core/dist-es/middleware-http-signing/httpSigningMiddleware.js?:25:20",
"fileName": "httpSigningMiddleware.js?",
"fileNameWithLine": "httpSigningMiddleware.js?:25",
"fileColumn": "20",
"fileLine": "25",
"filePath": "//revolver/./node_modules/@smithy/core/dist-es/middleware-http-signing/httpSigningMiddleware.js?",
"filePathWithLine": "/revolver/node_modules/@smithy/core/dist-es/middleware-http-signing/httpSigningMiddleware.js?:25",
"method": "async eval"
},
{
"fullFilePath": "webpack://revolver/./node_modules/@smithy/middleware-retry/dist-es/retryMiddleware.js?:41:46",
"fileName": "retryMiddleware.js?",
"fileNameWithLine": "retryMiddleware.js?:41",
"fileColumn": "46",
"fileLine": "41",
"filePath": "//revolver/./node_modules/@smithy/middleware-retry/dist-es/retryMiddleware.js?",
"filePathWithLine": "/revolver/node_modules/@smithy/middleware-retry/dist-es/retryMiddleware.js?:41",
"method": "async eval"
},
{
"fullFilePath": "webpack://revolver/./node_modules/@aws-sdk/middleware-logger/dist-es/loggerMiddleware.js?:9:26",
"fileName": "loggerMiddleware.js?",
"fileNameWithLine": "loggerMiddleware.js?:9",
"fileColumn": "26",
"fileLine": "9",
"filePath": "//revolver/./node_modules/@aws-sdk/middleware-logger/dist-es/loggerMiddleware.js?",
"filePathWithLine": "/revolver/node_modules/@aws-sdk/middleware-logger/dist-es/loggerMiddleware.js?:9",
"method": "async eval"
},
{
"fullFilePath": "webpack://revolver/./node_modules/@smithy/core/dist-es/pagination/createPaginator.js?:6:12",
"fileName": "createPaginator.js?",
"fileNameWithLine": "createPaginator.js?:6",
"fileColumn": "12",
"fileLine": "6",
"filePath": "//revolver/./node_modules/@smithy/core/dist-es/pagination/createPaginator.js?",
"filePathWithLine": "/revolver/node_modules/@smithy/core/dist-es/pagination/createPaginator.js?:6",
"method": "async makePagedClientRequest"
},
{
"fullFilePath": "webpack://revolver/./node_modules/@smithy/core/dist-es/pagination/createPaginator.js?:19:24",
"fileName": "createPaginator.js?",
"fileNameWithLine": "createPaginator.js?:19",
"fileColumn": "24",
"fileLine": "19",
"filePath": "//revolver/./node_modules/@smithy/core/dist-es/pagination/createPaginator.js?",
"filePathWithLine": "/revolver/node_modules/@smithy/core/dist-es/pagination/createPaginator.js?:19",
"method": "async paginateOperation"
},
{
"fullFilePath": "webpack://revolver/./lib/common.js?:16:22",
"fileName": "common.js?",
"fileNameWithLine": "common.js?:16",
"fileColumn": "22",
"fileLine": "16",
"filePath": "//revolver/./lib/common.js?",
"filePathWithLine": "/revolver/lib/common.js?:16",
"method": "async paginateAwsCall"
},
{
"fullFilePath": "webpack://revolver/./drivers/ec2.js?:171:35",
"fileName": "ec2.js?",
"fileNameWithLine": "ec2.js?:171",
"fileColumn": "35",
"fileLine": "171",
"filePath": "//revolver/./drivers/ec2.js?",
"filePathWithLine": "/revolver/drivers/ec2.js?:171",
"method": "async Ec2Driver.collect"
},
{},
{
"fullFilePath": "webpack://revolver/./lib/accountRevolver.js?:74:27",
"fileName": "accountRevolver.js?",
"fileNameWithLine": "accountRevolver.js?:74",
"fileColumn": "27",
"fileLine": "74",
"filePath": "//revolver/./lib/accountRevolver.js?",
"filePathWithLine": "/revolver/lib/accountRevolver.js?:74",
"method": "async AccountRevolver.loadResources"
},
{
"fullFilePath": "webpack://revolver/./lib/accountRevolver.js?:162:13",
"fileName": "accountRevolver.js?",
"fileNameWithLine": "accountRevolver.js?:162",
"fileColumn": "13",
"fileLine": "162",
"filePath": "//revolver/./lib/accountRevolver.js?",
"filePathWithLine": "/revolver/lib/accountRevolver.js?:162",
"method": "async AccountRevolver.revolve"
}
],
"accountId": "REMOVED",
"accountName": "aws-REMOVED-ap-southeast-2",
"_meta": {
"runtime": "Nodejs",
"runtimeVersion": "v20.16.0",
"hostname": "169.254.87.189",
"name": "aws-REMOVED-ap-southeast-2(522161664848)",
"parentNames": [
"revolver"
],
"date": "2024-09-10T15:30:48.922Z",
"logLevelId": 5,
"logLevelName": "ERROR"
}
}The stack trace above indicates that the SDK has retried 5 times already, for a totalRetryDelay of 4961ms
$SDK_MAX_RETRIES is set to 5
Revolver shouldn't fail to continue if an AWS api limit is reached. Need to add in the standard stuff around backoff and error handling across all AWS api calls.