Closed mishabruml closed 2 years ago
mishabruml
commented
2 years ago Commenting out the lines
custom:
esbuild:
plugins: esbuild-plugins.js
# external:
# - "@typedorm/core"
# - "@typedorm/common"
sourcemap: trueresolves this issue so it definitely appears to be related to the external dependencies. Whats interesting is that this project has deployed many times previously with this configuration, and is still working in CI
samchungy
commented
2 years ago The change I made in the dependency fix PR was adding -long to the npm command. Without it, it was impossible for me to tell if something was deduped in the tree.
I suppose you've possibly tried running npm ls -json -prod -long -all locally?
mishabruml
commented
2 years ago Yes I have. I have managed to resolve this issue by moving the typedorm dependencies from /common to /service-a, but that isn't ideal
mishabruml
commented
2 years ago {
"version": "1.0.0",
"name": "service-a",
"engines": {
"node": "^16.12.0",
"npm": "^8.1.0"
},
"description": "service-a",
"author": "ovo",
"license": "ISC",
"scripts": {
"tsc:check": "tsc -b --listFiles",
"eslint": "DEBUG=eslint:cli-engine eslint .",
"prettier": "prettier . --check",
"test": "jest",
"sls:deploy": "SLS_DEBUG=* rm -rf .build && sls deploy",
"sls:package": "rm -rf .build && sls package"
},
"devDependencies": {
"@aws-sdk/types": "^3.38.0",
"@emarketeer/esbuild-plugin-tsc": "^0.4.1",
"@tsconfig/node16": "^1.0.2",
"@types/aws-lambda": "^8.10.84",
"@types/http-errors": "^1.8.2",
"@types/jest": "^27.0.2",
"@types/node": "^14.17.20",
"esbuild": "^0.14.18",
"esbuild-plugin-tsc": "^0.3.0",
"jest": "^27.2.4",
"serverless": "^3.1.1",
"serverless-esbuild": "^1.23.4",
"serverless-offline": "^8.5.0",
"serverless-step-functions": "^3.5.1",
"ts-jest": "^27.0.5",
"ts-node": "^10.4.0"
},
"extraneous": false,
"path": "/Users/mishabruml/hm/homemoves-monorepo-poc/packages/service-a",
"_dependencies": {
"@faker-js/faker": "^6.0.0-alpha.7",
"@kafkajs/confluent-schema-registry": "^3.2.0",
"@middy/core": "^2.5.2",
"@middy/http-json-body-parser": "^2.5.7",
"@middy/http-response-serializer": "^2.5.7",
"@middy/ssm": "^2.5.2",
"@*******/avro-kafkajs": "^0.8.1",
"@*******/castle": "^0.8.1",
"@sinclair/typebox": "^0.23.3",
"@typedorm/common": "^1.14.9",
"@typedorm/core": "^1.14.9",
"@types/convict": "^6.1.1",
"avsc": "^5.7.3",
"convict": "^6.2.1",
"http-errors": "^2.0.0",
"kafkajs": "^1.15.0",
"reflect-metadata": "^0.1.13",
"uuid": "^8.3.2",
"zod": "^3.11.6"
},
"peerDependencies": {},
"problems": [
"missing: @faker-js/faker@^6.0.0-alpha.7, required by service-a@1.0.0",
"missing: @kafkajs/confluent-schema-registry@^3.2.0, required by service-a@1.0.0",
"missing: @middy/core@^2.5.2, required by service-a@1.0.0",
"missing: @middy/http-json-body-parser@^2.5.7, required by service-a@1.0.0",
"missing: @middy/http-response-serializer@^2.5.7, required by service-a@1.0.0",
"missing: @middy/ssm@^2.5.2, required by service-a@1.0.0",
"missing: @*******/avro-kafkajs@^0.8.1, required by service-a@1.0.0",
"missing: @*******/castle@^0.8.1, required by service-a@1.0.0",
"missing: @sinclair/typebox@^0.23.3, required by service-a@1.0.0",
"missing: @typedorm/common@^1.14.9, required by service-a@1.0.0",
"missing: @typedorm/core@^1.14.9, required by service-a@1.0.0",
"missing: @types/convict@^6.1.1, required by service-a@1.0.0",
"missing: avsc@^5.7.3, required by service-a@1.0.0",
"missing: convict@^6.2.1, required by service-a@1.0.0",
"missing: http-errors@^2.0.0, required by service-a@1.0.0",
"missing: kafkajs@^1.15.0, required by service-a@1.0.0",
"missing: reflect-metadata@^0.1.13, required by service-a@1.0.0",
"missing: uuid@^8.3.2, required by service-a@1.0.0",
"missing: zod@^3.11.6, required by service-a@1.0.0"
],
"dependencies": {
"@faker-js/faker": {
"required": "^6.0.0-alpha.7",
"missing": true,
"problems": [
"missing: @faker-js/faker@^6.0.0-alpha.7, required by service-a@1.0.0"
]
},
"@kafkajs/confluent-schema-registry": {
"missing": true,
"problems": [
"missing: @kafkajs/confluent-schema-registry@^3.2.0, required by service-a@1.0.0"
]
},
"@middy/core": {
"missing": true,
"problems": [
"missing: @middy/core@^2.5.2, required by service-a@1.0.0"
]
},
"@middy/http-json-body-parser": {
"missing": true,
"problems": [
"missing: @middy/http-json-body-parser@^2.5.7, required by service-a@1.0.0"
]
},
"@middy/http-response-serializer": {
"missing": true,
"problems": [
"missing: @middy/http-response-serializer@^2.5.7, required by service-a@1.0.0"
]
},
"@middy/ssm": {
"missing": true,
"problems": [
"missing: @middy/ssm@^2.5.2, required by service-a@1.0.0"
]
},
"@*******/avro-kafkajs": {
"missing": true,
"problems": [
"missing: @*******/avro-kafkajs@^0.8.1, required by service-a@1.0.0"
]
},
"@*******/castle": {
"missing": true,
"problems": [
"missing: @*******/castle@^0.8.1, required by service-a@1.0.0"
]
},
"@sinclair/typebox": {
"missing": true,
"problems": [
"missing: @sinclair/typebox@^0.23.3, required by service-a@1.0.0"
]
},
"@typedorm/common": {
"missing": true,
"problems": [
"missing: @typedorm/common@^1.14.9, required by service-a@1.0.0"
]
},
"@typedorm/core": {
"missing": true,
"problems": [
"missing: @typedorm/core@^1.14.9, required by service-a@1.0.0"
]
},
"@types/convict": {
"missing": true,
"problems": [
"missing: @types/convict@^6.1.1, required by service-a@1.0.0"
]
},
"avsc": {
"missing": true,
"problems": [
"missing: avsc@^5.7.3, required by service-a@1.0.0"
]
},
"convict": {
"missing": true,
"problems": [
"missing: convict@^6.2.1, required by service-a@1.0.0"
]
},
"http-errors": {
"missing": true,
"problems": [
"missing: http-errors@^2.0.0, required by service-a@1.0.0"
]
},
"kafkajs": {
"missing": true,
"problems": [
"missing: kafkajs@^1.15.0, required by service-a@1.0.0"
]
},
"reflect-metadata": {
"missing": true,
"problems": [
"missing: reflect-metadata@^0.1.13, required by service-a@1.0.0"
]
},
"uuid": {
"missing": true,
"problems": [
"missing: uuid@^8.3.2, required by service-a@1.0.0"
]
},
"zod": {
"missing": true,
"problems": [
"missing: zod@^3.11.6, required by service-a@1.0.0"
]
}
}
}@samchungy this is the output from npm ls -json -prod -long -all locally in service-a. It threw some errors as you can see
samchungy
commented
2 years ago Seeing as you're on node 16 I assume you're using npm v7? I know that npm v7 likes to complain if peerDependencies aren't installed. Do the external deps you've listed have peerDependencies?
mishabruml
commented
2 years ago node -v v16.12.0
npm -v 8.1.0
I need npm 8 for npm workspaces 😁
Do the external deps you've listed have peerDependencies?
"node_modules/@typedorm/common": {
"version": "1.14.9",
"resolved": "https://registry.npmjs.org/@typedorm/common/-/common-1.14.9.tgz",
"integrity": "sha512-rmLydYjpiYnq2PmotkkfiQCdSTeYRHNRa2bV8nB1PAcY6bKBnsa6ap5xD13T1OzsEl0qUBZBGJmRlIw9ImyvGg==",
"dependencies": {
"chalk": "^4.1.0",
"class-transformer": "0.4.0",
"debug": "^4.3.1",
"uuid": "^8.3.1"
},
"peerDependencies": {
"reflect-metadata": "^0.1.13"
}
},
"node_modules/@typedorm/core": {
"version": "1.14.9",
"resolved": "https://registry.npmjs.org/@typedorm/core/-/core-1.14.9.tgz",
"integrity": "sha512-xrvPacYZNY0YOja9nO5xxvsUM2AC+toINmL3GBzZAKPj3z76RNrwBLnOmVjWrH2kqYEQ7n3ejQiqscdEIXLz6w==",
"dependencies": {
"class-transformer": "0.4.0",
"fast-glob": "^3.2.4",
"ksuid": "^2.0.0",
"p-limit": "^3.1.0",
"uuid": "^8.3.1"
},
"peerDependencies": {
"@typedorm/common": "1.14.9",
"aws-sdk": "^2.799.0",
"reflect-metadata": "^0.1.13"
}
},Hmm since this is local. we might be able to log the error.
In node_modules/serverless-esbuild/dist/util.js can you add: console.log(stdout, stderr) between line 51 and 52 and see what it's yelling about? I have a hunch it's related to peer deps. I might be able to add this sort of logging in future for easier debugging
Apologies if I don't reply after this one for a while, currently night time over here 🌕
mishabruml
commented
2 years ago Thanks for looking into this. Here is the output. Some redaction due to private repos 😄
> sls:deploy
> SLS_DEBUG=* rm -rf .build && sls deploy
8.1.0
{
"version": "1.0.0",
"name": "service-a",
"engines": {
"node": "^16.12.0",
"npm": "^8.1.0"
},
"description": "service-a",
"author": "*****",
"license": "ISC",
"scripts": {
"tsc:check": "tsc -b --listFiles",
"eslint": "DEBUG=eslint:cli-engine eslint .",
"prettier": "prettier . --check",
"test": "jest",
"sls:deploy": "SLS_DEBUG=* rm -rf .build && sls deploy",
"sls:package": "rm -rf .build && sls package"
},
"devDependencies": {
"@aws-sdk/types": "^3.38.0",
"@emarketeer/esbuild-plugin-tsc": "^0.4.1",
"@tsconfig/node16": "^1.0.2",
"@types/aws-lambda": "^8.10.84",
"@types/http-errors": "^1.8.2",
"@types/jest": "^27.0.2",
"@types/node": "^14.17.20",
"esbuild": "^0.14.18",
"esbuild-plugin-tsc": "^0.3.0",
"jest": "^27.2.4",
"serverless": "^3.1.1",
"serverless-esbuild": "^1.23.4",
"serverless-iam-roles-per-function": "^3.2.0",
"serverless-offline": "^8.5.0",
"serverless-step-functions": "^3.5.1",
"ts-jest": "^27.0.5",
"ts-node": "^10.4.0"
},
"extraneous": false,
"path": "/Users/mishabruml/*****/**********/packages/service-a",
"_dependencies": {
"@faker-js/faker": "^6.0.0-alpha.7",
"@kafkajs/confluent-schema-registry": "^3.2.0",
"@middy/core": "^2.5.2",
"@middy/http-json-body-parser": "^2.5.7",
"@middy/http-response-serializer": "^2.5.7",
"@middy/ssm": "^2.5.2",
"@*****/avro-kafkajs": "^0.8.1",
"@*****/castle": "^0.8.1",
"@sinclair/typebox": "^0.23.3",
"@types/convict": "^6.1.1",
"avsc": "^5.7.3",
"convict": "^6.2.1",
"http-errors": "^2.0.0",
"kafkajs": "^1.15.0",
"reflect-metadata": "^0.1.13",
"uuid": "^8.3.2",
"zod": "^3.11.6"
},
"peerDependencies": {},
"problems": [
"missing: @faker-js/faker@^6.0.0-alpha.7, required by service-a@1.0.0",
"missing: @kafkajs/confluent-schema-registry@^3.2.0, required by service-a@1.0.0",
"missing: @middy/core@^2.5.2, required by service-a@1.0.0",
"missing: @middy/http-json-body-parser@^2.5.7, required by service-a@1.0.0",
"missing: @middy/http-response-serializer@^2.5.7, required by service-a@1.0.0",
"missing: @middy/ssm@^2.5.2, required by service-a@1.0.0",
"missing: @*****/avro-kafkajs@^0.8.1, required by service-a@1.0.0",
"missing: @*****/castle@^0.8.1, required by service-a@1.0.0",
"missing: @sinclair/typebox@^0.23.3, required by service-a@1.0.0",
"missing: @types/convict@^6.1.1, required by service-a@1.0.0",
"missing: avsc@^5.7.3, required by service-a@1.0.0",
"missing: convict@^6.2.1, required by service-a@1.0.0",
"missing: http-errors@^2.0.0, required by service-a@1.0.0",
"missing: kafkajs@^1.15.0, required by service-a@1.0.0",
"missing: reflect-metadata@^0.1.13, required by service-a@1.0.0",
"missing: uuid@^8.3.2, required by service-a@1.0.0",
"missing: zod@^3.11.6, required by service-a@1.0.0"
],
"dependencies": {
"@faker-js/faker": {
"required": "^6.0.0-alpha.7",
"missing": true,
"problems": [
"missing: @faker-js/faker@^6.0.0-alpha.7, required by service-a@1.0.0"
]
},
"@kafkajs/confluent-schema-registry": {
"missing": true,
"problems": [
"missing: @kafkajs/confluent-schema-registry@^3.2.0, required by service-a@1.0.0"
]
},
"@middy/core": {
"missing": true,
"problems": [
"missing: @middy/core@^2.5.2, required by service-a@1.0.0"
]
},
"@middy/http-json-body-parser": {
"missing": true,
"problems": [
"missing: @middy/http-json-body-parser@^2.5.7, required by service-a@1.0.0"
]
},
"@middy/http-response-serializer": {
"missing": true,
"problems": [
"missing: @middy/http-response-serializer@^2.5.7, required by service-a@1.0.0"
]
},
"@middy/ssm": {
"missing": true,
"problems": [
"missing: @middy/ssm@^2.5.2, required by service-a@1.0.0"
]
},
"@*****/avro-kafkajs": {
"missing": true,
"problems": [
"missing: @*****/avro-kafkajs@^0.8.1, required by service-a@1.0.0"
]
},
"@*****/castle": {
"missing": true,
"problems": [
"missing: @*****/castle@^0.8.1, required by service-a@1.0.0"
]
},
"@sinclair/typebox": {
"missing": true,
"problems": [
"missing: @sinclair/typebox@^0.23.3, required by service-a@1.0.0"
]
},
"@types/convict": {
"missing": true,
"problems": [
"missing: @types/convict@^6.1.1, required by service-a@1.0.0"
]
},
"avsc": {
"missing": true,
"problems": [
"missing: avsc@^5.7.3, required by service-a@1.0.0"
]
},
"convict": {
"missing": true,
"problems": [
"missing: convict@^6.2.1, required by service-a@1.0.0"
]
},
"http-errors": {
"missing": true,
"problems": [
"missing: http-errors@^2.0.0, required by service-a@1.0.0"
]
},
"kafkajs": {
"missing": true,
"problems": [
"missing: kafkajs@^1.15.0, required by service-a@1.0.0"
]
},
"reflect-metadata": {
"missing": true,
"problems": [
"missing: reflect-metadata@^0.1.13, required by service-a@1.0.0"
]
},
"uuid": {
"missing": true,
"problems": [
"missing: uuid@^8.3.2, required by service-a@1.0.0"
]
},
"zod": {
"missing": true,
"problems": [
"missing: zod@^3.11.6, required by service-a@1.0.0"
]
}
}
}
npm ERR! code ELSPROBLEMS
npm ERR! missing: @faker-js/faker@^6.0.0-alpha.7, required by service-a@1.0.0
npm ERR! missing: @kafkajs/confluent-schema-registry@^3.2.0, required by service-a@1.0.0
npm ERR! missing: @middy/core@^2.5.2, required by service-a@1.0.0
npm ERR! missing: @middy/http-json-body-parser@^2.5.7, required by service-a@1.0.0
npm ERR! missing: @middy/http-response-serializer@^2.5.7, required by service-a@1.0.0
npm ERR! missing: @middy/ssm@^2.5.2, required by service-a@1.0.0
npm ERR! missing: @*****/avro-kafkajs@^0.8.1, required by service-a@1.0.0
npm ERR! missing: @*****/castle@^0.8.1, required by service-a@1.0.0
npm ERR! missing: @sinclair/typebox@^0.23.3, required by service-a@1.0.0
npm ERR! missing: @types/convict@^6.1.1, required by service-a@1.0.0
npm ERR! missing: avsc@^5.7.3, required by service-a@1.0.0
npm ERR! missing: convict@^6.2.1, required by service-a@1.0.0
npm ERR! missing: http-errors@^2.0.0, required by service-a@1.0.0
npm ERR! missing: kafkajs@^1.15.0, required by service-a@1.0.0
npm ERR! missing: reflect-metadata@^0.1.13, required by service-a@1.0.0
npm ERR! missing: uuid@^8.3.2, required by service-a@1.0.0
npm ERR! missing: zod@^3.11.6, required by service-a@1.0.0
{
"error": {
"code": "ELSPROBLEMS",
"summary": "missing: @faker-js/faker@^6.0.0-alpha.7, required by service-a@1.0.0\nmissing: @kafkajs/confluent-schema-registry@^3.2.0, required by service-a@1.0.0\nmissing: @middy/core@^2.5.2, required by service-a@1.0.0\nmissing: @middy/http-json-body-parser@^2.5.7, required by service-a@1.0.0\nmissing: @middy/http-response-serializer@^2.5.7, required by service-a@1.0.0\nmissing: @middy/ssm@^2.5.2, required by service-a@1.0.0\nmissing: @*****/avro-kafkajs@^0.8.1, required by service-a@1.0.0\nmissing: @*****/castle@^0.8.1, required by service-a@1.0.0\nmissing: @sinclair/typebox@^0.23.3, required by service-a@1.0.0\nmissing: @types/convict@^6.1.1, required by service-a@1.0.0\nmissing: avsc@^5.7.3, required by service-a@1.0.0\nmissing: convict@^6.2.1, required by service-a@1.0.0\nmissing: http-errors@^2.0.0, required by service-a@1.0.0\nmissing: kafkajs@^1.15.0, required by service-a@1.0.0\nmissing: reflect-metadata@^0.1.13, required by service-a@1.0.0\nmissing: uuid@^8.3.2, required by service-a@1.0.0\nmissing: zod@^3.11.6, required by service-a@1.0.0",
"detail": ""
}
}
npm ERR! A complete log of this run can be found in:
npm ERR! /Users/mishabruml/.npm/_logs/2022-03-03T14_33_14_772Z-debug.log
Error:
Error: npm ls -json -prod -long -all failed with code 1
at ChildProcess.<anonymous> (/Users/mishabruml/*****/**********/node_modules/serverless-esbuild/dist/utils.js:54:24)
at ChildProcess.emit (node:events:390:28)
at ChildProcess.emit (node:domain:475:12)
at maybeClose (node:internal/child_process:1064:16)
at Process.ChildProcess._handle.onexit (node:internal/child_process:301:5)This is likely related to https://github.com/serverless-heaven/serverless-webpack/pull/782
mishabruml
commented
2 years ago thanks a lot for this
hmm. I tried adding
{ npmError: 'code ELSPROBLEMS', log: false } to the ignoredNpmErrors here https://github.com/floydspace/serverless-esbuild/blob/master/src/packagers/npm.ts#L124-L128
but received the same error regardless. this seems to be due to the stringmatching in the error handling function throwing a false negative for the block of JSON that is output to stderr. I am continuing some local testing and may submit a PR 👍
JSON emitted to stderr that needs ignoring:
{
"error": {
"code": "ELSPROBLEMS",
"summary": "missing: @faker-js/faker@^6.0.0-alpha.7, required by service-a@1.0.0\nmissing: @kafkajs/confluent-schema-registry@^3.2.0, required by service-a@1.0.0\nmissing: @middy/core@^2.5.2, required by service-a@1.0.0\nmissing: @middy/http-json-body-parser@^2.5.7, required by service-a@1.0.0\nmissing: @middy/http-response-serializer@^2.5.7, required by service-a@1.0.0\nmissing: @middy/ssm@^2.5.2, required by service-a@1.0.0\nmissing: @*****/avro-kafkajs@^0.8.1, required by service-a@1.0.0\nmissing: @*****/castle@^0.8.1, required by service-a@1.0.0\nmissing: @sinclair/typebox@^0.23.3, required by service-a@1.0.0\nmissing: @types/convict@^6.1.1, required by service-a@1.0.0\nmissing: avsc@^5.7.3, required by service-a@1.0.0\nmissing: convict@^6.2.1, required by service-a@1.0.0\nmissing: http-errors@^2.0.0, required by service-a@1.0.0\nmissing: kafkajs@^1.15.0, required by service-a@1.0.0\nmissing: reflect-metadata@^0.1.13, required by service-a@1.0.0\nmissing: uuid@^8.3.2, required by service-a@1.0.0\nmissing: zod@^3.11.6, required by service-a@1.0.0",
"detail": ""
}
}
github-actions[bot]
commented
2 years ago :tada: This issue has been resolved in version 1.26.1 :tada:
The release is available on:
Your semantic-release bot :package::rocket:
This error started spontaneously with no code or environment changes that i can work out. I have pulled down my master branch, which is working in circleci just fine, but locally I recieve the above error. I am not sure what actually causes this error as the stack trace is unclear. Is it to do with resolving dependencies across monorepo structure? For information, my project looks like:
Where the
packagesare managed by npm workspaces. It isservice-athat is the serverless project.service-a package.json
service-a/esbuild-plugins.js
service-a/serverless.yml
npm ls -json -prod -all --depth 0inservice-adir yields: