various npm modules being ignored

[jamesdixon]

This is a (Bug Report / Feature Proposal)

Maybe a bug, maybe a misunderstanding

Description

For bug reports:

• What went wrong?

Many of my modules defined under dependencies aren't being included in the build. For example, none of the lodash.* dependencies are included.

• What did you expect should have happened?

All modules should be included in the build

• What was the config you used?

// package.json
{
  "scripts": {
    "test": "ava -v -s",
    "coverage": "nyc npm test"
  },
  "devDependencies": {
    "ava": "^0.25.0",
    "aws-sdk": "^2.269.1",
    "aws-sdk-mock": "^4.0.0",
    "dotenv": "^6.0.0",
    "faker": "^4.1.0",
    "knex-cleaner": "^1.1.4",
    "lambda-tester": "^3.4.1",
    "lodash.times": "^4.3.2",
    "proxyquire": "^2.0.1",
    "serverless-offline": "^3.25.5",
    "serverless-plugin-deploy-environment": "^1.1.0",
    "serverless-webpack": "^5.2.0",
    "sinon": "^6.1.0",
    "standard": "^11.0.1",
    "webpack": "^4.15.1",
    "webpack-node-externals": "^1.7.2"
  },
  "dependencies": {
    "@sendgrid/client": "^6.3.0",
    "@sendgrid/mail": "^6.3.1",
    "cache-pug-templates": "^0.0.7",
    "email-templates": "^4.0.1",
    "joi": "^13.4.0",
    "knex": "^0.15.0",
    "lodash.camelcase": "^4.3.0",
    "lodash.chunk": "^4.2.0",
    "lodash.get": "^4.4.2",
    "lodash.groupby": "^4.6.0",
    "lodash.orderby": "^4.6.0",
    "lodash.pickby": "^4.6.0",
    "lodash.startcase": "^4.4.0",
    "lodash.uniqby": "^4.7.0",
    "moment": "^2.22.2",
    "pg": "^7.4.3",
    "pug": "^2.0.3",
    "redis": "^2.8.0",
    "rollbar": "^2.4.2",
    "rrule": "^2.2.9"
  },
  "ava": {
    "require": [
      "./dotenv",
      "dotenv/config"
    ]
  }
}

// serverless.ymlservice: email-service

provider:
  name: aws
  runtime: nodejs8.10
  region: us-east-2

package:
  individually: true

functions:
  sendEmail:
    handler: lib/sendEmail/index.handler
    events:
      - sqs: 
          arn: ${self:deployVariables.SEND_EMAIL_SQS_ARN}
          enabled: true
          batchSize: 1
  createEmail:
    handler: lib/createEmail/index.handler
    events:
      - sqs: 
          arn: ${self:deployVariables.CREATE_EMAIL_SQS_ARN}
          enabled: true
          batchSize: 1
  createCampaign:
    handler: lib/createCampaign/index.handler
    events:
      - sqs: 
          arn: ${self:deployVariables.CREATE_CAMPAIGN_SQS_ARN}
          enabled: true
          batchSize: 1
  sendGridEvent:
    handler: lib/sendGridEvent/index.handler
    events:
      - http:
          path: /sendGridEvent
          method: post

custom:
  deploy:
    environments: ${file(config/env.yml)}
    variables: ${file(config/variables.yml)}
  webpack:
    includeModules: true
    packager: 'yarn'

plugins:
  - serverless-offline
  - serverless-plugin-deploy-environment
  - serverless-webpack

// webpack.config.js
const nodeExternals = require('webpack-node-externals')
const slsw = require('serverless-webpack')
const Webpack = require('webpack')

module.exports = {
  entry: slsw.lib.entries,
  mode: slsw.lib.webpack.isLocal ? 'development' : 'production',
  target: 'node',
  devtool: 'source-map',
  optimization: {
    minimize: false
  },
  externals: [nodeExternals()],
  plugins: [
    // https://github.com/tj/consolidate.js/issues/295
    new Webpack.IgnorePlugin(/(atpl|bracket|dot|doT.js|dust|dustjs-linkedin|eco|ect|ejs|haml|haml-coffee|hamlet|hiredis|handlebars|hogan|htmling|jazz|jqtpl|just|liquor|lodash|marko|mote|mustache|nunjucks|plates|QEJS|ractive|react|slm|swig|swig|teacup|templayed|twig|liquid|toffee|underscore|vash|walrus|whiskers)/),
    // https://github.com/tgriesser/knex/issues/1446
    new Webpack.NormalModuleReplacementPlugin(/\.\.\/migrate/, '../util/noop.js'),
    new Webpack.NormalModuleReplacementPlugin(/\.\.\/seed/, '../util/noop.js'),
    new Webpack.IgnorePlugin(/mariasql/, /\/knex\//),
    new Webpack.IgnorePlugin(/mssql/, /\/knex\//),
    new Webpack.IgnorePlugin(/mysql/, /\/knex\//),
    new Webpack.IgnorePlugin(/mysql2/, /\/knex\//),
    new Webpack.IgnorePlugin(/oracle/, /\/knex\//),
    new Webpack.IgnorePlugin(/oracledb/, /\/knex\//),
    new Webpack.IgnorePlugin(/pg-query-stream/, /\/knex\//),
    new Webpack.IgnorePlugin(/sqlite3/, /\/knex\//),
    new Webpack.IgnorePlugin(/strong-oracle/, /\/knex\//),
    new Webpack.IgnorePlugin(/pg-native/, /\/pg\//)
  ]
}

• What stacktrace or error message from your provider did you see? I'm not seeing any errors, but when running sls deploy --stage beta, I see the following:

Serverless: Package lock found - Using locked versions Serverless: Packing external modules: @sendgrid/mail@^6.3.1, joi@^13.4.0, rollbar@^2.4.2, redis@^2.8.0, @sendgrid/client@^6.3.0, knex@^0.15.0 Serverless: Packaging service...

You can see this is just a small subset of my dependencies.

For feature proposals:

• What is the use case that should be solved. The more detail you describe this in the easier it is to understand for us.
• If there is additional config how would it look

Similar or dependent issue(s):

• 408

Additional Data

• Serverless-Webpack Version you're using: 5.2.0
• Webpack version you're using: 4.15
• Serverless Framework Version you're using: 1.28.0
• Operating System: OSX High Sierra
• Stack Trace (if available):

[HyperBrain]

Hi @jamesdixon , the plugin only packages dependencies that are used by the function code you deploy. These dependencies are detected by webpack and reported to the plugin. As a consequence the plugin only bundles dependencies that are reported by Webpack. So you get the best dependency optimization possible for each single function. Maybe webpack bundled some dependencies into the code - or it is not used in the code paths used for the functions. If the deployed code works properly this is a sign, that Webpack did it's job very well 😄 .

BTW: I saw a small glitch in your serverless.yml. You should correct the plugin order, because it matters when initializing the plugins:

plugins:
  - serverless-webpack
  - serverless-offline
  - serverless-plugin-deploy-environment

It is very important that sls-webpack is first, because offline should run the compiled code.

[jamesdixon]

Hi @HyperBrain - first, thank you for taking the time to reply! It's much appreciated.

I probably should have been more clear in my report. Various npm modules are not being included, which in turn is causing my functions to fail. I mentioned the previous issue with lodash.* functions not being included. Rather than import all of lodash, I've just been grabbing individual packages, such as lodash.chunk. When I run sls package, I see that the lodash library is included in full, but not the individual functions. I'm guessing one of my dependencies includes lodash and webpack sees that lodash.chunk is part of lodash and decides not to include it? If that is the case, my code calls require('lodash.chunk') which is why the error is being thrown.

I'd love to hear your thoughts.

Also, thanks for the catch on the issue with my serverless.yml file!

[jamesdixon]

I just opened up my handler that's output after packaging and noticed the following lines:

const AWS = __webpack_require__(7)
const Joi = __webpack_require__(6)
const KnexConfig = __webpack_require__(5)
const Knex = __webpack_require__(4)(KnexConfig["production"])
const SendGrid = __webpack_require__(3)
const UUID = __webpack_require__(2)
const { configureRollbar } = __webpack_require__(1)

const chunk = __webpack_require__(!(function webpackMissingModule() { var e = new Error("Cannot find module 'lodash.chunk'"); e.code = 'MODULE_NOT_FOUND'; throw e; }()))

It looks like webpack can't find lodash.chunk? So confused...lol

[HyperBrain]

Indeed - it is really a missing dependency. Maybe it's a bug in the plugin with dependency names containing a dot? I never used such names 😃 We should test, if other dependencies of the format xxx.yyy are also dropped silently.

[jamesdixon]

Actually seeing this with dependencies containing -. Going to see if there are other cases...

[jamesdixon]

Here's an example from another function:

const AWS = __webpack_require__(7)
const Email = __webpack_require__(!(function webpackMissingModule() { var e = new Error("Cannot find module 'email-templates'"); e.code = 'MODULE_NOT_FOUND'; throw e; }()))
const Joi = __webpack_require__(6)
const Path = __webpack_require__(5)
const Redis = __webpack_require__(4)
const cachePugTemplates = __webpack_require__(!(function webpackMissingModule() { var e = new Error("Cannot find module 'cache-pug-templates'"); e.code = 'MODULE_NOT_FOUND'; throw e; }()))
const { configureRollbar } = __webpack_require__(3)
const { getEnvPrefix } = __webpack_require__(1)

[HyperBrain]

Hmmm... It is strange that the missing modules are already "compiled" into the code. This hints to an issue with the webpack configuration, because otherwise they would only be missing in the deployed package, but not the compiled code, wouldn't they?

[jamesdixon]

I suppose so. My webpack knowledge is very very limited 😄

[jamesdixon]

I don't have any of the output or babel related code in my config. Is it something that's required?

[HyperBrain]

Maybe yes - I never used webpack without babel (and the babel loader). Maybe you can check the webpack4 example in the examples folder (of the plugin repo) and use that to setup a second variant of your project to see if that works.

[jamesdixon]

@HyperBrain finally got this working!

It turns out the issue was actually with the Webpack.IgnorePlugin calls in my webpack config. I had originally added those due to errors I was getting when building. For some reason, those errors no longer exist now that I've removed the plugins.

I appreciate your help with this. Sorry for the runaround!

[mskutin]

Got into this with local packages that obviously have dot symbol

TypeError

```log TypeError: someFunction is not a function at /Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/src/index.js:24:27 at Layer.handle [as handle_request] (/Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/express/lib/router/layer.js:95:1) at next (/Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/express/lib/router/route.js:137:1) at Route.dispatch (/Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/express/lib/router/route.js:112:1) at Layer.handle [as handle_request] (/Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/express/lib/router/layer.js:95:1) at /Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/express/lib/router/index.js:281:1 at Function.process_params (/Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/express/lib/router/index.js:335:1) at next (/Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/express/lib/router/index.js:275:1) at /Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/body-parser/lib/read.js:130:1 at invokeCallback (/Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/raw-body/index.js:224:1) at done (/Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/raw-body/index.js:213:1) at IncomingMessage.onEnd (/Users/ms/repo/awesome-mono/services/awesome-api/dist/service/src/webpack:/Users/ms/repo/awesome-mono/node_modules/raw-body/index.js:273:1) at IncomingMessage.emit (events.js:182:13) at IncomingMessage.EventEmitter.emit (domain.js:460:23) at endReadableNT (_stream_readable.js:1081:12) at process._tickCallback (internal/process/next_tick.js:63:19) ```

bundle.js

```js var someFunction = __webpack_require__(!(function webpackMissingModule() { var e = new Error("Cannot find module 'some'"); e.code = 'MODULE_NOT_FOUND'; throw e; }())); ```

index.js

```js const serverless = require('serverless-http') const bodyParser = require('body-parser') const express = require('express') const someFunction = require('./some') # <- "some.js" const app = express() app.post('/', function (req, res) { console.log('Entry request body: ', req.body) res.send({ status: 'ok', message: someFunction() }) } ```

some.js

```js const someFunction = () => { return console.log("Works!") } module.exports.someFunction = someFunction; ```

.babelrc

```js { "plugins": [ ["transform-runtime", { "polyfill": false, "regenerator": true }] ], "presets": [ ["env", { "modules": false, "targets": { "node": "8.10" }}], ["es2015", { "modules": false }], ["stage-2"] ] } ```

webpack.config.js

```js const path = require('path'); const slsw = require('serverless-webpack'); const CopyWebpackPlugin = require('copy-webpack-plugin') const nodeExternals = require('webpack-node-externals'); module.exports = { target: 'node', entry: slsw.lib.entries, plugins: [ new CopyWebpackPlugin([{ from: 'assets', to: 'assets' }]) ], module: { rules: [{ exclude: /node_modules/, test: /\.js$/, loader: 'babel-loader' }] }, stats: { colors: true }, devtool: 'source-map', mode: slsw.lib.webpack.isLocal ? "development" : "production", optimization: { minimize: false }, output: { libraryTarget: 'commonjs', path: path.join(__dirname, 'dist'), filename: '[name].js', }, externals: [nodeExternals({ whitelist: ['express', /^lodash/] })] }; ```

serverless.yml

```yaml # serverless.yml frameworkVersion: ">=1.0.0 <2.0.0" service: awesome-api custom: webpack: includeModules: true packager: 'yarn' forceExclude: - aws-sdk forceInclude: - some functions: app: handler: src/index.handler events: - http: ANY / - http: 'ANY {proxy+}' plugins: - serverless-webpack - serverless-offline - serverless-pseudo-parameters package: individually: true ```