Performance degradation webpack 5.3.2 + this plugin 5.2.2 on recompile

[woophi]

Upgrading to latest circular-dependency-plugin causes performance degradation on recompile in dev mode.

with plugin ~12s

without plugin ~2.5s

current webpack configuration:

// / <binding ProjectOpened='Watch - Development' />
import { cpus } from 'os';
import * as path from 'path';
import * as webpack from 'webpack';
import * as webpackDevServer from 'webpack-dev-server';
import { BundleAnalyzerPlugin } from 'webpack-bundle-analyzer';
import * as MiniCssExtractPlugin from 'mini-css-extract-plugin';
import TsCheckerPlugin = require('fork-ts-checker-webpack-plugin');
import TerserJSPlugin = require('terser-webpack-plugin');
import OptimizeCSSAssetsPlugin = require('optimize-css-assets-webpack-plugin');
import TsCheckerNotifierWebpackPlugin = require('fork-ts-checker-notifier-webpack-plugin');
import { writeFileSync } from 'fs';
const WebpackBuildNotifierPlugin = require('webpack-build-notifier');
const CircularDependencyPlugin = require('circular-dependency-plugin');
const threadLoader = require('thread-loader');

const CPU_COUNT = cpus().length;

const processCwd = process.cwd();

const circularDependencies = {
  count: 0,
  countInMluviiUiPackage: 0,
  dependencyCircles: {} as { [key: string]: string[] | string[][] }
};

interface Configuration extends webpack.Configuration {
  devServer?: webpackDevServer.Configuration;
}

const isProduction = process.argv[process.argv.indexOf('--mode') + 1] === 'production';
const analyzeBuild = process.argv.some(a => a === '--analyze');
const profilingBuild = isProduction && process.argv.some(a => a === '--profiling');

const profilingAliases = profilingBuild
  ? {
      'react-dom$': 'react-dom/profiling',
      'scheduler/tracing': 'scheduler/tracing-profiling'
    }
  : {};

const enforceCRLF = (text: string = '') => text.replace(/\r\n/gm, '\n').replace(/\n/gm, '\r\n');

const tsPool = {
  workers: CPU_COUNT - 2,
  // number of jobs a worker processes in parallel
  // defaults to 20
  workerParallelJobs: 10,
  // additional node.js arguments
  workerNodeArgs: ['--max-old-space-size=1024'],
  // Allow to respawn a dead worker pool
  // respawning slows down the entire compilation
  // and should be set to false for development
  poolRespawn: isProduction,
  // timeout for killing the worker processes when idle
  // defaults to 500 (ms)
  // can be set to Infinity for watching builds to keep workers alive
  poolTimeout: isProduction ? 500 : Infinity,
  // number of jobs the pool distributes to the workers
  // defaults to 200
  // decrease of less efficient but more fair distribution
  poolParallelJobs: 300
  // can be used to create different pools with elsewise identical options
  // name: "ts-pool"
};

threadLoader.warmup(tsPool, ['ts-loader']);

const devServer: webpackDevServer.Configuration = {
  port: 3000,
  https: true,
  disableHostCheck: true,
  headers: { 'Access-Control-Allow-Origin': '*' },
  contentBase: path.resolve(__dirname, '../../../dist')
};

const config: Configuration = {
  devServer,
  entry: {
    'app.operator': './src/ui/application/entrypoint.tsx',
    'app.guest': './src/ui/guest/app.tsx',
    'app.qrupload': './src/qrupload/qrupload.ts',
    'lib.edge': './src/edge.js',
    'lib.firefox': './src/firefox.js',
    'lib.chrome': './src/chrome.js',
    'lib.ie10': './src/ie10.js',
    'lib.ie11': './src/ie11.js',
    'lib.otherbrowser': './src/otherbrowser.js',
    'lib.safari': './src/safari.js',
    'lib.pdfworker': 'pdfjs-dist/build/pdf.worker'
  },
  target: ['web', 'es5'],
  output: {
    libraryTarget: 'this',
    path: path.resolve(__dirname, '../../../dist'),
    filename: 'js/[name].js',
    publicPath: '/appcontent/',
    chunkFilename: 'js/[name].js?[contenthash]',
    pathinfo: !isProduction
  },
  resolve: {
    alias: {
      src: path.resolve(__dirname, '../src/'),
      core: path.resolve(__dirname, '../src/core'),
      ui: path.resolve(__dirname, '../src/ui'),
      tslib: path.resolve(__dirname, '../../../node_modules/tslib'),
      '@mluvii/ui': path.resolve(__dirname, '../../../node_modules/@mluvii/ui/dist'),
      ...profilingAliases
    },
    extensions: ['.js', '.jsx', '.ts', '.tsx']
  },
  module: {
    rules: [
      {
        test: require.resolve('janus-gateway'),
        loader: 'exports-loader',
        options: {
          exports: 'Janus'
        }
      },
      {
        test: /src[\\/]ui[\\/].*\.url\.svg$/,
        loader: 'file-loader',
        options: {
          name: 'svg/[name].[ext]?[md5:hash:base62]'
        }
      },
      {
        test: /src[\\/]ui[\\/].*(?<!\.url)\.svg$/,
        exclude: /node_modules/,
        use: [
          {
            loader: 'ts-loader',
            options: {
              happyPackMode: true,
              compilerOptions: {
                jsx: 'preserve',
                allowJs: true,
                checkJs: false
              }
            }
          },
          'react-svg-loader'
        ]
      },
      {
        test: /\.json$/,
        loader: 'json-loader',
        exclude: /node_modules/,
        type: 'javascript/auto'
      },
      {
        test: [/\.tsx?$/, /\.js$/],
        exclude: /node_modules/,
        use: [
          {
            loader: 'thread-loader',
            options: tsPool
          },
          {
            loader: 'ts-loader',
            options: {
              happyPackMode: true
            }
          }
        ]
      },
      {
        // TODO: review webpack bundling rules
        test: /\.jsx?$/,
        exclude: /(node_modules)(?![/|\\](swiper|dom7|ssr\-window))/,
        use: [
          {
            loader: 'thread-loader',
            options: tsPool
          },
          {
            loader: 'ts-loader',
            options: {
              happyPackMode: true,
              compilerOptions: {
                allowJs: true,
                checkJs: false
              }
            }
          }
        ]
      },
      {
        test: /\.css$/,
        use: [MiniCssExtractPlugin.loader, 'css-loader']
      },
      {
        test: /\.(png|gif|jpg|jpeg)$/,
        loader: 'file-loader',
        options: {
          name: 'img/[name].[ext]?[md5:hash:base62]'
        }
      },
      {
        test: /fonts[\\/].*\.(woff|woff2|eot|ttf|svg)$/,
        loader: 'file-loader',
        options: {
          name: 'fonts/[name].[ext]?[md5:hash:base62]'
        }
      },
      {
        test: /\.mp3$/,
        loader: 'file-loader',
        options: {
          name: 'sounds/[name].[ext]?[md5:hash:base62]'
        }
      },
      {
        test: /\.html$/i,
        loader: 'html-loader',
        options: {
          minimize: true
        }
      },
      // https://github.com/webpack/webpack/issues/11467
      {
        test: /\.m?js/,
        resolve: {
          fullySpecified: false
        }
      }
    ],
    noParse: [/pdfjs-dist/]
  },
  externals: {
    './chrome/chrome_shim': 'webrtc_adapter_chrome_shim',
    './edge/edge_shim': 'webrtc_adapter_edge_shim',
    './firefox/firefox_shim': 'webrtc_adapter_firefox_shim',
    './safari/safari_shim': 'webrtc_adapter_safari_shim'
  },
  optimization: {
    minimizer: isProduction
      ? [
          new TerserJSPlugin({
            parallel: true,
            terserOptions: {
              sourceMap: true
            }
          }),
          new OptimizeCSSAssetsPlugin({})
        ]
      : undefined
  },
  mode: isProduction ? 'production' : 'development',
  devtool: isProduction ? false : 'eval-source-map',
  plugins: [
    new webpack.DefinePlugin({
      PRODUCTION: isProduction
    }),
    new webpack.ProvidePlugin({ adapter: 'webrtc-adapter' }),
    new webpack.DllReferencePlugin({
      context: path.join(__dirname, '..'),
      manifest: require('../../../dist/lib.base-manifest.json')
    }),
    isProduction &&
      new webpack.SourceMapDevToolPlugin({
        filename: '../source-map/[name].js.map',
        moduleFilenameTemplate: '[resource-path]',
        append: '\n//# sourceMappingURL=[name].js.map',
        exclude: /lib\..*.*/
      }),
    analyzeBuild &&
      new BundleAnalyzerPlugin({
        analyzerMode: 'static',
        generateStatsFile: true,
        openAnalyzer: false
      }),
    new CircularDependencyPlugin({
      exclude: /node_modules/,
      failOnError: false,
      allowAsyncCycles: false,
      cwd: processCwd,
      onStart() {
        circularDependencies.count = 0;
        circularDependencies.countInMluviiUiPackage = 0;
        circularDependencies.dependencyCircles = {};
      },
      onDetected({ compilation, paths }) {
        circularDependencies.count++;
        const [source, ...deps] = paths.map(pa => pa.replace(/\\/g, '/'));

        if (paths.some(p => /[\\/]ui[\\/]dist/.test(p))) {
          circularDependencies.countInMluviiUiPackage++;
          compilation.errors.push(
            new Error(
              `[CircularDependency] in ${path.join(processCwd, paths[0])}:\n ${deps.join(' -> ')}`
            )
          );
        }

        if (circularDependencies.dependencyCircles[source]) {
          circularDependencies.dependencyCircles[source] = [
            circularDependencies.dependencyCircles[source],
            deps
          ];
        } else {
          circularDependencies.dependencyCircles[source] = deps;
        }
      },
      onEnd({ compilation }: { compilation: any }) {
        if (circularDependencies.count > 0) {
          compilation.warnings.push(
            new Error(`Detected ${circularDependencies.count} cycles in dependency tree.`)
          );
        }
        if (circularDependencies.countInMluviiUiPackage > 0) {
          compilation.errors.push(
            new Error(
              `Detected ${circularDependencies.countInMluviiUiPackage} cycles in dependency tree of @mluvii/ui - please refactor code to eliminate them.`
            )
          );
        }
        if (!isProduction) {
          const content = JSON.stringify(circularDependencies, null, 2);
          writeFileSync(path.join(__dirname, '../../../circularDeps.json'), enforceCRLF(content), {
            encoding: 'utf-8'
          });
        }
      }
    }),
    new MiniCssExtractPlugin({
      filename: 'css/[name].css?[contenthash]'
    }),
    new webpack.IgnorePlugin({
      resourceRegExp: /^\.\/locale$/,
      contextRegExp: /moment$/
    }),
    !isProduction &&
      new WebpackBuildNotifierPlugin({
        title: 'Application built',
        suppressSuccess: true
      }),
    !isProduction &&
      new TsCheckerNotifierWebpackPlugin({
        title: 'Application',
        skipSuccessful: true
      }),
    new TsCheckerPlugin({
      async: !isProduction,
      typescript: {
        memoryLimit: 4096,
        configFile: path.resolve(__dirname, '../tsconfig.json'),
        diagnosticOptions: {
          declaration: false,
          global: false,
          syntactic: true,
          semantic: true
        },
        configOverwrite: {
          compilerOptions: {
            skipLibCheck: false,
            sourceMap: false,
            inlineSourceMap: false,
            declarationMap: false
          },
          include: [
            path.resolve(__dirname, '../src'),
            path.resolve(__dirname, '../node_modules/@mluvii/ui/dist')
          ]
        }
      }
    })
  ].filter(Boolean)
};

export default config;

in comparison with old versions of webpack and this plugin, recompile tooks ~6s "webpack": "^4.44.2" and "circular-dependency-plugin": "^5.2.0",

package.json

{
  "scripts": {
    "clean": "rimraf dist cache",
    "build:dll": "webpack --color --bail --config=./webpack/webpack.dll.config.js",
    "build:app": "webpack --color --bail --config=./webpack/webpack.config.ts",
    "build": "yarn build:dll --mode production && yarn build:app --mode production",
    "build:analyze": "yarn build:dll --mode production --analyze && yarn build:app --mode production --analyze",
    "build:profiling": "yarn build --profiling",
    "watch": "yarn clean && yarn && yarn build:dll --mode development && yarn build:app --mode development --watch",
    "start": "yarn && yarn build:dll --mode development && webpack-dev-server --mode development --config=./webpack/webpack.config.ts --no-inline --output-public-path https://localhost:3000/appcontent/",
    "measure": "webpack-dev-server --mode development --config=./webpack/speed-measure.ts --no-inline --output-public-path https://localhost:3000/appcontent/"
  },
  "devDependencies": {
    "@types/glob": "^5.0.35",
    "@types/moment-duration-format": "^2.2.2",
    "@types/node": "^8.9.1",
    "@types/react": "16.8.6",
    "@types/react-custom-scrollbars": "^4.0.5",
    "@types/react-dom": "^16.8.6",
    "@types/react-router": "^5.1.2",
    "@types/react-router-dom": "^5.1.0",
    "@types/redux-thunk": "^2.1.0",
    "@types/webpack-dev-server": "^3.11.1",
    "css-loader": "^5.0.0",
    "exports-loader": "^1.1.1",
    "file-loader": "^6.2.0",
    "fork-ts-checker-notifier-webpack-plugin": "^3.0.0",
    "fs": "^0.0.1-security",
    "glob": "^7.1.2",
    "html-loader": "^1.3.2",
    "mini-css-extract-plugin": "^1.2.1",
    "optimize-css-assets-webpack-plugin": "^5.0.4",
    "path": "^0.12.7",
    "prettier": "^2.1.1",
    "rimraf": "^2.6.2",
    "source-map-loader": "^1.1.2",
    "speed-measure-webpack-plugin": "^1.3.1",
    "style-loader": "^2.0.0",
    "terser-webpack-plugin": "^5.0.3",
    "ts-loader": "^8.0.7",
    "webpack-build-notifier": "^2.1.0",
    "webpack-bundle-analyzer": "^3.9.0",
    "webpack-cli": "^4.1.0",
    "webpack-dev-server": "^3.11.0",
    "worker-loader": "^3.0.5",
    "xlsx": "^0.14.0"
  },
  "dependencies": {
    "@aspnet/signalr": "1.1.4",
    "@sentry/browser": "^5.27.2",
    "@sentry/webpack-plugin": "^1.13.0",
    "@types/chrome": "^0.0.46",
    "@types/cytoscape": "^3.14.0",
    "@types/faker": "^4.1.9",
    "@types/hammerjs": "^2.0.35",
    "@types/i18next": "^13.0.0",
    "@types/markdown-it": "^10.0.2",
    "@types/pdfjs-dist": "^2.1.5",
    "@types/quill": "^1.3.10",
    "@types/ramda": "types/npm-ramda#dist",
    "@types/react-color": "^2.13.4",
    "@types/react-grid-layout": "^0.17.2",
    "@types/react-plotly.js": "^2.2.4",
    "@types/react-redux": "^7.1.9",
    "@types/react-select": "^2.0.2",
    "@types/react-sortable-hoc": "^0.6.1",
    "@types/react-textarea-autosize": "^4.3.3",
    "@types/react-transition-group": "^2.0.11",
    "@types/react-virtualized": "^9.8.0",
    "@types/react-youtube": "^7.4.1",
    "@types/redux-form": "^8.1.9",
    "@types/redux-thunk": "^2.1.0",
    "@types/rx-dom": "^7.0.0",
    "@types/service_worker_api": "^0.0.9",
    "@types/swiper": "^4.4.2",
    "adaptivecards": "^1.2.0",
    "autoprefixer": "^7.2.5",
    "axios": "^0.19.2",
    "botframework-directlinejs": "^0.13.1",
    "botframework-webchat": "^4.10.1",
    "botframework-webchat-component": "^4.10.1",
    "circular-dependency-plugin": "^5.2.2",
    "comlink": "^4.3.0",
    "connected-react-router": "^6.5.2",
    "core-js": "^3.4.7",
    "css-element-queries": "^1.0.2",
    "cytoscape": "^3.14.1",
    "cytoscape-canvas": "^3.0.1",
    "cytoscape-node-html-label": "^1.2.0",
    "cytoscape-panzoom": "^2.5.3",
    "eventbusjs": "^0.2.0",
    "faker": "^4.1.0",
    "fela": "^11.2.0",
    "fela-plugin-embedded": "^11.2.0",
    "fela-preset-web": "^11.2.0",
    "final-form": "^4.17.0",
    "final-form-arrays": "^3.0.1",
    "flubber": "^0.4.2",
    "font-awesome": "^4.7.0",
    "fork-ts-checker-webpack-plugin": "^5.2.1",
    "fp-ts": "^2.6.5",
    "gsap": "^3.1.1",
    "hammerjs": "^2.0.8",
    "history": "^4.10.1",
    "i18next": "^19.8.3",
    "immer": "^6.0.9",
    "intl": "^1.2.5",
    "io-ts": "^2.2.4",
    "janus-gateway": "git://github.com/meetecho/janus-gateway.git#v0.9.5",
    "json-loader": "^0.5.7",
    "konva": "^4.0.4",
    "markdown-it": "^12.0.2",
    "moment": "^2.18.1",
    "moment-duration-format": "^2.3.2",
    "moment-timezone": "^0.5.11",
    "papaparse": "^5.2.0",
    "paper": "^0.11.4",
    "pdfjs-dist": "^2.5.207",
    "plotly.js": "^1.54.0",
    "postcss-loader": "^2.1.0",
    "precss": "^3.1.0",
    "ramda": "^0.26.1",
    "react": "^16.12.0",
    "react-color": "^2.14.1",
    "react-copy-to-clipboard": "^5.0.0",
    "react-custom-scrollbars": "4.1.1",
    "react-dom": "^16.12.0",
    "react-draggable": "^3.3.0",
    "react-dropzone": "^7.0.1",
    "react-fela": "^11.2.0",
    "react-final-form": "^6.3.0",
    "react-final-form-arrays": "^3.1.1",
    "react-flip-move": "^3.0.4",
    "react-grid-layout": "^1.0.0",
    "react-hint": "^3.0.0",
    "react-i18next": "^11.7.3",
    "react-konva": "^16.9.0-0",
    "react-plotly.js": "^2.4.0",
    "react-qr-svg": "^2.3.0",
    "react-quill": "^1.3.3",
    "react-redux": "^7.1.1",
    "react-router": "^5.1.2",
    "react-router-dom": "^5.1.2",
    "react-select": "^2.0.0",
    "react-sortable-hoc": "^0.6.8",
    "react-spring": "^8.0.27",
    "react-svg-loader": "^3.0.3",
    "react-textarea-autosize": "^5.1.0",
    "react-transition-group": "^2.5.2",
    "react-virtualized": "^9.8.0",
    "react-youtube": "^7.4.0",
    "redux": "^4.0.4",
    "redux-devtools-extension": "^2.13.8",
    "redux-form": "^8.2.6",
    "redux-observable": "^1.2.0",
    "redux-thunk": "^2.3.0",
    "regenerator-runtime": "^0.13.3",
    "reselect": "^4.0.0",
    "rx-dom": "^7.0.3",
    "rxjs": "^6.5.3",
    "sip.js": "^0.17.1",
    "swiper": "^4.5.0",
    "thread-loader": "^3.0.1",
    "ts-node": "^4.1.0",
    "tslib": "^2.0.1",
    "twemoji": "^2.5.0",
    "typescript": "^4.0.3",
    "webpack": "^5.3.2",
    "webrtc-adapter": "6.4.8"
  }
}

[aackerman]

I recommend not using this in dev mode where you want/need fast compilation, 6 seconds is brutal enough.

I have no strong intentions to make this plugin faster because I recommend it to be used to trigge errors in CI and if the duration is 12 seconds in CI, that's not important enough to micro-optimize.

[zlk89]

Is it possible to make this plugin to run in separate thread, similar with the ForkTsCheckerWebpackPlugin?

[ko22009]

up