search

wallabyjs / public

Repository for Wallaby.js questions and issues
http://wallabyjs.com
760 stars 45 forks source link

Only runs subset of tests with default workers configuration #1515

Closed jkyoutsey closed 6 years ago

jkyoutsey commented 6 years ago

Issue description or question

If add

module.exports = function (wallaby) {
  ...
  return {
    workers: { initial: 1, regular: 1 },
    ...
  };
};

Then all 600+ of my tests run.

If I do not add it, then I get in the low 100 tests running. 104 this time, 111 next time, sometimes as much as 128. But never all 600+ tests.

Is this configuration costing me developer time? It seems to be just initial startup, and once running doesn't cause THAT much difference. But I'd like my initial wallaby startup to be as fast as possible.

Wallaby.js configuration file

var wallabyWebpack = require('wallaby-webpack');
var path = require('path');

var compilerOptions = Object.assign(
  require('./tsconfig.json').compilerOptions,
  require('./src/main/ui/tsconfig.spec.json').compilerOptions);

compilerOptions.module = 'CommonJs';

module.exports = function (wallaby) {

  var webpackPostprocessor = wallabyWebpack({
    entryPatterns: [
      'src/**/wallabyTest.js',
      'src/**/*spec.js'
    ],

    module: {
      rules: [
        { test: /\.css$/, loader: ['raw-loader', 'css-loader'] },
        { test: /\.html$/, loader: 'raw-loader' },
        { test: /\.ts$/, loader: '@ngtools/webpack', include: /node_modules/, query: { tsConfigPath: 'tsconfig.json' } },
        { test: /\.js$/, loader: 'angular2-template-loader', exclude: /node_modules/ },
        { test: /\.json$/, loader: 'json-loader' },
        { test: /\.styl$/, loaders: ['raw-loader', 'stylus-loader'] },
        { test: /\.less$/, loaders: ['raw-loader', 'less-loader'] },
        { test: /\.scss$|\.sass$/, loaders: ['raw-loader', 'sass-loader'] },
        { test: /\.(jpg|png)$/, loader: 'url-loader?limit=128000' }
      ]
    },

    resolve: {
      extensions: ['.js', '.ts'],
      modules: [
        path.join(wallaby.projectCacheDir, 'src/main/ui/app'),
        path.join(wallaby.projectCacheDir, 'src/main/ui'),
        'node_modules'
      ],
      alias: {
        // except for @angular, this should match tsconfig.spec.json
        '@environment': path.join(wallaby.projectCacheDir, 'src/main/ui/environments'),
        '@assets': path.join(wallaby.projectCacheDir, 'src/main/ui/assets'),
        "@utils": path.join(wallaby.projectCacheDir, 'src/main/ui/app/utils'),
        "@shared": path.join(wallaby.projectCacheDir, 'src/main/ui/app/shared'),
        "@models": path.join(wallaby.projectCacheDir, 'src/main/ui/app/models'),
        "@core": path.join(wallaby.projectCacheDir, 'src/main/ui/app/core'),
      }
    },
    node: {
      fs: 'empty',
      net: 'empty',
      tls: 'empty',
      dns: 'empty'
    }
  });

  return {
    workers: { initial: 1, regular: 1 },

    files: [
      { pattern: 'src/**/*.+(ts|css|less|scss|sass|styl|html|json|svg)', load: false },
      { pattern: 'src/**/*.d.ts', ignore: true },
      { pattern: 'src/**/*spec.ts', ignore: true }
    ],

    tests: [
      { pattern: 'src/**/*spec.ts', load: false },
      { pattern: 'src/**/*e2e-spec.ts', ignore: true }
    ],

    testFramework: 'jasmine',

    compilers: {
      '**/*.ts': wallaby.compilers.typeScript(compilerOptions)
    },

    middleware: function (app, express) {
      var path = require('path');
      app.use('/favicon.ico', express.static(path.join(__dirname, 'src/main/ui/favicon.ico')));
      app.use('/assets', express.static(path.join(__dirname, 'src/main/ui/assets')));
    },

    env: {
      kind: 'chrome'
    },

    postprocessor: webpackPostprocessor,

    setup: function () {
      window.__moduleBundler.loadTests();
    },

    debug: true
  };
};

Code editor or IDE name and version

Visual Studio Code January 2018

OS name and version

Mac OS-X High Sierra

ArtemGovorov commented 6 years ago

If I do not add it, then I get in the low 100 tests running. 104 this time, 111 next time, sometimes as much as 128. But never all 600+ tests.

This means some of your tests are failing when run in parallel (probably because some test files depend on each other in some way, so that only running them sequentially works).

What errors are displayed in Wallaby Failing tests output channel and Wallaby Console output channel when you run your tests without the workers: { initial: 1, regular: 1 }, setting?

Is this configuration costing me developer time? It seems to be just initial startup, and once running doesn't cause THAT much difference. But I'd like my initial wallaby startup to be as fast as possible.

It mostly affects startup time. Once running, wallaby runs affected tests incrementally and normally it's just one spec file, so using 1 worker doesn't affect incremental runs much, sometimes at all (depending on your project/tests structure).

ArtemGovorov commented 6 years ago

Please also see my email with some suggestions/questions. Could you also please

jkyoutsey commented 6 years ago

Most relevent bits first...

I'm using latest Chrome Version 64.0.3282.167 (Official Build) (64-bit). Fails with multiple workers. Electron works fine with multiple workers. Appears to be an issue with the latest ChromeHeadless? ChromeHeadless is what our CI server uses, so I'd really like to continue using it. I can ng test and everything works. But that's just the same as wallaby in a single worker.

Other bits you asked for:

Okay, so I added emitModulePaths: true and I get this output with multiple workers:

​​Uncaught Error: Cannot find module './src/main/ui/wallabyTest.js' (./src/main/ui/wallabyTest.js)​​
​​at http://localhost:61659/wallaby-webpack.js?1518702094975:1​​
​​Uncaught Error: Cannot find module '295' (/Users/me/src/app_om/branches/jky-70824-httpclient/node_modules/@angular/core/@angular/core/testing.es5.js)​​
​​at http://localhost:61659/wallaby-webpack.js?1518702094975:1​​
​​Uncaught Error: Cannot find module '297' (/Users/me/src/app_om/branches/jky-70824-httpclient/node_modules/@angular/core/@angular/core.es5.js)​​
​​at http://localhost:61659/wallaby-webpack.js?1518702094975:1​​

None of my jasmine specs are interdependent.

package.json

  "dependencies": {
    "@angular/animations": "^4.3.0",
    "@angular/common": "^4.3.0",
    "@angular/compiler": "^4.3.0",
    "@angular/core": "^4.3.0",
    "@angular/forms": "^4.3.0",
    "@angular/http": "^4.3.0",
    "@angular/platform-browser": "^4.3.0",
    "@angular/platform-browser-dynamic": "^4.3.0",
    "@angular/router": "^4.3.0",
    "@conclurer/date-to-format-pipe": "^1.1.3",
    "@ngrx/effects": "^4.1.1",
    "@ngrx/router-store": "^4.1.1",
    "@ngrx/store": "^4.1.1",
    "@ngrx/store-devtools": "^4.1.1",
    "@types/geojson": "^1.0.6",
    "@types/lodash": "^4.14.71",
    "bootstrap-sass": "^3.3.7",
    "c3": "^0.4.14",
    "chart.js": "^2.7.0",
    "classlist.js": "^1.1.20150312",
    "core-js": "^2.4.1",
    "d3-ng2-service": "^1.16.0",
    "esri-leaflet": "^2.1.2",
    "font-awesome": "^4.7.0",
    "intl": "^1.2.5",
    "leaflet": "^1.3.0",
    "leaflet.markercluster": "^1.2.0",
    "lodash": "^4.17.4",
    "moment": "^2.18.1",
    "ngrx-store-freeze": "^0.1.9",
    "ngx-components": "^1.0.1-beta.4",
    "ngx-devproxy": "^3.1.0",
    "ngx-tilegrid": "^1.0.3",
    "primeng": "^4.3.0",
    "reselect": "^3.0.1",
    "rxjs": "^5.1.0",
    "topojson-client": "^3.0.0",
    "web-animations-js": "^2.2.2",
    "zone.js": "^0.8.14"
  },
  "devDependencies": {
    "@angular-devkit/core": "0.0.29",
    "@angular/cli": "1.6.2",
    "@angular/compiler-cli": "^4.3.0",
    "@angular/language-service": "^4.3.0",
    "@types/jasmine": "2.5.45",
    "@types/leaflet": "^1.2.5",
    "@types/node": "^6.0.85",
    "angular2-template-loader": "^0.6.2",
    "codelyzer": "^4.1.0",
    "enhanced-resolve": "3.3.0",
    "jasmine-core": "~2.6.2",
    "jasmine-spec-reporter": "~4.1.0",
    "karma": "^1.7.1",
    "karma-chrome-launcher": "~2.1.1",
    "karma-cli": "~1.0.1",
    "karma-coverage-istanbul-reporter": "^1.2.1",
    "karma-ie-launcher": "^1.0.0",
    "karma-jasmine": "~1.1.0",
    "karma-jasmine-html-reporter": "^0.2.2",
    "karma-phantomjs-launcher": "~1.0.4",
    "karma-spec-reporter": "^0.0.30",
    "karma-verbose-reporter": "^0.0.6",
    "protractor": "~5.1.2",
    "ts-node": "~3.0.4",
    "tslint": "~5.3.2",
    "typescript": "~2.3.3",
    "wallaby-webpack": "^3.9.4"
  }

Commenting out all of my specs and re-adding them is a non-starter. There are hundreds. It would take far too long to be practical.

ArtemGovorov commented 6 years ago

Thanks for the provided info. Merging the issue into https://github.com/wallabyjs/public/issues/1514#issuecomment-366115013, you may more info and subscribe for updates there.