Support vitest bench

[polRk]

Issue description or question

Go an error if using vitest bench

Error: `bench()` is only available in benchmark mode. Run with `vitest bench` instead.

Wallaby diagnostics report

{
  editorVersion: '1.73.0',
  pluginVersion: '1.0.348',
  editorType: 'VSCode',
  osVersion: 'linux 5.4.0-1094-azure',
  nodeVersion: 'v18.11.0',
  coreVersion: '1.0.1342',
  checksum: 'ZDEyOGZmNzYyYWE4ODk3YzY1YTYxNjQ2NWY3ZTgxMzIsMTY2ODk4ODgwMDAwMCwx',
  config: {
    diagnostics: {
      vitest: {
        config: {
          allowOnly: true,
          watch: true,
          globals: false,
          environment: 'node',
          threads: true,
          clearMocks: false,
          restoreMocks: false,
          mockReset: false,
          include: [ '**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}' ],
          exclude: [ '**/node_modules/**', '**/dist/**', '**/cypress/**', '**/.{idea,git,cache,output,temp}/**', '**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress}.config.*' ],
          testTimeout: 5000,
          hookTimeout: 10000,
          teardownTimeout: 1000,
          isolate: true,
          watchExclude: [ '**/node_modules/**', '**/dist/**' ],
          forceRerunTriggers: [ '**/package.json/**', '**/{vitest,vite}.config.*/**' ],
          update: false,
          reporters: [ 'default' ],
          silent: false,
          ui: false,
          uiBase: '/__vitest__/',
          open: true,
          css: { include: [], modules: { classNameStrategy: 'stable' } },
          coverage: {
            provider: 'c8',
            enabled: false,
            clean: true,
            cleanOnRerun: false,
            reportsDirectory: './coverage',
            excludeNodeModules: true,
            exclude: [
              'coverage/**',
              'dist/**',
              'packages/*/test{,s}/**',
              '**/*.d.ts',
              'cypress/**',
              'test{,s}/**',
              'test{,-*}.{js,cjs,mjs,ts,tsx,jsx}',
              '**/*{.,-}test.{js,cjs,mjs,ts,tsx,jsx}',
              '**/*{.,-}spec.{js,cjs,mjs,ts,tsx,jsx}',
              '**/__tests__/**',
              '**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress}.config.*',
              '**/.{eslint,mocha,prettier}rc.{js,cjs,yml}'
            ],
            reporter: [ 'text', 'html', 'clover', 'json' ],
            allowExternal: false,
            extension: [
              '.js',  '.cjs',
              '.mjs', '.ts',
              '.tsx', '.jsx',
              '.vue', '.svelte'
            ]
          },
          fakeTimers: { loopLimit: 10000, shouldClearNativeTimers: true, toFake: [ 'setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'setImmediate', 'clearImmediate', 'Date' ] },
          maxConcurrency: 5,
          dangerouslyIgnoreUnhandledErrors: false,
          defines: {},
          root: '<rootDir>',
          mode: [],
          deps: { inline: [ {}, {}, {}, {}, {}, {}, '@nuxt/test-utils' ], registerNodeLoader: false },
          snapshotOptions: { snapshotFormat: {}, updateSnapshot: 'new' },
          setupFiles: [],
          cache: { dir: '<rootDir>/node_modules/.vitest' },
          sequence: {},
          package: {
            version: '0.24.5',
            urls: { hooks: 'file://<homeDir>/.vscode-remote/extensions/wallabyjs.wallaby-vscode-1.0.348/wallaby9c62ce/runners/node/hooks.mjs' },
            paths: { root: '<rootDir>/node_modules/vitest', dist: '<rootDir>/node_modules/vitest/dist' }
          }
        }
      }
    },
    testFramework: { version: 'vitest@0.14.0', configurator: 'vitest@0.14.0', reporter: 'vitest@0.14.0', starter: 'vitest@0.14.0', autoDetected: true },
    preserveComments: false,
    extractComments: true,
    files: [
      { pattern: '**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}', ignore: true, trigger: true, load: true, file: true },
      { pattern: '**/node_modules/**', ignore: true, trigger: true, load: true, file: true, test: true },
      { pattern: '**/dist/**', ignore: true, trigger: true, load: true, file: true, test: true },
      { pattern: '**/cypress/**', ignore: true, trigger: true, load: true, file: true, test: true },
      { pattern: '**/.{idea,git,cache,output,temp}/**', ignore: true, trigger: true, load: true, file: true, test: true },
      { pattern: '**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress}.config.*', ignore: true, trigger: true, load: true, file: true, test: true },
      { pattern: '**/*.*', ignore: false, trigger: true, load: true, order: 1 }
    ],
    tests: [
      { pattern: '**/node_modules/**', ignore: true, trigger: true, load: true, test: true, file: false },
      { pattern: '**/dist/**', ignore: true, trigger: true, load: true, test: true, file: false },
      { pattern: '**/cypress/**', ignore: true, trigger: true, load: true, test: true, file: false },
      { pattern: '**/.{idea,git,cache,output,temp}/**', ignore: true, trigger: true, load: true, test: true, file: false },
      { pattern: '**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress}.config.*', ignore: true, trigger: true, load: true, test: true, file: false },
      { pattern: '**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}', ignore: false, trigger: true, load: true, test: true, order: 2 }
    ],
    workers: { initial: 1, regular: 1, recycle: false },
    filesWithNoCoverageCalculated: [],
    runAllTestsInAffectedTestFile: false,
    updateNoMoreThanOneSnapshotPerTestFileRun: false,
    compilers: {},
    logLimits: { inline: { depth: 5, elements: 5000 }, values: { default: { stringLength: 200 }, autoExpand: { elements: 5000, stringLength: 8192, depth: 10 } } },
    preprocessors: {},
    maxConsoleMessagesPerTest: 100,
    autoConsoleLog: true,
    delays: { run: 0, edit: 100, update: 0 },
    teardown: undefined,
    hints: {
      ignoreCoverage: '__REGEXP /ignore coverage|istanbul ignore/',
      ignoreCoverageForFile: '__REGEXP /ignore file coverage/',
      commentAutoLog: '?',
      testFileSelection: { include: '__REGEXP /file\\.only/', exclude: '__REGEXP /file\\.skip/' }
    },
    automaticTestFileSelection: true,
    runSelectedTestsOnly: false,
    mapConsoleMessagesStackTrace: false,
    extensions: {},
    env: {
      type: 'node',
      params: { runner: '--experimental-loader=file://<homeDir>/.vscode-remote/extensions/wallabyjs.wallaby-vscode-1.0.348/wallaby9c62ce/runners/node/hooks.mjs' },
      runner: '/usr/local/bin/node',
      viewportSize: { width: 800, height: 600 },
      options: { width: 800, height: 600 },
      bundle: true
    },
    reportUnhandledPromises: true,
    slowTestThreshold: 75,
    lowCoverageThreshold: 80,
    runAllTestsWhenNoAffectedTests: true,
    symlinkNodeModules: true,
    configCode: 'auto.detect#-2054068591'
  },
  packageJSON: {
    dependencies: undefined,
    devDependencies: {
      vite: '3.2.2',
      vitest: '0.24.5',
    }
  },
  fs: { numberOfFiles: 82 },
  debug: []
}

[smcenlly]

We don't have any plans to support vitest benchmarks with Wallaby but Wallaby should not error in this case.

As of Wallaby v1.0.1343, any vitest bench definitions will be skipped by Wallaby.

Our thinking is that benchmarks can be expensive to run and in the context of Wallaby running your tests, they don't actually provide the kind of instant validation/feedback that is useful in the context of Wallaby executing your code. This is most likely why there is a separate run mode in vitest. Additionally, there are no assertions that are validated within the benchmarks and at this point in time vitest does not contain any way of validating the performance of your benchmarks.

If you really want Wallaby to report on test execution times (similar to bench), you can use the it function (instead of bench function) and Wallaby will report on the total test execution time. You could also measure and have assertion of test execution time within such a function if it is important to you.