Failed test Run with maxFailures flag comes out as having no failures

[lonely-caat]

Hi, awesome lib, forever grateful! My issue is when my test run is aborted due to having too many failures the reports comes out without failures.

  reporter: process.env.CI
    ? [
        [
          './node_modules/playwright-slack-report/dist/src/SlackReporter.js',
          {
            channels: ['tests'],
            sendResults: 'always', // "always" , "on-failure", "off"
            meta: [
              {
                key: 'ENVIRONMENT',
                value: process.env.BASE_URL,
              },

            ],
            disableUnfurl: true,
            slackLogLevel: LogLevel.DEBUG,
            showInThread: true,
          },
        ],
        ['html'],
      ]
    : [['dot'], ['list'], ['html']],

and

export default defineConfig({

  /* Run tests in files in parallel */
  fullyParallel: true,
  /* Fail the whole run after enough failures happen*/
  maxFailures: 20,

Comes out as

[ryanrosello-og]

I can't seem to repro this using similar config you provided. @lonely-caat , Can you confirm that you are using the latest version?

Config (set to maxFailures === 2)

// playwright.config.ts
import { PlaywrightTestConfig, devices } from '@playwright/test';

const config: PlaywrightTestConfig = {
  fullyParallel: true,
  maxFailures: 2,
  retries: 1,
  use: {
    trace: 'on',
    screenshot: 'on',
    video: 'on',
  },
  projects: [
    {
      name: 'Nightly_Regression',
      use: { ...devices['Desktop Chrome'] },
    },
  ],
  reporter: [
    [
      './node_modules/playwright-slack-report/dist/src/SlackReporter.js',
      {
        channels: ['qa-automation'],
        sendResults: 'always', // "always" , "on-failure", "off"
        showInThread: true,
        meta: [
          {
            key: 'BuildID',
            value: '0181ec91-6d2e-4ac9-8587-7612858a57a7',
          },
          {
            key: 'Branch',
            value: 'feat/create-secondary-account',
          },
          {
            key: 'HTML Results',
            value:
              '<https://buildkite-artifacts.playwright.dev/pw/23887/playwright-report/index.html|📊>',
          },
        ],
      },
    ],
    ['dot'],
    ['html'],
  ],
};
export default config;

Tests (should have 4 failures)

import { test, expect } from '@playwright/test';

test('@this has title', async ({ page }) => {
  await page.goto('https://playwright.dev/');

  // Expect a title "to contain" a substring.
  await expect(page).toHaveTitle(/Playwright/);
});
test('@this has title2', async ({ page }) => {
  await page.goto('https://playwright.dev/');

  // Expect a title "to contain" a substring.
  await expect(page).toHaveTitle(/Playwright/);
});
test.describe('serial', () => {
  test('@this has title FAIL 1', async ({ page }) => {
    await page.goto('https://playwright.dev/');

    // Expect a title "to contain" a substring.
    await expect(page).toHaveTitle(/Playwrightzz/, { timeout: 500 });
  });
  test('@this has title4', async ({ page }) => {
    await page.goto('https://playwright.dev/');

    // Expect a title "to contain" a substring.
    await expect(page).toHaveTitle(/Playwright/);
  });
});

test('@this get started link FAIL 2', async ({ page }) => {
  await page.goto('https://playwright.dev/');

  // Click the get started link.
  await page.getByRole('link', { name: 'Get started' }).click();

  // Expects page to have a heading with the name of Installation.
  await expect(
    page.getByRole('heading', { name: 'Installation_fail' }),
  ).toBeVisible({ timeout: 500 });
});
test('@this get started link2 FAIL 3', async ({ page }) => {
  await page.goto('https://playwright.dev/');

  // Click the get started link.
  await page.getByRole('link', { name: 'Get started' }).click();

  // Expects page to have a heading with the name of Installation.
  await expect(
    page.getByRole('heading', { name: 'Installation_fail' }),
  ).toBeVisible({ timeout: 500 });
});
test('@this get started link3 FAIL 4', async ({ page }, testInfo) => {
  await page.goto('https://playwright.dev/');

  // Click the get started link.
  await page.getByRole('link', { name: 'Get started' }).click();

  if (testInfo.retry > 0) {
    await expect(
      page.getByRole('heading', { name: 'Installation' }),
    ).toBeVisible({ timeout: 500 });
  } else {
    await expect(
      page.getByRole('heading', { name: 'Installation_fail' }),
    ).toBeVisible({ timeout: 500 });
  }
});

[github-actions[bot]]

This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.