.verify swallows custom assertion initialization errors

[michaelleb]

Description of the bug/issue

when I execute browser.verify.myCustomAssertion(), and there is an error inside myCustomAssertion definition function, I expect that the error will be visible in the test runner output console, but what happens is that no output is written for the custom assertion as if there was no such line in the test case at all. Also, no error message is logged to the test runner output console.

When the same custom assertion run via .assert, browser.assert.myCustomAssertion(), the error is visible

Steps to reproduce

1. setup new nightwatch project - it comes with sample for custom assertion and examples of simple google search page object.
2. create new custom assertion in file /custom-assertions/myCustomAssertion.js - paste below content
3. create new test file in /test/custom/testCustomAssertion.js - paste below content
4. while in project directory, run nightwatch test using npx nightwatch --group custom
5. see that there should be error but no error is visible in the console output
6. change the .verify to .assert in the test case, or uncomment the line with the .assert
7. run the test again
8. see that the error is visible in the console output, but it should have been visible in step 5 as well.

files content

// .../test/custom/testCustomAssertion.js
describe('Test Custom Assertion', () => {

  test('custom assertion should display error in the output console', async function testSouldShowError(browser){

    const searchPage = browser.page.google.search()
    await searchPage.navigate()

    await browser.verify.myCustomAssertion('foo') // <---- should show error in console but swallows it.

    // When running the same assertion via .assert, the error is visible in the 
    // output console as expected
    // await browser.assert.myCustomAssertion('foo') 
  })
});

//  .../custom-assertions/myCustomAssertion.js

/** A custom Nightwatch assertion. The assertion name is the filename.  */

exports.assertion = function myCustomAssertion(expected = 'ok') {
  /** when called, this function bound to AssertionInstance object */

  this.api = 'this line throws Error, which is swallowed when assertion called under .verify, but visible when called under .assert'
  // throw new Error('something went wrong')

  // Message to be displayed on the console while running this assertion.
  this.message = `Testing my custom assertion working`;

  // Expected value of the assertion, to be displayed in case of failure.
  this.expected = expected;

  // Given the result value (from `this.value` below), this function will
  // evaluate if the assertion has passed.
  this.evaluate = function (value) {

    return value === this.expected;
  };

  // Retrieve the value from the result object we got after running the
  // assertion command (defined below), which is to be evaluated against
  // the value passed into the assertion as the second argument.
  this.value = function (result) {
    return result.value;
  };

  // The command to be executed by the assertion runner, to find the actual
  // result. Nightwatch API is available as `this.api`.
  this.command = function (callback) {
    const result = {
      value: 'ok'
    }
    callback(result)
  };
};

Sample test

test/custom/
describe('Test Custom Assertion', () => {

  test('custom assertion should display error in the output console', async function testSouldShowError(browser){

    const searchPage = browser.page.google.search()
    await searchPage.navigate()

    await browser.verify.myCustomAssertion('foo') // <---- should show error in console but swallows it.

    // When running the same assertion via .assert, the error is visible in the 
    // output console as expected
    // await browser.assert.myCustomAssertion('foo') 
  })
});

### Command to run

```bash
npx nightwatch --group custom

Verbose Output

Running   default: custom/testCustomAssertion.js  
┌ ────────────────── ✔  default: custom/testCustomAssertion.js  ──────────────────────────────────────────────────────────────────────────┐
│                                                                                                                                         │
│   default [Test Custom Assertion] Test Suite                                                                                            │
│   default ────────────────────────────────────────────────────────────────────                                                          │
│   default Starting ChromeDriver...                                                                                                      │
│   default   Request POST /session                                                                                                       │
│   default {                                                                                                                             │
│   default      capabilities: {                                                                                                          │
│   default        firstMatch: [ {} ],                                                                                                    │
│   default        alwaysMatch: {                                                                                                         │
│   default          browserName: 'chrome',                                                                                               │
│   default          'goog:chromeOptions': {                                                                                              │
│   default            binary: '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'                                             │
│   default          }                                                                                                                    │
│   default        }                                                                                                                      │
│   default      }                                                                                                                        │
│   default   }                                                                                                                           │
│   default   Response 200 POST /session (475ms)                                                                                          │
│   default {                                                                                                                             │
│   default      value: {                                                                                                                 │
│   default        capabilities: {                                                                                                        │
│   default          acceptInsecureCerts: false,                                                                                          │
│   default          browserName: 'chrome',                                                                                               │
│   default          browserVersion: '123.0.6312.59',                                                                                     │
│   default          chrome: {                                                                                                            │
│   default            chromedriverVersion: '123.0.6312.58 (6b4b19e9dfbb93aa414dc045bd445287977d8d7a-refs/branch-heads/6312_46@{#3})',    │
│   default            userDataDir: '/var/folders/pt/rfwb8gk529d_wb4r8gf80r040000gn/T/.org.chromium.Chromium.WvNw90'                      │
│   default          },                                                                                                                   │
│   default          'fedcm:accounts': true,                                                                                              │
│   default          'goog:chromeOptions': { debuggerAddress: 'localhost:59178' },                                                        │
│   default          networkConnectionEnabled: false,                                                                                     │
│   default          pageLoadStrategy: 'normal',                                                                                          │
│   default          platformName: 'mac',                                                                                                 │
│   default          proxy: {},                                                                                                           │
│   default          setWindowRect: true,                                                                                                 │
│   default          strictFileInteractability: false,                                                                                    │
│   default          timeouts: { implicit: 0, pageLoad: 300000, script: 30000 },                                                          │
│   default          unhandledPromptBehavior: 'dismiss and notify',                                                                       │
│   default          'webauthn:extension:credBlob': true,                                                                                 │
│   default          'webauthn:extension:largeBlob': true,                                                                                │
│   default          'webauthn:extension:minPinLength': true,                                                                             │
│   default          'webauthn:extension:prf': true,                                                                                      │
│   default          'webauthn:virtualAuthenticators': true                                                                               │
│   default        },                                                                                                                     │
│   default        sessionId: 'c4dc1f2b361ac6af312aedcf000241c9'                                                                          │
│   default      }                                                                                                                        │
│   default   }                                                                                                                           │
│   default Using: chrome (123.0.6312.59) on MAC.                                                                                         │
│   default Received session with ID: c4dc1f2b361ac6af312aedcf000241c9                                                                    │
│   default                                                                                                                               │
│   default → Running [before]:                                                                                                           │
│   default → Completed [before].                                                                                                         │
│   default – custom assertion should display error in the output console                                                                 │
│   default → Running [beforeEach]:                                                                                                       │
│   default → Completed [beforeEach].                                                                                                     │
│   default → Running command: url ('https://google.no', )                                                                                │
│   default - Loading url: https://google.no                                                                                              │
│   default   Request POST /session/c4dc1f2b361ac6af312aedcf000241c9/url                                                                  │
│   default { url: 'https://google.no' }                                                                                                  │
│   default   Response 200 POST /session/c4dc1f2b361ac6af312aedcf000241c9/url (1675ms)                                                    │
│   default { value: null }                                                                                                               │
│   default ℹ Loaded url https://google.no in 1677ms                                                                                      │
│   default → Completed command: url ('https://google.no', ) (1677ms)                                                                     │
│   default → Running command: verify.myCustomAssertion ('foo')                                                                           │
│   default → Completed command: verify.myCustomAssertion ('foo') (0ms)                                                                   │
│   default → Running [afterEach]:                                                                                                        │
│   default → Completed [afterEach].                                                                                                      │
│   default ✔ default [Test Custom Assertion] custom assertion should display error in the output console (1.685s)                        │
│   default → Running [after]:                                                                                                            │
│   default → Completed [after].                                                                                                          │
│   default → Running command: end (true)                                                                                                 │
│   default → Running command: session ('delete', [Function])                                                                             │
│   default   Request DELETE /session/c4dc1f2b361ac6af312aedcf000241c9                                                                    │
│   default   Response 200 DELETE /session/c4dc1f2b361ac6af312aedcf000241c9 (52ms)                                                        │
│   default { value: null }                                                                                                               │
│   default → Completed command: end (true) (54ms)                                                                                        │
│   default → Completed command: session ('delete', [Function]) (54ms)                                                                    │
│                                                                                                                                         │
└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
 Wrote HTML report file to: /Users/rocket/code/inception/src/frontend/tests_output/nightwatch-html-report/index.html

 Wrote JSON report file to: /Users/rocket/code/inception/src/frontend/tests_output/custom/CHROME_123.0.6312.59__testCustomAssertion.json
 Wrote Rerun Json report file to: /Users/rocket/code/inception/src/frontend/tests_output/minimal_report.json
 Wrote XML report file to: /Users/rocket/code/inception/src/frontend/tests_output/custom/CHROME_123.0.6312.59__testCustomAssertion.xml
Tearing down...
 Analytics send event:
   {
     client_id: 'eaf1701e-381d-4976-a377-d2a5b865341a',
     non_personalized_ads: true,
     timestamp_micros: 1711281213939000,
     events: [
       {
         name: 'nw_test_run',
         params: {
           arg_parallel: 'undefined',
           browser_name: 'chrome',
           test_workers_enabled: true,
           use_xpath: false,
           is_bstack: false,
           test_runner: 'default',
           event_time: 1711281210950000,
           env_os: 'darwin/23.4.0/Apple M3 Pro',
           env_lang: 'en_US.UTF-8',
           env_nw_version: '3.4.1',
           env_node_version: 'node v20.11.1',
           test_env: 'default',
           run_id: '480ece06-e92e-4129-94cc-659c50236492'
         }
       },
       {
         name: 'nw_test_run',
         params: {
           arg_parallel: 'undefined',
           browser_name: 'chrome',
           test_workers_enabled: true,
           use_xpath: false,
           is_bstack: false,
           test_runner: 'default',
           event_time: 1711281211333000,
           env_os: 'darwin/23.4.0/Apple M3 Pro',
           env_lang: 'en_US.UTF-8',
           env_nw_version: '3.4.1',
           env_node_version: 'node v20.11.1',
           test_env: 'default',
           run_id: 'eeb1fa9c-deb0-4cf8-9c2a-ab83e5abfff2'
         }
       }
     ]
  }

Nightwatch Configuration

No response

Nightwatch.js Version

3.4.1

Node Version

20.11.1

Browser

chrome (123.0.6312.59) on MAC

Operating System

MacOs 14.4 (23E214)

Additional Information

No response

[Visbhavesh]

hey assign these issue to me and I will resolve these