Newman fails when run with WDIO/Mocha

[adzhanson]

Hi, I am pretty new to coding so this may well be a user issue. However I am trying to run a collection from mocha using the Newman npm library. When I do so only the first request in the collection gets run. I don't see any errors in the console log and the results are not reported. I am using the default cli reporter.

If i run the same collection from a command line it completes all requests as expected and the results are correctly reported.

This is the code I am running:

var newman = require('newman')
// call newman.run to pass `options` object and wait for callback
newman.run({
    collection: require('./Data/Test.postman_collection.json'),
    reporters: 'cli'
}, function (err) {
    if (err) { throw err; }
    console.log('collection run complete!');
});

Collection is:

{
    "info": {
        "name": "Test",
        "_postman_id": "a05de9a1-4678-0bec-2575-d8d708f080fd",
        "description": "",
        "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
    },
    "item": [
        {
            "name": "Delay 3s copy",
            "event": [
                {
                    "listen": "test",
                    "script": {
                        "id": "3bd8d84d-47ed-47da-ac12-fc7b6aa92947",
                        "type": "text/javascript",
                        "exec": [
                            ""
                        ]
                    }
                }
            ],
            "request": {
                "method": "GET",
                "header": [],
                "body": {},
                "url": {
                    "raw": "https://postman-echo.com/delay/1",
                    "protocol": "https",
                    "host": [
                        "postman-echo",
                        "com"
                    ],
                    "path": [
                        "delay",
                        "1"
                    ]
                },
                "description": null
            },
            "response": []
        },
        {
            "name": "Delay 3s copy copy",
            "event": [
                {
                    "listen": "test",
                    "script": {
                        "id": "3bd8d84d-47ed-47da-ac12-fc7b6aa92947",
                        "type": "text/javascript",
                        "exec": [
                            ""
                        ]
                    }
                }
            ],
            "request": {
                "method": "GET",
                "header": [],
                "body": {},
                "url": {
                    "raw": "https://postman-echo.com/delay/1",
                    "protocol": "https",
                    "host": [
                        "postman-echo",
                        "com"
                    ],
                    "path": [
                        "delay",
                        "1"
                    ]
                },
                "description": null
            },
            "response": []
        }
    ]
}

1. Newman Version - 3.9.3
2. OS details ubuntu 16:
3. Using Newman as a library

[kunagpal]

Could you share some Mocha test specs that would help us reproduce this behaviour? A snippet of the form below should run all collection requests correctly:

var newman = require('newman');

describe('newman', function () {
  // done on the next line is a function that marks the current test as asynchronous
  it('should run collection correctly', function (done) {
    this.timeout(1e4); // 10 seconds
    newman.run({
      collection: require('./Data/Test.postman_collection.json'),
      reporters: 'cli'
    }, function (err, summary) {
      if (err) { return done(err); }
      if (summary.run.failures.length) { return done(new Error('One or more collection tests failed')); }
      done();
    });
  });
});

[adzhanson]

Hi kunagpal

Thankyou for your response. I am using WebdriverIO to run the mocha framework. The WDIO config is here:

const env = require('process').env

const inBrowserstack = (env.E2E_ENV === 'browserstack')

let WdioTestRailReporter = require('./node_modules/wdio-testrail-reporter/lib/wdio-testrail-reporter')

exports.config = {
  //
  // ==================
  // Specify Test Files
  // ==================
  // Define which test specs should run. The pattern is relative to the directory
  // from which `wdio` was called. Notice that, if you are calling `wdio` from an
  // NPM script (see https://docs.npmjs.com/cli/run-script) then the current working
  // directory is where your package.json resides, so `wdio` will be called from there.
  //
  specs: [
'./postman/smokeTestPostman.spec.js',
],
  // Patterns to exclude.
  exclude: [
  ],
  suites: {
  },
  //
  // ============
  // Capabilities
  // ============
  // Define your capabilities here. WebdriverIO can run multiple capabilities at the same
  // time. Depending on the number of capabilities, WebdriverIO launches several test
  // sessions. Within your capabilities you can overwrite the spec and exclude options in
  // order to group specific specs to a specific capability.
  //
  // First, you can define how many instances should be started at the same time. Let's
  // say you have 3 different capabilities (Chrome, Firefox, and Safari) and you have
  // set maxInstances to 1; wdio will spawn 3 processes. Therefore, if you have 10 spec
  // files and you set maxInstances to 10, all spec files will get tested at the same time
  // and 30 processes will get spawned. The property handles how many capabilities
  // from the same test should run tests.
  //
  maxInstances: 10,
  //
  // If you have trouble getting all important capabilities together, check out the
  // Sauce Labs platform configurator - a great tool to configure your capabilities:
  // https://docs.saucelabs.com/reference/platforms-configurator
  //
  capabilities: [{
    maxInstances: 3,
    browserName: 'chrome',
    chromeOptions: {
      args: inBrowserstack || env.IN_BROWSER ? ['--ignore-certificate-errors', 'window-size=1920,1080'] : ['--headless', '--ignore-certificate-errors'] 
    },
    acceptSslCerts: true,
    'browserstack.local': inBrowserstack,
    'browserstack.debug': inBrowserstack,
    'browserstack.video': inBrowserstack,
    'browserstack.networkLogs': inBrowserstack,
  }],
  //
  // ===================
  // Test Configurations
  // ===================
  // Define all options that are relevant for the WebdriverIO instance here
  //
  // By default WebdriverIO commands are executed in a synchronous way using
  // the wdio-sync package. If you still want to run your tests in an async way
  // e.g. using promises you can set the sync option to false.
  sync: true,
  //
  // Level of logging verbosity: silent | verbose | command | data | result | error
  logLevel: 'error',
  //
  // Enables colors for log output.
  coloredLogs: true,
  //
  // If you only want to run your tests until a specific amount of tests have failed use
  // bail (default is 0 - don't bail, run all tests).
  bail: 0,
  //
  // Saves a screenshot to a given path if a command fails.
  //screenshotPath: './errorShots/',
  //
  // Set a base URL in order to shorten url command calls. If your url parameter starts
  // with "/", then the base url gets prepended.
  baseUrl: 'http://localhost',
  //
  // Default timeout for all waitFor* commands.
  waitforTimeout: 120000,
  //
  // Default timeout in milliseconds for request
  // if Selenium Grid doesn't send response
  connectionRetryTimeout: 10000,
  //
  // Default request retries count
  connectionRetryCount: 3,
  //
  // Initialize the browser instance with a WebdriverIO plugin. The object should have the
  // plugin name as key and the desired plugin options as properties. Make sure you have
  // the plugin installed before running any tests. The following plugins are currently
  // available:
  // WebdriverCSS: https://github.com/webdriverio/webdrivercss
  // WebdriverRTC: https://github.com/webdriverio/webdriverrtc
  // Browserevent: https://github.com/webdriverio/browserevent
  // plugins: {
  //     webdrivercss: {
  //         screenshotRoot: 'my-shots',
  //         failedComparisonsRoot: 'diffs',
  //         misMatchTolerance: 0.05,
  //         screenWidth: [320,480,640,1024]
  //     },
  //     webdriverrtc: {},
  //     browserevent: {}
  // },
  //
  // Test runner services
  // Services take over a specific job you don't want to take care of. They enhance
  // your test setup with almost no effort. Unlike plugins, they don't add new
  // commands. Instead, they hook themselves up into the test process.
  services: inBrowserstack ? ['browserstack'] : ['selenium-standalone'],
  user: inBrowserstack ? env.BROWSERSTACK_USERNAME : undefined,
  key: inBrowserstack ? env.BROWSERSTACK_ACCESS_KEY : undefined,
  browserstackLocal: inBrowserstack,
  browserstackOpts: {
    'local-proxy-host': 'xxxx.xxxx',
    'local-proxy-port': 8080,
    'verbose': 3,
    'force-proxy': true
  },
  //
  // Framework you want to run your specs with.
  // The following are supported: Mocha, Jasmine, and Cucumber
  // see also: http://webdriver.io/guide/testrunner/frameworks.html
  //
  // Make sure you have the wdio adapter package for the specific framework installed
  // before running any tests.
  framework: 'mocha',
  //
  // Test reporter for stdout.
  // The only one supported by default is 'dot'
  // see also: http://webdriver.io/guide/testrunner/reporters.html
  reporters: ['spec', 'junit'],
  reporterOptions: {
    junit: {
      outputDir: './'
    }
  },
  //
  // Test reporter for testRail.
  // The only one supported by default is 'dot'
  // see also: http://webdriver.io/guide/testrunner/reporters.html
  reporters: ['spec', WdioTestRailReporter],
  testRailsOptions: {
    domain: "xxxxxxxx.xxxxxxx.com",
    username: "xxxxxxx@xxxxxxxx.com",
    password: "xxxxxxxxx",
    projectId: 222,
    suiteId: 15975,
    runName: "Smoke Test"
  },
  //
  // Options to be passed to Mocha.
  // See the full list at http://mochajs.org/
  mochaOpts: {
    ui: 'bdd',
    require: [],
    compilers: ['js:babel-core/register'],
    timeout: 130000,
    // If the IN_BROWSER environment variable is set then disable timeouts
    // as it's likely debug is desirable which conflicts with timeouts closing
    // the browser.
    enableTimeouts: env.IN_BROWSER ? false : true,
  },
  //
  // =====
  // Hooks
  // =====
  // WebdriverIO provides several hooks you can use to interfere with the test process in order to enhance
  // it and to build services around it. You can either apply a single function or an array of
  // methods to it. If one of them returns with a promise, WebdriverIO will wait until that promise got
  // resolved to continue.
  /**
   * Gets executed once before all workers get launched.
   * @param {Object} config wdio configuration object
   * @param {Array.<Object>} capabilities list of capabilities details
   */
  // onPrepare: function (config, capabilities) {
  // },
  /**
   * Gets executed just before initialising the webdriver session and test framework. It allows you
   * to manipulate configurations depending on the capability or spec.
   * @param {Object} config wdio configuration object
   * @param {Array.<Object>} capabilities list of capabilities details
   * @param {Array.<String>} specs List of spec file paths that are to be run
   */
  // beforeSession: function (config, capabilities, specs) {
  // },
  /**
   * Gets executed before test execution begins. At this point you can access to all global
   * variables like `browser`. It is the perfect place to define custom commands.
   * @param {Array.<Object>} capabilities list of capabilities details
   * @param {Array.<String>} specs List of spec file paths that are to be run
   */
  // before: function (capabilities, specs) {
  // },
  /**
   * Hook that gets executed before the suite starts
   * @param {Object} suite suite details
   */
  // beforeSuite: function (suite) {
  // },
  /**
   * Hook that gets executed _before_ a hook within the suite starts (e.g. runs before calling
   * beforeEach in Mocha)
   */
  // beforeHook: function () {
  // },
  /**
   * Hook that gets executed _after_ a hook within the suite starts (e.g. runs after calling
   * afterEach in Mocha)
   */
  // afterHook: function () {
  // },
  /**
   * Function to be executed before a test (in Mocha/Jasmine) or a step (in Cucumber) starts.
   * @param {Object} test test details
   */
   //beforeTest: function (test) {
   //},
  /**
   * Runs before a WebdriverIO command gets executed.
   * @param {String} commandName hook command name
   * @param {Array} args arguments that command would receive
   */
  // beforeCommand: function (commandName, args) {
  // },
  /**
   * Runs after a WebdriverIO command gets executed
   * @param {String} commandName hook command name
   * @param {Array} args arguments that command would receive
   * @param {Number} result 0 - command success, 1 - command error
   * @param {Object} error error object if any
   */
  // afterCommand: function (commandName, args, result, error) {
  // },
  /**
   * Function to be executed after a test (in Mocha/Jasmine) or a step (in Cucumber) starts.
   * @param {Object} test test details
   */
//   afterTest: function (test) {
 //  },
  /**
   * Hook that gets executed after the suite has ended
   * @param {Object} suite suite details
   */
  // afterSuite: function (suite) {
  // },
  /**
   * Gets executed after all tests are done. You still have access to all global variables from
   * the test.
   * @param {Number} result 0 - test pass, 1 - test fail
   * @param {Array.<Object>} capabilities list of capabilities details
   * @param {Array.<String>} specs List of spec file paths that ran
   */
  // after: function (result, capabilities, specs) {
  // },
  /**
   * Gets executed right after terminating the webdriver session.
   * @param {Object} config wdio configuration object
   * @param {Array.<Object>} capabilities list of capabilities details
   * @param {Array.<String>} specs List of spec file paths that ran
   */
  // afterSession: function (config, capabilities, specs) {
  // },
  /**
   * Gets executed after all workers got shut down and the process is about to exit. It is not
   * possible to defer the end of the process using a promise.
   * @param {Object} exitCode 0 - success, 1 - fail
   */
  // onComplete: function(exitCode) {
  // }

  // This flag is to enable/disable browser debugging in WDIO env
  debug: false
}

[adzhanson]

Hi @kunagpal

I was just wondering if you had a chance to look into this?

[kunagpal]

Hey @adzhanson, apologies for the delay. After reading up on WDIO in more depth, I see that this is an attempt to get Newman to work in a browser environment. This is currently not supported (and probably never be), as Newman needs to have IO access (disk, network, etc) in order to work correctly.