Tests stop after initial test run when using browserstack conf files and --tag

[GrayedFox]

Using the provided examples (updated for ES6) I am running into a strange error, namely: only the first test that belongs to a group using the built in Nightwatch tag system will run, before the test runner stops and exits. However running the same set of tests locally (without browserstack) will properly run all tests that are tagged with the given tag.

For example, this, using browserstack, will only run the first test that belongs to a tagged group:

./localRunner --tag myTag --env chrome,firefox --config conf/browserstack-parallel-local.conf.js

Will run and complete the 1st test (on both browsers) which matches the given tag before exiting. I can see the results both in my local console and at browserstack.com/automate.

However running the same command using the built in test runner and without the browserstack config behaves as expected:

nightwatch --tag myTag --env chrome,firefox

Not sure how or why the Browserstack test runner differs exactly but I'm using the one provided. Posting here for clarity:

#!/usr/bin/env node

const Nightwatch = require('nightwatch')
const browserstack = require('browserstack-local')
let bsLocal

const startLocalBrowserstack = (localBrowserstackInstance) => {
  localBrowserstackInstance.start({ 'key': process.env.BROWSERSTACK_ACCESS_KEY, 'force': true }, (error) => {
    if (error) {
      throw error
    } else {
      Nightwatch.cli( (argv) => {
        Nightwatch.CliRunner(argv)
          .setup(null, () => {
            localBrowserstackInstance.stop( () => {} )
          })
          .runTests(() => {
            localBrowserstackInstance.stop( () => {} )
          })
      })
    }
  })
}

try {
  process.mainModule.filename = './node_modules/.bin/nightwatch'
  Nightwatch.bs_local = bsLocal = new browserstack.Local()
  startLocalBrowserstack(bsLocal)
} catch (err) {
  console.log('There was an error while starting the test runner:\n\n')
  process.stderr.write(err.stack + '\n')
  process.exit(2)
}

Adding my conf file for clarity too:

module.exports = (function(settings) {

  const selenium = {
    'start_process': false,
    'host': 'hub-cloud.browserstack.com',
    'port': 80
  }

  const commonCapabilities = {
    'build': 'nightwatch-browserstack',
    'browserstack.user': process.env.BROWSERSTACK_USERNAME,
    'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY,
    'acceptSslCerts': true,
    'browserstack.debug': true,
    'browserstack.local': true
  }

  const testWorkers = {
    'enabled': false,
    'workers': 'auto'
  }

  const testSettings = {
    default: {},
    chrome: {
      desiredCapabilities: {
        browser: 'chrome',
        browserName: 'Chrome',
        browser_version: '56.0',
        os: 'Windows',
        os_version: '7',
        resolution: '1024x768'
      }
    },
    firefox: {
      desiredCapabilities: {
        browser: 'firefox',
        browserName: 'Firefox',
        browser_version: '51.0',
        os: 'Windows',
        os_version: '7',
        resolution: '1024x768'
      }
    },
    safari: {
      desiredCapabilities: {
        browser: 'safari',
        browserName: 'Safari',
        browser_version: '9.1',
        os: 'OS X',
        os_version: 'El Capitan',
        resolution: '1024x768'
      }
    },
    ie: {
      desiredCapabilities: {
        browser: 'internet explorer',
        browserName: 'IE',
        browser_version: '11.0',
        os: 'Windows',
        os_version: '7',
        resolution: '1024x768'
      }
    }
  }

  for (const profile in testSettings) {
    testSettings[profile]['selenium_host'] = selenium.host
    testSettings[profile]['selenium_port'] = selenium.port
    testSettings[profile]['desiredCapabilities'] = testSettings[profile]['desiredCapabilities'] || {}
    for (const capability in commonCapabilities) {
      testSettings[profile]['desiredCapabilities'][capability] = testSettings[profile]['desiredCapabilities'][capability] || commonCapabilities[capability]
    }
  }

  settings.selenium = selenium
  settings.test_workers = testWorkers
  settings.test_settings = testSettings

  return settings

})(require('../nightwatch.json'))