search

theintern / intern

A next-generation code testing stack for JavaScript.
https://theintern.io/
Other
4.36k stars 309 forks source link

Different behavior for `browserVersion` and `version` #1113

Open jason0x43 opened 4 years ago

jason0x43 commented 4 years ago

In Intern master, browserVersion and version are being handled differently, at least for MS Edge Chromium.

To test, create a test config with

"environments": {"browserName": "MicrosoftEdge", "version": "insider preview" }

Run npm test -- --config <your config> --show-config and note that the drivers list contains MicrosoftEdge.

Create or update a config to use "browserVersion": "insider preview", run the test command, and note that the drivers list contains MicrosoftEdgeChromium (which is what it's supposed to contain).

jason0x43 commented 4 years ago

This is also an issue with browserStack, at least with Intern 4.x. { "browserName": "safari", "browserVersion": "latest" } gives an OS/Browser combination invalid error, whereas { "browserName": "safari", "version": "latest" } works as intended.

samends commented 4 years ago

Hey @jason0x43 I created a PR to resolve this at #1117 however I'm unsure how to test to see if it worked. Where am I suppose to look for the driver list? I updated the intern.json file and ran npm test -- --show-config and got the output 

{
  "$schema": "./schemas/config.json",
  "bail": false,
  "basePath": "/Users/sam/projects/intern/",
  "baseline": false,
  "benchmark": false,
  "browser": {
    "plugins": [
      {
        "script": "tests/support/globalUi.ts"
      },
      {
        "script": "tests/support/browserDom.ts"
      }
    ],
    "reporters": [],
    "suites": [
      "tests/benchmark/example.ts",
      "tests/unit/common/lib/Evented.ts",
      "tests/unit/common/lib/global.ts",
      "tests/unit/common/lib/request.ts",
      "tests/unit/common/lib/util.ts",
      "tests/unit/core/lib/BenchmarkTest.ts",
      "tests/unit/core/lib/Channel.ts",
      "tests/unit/core/lib/Deferred.ts",
      "tests/unit/core/lib/Environment.ts",
      "tests/unit/core/lib/Suite.ts",
      "tests/unit/core/lib/Test.ts",
      "tests/unit/core/lib/browser/util.ts",
      "tests/unit/core/lib/channels/Base.ts",
      "tests/unit/core/lib/channels/Http.ts",
      "tests/unit/core/lib/channels/WebSocket.ts",
      "tests/unit/core/lib/common/ErrorFormatter.ts",
      "tests/unit/core/lib/common/console.ts",
      "tests/unit/core/lib/common/path.ts",
      "tests/unit/core/lib/common/util.ts",
      "tests/unit/core/lib/executors/Browser.ts",
      "tests/unit/core/lib/executors/Executor.ts",
      "tests/unit/core/lib/interfaces/bdd.ts",
      "tests/unit/core/lib/interfaces/benchmark.ts",
      "tests/unit/core/lib/interfaces/object.ts",
      "tests/unit/core/lib/interfaces/tdd.ts",
      "tests/unit/core/lib/reporters/Console.ts",
      "tests/unit/core/lib/reporters/Dom.ts",
      "tests/unit/core/lib/reporters/Html.ts",
      "tests/unit/core/loaders/default.ts",
      "tests/unit/core/loaders/dojo.ts",
      "tests/unit/core/loaders/dojo2.ts",
      "tests/unit/core/loaders/esm.ts",
      "tests/unit/core/loaders/systemjs.ts"
    ]
  },
  "capabilities": {
    "idle-timeout": 60,
    "name": "intern"
  },
  "connectTimeout": 30000,
  "coverage": [
    "!src/browser/**/*.ts",
    "src/**/*.ts"
  ],
  "coverageVariable": "__coverage__",
  "debug": false,
  "defaultTimeout": 30000,
  "description": "Run tests on Node, excluding the benchmark tests",
  "environments": [
    {
      "browserName": "node"
    }
  ],
  "filterErrorStack": false,
  "functionalCoverage": true,
  "functionalRetries": 1,
  "functionalSuites": [
    "tests/functional/lib/ProxiedSession.ts",
    "tests/functional/webdriver/Command.ts",
    "tests/functional/webdriver/Element.ts",
    "tests/functional/webdriver/Server.ts",
    "tests/functional/webdriver/Session.ts",
    "tests/functional/webdriver/helpers/pollUntil.ts",
    "tests/functional/webdriver/helpers/pollUntilTruthy.ts",
    "tests/functional/webdriver/support/util.ts"
  ],
  "functionalTimeouts": {},
  "grep": "/(?:)/",
  "heartbeatInterval": 60,
  "instrumenterOptions": {},
  "internPath": "/Users/sam/projects/intern/_testIntern/",
  "loader": {
    "script": "default"
  },
  "maxConcurrency": null,
  "name": "node",
  "node": {
    "plugins": [
      {
        "script": "tests/support/globalUi.ts"
      },
      {
        "script": "tests/support/nodeDom.ts"
      },
      {
        "script": "source-map-support/register"
      }
    ],
    "reporters": [
      {
        "name": "runner",
        "options": {
          "hidePassed": ".",
          "hideSkipped": false
        }
      }
    ],
    "suites": [
      "tests/benchmark/example.ts",
      "tests/integration/tunnels/BrowserStackTunnel.ts",
      "tests/integration/tunnels/CrossBrowserTestingTunnel.ts",
      "tests/integration/tunnels/SauceLabsTunnel.ts",
      "tests/integration/tunnels/SeleniumTunnel.ts",
      "tests/integration/tunnels/TestingBotTunnel.ts",
      "tests/unit/common/lib/Evented.ts",
      "tests/unit/common/lib/global.ts",
      "tests/unit/common/lib/request.ts",
      "tests/unit/common/lib/util.ts",
      "tests/unit/core/bin/intern.ts",
      "tests/unit/core/lib/BenchmarkTest.ts",
      "tests/unit/core/lib/Channel.ts",
      "tests/unit/core/lib/Deferred.ts",
      "tests/unit/core/lib/Environment.ts",
      "tests/unit/core/lib/ProxiedSession.ts",
      "tests/unit/core/lib/Server.ts",
      "tests/unit/core/lib/Suite.ts",
      "tests/unit/core/lib/Test.ts",
      "tests/unit/core/lib/channels/Base.ts",
      "tests/unit/core/lib/channels/Http.ts",
      "tests/unit/core/lib/channels/WebSocket.ts",
      "tests/unit/core/lib/common/ErrorFormatter.ts",
      "tests/unit/core/lib/common/console.ts",
      "tests/unit/core/lib/common/path.ts",
      "tests/unit/core/lib/common/util.ts",
      "tests/unit/core/lib/executors/Executor.ts",
      "tests/unit/core/lib/executors/Node.ts",
      "tests/unit/core/lib/interfaces/bdd.ts",
      "tests/unit/core/lib/interfaces/benchmark.ts",
      "tests/unit/core/lib/interfaces/object.ts",
      "tests/unit/core/lib/interfaces/tdd.ts",
      "tests/unit/core/lib/middleware/filterUrl.ts",
      "tests/unit/core/lib/middleware/finalError.ts",
      "tests/unit/core/lib/middleware/instrument.ts",
      "tests/unit/core/lib/middleware/post.ts",
      "tests/unit/core/lib/middleware/resolveSuites.ts",
      "tests/unit/core/lib/middleware/unhandled.ts",
      "tests/unit/core/lib/node/ErrorFormatter.ts",
      "tests/unit/core/lib/node/util.ts",
      "tests/unit/core/lib/reporters/Benchmark.ts",
      "tests/unit/core/lib/reporters/Cobertura.ts",
      "tests/unit/core/lib/reporters/Console.ts",
      "tests/unit/core/lib/reporters/Coverage.ts",
      "tests/unit/core/lib/reporters/HtmlCoverage.ts",
      "tests/unit/core/lib/reporters/JUnit.ts",
      "tests/unit/core/lib/reporters/JsonCoverage.ts",
      "tests/unit/core/lib/reporters/Lcov.ts",
      "tests/unit/core/lib/reporters/Pretty.ts",
      "tests/unit/core/lib/reporters/Reporter.ts",
      "tests/unit/core/lib/reporters/Runner.ts",
      "tests/unit/core/lib/reporters/Simple.ts",
      "tests/unit/core/lib/reporters/TeamCity.ts",
      "tests/unit/core/lib/reporters/TextCoverage.ts",
      "tests/unit/core/loaders/default.ts",
      "tests/unit/core/loaders/dojo.ts",
      "tests/unit/core/loaders/dojo2.ts",
      "tests/unit/core/loaders/esm.ts",
      "tests/unit/core/loaders/systemjs.ts",
      "tests/unit/core/tasks/intern.ts",
      "tests/unit/index.ts",
      "tests/unit/tunnels/BrowserStackTunnel.ts",
      "tests/unit/tunnels/CrossBrowserTestingTunnel.ts",
      "tests/unit/tunnels/SauceLabsTunnel.ts",
      "tests/unit/tunnels/SeleniumTunnel.ts",
      "tests/unit/tunnels/TestingBotTunnel.ts",
      "tests/unit/tunnels/Tunnel.ts",
      "tests/unit/tunnels/lib/util.ts",
      "tests/unit/webdriver/lib/util.ts"
    ],
    "tsconfig": "tsconfig.json"
  },
  "plugins": [],
  "reporters": [
    {
      "name": "runner"
    }
  ],
  "runInSync": false,
  "serveOnly": false,
  "serverPort": 9000,
  "serverUrl": "http://localhost:9000/",
  "sessionId": "",
  "showConfig": true,
  "socketPort": 9001,
  "tunnel": "selenium",
  "tunnelOptions": {
    "drivers": [],
    "tunnelId": "1582938871769"
  }
}
jason0x43 commented 4 years ago

tunnelOptions.drivers is the driver list. However, it won't be automatically populated if there are no browsers in the environments list, so you need to create a test config (or use an existing one) that has one or more browsers in the environments list.

You can verify the original issue by just providing an environment on the command line:

$ npm test -- --show-config --environments '{"browserName":"MicrosoftEdge", "version":"insider preview"}'

and

$ npm test -- --show-config --environments '{"browserName":"MicrosoftEdge", "browserVersion":"insider preview"}'

In current master, the first will give you a tunnelOptions.drivers value of

  "tunnelOptions": {
    "drivers": [
      {
        "name": "MicrosoftEdge"
      }
    ],
    "tunnelId": "1583422526883"
  }

while the second gives you

  "tunnelOptions": {
    "drivers": [
      {
        "name": "MicrosoftEdgeChromium"
      }
    ],
    "tunnelId": "1583422594091"
  }

With this PR, both commands return the expected driver of "MicrosoftEdgeChromium". 🎉