Question: Running tests for multiple view ports?

[jcraggs]

First off, this is really cool and easy to use software, great work!

The only issue I'm having is that I've assigned two viewport sizes in the backstop.js file but the tests undertaken by backstop only test both of the view sizes for one my test functions. The rest of the tests after the first one just look at the first viewport size in the array.

I call "await backstop(assert)" 4 times, but only once in each test function in the test file.

What am I doing wrong?

The sanitised output from the html_report config.js is that I have 5 tests (instead of the expected 8):

report({
  "testSuite": "BackstopJS",
  "tests": [
    {
      "pair": {
        "reference": "../bitmaps_reference/ember-backstoptest_Acceptance__login__user_can_visit_login__assert0_0_document_0_zebraDeviceView.png",
        "test": "../bitmaps_test/20200805-171235/ember-backstoptest_Acceptance__login__user_can_visit_login__assert0_0_document_0_zebraDeviceView.png",
        "fileName": "ember-backstoptest_Acceptance__login__user_can_visit_login__assert0_0_document_0_zebraDeviceView.png",
        "label": "Acceptance | login | user can visit login | assert0",
        "viewportLabel": "zebraDeviceView",
      },
      "status": "pass"
    },
    {
      "pair": {
        "reference": "../bitmaps_reference/ember-backstoptest_Acceptance__login__user_can_visit_login__assert0_0_document_1_webview.png",
        "test": "../bitmaps_test/20200805-171235/ember-backstoptest_Acceptance__login__user_can_visit_login__assert0_0_document_1_webview.png",
        "fileName": "ember-backstoptest_Acceptance__login__user_can_visit_login__assert0_0_document_1_webview.png",
        "label": "Acceptance | login | user can visit login | assert0",
        "viewportLabel": "webview",
      },
      "status": "pass"
    },
    {
      "pair": {
        "reference": "../bitmaps_reference/ember-backstoptest_Acceptance__login__error_message_displayed_when_invalid_credentials__assert0_0_document_0_zebraDeviceView.png",
        "test": "../bitmaps_test/20200805-171235/ember-backstoptest_Acceptance__login__error_message_displayed_when_invalid_credentials__assert0_0_document_0_zebraDeviceView.png",
        "fileName": "ember-backstoptest_Acceptance__login__error_message_displayed_when_invalid_credentials__assert0_0_document_0_zebraDeviceView.png",
        "label": "Acceptance | login | error message displayed when invalid credentials | assert0",
        "viewportLabel": "zebraDeviceView",
      },
      "status": "pass"
    },
    {
      "pair": {
        "reference": "../bitmaps_reference/ember-backstoptest_Acceptance__login__user_should_be_redirected_to_change-password_when_their_password_has_expired__assert0_0_document_0_zebraDeviceView.png",
        "test": "../bitmaps_test/20200805-171235/ember-backstoptest_Acceptance__login__user_should_be_redirected_to_change-password_when_their_password_has_expired__assert0_0_document_0_zebraDeviceView.png",
        "fileName": "ember-backstoptest_Acceptance__login__user_should_be_redirected_to_change-password_when_their_password_has_expired__assert0_0_document_0_zebraDeviceView.png",
        "label": "Acceptance | login | user should be redirected to change-password when their password has expired | assert0",
        "viewportLabel": "zebraDeviceView",
      },
      "status": "pass"
    },
    {
      "pair": {
        "reference": "../bitmaps_reference/ember-backstoptest_Acceptance__login__account_suspended_message_shows_when_user_account_has_been_suspended__assert0_0_document_0_zebraDeviceView.png",
        "test": "../bitmaps_test/20200805-171235/ember-backstoptest_Acceptance__login__account_suspended_message_shows_when_user_account_has_been_suspended__assert0_0_document_0_zebraDeviceView.png",
        "fileName": "ember-backstoptest_Acceptance__login__account_suspended_message_shows_when_user_account_has_been_suspended__assert0_0_document_0_zebraDeviceView.png",
        "label": "Acceptance | login | account suspended message shows when user account has been suspended | assert0",
        "viewportLabel": "zebraDeviceView",
      },
      "status": "pass"
    }
  ],
  "id": "ember-backstop test"
});

Backstop.js file with the two viewports defined:

module.exports = {
  id: `ember-backstop test`,
  viewports: [
    {
      label: 'zebraDeviceView',
      width: 360,
      height: 570,
    },
    {
      label: 'webview',
      width: 1440,
      height: 900,
    }
  ],
  onReadyScript: `puppet/onReady.js`,
  scenarios: [
    {
      label: '{testName}',
      url: '{origin}/backstop/dview/{testId}/{scenarioId}',
      delay: 500,
    },
  ],
  paths: {
    bitmaps_reference: 'backstop_data/bitmaps_reference',
    bitmaps_test: 'backstop_data/bitmaps_test',
    engine_scripts: 'backstop_data/engine_scripts',
    html_report: 'backstop_data/html_report',
    ci_report: 'backstop_data/ci_report',
  },
  report: [],
  engine: 'puppet',
  engineOptions: {
    args: ['--no-sandbox'],
  },
  asyncCaptureLimit: 10,
  asyncCompareLimit: 50,
  debug: false,
  debugWindow: false,
};

[garris]

Hi -- sorry for the late response! Unfortunately, ember-backstop doesn't allow setting viewports like this because the viewport also relies on Testem's settings. You would have to change the viewport settings in your ember test env and then pass that viewport state in your backstop(assert, { scenario: { viewports: ... } });

[jcraggs]

Thanks for this Garris, its now working after passing in the viewport state. Also removed the individual objects from the backstop.js viewports array as going forward I will define the specific screen size I want testing on each call of the backstop(assert ...).

For others benefit, essentially I had to call the await backstop for each screen size I wanted to test e.g. to test two screen sizes:

await backstop(assert, {
  scenario: {
    "viewports": [
      {
         "label": 'webview',
         "width": 1440,
          "height": 900
       }
     ]
   }
});

await backstop(assert, {
  scenario: {
    "viewports": [
      {
         "label": 'zebraDeviceView',
         "width": 360,
          "height": 570
       }
     ]
   }
});

For cleaner code you can also import those viewport objects from a separately defined js file e.g.:

So now the test module file has this:

import viewports from '../viewports';
...

await backstop(assert, { scenario: viewports.webview })
await backstop(assert, { scenario: viewports.zebraDeviceView })

in a new viewports.js file you'll need this:

export default {
  webview: {
    "viewports": [
      {
        "label": 'webview',
        "width": 1440,
        "height": 900
      }
    ]
  },
  zebraDeviceView: {
    "viewports": [
      {
        "label": 'zebraDeviceView',
        "width": 360,
        "height": 570
      }
    ]
  }
};