search

codeceptjs / CodeceptJS

Supercharged End 2 End Testing Framework for NodeJS
http://codecept.io
MIT License
4.11k stars 725 forks source link

tryTo does not appear on Allure report #3005

Closed nelsonthedev closed 1 year ago

nelsonthedev commented 3 years ago

What are you trying to achieve?

I want to indicate that a step has failed in a tryTo session, to let the user know that this step is not necessarily required

What do you get instead?

I get a failed step only, which makes confusing a report which says a test passed but has failed steps. Also, the failed tryTo step does not indicate from which step it belongs to:

image

Provide console output if related. Use --verbose mode for more details.

➜  ump git:(develop) ✗ yarn run test:e2e-ios
yarn run v1.22.10
$ npx codeceptjs run -c codecept.conf.ios.js --verbose -p allure
CodeceptJS v3.0.7
Helpers: DeviceEventsHelper, SwipeHelper
Plugins: screenshotOnFail, retryFailedStep, allure, tryTo

uMP Smoke Tests --
    [1]  Starting recording promises
  Launch the app and use main screens
    setupView: isLoaded 
      I wait for visible {"ios":"~setupHeader","android":"#setupHeader"}, 25
      I wait for visible {"ios":"~nextBtn","android":"#nextBtn"}, 25
    setupView: save 
      I wait for element {"ios":"~nextBtn","android":"#nextBtn"}, 25
      I tap {"ios":"~nextBtn","android":"#nextBtn"}
    setupView: notifications 
      I wait for element {"ios":"~notificationNoButton","android":""}
      I tap {"ios":"~notificationNoButton","android":""}
    homeView: isLoaded 
      I wait for visible {"ios":"~ajHeaderLogo","android":"#ajHeaderLogo"}, 25
      I wait for visible {"ios":"~Top Stories","android":"#Top Stories"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"postTitle\"])[1]","android":"//*[@resource-id=\"postTitle\"]"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"postDate\"])[1]","android":"//*[@resource-id=\"postDate\"]"}, 25
    menuTabView: isLoaded 
      I wait for visible {"ios":"~homeTab","android":"#homeTab"}, 25
      I wait for visible {"ios":"~watchTab","android":"#watchTab"}, 25
      I wait for visible {"ios":"~sectionsTab","android":"#sectionsTab"}, 25
    [1] Starting <tryTo> session
    I wait for visible {"ios":"(//XCUIElementTypeOther[@name=\"playIcon\"])[1]","android":"//*[@resource-id=\"playIcon\"]"}
    [1] <tryTo> Error | Error: element ((//XCUIElementTypeOther[@name="playIcon"])[1]) still not visible after 1 sec
 › Unsuccesful try > Error: element ((//XCUIElementTypeOther[@name="playIcon"])[1]) still not visible after 1 sec
    [1] <tryTo> Finalize <tryTo> session
    homeView: openPost 
      I tap {"ios":"(//XCUIElementTypeStaticText[@name=\"postTitle\"])[1]","android":"//*[@resource-id=\"postTitle\"]"}
    postView: tapToCloseDismiss 
      I wait for visible {"ios":"~tapToCloseLabel","android":"#tapToCloseLabel"}, 25
      I tap {"ios":"~tapToCloseLabel","android":"#tapToCloseLabel"}
    postView: isLoaded , , "home"
      I wait for visible {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}, 25
      I wait for visible {"ios":"~imageCaption","android":"#imageCaption"}, 25
      I wait for visible {"ios":"//XCUIElementTypeStaticText[@name=\"postTitle\"]","android":"//*[@resource-id=\"postTitle\"]"}, 25
      I wait for visible {"ios":"~postExcerpt","android":"#postExcerpt"}, 25
      I swipe to element {"ios":"~postDate","android":"#postDate"}, , {"ios":"~homeTab","android":"#homeTab"}
      I wait for visible {"ios":"~postDate","android":"#postDate"}, 25
      I swipe to element {"ios":"(//XCUIElementTypeButton[@name=\"shareButton\"])[1]","android":"//*[@resource-id=\"shareButton\"]"}, , {"ios"…
      I wait for visible {"ios":"(//XCUIElementTypeButton[@name=\"shareButton\"])[1]","android":"//*[@resource-id=\"shareButton\"]"}, 25
      I swipe to element {"ios":"~postContent","android":"#postContent"}, , {"ios":"~homeTab","android":"#homeTab"}
      I wait for visible {"ios":"~postContent","android":"#postContent"}, 25
    menuTabView: navigateTo "SECTIONS"
      I wait for element {"ios":"~sectionsTab","android":"#sectionsTab"}
      I tap {"ios":"~sectionsTab","android":"#sectionsTab"}
    sectionsView: isLoaded 
      I wait for visible {"ios":"~searchInput","android":"//*[@class=\"android.widget.EditText\"]"}, 25
      I wait for visible {"ios":"~searchIcon","android":"#searchIcon"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"sectionTitle\"])[1]","android":"//*[@resource-id=\"sectionTitle\"]"}, 25…
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"sectionPostTitle\"])[1]","android":"//*[@resource-id=\"sectionPostTitle\…
      I wait for visible {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}, 25
    sectionsView: openTopSection 
      I tap {"ios":"(//XCUIElementTypeStaticText[@name=\"sectionTitle\"])[1]","android":"//*[@resource-id=\"sectionTitle\"]"}
    sectionView: isLoaded 
      I wait for visible {"ios":"~Back","android":"//android.widget.ImageButton[@content-desc=\"Navigate up\"]"}, 25
      I wait for visible {"ios":"//XCUIElementTypeNavigationBar[@name]","android":""}, 25
      I wait for visible {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"postTitle\"])[1]","android":"//*[@resource-id=\"postTitle\"]"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"postDate\"])[1]","android":"//*[@resource-id=\"postDate\"]"}, 25
    [1] Starting <tryTo> session
    I wait for visible {"ios":"(//XCUIElementTypeOther[@name=\"playIcon\"])[1]","android":"//*[@resource-id=\"playIcon\"]"}
    [1] <tryTo> Error | Error: element ((//XCUIElementTypeOther[@name="playIcon"])[1]) still not visible after 1 sec
 › Unsuccesful try > Error: element ((//XCUIElementTypeOther[@name="playIcon"])[1]) still not visible after 1 sec
    [1] <tryTo> Finalize <tryTo> session
    sectionView: openPost 
      I tap {"ios":"(//XCUIElementTypeStaticText[@name=\"postTitle\"])[1]","android":"//*[@resource-id=\"postTitle\"]"}
    postView: isLoaded true, , "section"
      I wait for visible {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}, 25
      I wait for visible {"ios":"~imageCaption","android":"#imageCaption"}, 25
      I wait for visible {"ios":"//XCUIElementTypeStaticText[@name=\"postTitle\"]","android":"//*[@resource-id=\"postTitle\"]"}, 25
      I wait for visible {"ios":"~postExcerpt","android":"#postExcerpt"}, 25
      I swipe to element {"ios":"~postDate","android":"#postDate"}, {"ios":"//XCUIElementTypeNavigationBar[@name]","android":"//android.widget…
      I wait for visible {"ios":"~postDate","android":"#postDate"}, 25
      I swipe to element {"ios":"(//XCUIElementTypeButton[@name=\"shareButton\"])[1]","android":"//*[@resource-id=\"shareButton\"]"}, {"ios":"…
      I wait for visible {"ios":"(//XCUIElementTypeButton[@name=\"shareButton\"])[1]","android":"//*[@resource-id=\"shareButton\"]"}, 25
      I swipe to element {"ios":"~postContent","android":"#postContent"}, {"ios":"//XCUIElementTypeNavigationBar[@name]","android":"//android.…
    [1] Retrying... Attempt #2
      I wait for visible {"ios":"~postContent","android":"#postContent"}, 25
    postView: back 
      I tap {"ios":"~Back","android":"//android.widget.ImageButton[@content-desc=\"Navigate up\"]"}
    sectionsView: isLoaded 
      I wait for visible {"ios":"~searchInput","android":"//*[@class=\"android.widget.EditText\"]"}, 25
      I wait for visible {"ios":"~searchIcon","android":"#searchIcon"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"sectionTitle\"])[1]","android":"//*[@resource-id=\"sectionTitle\"]"}, 25…
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"sectionPostTitle\"])[1]","android":"//*[@resource-id=\"sectionPostTitle\…
      I wait for visible {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}, 25
    searchView: searchFor "Tesla beats earnings forecasts despite supply chain crisis"
      I fill field {"ios":"~searchInput","android":"//*[@class=\"android.widget.EditText\"]"}, "Tesla beats earnings forecasts despite supply …
      I tap {"ios":"~searchIcon","android":"#searchIcon"}
      I tap {"ios":"~searchIcon","android":"#searchIcon"}
    searchResultView: isLoaded 
      I wait for visible {"ios":"~Back","android":"//android.widget.ImageButton[@content-desc=\"Navigate up\"]"}, 25
      I wait for visible {"ios":"//XCUIElementTypeNavigationBar[@name]","android":""}, 25
      I wait for visible {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"postTitle\"])[1]","android":"//*[@resource-id=\"postTitle\"]"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"postDate\"])[1]","android":"//*[@resource-id=\"postDate\"]"}, 25
    [1] Starting <tryTo> session
    I wait for visible {"ios":"(//XCUIElementTypeOther[@name=\"playIcon\"])[1]","android":"//*[@resource-id=\"playIcon\"]"}
    [1] <tryTo> Finalize <tryTo> session
    searchResultView: openPost true
      I tap {"ios":"(//XCUIElementTypeOther[@name=\"playIcon\"])[1]","android":"//*[@resource-id=\"playIcon\"]"}
    [1] Starting <tryTo> session
    I wait for element {"ios":"~loadingAnimation","android":"//*[@resource-id=\"loadingAnimation\"]"}, 25
    I wait for invisible {"ios":"~loadingAnimation","android":"//*[@resource-id=\"loadingAnimation\"]"}, 25
    [1] <tryTo> Finalize <tryTo> session
    postView: isLoaded , true, "watch"
      I wait for visible {"ios":"~videoPlayer","android":"#videoPlayer"}, 25
      I tap {"ios":"~videoPlayer","android":"#videoPlayer"}
      I wait for visible {"ios":"~mute/unmute","android":"//android.widget.Button[@content-desc=\"mute/unmute\"]"}, 25
      I tap {"ios":"~mute/unmute","android":"//android.widget.Button[@content-desc=\"mute/unmute\"]"}
      I wait for visible {"ios":"~imageCaption","android":"#imageCaption"}, 25
      I wait for visible {"ios":"//XCUIElementTypeStaticText[@name=\"postTitle\"]","android":"//*[@resource-id=\"postTitle\"]"}, 25
      I wait for visible {"ios":"~postExcerpt","android":"#postExcerpt"}, 25
      I swipe to element {"ios":"~postDate","android":"#postDate"}, {"ios":"//XCUIElementTypeNavigationBar[@name]","android":"//android.widget…
      I wait for visible {"ios":"~postDate","android":"#postDate"}, 25
      I swipe to element {"ios":"(//XCUIElementTypeButton[@name=\"shareButton\"])[1]","android":"//*[@resource-id=\"shareButton\"]"}, {"ios":"…
      I wait for visible {"ios":"(//XCUIElementTypeButton[@name=\"shareButton\"])[1]","android":"//*[@resource-id=\"shareButton\"]"}, 25
      I swipe to element {"ios":"~postContent","android":"#postContent"}, {"ios":"//XCUIElementTypeNavigationBar[@name]","android":"//android.…
      I wait for visible {"ios":"~postContent","android":"#postContent"}, 25
    postView: back 
      I tap {"ios":"~Back","android":"//android.widget.ImageButton[@content-desc=\"Navigate up\"]"}
    sectionsView: isLoaded 
      I wait for visible {"ios":"~searchInput","android":"//*[@class=\"android.widget.EditText\"]"}, 25
      I wait for visible {"ios":"~searchIcon","android":"#searchIcon"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"sectionTitle\"])[1]","android":"//*[@resource-id=\"sectionTitle\"]"}, 25…
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"sectionPostTitle\"])[1]","android":"//*[@resource-id=\"sectionPostTitle\…
      I wait for visible {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}, 25
      I tap {"ios":"~Back","android":"//android.widget.ImageButton[@content-desc=\"Navigate up\"]"}
    menuTabView: navigateTo "WATCH"
      I wait for element {"ios":"~watchTab","android":"#watchTab"}
      I tap {"ios":"~watchTab","android":"#watchTab"}
    watchView: isLoaded 
      I wait for visible {"ios":"~videoPlayer","android":"#videoPlayer"}, 25
      I wait for visible {"ios":"~liveIconContainer","android":"#liveIconContainer"}, 25
      I wait for visible {"ios":"~currentProgramText","android":"#currentProgramText"}, 25
      I wait for visible {"ios":"~nextProgramStartText","android":"#nextProgramStartText"}, 25
      I wait for visible {"ios":"~nextProgramNameText","android":"#nextProgramNameText"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeStaticText[@name=\"Latest Videos\"])","android":"#Latest Videos"}, 25
      I wait for visible {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}, 25
    watchView: openTopPost 
      I tap {"ios":"(//XCUIElementTypeImage[@name=\"postImage\"])[1]","android":"//*[@resource-id=\"postImage\"]"}
    [1] Starting <tryTo> session
    I wait for element {"ios":"~loadingAnimation","android":"//*[@resource-id=\"loadingAnimation\"]"}, 25
    I wait for invisible {"ios":"~loadingAnimation","android":"//*[@resource-id=\"loadingAnimation\"]"}, 25
    [1] <tryTo> Finalize <tryTo> session
    postView: isLoaded , true, "watch"
      I wait for visible {"ios":"(//XCUIElementTypeOther[@name=\"videoPlayer\"])[2]","android":"#videoPlayer"}, 25
      I tap {"ios":"(//XCUIElementTypeOther[@name=\"videoPlayer\"])[2]","android":"#videoPlayer"}
      I wait for visible {"ios":"~mute/unmute","android":"//android.widget.Button[@content-desc=\"mute/unmute\"]"}, 25
      I tap {"ios":"~mute/unmute","android":"//android.widget.Button[@content-desc=\"mute/unmute\"]"}
      I wait for visible {"ios":"~imageCaption","android":"#imageCaption"}, 25
      I wait for visible {"ios":"//XCUIElementTypeStaticText[@name=\"postTitle\"]","android":"//*[@resource-id=\"postTitle\"]"}, 25
      I wait for visible {"ios":"~postExcerpt","android":"#postExcerpt"}, 25
      I swipe to element {"ios":"~postDate","android":"#postDate"}, , {"ios":"~homeTab","android":"#homeTab"}
      I wait for visible {"ios":"~postDate","android":"#postDate"}, 25
      I swipe to element {"ios":"(//XCUIElementTypeButton[@name=\"shareButton\"])[1]","android":"//*[@resource-id=\"shareButton\"]"}, , {"ios"…
      I wait for visible {"ios":"(//XCUIElementTypeButton[@name=\"shareButton\"])[1]","android":"//*[@resource-id=\"shareButton\"]"}, 25
      I swipe to element {"ios":"~postContent","android":"#postContent"}, , {"ios":"~homeTab","android":"#homeTab"}
      I wait for visible {"ios":"~postContent","android":"#postContent"}, 25
  ✔ OK in 131219ms

  OK  | 1 passed   // 2m
✨  Done in 145.03s.

Provide test source code if related

Scenario('Launch the app and use main screens', async () => {
  let isVideoPost: boolean = false;
  //SETUP
  setupView.isLoaded();
  setupView.save();
  setupView.notifications(false);

  //HOME
  homeView.isLoaded();
  menuTabView.isLoaded();

  isVideoPost = await homeView.hasVideoPostVisible();

  homeView.openPost(isVideoPost);

  //POST
  postView.tapToCloseDismiss();

  postView.isLoaded(false, isVideoPost, 'home');

  //SECTIONS
  menuTabView.navigateTo('SECTIONS');
  sectionsView.isLoaded();

  //SECTION
  sectionsView.openTopSection();
  sectionView.isLoaded();
  isVideoPost = await sectionView.hasVideoPostVisible();
  sectionView.openPost(isVideoPost);
  postView.isLoaded(true, isVideoPost, 'section');
  postView.back();
  sectionsView.isLoaded();

  //SEARCH
  searchView.searchFor(
    'Tesla beats earnings forecasts despite supply chain crisis',
  );
  searchResultView.isLoaded();
  isVideoPost = await searchResultView.hasVideoPostVisible();
  searchResultView.openPost(isVideoPost);
  postView.isLoaded(true, isVideoPost, 'search');
  postView.back();
  sectionsView.isLoaded();

  //WATCH
  menuTabView.navigateTo('WATCH');

  isVideoPost = true;
  watchView.isLoaded();
  watchView.openTopPost();
  postView.isLoaded(false, isVideoPost, 'watch');
});
const {I} = inject();

const tryTo = codeceptjs.container.plugins('tryTo');

export = {
  fields: {
    ajHeaderLogo: {ios: '~ajHeaderLogo', android: '#ajHeaderLogo'},
    topStoriesLabel: {ios: '~Top Stories', android: '#Top Stories'},
    postImage: {
      ios: '(//XCUIElementTypeImage[@name="postImage"])[1]',
      android: '//*[@resource-id="postImage"]',
    },
    postTitle: {
      ios: '(//XCUIElementTypeStaticText[@name="postTitle"])[1]',
      android: '//*[@resource-id="postTitle"]',
    },
    postDate: {
      ios: '(//XCUIElementTypeStaticText[@name="postDate"])[1]',
      android: '//*[@resource-id="postDate"]',
    },
    playIcon: {
      ios: '(//XCUIElementTypeOther[@name="playIcon"])[1]',
      android: '//*[@resource-id="playIcon"]',
    },
  },

  isLoaded() {
    I.waitForVisible(this.fields.ajHeaderLogo, 25);
    I.waitForVisible(this.fields.topStoriesLabel, 25);
    I.waitForVisible(this.fields.postImage, 25);
    I.waitForVisible(this.fields.postTitle, 25);
    I.waitForVisible(this.fields.postDate, 25);
  },

  hasVideoPostVisible() {
    return tryTo(() => {
      I.waitForVisible(this.fields.playIcon);
    });
  },

  openPost(isVideo: boolean) {
    if (isVideo) {
      I.tap(this.fields.playIcon);
    } else {
      I.tap(this.fields.postTitle);
    }
  },
};

Details

require('ts-node/register');

exports.config = {
  tests: './__tests__/e2e/*_test.ts',
  output: './__tests__/e2e/output',
  helpers: {
    DeviceEventsHelper: {
      require: './__tests__/e2e/helpers/device-events-helper.js',
      app: '',
      platform: 'iOS',
      device: 'iPhone 11',
      desiredCapabilities: {
        automationName: 'XCUITest',
        platformVersion: '13.3',
      },
    },
    SwipeHelper: {
      require: './__tests__/e2e/helpers/swipe-helper.js',
    },
  },
  include: {
    setupView: './__tests__/e2e/views/setup-view.ts',
    homeView: './__tests__/e2e/views/home-view.ts',
    menuTabView: './__tests__/e2e/views/menu-tab-view.ts',
    postView: './__tests__/e2e/views/post-view.ts',
    watchView: './__tests__/e2e/views/watch-view.ts',
    sectionsView: './__tests__/e2e/views/sections-view.ts',
    sectionView: './__tests__/e2e/views/section-view.ts',
    settingsView: './__tests__/e2e/views/settings-view.ts',
    searchView: './__tests__/e2e/views/search-view.ts',
    searchResultView: './__tests__/e2e/views/search-result-view.ts',
  },
  bootstrap: null,
  mocha: {},
  plugins: {
    pauseOnFail: {},
    retryFailedStep: {
      enabled: true,
      retries: 20,
    },
    allure: {
      outputDir: '__tests__/e2e/output/report',
      enabled: true,
    },
    tryTo: {
      enabled: true,
    },
    screenshotOnFail: {
      enabled: true,
      outputDir: '__tests__/e2e/output/report',
    },
  },
};
kobenguyent commented 1 year ago

allure plugin is now maintained by allure team https://github.com/allure-framework/allure-js/tree/master/packages/allure-codeceptjs