search

webdriverio / webdriverio

Next-gen browser and mobile automation test framework for Node.js
http://webdriver.io
MIT License
9.04k stars 2.5k forks source link

hook title differs in reporter and config #6336

Closed sgordiev closed 1 year ago

sgordiev commented 3 years ago

[//]: # NOTE: This repository only maintains packages that are listed in the Readme. Please make sure that your issue is directly caused by one of these packages and if not file an issue in the correct 3rd party package repository.

Environment (please complete the following information):

Config of WebdriverIO

const getLogger = require('@wdio/logger').default;
const setScreenWidth = require('../../src/utils/helpers/screen');
const log = getLogger('root');
// const { TimelineService } = require('wdio-timeline-reporter/timeline-service');
log.setLevel('info');
const reportportal = require('../../src/utils/report-portal/reporter');
const RpService = require("../../src/utils/report-portal/service");
/**
* Loades all typedi containers before tests starts
*/
require('../../classloader');
const path = require('path');
const fs = require('fs');

//Report Portal configuration
const conf = {
  reportPortalClientConfig: { // report portal settings
    token: 'token,
    endpoint: 'endpoint,
    launch: 'lunch,
    project: 'project,
    mode: 'DEFAULT',
    debug: false,
    description: "Launch description text",
    attributes: [{key:"test_tag", value: "test_value"}],
    // headers: {"foo": "bar"} // optional headers for internal http client
  },
  reportSeleniumCommands: false, // add selenium commands to log
  seleniumCommandsLogLevel: 'debug', // log level for selenium commands
  autoAttachScreenshots: false, // automatically add screenshots
  screenshotsLogLevel: 'info', // log level for screenshots
  parseTagsFromTestTitle: false, // parse strings like `@foo` from titles and add to Report Portal
  cucumberNestedSteps: false, // report cucumber steps as Report Portal steps
  autoAttachCucumberFeatureToScenario: false, // requires cucumberNestedSteps to be true for use
  isSauseLabRun: false, // automatically add SauseLab ID to rp tags.
}
exports.common_config = {

  //
  // ====================
  // Runner Configuration
  // ====================

  services: [
    // [TimelineService],
    [RpService, {}]
  ],
  reporters: [
    'spec',
    // ['timeline',
    //   { outputDir: './reports',
    //     embedImages: false,
    //     screenshotStrategy: 'none'
    //   }],
      [reportportal, conf]
  ],
  specs: [
    './src/tests/**/*.js',
  ],
  suites: {
    smoke: [
      './src/tests/test.js'
    ],
  },

  //
  // ===================
  // Test Configurations
  // ===================
  // Define all options that are relevant for the WebdriverIO instance here
  //
  // Level of logging verbosity: trace | debug | info | warn | error | silent
  logLevel: 'info',

  // If you only want to run your tests until a specific amount of tests have failed use
  // bail (default is 0 - don't bail, run all tests).
  bail: 0,
  //
  // Set a base URL in order to shorten url command calls. If your `url` parameter starts
  // with `/`, the base url gets prepended, not including the path portion of your baseUrl.
  // If your `url` parameter starts without a scheme or `/` (like `some/path`), the base url
  // gets prepended directly.
  baseUrl: '0.0.0.0',
  //
  // Default timeout for all waitFor* commands.
  // waitforTimeout: 30000,
  //
  // Default timeout in milliseconds for request
  // if browser driver or grid doesn't send response
  connectionRetryTimeout: 240000,
  //
  // Default request retries count
  connectionRetryCount: 0,
  //
  // Test runner services
  // Services take over a specific job you don't want to take care of. They enhance
  // your test setup with almost no effort. Unlike plugins, they don't add new
  // commands. Instead, they hook themselves up into the test process.

  seleniumLogs: 'logs',
  framework: 'mocha',

  //
  // Options to be passed to Mocha.
  // See the full list at http://mochajs.org/
  mochaOpts: {
    ui: 'bdd',
    timeout: 3600000
  },

  /**
   * Gets executed before test execution begins. At this point you can access to all global
   * variables like `browser`. It is the perfect place to define custom commands.
   * @param {Array.<Object>} capabilities list of capabilities details
   * @param {Array.<String>} specs List of spec file paths that are to be run
   */
  before: async function (capabilities, specs) {
    /**
    * Update global object with chai bdd and lodash.
    */
    const chai = require('chai');
    const adsKeys = require('../../chai-plugins/ads-keys');
    chai.use(adsKeys);
    global.expect = chai.expect;
    await setScreenWidth(1350);
  },
  // afterTest: async function (test, context, { error, result, duration, passed, retries }) {
  //   log.info('passed', passed);
  //   log.info('error', error);
  //   if (error&&(error.message !=='sync skip; aborting execution')) {
  //     log.info('Gather information for error');
  //     const CONTAINER = require('typedi').Container;
  //     const rp = CONTAINER.get('ReportModel');
  //     await rp.addScreenShot('screenshot');
  //     // await rp.addHar('har');
  //     // await rp.addPageSource('source');
  //   }
  // },
  // afterHook: async function (test, context, { error, result, duration, passed, retries }/*, stepData, world*/) {
  //   if (error&&(error.message !=='sync skip; aborting execution')) {
  //     log.info('Gather information for error');
  //     const CONTAINER = require('typedi').Container;
  //     const rp = CONTAINER.get('ReportModel');
  //     await rp.addScreenShot('screenshot');
  //     // await rp.addHar('har');
  //     // await rp.addPageSource('source');
  //   }
  // },
  afterTest: async function (test, context, { error, result, duration, passed, retries }) {
    log.info('passed', passed);
    log.info('error', error);
    if (error&&(error.message !=='sync skip; aborting execution')) {
      log.info('Gather information for error');
      const filename = `screnshot+_${Date.now()}.png`;
      const outputFile = path.join(__dirname, filename);
      await browser.saveScreenshot(outputFile);
      await reportportal.sendFileToTest(test, `info`, filename, fs.readFileSync(outputFile));
      // await rp.addHar('har');
      // await rp.addPageSource('source');
    }
  },
  afterHook: async function (test, context, { error, result, duration, passed, retries }/*, stepData, world*/) {
    log.info('afterHook args',arguments);
    if (error&&(error.message !=='sync skip; aborting execution')) {
      log.info('Gather information for error');
      const filename = `screnshot_${Date.now()}.png`;
      const outputFile = path.join(__dirname, filename);
      await browser.saveScreenshot(outputFile);
      await reportportal.sendFile( 'info', filename, fs.readFileSync(outputFile));
      // const CONTAINER = require('typedi').Container;
      // const rp = CONTAINER.get('ReportModel');
      // await rp.addScreenShot('screenshot');
      // await rp.addHar('har');
      // await rp.addPageSource('source');
    }
  },
  onComplete: async function (exitCode, config, capabilities, results) {

  },

};

Describe the bug If create custom reporter and use 'onHookStart' 'onHookEnd' hook.title will be different then in 'afterHook' test.title

To Reproduce

src/utils/reporter/reporter.js

let Reporter = require( "@wdio/reporter").default;
class MyReporter extends Reporter {
   onHookStart(hook) {
     if(!hook.title.includes('root')){
          log.info('onHookStart ',hook.title);
     }
}

   onHookEnd(hook) {
     if(!hook.title.includes('root')){
          log.info('onHookEnd ',hook.title);
     }
}
}

config/wdio-conf/common.js


  afterHook: async function (test, context, { error, result, duration, passed, retries }/*, stepData, world*/) {
    log.info('afterHook title', test.title);
  },

Create custom reporter with code above and config with code above. Create a tests with 'after' hook. Run test and see that title in reporter and in afterHook of config won't match

[Include code or an example repository that can easily be set up]

Expected behavior titles should match

Log [0-0] 2021-01-21T18:47:14.326Z WARN wdio-reportportal-reporter: hook.title "after all" hook for Check ads on LiveCoveragePage auth [0-0] 2021-01-21T18:47:14.326Z WARN wdio-reportportal-reporter: test.title "after all" hook for "should have Symbol(AD_LIVECOVERAGE)" Additional context Add any other context about the problem here.

Please use the correct markdown

christian-bromann commented 3 years ago

@sgordiev can you provide a minimal reproducible example for this?

sgordiev commented 3 years ago

@sgordiev can you provide a minimal reproducible example for this?

I thought I gave the steps. Use report portal reporter it already exists, update its code in onHookEnd(hook){ if(!hook.title.includes('root')){ log.info('onHookEnd ',hook.title); }} to see hook title, create any simple test with after hook in mocha, in afterhock check context for hook name. One will see that in wdioconf.js afterhook its name differs from name in report onHookEnd. If you need code, than I need time, cause right now I am a bit busy with other tasks. Its one month passed since I created a ticket and management shifted me to another task

christian-bromann commented 3 years ago

I see, thanks! Any contributions or help on this would be appreciated.

christian-bromann commented 1 year ago

I did some investigations here and it seems that beforeHook and afterHook are called through the testFrameworkFnWrapper and basically wrap every hook called within a spec. The reporter hookStart and hookEnd events are based on Mocha specific events emitted by the Mocha runner.

My recommendation here: somehow pass in the worker reporter into testFrameworkFnWrapper and emit these events from the wrapped hook. Or maybe a simpler solution is to register these hooks by the worker reporter and propagate the same data as event.

christian-bromann commented 1 year ago

I will go ahead and close this due to inactivity. Hook information and reporter data come from different sources and can't be seen as equal. I am missing the use case here and a concrete reproducible example. Happy to re-open if there is any interest.