issues with sendLogToTest and sendFileToTest

igerasimov23 commented 5 years ago

The problem

when usingsendLogToTest and sendFileToTest data not always displays in reportportal(roughly 20% of the time does not displays)

Environment

WebdriverIO version: v.4
wdio-reportportal-version version: 0.0.17
Node.js version: 8.1.4
WDIO Testrunner and version(mocha\cucumber\jasmine): mocha
Additional wdio packages used (if applicable):

Details

I started to see when whole suite execuites, and using in afterTest sendLogToTest and sendFileToTest data does not displays on reportportal(roughly 20% of the time )

wdio.conf.js

const util = require('./utils');
const globals = require('./globals');

const reporter = require('wdio-reportportal-reporter');

const chai = require('chai');
const assertArrays = require('chai-arrays');
const assertions = require('./customCommands/assertions');

exports.config = {
 host: util.setHost(),
 port: 4444,
 path: '/wd/hub',

 sync: true,
 //
 // Level of logging verbosity: silent | verbose | command | data | result | error
 logLevel: 'data',
 //
 // Enables colors for log output.
 coloredLogs: true,
 //
 // Warns when a deprecated command is used
 deprecationWarnings: true,
 bail: 0,
 //
 baseUrl: util.setUrl(),
 //
 // Default timeout for all waitFor* commands.
 waitforTimeout: globals.baseConfig.waitForTimeout,

 // Default timeout in milliseconds for request
 // if Selenium Grid doesn't send response
 connectionRetryTimeout: process.env.CONNECTRETRYTIMEOUT || 90000,
 //
 // Default request retries count
 connectionRetryCount: 3,

 framework: 'mocha',
 maxInstances: process.env.PARALLELTHREADS || 20,

 capabilities: [
   {
     userCred: globals.userCredArray.cred9,
     version: '',
     browserName: 'chrome',

     specs: [
       './test/features/footer/test-footer.js'
     ],
   }
 ],

 services: [ 'selenium-standalone'],
 reporters: ['reportportal'],
 reporterOptions: {
   reportportal: {
     rpConfig: {
       token: process.env.RP_UUID,
       endpoint: 'https://testreportingsystem.nyt.net/api/v1',
       launch: process.env.JOB_NAME,
       project: 'cooking',
       mode: 'DEFAULT',
       debug: false

     },
     enableSeleniumCommandReporting: true,
     enableScreenshotsReporting: false,
     seleniumCommandsLogLevel: 'debug',
     screenshotsLogLevel: 'info',
     enableRetriesWorkaround: true,
     parseTagsFromTestTitle: true,
   }
 },

 mochaOpts: {
   timeout: process.env.MOCHATIMEOUT || 900000,
   ui: 'bdd',
   grep: process.env.npm_config_grep,
   compilers: ['js:babel-register'],
 },

 beforeSession: function (config, capabilities, specs) {
   //runs block for every capability
   exports.userCred = capabilities.userCred;
   // get sessions and user for parallel in the future

 },

 before: function () {
   chai.config.includeStack = true;
   global.expect = chai.expect;
   global.AssertionError = chai.AssertionError;
   global.Assertion = chai.Assertion;
   global.assert = assertions;

   let options = {defaultWait: 5000};
   let chaiWebdriver = require('chai-webdriverio').default;
   chai.use(chaiWebdriver(browser, options));
   chai.use(assertArrays);

 },

 beforeSuite: function (suite) {

   //setting viewport size
   util.setViewportSize();
 },

 afterTest: function (test) {
   // logging Pass or Fail for test
   if (test.passed) {
     let passMsg = `******* TEST '${test.title}' PASSED ******* `;
     console.log('info', passMsg);
     reporter.sendLogToTest(test, 'info', passMsg)
   } else {
     let failMsg = `******* TEST '${test.title}' FAILED ******* `;
     console.log(failMsg);
     reporter.sendLogToTest(test, 'info', failMsg);
     const screenshot =  browser.saveScreenshot();
     reporter.sendFileToTest(test, 'info', `${test.title}.png`, screenshot);
   }

 },

onComplete: async function (exitCode, config, capabilities) {
    return new Promise(resolve => setTimeout(resolve, 5000));
  }
};

BorisOsipov commented 5 years ago

@igerasimov23

wdio-reportportal-version version: 0.0.17

1) Can you updates to the latest version and check? 2) can you enable debug logs and review tests stdout logs for warn messages like https://github.com/BorisOsipov/wdio-reportportal-reporter/blob/master/lib/reporter.ts#L280 I need to know is there such log messages for items with missing logs on rp server.

Debug logs can be enabled by config

 reporterOptions: {
   reportportal: {
    debug: true,
...

igerasimov23 commented 5 years ago

After 2 hours narrowing down why we don't get screenshot when tests fails. I finally found issue: looks like if test case name has symbol / it somehow will not post a screenshot: Example:

describe('test free trial permissions', function () {
  this.retries(1);

  it('Verify saving/unsaving a recipe from RDP for FT user -env prod -feature freetrial', function () {
    browser.navigateTo('https://www.google.com');
    assert.isTrue(false, 'test should fail!!!!!')
  });
});

Note: debugging did not show anything

Start launch 42iggtgg2cjr8adsss
This test will run on the local selenium node.
[16:43:33]  DATA        {"desiredCapabilities":{"javascriptEnabled":true,"locationContextEnabled":true,"handlesAlerts":true,"rotatable":true,"browserName":"chrome","platform":"mac","version":"","build":"WebdriverioCooking : Tue Jan 22 2019 16:43:30 GMT-0500 (EST)","parentTunnel":"SauceRoot","tunnelIdentifier":"timestunnel","_":"8 more keys: [\"extendedDebugging\",\"commandTimeout\",\"idleTimeout\",\"screenResolution\",\"userCred\",\"specs\",\"loggingPrefs\",\"requestOrigins\"]"}}
[16:43:35]  DATA        {"type":"page load","ms":120000}
Start test item 42iggtgg2cjr8adwwn
Navigating to url: 'https://www.google.com'
[16:43:35]  DATA        {"url":"https://www.google.com/"}
Success start item 42iggtgg2cjr8adwwn
Start test item 42iggtgg2cjr8adwwy
Success start item 42iggtgg2cjr8adwwy
Save log 42iggtgg2cjr8adwx0
Save log 42iggtgg2cjr8adwx2
Successfully save log 42iggtgg2cjr8adwx0
Successfully save log 42iggtgg2cjr8adwx2
******* TEST 'Verify saving/unsaving a recipe from RDP for FT user -env prod -feature freetrial' FAILED *******
FSave log 42iggtgg2cjr8adxbh
[16:43:36]  DATA        {}
Save log 42iggtgg2cjr8adxbj
Successfully save log 42iggtgg2cjr8adxbh
Finish test item 42iggtgg2cjr8adwwy
Successfully save log 42iggtgg2cjr8adxbj
Save log 42iggtgg2cjr8adxpm
[16:43:36]  DATA        {}
------------------------------------------------------------------
[chrome  mac #0-0] Session ID: dd0f63c0729ecdc0d200daf1f45d8ce5
[chrome  mac #0-0] Spec: /Users/208821/Projects/np-cooking/api/test/e2e/test/users/free-trial/test-free-trial-permissions.js
[chrome  mac #0-0] Running: chrome on mac
[chrome  mac #0-0]
[chrome  mac #0-0] test free trial permissions
[chrome  mac #0-0]   1) Verify saving/unsaving a recipe from RDP for FT user -env prod -feature freetrial
[chrome  mac #0-0]
[chrome  mac #0-0]
[chrome  mac #0-0] 1 failing (3s)
[chrome  mac #0-0]
[chrome  mac #0-0] 1) test free trial permissions Verify saving/unsaving a recipe from RDP for FT user -env prod -feature freetrial:
[chrome  mac #0-0] test should fail!!!!!: expected false to be true
[chrome  mac #0-0] AssertionError: test should fail!!!!!: expected false to be true
[chrome  mac #0-0]     at Assertion.<anonymous> (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/chai/lib/chai/core/assertions.js:659:10)
[chrome  mac #0-0]     at Assertion.propertyGetter (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/chai/lib/chai/utils/addProperty.js:62:29)
[chrome  mac #0-0]     at Object.get (<anonymous>)
[chrome  mac #0-0]     at Object.proxyGetter [as get] (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/chai/lib/chai/utils/proxify.js:86:22)
[chrome  mac #0-0]     at Function.assert.isTrue (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/chai/lib/chai/interface/assert.js:335:52)
[chrome  mac #0-0]     at Object.exports.isTrue (/Users/208821/Projects/np-cooking/api/test/e2e/customCommands/assertions.js:38:15)
[chrome  mac #0-0]     at Context.<anonymous> (/Users/208821/Projects/np-cooking/api/test/e2e/test/users/free-trial/test-free-trial-permissions.js:8:12)
[chrome  mac #0-0]     at new Promise (<anonymous>)
[chrome  mac #0-0]     at new F (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/core-js/library/modules/_export.js:36:28)
[chrome  mac #0-0]

0 passing (6.70s)
1 failing

1) test free trial permissions Verify saving/unsaving a recipe from RDP for FT user -env prod -feature freetrial:
test should fail!!!!!: expected false to be true
running chrome on mac
AssertionError: test should fail!!!!!: expected false to be true
    at Assertion.<anonymous> (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/chai/lib/chai/core/assertions.js:659:10)
    at Assertion.propertyGetter (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/chai/lib/chai/utils/addProperty.js:62:29)
    at Object.get (<anonymous>)
    at Object.proxyGetter [as get] (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/chai/lib/chai/utils/proxify.js:86:22)
    at Function.assert.isTrue (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/chai/lib/chai/interface/assert.js:335:52)
    at Object.exports.isTrue (/Users/208821/Projects/np-cooking/api/test/e2e/customCommands/assertions.js:38:15)
    at Context.<anonymous> (/Users/208821/Projects/np-cooking/api/test/e2e/test/users/free-trial/test-free-trial-permissions.js:8:12)
    at new Promise (<anonymous>)
    at new F (/Users/208821/Projects/np-cooking/api/test/e2e/node_modules/core-js/library/modules/_export.js:36:28)

Wrote xunit report "WDIO.xunit.chrome.mac.0-0.xml" to [./].
Success finish item 42iggtgg2cjr8adwwy
Successfully save log 42iggtgg2cjr8adxpm
Finish test item 42iggtgg2cjr8adwwn
Success finish item 42iggtgg2cjr8adwwn
Finish launch 42iggtgg2cjr8adsss
Success finish launch 42iggtgg2cjr8adsss

BorisOsipov commented 5 years ago

@igerasimov23 yes good catch! It is a bug in https://github.com/reportportal/client-javascript/blob/master/lib/report-portal-client.js#L487

multipart/form-data doesn't allow special symbols. Will prepare workaround

BorisOsipov commented 5 years ago

@igerasimov23 I published fix in version 0.0.22

igerasimov23 commented 5 years ago

Tested 0.0.22 and it looks good

BorisOsipov commented 5 years ago

Ok. close then.

BorisOsipov / wdio-reportportal-reporter