michaelleeallen / mocha-junit-reporter

A JUnit XML reporter for mocha.
MIT License
144 stars 153 forks source link

Weird behavior of nested suites in xml output #161

Open DanaGoyette opened 2 years ago

DanaGoyette commented 2 years ago

I'm testing out the junit reporter with Cypress, and there are some weird behaviors with regards to the root suite, especially when using nested suites.

The root suite seems to be empty, and ends up being named ` (empty string) even if I have setrootSuiteName. This seems to be becausefullSuiteTitle()assumes the passed-in suite is not the root itself:var title = [ suite.title ];`

If I set useFullSuiteTitle to true, there's no way to omit the root name, because the || operator overrides the empty string:

config.rootSuiteTitle = config.rootSuiteTitle || 'Root Suite';
config.testsuitesTitle = config.testsuitesTitle || 'Mocha Tests';

Given this sample test:

context('Outer1', () => {
  context('InnerA', () => {
    it('Bad', () => { expect(false).to.be.true })
    it('Good', () => {})
    xit('Pending', () => {})
  })
  context('InnerB', () => {
    it('Bad', () => { expect(false).to.be.true })
    it('Good', () => {})
    xit('Pending', () => {})
  })
})
context('Outer2', () => {
  context('InnerA', () => {
    it('Bad', () => { expect(false).to.be.true })
    it('Good', () => {})
    xit('Pending', () => {})
  })
  context('InnerB', () => {
    it('Bad', () => { expect(false).to.be.true })
    it('Good', () => {})
    xit('Pending', () => {})
  })
})

Assuming the config:

{
  "testCaseSwitchClassnameAndName": true,
  "useFullSuiteTitle": true,
  "suiteTitleSeparatedBy": " -- ",
  "rootSuiteTitle": ""
}

What I expect (though the last piece of the classname is redundant):

<testsuite name="Outer1 -- InnerA" tests="3">
    <testcase name="Bad" classname="Outer1 -- InnerA -- Bad">…</testcase>
<testsuite>

What I get:

<testsuite name="" tests="0">
</testsuite>
<testsuite name="Root Suite -- Outer1 -- InnerA" tests="3">
    <testcase name="Bad" classname="Root Suite Outer1 InnerA">…</testcase>
</testsuite>

Full spew, with "useFullSuiteTitle": true:

<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="Mocha Tests" time="1.1380" tests="12" failures="4" skipped="4">
  <testsuite name="" timestamp="2022-02-03T01:13:51" tests="0" file="cypress/integration/sample.test.ts" time="0.0000" failures="0">
  </testsuite>
  <testsuite name="Root Suite -- Outer1" timestamp="2022-02-03T01:13:51" tests="0" time="0.0000" failures="0">
  </testsuite>
  <testsuite name="Root Suite -- Outer1 -- InnerA" timestamp="2022-02-03T01:13:51" tests="3" time="0.2290" failures="1">
    <testcase name="Bad" time="0.0000" classname="Outer1 InnerA Bad">
      <failure message="expected false to be true" type="AssertionError"><![CDATA[AssertionError: expected false to be true
    at Context.eval (http://localhost:3001/__cypress/tests?p=cypress/integration/sample.test.ts:98:52)]]></failure>
    </testcase>
    <testcase name="Good" time="0.0290" classname="Outer1 InnerA Good">
    </testcase>
  </testsuite>
  <testsuite name="Root Suite -- Outer1 -- InnerB" timestamp="2022-02-03T01:13:51" tests="3" time="0.2310" failures="1">
    <testcase name="Bad" time="0.0000" classname="Outer1 InnerB Bad">
      <failure message="expected false to be true" type="AssertionError"><![CDATA[AssertionError: expected false to be true
    at Context.eval (http://localhost:3001/__cypress/tests?p=cypress/integration/sample.test.ts:104:32)]]></failure>
    </testcase>
    <testcase name="Good" time="0.0240" classname="Outer1 InnerB Good">
    </testcase>
  </testsuite>
  <testsuite name="Root Suite -- Outer2" timestamp="2022-02-03T01:13:52" tests="0" time="0.0000" failures="0">
  </testsuite>
  <testsuite name="Root Suite -- Outer2 -- InnerA" timestamp="2022-02-03T01:13:52" tests="3" time="0.2290" failures="1">
    <testcase name="Bad" time="0.0000" classname="Outer2 InnerA Bad">
      <failure message="expected false to be true" type="AssertionError"><![CDATA[AssertionError: expected false to be true
    at Context.eval (http://localhost:3001/__cypress/tests?p=cypress/integration/sample.test.ts:112:52)]]></failure>
    </testcase>
    <testcase name="Good" time="0.0260" classname="Outer2 InnerA Good">
    </testcase>
  </testsuite>
  <testsuite name="Root Suite -- Outer2 -- InnerB" timestamp="2022-02-03T01:13:52" tests="3" time="0.2880" failures="1">
    <testcase name="Bad" time="0.0000" classname="Outer2 InnerB Bad">
      <failure message="expected false to be true" type="AssertionError"><![CDATA[AssertionError: expected false to be true
    at Context.eval (http://localhost:3001/__cypress/tests?p=cypress/integration/sample.test.ts:117:52)]]></failure>
    </testcase>
    <testcase name="Good" time="0.0280" classname="Outer2 InnerB Good">
    </testcase>
  </testsuite>
</testsuites>

Full spew, with "useFullSuiteTitle": false:

<?xml version="1.0" encoding="UTF-8"?>
<testsuites name="Mocha Tests" time="1.1570" tests="12" failures="4" skipped="4">
  <testsuite name="Root Suite" timestamp="2022-02-03T01:14:37" tests="0" file="cypress/integration/sample.test.ts" time="0.0000" failures="0">
  </testsuite>
  <testsuite name="Outer1" timestamp="2022-02-03T01:14:38" tests="0" time="0.0000" failures="0">
  </testsuite>
  <testsuite name="InnerA" timestamp="2022-02-03T01:14:38" tests="3" time="0.2420" failures="1">
    <testcase name="Bad" time="0.0000" classname="Outer1 InnerA Bad">
      <failure message="expected false to be true" type="AssertionError"><![CDATA[AssertionError: expected false to be true
    at Context.eval (http://localhost:3001/__cypress/tests?p=cypress/integration/sample.test.ts:98:52)]]></failure>
    </testcase>
    <testcase name="Good" time="0.0270" classname="Outer1 InnerA Good">
    </testcase>
  </testsuite>
  <testsuite name="InnerB" timestamp="2022-02-03T01:14:38" tests="3" time="0.2220" failures="1">
    <testcase name="Bad" time="0.0000" classname="Outer1 InnerB Bad">
      <failure message="expected false to be true" type="AssertionError"><![CDATA[AssertionError: expected false to be true
    at Context.eval (http://localhost:3001/__cypress/tests?p=cypress/integration/sample.test.ts:104:32)]]></failure>
    </testcase>
    <testcase name="Good" time="0.0220" classname="Outer1 InnerB Good">
    </testcase>
  </testsuite>
  <testsuite name="Outer2" timestamp="2022-02-03T01:14:38" tests="0" time="0.0000" failures="0">
  </testsuite>
  <testsuite name="InnerA" timestamp="2022-02-03T01:14:38" tests="3" time="0.2300" failures="1">
    <testcase name="Bad" time="0.0000" classname="Outer2 InnerA Bad">
      <failure message="expected false to be true" type="AssertionError"><![CDATA[AssertionError: expected false to be true
    at Context.eval (http://localhost:3001/__cypress/tests?p=cypress/integration/sample.test.ts:112:52)]]></failure>
    </testcase>
    <testcase name="Good" time="0.0270" classname="Outer2 InnerA Good">
    </testcase>
  </testsuite>
  <testsuite name="InnerB" timestamp="2022-02-03T01:14:38" tests="3" time="0.3020" failures="1">
    <testcase name="Bad" time="0.0000" classname="Outer2 InnerB Bad">
      <failure message="expected false to be true" type="AssertionError"><![CDATA[AssertionError: expected false to be true
    at Context.eval (http://localhost:3001/__cypress/tests?p=cypress/integration/sample.test.ts:117:52)]]></failure>
    </testcase>
    <testcase name="Good" time="0.0340" classname="Outer2 InnerB Good">
    </testcase>
  </testsuite>
</testsuites>
babalubas090 commented 2 years ago

I'm having the same issue here

louneskmt commented 2 years ago

Same here

cgraham-rs commented 1 year ago

Being unable to lose the rootSuiteName makes for some ugly and unnecessary text if you're outputting XML to be imported into something like TestRail.

image