search

adamgruber / mochawesome-report-generator

Standalone mochawesome report generator. Just add test data.
MIT License
231 stars 90 forks source link

Invalid value "pending" supplied to /suites/suites/0/tests/0/state: TestState #111

Closed stephenchan9 closed 5 years ago

stephenchan9 commented 5 years ago

I am getting this issue that occurs for tests that are in a skipped/pending state. I believe it has to do with the skipped/pending value for one of my tests. If I remove that test, using "marge" and generating the report works as expected with the JSON.

"mocha": "^5.2.0",
"mochawesome": "^3.0.3",

I am using this in CLI

marge ${uuid}.json --reportFilename FuncTestReport --reportDir mochaReports --inline true
{
  "stats": {
    "suites": 2,
    "tests": 4,
    "passes": 3,
    "pending": 0,
    "failures": 1,
    "start": "2018-10-02T18:20:55.891Z",
    "end": "2018-10-02T18:24:28.055Z",
    "duration": 212164,
    "testsRegistered": 4,
    "passPercent": 75,
    "pendingPercent": 0,
    "other": 0,
    "hasOther": false,
    "skipped": 0,
    "hasSkipped": false,
    "passPercentClass": "warning",
    "pendingPercentClass": "danger"
  },
  "suites": {
    "uuid": "662d9420-c670-11e8-935c-6f7f14b4331d",
    "title": "",
    "fullFile": "",
    "file": "",
    "beforeHooks": [],
    "afterHooks": [],
    "tests": [],
    "suites": [{
      "uuid": "8dda2435-40d5-4108-b54d-c521301ffb9c",
      "title": "Image Diff 1",
      "fullFile": "",
      "file": "",
      "beforeHooks": [],
      "afterHooks": [],
      "tests": [{
        "title": "Draw Polygon and check for Image diff. Test should fail",
        "fullTitle": "Image Diff 1 Draw Polygon and check for Image diff. Test should fail",
        "duration": 64868,
        "state": "failed",
        "pass": false,
        "fail": true,
        "pending": false,
        "code": "cy.viewport(1280, 720);\ncy.get(\"#drawIcon\").click();\ncy.get(\".insertText\").contains(\"Draw Polygon\").should(\"be.visible\").click();\ncy.get(\"#MapPanel #geoStyler\").should(\"be.visible\");\ncy.wait(7000);\ncy.matchImageSnapshot(\"drawPolygon\", { capture: \"fullPage\", scale: true });\n// Draw the Polygon.\ncy.get(\"#labelCanvasId\").click(1000, 850, {\n  force: true\n});\ncy.get(\"#labelCanvasId\").click(1250, 750, {\n  force: true\n});\ncy.get(\"#labelCanvasId\").click(1150, 550, {\n  force: true\n});\ncy.get(\"#labelCanvasId\").click(1350, 650, {\n  force: true\n});\ncy.get(\"#labelCanvasId\").dblclick();\n// End draw\ncy.get(\"#geoStyler_accept\").click();\ncy.wait(10000);\n// Compare the snapshots\ncy.matchImageSnapshot(\"drawPolygon\", { capture: \"fullPage\", scale: true });",
        "err": {
          "message": "Error: Screenshot was 58.04698350694445% different from saved snapshot with 534961 different pixels.\n  See diff for details: C:\\SStream Client\\lib\\3.0\\cypress\\snapshots\\funcTests\\testcases\\drawIcon_toolbar.js\\__diff_output__\\drawPolygon-diff.png",
          "estack": "Error: Screenshot was 58.04698350694445% different from saved snapshot with 534961 different pixels.\n  See diff for details: C:\\SStream Client\\lib\\3.0\\cypress\\snapshots\\funcTests\\testcases\\drawIcon_toolbar.js\\__diff_output__\\drawPolygon-diff.png\n    at Context.cy.task.then (https://localhost:44344/__cypress/tests?p=cypress\\support\\index.js-159:27206:15)"
        },
        "isRoot": false,
        "uuid": "7a775cc6-863f-4ee5-9da2-661132c70f3a",
        "isHook": false,
        "skipped": false,
        "timedOut": false
      }],
      "suites": [],
      "passes": [],
      "failures": ["7a775cc6-863f-4ee5-9da2-661132c70f3a"],
      "pending": [],
      "skipped": [],
      "duration": 64868,
      "root": false,
      "rootEmpty": false,
      "_timeout": 2000
    }, {
      "uuid": "804a63e9-7ce7-4644-99fa-737c76c86ba6",
      "title": "Parcel Tests",
      "fullFile": "",
      "file": "",
      "beforeHooks": [],
      "afterHooks": [],
      "tests": [{
        "title": "Parcel Copied 1.1: Verify Parcel is copied",
        "fullTitle": "Parcel Tests Parcel Copied 1.1: Verify Parcel is copied",
        "duration": 65983,
        "state": "passed",
        "speed": "slow",
        "pass": true,
        "fail": false,
        "pending": false,
        "code": "\ncy.searchAddress(address);\ncy.get(\".js-popout_card\").click();\n// Poput should appear\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-SubjectSummary\\\"]\").then(function ($modal) {\n  var display = $modal.css(\"display\");\n  cy.expect(display).to.equal(\"block\");\n});\ncy.wait(5000);\ncy.get(\"body\").screenshot(\"notCopied\");\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-SubjectSummary\\\"] .ui-icon-closethick\").click();\n// Popout should not appear.\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-SubjectSummary\\\"]\").then(function ($modal) {\n  var display = $modal.css(\"display\");\n  cy.expect(display).to.equal(\"none\");\n});\ncy.wait(5000);\n// Compare the snapshots test should fail.\ncy.get(\"body\").screenshot(\"Copied\");\n// Close all dialogs\ncy.get(\".js-close_card > .fa\").click();",
        "err": {},
        "isRoot": false,
        "uuid": "f4050710-5d42-4537-8048-8857aeb13f3b",
        "isHook": false,
        "skipped": false,
        "timedOut": false
      }, {
        "title": "Shared Documents 1.2: Manage Documents command. Upload a file and save.",
        "fullTitle": "Parcel Tests Shared Documents 1.2: Manage Documents command. Upload a file and save.",
        "duration": 12100,
        "state": "passed",
        "speed": "slow",
        "pass": true,
        "fail": false,
        "pending": false,
        "code": "\ncy.searchAddress(address);\n// ---Wait for the parcel to load\ncy.get(\".title\").contains(\"Shared Documents\").click();\ncy.get(\".command_name\").contains(\"Manage Documents\").scrollIntoView().click();\ncy.get(\".ui-dialog-title\").contains(\"Attachments\").should(\"be.visible\");\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-4\\\"] > .ui-dialog-buttonpane > .ui-dialog-buttonset > :nth-child(1) > .ui-button-text\").contains(\"Add\").last().click();\ncy.upload_file2(\"TestAutomation.csv\", \"#attach_doc_browse_file\");\ncy.get(\"#attach_doc_desc\").type(\"Testing upload\");\ncy.get(\".ui-dialog-title\").contains(\"Attach a file\").should(\"be.visible\").then(function ($modal) {\n  var id = $modal.attr(\"id\");\n  // Click the Ok Button\n  cy.get(\"[aria-labelledby=\" + id + \"] .ui-button-text\").contains(\"OK\").click();\n});\ncy.get(\"#attach_doc_list_dialog a\").contains(\"TestAutomation.csv\").should(\"be.visible\");\n// Remove the created attachment.\ncy.get(\"#attach_doc_list_dialog [title=\\\"Remove from list\\\"]\").click();\ncy.get(\".ui-button-text\").contains(\"Yes\").click();\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-4\\\"] > .ui-dialog-buttonpane > .ui-dialog-buttonset > :nth-child(2) > .ui-button-text\").contains(\"Close\").click();",
        "err": {},
        "isRoot": false,
        "uuid": "76f3a940-343f-4b27-85c3-f583f0765fd6",
        "isHook": false,
        "skipped": false,
        "timedOut": false
      }, {
        "title": "Shared Notes: Manage Notes. Upload a note and save.",
        "fullTitle": "Parcel Tests Shared Notes: Manage Notes. Upload a note and save.",
        "duration": 11819,
        "state": "passed",
        "speed": "slow",
        "pass": true,
        "fail": false,
        "pending": false,
        "code": "\ncy.searchAddress(address);\ncy.get(\".title\").contains(\"Shared Notes\").click();\ncy.get(\".command_name\").contains(\"Manage Notes\").scrollIntoView().click();\ncy.get(\".ui-dialog-title\").contains(\"Notes\").should(\"be.visible\");\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-3\\\"] > .ui-dialog-buttonpane > .ui-dialog-buttonset > :nth-child(1) > .ui-button-text\").contains(\"Add\").last().click({\n  force: true\n});\ncy.get(\".editAttributeLabel\").contains(\"Title\").next().clear().type(title);\ncy.get(\".editAttributeLabel\").contains(\"type\").next().clear().type(noteType);\ncy.get(\".editAttributeLabel\").contains(\"Note\").next().clear().type(note);\ncy.get(\".editAttributeSubmit\").click();\ncy.get(\"#AddNotesListDiv label u\").first().contains(title).should(\"be.visible\");\n// Remove the created attachment.\ncy.get(\"#AddNotesListDiv [title=\\\"Remove from list\\\"]\").click();\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-3\\\"] > .ui-dialog-buttonpane > .ui-dialog-buttonset > :nth-child(2) > .ui-button-text\").contains(\"Close\").click();",
        "err": {},
        "isRoot": false,
        "uuid": "7cf4f226-795d-409f-8d91-09d111cb5ad1",
        "isHook": false,
        "skipped": false,
        "timedOut": false
      }],
      "suites": [],
      "passes": ["f4050710-5d42-4537-8048-8857aeb13f3b", "76f3a940-343f-4b27-85c3-f583f0765fd6", "7cf4f226-795d-409f-8d91-09d111cb5ad1"],
      "failures": [],
      "pending": [],
      "skipped": [],
      "duration": 89902,
      "root": false,
      "rootEmpty": false,
      "_timeout": 2000
    }],
    "passes": [],
    "failures": [],
    "pending": [],
    "skipped": [],
    "duration": 0,
    "root": true,
    "rootEmpty": true,
    "_timeout": 2000
  },
  "copyrightYear": 2018
}
adamgruber commented 5 years ago

If you run npm ls mochawesome-report-generator, what version is installed?

stephenchan9 commented 5 years ago

If you run npm ls mochawesome-report-generator, what version is installed?

-- mochawesome@3.0.3
-- mochawesome-report-generator@3.1.3
adamgruber commented 5 years ago

So using the JSON above with the CLI works. Can you provide a simple test that reproduces the issue?

stephenchan9 commented 5 years ago

Hmm you're correct. the above JSON works correctly. Here's an updated one which is giving me issues.

{
  "stats": {
    "suites": 2,
    "tests": 4,
    "passes": 3,
    "pending": 1,
    "failures": 0,
    "start": "2018-10-04T17:29:22.538Z",
    "end": "2018-10-04T17:32:39.970Z",
    "duration": 197432,
    "testsRegistered": 3,
    "passPercent": 150,
    "pendingPercent": 33.3,
    "other": 0,
    "hasOther": false,
    "skipped": 0,
    "hasSkipped": false,
    "passPercentClass": "success",
    "pendingPercentClass": "danger"
  },
  "suites": {
    "uuid": "7f4ab000-c7fb-11e8-9143-33aa91f517b9",
    "title": "",
    "fullFile": "",
    "file": "",
    "beforeHooks": [],
    "afterHooks": [],
    "tests": [],
    "suites": [{
      "uuid": "9851cc5c-02ed-4b43-ae34-c4406270abf9",
      "title": "Image Diff 1",
      "fullFile": "",
      "file": "",
      "beforeHooks": [],
      "afterHooks": [],
      "tests": [{
        "title": "Draw Polygon and check for Image diff. Test should fail",
        "fullTitle": "Image Diff 1 Draw Polygon and check for Image diff. Test should fail",
        "duration": 0,
        "state": "pending",
        "pass": false,
        "fail": false,
        "pending": true,
        "code": "",
        "err": {},
        "isRoot": false,
        "uuid": "f66916af-e8f6-4cb8-a1b2-c2b8e77898e4",
        "isHook": false,
        "skipped": false,
        "timedOut": false
      }],
      "suites": [],
      "passes": [],
      "failures": [],
      "pending": ["f66916af-e8f6-4cb8-a1b2-c2b8e77898e4"],
      "skipped": [],
      "duration": null,
      "root": false,
      "rootEmpty": false,
      "_timeout": 2000
    }, {
      "uuid": "c0d62fb7-1bef-421b-b069-1cc62a6751da",
      "title": "Parcel Tests",
      "fullFile": "",
      "file": "",
      "beforeHooks": [],
      "afterHooks": [],
      "tests": [{
        "title": "Parcel Copied 1.1: Verify Parcel is copied",
        "fullTitle": "Parcel Tests Parcel Copied 1.1: Verify Parcel is copied",
        "duration": 67585,
        "state": "passed",
        "speed": "slow",
        "pass": true,
        "fail": false,
        "pending": false,
        "code": "\ncy.searchAddress(address);\ncy.get(\".js-popout_card\").click();\n// Poput should appear\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-SubjectSummary\\\"]\").then(function ($modal) {\n  var display = $modal.css(\"display\");\n  cy.expect(display).to.equal(\"block\");\n});\ncy.wait(5000);\ncy.get(\"body\").screenshot(\"notCopied\");\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-SubjectSummary\\\"] .ui-icon-closethick\").click();\n// Popout should not appear.\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-SubjectSummary\\\"]\").then(function ($modal) {\n  var display = $modal.css(\"display\");\n  cy.expect(display).to.equal(\"none\");\n});\ncy.wait(5000);\n// Compare the snapshots test should fail.\ncy.get(\"body\").screenshot(\"Copied\");\n// Close all dialogs\ncy.get(\".js-close_card > .fa\").click();",
        "err": {},
        "isRoot": false,
        "uuid": "0fe4de10-91a0-4fc3-933d-726478ea36e3",
        "isHook": false,
        "skipped": false,
        "timedOut": false
      }, {
        "title": "Shared Documents 1.2: Manage Documents command. Upload a file and save.",
        "fullTitle": "Parcel Tests Shared Documents 1.2: Manage Documents command. Upload a file and save.",
        "duration": 12851,
        "state": "passed",
        "speed": "slow",
        "pass": true,
        "fail": false,
        "pending": false,
        "code": "\ncy.searchAddress(address);\ncy.get(\".title\").contains(\"Shared Documents\").click();\ncy.get(\".command_name\").contains(\"Manage Documents\").scrollIntoView().click();\ncy.get(\".ui-dialog-title\").contains(\"Attachments\").should(\"be.visible\");\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-4\\\"] > .ui-dialog-buttonpane > .ui-dialog-buttonset > :nth-child(1) > .ui-button-text\").contains(\"Add\").last().click();\ncy.upload_file2(\"TestAutomation.csv\", \"#attach_doc_browse_file\");\ncy.get(\"#attach_doc_desc\").type(\"Testing upload\");\ncy.get(\".ui-dialog-title\").contains(\"Attach a file\").should(\"be.visible\").then(function ($modal) {\n  var id = $modal.attr(\"id\");\n  // Click the Ok Button\n  cy.get(\"[aria-labelledby=\" + id + \"] .ui-button-text\").contains(\"OK\").click();\n});\ncy.get(\"#attach_doc_list_dialog a\").contains(\"TestAutomation.csv\").should(\"be.visible\");\n// Remove the created attachment.\ncy.get(\"#attach_doc_list_dialog [title=\\\"Remove from list\\\"]\").click();\ncy.get(\".ui-button-text\").contains(\"Yes\").click();\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-4\\\"] > .ui-dialog-buttonpane > .ui-dialog-buttonset > :nth-child(2) > .ui-button-text\").contains(\"Close\").click();",
        "err": {},
        "isRoot": false,
        "uuid": "9216292d-16ce-4e84-bbb8-2b36ac466c0b",
        "isHook": false,
        "skipped": false,
        "timedOut": false
      }, {
        "title": "Shared Notes: Manage Notes. Upload a note and save.",
        "fullTitle": "Parcel Tests Shared Notes: Manage Notes. Upload a note and save.",
        "duration": 12724,
        "state": "passed",
        "speed": "slow",
        "pass": true,
        "fail": false,
        "pending": false,
        "code": "\ncy.searchAddress(address);\ncy.get(\".title\").contains(\"Shared Notes\").click();\ncy.get(\".command_name\").contains(\"Manage Notes\").scrollIntoView().click();\ncy.get(\".ui-dialog-title\").contains(\"Notes\").should(\"be.visible\");\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-3\\\"] > .ui-dialog-buttonpane > .ui-dialog-buttonset > :nth-child(1) > .ui-button-text\").contains(\"Add\").last().click({\n  force: true\n});\ncy.get(\".editAttributeLabel\").contains(\"Title\").next().clear().type(title);\ncy.get(\".editAttributeLabel\").contains(\"type\").next().clear().type(noteType);\ncy.get(\".editAttributeLabel\").contains(\"Note\").next().clear().type(note);\ncy.get(\".editAttributeSubmit\").click();\ncy.get(\"#AddNotesListDiv label u\").first().contains(title).should(\"be.visible\");\n// Remove the created attachment.\ncy.get(\"#AddNotesListDiv [title=\\\"Remove from list\\\"]\").click();\ncy.get(\"[aria-labelledby=\\\"ui-dialog-title-3\\\"] > .ui-dialog-buttonpane > .ui-dialog-buttonset > :nth-child(2) > .ui-button-text\").contains(\"Close\").click();",
        "err": {},
        "isRoot": false,
        "uuid": "6f9b953b-17e4-4ab7-886e-8a366cdb3ff2",
        "isHook": false,
        "skipped": false,
        "timedOut": false
      }],
      "suites": [],
      "passes": ["0fe4de10-91a0-4fc3-933d-726478ea36e3", "9216292d-16ce-4e84-bbb8-2b36ac466c0b", "6f9b953b-17e4-4ab7-886e-8a366cdb3ff2"],
      "failures": [],
      "pending": [],
      "skipped": [],
      "duration": 93160,
      "root": false,
      "rootEmpty": false,
      "_timeout": 2000
    }],
    "passes": [],
    "failures": [],
    "pending": [],
    "skipped": [],
    "duration": 0,
    "root": true,
    "rootEmpty": true,
    "_timeout": 2000
  },
  "copyrightYear": 2018
}

I was able to isolate the issue and it occurs at: "state": "pending", If this is either "pending" or "skipped" the report cannot be created.

adamgruber commented 5 years ago

Ok, so the change to allow pending and skipped states is straightforward. But here's what's odd, I can't seem to find any place in mocha's codebase where the test state would ever be set to pending or skipped. Same inside mochawesome. From what I can tell state should only ever be passed or failed. Can you provide a repo with some sample tests that gets the JSON to look like that?

stephenchan9 commented 5 years ago

Ok, so the change to allow pending and skipped states is straightforward. But here's what's odd, I can't seem to find any place in mocha's codebase where the test state would ever be set to pending or skipped. Same inside mochawesome. From what I can tell state should only ever be passed or failed. Can you provide a repo with some sample tests that gets the JSON to look like that?

https://github.com/stephenchan9/cypressTest I am running my tests using Cypress. Here's a small test for the issue.

adamgruber commented 5 years ago

I am running my tests using Cypress.

Ah, that's a key detail. Looks like Cypress alters the mocha output so it can end up slightly different from what mochawesome expects.