Cy.visit from a component spec is not allowed and code coverage broken. Happening in integration tests suitcase.

[JairAviles]

Current behavior

I did follow all the Migrating to Cypress 7.0 walkthrough from version 6.40. My project uses as well the component unit test react library for the experimental feature in previous major versions and was running fine.

After some issues with the Webpack configuration (solved by installing the @cypress Webpack dependencies), I'm having the issue when running any of my integration test suite cases that execute the cy.visit command defined in the beforeEach hook.

As a matter of fact, the code coverage is broken. Used to work before the version change and also followed the Code coverage guide and the instrumentation steps with babel plugin.

Desired behavior

End to end testing and component unit testing run smoothly and code coverage generate report.

Test code to reproduce

// src/scripts/cypress.js

const cypress = require('cypress');
const marge = require('mochawesome-report-generator');
const { merge } = require('mochawesome-merge');
require('mochawesome');

cypress.run({ reporter: 'mochawesome' }).then(
    () => {
        generateReport();
    },
    (error) => {
        generateReport();
        console.error(error);
        process.exit(1);
    }
);

function generateReport(options) {
    return merge({
        files: ['./mochawesome-report/*.json']
    }).then((report) => marge.create(report, { saveJson: true }));
}

cypress.json

{
    "projectId": "my-project",
    "baseUrl": "https://localhost:3000/",
    "chromeWebSecurity": false,
    "component": {
        "componentFolder": "src",
        "testFiles": "**/*cy.spec.js"
    },
    "video": false,
    "reporterOptions": { "overwrite": false },
    "retries": {
        "runMode": 2,
        "openMode": 1
    }
}

package.json

...
{
    "name": "my-project",
    "version": "0.1.0",
    "private": true,
    "dependencies": {
        "@cypress/code-coverage": "^3.9.5",
        "@loadable/component": "^5.14.1",
        "@material-ui/core": "^4.11.3",
        "@material-ui/icons": "^4.11.2",
        "@material-ui/lab": "^4.0.0-alpha.57",
        "@okta/okta-auth-js": "^4.2.0",
        "@okta/okta-react": "^4.0.0",
        "@redux-offline/redux-offline": "^2.6.0",
        "@reduxjs/toolkit": "^1.4.0",
        "@testing-library/jest-dom": "^5.11.5",
        "@testing-library/react": "^11.1.0",
        "@testing-library/user-event": "^12.1.10",
        "big.js": "^6.0.3",
        "history": "^5.0.0",
        "immer": "^9.0.1",
        "notistack": "^1.0.7",
        "react": "^17.0.1",
        "react-dom": "^17.0.1",
        "react-dropzone": "^11.2.2",
        "react-icons": "^3.11.0",
        "react-number-format": "^4.4.1",
        "react-redux": "^7.2.2",
        "react-router-dom": "^5.2.0",
        "react-scripts": "^4.0.3",
        "react-table": "^7.6.1",
        "redux": "^4.0.5",
        "redux-dynamic-modules-core": "^5.2.3",
        "redux-dynamic-modules-react": "^5.2.3",
        "redux-dynamic-modules-saga": "^5.2.3",
        "redux-logger": "^3.0.6",
        "redux-saga": "^1.1.3",
        "uuid": "^8.3.1",
        "whatwg-fetch": "^3.4.1"
    },
    "scripts": {
        "start": "npm-run-all buildconf startdev",
        "buildconf": "npm run build:env",
        "startdev": "HTTPS=true PORT=3000 react-scripts -r @cypress/instrument-cra start",
        "ci:start": "npm run build:env && HTTPS=true PORT=3030 react-scripts start",
        "cy:run": "node scripts/cypress.js",
        "cy:ci": "START_SERVER_AND_TEST_INSECURE=1 start-server-and-test start https-get://localhost:3030 cy:run",
        "build:env": "node scripts/set-env.js DIR='./public'",
        "build": "react-scripts build",
        "test": "react-scripts test",
        "eject": "react-scripts eject",
        "clean": "prettier --write .",
        "e2e": "node_modules/.bin/cypress open --browser chrome"
    },
    "eslintConfig": {
        "extends": [
            "react-app",
            "eslint:recommended",
            "plugin:react/recommended",
            "plugin:cypress/recommended"
        ]
    },
    "husky": {
        "hooks": {
            "pre-commit": "lint-staged"
        }
    },
    "browserslist": {
        "production": [
            ">0.2%",
            "not dead",
            "not op_mini all"
        ],
        "development": [
            "last 1 chrome version",
            "last 1 firefox version",
            "last 1 safari version"
        ]
    },
    "devDependencies": {
        "@cypress/instrument-cra": "^1.4.0",
        "cypress": "^6.9.1",
        "cypress-file-upload": "^5.0.7",
        "cypress-pipe": "^2.0.0",
        "cypress-react-unit-test": "^4.17.0",
        "dotenv": "^8.2.0",
        "eslint-plugin-cypress": "^2.11.3",
        "eslint-plugin-react": "^7.23.2",
        "istanbul-lib-coverage": "^3.0.0",
        "lint-staged": "^10.5.0",
        "mocha": "^8.4.0",
        "mochawesome": "^6.2.2",
        "mochawesome-merge": "^4.2.0",
        "mochawesome-report-generator": "^5.2.0",
        "npm-run-all": "^4.1.5",
        "nyc": "^15.1.0",
        "prettier": "^2.1.2",
        "start-server-and-test": "^1.11.7"
    },
    "lint-staged": {
        "src/**/*.{js,jsx,ts,tsx,json,css,scss,md}": [
            "prettier --write"
        ]
    },
    "browser": {
        "fs": false
    },
    "nyc": {
        "branches": 80,
        "clean": true,
        "exclude": [
            "build/*"
        ],
        "extension": [
            ".js"
        ],
        "functions": 80,
        "include": [
            "src/**/*.js"
        ],
        "lines": 80,
        "reporter": [
            "lcov",
            "json-summary",
            "text-summary"
        ],
        "statements": 80
    }
}

cypress/plugins/index.js

/// <reference types="cypress" />
// ***********************************************************
// This example plugins/index.js can be used to load plugins
//
// You can change the location of this file or turn off loading
// the plugins file with the 'pluginsFile' configuration option.
//
// You can read more here:
// https://on.cypress.io/plugins-guide
// ***********************************************************

// This function is called when a project is opened or re-opened (e.g. due to
// the project's config changing)

/**
 * @type {Cypress.PluginConfig}
 */

const path = require('path');
const fs = require('fs');
const webpackPreprocessor = require('@cypress/webpack-preprocessor')

// place downloads into Cypress.env('downloads_directory') folder
const downloadDirectory = path.join(__dirname, '..', 'downloads');

module.exports = (on, config) => {
    if (config.testingType === 'component') {
        require('@cypress/react/plugins/react-scripts')(on, config)
      }
    // custom tasks for sending and reporting code coverage
    require('@cypress/code-coverage/task')(on, config);

    // add other tasks to be registered here
    on('file:preprocessor', webpackPreprocessor())
    on('task', {
        clearDownloads() {
            console.log(`  📁 Clearing folder ${downloadDirectory}`);
            fs.rmdirSync(downloadDirectory, { recursive: true });
            return null;
        }
    });

    // IMPORTANT to return the config object
    // with the any changed environment variables
    return config;
};

home.spec.js

// cypress/integration/home/home.spec/js
'use strict';

describe('Home Page', function () {
      beforeEach(function () {
        cy.visit('/');
      });

      it('should display application name', function () {
        cy.get('.MuiTypography-h1').contains(
            'My Home page name'
        );
      });

});

Versions

Bumped up Cypress version from ^6.4.0 to 7.3.0

[lmiller1990]

I tried to reproduce this using the above information but I didn't have any luck. I had to make some changes to the files to get it to run.

Here's exactly what I have:

cypress.json

{}

package.json

{
  "dependencies": {
    "cypress": "^7.3.0",
    "mocha": "^8.4.0",
    "mochawesome": "^6.2.2",
    "mochawesome-merge": "^4.2.0",
    "mochawesome-report-generator": "^5.2.0"
  },
  "scripts": {
    "e2e": "cypress open --browser chrome",
    "cy:run": "node src/scripts/cypress.js"
  }
}

cypress/integration/home/home.spec.js

// cypress/integration/home/home.spec/js
'use strict';

describe('Home Page', function () {
      beforeEach(function () {
        cy.visit('http://localhost:3000');
      });

      it('should display application name', function () {
        cy.get('.MuiTypography-h1').contains(
            'My Home page name'
        );
      });

});

src/scripts/cypress.js

const cypress = require('cypress');
const marge = require('mochawesome-report-generator');
const { merge } = require('mochawesome-merge');
require('mochawesome');

cypress.run({ reporter: 'mochawesome' }).then(
    () => {
        generateReport();
    },
    (error) => {
        generateReport();
        console.error(error);
        process.exit(1);
    }
);

function generateReport(options) {
    return merge({
        files: ['./mochawesome-report/*.json']
    }).then((report) => marge.create(report, { saveJson: true }));
}

Running yarn cy:run, I get an error about http://localhost:3000 not existing (because it doesn't). I was expecting the "cannot use cy.visit error.

Can you provide either a repository replicating your problem, or some additional information such as:

• cypress.json
• cypress/plugins/index.js
• package.json (it it's entirety)
• any other configuration or information about your setup

[mwren-mshanken]

I've been setting up component testing, and ran into similar behavior.

I was exporting a custom mount in cypress/support/index.js

export { mountTheme as mount } from './test-utils.tsx';

importing this manually in my test files solved this issue. Posting here since I ran into the same end result, but am not sure if you have the same root cause.

In my individual test files I import:

import { mountTheme as mount } from '../support/test-utils';

I'd tried manually passing testingType in my config files, combining my config files, etc, and nothing helped. This was causing every run to run as component

Throwing this out there since this may lead you to a similar solution.

[JairAviles]

@lmiller1990 Thank you for your response. I updated my message above with the requested information. No further config file besides what I updated. Hope this would help

[JessicaSachs]

Thanks for your patience!

A few things:

1. cy.visit will not work inside of component tests. cy.visit navigates the entire web page, which will break Component Testing because it kills the spec execution. The spec is executing in the same window as the Component, so navigating w/ cy.visit causes your tests to stop completely.

2. You're using cypress-react-unit-test which has been deprecated since ~February. Try the new documentation, please :-)

3. cypress.run is via the Cypress Node API and it looks like you're trying to use E2E with it. Can you try via the CLI? yarn cypress run-ct or yarn cypress open-ct are the dedicated commands for component testing.

Lastly, a standalone repository with the failure would be ideal. That way, we can reproduce the issue and fix it :-)

[ericblade]

cy.visit will not work inside of component tests.

... noting here that the problem appears to be that they are not using the component tester, but are erroring as if they were.

also, came here because I'm hoping to find a way to defeat this error, because i am trying to find a way use the component tester outside of a cra app, which doesn't seem to be possible.

[lmiller1990]

@JairAviles I was still not able to reproduce your scenario. If you could post a repo I could clone, that would help a lot. Happy to help out, but I copied your config and I wasn't having any luck.

@ericblade please post your repo and I will help you get it working. I'm not clear on what you mean by "outside of a cra app", though. Can you clarify? A full repo I can clone and try out would be best - I could even send you a PR helping out, assuming I can find the problem.

[ericblade]

component tester only seems to work if you have created the repo with create-react-app, after some more poking around, i don't think it's possible to get around. My specific use case is in wanting to test a separate react component, that is just pure typescript. I can import the component directly into my test suite, but t here's no way that I can find to actually perform tests on it that way.

this is completely unrelated to the actual topic, i had glommed onto it a bit because i was getting a similar error message, but it's an entirely different use case that doesn't appear supported, and i suspect it might require major surgery to get there. (but i'd love to be wrong)

[lmiller1990]

We are in no way coupled to CRA - I have used Cypress with a basic Webpack config I wrote myself. I don't understand what you mean by a "separate React component"

can import the component directly into my test suite, but there's no way that I can find to actually perform tests on it that way.

Why can't you perform a test? What is preventing this?

The dependencies are a bit dated but I have a minimal example working with React here that is not using CRA - just a super simple webpack config.

If you can provide some code showing your problem, or at least describe your environment a little more, I'm happy to help you work through it.

[ericblade]

I don't have any particular need to use webpack for anything at all, but i couldn't find any way to get component testing to work without connecting it to a webpack config and a webpack server thing

I was under impression that I could literally just import the component source in a .spec file, and test it from there. I don't think that there's any route to there ,from where cypress is at right now.

i mean literally

import ComponentToTest from './ComponentToTest';

describe('test the component', () => {
    beforeEach(() => mount(<ComponentToTest />);
    it('tests', () => {
        ... perform test ...
    });
});

[lmiller1990]

Right, I understand.

• You will nee a dev server, so either webpack or vite is required at this point. It's possible we will support others, or even a "batteries included" one out of the box in the future, so you can just import and test like you are suggesting.
• If your component is just plain old TS, this will work, as long as you have a minimal webpack/vite dev server set up. Docs.

Hopefully this clears things up!

[ericblade]

You will nee a dev server, so either webpack or vite is required at this point. It's possible we will support others, or even a "batteries included" one out of the box in the future, so you can just import and test like you are suggesting.

if I may ask -- although this is totally not the right place to do so, i don't know where would be :-) -- fails is mount() when attempting to do that. Mount fails due to not being able to figure out how to mount it. I was able to get around that error, by manually creating a div with the right id that mount() was looking for, and attaching it to the document, but then cy functions on it failed. So, I suspect that either I was very close to getting it working and perhaps just missing some setup phase, or that it's very far away from being doable. So, the question is -- is it a significant design problem, or is it something else? (and if you have any insight on if it might be some small details missing in a setup phase, i'd be more than happy to do some digging on it)

[lmiller1990]

Right, yes, you could theoretically build your own little dev-server like setup if you had enough time on your hands and reverse engineered how it all works. You could grab the id from the mount utils package, too. The problem you will have is, even if you successfully mount your component - what happens when you have your spec/update your component? Without a dev server, you will have no way to serve the latest assets.

Or is your goal to facilitate testing using npx cypress run-ct? You could do a kind of "one shot" spec, without the Cypress UI etc - at this point, I'd really start probing and figuring out what you actually want to accomplish. What is the actual goal here? You want to avoid using a bundler and dev server? How are your building your React app for production without one?

[ericblade]

I don't entirely understand why there even needs to be a server involved -- I can directly import the modules into the .spec file.

Just doing a simple const x = document.createElement("div"); x.id = '__cy_root'; document.appendChild(x); mount(<MyComponent />) appears to work code wise (it doesn't error) but it doesn't visualize anything, and calling any cy test function on it after that fails. I didn't try to dig down any further because no error messages other than "test failed" doesn't really give me anything to dig around in in a giant source tree i'm unfamiliar with :)

You want to avoid using a bundler and dev server? How are your building your React app for production without one?

It's just a repo with a single component in it, intended for consumption in an application, but it's not an application on it's own. It just uses TSC to build the npm package.

https://github.com/ericblade/react-currency-input

[lmiller1990]

Now that I can see the repo, I understand a lot better.

All the things I was mentioning with dev-servers was with our component testing runner in mind. It's still in alpha and was only released about six months ago. It's more or less the same as the E2E runner, but under the hood it uses a dev-server, so you don't need a server of your own. If you were to set this up, you wouldn't need any of the code in your examples directory, including your python server.

EDIT here is a PR against your repo showing how you could use Cypress component testing (npx cypress open-ct, as opposed to npx cypress open), for your component, and why you don't need a server or cy.visit to test a component in isolation: https://github.com/ericblade/react-currency-input/pull/23. I think this will simplify your testing.

[ericblade]

Appreciated that you've given me a webpack config that will work for that, because webpack is one of those tools that I have absolutely no useful knowledge of.

What's in the repo test/example wise is what I came up with because I didn't see an obvious way to use the component tester without also installing and configuring webpack, which is what I was trying to find out if there's any way to avoid.

So, the question then is "if you completely remove webpack and that plugin that starts a webpack-dev-server .. what is still preventing you from just running cy.get('input') on the component? and can whatever is preventing that be worked around?

[lmiller1990]

I noticed the project you linked me already had webpack in the package.json - it looks like someone was using it at one point, maybe the original author.

Either way, what you are doing should work fine, you can create a mini static website that loads a little app with your bundled asset to test. I think the component test runner is a much nicer experience, but if you feel strongly about using the approach you have committed to, I could make a PR and see if I can get it sorted for you?

[ericblade]

what i want to know, is what prevents one from using the component test-runner without a webpack dev server? i can make the tests run but i can't make them pass.

i feel like we're not quite speaking at the same level, because you keep saying 'heres how to use the component tester' and i keep asking 'why cant i use the component tester in a slightly different way? why does it need to have a server?' :-) ... 'why cant i just mount the component to test in the empty document?'

[cypress-app-bot]

This issue has not had any activity in 180 days. Cypress evolves quickly and the reported behavior should be tested on the latest version of Cypress to verify the behavior is still occurring. It will be closed in 14 days if no updates are provided.

[cypress-app-bot]

This issue has been closed due to inactivity.