Closed zernie closed 3 months ago
@zernie the error is coming from the cypress action, so I'm not sure what's causing that.
Could you share your cypress.config.js
file? The only Chromatic specific thing in there should be the installPlugin
command.
@winkerVSbecks sure:
const { defineConfig } = require('cypress')
const { installPlugin } = require('@chromatic-com/cypress')
module.exports = defineConfig({
projectId: 'XXX',
env: {
assetDomains: ['https://ik.imagekit.io']
},
e2e: {
setupNodeEvents(on, config) {
installPlugin(on, config)
},
defaultCommandTimeout: 60000,
video: true,
includeShadowDom: true,
chromeWebSecurity: false,
// MacBook 13' resolution
viewportHeight: 800,
viewportWidth: 1280
}
})
package.json:
{
"name": "cypress",
"version": "1.0.0",
"description": "e2e",
"main": "index.js",
"private": true,
"engines": {
"node": "^18.18.0 || ^20.9.0 || >=21.1.0"
},
"devDependencies": {
"@chromatic-com/cypress": "^0.6.6",
"chromatic": "^11.3.1",
"cypress": "13.8.1",
"eslint": "^9.2.0",
"@eslint/js": "^9.2.0",
"@eslint/eslintrc": "^3.0.2",
"eslint-config-standard": "^17.0.0",
"eslint-plugin-cypress": "^3.2.0",
"eslint-plugin-import": "^2.29.1"
},
"scripts": {
"lint:js": "eslint \"cypress/**/*.js\"",
"cypress:open": "NODE_OPTIONS=--openssl-legacy-provider ELECTRON_EXTRA_LAUNCH_ARGS=--remote-debugging-port=9222 cypress open",
"cypress:run": "NODE_OPTIONS=--openssl-legacy-provider ELECTRON_EXTRA_LAUNCH_ARGS=--remote-debugging-port=9222 cypress run",
"chromatic": "chromatic --cypress --exit-zero-on-changes"
}
}
It happens quite randomly though on different specs. I've tried updating the packages to the latest versions and it didn't help.
Given that it only happens after a successful test I'm 100% positive that it's a snapshot that fails.
perhaps watcher.idle()
is actually needed?
@zernie thanks, we'll investigate
@winkerVSbecks hey, any updates here? We had to fully disable chromatic because of this issue :(
@zernie I think your hint about possibly requiring watcher.idle()
is on the right path, but I'm having a very difficult time reproducing this to confirm.
package.json:
{
"dependencies": {
"@absinthe/socket": "^0.2.1",
"@ant-design/icons": "^5.1.4",
"@ant-design/plots": "^1.2.5",
"@auth0/auth0-react": "^1.10.1",
"@datadog/browser-logs": "^5.9.0",
"@datadog/browser-rum": "^4.38.0",
"@dnd-kit/core": "^6.0.8",
"@dnd-kit/sortable": "^7.0.2",
"@dnd-kit/utilities": "^3.2.1",
"@graphiql/toolkit": "^0.8.0",
"@oneschema/react": "^0.4.7",
"@rails/activestorage": "^7.0.8",
"@rc-component/mini-decimal": "^1.0.1",
"@react-hookz/web": "^23.0.0",
"@reduxjs/toolkit": "^1.7.1",
"@sentry/core": "^7.73.0",
"@sentry/react": "7.73.0",
"@sentry/tracing": "7.73.0",
"@stripe/react-stripe-js": "^1.8.1",
"@stripe/stripe-js": "^1.32.0",
"@types/rails__activestorage": "^7.0.1",
"@types/react-redux": "^7.1.25",
"@types/redux-logger": "^3.0.9",
"ajv": "^6.12.6",
"antd": "^5.8.4",
"antd-img-crop": "^4.12.2",
"antd-style": "https://github.com/GaaS-Holdings/antd-style.git#archive-release-2",
"axios": "0.24.0",
"babel-plugin-polyfill-corejs2": "^0.4.8",
"circular-dependency-plugin": "^5.2.2",
"classnames": "^2.3.1",
"csstype": "3.0.10",
"date-fns": "2.27.0",
"date-fns-tz": "^2.0.0",
"dayjs": "^1.11.10",
"dayjs-plugin-utc": "^0.1.2",
"es-cookie": "^1.3.2",
"graphiql": "^2.0.7",
"graphql": "^16.8.1",
"graphql-tag": "^2.12.6",
"highcharts": "^11.4.1",
"highcharts-react-official": "^3.2.1",
"history": "^5.3.0",
"js-cookie": "^3.0.1",
"jwt-decode": "3.1.2",
"launchdarkly-js-client-sdk": "^3.1.1",
"lodash": "^4.17.21",
"masonic": "^3.7.0",
"papaparse": "^5.3.2",
"phoenix": "^1.7.11",
"pluralize": "^8.0.0",
"rc-picker": "^2.6.10",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-inlinesvg": "^2.3.0",
"react-intl": "5.24.0",
"react-markdown": "^9.0.1",
"react-redux": "^7.2.6",
"react-resizable": "^3.0.5",
"react-retool": "^1.4.0",
"react-router-dom": "^6.18.0",
"redux-logger": "^3.0.6",
"redux-saga": "^1.1.3",
"remark-gfm": "^4.0.0",
"uuid": "^8.3.2"
},
"devDependencies": {
"@babel/core": "^7.24.0",
"@babel/plugin-proposal-class-properties": "^7.16.5",
"@babel/plugin-proposal-private-methods": "^7.16.5",
"@babel/plugin-proposal-private-property-in-object": "^7.21.11",
"@babel/plugin-syntax-dynamic-import": "^7.8.3",
"@babel/plugin-transform-destructuring": "^7.16.5",
"@babel/plugin-transform-regenerator": "^7.23.3",
"@babel/plugin-transform-runtime": "7.16.5",
"@babel/preset-env": "^7.23.2",
"@babel/preset-react": "7.16.5",
"@babel/preset-typescript": "^7.16.5",
"@babel/traverse": "^7.24.0",
"@chromatic-com/storybook": "^1.2.26",
"@graphql-codegen/add": "^3.2.1",
"@graphql-codegen/cli": "^2.11.8",
"@graphql-codegen/typescript": "^2.7.3",
"@graphql-codegen/typescript-operations": "^2.5.3",
"@pmmmwh/react-refresh-webpack-plugin": "^0.5.10",
"@rails/webpacker": "5.4.4",
"@storybook/addon-controls": "^8.0.0",
"@storybook/addon-docs": "^8.0.0",
"@storybook/addon-essentials": "^8.0.0",
"@storybook/addon-interactions": "^8.0.0",
"@storybook/addon-links": "^8.0.0",
"@storybook/addon-onboarding": "^8.0.0",
"@storybook/addon-webpack5-compiler-swc": "^1.0.2",
"@storybook/blocks": "^8.0.0",
"@storybook/react": "^8.0.0",
"@storybook/react-webpack5": "^8.0.0",
"@storybook/test": "^8.0.0",
"@testing-library/jest-dom": "^5.16.5",
"@testing-library/react": "^14.0.0",
"@trivago/prettier-plugin-sort-imports": "^4.3.0",
"@types/absinthe__socket": "^0.2.3",
"@types/codemirror": "^5.60.5",
"@types/intercom-web": "^2.8.15",
"@types/jest": "27.0.3",
"@types/js-cookie": "^3.0.2",
"@types/lodash": "^4.14.178",
"@types/papaparse": "^5.3.4",
"@types/pluralize": "^0.0.29",
"@types/react": "^18.2.6",
"@types/react-dom": "^18.2.4",
"@types/react-resizable": "^3.0.4",
"@types/redux-mock-store": "^1.0.3",
"@types/segment-analytics": "^0.0.34",
"@types/uuid": "^8.3.4",
"@types/webpack-env": "^1.16.3",
"@typescript-eslint/eslint-plugin": "^5.59.9",
"@typescript-eslint/parser": "^5.59.9",
"@webpack-cli/serve": "^1.6.0",
"babel-eslint": "^10.1.0",
"babel-jest": "27.4.5",
"babel-loader": "^8.2.3",
"babel-plugin-import": "^1.13.5",
"babel-plugin-lodash": "^3.3.4",
"babel-plugin-transform-react-jsx": "^6.24.1",
"babel-plugin-transform-react-remove-prop-types": "^0.4.24",
"brotli-size": "^4.0.0",
"bundlewatch": "^0.3.3",
"chromatic": "^11.3.0",
"compression-webpack-plugin": "^10.0.0",
"copy-webpack-plugin": "^11.0.0",
"css-loader": "^6.8.1",
"css-minimizer-webpack-plugin": "^5.0.1",
"css-modules-typescript-loader": "^4.0.1",
"dotenv": "^16.3.1",
"dotenv-webpack": "^8.0.1",
"eslint": "8.5.0",
"eslint-config-airbnb-typescript": "16.1.0",
"eslint-config-preact": "^1.3.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-babel": "^5.3.1",
"eslint-plugin-import": "^2.25.3",
"eslint-plugin-jsx-a11y": "^6.5.1",
"eslint-plugin-local": "^4.2.2",
"eslint-plugin-react": "^7.28.0",
"eslint-plugin-react-hooks": "^4.3.0",
"eslint-plugin-storybook": "^0.8.0",
"file-loader": "^6.2.0",
"identity-obj-proxy": "^3.0.0",
"jest": "^27.4.5",
"jest-css-modules": "^2.1.0",
"jest-junit": "13.0.0",
"less": "^4.1.3",
"less-loader": "7.3.0",
"mini-css-extract-plugin": "^2.7.6",
"msw": "^2.2.14",
"prettier": "^2.7.1",
"react-docgen-typescript": "^2.2.2",
"react-docgen-typescript-plugin": "^1.0.5",
"react-refresh": "^0.14.0",
"redux-mock-store": "^1.5.4",
"resolve-url-loader": "4.0.0",
"sass": "^1.63.6",
"sass-loader": "^13.3.2",
"storybook": "^8.0.5",
"storybook-addon-react-docgen": "^1.2.44",
"storybook-msw-addon": "^2.1.4",
"stream-browserify": "^3.0.0",
"style-loader": "^3.3.3",
"terser-webpack-plugin": "^5.3.9",
"ts-jest": "^27.1.2",
"typescript": "^5.0.4",
"typings-for-css-modules-loader": "^1.7.0",
"webpack": "^5.88.0",
"webpack-assets-manifest": "^3.1.1",
"webpack-bundle-analyzer": "^4.6.1",
"webpack-cli": "^5.1.4",
"webpack-dev-server": "^4.15.1"
}
"engines": {
"node": "v20"
}
}
webpack.config.js
:
const path = require('path')
const DotenvPlugin = require('dotenv-webpack')
const ReactRefreshPlugin = require('@pmmmwh/react-refresh-webpack-plugin')
const { BundleAnalyzerPlugin } = require('webpack-bundle-analyzer')
// const CircularDependencyPlugin = require('circular-dependency-plugin')
const CopyWebpackPlugin = require('copy-webpack-plugin')
const WebpackAssetsManifest = require('webpack-assets-manifest')
require('dotenv').config()
const parsedWebpackDevUrl = new URL(process.env.WEBPACK_PROXY_URL || 'ws://0.0.0.0:3035/ws')
const WEBPACK_URL_FROM_BROWSER = (process.env.WEBPACK_URL_FROM_BROWSER || 'http://localhost:3035/').replace(
/\/$|$/,
'/'
) // `replace` is to ensure trailing slash
module.exports = {
mode: 'development',
entry: {
app: './app/javascript/packs/app.tsx',
graphiql_helper: './app/javascript/packs/graphiql_helper.tsx',
graphiql: './app/javascript/packs/graphiql.tsx',
public_app: './app/javascript/packs/public_app.tsx',
admins_app: './app/javascript/packs/admins_app.tsx'
},
output: {
filename: '[name]-[contenthash]-bundle.js',
path: path.resolve(__dirname, '/public/packs'),
publicPath: WEBPACK_URL_FROM_BROWSER,
clean: true
},
resolve: {
extensions: ['.tsx', '.ts', '.jsx', '.js', '.scss', '.css', '.module.scss', '.png', '.svg', '.jpeg', '.jpg'],
alias: {
},
modules: ['/app/javascript', 'node_modules'],
fallback: {
stream: false
}
},
module: {
rules: [
{
test: /\.(ts|tsx)$/,
exclude: /node_modules/,
use: ['babel-loader']
},
{
test: /\.(scss|css)$/,
use: [
'style-loader',
'css-modules-typescript-loader',
{
loader: 'css-loader',
options: {
importLoaders: 2,
modules: {
auto: true,
localIdentName: '[name]__[local]__[hash:base64:5]'
},
sourceMap: true
}
},
'sass-loader'
]
},
// Images: Copy image files to build folder
{ test: /\.(?:ico|gif|png|jpg|jpeg)$/i, type: 'asset/resource' },
// Fonts and SVGs: Inline files
{ test: /\.(woff(2)?|eot|ttf|otf|svg|)$/, type: 'asset/inline' }
]
},
plugins: [
new ReactRefreshPlugin(),
new DotenvPlugin(),
new WebpackAssetsManifest({
// [INFO]: !!! it's important to keep "webpack-assets-manifest" package of version "^3.1.1"
// [REASON]: Webpacker logic that generates <script>'s and CSS <link>'s relies on
// the manifest, produced by this specific version of plugin.
entrypoints: true,
entrypointsUseAssets: true,
publicPath: WEBPACK_URL_FROM_BROWSER
}),
new CopyWebpackPlugin({
// [INFO]: the purpose:
// -> we copy these files directly instead of preprocessing with webpack
// -> and then include it directly in the `...index.ejs` pages
// the reason:
// -> I've tried to do it via `config.entry`, but it results in error when the
// compiled code is being executed
patterns: [
{
from: './app/javascript/packs/shared/analytics.js',
to: 'shared/',
transform(content) {
const transformedContent = content
.toString()
.replace('__SEGMENT_CLIENT_WRITE_API_KEY__', JSON.stringify(process.env.SEGMENT_CLIENT_WRITE_API_KEY))
return transformedContent
}
},
{ from: './app/javascript/packs/shared/customerio.js', to: 'shared/' }
]
}),
// new CircularDependencyPlugin({
// // exclude detection of files based on a RegExp
// exclude: /a\.js|node_modules/,
// // add errors to webpack instead of warnings
// failOnError: false,
// // allow import cycles that include an asyncronous import,
// // e.g. via import(/* webpackMode: "weak" */ './file.js')
// allowAsyncCycles: false,
// // set the current working directory for displaying module paths
// cwd: process.cwd(),
// }),
process.env.BUNDLE_ANALYZE && new BundleAnalyzerPlugin()
].filter(Boolean),
devtool: 'cheap-module-source-map',
devServer: {
hot: true,
allowedHosts: 'all',
host: '0.0.0.0',
port: 3035,
https: false,
headers: { 'Access-Control-Allow-Origin': '*' },
client: {
overlay: false,
webSocketURL: {
hostname: parsedWebpackDevUrl.hostname,
pathname: parsedWebpackDevUrl.pathname,
protocol: parsedWebpackDevUrl.protocol,
port: parsedWebpackDevUrl.protocol === 'wss:' ? parsedWebpackDevUrl.port || 443 : parsedWebpackDevUrl.port
}
}
},
optimization: {
splitChunks: {
chunks: 'all'
}
}
}
an example test & helpers:
export const aliasGraphql = () => {
cy.intercept('POST', `${Cypress.env('host')}/graphql`, (req) => {
const pattern = /^(?<operation>(query|mutation)) (?<operationName>[a-zA-z]+)(\(|( \{))/
const match = pattern.exec(req.body.query)
if (match) {
const { operation, operationName } = match.groups as { operation: string; operationName: string }
if (operation === 'query') {
req.alias = `gql${operationName}Query`
} else if (operation === 'mutation') {
req.alias = `gql${operationName}Mutation`
}
}
})
}
export const openPage = (route: string) => {
aliasGraphql()
return cy.visit(`${Cypress.env('host')}${route}?token=${Cypress.env('accessToken')}`)
}
describe('Root', () => {
beforeEach(() => {
openPage('/')
})
it('Renders cards correctly', () => {
cy.wait('@gqlGetMediaItemsQuery');
})
})
@andrewortwein will that do? I'm afraid I don't have any test artifacts to give you since GitHub didn't save them and I can't reproduce this issue on my laptop.
also, do you think it's possible that upgrading Nodejs version can help here? I think we're on 18.x
@zernie thanks for providing all this info. As you mentioned we aren't properly awaiting network idleness yet. We'll take a look at how feasible it'll be to add that in and will get back to you!
hey @skitterm, any updates on this issue? :)
This is holding us up as well. 😅 Only solutions we have found so far is to add waits at the end of our tests, which is not a real solution.
While also not a great solution, we are having promising results using this plugin as a stopgap: https://github.com/bahmutov/cypress-network-idle
We stuck this in our e2e.js
file
afterEach(() => {
cy.waitForNetworkIdle(100);
});
Hey folks! we haven't moved on this yet but clearly this is a blocking issue and pain point. We'll prioritize this higher and try to get to this within the next week or two!
Hi @skitterm, we've met the same issue, is there any update on this issue?
Hi @Chaos0929, I've begun working on the fix yesterday and hope to have it available within a few days!
As an aside, if you're able to share a spec file (with production version of the URL under test) and Cypress config file, that'd be great! We want to make sure we're repro-ing the exact issue you all are seeing.
Same goes for @3CordGuy and @zernie, if you have a spec file you can share (with a publicly-accessible site-under-test URL so I can repro), that'd be great!
@Chaos0929 @3CordGuy @zernie we've shipped a fix for this -- you'll need to upgrade @chromatic-com/cypress
to version 0.6.14
or greater. Give it a try and let us know if you still run into any issues!
Here's my config:
One of the jobs randomly fails with the following error AFTER it was run: