ember-release ember-try scenario is failing

[kellyselden]

https://travis-ci.org/kellyselden/ember-async-action-container/jobs/370742585#L2660

=== Scenario: ember-release ====================================================
Missing npm packages: 
Package: ember-source
  * Specified: https://s3.amazonaws.com/builds.emberjs.com/release/shas/cc511bb5280d5697f2c07379fafe506644df6855.tgz
  * Installed: (not installed)
Run `npm install` to install missing dependencies.
=================================================================================

I think the 3.1.1 tarball failed to publish again.

[rwjblue]

There is a valid tarball at that URL (just downloaded it locally to confirm).

[kellyselden]

This is very confusing. I just reran again to make sure and got the same error. https://travis-ci.org/kellyselden/ember-async-action-container/jobs/370742585

[rwjblue]

Did some debugging (with @kategengler) and this seems like an npm bug. Basically, the ember-source@3.1.1 published tag (which your addon was already using due to ~3.1.1 in its package.json) and the current release tarball are exactly the same (sha512 integrity checksum matches), and npm is getting confused and pruning out the newly installed package.

detailed log

``` ~/s/s/ember-async-action-container ❯❯❯ DEBUG=ember-try:* ember try:one ember-release --skip-cleanup ember-try:commands:try-one Scenario argument: ember-release +0ms ember-try:commands:try-one Command options: ember-try:commands:try-one { skipCleanup: true, configPath: 'config/ember-try.js', checkForUpdates: false, disableAnalytics: false } +1ms ember-try:commands:try-one Command specified on command line: +1ms ember-try:commands:try-one Config: {"npmOptions":["--verbose"],"scenarios":[{"name":"ember-lts-2.12","npm":{"devDependencies":{"ember-source":"~2.12.0"}}},{"name":"ember-lts-2.16","npm":{"devDependencies":{"ember-source":"~2.16.0"}}},{"name":"ember-lts-2.18","npm":{"devDependencies":{"ember-source":"~2.18.0"}}},{"name":"ember-release","npm":{"devDependencies":{"ember-source":"https://s3.amazonaws.com/builds.emberjs.com/release/shas/cc511bb5280d5697f2c07379fafe506644df6855.tgz"}}},{"name":"ember-beta","npm":{"devDependencies":{"ember-source":"https://s3.amazonaws.com/builds.emberjs.com/beta/shas/a607badb43df439a37e98527cac036f1492d4c36.tgz"}}},{"name":"ember-canary","npm":{"devDependencies":{"ember-source":"https://s3.amazonaws.com/builds.emberjs.com/canary/shas/24b56fae2b90cc32f9fffde3a2c69d067bb6a2ed.tgz"}}},{"name":"ember-default","npm":{"devDependencies":{}}}]} +262ms ember-try:task:try-each DependencyManagerAdapters: npm,bower +49ms ember-try:dependency-manager-adapter:npm Backing up package.json and node_modules +1ms ember-try:task:try-each Scenario Manager setup +7s ember-try:dependency-manager-adapter:npm Changing to dependency set: {"devDependencies":{"ember-source":"https://s3.amazonaws.com/builds.emberjs.com/release/shas/cc511bb5280d5697f2c07379fafe506644df6855.tgz"}} +1ms ember-try:dependency-manager-adapter:npm Write package.json with: ember-try:dependency-manager-adapter:npm {"name":"ember-async-action-container","version":"1.0.0","description":"An async state wrapper","keywords":["ember-addon"],"license":"MIT","author":"Kelly Selden","directories":{"doc":"doc","test":"tests"},"repository":"https://github.com/kellyselden/ember-async-action-container.git","scripts":{"build":"ember build","lint:js":"eslint ./*.js addon addon-test-support app config lib server test-support tests","start":"ember serve","test":"ember test","test:all":"ember try:each"},"dependencies":{"ember-cli-babel":"^6.6.0","ember-cli-htmlbars":"^2.0.1","ember-defer":"^1.0.1"},"devDependencies":{"broccoli-asset-rev":"^2.4.5","ember-ajax":"^3.0.0","ember-cli":"~3.1.2","ember-cli-dependency-checker":"^2.0.0","ember-cli-eslint":"^4.2.1","ember-cli-htmlbars-inline-precompile":"^1.0.0","ember-cli-inject-live-reload":"^1.4.1","ember-cli-qunit":"^4.1.1","ember-cli-shims":"^1.2.0","ember-cli-sri":"^2.1.0","ember-cli-uglify":"^2.0.0","ember-disable-prototype-extensions":"^1.1.2","ember-export-application-global":"^2.0.0","ember-load-initializers":"^1.0.0","ember-maybe-import-regenerator":"^0.1.6","ember-resolver":"^4.0.0","ember-source":"https://s3.amazonaws.com/builds.emberjs.com/release/shas/cc511bb5280d5697f2c07379fafe506644df6855.tgz","ember-source-channel-url":"^1.0.1","ember-try":"^0.2.23","eslint-plugin-ember":"^5.0.0","eslint-plugin-node":"^6.0.1","loader.js":"^4.2.3","qunit-dom":"^0.6.2"},"engines":{"node":"^4.5 || 6.* || >= 7.*"},"ember-addon":{"configPath":"tests/dummy/config"}} +0ms ember-try:dependency-manager-adapter:npm Run npm install with options --verbose +1ms ember-try:utils:run spawned with sh [ '-c', 'npm install --verbose --no-shrinkwrap' ] { cwd: '/Users/rjackson/src/sandbox/ember-async-action-container', stdio: 'inherit' } +4ms npm info it worked if it ends with ok npm verb cli [ '/Users/rjackson/.nvm/versions/node/v9.4.0/bin/node', npm verb cli '/Users/rjackson/.nvm/versions/node/v9.4.0/bin/npm', npm verb cli 'install', npm verb cli '--verbose', npm verb cli '--no-shrinkwrap' ] npm info using npm@5.7.1 npm info using node@v9.4.0 npm verb npm-session e72b9e15a9e56d18 npm info lifecycle ember-async-action-container@1.0.0~preinstall: ember-async-action-container@1.0.0 npm http fetch GET 200 https://s3.amazonaws.com/builds.emberjs.com/release/shas/cc511bb5280d5697f2c07379fafe506644df6855.tgz 130ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/broccoli-funnel 226ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/ember-cli-get-component-path-option 226ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/broccoli-merge-trees 235ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/ember-cli-path-utils 236ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/jquery 238ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/ember-cli-valid-component-name 241ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/ember-router-generator 243ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/recast 45ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/esprima 49ms (from cache) npm http fetch GET 304 https://registry.npmjs.org/ast-types 64ms (from cache) npm verb correctMkdir /Users/rjackson/.npm/_locks correctMkdir not in flight; initializing npm verb lock using /Users/rjackson/.npm/_locks/staging-fc8cfbb67d9fe01d.lock for /Users/rjackson/src/sandbox/ember-async-action-container/node_modules/.staging npm verb unlock done using /Users/rjackson/.npm/_locks/staging-fc8cfbb67d9fe01d.lock for /Users/rjackson/src/sandbox/ember-async-action-container/node_modules/.staging npm info linkStuff ember-async-action-container@1.0.0 npm verb linkBins ember-async-action-container@1.0.0 npm verb linkMans ember-async-action-container@1.0.0 npm info lifecycle ember-async-action-container@1.0.0~install: ember-async-action-container@1.0.0 npm info lifecycle ember-async-action-container@1.0.0~postinstall: ember-async-action-container@1.0.0 npm info lifecycle ember-async-action-container@1.0.0~prepublish: ember-async-action-container@1.0.0 npm info lifecycle ember-async-action-container@1.0.0~prepare: ember-async-action-container@1.0.0 npm verb saving [] up to date in 6.857s npm verb exit [ 0, true ] npm info lifecycle undefined~preshrinkwrap: undefined npm info ok ember-try:utils:run Process exited 0 +7s ember-try:dependency-manager-adapter:npm Run npm prune +0ms ember-try:utils:run spawned with sh [ '-c', 'npm prune --verbose --no-shrinkwrap' ] { cwd: '/Users/rjackson/src/sandbox/ember-async-action-container', stdio: 'inherit' } +4ms npm info it worked if it ends with ok npm verb cli [ '/Users/rjackson/.nvm/versions/node/v9.4.0/bin/node', npm verb cli '/Users/rjackson/.nvm/versions/node/v9.4.0/bin/npm', npm verb cli 'prune', npm verb cli '--verbose', npm verb cli '--no-shrinkwrap' ] npm info using npm@5.7.1 npm info using node@v9.4.0 npm verb npm-session 36a1b54cb54a6760 npm verb correctMkdir /Users/rjackson/.npm/_locks correctMkdir not in flight; initializing npm verb lock using /Users/rjackson/.npm/_locks/staging-fc8cfbb67d9fe01d.lock for /Users/rjackson/src/sandbox/ember-async-action-container/node_modules/.staging npm info lifecycle ember-source@3.1.1~preuninstall: ember-source@3.1.1 npm info lifecycle ember-source@3.1.1~uninstall: ember-source@3.1.1 npm verb unbuild rmStuff ember-source@3.1.1 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm info lifecycle ember-source@3.1.1~postuninstall: ember-source@3.1.1 npm info lifecycle ember-router-generator@1.2.3~preuninstall: ember-router-generator@1.2.3 npm info lifecycle ember-router-generator@1.2.3~uninstall: ember-router-generator@1.2.3 npm verb unbuild rmStuff ember-router-generator@1.2.3 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm info lifecycle ember-router-generator@1.2.3~postuninstall: ember-router-generator@1.2.3 npm info lifecycle recast@0.11.23~preuninstall: recast@0.11.23 npm info lifecycle recast@0.11.23~uninstall: recast@0.11.23 npm verb unbuild rmStuff recast@0.11.23 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm info lifecycle recast@0.11.23~postuninstall: recast@0.11.23 npm info lifecycle esprima@3.1.3~preuninstall: esprima@3.1.3 npm info lifecycle esprima@3.1.3~uninstall: esprima@3.1.3 npm verb unbuild rmStuff esprima@3.1.3 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm verb unbuild rmStuff in node_modules/recast/node_modules npm info lifecycle esprima@3.1.3~postuninstall: esprima@3.1.3 npm info lifecycle jquery@3.3.1~preuninstall: jquery@3.3.1 npm info lifecycle jquery@3.3.1~uninstall: jquery@3.3.1 npm verb unbuild rmStuff jquery@3.3.1 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm info lifecycle jquery@3.3.1~postuninstall: jquery@3.3.1 npm info lifecycle broccoli-merge-trees@2.0.0~preuninstall: broccoli-merge-trees@2.0.0 npm info lifecycle broccoli-merge-trees@2.0.0~uninstall: broccoli-merge-trees@2.0.0 npm verb unbuild rmStuff broccoli-merge-trees@2.0.0 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm verb unbuild rmStuff in node_modules/ember-source/node_modules npm info lifecycle broccoli-merge-trees@2.0.0~postuninstall: broccoli-merge-trees@2.0.0 npm info lifecycle broccoli-funnel@2.0.1~preuninstall: broccoli-funnel@2.0.1 npm info lifecycle broccoli-funnel@2.0.1~uninstall: broccoli-funnel@2.0.1 npm verb unbuild rmStuff broccoli-funnel@2.0.1 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm verb unbuild rmStuff in node_modules/ember-source/node_modules npm info lifecycle broccoli-funnel@2.0.1~postuninstall: broccoli-funnel@2.0.1 npm info lifecycle ember-cli-valid-component-name@1.0.0~preuninstall: ember-cli-valid-component-name@1.0.0 npm info lifecycle ember-cli-valid-component-name@1.0.0~uninstall: ember-cli-valid-component-name@1.0.0 npm verb unbuild rmStuff ember-cli-valid-component-name@1.0.0 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm info lifecycle ember-cli-valid-component-name@1.0.0~postuninstall: ember-cli-valid-component-name@1.0.0 npm info lifecycle ember-cli-path-utils@1.0.0~preuninstall: ember-cli-path-utils@1.0.0 npm info lifecycle ember-cli-path-utils@1.0.0~uninstall: ember-cli-path-utils@1.0.0 npm verb unbuild rmStuff ember-cli-path-utils@1.0.0 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm info lifecycle ember-cli-path-utils@1.0.0~postuninstall: ember-cli-path-utils@1.0.0 npm info lifecycle ember-cli-get-component-path-option@1.0.0~preuninstall: ember-cli-get-component-path-option@1.0.0 npm info lifecycle ember-cli-get-component-path-option@1.0.0~uninstall: ember-cli-get-component-path-option@1.0.0 npm verb unbuild rmStuff ember-cli-get-component-path-option@1.0.0 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm info lifecycle ember-cli-get-component-path-option@1.0.0~postuninstall: ember-cli-get-component-path-option@1.0.0 npm info lifecycle ast-types@0.9.6~preuninstall: ast-types@0.9.6 npm info lifecycle ast-types@0.9.6~uninstall: ast-types@0.9.6 npm verb unbuild rmStuff ast-types@0.9.6 from /Users/rjackson/src/sandbox/ember-async-action-container/node_modules npm info lifecycle ast-types@0.9.6~postuninstall: ast-types@0.9.6 npm verb unlock done using /Users/rjackson/.npm/_locks/staging-fc8cfbb67d9fe01d.lock for /Users/rjackson/src/sandbox/ember-async-action-container/node_modules/.staging removed 11 packages in 4.958s npm verb exit [ 0, true ] npm info ok ember-try:utils:run Process exited 0 +6s ember-try:dependency-manager-adapter:npm Switched to dependencies: ember-try:dependency-manager-adapter:npm [ { name: 'ember-source', versionExpected: 'https://s3.amazonaws.com/builds.emberjs.com/release/shas/cc511bb5280d5697f2c07379fafe506644df6855.tgz', versionSeen: null, packageManager: 'npm' } ] +1ms ember-try:dependency-manager-adapter:bower Changing to dependency set: {} +0ms === Scenario: ember-release ==================================================== ember-try:task:try-each With: ember-try:task:try-each { scenario: 'ember-release', allowedToFail: false, dependencyState: [ { name: 'ember-source', versionExpected: 'https://s3.amazonaws.com/builds.emberjs.com/release/shas/cc511bb5280d5697f2c07379fafe506644df6855.tgz', versionSeen: null, packageManager: 'npm' } ], command: 'ember test' } +2ms ember-try:utils:run spawned with sh [ '-c', 'node "/Users/rjackson/src/sandbox/ember-async-action-container/node_modules/ember-cli/bin/ember" test' ] { cwd: '/Users/rjackson/src/sandbox/ember-async-action-container', stdio: 'inherit' } +8ms Missing npm packages: Package: ember-source * Specified: https://s3.amazonaws.com/builds.emberjs.com/release/shas/cc511bb5280d5697f2c07379fafe506644df6855.tgz * Installed: (not installed) Run `npm install` to install missing dependencies. Stack Trace and Error Report: /var/folders/mt/yt2qjpl93ls98lrkrs81rzch000pzv/T/error.dump.cc4d3dd87e00916e3848f2f0efc332e0.log An error occurred in the constructor for ember-cli-dependency-checker at /Users/rjackson/src/sandbox/ember-async-action-container/node_modules/ember-cli-dependency-checker ember-try:utils:run Process exited 1 +1s Result: false --- ember-try:task:try-each Cleanup +2ms ember-try:task:try-each Skip ScenarioManager cleanup +0ms ember-try:task:try-each Output results +0ms ------ RESULTS ------ Scenario ember-release: FAIL Command run: ember test ┌────────────────────┬────────────────────┬──────────────────────────────┬──────────┐ │ Dependency │ Expected │ Used │ Type │ ├────────────────────┼────────────────────┼──────────────────────────────┼──────────┤ │ ember-source │ https://s3.amazon… │ Not Installed │ npm │ └────────────────────┴────────────────────┴──────────────────────────────┴──────────┘ 1 scenarios failed 0 scenarios succeeded 1 scenarios run ember-try:task:try-each Exit 1 +27ms ```

[kellyselden]

woah that explains it! since it was a brand new addon.

[rwjblue]

We can "fix" it by retriggering the CI build for the release branch (because the tarball would generate and have a very small difference if only in timestamps from 3.1.1)...

Kinda sucks though 😭

[kellyselden]

yeah. Can we mitigate in ember-try that says "if shas match, no-op"? at least until npm fixes.

[rwjblue]

There are a few possible mitigations:

1. remove node_modules before attempting the npm install after updating the package.json
2. remove node_modules/<specific packages configured in ember-try config> after updating the package.json
3. allow disabling npm prune after ember-try has npm install'ed with the new deps

I think the second seems the most likely to succeed...

Thoughts @kategengler ?

[kategengler]

I like the second, but think that is what it originally did, and moved to prune for some reason.

On Tue, Apr 24, 2018 at 6:15 PM Robert Jackson notifications@github.com wrote:

There are a few possible mitigations:

1. remove node_modules before attempting the npm install after updating the package.json
2. remove node_modules/<specific packages configured in ember-try config> after updating the package.json
3. allow disabling npm prune after ember-try has npm install'ed with the new deps

I think the second seems the most likely to succeed...

Thoughts @kategengler https://github.com/kategengler ?

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub https://github.com/emberjs/ember.js/issues/16580#issuecomment-384097206, or mute the thread https://github.com/notifications/unsubscribe-auth/AAbHOp1fpCqiRH5pamKlbD9UDBcNoU2pks5tr6P8gaJpZM4TiVXA .

[yininge]

Since we have an open PR to avoid the problem, let's labeled this with has PR Perhaps we can close this once the PR is merged? What do you think? @Panman8201 @kategengler @kellyselden @rwjblue

[kategengler]

@yininge Which PR is for this? I haven't seen one

[yininge]

@kategengler sorry it is my bad. I saw an issue and thought it is a PR.

[rwjblue]

FWIW, I think that @kategengler and I figured out a more general possible solution here. Hopefully one of us has time to implement in the next week or two. 😩

[kategengler]

We believe we've fixed this by changing the way Ember publishes builds to s3 (for use in ember-try among other places.

@kellyselden Can you confirm this is fixed?

[kellyselden]

I haven't seen it for a while.

[Panman82]

FWIW, I seen this twice and it fixes itself after the next release. So it would be tough to say now, without it being an issue with 3.6.1 (current release).

[rwjblue]

Agreed @Panman8201. If folks do see it again, please feel free to chime in here or open up a new issue...

[MrAvantiC]

Hi there, unfortunately I'm discovering this at the moment when trying to deploy an Ember application to Netlify.

Version-Info ember version --verbose && npm --version

ember-cli: 3.8.0-beta.1
http_parser: 2.8.0
node: 10.15.0
v8: 6.8.275.32-node.45
uv: 1.23.2
zlib: 1.2.11
ares: 1.15.0
modules: 64
nghttp2: 1.34.0
napi: 3
openssl: 1.1.0j
icu: 62.1
unicode: 11.0
cldr: 33.1
tz: 2018e
os: darwin x64
6.7.0

Dependencies:

  "devDependencies": {
    "@ember-decorators/babel-transforms": "^5.1.2",
    "@ember/optional-features": "^0.6.3",
    "babel-eslint": "^10.0.1",
    "broccoli-asset-rev": "^2.7.0",
    "ember-auto-import": "^1.2.19",
    "ember-bind-helper": "^0.3.2",
    "ember-cli": "github:ember-cli/ember-cli",
    "ember-cli-app-version": "^3.2.0",
    "ember-cli-babel": "^7.1.0",
    "ember-cli-dependency-checker": "^3.0.0",
    "ember-cli-eslint": "^4.2.3",
    "ember-cli-fastboot": "^2.0.4",
    "ember-cli-htmlbars": "^3.0.0",
    "ember-cli-htmlbars-inline-precompile": "^1.0.3",
    "ember-cli-inject-live-reload": "^1.8.2",
    "ember-cli-markdown-resolver": "^0.1.3",
    "ember-cli-showdown": "^4.4.4",
    "ember-cli-sri": "^2.1.1",
    "ember-cli-stylus": "^1.0.7",
    "ember-cli-template-lint": "^1.0.0-beta.1",
    "ember-cli-uglify": "^2.1.0",
    "ember-decorators": "^5.1.2",
    "ember-export-application-global": "^2.0.0",
    "ember-load-initializers": "^1.1.0",
    "ember-maybe-import-regenerator": "^0.1.6",
    "ember-prism": "^0.4.0",
    "ember-qunit": "^3.4.1",
    "ember-resolver": "^5.0.1",
    "eslint-plugin-ember": "^5.2.0",
    "loader.js": "^4.7.0",
    "prember": "^1.0.2",
    "prettier": "^1.16.4",
    "qunit-dom": "^0.8.0",
    "sparkles-component": "^1.3.0",
    "sparkles-decorators": "^0.0.6"
  },

As soon as Netlify installs the dependencies and runs ember build -e production I get the error. I can also reproduce this locally by removing my node_modules, running npm install and starting the development server.

I tried adding "ember-source": "https://s3.amazonaws.com/builds.emberjs.com/canary/shas/e7a144b1bf432769383c3360bbcc44a68e2b9522.tgz" explicitly as a dependency but that did not work either.

The only way to get the deployment working is by running npm install ember-source && ember build -e production instead of just ember build -e production

...but that doesn't feel quite right. :/