digidem / mapeo-core-next

The upcoming version of Mapeo Core
MIT License
7 stars 1 forks source link

chore: fix lockfile to include platform-specific binaries #654

Closed achou11 closed 1 month ago

achou11 commented 1 month ago

Restores the inclusion of all platform-specific binaries for @node-rs/crc32 and @bufbuild. Without this, I was not able to run the code locally (usually as tests) on macOS, with an error looking like this:

node:internal/modules/cjs/loader:1140
  const err = new Error(message);
              ^

Error: Cannot find module '@node-rs/crc32-darwin-x64'
Require stack:
- /Users/andrewchou/GitHub/digidem/mapeo-core-next/node_modules/@node-rs/crc32/index.js
- /Users/andrewchou/GitHub/digidem/mapeo-core-next/node_modules/yauzl-promise/lib/zip.js
- /Users/andrewchou/GitHub/digidem/mapeo-core-next/node_modules/yauzl-promise/lib/index.js
    at Module._resolveFilename (node:internal/modules/cjs/loader:1140:15)
    at Module._load (node:internal/modules/cjs/loader:981:27)
    at Module.require (node:internal/modules/cjs/loader:1231:19)
    at require (node:internal/modules/helpers:177:18)
    at Object.<anonymous> (/Users/andrewchou/GitHub/digidem/mapeo-core-next/node_modules/@node-rs/crc32/index.js:105:29)
    at Module._compile (node:internal/modules/cjs/loader:1364:14)
    at Module._extensions..js (node:internal/modules/cjs/loader:1422:10)
    at Module.load (node:internal/modules/cjs/loader:1203:32)
    at Module._load (node:internal/modules/cjs/loader:1019:12)
    at Module.require (node:internal/modules/cjs/loader:1231:19) {
  code: 'MODULE_NOT_FOUND',
  requireStack: [
    '/Users/andrewchou/GitHub/digidem/mapeo-core-next/node_modules/@node-rs/crc32/index.js',
    '/Users/andrewchou/GitHub/digidem/mapeo-core-next/node_modules/yauzl-promise/lib/zip.js',
    '/Users/andrewchou/GitHub/digidem/mapeo-core-next/node_modules/yauzl-promise/lib/index.js'
  ]
}

Git bisect showed that these deps were removed from the lockfile via 9fa05aa99eef1bd2a9e0dab9cc6dd0a7b8069d03. @tomasciccola I'd be curious if there's some different behavior from npm that occurs when you install/update deps? I've run into this problem before where the lockfile unexpectedly changes this dep after you've made updates 🤔

Additionally, wondering what the best way of avoiding this issue moving forward is. Otherwise it will probably keep happening.

achou11 commented 1 month ago

worth keeping in mind that this could very much be some kind of bug around packaging for @node-rs/crc32 (yauzl-promise depends on a version that's about 3 minor releases older)

tomasciccola commented 1 month ago

Mhm! interesting. Yeah, I know my lockfiles have messed you in the past... Do you think installing some dep (like yauzl, i.e.) with --verbose may help??

achou11 commented 1 month ago

I'm surprised npm ci doesn't catch this. From its docs:

If dependencies in the package lock do not match those in package.json, npm ci will exit with an error, instead of updating the package lock.

I would expect that to have caught this...

If i understand that snippet, I wouldn't expect this to be caught. The binaries are a transitive dependency of yauzl-promise via @node-rs/crc32 and are marked as optional dependencies for @node-rs/crc32.

I'm just kind of confused about how to prevent this from happening - seems like it's kind of dependent on the machine that installs/updates the deps, plus some hidden behavior of npm depending on OS (e.g. seems like in Tomas' PR, it kept the binaries needed for Linux and pruned everything else)

EvanHahn commented 1 month ago

I wonder if npm config ls shows anything different on Tomás's machine?

tomasciccola commented 1 month ago

I know this is already close, but this is a dump of npm config list --json. I don't see anything suspicious though...

{
  "json": true,
  "access": null,
  "all": false,
  "allow-same-version": false,
  "also": null,
  "audit": true,
  "audit-level": null,
  "auth-type": "web",
  "before": null,
  "bin-links": true,
  "browser": null,
  "ca": null,
  "cache": "/home/szgy/.npm",
  "cache-max": null,
  "cache-min": 0,
  "cafile": null,
  "call": "",
  "cert": null,
  "cidr": null,
  "color": true,
  "commit-hooks": true,
  "cpu": null,
  "os": null,
  "depth": null,
  "description": true,
  "dev": false,
  "diff": [],
  "diff-ignore-all-space": false,
  "diff-name-only": false,
  "diff-no-prefix": false,
  "diff-dst-prefix": "b/",
  "diff-src-prefix": "a/",
  "diff-text": false,
  "diff-unified": 3,
  "dry-run": false,
  "editor": "nvim",
  "engine-strict": false,
  "fetch-retries": 2,
  "fetch-retry-factor": 10,
  "fetch-retry-maxtimeout": 60000,
  "fetch-retry-mintimeout": 10000,
  "fetch-timeout": 300000,
  "force": false,
  "foreground-scripts": false,
  "format-package-lock": true,
  "fund": true,
  "git": "git",
  "git-tag-version": true,
  "global": false,
  "globalconfig": "/home/szgy/.config/nvm/versions/node/v18.19.1/etc/npmrc",
  "global-style": false,
  "heading": "npm",
  "https-proxy": null,
  "if-present": false,
  "ignore-scripts": false,
  "include": [],
  "include-staged": false,
  "include-workspace-root": false,
  "init-author-email": "",
  "init-author-name": "",
  "init-author-url": "",
  "init-license": "ISC",
  "init-module": "/home/szgy/.npm-init.js",
  "init-version": "1.0.0",
  "init.author.email": "",
  "init.author.name": "",
  "init.author.url": "",
  "init.license": "ISC",
  "init.module": "/home/szgy/.npm-init.js",
  "init.version": "1.0.0",
  "install-links": false,
  "install-strategy": "hoisted",
  "key": null,
  "legacy-bundling": false,
  "legacy-peer-deps": false,
  "link": false,
  "local-address": null,
  "sbom-format": null,
  "sbom-type": "library",
  "location": "user",
  "lockfile-version": null,
  "loglevel": "notice",
  "logs-dir": null,
  "logs-max": 10,
  "long": false,
  "maxsockets": 15,
  "message": "%s",
  "node-options": null,
  "noproxy": [
    ""
  ],
  "offline": false,
  "omit": [],
  "omit-lockfile-registry-resolved": false,
  "only": null,
  "optional": null,
  "otp": null,
  "package": [],
  "package-lock": true,
  "package-lock-only": false,
  "pack-destination": ".",
  "parseable": false,
  "prefer-dedupe": false,
  "prefer-offline": false,
  "prefer-online": false,
  "prefix": "/home/szgy/.config/nvm/versions/node/v18.19.1",
  "preid": "",
  "production": null,
  "progress": true,
  "provenance": false,
  "provenance-file": null,
  "proxy": null,
  "read-only": false,
  "rebuild-bundle": true,
  "registry": "https://registry.npmjs.org/",
  "replace-registry-host": "npmjs",
  "save": true,
  "save-bundle": false,
  "save-dev": false,
  "save-exact": false,
  "save-optional": false,
  "save-peer": false,
  "save-prefix": "^",
  "save-prod": false,
  "scope": "",
  "script-shell": null,
  "searchexclude": "",
  "searchlimit": 20,
  "searchopts": "",
  "searchstaleness": 900,
  "shell": "/usr/bin/bash",
  "shrinkwrap": true,
  "sign-git-commit": false,
  "sign-git-tag": false,
  "strict-peer-deps": false,
  "strict-ssl": true,
  "tag": "latest",
  "tag-version-prefix": "v",
  "timing": false,
  "umask": 0,
  "unicode": true,
  "update-notifier": true,
  "usage": false,
  "user-agent": "npm/{npm-version} node/{node-version} {platform} {arch} workspaces/{workspaces} {ci}",
  "userconfig": "/home/szgy/.npmrc",
  "version": false,
  "versions": false,
  "viewer": "man",
  "which": null,
  "workspace": [],
  "workspaces": null,
  "workspaces-update": true,
  "yes": null,
  "npm-version": "10.2.4"
}