error - Error: Cannot find module './dist/tiktoken-node.darwin-arm64.node'

[danrasmuson]

My code...

const tiktoken = require('tiktoken-node')
let enc = tiktoken.getEncoding("gpt2")
console.assert(enc.decode(enc.encode("hello world")) == "hello world")

My error...

error - Error: Cannot find module './dist/tiktoken-node.darwin-arm64.node'
Require stack:
- /Users/danielrasmuson/code/humata/app/node_modules/tiktoken-node/index.cjs
- /Users/danielrasmuson/code/humata/app/.next/server/pages/api/hello.js
- /Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/next-server.js
- /Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/dev/next-dev-server.js
- /Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/next.js
- /Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/lib/start-server.js
- /Users/danielrasmuson/code/humata/app/node_modules/next/dist/cli/next-dev.js
- /Users/danielrasmuson/code/humata/app/node_modules/next/dist/lib/commands.js
- /Users/danielrasmuson/code/humata/app/node_modules/next/dist/bin/next
    at Function.Module._resolveFilename (node:internal/modules/cjs/loader:985:15)
    at Function.mod._resolveFilename (/Users/danielrasmuson/code/humata/app/node_modules/next/dist/build/webpack/require-hook.js:23:32)
    at Function.Module._load (node:internal/modules/cjs/loader:833:27)
    at Module.require (node:internal/modules/cjs/loader:1057:19)
    at require (node:internal/modules/cjs/helpers:103:18)
    at Object.<anonymous> (/Users/danielrasmuson/code/humata/app/node_modules/tiktoken-node/index.cjs:13:25)
    at Module._compile (node:internal/modules/cjs/loader:1155:14)
    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1209:10)
    at Module.load (node:internal/modules/cjs/loader:1033:32)
    at Function.Module._load (node:internal/modules/cjs/loader:868:12)
    at Module.require (node:internal/modules/cjs/loader:1057:19)
    at require (node:internal/modules/cjs/helpers:103:18)
    at Object.tiktoken-node (/Users/danielrasmuson/code/humata/app/.next/server/pages/api/hello.js:22:18)
    at __webpack_require__ (/Users/danielrasmuson/code/humata/app/.next/server/webpack-api-runtime.js:33:42)
    at eval (webpack-internal:///(api)/./src/pages/api/hello.ts:5:18)
    at Object.(api)/./src/pages/api/hello.ts (/Users/danielrasmuson/code/humata/app/.next/server/pages/api/hello.js:32:1)
    at __webpack_require__ (/Users/danielrasmuson/code/humata/app/.next/server/webpack-api-runtime.js:33:42)
    at __webpack_exec__ (/Users/danielrasmuson/code/humata/app/.next/server/pages/api/hello.js:42:39)
    at /Users/danielrasmuson/code/humata/app/.next/server/pages/api/hello.js:43:28
    at Object.<anonymous> (/Users/danielrasmuson/code/humata/app/.next/server/pages/api/hello.js:46:3)
    at Module._compile (node:internal/modules/cjs/loader:1155:14)
    at Object.Module._extensions..js (node:internal/modules/cjs/loader:1209:10)
    at Module.load (node:internal/modules/cjs/loader:1033:32)
    at Function.Module._load (node:internal/modules/cjs/loader:868:12)
    at Module.require (node:internal/modules/cjs/loader:1057:19)
    at require (node:internal/modules/cjs/helpers:103:18)
    at DevServer.runApi (/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/next-server.js:481:34)
    at DevServer.handleApiRequest (/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/next-server.js:827:21)
    at processTicksAndRejections (node:internal/process/task_queues:96:5)
    at async Object.fn (/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/next-server.js:751:37)
    at async Router.execute (/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/router.js:253:36)
    at async DevServer.run (/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/base-server.js:384:29)
    at async DevServer.run (/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/dev/next-dev-server.js:743:20)
    at async DevServer.handleRequest (/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/base-server.js:322:20) {
  code: 'MODULE_NOT_FOUND',
  requireStack: [
    '/Users/danielrasmuson/code/humata/app/node_modules/tiktoken-node/index.cjs',
    '/Users/danielrasmuson/code/humata/app/.next/server/pages/api/hello.js',
    '/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/next-server.js',
    '/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/dev/next-dev-server.js',
    '/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/next.js',
    '/Users/danielrasmuson/code/humata/app/node_modules/next/dist/server/lib/start-server.js',
    '/Users/danielrasmuson/code/humata/app/node_modules/next/dist/cli/next-dev.js',
    '/Users/danielrasmuson/code/humata/app/node_modules/next/dist/lib/commands.js',
    '/Users/danielrasmuson/code/humata/app/node_modules/next/dist/bin/next'
  ]
}

Any suggestions for debugging?

[danrasmuson]

I can confirm here that ./dist/tiktoken-node.darwin-arm64.node is not present in my dist file

[ceifa]

Hey! Thanks for reporting this. This is happening because the MacOS version for Apple Silicon processors was not compiled, and I would need a MacOS to fix this, but I don't have one.

If you want to contribute, you can compile and open a PR for us, take this one as reference: https://github.com/ceifa/steamworks.js/pull/17