Vercel edge runtime: Module not found

[louis030195]

Hi, amazing work @dqbd making tiktoken JS compatible :)

I have an issue running it on vercel edge runtime though, been trying different things without success :(

const config = {
    webpack(config, { isServer, dev }) {
        config.experiments = {
            asyncWebAssembly: true,
            layers: true,
        };

        return config;
    },
};

module.exports = config;

// @ts-expect-error
import wasm from "@dqbd/tiktoken/lite/tiktoken_bg.wasm?module";
import model from "@dqbd/tiktoken/encoders/cl100k_base.json";
import { init, Tiktoken } from "@dqbd/tiktoken/lite/init";
export async function splitText(
    text: string,
    {
        maxTokens = MAX_CHUNK_LENGTH,
        chunkOverlap = CHUNK_OVERLAP,
        encodingName = EMBEDDING_ENCODING,
    }: { maxTokens?: number; chunkOverlap?: number; encodingName?: any },
    callback?: (chunk: SplitTextChunk) => void
): Promise<SplitTextChunk[]> {
    if (chunkOverlap >= maxTokens) {
        throw new Error('Cannot have chunkOverlap >= chunkSize')
    }
    await init((imports) => WebAssembly.instantiate(wasm, imports));

    const encoding = new Tiktoken(
        model.bpe_ranks,
        model.special_tokens,
        model.pat_str
    );

    const input_ids = encoding.encode(text)
    const chunkSize = maxTokens

    let start_idx = 0
    let cur_idx = Math.min(start_idx + chunkSize, input_ids.length)
    let chunk_ids = input_ids.slice(start_idx, cur_idx)

    const decoder = new TextDecoder()
    const chunks = []

    while (start_idx < input_ids.length) {
        const chunk = decoder.decode(encoding.decode(chunk_ids))
        const chunkItem = { chunk, start: start_idx, end: cur_idx }
        chunks.push(chunkItem)
        callback && callback(chunkItem)
        start_idx += chunkSize - chunkOverlap
        cur_idx = Math.min(start_idx + chunkSize, input_ids.length)
        chunk_ids = input_ids.slice(start_idx, cur_idx)
    }
    encoding.free()
    return chunks
}

export const config = {
    runtime: "edge",
};
// ....

Module not found: Package path ./lite/tiktoken_bg.wasm?module is not exported from package /workspaces/app/node_modules/@dqbd/tiktoken (see exports field in /workspaces/app/node_modules/@dqbd/tiktoken/package.json) 5 | // @ts-expect-error

6 | import wasm from "@dqbd/tiktoken/lite/tiktoken_bg.wasm?module"; 7 | import model from "@dqbd/tiktoken/encoders/cl100k_base.json"; 8 | import { init, Tiktoken } from "@dqbd/tiktoken/lite/init"; 9 |

https://nextjs.org/docs/messages/module-not-found

Import trace for requested module: ./node_modules/next/dist/build/webpack/loaders/next-edge-function-loader.js?absolutePagePath=%2Fworkspaces%2Fapp%2Fpages%2Fapi%2Fask.tsx&page=%2Fapi%2Fask&rootDir=%2Fworkspaces%2Fapp! wait - compiling... error - ./pages/api/ask.tsx:6:0 Module not found: Package path ./lite/tiktoken_bg.wasm?module is not exported from package /workspaces/app/node_modules/@dqbd/tiktoken (see exports field in /workspaces/app/node_modules/@dqbd/tiktoken/package.json)

Any idea of whats wrong?

[dqbd]

Hi! That's seems weird, will take a look :)

[dqbd]

@louis030195 It seems like you need to pass a relative URL to the WASM file, as there is a conflict with the exports field. Will look into it, but in the meantime, the following should work:

// @ts-expect-error
import wasm from "../../../../node_modules/@dqbd/tiktoken/lite/tiktoken_bg.wasm?module";
import model from "@dqbd/tiktoken/encoders/cl100k_base.json";
import { init, Tiktoken } from "@dqbd/tiktoken/lite/init";

export const config = { runtime: "edge" };

async function handler(req: Request) {
  await init((imports) => WebAssembly.instantiate(wasm, imports));

  const encoding = new Tiktoken(
    model.bpe_ranks,
    model.special_tokens,
    model.pat_str
  );

  const tokens = encoding.encode("hello world");
  encoding.free();

  return new Response(`${tokens}`);
}

export default handler;

Update: I believe this is more of an issue of Next.JS, as providing a new exports for tiktoken_bg.wasm?module causes an another error in Next.JS (Failed to parse URL...)

[louis030195]

@dqbd thanks a lot for the quick response, it works :)

[dqbd]

Another possible solution is to enforce the next-middleware-wasm-loader as the loader for the file.

// @ts-expect-error
import wasm from "!next-middleware-wasm-loader!@dqbd/tiktoken/lite/tiktoken_bg.wasm?module";
import model from "@dqbd/tiktoken/encoders/cl100k_base.json";
import { init, Tiktoken } from "@dqbd/tiktoken/lite/init";

export const config = { runtime: "edge" };

async function handler(req: Request) {
  await init((imports) => WebAssembly.instantiate(wasm, imports));

  const encoding = new Tiktoken(
    model.bpe_ranks,
    model.special_tokens,
    model.pat_str
  );

  const tokens = encoding.encode("hello world");
  encoding.free();

  return new Response(`${tokens}`);
}

export default handler;

[dqbd]

Finally, the issue should be fixed in 1.0.4, no need to use relative paths or force a different loader.