Decoder crashes on medium data (~50MB)

[inklesspen1rus]

Here is example of project: ZstdBug.zip

It crashes when I run it (npm run test). Error on last test: errLog.txt

I tested asm and wasm decoders, both are broken. Also on Mozilla Firefox I got correct work with "data" file and ZstdStream decodec but missing chunks with ZstdSimple decodec

npm run test writes stderr into errLog.txt

[tamascsaba]

Thank you very much for sending the error report, @inklesspen1rus. This week, I will take a look at it.

[inklesspen1rus]

Thank you very much for sending the error report, @inklesspen1rus. This week, I will take a look at it.

Is it still "this" week? =) I guess you just does not pass ALLOW_MEMORY_GROW option to emcc

[6XGate]

I'm seeing this exact same issue with ArchLinux packages. For example from the ArchLinux Chromium package.

Here is some example code I used:

zstd.ts

import { ZstdInit } from "@oneidentity/zstd-js";
import transformStream from "easy-transform-stream";
import { isArrayBuffer, isArrayBufferView, isDataView, isTypedArray } from "node:util/types";

const { ZstdStream } = await ZstdInit()

function toBuffer(chunk: unknown) {
  if (typeof chunk === 'string') return Buffer.from(chunk)
  if (isArrayBuffer(chunk)) return Buffer.from(chunk)
  if (isArrayBufferView(chunk) || isDataView(chunk) || isTypedArray(chunk)) return Buffer.from(chunk.buffer)
  throw new TypeError('Chunk must be a string, buffer, typed array, array buffer, or view of those')
}

export function createCompress () {
  return transformStream(async chunk => ZstdStream.compress(toBuffer(chunk)))
}

export function createDecompress () {
  return transformStream(async chunk => ZstdStream.decompress(toBuffer(chunk)))
}

package.ts

import { createReadStream } from "node:fs"
import { createDecompress } from "./zstd.js"
import { extract } from "tar-stream"
import { posix } from "node:path/posix"
import type { Readable } from "node:stream"
import { createInterface } from "node:readline"

async function readPackageInfo(stream: Readable) {
  const reader = createInterface(stream)
  for await (const line of reader) {
    console.log(line)
  }
}

export async function getPackageInfo(filename: string) {
  const file = createReadStream(filename)
  const decompress = createDecompress()
  const stream = extract()
  file.pipe(decompress).pipe(stream)

  for await (const entry of stream) {
    if (posix.basename(entry.header.name) === '.PKGINFO') {
      return readPackageInfo(entry)
    } else {
      entry.resume()
    }
  }

  return null
}

index.ts

import process from 'node:process'
import { getPackageInfo } from './package.js'
async function main() {
  await getPackageInfo('path/to/chromium-121.0.6167.184-1-x86_64.pkg.tar.zst')
  return 0
}
process.exitCode = await main()