Issue with transcoding into aac / m4a [Attached reproable code]

[kartikdutt18]

I'm trying to convert an aac file into an mp4 / m4a / aac file. I've made changes in the test-decode-audio.js and using that to transcode the aac file.

Issue: It fails in the libav.ff_write_multi step with a division by zero error. On debugging (compiling with g3 flag instead), the error occurs when frac_add method is called by compute_muxer_pkt_fields and it fails there. Stack: av_interleaved_write_frame -> write_packets_common -> write_packet_common -> compute_muxer_pkt_fields -> frac_add (where it throws)

Hi @Yahweasel, Can you please help me figure out the issue here. Thanks! I've attached code, flags used and some other things I tried.

Code file:

if (typeof process !== "undefined") {
  // Node.js
  LibAV = require("../libav-<version>-all.js");
  fs = require("fs");
  OpusExa = Function(
    fs.readFileSync("exa.opus.js", "utf8") + "return OpusExa;"
  )();
}

function makeCodecOpts(frame, libav) {
  return {
    ctx: {
      bit_rate: 128000,
      sample_fmt: frame?.format ?? libav.AV_SAMPLE_FMT_FLTP,
      sample_rate: 48000,
      channel_layout: frame?.channel_layout,
      channels: frame?.channels,
      frame_size: frame?.nb_samples,
    },
    time_base: [1, 48000],
    options: {
      rc_mode: 'timestamp',
    },
  };
}

async function readFile(libav, audio_stream_idx, formatCtx, pkt) {
  const packets = [];
  for (;;) {
    const ret = await libav.ff_read_multi(formatCtx, pkt, undefined, {
      limit: 100,
    });
    if (ret[1][audio_stream_idx] !== undefined) {
      for (const p of ret[1][audio_stream_idx]) {
        packets.push(p);
      }
    }

    if (ret[0] === libav.AVERROR_EOF) {
      break;
    }
  }
  return packets;
}

async function decodeAudio(libav) {
  const response = await fetch(
    "https://osizewuspersimmon001.blob.core.windows.net/audios/publish/e4d72f8e-cb51-4784-b728-9b9b2ce72c24/fun_Block_Party_MSFT_MSTR_64.aac"
  );
  const data = new Uint8Array(await response.arrayBuffer());
  const filename = `AudioPlayer0.aac`;
  await libav.writeFile(filename, new Uint8Array(data));
  const readDemuxer = await libav.ff_init_demuxer_file(filename);
  const formatCtx = readDemuxer[0];
  let audioStream;
  for (const s of readDemuxer[1] /* readDemuxer = [cntx, streams] */) {
    if (s.codec_type === libav.AVMEDIA_TYPE_AUDIO) {
      audioStream = s;
      break;
    }
  }

  // await libav.AVStream_duration_s(, 83)
  if (audioStream === undefined) {
    // No audio to add.
    throw new Error("No audio stream");
  }

  const rDecoder = await libav.ff_init_decoder(
    audioStream.codec_id,
    audioStream.codecpar
  );
  const c = rDecoder[1];
  const pkt = rDecoder[2];
  const frame = rDecoder[3];
  // await libav.AVPacket_duration_s(pkt, 83);
  const packets = await readFile(libav, audioStream.index, formatCtx, pkt);
  const frames = await libav.ff_decode_multi(c, pkt, frame, packets, true);
  return {
    frames,
    c,
    pkt,
    frame,
    formatCtx,
    timebase: { num: audioStream.time_base_num, den: audioStream.time_base_den}
  };
}

async function RunExampleToTranscodeMp4(libav) {

  // Decode audio
  const audioEncoderData = await decodeAudio(libav);

  // Init encoder
  const encoder = await libav.ff_init_encoder(
    "aac",
    makeCodecOpts(
      audioEncoderData.frames.length > 0
        ? audioEncoderData.frames[0]
        : undefined,
      libav
    )
  );
  const encoderAVCodecContext = encoder[1];
  const encoderFramePointer = encoder[2];
  const encoderPacketHandler = encoder[3];

  const filename = "tmp.aac";
  const muxR = await libav.ff_init_muxer(
    { filename: filename, open: true },
    [encoderAVCodecContext, audioEncoderData.timebase.num, audioEncoderData.timebase.den]
  );
  const destinationContext = muxR[0];
  const pb = muxR[2];
  await libav.avformat_write_header(destinationContext, 0);

  const retL = await libav.ff_add_stream(
    destinationContext,
    encoderAVCodecContext,
    1,
    48000
  );
  const streamPacketHandler = retL[1];
  const streamIdx = retL[0];

  // Write frames
  let packets = await libav.ff_encode_multi(
    encoderAVCodecContext,
    encoderFramePointer,
    encoderPacketHandler,
    audioEncoderData.frames,
    true
  );

  for (const p of packets) {
      p.stream_index = streamIdx;
      // To Do: Figure out pts logic for safari as it will export without audio if pts aren't correct.
  }

  packets = packets.sort((p1, p2) => {
    return (p1.dts ?? 0) - (p2.dts ?? 0);
  });

  await libav.ff_write_multi(
    destinationContext,
    streamPacketHandler,
    packets,
    true
  );
  await libav.ff_write_multi(destinationContext, streamPacketHandler, [], true);
  await libav.ff_free_encoder(
    encoderAVCodecContext,
    encoderFramePointer,
    encoderPacketHandler
  );

  // Export to file.
  await libav.av_write_trailer(destinationContext);
  await libav.ff_free_muxer(destinationContext, pb);
  const file = await libav.readFile(filename);

  // Some code to download, but it fails before this as shown in stack.
  const blob = new Blob([file.buffer], { type: "video/mp4" });
  const a = document.createElement("a");
  a.href = URL.createObjectURL(blob);
  document.body.appendChild(a);
}

function main() {
  LibAV.LibAV().then(async function (libav) {
    libav.ff_add_stream = async function (
      oc,
      codecparms,
      time_base_num,
      time_base_den
    ) {
      const st = await libav.avformat_new_stream(oc, 0);
      if (st === 0) throw new Error("Could not allocate stream");
      const codecpar = await libav.AVStream_codecpar(st);
      await libav.AVStream_time_base_s(st, time_base_num, time_base_den);
      const ret = await libav.avcodec_parameters_from_context(
        codecpar,
        codecparms
      );
      if (ret < 0)
        throw new Error(
          "Could not copy the stream parameters: " + libav.ff_error(ret)
        );
      await libav.AVStream_time_base_s(st, time_base_num, time_base_den);
      const pkt = await libav.av_packet_alloc();
      if (pkt === 0) throw new Error("Could not allocate packet");
      const sti = await libav.AVFormatContext_nb_streams(oc) - 1;
      return [sti, pkt];
    };
    return await RunExampleToTranscodeMp4(libav);
  });
}

main();

Flags used for compiling ffmpeg:

--enable-protocol=data --enable-protocol=file
--enable-filter=aresample
--enable-decoder=aac
--enable-encoder=aac
--enable-libopenh264
--enable-muxer=mp4
--enable-parser=aac
--enable-demuxer=mp4
--enable-demuxer=mov
--enable-muxer=adts
--enable-demuxer=aac
--enable-decoder=libopenh264
--enable-encoder=libopenh264
--enable-filter=acompressor --enable-filter=adeclick --enable-filter=adeclip
--enable-filter=aecho --enable-filter=afade --enable-filter=aformat
--enable-filter=agate --enable-filter=alimiter --enable-filter=amix
--enable-filter=apad --enable-filter=atempo --enable-filter=atrim
--enable-filter=bandpass --enable-filter=bandreject --enable-filter=dynaudnorm
--enable-filter=equalizer --enable-filter=loudnorm --enable-filter=pan
--enable-filter=amix --enable-filter=volume
--enable-demuxer=ogg
--enable-muxer=ogg
--enable-demuxer=matroska
--enable-muxer=matroska --enable-muxer=webm
--enable-muxer=ipod
--enable-muxer=mov
--enable-demuxer=flac
--enable-muxer=flac
--enable-parser=flac
--enable-decoder=flac
--enable-encoder=flac
--enable-demuxer=mp3
--enable-muxer=mp3
--enable-decoder=mp3
--enable-decoder=pcm_s16le --enable-decoder=pcm_s24le
--enable-demuxer=wav
--enable-encoder=pcm_s16le --enable-encoder=pcm_s24le
--enable-muxer=wav
--enable-demuxer=pcm_f32le
--enable-muxer=pcm_f32le
--enable-decoder=pcm_f32le
--enable-encoder=pcm_f32le
--enable-filter=amix --enable-filter=volume --enable-filter=anull

Other things I tried:

1. Setting pts / dts before decoding (no change).
2. Manually setting duration and time_base using AVStream_time_base_s, AVStream_duration_s (no change).
3. Adding other audio related flags aside from those related to aac (no change).

[kartikdutt18]

Hi @Yahweasel, I also tried adding a is zero check in ffmpeg (by applying another patch), however that further results in another error. Check added:

} else if (num >= den && den > 0) { f->val += num / den; num = num % den; }

New error:

test-decode-audio.html:1 Uncaught (in promise) RuntimeError: null function or function signature mismatch RuntimeError: null function or function signature mismatch at interleaved_write_packet (http://localhost:8080/libav-mp4-aac.wasm.wasm:wasm-function[1174]:0x156639) at write_packet_common (http://localhost:8080/libav-mp4-aac.wasm.wasm:wasm-function[1172]:0x1560af) at write_packets_common (http://localhost:8080/libav-mp4-aac.wasm.wasm:wasm-function[1167]:0x154bcb) at av_interleaved_write_frame (http://localhost:8080/libav-mp4-aac.wasm.wasm:wasm-function[1173]:0x156364) at ret. (http://localhost:8080/libav-mp4-aac.wasm.js:5851:35)

Can you please help me out with this issue, thank you!

[kartikdutt18]

RuntimeError: null function or function signature mismatch

Ahh, so this implies out of memory access. Maybe? Or is it a type thing?

[Yahweasel]

After an hour of wasting my time, I have determined nothing more than the fact that the bug is not mine. libav.js is a binding; if you believe there's a bug in libav itself, you're barking up the wrong tree. Personally, I think the way you create your streams is probably incorrect in some subtle way, but I've expended all the time I'm willing to to debug it. Do not open issue reports on this repository unless the issue is with this repository's code.