membraneframework / membrane_core

The core of the Membrane Framework, advanced multimedia processing framework
https://membrane.stream
Apache License 2.0
1.22k stars 34 forks source link

Opus encoder outputs non-monotonic timestamps #834

Open mat-hek opened 1 week ago

mat-hek commented 1 week ago

In the example below, timestamps that come to the sink aren't monotonic. After commenting out the Opus encoder, everything works. One of the previous elements may mess up the timestamps too, however, they're monotonic at least. Fixture: bun10s_a.mp4

Mix.install([
  :membrane_file_plugin,
  :membrane_aac_plugin,
  :membrane_aac_fdk_plugin,
  :membrane_ffmpeg_swresample_plugin,
  :membrane_opus_plugin,
  :membrane_mp4_plugin
])

defmodule CheckTimestamps do
  use Membrane.Sink

  def_input_pad :input, accepted_format: _any

  @impl true
  def handle_init(_ctx, _opts) do
    {[], %{last_pts: nil}}
  end

  @impl true
  def handle_buffer(:input, buffer, _ctx, state) do
    last_pts = state.last_pts || buffer.pts

    if buffer.pts < last_pts do
      raise "Non-monotonic PTS, diff: #{buffer.pts - last_pts}"
    end

    {[], %{state | last_pts: buffer.pts}}
  end
end

defmodule Rep do
  import Membrane.ChildrenSpec
  require Membrane.Pad, as: Pad
  require Membrane.RCPipeline

  def run() do
    p = Membrane.RCPipeline.start_link!()

    Membrane.RCPipeline.subscribe(p, _all)

    Membrane.RCPipeline.exec_actions(p,
      spec:
        child(%Membrane.File.Source{location: "bun10s_a.mp4"})
        |> child(Membrane.MP4.Demuxer.ISOM)
        |> via_out(Pad.ref(:output, 1))
        |> child(Membrane.AAC.Parser)
        |> child(Membrane.AAC.FDK.Decoder)
        |> child(%Membrane.FFmpeg.SWResample.Converter{
          output_stream_format: %Membrane.RawAudio{
            sample_format: :s16le,
            sample_rate: 48_000,
            channels: 2
          }
        })
        |> child(Membrane.Opus.Encoder)
        |> child(:sink, CheckTimestamps)
    )

    Membrane.RCPipeline.await_end_of_stream(p, :sink)
  end
end

Rep.run()