Chromium WebCodecs 'opus' implementation does not work

guest271314 commented 3 years ago

So far I have concluded WebCodecs 'opus' implementation does not work, either live-stream or non-live-stream

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="utf-8" />
    <title>
      WebCodecs Breakout Box support for Audio (AudioEncoder+AudioDecoder) is
      broken: Outputs gaps and glitches in playback
    </title>
    <style>
      body *:not(script) {
        display: block;
        position: relative;
        padding: 8px;
      }
    </style>
  </head>

  <body>
    <h1>Click</h1>
    <audio id="audio" controls autoplay></audio>
    <script>
      // https://bugs.chromium.org/p/chromium/issues/detail?id=1184070
      async function main(buffer) {
        const ac = new AudioContext({ latencyHint: 0.5, sampleRate: 48000 });
        let ab = await ac.decodeAudioData(buffer);
        let tmp = ab.getChannelData(0);
        ab = new AudioBuffer({
          length: ab.length,
          sampleRate: ab.sampleRate,
          numberOfChannels: 2,
        });
        for (let i = 0; i < ab.numberOfChannels; i++) {
          ab.getChannelData(i).set(tmp.slice(0));
        }
        console.log(ab.getChannelData(0), ab.getChannelData(1));
        const msd = new MediaStreamAudioDestinationNode(ac, {
          channelCount: 2,
        });

        const { stream } = msd;
        const [track] = stream.getTracks();
        // await track.applyConstraints({ channelCount:1 });
        const absn = new AudioBufferSourceNode(ac, { buffer: ab });
        absn.connect(msd);
        absn.start();

        const settings = track.getSettings();
        console.log(settings);
        let duration = 0;
        let frames = [];
        let rsController;
        let rs = new ReadableStream({
          start(c) {
            return (rsController = c);
          },
        });
        let base_timestamp = null;
        const processor = new MediaStreamTrackProcessor(track);
        const reader = processor.readable.getReader();
        const generator = new MediaStreamTrackGenerator({ kind: 'audio' });
        generator.onmute = generator.onunmute = generator.onended = (e) =>
          console.log(e);
        // const writer = generator.writable.getWriter();
        const audio = document.getElementById('audio');
        let recorder;
        audio.onplay = () => {
          console.log('play');
          recorder = new MediaRecorder(audio.srcObject);
          recorder.start();
          recorder.ondataavailable = ({ data }) =>
            console.log(URL.createObjectURL(data));
        };
        const decoder = new AudioDecoder({
          error(e) {
            console.log(e);
          },
          output(frame) {
            rsController.enqueue(frame);
          },
        });

        console.log(decoder);

        const encoder = new AudioEncoder({
          error(e) {
            console.log(e);
          },
          output(chunk, config) {
            if (config) {
              console.log(config);
              frames.push(config);
            }
            frames.push(chunk);
          },
        });

        const config = {
          numberOfChannels: settings.channelCount,
          sampleRate: settings.sampleRate,
          codec: 'opus',
          bitrate: 128000,
        };

        encoder.configure(config);

        reader
          .read()
          .then(async function process({ value, done }) {
            if (
              frames.length >= Math.min(ab.length / value.buffer.length) / 2 &&
              processor.readable.locked
            ) {
              // console.log(frames);
              encoder.close();

              audio.ontimeupdate = async () => {
                console.log(audio.currentTime);
                if (
                  audio.currentTime >= duration + 0.5 &&
                  processor.readable.locked
                ) {
                  console.log(processor, reader);
                  try {
                    reader.releaseLock();
                    console.log(
                      'processor.cancel',
                      await processor.readable.cancel()
                    );
                  } catch (e) {
                    console.log(e);
                  }
                  generator.stop();
                  rsController.close();
                  track.stop();
                  recorder.stop();
                }
              };
              // expected output 'Test'
              audio.srcObject = new MediaStream([generator]);
              decoder.configure(frames.shift());
              for (const frame of frames) {
                await scheduler.postTask(() => decoder.decode(frame));
              }
              console.log(
                await Promise.allSettled([
                  rs.pipeTo(generator.writable).catch(console.error),
                  reader.closed,
                ])
              );
              decoder.close();
            } else {
              let frame = value;
              // https://bugs.chromium.org/p/chromium/issues/detail?id=1181547
              // https://wc-audio.glitch.me/
              /*
               if (base_timestamp === null)
                base_timestamp = frame.timestamp;
               let rebased_frame = new AudioFrame({timestamp: (frame.timestamp - base_timestamp), buffer: frame.buffer });
               frame = rebased_frame;
               */
              duration += frame.buffer.duration;
              // console.log(frame.timestamp);
              await scheduler.postTask(() => encoder.encode(frame));
              return reader.read().then(process);
            }
          })
          .then(() => console.log('Done encoding and decoding.'))

          .catch(console.error);
      }
      const input = document.querySelector('h1');
      input.onclick = async () => {
        main(await (await fetch('./test.wav')).arrayBuffer());
      };
    </script>
  </body>
</html>

plnkr https://plnkr.co/edit/yY4CZbPNnZMT7Kwd?preview

Are there any working examples of AudioEncoder and AudioDecoder actually outputting the expected result?

tguilbert-google commented 3 years ago

For questions about demos, this should be moved to the official WebCodecs repro (we will be aggregating all demos there in the relatively near future).

The problem with opus is most likely crbug.com/1181547, since we might drop frames if there are incoherent timestamps. I will ping people related to the bug.

guest271314 commented 3 years ago

WICG banned be. Feel free to move the issue to official WebCodecs repository. Non-live decoding example outputs the same result https://plnkr.co/edit/QR92LDMQnyE4xpi9.

I do not gather the necessity of timestamps with regard to audio input.

tguilbert-google / tguilbert-google.github.io

Chromium WebCodecs 'opus' implementation does not work #6