How to set correct frameRate or frameDuration when writing images from different input tracks to the same file?

Background

Use webm-writer-js to add frames from multiple input video tracks potentially containing different pixel dimensions.

Commented lines in this.addFrame() to avoid error being thrown

this.addFrame = function(canvas) {
// if (writtenHeader) {
//   if (canvas.width != videoWidth || canvas.height != videoHeight) {
//      throw "Frame size differs from previous frames";
//    }
// } else {
  videoWidth = canvas.width;
  videoHeight = canvas.height;
  writeHeader();
  writtenHeader = true;
// }

The resulting video frame rate is not consistent relevant to the multiple input tracks.

Code

<!DOCTYPE html>
<html>

<head>
  <script src="webm-writer.js"></script>
</head>

<body>
  <video controls autoplay></video>
  <canvas></canvas>
  <script>

    let width;
    let height;
    let ctx;
    let controller;
    let done;
    const canvas = document.querySelector("canvas");

    const video = document.querySelector("video");
    const videoWriter = new WebMWriter({
      quality: 0.9, // WebM image quality from 0.0 (worst) to 1.0 (best)
      fileWriter: null, // FileWriter in order to stream to a file instead of buffering to memory (optional)
      fd: null, // Node.js file handle to write to instead of buffering to memory (optional)
      // You must supply one of:
      frameDuration: null, // Duration of frames in milliseconds
      frameRate: 30 // Number of frames per second
    });
    video.onloadedmetadata = e => {
      console.log(e.target.duration);
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;
      console.log(canvas.width, canvas.height);
      if (!ctx) {
        ctx = canvas.getContext("2d");
        ctx.globalComposite = "copy";
      }
    }
    video.onplay = async e => {
      let frameRate = 0;

      const processData = async({
          done
        }) =>
        done 
        ? await reader.closed
        : (await new Promise(resolve => setTimeout(resolve, !frameRate ? frameRate : (frameRate = 1000/30))), ctx.drawImage(video, 0, 0), videoWriter.addFrame(canvas), processData(await reader.read()));

      const rs = new ReadableStream({
        start(c) {
          return controller = c;
        },
        pull(_) {
          controller.enqueue(null);
        }
      });
      const reader = rs.getReader();
      done = processData(await reader.read());
    };
    (async() => {
      const urls = Promise.all([{
        from: 0,
        to: 4,
        src: "https://upload.wikimedia.org/wikipedia/commons/a/a4/Xacti-AC8EX-Sample_video-001.ogv"
      }, {
        from: 10,
        to: 20,
        src: "https://mirrors.creativecommons.org/movingimages/webm/ScienceCommonsJesseDylan_240p.webm#t=10,20"
      }, {
        from: 55,
        to: 60,
        src: "https://nickdesaulniers.github.io/netfix/demo/frag_bunny.mp4"
      }, {
        from: 0,
        to: 5,
        src: "https://raw.githubusercontent.com/w3c/web-platform-tests/master/media-source/mp4/test.mp4"
      }, {
        from: 0,
        to: 5,
        src: "https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerBlazes.mp4"
      }, {
        from: 0,
        to: 5,
        src: "https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerJoyrides.mp4"
      }, {
        from: 0,
        to: 6,
        src: "https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerMeltdowns.mp4#t=0,6"
      }].map(async({
        from,
        to,
        src
      }) => {
        try {
          const request = await fetch(src);
          const blob = await request.blob();
          const blobURL = URL.createObjectURL(blob);
          const url = new URL(src);
          console.log(url.hash);
          return blobURL + (url.hash || `#t=${from},${to}`);
        } catch (e) {
          throw e;
        }
      }));
      let media = await urls;
      for (const blobURL of media) {
        await new Promise(async resolve => {
          video.addEventListener("pause", e => {
            controller.close();
            done.then(resolve);
          }, {
            once: true
          });
          video.src = blobURL;
        });
      }
      const blob = await videoWriter.complete();
      video.remove();
      canvas.remove();
      const videoStream = document.createElement("video");
      videoStream.onloadedmetadata = e => console.log(videoStream.duration);
      videoStream.onended = e => console.log(videoStream.duration, videoStream.currentTime);
      videoStream.controls = videoStream.autoplay = true;
      document.body.appendChild(videoStream);
      const src = URL.createObjectURL(blob);
      console.log(src);
      videoStream.src = src;

    })();
  </script>
</body>

</html>

Issue

plnkr https://plnkr.co/edit/4JxS4O?p=preview

The expected duration of the resulting WebM file is 41-42 seconds. The actual duration is 27-28.

thenickdude / webm-writer-js

How to set correct frameRate or frameDuration when writing images from different input tracks to the same file? #6