Real-time Audio Processing

[hdaneshfar]

Hello Dani , I congratulate you for create this project. Please work on Real-time processing and add this feature to the project ! best regards

[YoniH]

Hello Dani , I congratulate you for create this project. Please work on Real-time processing and add this feature to the project ! best regards

+1

[YoniH]

Yes, that would be amazing. I'm trying to make it work in real time too. I created an AudioWorkletNode with an AudioWorkletProcessor, trying to make a custom time stretch and pitch shift node. I call rubberband_process and rubberband_retrieve in the processor, and getting memory errors all the time. A demo would be great!

[daniela-artlist]

+1 for real time

[Daninet]

This should be already possible without any modifications to the library. All required functions are already exposed to JavaScript. However, because real-time audio is not currently useful in my use case, I cannot afford to devote more time to it. 

I'm available as a consultant if you need help.

[daniela-artlist]

@Daninet Thank!, will take a look into it and contact you if needed :)

[tomduncalf]

I've made a really rough AudioWorklet which will real-time timestretch. The code is very rough, I'll post a tidied up version if I get round to it but though I'd share this while I remember! It's currently only mono, I just didn't get round to making it stereo yet.

It's essentially the code in worker.js, transposed to work in a worklet. It uses the "pull" model, i.e. whenever we run out of timestretched samples to play, we ask Rubberband to stretch the next chunk. It seems to perform pretty well on my M1 Mac.

Thanks for sharing this library @Daninet, really amazing to be able to timestertch in the browser :)

Here's how you use it:

audioContext = new AudioContext();

await audioContext.audioWorklet.addModule(
  "processors/timestretch-playback-processor.js"
);
const module = await WebAssembly.compileStreaming(
  fetch("/rubberband/rubberband.wasm")
);

const timetretchedSoundSource = new AudioWorkletNode(
  audioContext,
  "timestretch-playback-processor",
  {
    processorOptions: {
      buffer: soundSource.buffer!.getChannelData(0),
      module,
      playbackRate: 1.1,
      pitch: 1,
      sampleRate: audioContext.sampleRate,
    },
  }
);

timetretchedSoundSource.connect(audioContext.destination);

// Send a message to start playback

sourceNode.port.postMessage({
  type: "START",
  time: 0,
  offset: s.song.meta.firstBeat,
})

Here's the worklet source:

// This is rubberband.umd.min.js
/*!
 * rubberband-wasm v2.0.2-beta.0 (https://www.npmjs.com/package/rubberband-wasm)
 * (c) Dani Biro
 * @license GPLv2
 */

!(function (e, t) {
  "object" == typeof exports && "undefined" != typeof module
    ? t(exports)
    : "function" == typeof define && define.amd
    ? define(["exports"], t)
    : t(
        ((e =
          "undefined" != typeof globalThis
            ? globalThis
            : e || self).rubberband = {})
      );
})(this, function (e) {
  "use strict";
  var t, r;
  (e.RubberBandOption = void 0),
    ((t = e.RubberBandOption || (e.RubberBandOption = {}))[
      (t.RubberBandOptionProcessOffline = 0)
    ] = "RubberBandOptionProcessOffline"),
    (t[(t.RubberBandOptionProcessRealTime = 1)] =
      "RubberBandOptionProcessRealTime"),
    (t[(t.RubberBandOptionStretchElastic = 0)] =
      "RubberBandOptionStretchElastic"),
    (t[(t.RubberBandOptionStretchPrecise = 16)] =
      "RubberBandOptionStretchPrecise"),
    (t[(t.RubberBandOptionTransientsCrisp = 0)] =
      "RubberBandOptionTransientsCrisp"),
    (t[(t.RubberBandOptionTransientsMixed = 256)] =
      "RubberBandOptionTransientsMixed"),
    (t[(t.RubberBandOptionTransientsSmooth = 512)] =
      "RubberBandOptionTransientsSmooth"),
    (t[(t.RubberBandOptionDetectorCompound = 0)] =
      "RubberBandOptionDetectorCompound"),
    (t[(t.RubberBandOptionDetectorPercussive = 1024)] =
      "RubberBandOptionDetectorPercussive"),
    (t[(t.RubberBandOptionDetectorSoft = 2048)] =
      "RubberBandOptionDetectorSoft"),
    (t[(t.RubberBandOptionPhaseLaminar = 0)] = "RubberBandOptionPhaseLaminar"),
    (t[(t.RubberBandOptionPhaseIndependent = 8192)] =
      "RubberBandOptionPhaseIndependent"),
    (t[(t.RubberBandOptionThreadingAuto = 0)] =
      "RubberBandOptionThreadingAuto"),
    (t[(t.RubberBandOptionThreadingNever = 65536)] =
      "RubberBandOptionThreadingNever"),
    (t[(t.RubberBandOptionThreadingAlways = 131072)] =
      "RubberBandOptionThreadingAlways"),
    (t[(t.RubberBandOptionWindowStandard = 0)] =
      "RubberBandOptionWindowStandard"),
    (t[(t.RubberBandOptionWindowShort = 1048576)] =
      "RubberBandOptionWindowShort"),
    (t[(t.RubberBandOptionWindowLong = 2097152)] =
      "RubberBandOptionWindowLong"),
    (t[(t.RubberBandOptionSmoothingOff = 0)] = "RubberBandOptionSmoothingOff"),
    (t[(t.RubberBandOptionSmoothingOn = 8388608)] =
      "RubberBandOptionSmoothingOn"),
    (t[(t.RubberBandOptionFormantShifted = 0)] =
      "RubberBandOptionFormantShifted"),
    (t[(t.RubberBandOptionFormantPreserved = 16777216)] =
      "RubberBandOptionFormantPreserved"),
    (t[(t.RubberBandOptionPitchHighSpeed = 0)] =
      "RubberBandOptionPitchHighSpeed"),
    (t[(t.RubberBandOptionPitchHighQuality = 33554432)] =
      "RubberBandOptionPitchHighQuality"),
    (t[(t.RubberBandOptionPitchHighConsistency = 67108864)] =
      "RubberBandOptionPitchHighConsistency"),
    (t[(t.RubberBandOptionChannelsApart = 0)] =
      "RubberBandOptionChannelsApart"),
    (t[(t.RubberBandOptionChannelsTogether = 268435456)] =
      "RubberBandOptionChannelsTogether"),
    (e.RubberBandPresetOption = void 0),
    ((r = e.RubberBandPresetOption || (e.RubberBandPresetOption = {}))[
      (r.DefaultOptions = 0)
    ] = "DefaultOptions"),
    (r[(r.PercussiveOptions = 1056768)] = "PercussiveOptions");
  class n {
    constructor() {}
    static async initialize(e) {
      if ("undefined" == typeof WebAssembly)
        throw new Error("WebAssembly is not supported in this environment!");
      let t = {};
      const r = (...e) => (console.error("WASI called with params", e), 52);
      let s = [];
      const i = await WebAssembly.instantiate(e, {
          env: {
            emscripten_notify_memory_growth: () => {
              (t.HEAP8 = new Uint8Array(i.exports.memory.buffer)),
                (t.HEAP32 = new Uint32Array(i.exports.memory.buffer));
            },
          },
          wasi_snapshot_preview1: {
            proc_exit: (...e) => r("proc_exit", e),
            fd_read: (...e) => r("fd_read", e),
            fd_write: (e, r, n, i) => {
              if (e > 2) return 52;
              let b = 0;
              for (let e = 0; e < n; e++) {
                const e = t.HEAP32[r >> 2],
                  n = t.HEAP32[(r + 4) >> 2];
                r += 8;
                for (let r = 0; r < n; r++) {
                  const n = t.HEAP8[e + r];
                  0 === n || 10 === n
                    ? (console.log(s.join("")), (s.length = 0))
                    : s.push(String.fromCharCode(n));
                }
                b += n;
              }
              return (t.HEAP32[i >> 2] = b), 0;
            },
            fd_seek: (...e) => r("fd_seek", e),
            fd_close: (...e) => r("fd_close", e),
            environ_sizes_get: (e, t) => 52,
            environ_get: (...e) => r("environ_get", e),
            clock_time_get: (...e) => r("clock_time_get", e),
          },
        }),
        b = i.exports;
      (t.HEAP8 = new Uint8Array(i.exports.memory.buffer)),
        (t.HEAP32 = new Uint32Array(i.exports.memory.buffer)),
        b._initialize();
      const a = { heap: t, exports: b },
        o = new n();
      return (o.wasm = a), o;
    }
    malloc(e) {
      return this.wasm.exports.wasm_malloc(e);
    }
    memWrite(e, t) {
      const r =
        t instanceof Uint8Array
          ? t
          : new Uint8Array(t.buffer, t.byteOffset, t.byteLength);
      this.wasm.heap.HEAP8.set(r, e);
    }
    memWritePtr(e, t) {
      const r = new Uint8Array(4);
      new DataView(r.buffer).setUint32(0, t, !0),
        this.wasm.heap.HEAP8.set(r, e);
    }
    memReadU8(e, t) {
      return this.wasm.heap.HEAP8.subarray(e, e + t);
    }
    memReadF32(e, t) {
      const r = this.memReadU8(e, 4 * t);
      return new Float32Array(r.buffer, r.byteOffset, t);
    }
    free(e) {
      this.wasm.exports.wasm_free(e);
    }
    rubberband_new(e, t, r, n, s) {
      return this.wasm.exports.rb_new(e, t, r, n, s);
    }
    rubberband_delete(e) {
      this.wasm.exports.rb_delete(e);
    }
    rubberband_reset(e) {
      this.wasm.exports.rb_reset(e);
    }
    rubberband_set_time_ratio(e, t) {
      this.wasm.exports.rb_set_time_ratio(e, t);
    }
    rubberband_set_pitch_scale(e, t) {
      this.wasm.exports.rb_set_pitch_scale(e, t);
    }
    rubberband_get_time_ratio(e) {
      return this.wasm.exports.rb_get_time_ratio(e);
    }
    rubberband_get_pitch_scale(e) {
      return this.wasm.exports.rb_get_pitch_scale(e);
    }
    rubberband_get_latency(e) {
      return this.wasm.exports.rb_get_latency(e);
    }
    rubberband_set_transients_option(e, t) {
      this.wasm.exports.rb_set_transients_option(e, t);
    }
    rubberband_set_detector_option(e, t) {
      this.wasm.exports.rb_set_detector_option(e, t);
    }
    rubberband_set_phase_option(e, t) {
      this.wasm.exports.rb_set_phase_option(e, t);
    }
    rubberband_set_formant_option(e, t) {
      this.wasm.exports.rb_set_formant_option(e, t);
    }
    rubberband_set_pitch_option(e, t) {
      this.wasm.exports.rb_set_pitch_option(e, t);
    }
    rubberband_set_expected_input_duration(e, t) {
      this.wasm.exports.rb_set_expected_input_duration(e, t);
    }
    rubberband_get_samples_required(e) {
      return this.wasm.exports.rb_get_samples_required(e);
    }
    rubberband_set_max_process_size(e, t) {
      this.wasm.exports.rb_set_max_process_size(e, t);
    }
    rubberband_set_key_frame_map(e, t, r, n) {
      this.wasm.exports.rb_set_key_frame_map(e, t, r, n);
    }
    rubberband_study(e, t, r, n) {
      this.wasm.exports.rb_study(e, t, r, n);
    }
    rubberband_process(e, t, r, n) {
      this.wasm.exports.rb_process(e, t, r, n);
    }
    rubberband_available(e) {
      return this.wasm.exports.rb_available(e);
    }
    rubberband_retrieve(e, t, r) {
      return this.wasm.exports.rb_retrieve(e, t, r);
    }
    rubberband_get_channel_count(e) {
      return this.wasm.exports.rb_get_channel_count(e);
    }
    rubberband_calculate_stretch(e) {
      this.wasm.exports.rb_calculate_stretch(e);
    }
    rubberband_set_debug_level(e, t) {
      this.wasm.exports.rb_set_debug_level(e, t);
    }
    rubberband_set_default_debug_level(e) {
      this.wasm.exports.rb_set_default_debug_level(e);
    }
  }
  (e.RubberBandInterface = n),
    Object.defineProperty(e, "__esModule", { value: !0 });
});

// Ported from worker.js by Tom Duncalf
//
// Sources:
// https://breakfastquay.com/rubberband/integration.html#:~:text=For%20real%2Dtime%20use%20especially,samples)%20is%20allowed%20to%20vary.
// https://breakfastquay.com/rubberband/code-doc/index.html
class TimestretchPlaybackProcessor extends AudioWorkletProcessor {
  static get parameterDescriptors() {
    return [{ name: "playback", defaultValue: 1 }];
  }

  constructor(options) {
    super(options);

    // TODO stereo
    this.sampleRate = options.processorOptions.sampleRate;
    this.channelBuffers = [options.processorOptions.buffer];
    this.module = options.processorOptions.module;
    this.playbackRate = options.processorOptions.playbackRate || 1;

    const pitchSemitones = Math.pow(2, options.processorOptions.pitch / 12);
    this.pitch = pitchSemitones || 1;

    this.currentPositionSamples = 0;

    // TODO what size should this be? This could be a ring buffer but for now we
    // store the entire timestretched track - could be used to save CPU if you
    // play it again
    this.outputBuffers = this.channelBuffers.map(
      () => new Float32Array(9999999)
    );
    this.outputBufferEnd = 0;

    this.write = 0;
    this.read = 0;

    this.setupRubberband();

    this.port.onmessage = (event) => {
      if (event.data.type === "START") {
        this.startOffset = Math.round(event.data.offset * this.sampleRate);
        this.read = this.startOffset;
        this.startTime = event.data.time;
      }
    };

    this.startTime = Number.POSITIVE_INFINITY;
  }

  setupRubberband = async () => {
    this.rbApi = await rubberband.RubberBandInterface.initialize(this.module);

    this.rbState = this.rbApi.rubberband_new(
      this.sampleRate,
      this.channelBuffers.length,
      1, // = realtime
      1,
      1
    );
    this.rbApi.rubberband_set_pitch_scale(this.rbState, this.pitch);
    this.rbApi.rubberband_set_time_ratio(this.rbState, this.playbackRate);

    this.samplesRequired = this.rbApi.rubberband_get_samples_required(
      this.rbState
    );

    // create the buffers?
    this.channelArrayPtr = this.rbApi.malloc(this.channelBuffers.length * 4);
    this.channelDataPtr = [];
    for (let channel = 0; channel < this.channelBuffers.length; channel++) {
      const bufferPtr = this.rbApi.malloc(this.samplesRequired * 4);
      this.channelDataPtr.push(bufferPtr);
      this.rbApi.memWritePtr(this.channelArrayPtr + channel * 4, bufferPtr);
    }
  };

  tryRetrieve = (final = false) => {
    while (1) {
      const available = this.rbApi.rubberband_available(this.rbState);
      if (available < 1) break;
      if (!final && available < this.samplesRequired) break;

      const requested = Math.min(this.samplesRequired, available);
      const recv = this.rbApi.rubberband_retrieve(
        this.rbState,
        this.channelArrayPtr,
        requested
      );

      this.outputBufferEnd += requested;

      this.channelDataPtr.forEach((ptr, i) =>
        this.outputBuffers[i].set(this.rbApi.memReadF32(ptr, recv), this.write)
      );

      this.write += recv;
    }
  };

  timestretchNextChunk = () => {
    this.samplesRequired = this.rbApi.rubberband_get_samples_required(
      this.rbState
    );

    this.channelBuffers.forEach((buf, i) => {
      this.rbApi.memWrite(
        this.channelDataPtr[i],
        buf.subarray(this.read, this.read + this.samplesRequired)
      );
    });

    // and process them
    const remaining = Math.min(
      this.samplesRequired,
      this.channelBuffers[0].length - this.read
    );
    this.read += remaining;
    let isFinal = false;

    this.rbApi.rubberband_process(
      this.rbState,
      this.channelArrayPtr,
      remaining,
      0 //isFinal ? 0 : 1
    );

    this.samplesAvailable = this.rbApi.rubberband_available(this.rbState);

    this.tryRetrieve();
  };

  process(inputs, outputs, parameters) {
    // We've run out of audio in the output buffer
    while (this.outputBufferEnd - this.currentPositionSamples < 128) {
      this.timestretchNextChunk();
    }

    if (currentTime < this.startTime) {
      return true;
    }

    const output = outputs[0];

    output.forEach((channel) => {
      for (let i = 0; i < channel.length; i++) {
        if (this.currentPositionSamples < this.outputBuffers[0].length) {
          channel[i] = this.outputBuffers[0][this.currentPositionSamples];
          // channel[i] = this.channelBuffers[0][this.currentPositionSamples];
          this.currentPositionSamples += 1;
        } else {
          channel[i] = 0;
        }
      }
    });

    return true;
  }
}

registerProcessor(
  "timestretch-playback-processor",
  TimestretchPlaybackProcessor
);