Vanilla JS example

[flatsiedatsie]

It would be great if there was a minimal vanilla JS example of how to use this in a project. I don't use React, so the current output of npm install is very difficult to extract any understanding from.

[xenova]

Check out the whisper-tiny model card for example usage of the model: https://huggingface.co/Xenova/whisper-tiny.en

You'll need to implement the additional UI features and functionality yourself (e.g., mic input), though. But there are resources online to assist you!

Example: Transcribe English.

// npm i @xenova/transformers
import { pipeline } from '@xenova/transformers';

let url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav';

// Create translation pipeline
let transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
let output = await transcriber(url);
// { text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country." }

Example: Transcribe English w/ timestamps.

// npm i @xenova/transformers
import { pipeline } from '@xenova/transformers';

let url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav';

// Create translation pipeline
let transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
let output = await transcriber(url, { return_timestamps: true });
// {
//   text: " And so my fellow Americans ask not what your country can do for you, ask what you can do for your country."
//   chunks: [
//     { timestamp: [0, 8],  text: " And so my fellow Americans ask not what your country can do for you" }
//     { timestamp: [8, 11], text: " ask what you can do for your country." }
//   ]
// }

Example: Transcribe English w/ word-level timestamps.

// npm i @xenova/transformers
import { pipeline } from '@xenova/transformers';

let url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav';

// Create translation pipeline
let transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
let output = await transcriber(url, { return_timestamps: 'word' });
// {
//   "text": " And so my fellow Americans ask not what your country can do for you ask what you can do for your country.",
//   "chunks": [
//     { "text": " And", "timestamp": [0, 0.78] },
//     { "text": " so", "timestamp": [0.78, 1.06] },
//     { "text": " my", "timestamp": [1.06, 1.46] },
//     ...
//     { "text": " for", "timestamp": [9.72, 9.92] },
//     { "text": " your", "timestamp": [9.92, 10.22] },
//     { "text": " country.", "timestamp": [10.22, 13.5] }
//   ]
// }

[flatsiedatsie]

Thanks!

In the end I extracted the necessary part from the React version.

whisper_worker.js

import { pipeline,env } from './js/transformers.js';
console.log("WHISPER WEB WORKER EXISTS");

env.allowLocalModels = false;

addEventListener('message', async (event) => {
    console.log("WHISPER WEB WORKER: RECEIVED MESSAGE");
    console.log("WHISPER WEB WORKER: event.data: ", event.data);

    const message = event.data;
    let task = message.task;

    // Do some work...
    // TODO use message data
    try{

        let transcript = await transcribe(
            message.task.recorded_audio,
            message.model,
            message.multilingual,
            message.quantized,
            message.subtask,
            message.language,
        );
        console.log("WHISPER WEB WORKER: TRANSCRIPTION RESULT: ", transcript);
        if (transcript === null){
            console.error("WHISPER WEB WORKER: transcription was null");
        }
        if (typeof transcript === 'undefined'){
            console.error("WHISPER WEB WORKER: transcription was undefined??");
        }

        delete task.recorded_audio;
        task['transcript'] = transcript;
        // Send the result back to the main thread
        self.postMessage({
            task: task,
            status: "complete",
            //task: "automatic-speech-recognition",
            transcript: transcript,
        });

    }catch(e){
        console.error("ERROR: whisper worker: ", e);
    }

});

// Define model factories
// Ensures only one model is created of each type

class PipelineFactory {
    static task = null;
    static model = null;
    static quantized = null;
    static instance = null;

    constructor(tokenizer, model, quantized) {
        this.tokenizer = tokenizer;
        this.model = model;
        this.quantized = quantized;
    }

    static async getInstance(progress_callback = null) {
        if (this.instance === null) {
            this.instance = pipeline(this.task, this.model, {
                quantized: this.quantized,
                progress_callback,

                // For medium models, we need to load the `no_attentions` revision to avoid running out of memory
                revision: this.model.includes("/whisper-medium") ? "no_attentions" : "main"
            });
        }

        return this.instance;
    }
}

class AutomaticSpeechRecognitionPipelineFactory extends PipelineFactory {
    static task = "automatic-speech-recognition";
    static model = null;
    static quantized = null;
}

const transcribe = async (
    audio,
    model,
    multilingual,
    quantized,
    subtask,
    language,
) => {
    console.log("in transcribe. audio: ", audio);
    console.log("whisper web worker: in transcribe.  model,multilingual,quantized,subtask,language: ", model, multilingual, quantized, subtask, language);

    let output = null;

    try{
        const isDistilWhisper = model.startsWith("distil-whisper/");

        let modelName = model;
        if (!isDistilWhisper && !multilingual) {
            modelName += ".en"
        }

        const p = AutomaticSpeechRecognitionPipelineFactory;
        if (p.model !== modelName || p.quantized !== quantized) {
            // Invalidate model if different
            p.model = modelName;
            p.quantized = quantized;

            if (p.instance !== null) {
                (await p.getInstance()).dispose();
                p.instance = null;
            }
        }

        // Load transcriber model
        let transcriber = await p.getInstance((data) => {
            console.log("whisper web worker: posting something back: ", data);
            self.postMessage(data);
        });

        const time_precision =
            transcriber.processor.feature_extractor.config.chunk_length /
            transcriber.model.config.max_source_positions;

        // Storage for chunks to be processed. Initialise with an empty chunk.
        let chunks_to_process = [
            {
                tokens: [],
                finalised: false,
            },
        ];

        // TODO: Storage for fully-processed and merged chunks
        // let decoded_chunks = [];

        function chunk_callback(chunk) {
            console.log("in whisper chunk callback. chunk: ", chunk);
            let last = chunks_to_process[chunks_to_process.length - 1];

            // Overwrite last chunk with new info
            Object.assign(last, chunk);
            last.finalised = true;

            // Create an empty chunk after, if it not the last chunk
            if (!chunk.is_last) {
                chunks_to_process.push({
                    tokens: [],
                    finalised: false,
                });
            }
        }

        // Inject custom callback function to handle merging of chunks
        function callback_function(item) {
            //console.log("whisper_worker: COMPLETE?  item: ", item);
            let last = chunks_to_process[chunks_to_process.length - 1];

            // Update tokens of last chunk
            last.tokens = [...item[0].output_token_ids];

            // Merge text chunks
            // TODO optimise so we don't have to decode all chunks every time
            let data = transcriber.tokenizer._decode_asr(chunks_to_process, {
                time_precision: time_precision,
                return_timestamps: true,
                force_full_sequences: false,
            });

            self.postMessage({
                status: "update",
                //task: "automatic-speech-recognition",
                data: data,
            });
        }

        // Actually run transcription
        output = await transcriber(audio, {

            // Greedy
            top_k: 0,
            do_sample: false,

            // Sliding window
            chunk_length_s: isDistilWhisper ? 20 : 30,
            stride_length_s: isDistilWhisper ? 3 : 5,

            // Language and task
            language: language,
            task: subtask,

            // Return timestamps
            return_timestamps: true,
            force_full_sequences: false,

            // Callback functions
            callback_function: callback_function, // after each generation step
            chunk_callback: chunk_callback, // after each chunk is processed
        })

        .catch((error) => {
            console.error("ERROR, actually running whisper failed");

            return null;
        });

        console.log("beyond WHISPER transcribe. output: ", output);

    }
    catch(e){
        console.error("Whisper worker: error in transcribe function: ", e);
    }

    return output;
};

Which I start with this code in a JS module:

window.whisper_worker = null;
window.whisper_worker_busy = false;
let whisper_worker_error_count = 0;

//
//   WHISPER
//

function create_whisper_worker(){
    console.log("in create_whisper_worker");

    window.whisper_worker = null;
    window.whisper_worker = new Worker('./whisper_worker.js', {
      type: 'module'
    });

    console.log("whisper_module: window.whisper_worker: ", window.whisper_worker);

    window.whisper_worker.addEventListener('message', e => {
        //console.log("whisper_module: received message from whisper_worker: ", e.data);

        if(typeof e.data.status == 'string'){
            if(e.data.status == 'progress'){
                //console.log("whisper worker sent download percentage: ", e.data.progress);
                let whisper_progress_el = document.getElementById('download-progress-whisper');
                if(whisper_progress_el == null){
                    console.error("whisper (down)load progress element is missing");
                    add_chat_message("whisper",'download_progress#setting---');
                }
                else{
                    //console.log("updating whisper (down)load progress");
                    whisper_progress_el.value = e.data.progress / 100;
                }

            }
            else if(e.data.status == 'ready'){
                console.log("whisper worker sent ready message");
                window.whisper_worker_busy = false;
                add_chat_message("whisper",get_translation('Voice_recognition_has_loaded'));
                let whisper_progress_el = document.getElementById('download-progress-whisper');
                if(whisper_progress_el){
                    whisper_progress_el.classList.add('download-complete-chat-message');
                }
                else{
                    console.error("whisper became ready, but cannot find loading progress indicator element");
                }
            }
            else if(e.data.status == 'initiate'){
                console.log("whisper worker sent initiate message");
            }
            else if(e.data.status == 'download'){
                console.log("whisper worker sent download message");
                add_chat_message("whisper","(down)loading: " + e.data.file);
            }

            else if(e.data.status == 'update'){
                if(typeof e.data.data == 'object' && e.data.data != null && e.data.data.length){
                    set_chat_status(e.data.data[0],2);
                }

            }

            else if(e.data.status == 'complete'){
                window.whisper_worker_busy = false;
                console.log('GOT WHISPER COMPLETE.  e.data: ', e.data);
                console.log('GOT WHISPER COMPLETE.  e.data.transcript: ', e.data.transcript);
                console.log('GOT WHISPER COMPLETE.  e.data.task: ', e.data.task);

                if(e.data.transcript == null){
                    console.warn("whisper recognition failed. If this is the first run, that's normal.");
                    set_state(LISTENING);
                }
                else if(typeof e.data.transcript != 'undefined'){
                    console.log("whisper returned transcription text: ", e.data.transcript);

                    if(Array.isArray(e.data.transcript)){
                        console.log("typeof transcription is array");
                    }
                    else if(typeof e.data.transcript == 'object'){
                        if(typeof e.data.transcript.text == 'string'){
                            console.log("GOT TEXT: ", e.data.transcript.text);
                        }
                    }
                }
                else{
                    console.log("transcript was not in whisper e.data");
                }

                //add_chat_message("whisper","(down)loading: " + e.data.file);
            }
            else{
                console.log("whisper worker sent a content message");
                window.whisper_worker_busy = false;

                if(e.data.data == null){
                    console.warn("whisper recognition failed. If this is the first run, that's normal.");
                    set_state(LISTENING);
                }
            }
        }

            if(window.enable_microphone == false){
                console.log("whisper worker returned audio file, but in the meantime enable_microphone was disabled. Throwing away the data.");
            }
            else{

                /*

                if(window.whisper_queue.length){
                    console.log("whisper worker done, but there is more work to do. Sentences still in whisper_queue: ", window.whisper_queue.length);
                    let next_sentence = window.whisper_queue[0][0] + window.whisper_queue[0][1]; // sentence plus punctuation mark
                    window.whisper_queue.splice(0,1);

                    whisper_worker.postMessage({'whisper_counter':window.whisper_counter,'sentence':next_sentence});
                    window.whisper_counter++;
                }
                else{
                    console.log("whisper worker was done, and there are no more sentences in the whisper queue. Worker is now idle.");
                    window.whisper_worker_busy = false;
                }
                */
            }

    });

    window.whisper_worker.addEventListener('error', (error) => {
        console.error("ERROR: whisper_worker sent error. terminating!. Error was: ", error, error.message);
        whisper_worker_error_count++;

        window.whisper_worker.terminate();
        window.whisper_worker_busy = false;
        if(typeof error != 'undefined' && whisper_worker_error_count < 10){
            setTimeout(() => {
                console.log("attempting to restart whisper worker");
                create_whisper_worker();
            },1000);
        }
        else{
            console.error("whisper_worker errored out");
        }
    });
}

// create whisper worker
create_whisper_worker();

//
//  Send audio buffer to whisper worker
//
function do_whisper_web(task,language=null){
    console.log("in do_whisper_web. task: ", task);

    if(window.whisper_worker_busy){
        console.error("do_whisper_web was called while whisper worker was busy. Aborting.");
        return
    }

    if(typeof task.recorded_audio == 'undefined'){
        console.error("do_whisper_web: task did not contain recorded_audio. Aborting.");
        return
    }

    task.state == 'stt_in_progress';

    let multilingual = false;
    if(typeof language == 'string'){
        if(language != 'en'){
            multilingual = true;
        }
    }
    const quantized = false;
    const model = "Xenova/whisper-tiny";

    const subtask = null;

    console.log("do_whisper_web: sending audio to whisper worker: ", task.recorded_audio);

    window.whisper_worker.postMessage({
        task:task,
        model,
        multilingual,
        quantized,
        subtask: multilingual ? subtask : null,
        language:
            multilingual && language !== "auto" ? language : null,
    });

}
window.do_whisper_web = do_whisper_web;

[hadfield]

I had a similar desire to use this in vanilla javascript form and had extracted some code from the react version to give it a try, but this version is much nicer. thanks for posting it! A rough version of whisper + vanilla js is in the repo: https://github.com/vital-ai/vital-stt-js i wanted to test this out in combination with a browser based wake word detector, which I posted here: https://github.com/chat-ai-app/chat-ai-assistant-demo with webapp posted here: https://demo-voice.chat.ai/