Provide a way to create a token provider instance only

[ctron]

Having a setup with Rust and WASM, I do have the problem that I can't pass in the monaco parameter to the wireGrammar function. There is no type/instance for the namespace.

However, using an existing wrapper for the monco editor, I already have the API available for calling monaco.languages.setTokensProvider. If I would have an instance to set.

Unfortunately, this is hidden in the wireTmGrammars. It would be great if the API could be extended, to offer some createProvider function, which creates the provider instance, but not setting it.

Something like:

export function createProvider(grammar: IGrammar, editor?: monacoNsps.editor.ICodeEditor) : TokensProvider {
    return {
        getInitialState: () => new TokenizerState(INITIAL),
        tokenize: (line: string, state: TokenizerState) => {
            const res = grammar.tokenizeLine(line, state.ruleStack)
            return {
                endState: new TokenizerState(res.ruleStack),
                tokens: res.tokens.map(token => ({
                    ...token,
                    // TODO: At the moment, monaco-editor doesn't seem to accept array of scopes
                    scopes: editor ? TMToMonacoToken(editor, token.scopes) : token.scopes[token.scopes.length - 1]
                })),
            }
        }
    }
}

export function wireTmGrammars(monaco: typeof monacoNsps, registry: Registry, languages: Map<string, string>, editor?: monacoNsps.editor.ICodeEditor) {
    return Promise.all(
        Array.from(languages.keys())
            .map(async (languageId) => {
                const grammar = await registry.loadGrammar(languages.get(languageId))
                const provider = createProvider(grammar, editor);
                monaco.languages.setTokensProvider(languageId, provider)
            })
    )
}

This would allow one to re-use as much code as possible, but performing the task of setting the tokens provider manually.