how to add custom speech in botframe work

[Tamoormughal]

I am working on Custom speech in bot framework.I add Bing simple speech it is working on my website: here is link :https://envolvebot.azurewebsites.net/

but now i want to know how can we add custom speech (LUIS custom speech) in our webchat bot? I create custom speech using : https://westus.cris.ai/Home/CustomSpeech Now i want to add it and want it to work like bing speech .

i am using following code.

`/! modernizr 3.6.0 (Custom Build) | MIT

• https://modernizr.com/download/?-getusermedia-speechsynthesis !*/ !function (e, n, s) { function i(e, n) { return typeof e === n } function o() { var e, n, s, o, a, f, d; for (var c in t) if (t.hasOwnProperty(c)) { if (e = [], n = t[c], n.name && (e.push(n.name.toLowerCase()), n.options && n.options.aliases && n.options.aliases.length)) for (s = 0; s < n.options.aliases.length; s++)e.push(n.options.aliases[s].toLowerCase()); for (o = i(n.fn, "function") ? n.fn() : n.fn, a = 0; a < e.length; a++)f = e[a], d = f.split("."), 1 === d.length ? Modernizr[d[0]] = o : (!Modernizr[d[0]] || Modernizr[d[0]] instanceof Boolean || (Modernizr[d[0]] = new Boolean(Modernizr[d[0]])), Modernizr[d[0]][d[1]] = o), r.push((o ? "" : "no-") + d.join("-")) } } var t = [], a = { _version: "3.6.0", _config: { classPrefix: "", enableClasses: !0, enableJSClass: !0, usePrefixes: !0 }, _q: [], on: function (e, n) { var s = this; setTimeout(function () { n(s[e]) }, 0) }, addTest: function (e, n, s) { t.push({ name: e, fn: n, options: s }) }, addAsyncTest: function (e) { t.push({ name: null, fn: e }) } }, Modernizr = function () { }; Modernizr.prototype = a, Modernizr = new Modernizr, Modernizr.addTest("speechsynthesis", "SpeechSynthesisUtterance" in e), Modernizr.addTest("getUserMedia", "mediaDevices" in navigator && "getUserMedia" in navigator.mediaDevices); var r = []; o(), delete a.addTest, delete a.addAsyncTest; for (var f = 0; f < Modernizr._q.length; f++)Modernizr._q[f](); e.Modernizr = Modernizr }(window, document);

  // Necessary for safari // Safari will only speak after speaking from a button click var isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); const params = BotChat.queryParams(location.search); debugger; function SpeakText() { var msg = new SpeechSynthesisUtterance(); window.speechSynthesis.speak(msg);

  document.getElementsByClassName("wc-mic")[0].removeEventListener("click", SpeakText);

  }

  if (isSafari) {

  window.addEventListener("load", function () {
      document.getElementsByClassName("wc-mic")[0].addEventListener("click", SpeakText);
  });

  }

  var femaleVoice = null;

  if (Modernizr.speechsynthesis) { var baseSpeechPrototype = SpeechSynthesisUtterance.prototype; SpeechSynthesisUtterance = function (msg) { var utterance = new baseSpeechPrototype.constructor(msg);

      if (femaleVoice != null)
          utterance.voice = femaleVoice;
  
      return utterance;
  }
  
  window.speechSynthesis.onvoiceschanged = function () {
      var voice = "Microsoft Zira";
      var safariVoice = "Samantha";
  
      var voices = window.speechSynthesis.getVoices();
      voices.forEach(function (v) {
          if (v.name.startsWith(voice))
              femaleVoice = v;
          else if (v.name == safariVoice && v.lang == "en-US")
              femaleVoice = v;
      })
  }

  }

  //// Needed to change between the two audio contexts var AudioContext = window.AudioContext || window.webkitAudioContext;

  //// Sets the old style getUserMedia to use the new style that is supported in more browsers even though the framework uses the new style if (window.navigator.mediaDevices.getUserMedia && !window.navigator.getUserMedia) { window.navigator.getUserMedia = function (constraints, successCallback, errorCallback) { window.navigator.mediaDevices.getUserMedia(constraints) .then(function (e) { successCallback(e); }) .catch(function (e) { errorCallback(e); }); }; }

  const bot = { id: params['botid'] || 'botid', name: params['botname'] || 'botname' };

  window.botchatDebug = params['debug'] && params['debug'] === 'true';

  var speechOptions; if (Modernizr.speechsynthesis) { speechOptions = { speechRecognizer: new CognitiveServices.SpeechRecognizer({ fetchCallback: function (authFetchEventId) { return getBingToken() }, fetchOnExpiryCallback: function (authFetchEventId) { getBingToken() } }), speechSynthesizer: new BotChat.Speech.BrowserSpeechSynthesizer() }; } else if (Modernizr.getusermedia) { speechOptions = { speechRecognizer: new CognitiveServices.SpeechRecognizer({ fetchCallback: function (authFetchEventId) { return getBingToken() }, fetchOnExpiryCallback: function (authFetchEventId) { getBingToken() } }), speechSynthesizer: null }; } else { speechOptions = null; }

  // for bot

  var botConnection = new BotChat.DirectLine({ domain: params['domain'], //token: document.getElementById("directLineToken").value, secret: 'Secret', webSocket: params['webSocket'] && params['webSocket'] === 'true' // defaults to true });

  BotChat.App({
      sendTyping: true,
      locale: params['locale'],
      resize: 'detect',
      speechOptions: speechOptions,
      user: { id:'ID' name: "You" },
      bot: { id: 'Vera', name: "Vera" },
      botConnection: botConnection
  }, document.getElementById('BotChatElement'));

  function getBingToken() { // Normally this token fetch is done from your secured backend to avoid exposing the API key and this call // would be to your backend, or to retrieve a token that was served as part of the original page. return fetch( 'https://api.cognitive.microsoft.com/sts/v1.0/issueToken', { headers: { 'Ocp-Apim-Subscription-Key': 'Key' }, method: 'POST' } //).then(res => res.text()); ).then(function (res) { return res.text() }); }`

aNY BODY Know how to do this ?

Thank in advance .

[compulim]

The existing SDK we consume is based on Bing Speech. And unfortunately, their SDK does not support CRIS.

Speech Services team just release a new SDK at Ignite (9/24). Their new SDK support their recently GA-ed Speech Services and CRIS. There are some works for us to do in order to consumer their new SDK.

[sgellock]

dup of #1217

[soso-maitha]

The existing SDK we consume is based on Bing Speech. And unfortunately, their SDK does not support CRIS.

Speech Services team just release a new SDK at Ignite (9/24). Their new SDK support their recently GA-ed Speech Services and CRIS. There are some works for us to do in order to consumer their new SDK.

@compulim Do you have any idea when Bot Framework WebChat will be supported with Custom Speech? I need this importantly.

[soso-maitha]

@Tamoormughal check this tutorial https://aischool.microsoft.com/en-us/conversational/learning-paths/building-an-intelligent-bot/adding-custom-speech-support , this is the bot code they are using for custom speech https://github.com/Microsoft/BuildAnIntelligentBot but this require a lot of work to integrate with your bot specially if you are already done with development. Also check this one it is very straight forward, i will try on my bot and will let you know https://channel9.msdn.com/Blogs/MVP-Azure/Cognitive-Services-Episode-9-Build-the-ultimate-chatbot-with-Custom-Speech

[compulim]

@soso-maitha this is the top priority work in our current cycle. We want to be as transparent as possible on this work. We are tracking the effort at #1336 and we will keep updating the work item as we progress thru.

I don't have a definite date for now because we are still early in development stage. We want to target 12/15 but there are some risks involved, which is outlined in the issue #1336. Can you subscribe thru that issue to track the work?

Btw, GA = general available, and in other words, it means the feature/product go production.

[soso-maitha]

@Tamoormughal Hi! I found the solution! Please follow these steps:

1. Copy the all the code from CognitiveServices.js file from this project https://github.com/Microsoft/BuildAnIntelligentBot and replace it with the original CognitiveServices.js you have from WebChat (for me I prettified the code using https://www.prettifyjs.net/ in order to understand it first . You will notice that it considers both Custom Speech CRIS (search for wss://westus.stt.speech.microsoft.com) and Bing Speech (search for wss://speech.platform.bing.com) while in the standard webchat you will find only Bing speech url in the CognitiveServices.js)

2. In the speechOptions , change the SpeechRecognizer to the following: const speechOptions = { speechRecognizer: new CognitiveServices.SpeechRecognizer({ subscriptionKey: 'yourCustomSpeechSubscriptionKey', recognitionAPI: 1, endpointId: 'yourCustomSpeechEndPointId', }),

Notes:

• recognitionAPI: value 1 means custom speech
• the custom speech key and point Id you should get it from your CRIS page after doing all the steps (language model, acoustic model..etc)

I tested it and it is working without issues! :) let me know if you need any further help.

regards,

[Tamoormughal]

hi @soso-maitha bundle of thanks it is working . one issue is this is not working on safari and iPhone. if you have any material kindly share it . Thanks in advance .

[compulim]

@Tamoormughal Cognitive Services use WebRTC for streaming sound clips. And there are some limitations on iOS:

• Requires iOS 11 or up
• Requires Safari (not Edge/Chrome, not WebView in apps)
• Requires a tap to start the recognition for the first time after page load (for user privacy reason)
• Requires HTTPS or localhost

Unfortunately, these are hard limitations due to platform restrictions.

[jchomarat]

Hello - I am not sure what is the status of this issue - but we also needed to inject a custom speech in the bot, via the webchat client (using the speech endpointId) - so we instantiated a webchat with an empty store, added a mic button, and attach events to it to actually call the speechSDK (microsoft.cognitiveservices.speech.sdk.bundle.js)

speechConfig = SpeechSDK.SpeechConfig.fromAuthorizationToken(authorizationToken, region); speechConfig.speechRecognitionLanguage = locale; speechConfig.endpointId = endpointId; reco = new SpeechSDK.SpeechRecognizer(speechConfig, audioConfig);

and handle all the events on the reco object.

The output is written in the chat box, so we just need to hit "enter" (we could also do it via actions) to send the corrected text to the bot.

It is far from being an elegant solution, but it does the job.

Any idea on when we will be able to use the standard webchat client with the ability to send a custom speech?

Best Julien