From 0a061149dd8c143c1300ec721bf20f5ac423d740 Mon Sep 17 00:00:00 2001 From: Rohini-Microsoft Date: Mon, 27 May 2024 10:11:58 +0530 Subject: [PATCH 1/3] added logic for slight pause --- code/frontend/src/pages/chat/Chat.tsx | 39 +++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/code/frontend/src/pages/chat/Chat.tsx b/code/frontend/src/pages/chat/Chat.tsx index b0fb1b524..3fef73eba 100644 --- a/code/frontend/src/pages/chat/Chat.tsx +++ b/code/frontend/src/pages/chat/Chat.tsx @@ -130,25 +130,48 @@ const Chat = () => { return abortController.abort(); }; - + let recognizedTextBuffer = ''; const startSpeechRecognition = async () => { if (!isRecognizing) { setIsRecognizing(true); recognizerRef.current = await multiLingualSpeechRecognizer(); // Store the recognizer in the ref - recognizerRef.current.recognized = (s, e) => { if (e.result.reason === ResultReason.RecognizedSpeech) { const recognized = e.result.text; - setUserMessage(recognized); - setRecognizedText(recognized); + recognizedTextBuffer += recognized; + // document.getElementById('outputElement').value = recognizedTextBuffer.trim(); + // document.getElementById("outputElement").setText( recognizedTextBuffer.trim()); + setUserMessage(recognizedTextBuffer.trim()); + setRecognizedText(recognizedTextBuffer.trim()); } + // if (e.result.reason === ResultReason.RecognizedSpeech) { + // console.log(`Recognized: ${e.result.text}`); + // const recognized = e.result.text; + // setUserMessage(recognized); + // setRecognizedText(recognized); + // } else if (e.result.reason === ResultReason.NoMatch) { + // console.error("No speech could be recognized."); + // } else if (e.result.reason === ResultReason.Canceled) { + // console.error(`Recognition was canceled. Error details: ${e.result.errorDetails}`); + // } }; - recognizerRef.current.startContinuousRecognitionAsync(() => { - setIsRecognizing(true); - setIsListening(true); - }); + recognizerRef.current.startContinuousRecognitionAsync( + () => { + console.log("Recognition started"); + setIsRecognizing(true); + setIsListening(true); + }, + error => { + console.error(`Error starting recognition: ${error}`); + } + ); + + recognizerRef.current.recognizing = (s, e) => { + console.log(`Recognizing: ${e.result.text}`); + }; + } }; From d4f7e884502566c319c5a4d7ea46d78ddd3b6b2a Mon Sep 17 00:00:00 2001 From: Rohini-Microsoft Date: Mon, 27 May 2024 17:24:23 +0530 Subject: [PATCH 2/3] append text after slight pause. disable the textarea when microphone is on --- .../QuestionInput/QuestionInput.tsx | 6 ++- code/frontend/src/pages/chat/Chat.tsx | 37 ++++++++++--------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/code/frontend/src/components/QuestionInput/QuestionInput.tsx b/code/frontend/src/components/QuestionInput/QuestionInput.tsx index 4cf363d97..0adb5ca7e 100644 --- a/code/frontend/src/components/QuestionInput/QuestionInput.tsx +++ b/code/frontend/src/components/QuestionInput/QuestionInput.tsx @@ -35,12 +35,14 @@ export const QuestionInput = ({ const [liveRecognizedText, setLiveRecognizedText] = useState(""); const [microphoneIconActive, setMicrophoneIconActive] = useState(false); - + const [isTextAreaDisabled, setIsTextAreaDisabled] = useState(false); useEffect(() => { if (isRecognizing) { setLiveRecognizedText(recognizedText); + setIsTextAreaDisabled(true) setMicrophoneIconActive(true); // Set microphone icon to active (blue) } else { + setIsTextAreaDisabled(false) setMicrophoneIconActive(false); // Set microphone icon to inactive } }, [recognizedText, isRecognizing]); @@ -81,6 +83,8 @@ export const QuestionInput = ({ {/* Text Input Field */} { return abortController.abort(); }; - let recognizedTextBuffer = ''; + // Buffer to store recognized text + let recognizedTextBuffer = ""; + let currentSentence = ""; + const startSpeechRecognition = async () => { if (!isRecognizing) { setIsRecognizing(true); @@ -138,28 +141,26 @@ const Chat = () => { recognizerRef.current = await multiLingualSpeechRecognizer(); // Store the recognizer in the ref recognizerRef.current.recognized = (s, e) => { if (e.result.reason === ResultReason.RecognizedSpeech) { - const recognized = e.result.text; - recognizedTextBuffer += recognized; - // document.getElementById('outputElement').value = recognizedTextBuffer.trim(); - // document.getElementById("outputElement").setText( recognizedTextBuffer.trim()); - setUserMessage(recognizedTextBuffer.trim()); - setRecognizedText(recognizedTextBuffer.trim()); + let recognizedText = e.result.text.trim(); + + // Append current sentence to buffer if it's not empty + if (currentSentence) { + recognizedTextBuffer += ` ${currentSentence.trim()}`; + currentSentence = ""; + } + + // Start new sentence + currentSentence += ` ${recognizedText}`; + + //set text in textarea + setUserMessage((recognizedTextBuffer + currentSentence).trim()); + setRecognizedText((recognizedTextBuffer + currentSentence).trim()); + } - // if (e.result.reason === ResultReason.RecognizedSpeech) { - // console.log(`Recognized: ${e.result.text}`); - // const recognized = e.result.text; - // setUserMessage(recognized); - // setRecognizedText(recognized); - // } else if (e.result.reason === ResultReason.NoMatch) { - // console.error("No speech could be recognized."); - // } else if (e.result.reason === ResultReason.Canceled) { - // console.error(`Recognition was canceled. Error details: ${e.result.errorDetails}`); - // } }; recognizerRef.current.startContinuousRecognitionAsync( () => { - console.log("Recognition started"); setIsRecognizing(true); setIsListening(true); }, From d641690f32edb1c62079337fd3253ec5f1a978e3 Mon Sep 17 00:00:00 2001 From: Rohini-Microsoft Date: Mon, 27 May 2024 17:31:44 +0530 Subject: [PATCH 3/3] remove the extra lines --- code/frontend/src/pages/chat/Chat.tsx | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/code/frontend/src/pages/chat/Chat.tsx b/code/frontend/src/pages/chat/Chat.tsx index da8311b02..17f09de88 100644 --- a/code/frontend/src/pages/chat/Chat.tsx +++ b/code/frontend/src/pages/chat/Chat.tsx @@ -137,25 +137,20 @@ const Chat = () => { const startSpeechRecognition = async () => { if (!isRecognizing) { setIsRecognizing(true); - recognizerRef.current = await multiLingualSpeechRecognizer(); // Store the recognizer in the ref recognizerRef.current.recognized = (s, e) => { if (e.result.reason === ResultReason.RecognizedSpeech) { let recognizedText = e.result.text.trim(); - // Append current sentence to buffer if it's not empty if (currentSentence) { recognizedTextBuffer += ` ${currentSentence.trim()}`; currentSentence = ""; } - // Start new sentence currentSentence += ` ${recognizedText}`; - //set text in textarea setUserMessage((recognizedTextBuffer + currentSentence).trim()); setRecognizedText((recognizedTextBuffer + currentSentence).trim()); - } }; @@ -168,11 +163,6 @@ const Chat = () => { console.error(`Error starting recognition: ${error}`); } ); - - recognizerRef.current.recognizing = (s, e) => { - console.log(`Recognizing: ${e.result.text}`); - }; - } };