Language format conversion improvements (#709)

yujonglee · web-flow · commit ca5deb27dfec · 2025-05-03T13:55:34.000-07:00
diff --git a/crates/language/src/error.rs b/crates/language/src/error.rs
@@ -3,7 +3,7 @@ use serde::{ser::Serializer, Serialize};
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("Not supported language: {0}")]
-    NotSupportedLanguage(codes_iso_639::part_1::LanguageCode),
+    NotSupportedLanguage(String),
 }
 
 impl Serialize for Error {
diff --git a/crates/language/src/lib.rs b/crates/language/src/lib.rs
@@ -28,13 +28,11 @@ impl std::ops::Deref for Language {
     }
 }
 
-impl Language {
-    pub fn iso639(&self) -> ISO639 {
-        self.iso639
-    }
+#[cfg(feature = "whisper")]
+impl TryInto<hypr_whisper::Language> for Language {
+    type Error = Error;
 
-    #[cfg(feature = "whisper")]
-    pub fn for_whisper(self) -> Result<hypr_whisper::Language, Error> {
+    fn try_into(self) -> Result<hypr_whisper::Language, Self::Error> {
         use hypr_whisper::Language as WL;
 
         match self.iso639 {
@@ -135,9 +133,127 @@ impl Language {
             ISO639::Yi => Ok(WL::Yi),
             ISO639::Yo => Ok(WL::Yo),
             ISO639::Zh => Ok(WL::Zh),
-            _ => Err(Error::NotSupportedLanguage(self.iso639)),
+            _ => Err(Error::NotSupportedLanguage(self.to_string())),
         }
     }
+}
+
+#[cfg(feature = "whisper")]
+impl TryInto<Language> for hypr_whisper::Language {
+    type Error = Error;
+
+    fn try_into(self) -> Result<Language, Self::Error> {
+        use hypr_whisper::Language as WL;
+
+        let iso639 = match self {
+            WL::Af => ISO639::Af,
+            WL::Am => ISO639::Am,
+            WL::Ar => ISO639::Ar,
+            WL::As => ISO639::As,
+            WL::Az => ISO639::Az,
+            WL::Ba => ISO639::Ba,
+            WL::Be => ISO639::Be,
+            WL::Bg => ISO639::Bg,
+            WL::Bn => ISO639::Bn,
+            WL::Bo => ISO639::Bo,
+            WL::Br => ISO639::Br,
+            WL::Bs => ISO639::Bs,
+            WL::Ca => ISO639::Ca,
+            WL::Cs => ISO639::Cs,
+            WL::Cy => ISO639::Cy,
+            WL::Da => ISO639::Da,
+            WL::De => ISO639::De,
+            WL::El => ISO639::El,
+            WL::En => ISO639::En,
+            WL::Es => ISO639::Es,
+            WL::Et => ISO639::Et,
+            WL::Eu => ISO639::Eu,
+            WL::Fa => ISO639::Fa,
+            WL::Fi => ISO639::Fi,
+            WL::Fo => ISO639::Fo,
+            WL::Fr => ISO639::Fr,
+            WL::Gl => ISO639::Gl,
+            WL::Gu => ISO639::Gu,
+            WL::Ha => ISO639::Ha,
+            WL::He => ISO639::He,
+            WL::Hi => ISO639::Hi,
+            WL::Hr => ISO639::Hr,
+            WL::Ht => ISO639::Ht,
+            WL::Hu => ISO639::Hu,
+            WL::Hy => ISO639::Hy,
+            WL::Id => ISO639::Id,
+            WL::Is => ISO639::Is,
+            WL::It => ISO639::It,
+            WL::Ja => ISO639::Ja,
+            WL::Jw => ISO639::Jv,
+            WL::Ka => ISO639::Ka,
+            WL::Kk => ISO639::Kk,
+            WL::Km => ISO639::Km,
+            WL::Kn => ISO639::Kn,
+            WL::Ko => ISO639::Ko,
+            WL::La => ISO639::La,
+            WL::Lb => ISO639::Lb,
+            WL::Lo => ISO639::Lo,
+            WL::Lt => ISO639::Lt,
+            WL::Lv => ISO639::Lv,
+            WL::Mg => ISO639::Mg,
+            WL::Mi => ISO639::Mi,
+            WL::Mk => ISO639::Mk,
+            WL::Ml => ISO639::Ml,
+            WL::Mn => ISO639::Mn,
+            WL::Mr => ISO639::Mr,
+            WL::Ms => ISO639::Ms,
+            WL::Mt => ISO639::Mt,
+            WL::My => ISO639::My,
+            WL::Ne => ISO639::Ne,
+            WL::Nl => ISO639::Nl,
+            WL::Nn => ISO639::Nn,
+            WL::No => ISO639::No,
+            WL::Oc => ISO639::Oc,
+            WL::Pa => ISO639::Pa,
+            WL::Pl => ISO639::Pl,
+            WL::Ps => ISO639::Ps,
+            WL::Pt => ISO639::Pt,
+            WL::Ro => ISO639::Ro,
+            WL::Ru => ISO639::Ru,
+            WL::Sa => ISO639::Sa,
+            WL::Sd => ISO639::Sd,
+            WL::Si => ISO639::Si,
+            WL::Sk => ISO639::Sk,
+            WL::Sl => ISO639::Sl,
+            WL::Sn => ISO639::Sn,
+            WL::So => ISO639::So,
+            WL::Sq => ISO639::Sq,
+            WL::Sr => ISO639::Sr,
+            WL::Su => ISO639::Su,
+            WL::Sv => ISO639::Sv,
+            WL::Sw => ISO639::Sw,
+            WL::Ta => ISO639::Ta,
+            WL::Te => ISO639::Te,
+            WL::Tg => ISO639::Tg,
+            WL::Th => ISO639::Th,
+            WL::Tk => ISO639::Tk,
+            WL::Tl => ISO639::Tl,
+            WL::Tr => ISO639::Tr,
+            WL::Tt => ISO639::Tt,
+            WL::Uk => ISO639::Uk,
+            WL::Ur => ISO639::Ur,
+            WL::Uz => ISO639::Uz,
+            WL::Vi => ISO639::Vi,
+            WL::Yi => ISO639::Yi,
+            WL::Yo => ISO639::Yo,
+            WL::Zh => ISO639::Zh,
+            _ => return Err(Error::NotSupportedLanguage(self.to_string())),
+        };
+
+        Ok(Language { iso639 })
+    }
+}
+
+impl Language {
+    pub fn iso639(&self) -> ISO639 {
+        self.iso639
+    }
 
     #[cfg(feature = "deepgram")]
     pub fn for_deepgram(self) -> Result<deepgram::common::options::Language, Error> {
@@ -178,7 +294,7 @@ impl Language {
             ISO639::Uk => Ok(DG::uk),
             ISO639::Vi => Ok(DG::vi),
             ISO639::Zh => Ok(DG::zh),
-            _ => Err(Error::NotSupportedLanguage(self.iso639)),
+            _ => Err(Error::NotSupportedLanguage(self.to_string())),
         }
     }
 
@@ -218,7 +334,7 @@ impl Language {
             ISO639::Uk => Ok(String::from("стенограма")),
             ISO639::Vi => Ok(String::from("bản ghi")),
             ISO639::Zh => Ok(String::from("文字记录")),
-            _ => Err(Error::NotSupportedLanguage(self.iso639)),
+            _ => Err(Error::NotSupportedLanguage(self.to_string())),
         }
     }
 }
diff --git a/crates/stt/Cargo.toml b/crates/stt/Cargo.toml
@@ -13,7 +13,7 @@ local = []
 hypr-audio-utils = { workspace = true }
 hypr-clova = { path = "../clova", package = "clova" }
 hypr-db-user = { workspace = true }
-hypr-language = { workspace = true, features = ["deepgram"] }
+hypr-language = { workspace = true, features = ["deepgram", "whisper"] }
 hypr-listener-interface = { workspace = true }
 hypr-rtzr = { path = "../rtzr", package = "rtzr" }
 hypr-whisper = { workspace = true, features = ["cloud"] }
diff --git a/crates/stt/src/realtime/mod.rs b/crates/stt/src/realtime/mod.rs
@@ -64,8 +64,9 @@ impl ClientBuilder {
 
 #[derive(Debug)]
 pub enum MultiClient {
-    Deepgram(DeepgramClient),
     Clova(hypr_clova::realtime::Client),
+    Deepgram(DeepgramClient),
+    Whisper(hypr_whisper::cloud::WhisperClient),
 }
 
 #[derive(Debug, Clone)]
@@ -81,20 +82,29 @@ impl Client {
 
     pub async fn for_language(&self, language: hypr_language::Language) -> MultiClient {
         match language.iso639() {
+            // hypr_language::ISO639::Ko => {
+            //     let clova = hypr_clova::realtime::Client::builder()
+            //         .api_key(self.clova_api_key.as_ref().unwrap())
+            //         .keywords(vec!["하이퍼노트".to_string()])
+            //         .build()
+            //         .await
+            //         .unwrap();
+            //     MultiClient::Clova(clova)
+            // }
             hypr_language::ISO639::Ko => {
-                let clova = hypr_clova::realtime::Client::builder()
-                    .api_key(self.clova_api_key.as_ref().unwrap())
-                    .keywords(vec!["하이퍼노트".to_string()])
-                    .build()
-                    .await
-                    .unwrap();
-                MultiClient::Clova(clova)
+                let whisper = hypr_whisper::cloud::WhisperClient::builder()
+                    .api_base(std::env::var("WHISPER_API_BASE").unwrap())
+                    .api_key(std::env::var("WHISPER_API_KEY").unwrap())
+                    .language(language.try_into().unwrap())
+                    .build();
+
+                MultiClient::Whisper(whisper)
             }
             hypr_language::ISO639::En => {
                 let deepgram = DeepgramClient::builder()
                     .api_key(self.deepgram_api_key.as_ref().unwrap())
                     .keywords(vec!["Hyprnote".to_string()])
-                    .language(language.into())
+                    .language(language)
                     .build()
                     .unwrap();
 
@@ -118,8 +128,9 @@ where
         crate::Error,
     > {
         match self {
-            MultiClient::Deepgram(client) => Ok(Box::new(client.transcribe(stream).await?)),
             MultiClient::Clova(client) => Ok(Box::new(client.transcribe(stream).await?)),
+            MultiClient::Deepgram(client) => Ok(Box::new(client.transcribe(stream).await?)),
+            MultiClient::Whisper(client) => Ok(Box::new(client.transcribe(stream).await?)),
         }
     }
 }
diff --git a/crates/ws/src/client.rs b/crates/ws/src/client.rs
@@ -68,15 +68,21 @@ impl WebSocketClient {
 
         let output_stream = async_stream::stream! {
             while let Some(msg_result) = ws_receiver.next().await {
-                if let Ok(msg) = msg_result {
-                    match msg {
-                        Message::Text(_) | Message::Binary(_) => {
+                match msg_result {
+                    Ok(msg) => {
+                        match msg {
+                            Message::Text(_) | Message::Binary(_) => {
                             if let Some(output) = T::from_message(msg) {
                                 yield output;
                             }
                         },
                         Message::Ping(_) | Message::Pong(_) | Message::Frame(_) => continue,
-                        Message::Close(_) => break,
+                            Message::Close(_) => break,
+                        }
+                    }
+                    Err(e) => {
+                        tracing::error!("ws_receiver_failed: {:?}", e);
+                        break;
                     }
                 }
             }
diff --git a/plugins/listener/src/fsm.rs b/plugins/listener/src/fsm.rs
@@ -221,7 +221,7 @@ impl Session {
                     wav.write_sample(sample).unwrap();
                     wav.write_sample(sample).unwrap();
                     if let Err(e) = mixed_tx.send(sample).await {
-                        tracing::error!("mixed_tx_send_error: {:?}", e);
+                        tracing::error!("mixed_tx_send_error: {:?}", e.0);
                     }
                 }
             }
diff --git a/plugins/local-stt/src/server.rs b/plugins/local-stt/src/server.rs
@@ -119,10 +119,11 @@ async fn listen(
         .ok_or(StatusCode::TOO_MANY_REQUESTS)?;
 
     let model_path = state.model_type.model_path(&state.model_cache_dir);
+    let language = params.language.try_into().unwrap();
 
     let model = hypr_whisper::local::Whisper::builder()
         .model_path(model_path.to_str().unwrap())
-        .language(params.language.for_whisper().unwrap())
+        .language(language)
         .static_prompt(&params.static_prompt)
         .dynamic_prompt(&params.dynamic_prompt)
         .build();

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@ use serde::{ser::Serializer, Serialize};`
`3`	`3`	`#[derive(Debug, thiserror::Error)]`
`4`	`4`	`pub enum Error {`
`5`	`5`	`#[error("Not supported language: {0}")]`
`6`		`- NotSupportedLanguage(codes_iso_639::part_1::LanguageCode),`
	`6`	`+ NotSupportedLanguage(String),`
`7`	`7`	`}`
`8`	`8`
`9`	`9`	`impl Serialize for Error {`
Original file line number	Diff line number	Diff line change
`@@ -221,7 +221,7 @@ impl Session {`
`221`	`221`	`wav.write_sample(sample).unwrap();`
`222`	`222`	`wav.write_sample(sample).unwrap();`
`223`	`223`	`if let Err(e) = mixed_tx.send(sample).await {`
`224`		`- tracing::error!("mixed_tx_send_error: {:?}", e);`
	`224`	`+ tracing::error!("mixed_tx_send_error: {:?}", e.0);`
`225`	`225`	`}`
`226`	`226`	`}`
`227`	`227`	`}`