Add language identification swiftui demo (#729)

Author: yujinqiu

ios-swiftui/SherpaOnnx/SherpaOnnx/Model.swift

@@ -48,6 +48,28 @@ func getBilingualStreamingZhEnParaformer() -> SherpaOnnxOnlineModelConfig {
   )
 }
 
+// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html#tiny-en
+//
+func getLanguageIdentificationTiny() -> SherpaOnnxSpokenLanguageIdentificationConfig
+ {
+  let encoder = getResource("tiny-encoder.int8", "onnx")
+  let decoder = getResource("tiny-decoder.int8", "onnx")
+    
+    let whisperConfig = sherpaOnnxSpokenLanguageIdentificationWhisperConfig(
+      encoder: encoder,
+      decoder: decoder
+    )
+
+    let config = sherpaOnnxSpokenLanguageIdentificationConfig(
+      whisper: whisperConfig,
+      numThreads: 1,
+      debug: 1,
+      provider: "cpu"
+    )
+    return config
+}
+
+
 /// Please refer to
 /// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
 /// to add more models if you need

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID.xcodeproj/project.pbxproj

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEDidComputeMac32BitWarning</key>
+	<true/>
+</dict>
+</plist>

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID.xcodeproj/project.xcworkspace/contents.xcworkspacedata

@@ -0,0 +1,11 @@
+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -0,0 +1,14 @@
+{
+  "images" : [
+    {
+      "filename" : "k2-1024x1024.png",
+      "idiom" : "universal",
+      "platform" : "ios",
+      "size" : "1024x1024"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID/Assets.xcassets/AccentColor.colorset/Contents.json

@@ -0,0 +1,13 @@
+{
+  "images" : [
+    {
+      "idiom" : "universal",
+      "platform" : "ios",
+      "size" : "1024x1024"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID/Assets.xcassets/AppIcon 1.appiconset/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID/Assets.xcassets/AppIcon 1.appiconset/k2-1024x1024.png

@@ -0,0 +1,46 @@
+//
+//  ContentView.swift
+//  SherpaOnnxLangID
+//
+//  Created by knight on 2024/4/1.
+//
+
+import SwiftUI
+
+struct ContentView: View {
+    @StateObject var viewModel = ViewModel()
+
+    var body: some View {
+        VStack {
+            Text("ASR with Next-gen Kaldi")
+                .font(.title)
+            if viewModel.status == .stop {
+                Text("See https://github.com/k2-fsa/sherpa-onnx")
+                Text("Press the Start button to run!")
+            }
+            if viewModel.status == .recording {
+                Text("Stop will show recording language.")
+            }
+            Spacer()
+            Text("Recording language is: \(viewModel.language)")
+                .frame(maxWidth: .infinity)
+            Spacer()
+            Button {
+                toggleRecorder()
+            } label: {
+                Text(viewModel.status == .stop ? "Start" : "Stop")
+            }
+        }
+        .padding()
+    }
+
+    private func toggleRecorder() {
+        Task {
+            await viewModel.toggleRecorder()
+        }
+    }
+}
+
+#Preview {
+    ContentView()
+}

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID/Assets.xcassets/AppIcon.appiconset/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID/Assets.xcassets/Contents.json

@@ -0,0 +1,17 @@
+//
+//  SherpaOnnxLangIDApp.swift
+//  SherpaOnnxLangID
+//
+//  Created by knight on 2024/4/1.
+//
+
+import SwiftUI
+
+@main
+struct SherpaOnnxLangIDApp: App {
+    var body: some Scene {
+        WindowGroup {
+            ContentView()
+        }
+    }
+}

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID/ContentView.swift

@@ -0,0 +1,131 @@
+//
+//  ViewModel.swift
+//  SherpaOnnxLangID
+//
+//  Created by knight on 2024/4/1.
+//
+
+import SwiftUI
+import AVFoundation
+
+enum Status {
+    case stop
+    case recording
+}
+
+@MainActor
+class ViewModel:ObservableObject {
+    @Published var status: Status = .stop
+
+    @Published var language: String = ""
+    
+    var languageIdentifier: SherpaOnnxSpokenLanguageIdentificationWrapper? = nil
+    var audioEngine: AVAudioEngine? = nil
+    
+    var voices: [Float] = []
+
+    init() {
+        initRecorder()
+        initRecognizer()
+    }
+    
+    private func initRecognizer() {
+        var config =  getLanguageIdentificationTiny()
+        self.languageIdentifier = SherpaOnnxSpokenLanguageIdentificationWrapper(config: &config)
+    }
+    
+    private func initRecorder() {
+        print("init recorder")
+        audioEngine = AVAudioEngine()
+        let inputNode = self.audioEngine?.inputNode
+        let bus = 0
+        let inputFormat = inputNode?.outputFormat(forBus: bus)
+        let outputFormat = AVAudioFormat(
+            commonFormat: .pcmFormatFloat32,
+            sampleRate: 16000, channels: 1,
+            interleaved: false)!
+
+        let converter = AVAudioConverter(from: inputFormat!, to: outputFormat)!
+
+        inputNode!.installTap(
+            onBus: bus,
+            bufferSize: 1024,
+            format: inputFormat
+        ) {
+            (buffer: AVAudioPCMBuffer, when: AVAudioTime) in
+            var newBufferAvailable = true
+
+            let inputCallback: AVAudioConverterInputBlock = {
+                inNumPackets, outStatus in
+                if newBufferAvailable {
+                    outStatus.pointee = .haveData
+                    newBufferAvailable = false
+
+                    return buffer
+                } else {
+                    outStatus.pointee = .noDataNow
+                    return nil
+                }
+            }
+
+            let convertedBuffer = AVAudioPCMBuffer(
+                pcmFormat: outputFormat,
+                frameCapacity:
+                    AVAudioFrameCount(outputFormat.sampleRate)
+                * buffer.frameLength
+                / AVAudioFrameCount(buffer.format.sampleRate))!
+
+            var error: NSError?
+            let _ = converter.convert(
+                to: convertedBuffer,
+                error: &error, withInputFrom: inputCallback)
+
+            // TODO(fangjun): Handle status != haveData
+
+            let array = convertedBuffer.array()
+            if !array.isEmpty {
+                self.voices.append(contentsOf: array)
+            }
+        }
+    }
+    
+    public func toggleRecorder() async{
+        if status == .stop {
+            await startRecorder()
+        } else {
+            await stopRecorder()
+        }
+    }
+
+    private func startRecorder() async {
+        await MainActor.run {
+            self.language = ""
+        }
+        if !self.voices.isEmpty {
+            self.voices = []
+        }
+        do {
+            try self.audioEngine?.start()
+            status = .recording
+            print("started")
+        } catch let error as NSError {
+            print("Got an error starting audioEngine: \(error.domain), \(error)")
+        }
+    }
+
+    private func stopRecorder() async {
+        audioEngine?.stop()
+        print("stopped, and begin identify language")
+        await self.identify()
+        status = .stop
+    }
+    
+    private func identify() async {
+        let result = self.languageIdentifier?    .decode(samples: self.voices)
+        if let language = result?.lang {
+            await MainActor.run {
+                self.language = language
+            }
+        }
+    }
+}

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID/Preview Content/Preview Assets.xcassets/Contents.json

@@ -0,0 +1,36 @@
+//
+//  SherpaOnnxLangIDTests.swift
+//  SherpaOnnxLangIDTests
+//
+//  Created by knight on 2024/4/1.
+//
+
+import XCTest
+@testable import SherpaOnnxLangID
+
+final class SherpaOnnxLangIDTests: XCTestCase {
+
+    override func setUpWithError() throws {
+        // Put setup code here. This method is called before the invocation of each test method in the class.
+    }
+
+    override func tearDownWithError() throws {
+        // Put teardown code here. This method is called after the invocation of each test method in the class.
+    }
+
+    func testExample() throws {
+        // This is an example of a functional test case.
+        // Use XCTAssert and related functions to verify your tests produce the correct results.
+        // Any test you write for XCTest can be annotated as throws and async.
+        // Mark your test throws to produce an unexpected failure when your test encounters an uncaught error.
+        // Mark your test async to allow awaiting for asynchronous code to complete. Check the results with assertions afterwards.
+    }
+
+    func testPerformanceExample() throws {
+        // This is an example of a performance test case.
+        self.measure {
+            // Put the code you want to measure the time of here.
+        }
+    }
+
+}

ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID/SherpaOnnxLangIDApp.swift

@@ -0,0 +1,41 @@
+//
+//  SherpaOnnxLangIDUITests.swift
+//  SherpaOnnxLangIDUITests
+//
+//  Created by knight on 2024/4/1.
+//
+
+import XCTest
+
+final class SherpaOnnxLangIDUITests: XCTestCase {
+
+    override func setUpWithError() throws {
+        // Put setup code here. This method is called before the invocation of each test method in the class.
+
+        // In UI tests it is usually best to stop immediately when a failure occurs.
+        continueAfterFailure = false
+
+        // In UI tests it’s important to set the initial state - such as interface orientation - required for your tests before they run. The setUp method is a good place to do this.
+    }
+
+    override func tearDownWithError() throws {
+        // Put teardown code here. This method is called after the invocation of each test method in the class.
+    }
+
+    func testExample() throws {
+        // UI tests must launch the application that they test.
+        let app = XCUIApplication()
+        app.launch()
+
+        // Use XCTAssert and related functions to verify your tests produce the correct results.
+    }
+
+    func testLaunchPerformance() throws {
+        if #available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 7.0, *) {
+            // This measures how long it takes to launch your application.
+            measure(metrics: [XCTApplicationLaunchMetric()]) {
+                XCUIApplication().launch()
+            }
+        }
+    }
+}