Inverse text normalization API of streaming ASR for various programming languages (#1022)

Author: csukuangfj

.github/scripts/test-dart.sh

@@ -66,7 +66,9 @@ echo '----------streaming zipformer ctc----------'
 rm -rf sherpa-onnx-*
 
 echo '----------streaming zipformer transducer----------'
+./run-zipformer-transducer-itn.sh
 ./run-zipformer-transducer.sh
+rm -f itn*
 rm -rf sherpa-onnx-*
 
 echo '----------streaming NeMo transducer----------'

.github/scripts/test-dot-net.sh

@@ -2,7 +2,13 @@
 
 cd dotnet-examples/
 
-cd ./offline-decode-files
+cd ./online-decode-files
+./run-transducer-itn.sh
+./run-zipformer2-ctc.sh
+./run-transducer.sh
+./run-paraformer.sh
+
+cd ../offline-decode-files
 ./run-paraformer-itn.sh
 ./run-telespeech-ctc.sh
 ./run-nemo-ctc.sh
@@ -27,11 +33,6 @@ cd ../streaming-hlg-decoding/
 cd ../spoken-language-identification
 ./run.sh
 
-cd ../online-decode-files
-./run-zipformer2-ctc.sh
-./run-transducer.sh
-./run-paraformer.sh
-
 cd ../offline-tts
 ./run-aishell3.sh
 ./run-piper.sh

.github/scripts/test-nodejs-addon-npm.sh

@@ -70,6 +70,13 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/s
 tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
 rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
 
+rm -f itn*
+
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn-zh-number.wav
+
+node test_asr_streaming_transducer_itn.js
+
 node test_asr_streaming_transducer.js
 
 rm -rf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20
@@ -120,6 +127,8 @@ rm sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2
 
 node ./test_asr_non_streaming_paraformer.js
 
+rm -f itn*
+
 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst
 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn-zh-number.wav
 

.github/scripts/test-nodejs-npm.sh

@@ -15,6 +15,8 @@ curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/s
 ls -lh
 tar xvf sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2
 rm sherpa-onnx-paraformer-zh-2023-03-28.tar.bz2
+
+rm -f itn*
 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn-zh-number.wav
 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst
 node ./test-offline-paraformer-itn.js
@@ -57,7 +59,15 @@ rm -rf sherpa-onnx-streaming-paraformer-bilingual-zh-en
 curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
 tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
 rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
+
+rm -f itn*
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn-zh-number.wav
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst
+
+node ./test-online-transducer-itn.js
+
 node ./test-online-transducer.js
+
 rm -rf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20
 
 curl -LS -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2

.github/workflows/build-wheels-aarch64.yaml

@@ -2,6 +2,8 @@ name: build-wheels-aarch64
 
 on:
   push:
+    branches:
+      - wheel
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
   workflow_dispatch:

.github/workflows/build-wheels-armv7l.yaml

@@ -2,6 +2,8 @@ name: build-wheels-armv7l
 
 on:
   push:
+    branches:
+      - wheel
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
   workflow_dispatch:

.github/workflows/build-wheels-linux.yaml

@@ -2,6 +2,8 @@ name: build-wheels-linux
 
 on:
   push:
+    branches:
+      - wheel
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
   workflow_dispatch:

.github/workflows/build-wheels-macos-arm64.yaml

@@ -2,6 +2,8 @@ name: build-wheels-macos-arm64
 
 on:
   push:
+    branches:
+      - wheel
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
   workflow_dispatch:
@@ -84,7 +86,7 @@ jobs:
         run: |
           opts='--break-system-packages'
           v=${{ matrix.python-version }}
-          if [[ $v == cp38 || $v == cp39 ]]; then
+          if [[ $v == cp37 || $v == cp38 || $v == cp39 ]]; then
             opts=''
           fi
 

.github/workflows/build-wheels-macos-x64.yaml

@@ -101,7 +101,7 @@ jobs:
         run: |
           opts='--break-system-packages'
           v=${{ matrix.python-version }}
-          if [[ $v == cp38 || $v == cp39 ]]; then
+          if [[ $v == cp37 || $v == cp38 || $v == cp39 ]]; then
             opts=''
           fi
 

.github/workflows/build-wheels-win32.yaml

@@ -2,6 +2,8 @@ name: build-wheels-win32
 
 on:
   push:
+    branches:
+      - wheel
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
   workflow_dispatch:

.github/workflows/run-java-test.yaml

@@ -173,6 +173,9 @@ jobs:
         shell: bash
         run: |
           cd ./java-api-examples
+          ./run-inverse-text-normalization-transducer.sh
+          rm -rf sherpa-onnx-streaming-*
+
           ./run-streaming-decode-file-ctc.sh
           # Delete model files to save space
           rm -rf sherpa-onnx-streaming-*

.github/workflows/test-go.yaml

@@ -187,6 +187,9 @@ jobs:
           ./run-transducer.sh
           rm -rf sherpa-onnx-streaming-zipformer-en-2023-06-26
 
+          ./run-transducer-itn.sh
+          rm -rf sherpa-onnx-streaming-*
+
           echo "Test paraformer"
           ./run-paraformer.sh
           rm -rf sherpa-onnx-streaming-paraformer-bilingual-zh-en

CMakeLists.txt

@@ -7,7 +7,8 @@ project(sherpa-onnx)
 # Remember to update
 # ./nodejs-addon-examples
 # ./dart-api-examples/
-set(SHERPA_ONNX_VERSION "1.9.30")
+# ./sherpa-onnx/flutter/CHANGELOG.md
+set(SHERPA_ONNX_VERSION "1.10.0")
 
 # Disable warning about
 #

android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt

@@ -196,6 +196,9 @@ class MainActivity : AppCompatActivity() {
         // See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
         // for a list of available models
         val type = 0
+        var ruleFsts : String?
+        ruleFsts = null
+
         Log.i(TAG, "Select model type $type")
         val config = OnlineRecognizerConfig(
             featConfig = getFeatureConfig(sampleRate = sampleRateInHz, featureDim = 80),
@@ -205,6 +208,10 @@ class MainActivity : AppCompatActivity() {
             enableEndpoint = true,
         )
 
+        if (ruleFsts != null) {
+            config.ruleFsts = ruleFsts
+        }
+
         recognizer = OnlineRecognizer(
             assetManager = application.assets,
             config = config,

android/SherpaOnnx2Pass/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt

@@ -194,13 +194,18 @@ class MainActivity : AppCompatActivity() {
         // See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
         // for a list of available models
         val firstType = 9
+        val firstRuleFsts: String?
+        firstRuleFsts = null
         Log.i(TAG, "Select model type $firstType for the first pass")
         val config = OnlineRecognizerConfig(
             featConfig = getFeatureConfig(sampleRate = sampleRateInHz, featureDim = 80),
             modelConfig = getModelConfig(type = firstType)!!,
             endpointConfig = getEndpointConfig(),
             enableEndpoint = true,
         )
+        if (firstRuleFsts != null) {
+            config.ruleFsts = firstRuleFsts;
+        }
 
         onlineRecognizer = OnlineRecognizer(
             assetManager = application.assets,
@@ -213,13 +218,19 @@ class MainActivity : AppCompatActivity() {
         // See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
         // for a list of available models
         val secondType = 0
+        var secondRuleFsts: String?
+        secondRuleFsts = null
         Log.i(TAG, "Select model type $secondType for the second pass")
 
         val config = OfflineRecognizerConfig(
             featConfig = getFeatureConfig(sampleRate = sampleRateInHz, featureDim = 80),
             modelConfig = getOfflineModelConfig(type = secondType)!!,
         )
 
+        if (secondRuleFsts != null) {
+            config.ruleFsts = secondRuleFsts
+        }
+
         offlineRecognizer = OfflineRecognizer(
             assetManager = application.assets,
             config = config,

android/SherpaOnnxVadAsr/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt

@@ -200,12 +200,17 @@ class MainActivity : AppCompatActivity() {
         // See https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
         // for a list of available models
         val asrModelType = 0
+        val asrRuleFsts: String?
+        asrRuleFsts = null
         Log.i(TAG, "Select model type ${asrModelType} for ASR")
 
         val config = OfflineRecognizerConfig(
             featConfig = getFeatureConfig(sampleRate = sampleRateInHz, featureDim = 80),
             modelConfig = getOfflineModelConfig(type = asrModelType)!!,
         )
+        if (asrRuleFsts != null) {
+            config.ruleFsts = asrRuleFsts;
+        }
 
         offlineRecognizer = OfflineRecognizer(
             assetManager = application.assets,

cmake/kaldi-decoder.cmake

@@ -63,36 +63,24 @@ function(download_kaldi_decoder)
       kaldi-decoder-core
       kaldifst_core
       fst
+      fstfar
     DESTINATION ..)
-    if(SHERPA_ONNX_ENABLE_TTS)
-      install(TARGETS
-        fstfar
-      DESTINATION ..)
-    endif()
   else()
     install(TARGETS
       kaldi-decoder-core
       kaldifst_core
       fst
+      fstfar
     DESTINATION lib)
-    if(SHERPA_ONNX_ENABLE_TTS)
-      install(TARGETS
-        fstfar
-      DESTINATION lib)
-    endif()
   endif()
 
   if(WIN32 AND BUILD_SHARED_LIBS)
     install(TARGETS
       kaldi-decoder-core
       kaldifst_core
       fst
+      fstfar
     DESTINATION bin)
-    if(SHERPA_ONNX_ENABLE_TTS)
-      install(TARGETS
-        fstfar
-      DESTINATION bin)
-    endif()
   endif()
 endfunction()
 

dart-api-examples/non-streaming-asr/pubspec.yaml

@@ -10,7 +10,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.9.30
+  sherpa_onnx: ^1.10.0
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/streaming-asr/bin/zipformer-transducer.dart

@@ -15,6 +15,7 @@ void main(List<String> arguments) async {
     ..addOption('decoder', help: 'Path to decoder model')
     ..addOption('joiner', help: 'Path to joiner model')
     ..addOption('tokens', help: 'Path to tokens.txt')
+    ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '')
     ..addOption('input-wav', help: 'Path to input.wav to transcribe');
 
   final res = parser.parse(arguments);
@@ -31,6 +32,7 @@ void main(List<String> arguments) async {
   final decoder = res['decoder'] as String;
   final joiner = res['joiner'] as String;
   final tokens = res['tokens'] as String;
+  final ruleFsts = res['rule-fsts'] as String;
   final inputWav = res['input-wav'] as String;
 
   final transducer = sherpa_onnx.OnlineTransducerModelConfig(
@@ -45,7 +47,10 @@ void main(List<String> arguments) async {
     debug: true,
     numThreads: 1,
   );
-  final config = sherpa_onnx.OnlineRecognizerConfig(model: modelConfig);
+  final config = sherpa_onnx.OnlineRecognizerConfig(
+    model: modelConfig,
+    ruleFsts: ruleFsts,
+  );
   final recognizer = sherpa_onnx.OnlineRecognizer(config);
 
   final waveData = sherpa_onnx.readWave(inputWav);

dart-api-examples/streaming-asr/pubspec.yaml

@@ -11,7 +11,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.9.30
+  sherpa_onnx: ^1.10.0
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/streaming-asr/run-zipformer-transducer-itn.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -ex
+
+dart pub get
+
+if [ ! -f ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
+  tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
+  rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2
+fi
+
+if [ ! -f ./itn-zh-number.wav ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn-zh-number.wav
+fi
+
+if [ ! -f ./itn_zh_number.fst ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst
+fi
+
+dart run \
+  ./bin/zipformer-transducer.dart \
+  --encoder ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.int8.onnx \
+  --decoder ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx \
+  --joiner ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.int8.onnx \
+  --tokens ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt \
+  --rule-fsts ./itn_zh_number.fst \
+  --input-wav ./itn-zh-number.wav

dart-api-examples/tts/pubspec.yaml

@@ -8,7 +8,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.9.30
+  sherpa_onnx: ^1.10.0
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/vad/pubspec.yaml

@@ -9,7 +9,7 @@ environment:
   sdk: ^3.4.0
 
 dependencies:
-  sherpa_onnx: ^1.9.30
+  sherpa_onnx: ^1.10.0
   path: ^1.9.0
   args: ^2.5.0