Skip to content

Support decoding multiple streams in Java API. #2149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 25, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/run-java-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,8 @@ jobs:
rm -rf sherpa-onnx-fire-red-*

./run-non-streaming-decode-file-whisper.sh

./run-non-streaming-decode-file-whisper-multiple.sh
rm -rf sherpa-onnx-whisper-*

./run-non-streaming-decode-file-nemo.sh
Expand Down
59 changes: 59 additions & 0 deletions java-api-examples/NonStreamingDecodeFileWhisperMultiple.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright 2025 Xiaomi Corporation

// This file shows how to use an offline whisper, i.e., non-streaming whisper,
// to decode files.
import com.k2fsa.sherpa.onnx.*;

public class NonStreamingDecodeFileWhisperMultiple {
public static void main(String[] args) {
// please refer to
// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html
// to download model files
String encoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx";
String decoder = "./sherpa-onnx-whisper-tiny.en/tiny.en-decoder.int8.onnx";
String tokens = "./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt";

String waveFilename0 = "./sherpa-onnx-whisper-tiny.en/test_wavs/0.wav";
String waveFilename1 = "./sherpa-onnx-whisper-tiny.en/test_wavs/1.wav";

WaveReader reader0 = new WaveReader(waveFilename0);
WaveReader reader1 = new WaveReader(waveFilename1);

OfflineWhisperModelConfig whisper =
OfflineWhisperModelConfig.builder().setEncoder(encoder).setDecoder(decoder).build();

OfflineModelConfig modelConfig =
OfflineModelConfig.builder()
.setWhisper(whisper)
.setTokens(tokens)
.setNumThreads(1)
.setDebug(true)
.build();

OfflineRecognizerConfig config =
OfflineRecognizerConfig.builder()
.setOfflineModelConfig(modelConfig)
.setDecodingMethod("greedy_search")
.build();

OfflineRecognizer recognizer = new OfflineRecognizer(config);
OfflineStream stream0 = recognizer.createStream();
stream0.acceptWaveform(reader0.getSamples(), reader0.getSampleRate());

OfflineStream stream1 = recognizer.createStream();
stream1.acceptWaveform(reader1.getSamples(), reader1.getSampleRate());

OfflineStream[] ss = new OfflineStream[] {stream0, stream1};
recognizer.decode(ss);

String text0 = recognizer.getResult(stream0).getText();
String text1 = recognizer.getResult(stream1).getText();

System.out.printf("filename0:%s\nresult0:%s\n\n", waveFilename0, text0);
System.out.printf("filename1:%s\nresult1:%s\n\n", waveFilename1, text1);

stream0.release();
stream1.release();
recognizer.release();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#!/usr/bin/env bash

set -ex

if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then
mkdir -p ../build
pushd ../build
cmake \
-DSHERPA_ONNX_ENABLE_PYTHON=OFF \
-DSHERPA_ONNX_ENABLE_TESTS=OFF \
-DSHERPA_ONNX_ENABLE_CHECK=OFF \
-DBUILD_SHARED_LIBS=ON \
-DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
-DSHERPA_ONNX_ENABLE_JNI=ON \
..

make -j4
ls -lh lib
popd
fi

if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then
pushd ../sherpa-onnx/java-api
make
popd
fi

if [ ! -f ./sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt ]; then
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2

tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2
rm sherpa-onnx-whisper-tiny.en.tar.bz2
fi

java \
-Djava.library.path=$PWD/../build/lib \
-cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
NonStreamingDecodeFileWhisperMultiple.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@ public void decode(OfflineStream s) {
decode(ptr, s.getPtr());
}

public void decode(OfflineStream[] ss) {
long[] streamPtrs = new long[ss.length];
for (int i = 0; i < ss.length; ++i) {
streamPtrs[i] = ss[i].getPtr();
}
decodeStreams(ptr, streamPtrs);
}

public OfflineStream createStream() {
long p = createStream(ptr);
return new OfflineStream(p);
Expand Down Expand Up @@ -55,5 +63,7 @@ public OfflineRecognizerResult getResult(OfflineStream s) {

private native void decode(long ptr, long streamPtr);

private native void decodeStreams(long ptr, long[] streamPtrs);

private native Object[] getResult(long streamPtr);
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@ public void decode(OnlineStream s) {
decode(ptr, s.getPtr());
}

public void decode(OnlineStream[] ss) {
long[] streamPtrs = new long[ss.length];
for (int i = 0; i < ss.length; ++i) {
streamPtrs[i] = ss[i].getPtr();
}
decodeStreams(ptr, streamPtrs);
}

public boolean isReady(OnlineStream s) {
return isReady(ptr, s.getPtr());
}
Expand Down Expand Up @@ -68,6 +76,8 @@ public OnlineRecognizerResult getResult(OnlineStream s) {

private native void decode(long ptr, long streamPtr);

private native void decodeStreams(long ptr, long[] streamPtrs);

private native boolean isEndpoint(long ptr, long streamPtr);

private native boolean isReady(long ptr, long streamPtr);
Expand Down
28 changes: 25 additions & 3 deletions sherpa-onnx/jni/offline-recognizer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -366,21 +366,43 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_createStream(JNIEnv * /*env*/,

SHERPA_ONNX_EXTERN_C
JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_decode(
JNIEnv *env, jobject /*obj*/, jlong ptr, jlong streamPtr) {
JNIEnv *env, jobject /*obj*/, jlong ptr, jlong stream_ptr) {
SafeJNI(env, "OfflineRecognizer_decode", [&] {
if (!ValidatePointer(env, ptr, "OfflineRecognizer_decode",
"OfflineRecognizer pointer is null.") ||
!ValidatePointer(env, streamPtr, "OfflineRecognizer_decode",
!ValidatePointer(env, stream_ptr, "OfflineRecognizer_decode",
"OfflineStream pointer is null.")) {
return;
}

auto recognizer = reinterpret_cast<sherpa_onnx::OfflineRecognizer *>(ptr);
auto stream = reinterpret_cast<sherpa_onnx::OfflineStream *>(streamPtr);
auto stream = reinterpret_cast<sherpa_onnx::OfflineStream *>(stream_ptr);
recognizer->DecodeStream(stream);
});
}

SHERPA_ONNX_EXTERN_C
JNIEXPORT void JNICALL
Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_decodeStreams(
JNIEnv *env, jobject /*obj*/, jlong ptr, jlongArray stream_ptrs) {
SafeJNI(env, "OfflineRecognizer_decode_streams", [&] {
if (!ValidatePointer(env, ptr, "OfflineRecognizer_decode_streams",
"OfflineRecognizer pointer is null.")) {
return;
}

auto recognizer = reinterpret_cast<sherpa_onnx::OfflineRecognizer *>(ptr);

jlong *p = env->GetLongArrayElements(stream_ptrs, nullptr);
jsize n = env->GetArrayLength(stream_ptrs);

auto ss = reinterpret_cast<sherpa_onnx::OfflineStream **>(p);
recognizer->DecodeStreams(ss, n);

env->ReleaseLongArrayElements(stream_ptrs, p, JNI_ABORT);
});
}

SHERPA_ONNX_EXTERN_C
JNIEXPORT jobjectArray JNICALL
Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_getResult(JNIEnv *env,
Expand Down
16 changes: 16 additions & 0 deletions sherpa-onnx/jni/online-recognizer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,22 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OnlineRecognizer_decode(
recognizer->DecodeStream(stream);
}

SHERPA_ONNX_EXTERN_C
JNIEXPORT void JNICALL
Java_com_k2fsa_sherpa_onnx_OnlineRecognizer_decodeStreams(
JNIEnv *env, jobject /*obj*/, jlong ptr, jlongArray stream_ptrs) {
auto recognizer = reinterpret_cast<sherpa_onnx::OnlineRecognizer *>(ptr);

jlong *p = env->GetLongArrayElements(stream_ptrs, nullptr);
jsize n = env->GetArrayLength(stream_ptrs);

auto ss = reinterpret_cast<sherpa_onnx::OnlineStream **>(p);

recognizer->DecodeStreams(ss, n);

env->ReleaseLongArrayElements(stream_ptrs, p, JNI_ABORT);
}

SHERPA_ONNX_EXTERN_C
JNIEXPORT jlong JNICALL
Java_com_k2fsa_sherpa_onnx_OnlineRecognizer_createStream(JNIEnv *env,
Expand Down
Loading