Add Java and Kotlin API for VoxLock (#4107)

This commit is contained in:
Fangjun Kuang authored and GitHub committed 2026-10-26 19:40:49 +08:00
1 parent 42e1e0d077
commit 687a26126c
20 files changed
+922

No files matched your search

+26
View File
@@ -78,3 +78,29 @@ jobs:
with:
name: tts-files-${{ matrix.os }}
path: kotlin-api-examples/test-*.wav
- uses: actions/upload-artifact@v4
with:
name: test-offline-speech-denoiser-voxlock-${{ matrix.os }}
# run.sh keeps the input and output wavs of each test in a folder
# named after the test file
path: kotlin-api-examples/test_offline_speech_denoiser_voxlock
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-offline-speech-denoiser-voxlock-with-voice-sample-${{ matrix.os }}
path: kotlin-api-examples/test_offline_speech_denoiser_voxlock_with_voice_sample
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-online-speech-denoiser-voxlock-${{ matrix.os }}
path: kotlin-api-examples/test_online_speech_denoiser_voxlock
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-online-speech-denoiser-voxlock-with-voice-sample-${{ matrix.os }}
path: kotlin-api-examples/test_online_speech_denoiser_voxlock_with_voice_sample
if-no-files-found: warn
+35
View File
@@ -158,6 +158,41 @@ jobs:
cd ./java-api-examples
./run-version-test.sh
- name: Run speech enhancement (VoxLock)
shell: bash
run: |
cd ./java-api-examples
./run-non-streaming-speech-enhancement-voxlock.sh
./run-non-streaming-speech-enhancement-voxlock-with-voice-sample.sh
./run-streaming-speech-enhancement-voxlock.sh
./run-streaming-speech-enhancement-voxlock-with-voice-sample.sh
- uses: actions/upload-artifact@v4
with:
name: NonStreamingSpeechEnhancementVoxLock-${{ matrix.os }}
# each run-*.sh keeps its input and output wavs in a folder named
# after the example
path: java-api-examples/NonStreamingSpeechEnhancementVoxLock
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: NonStreamingSpeechEnhancementVoxLockWithVoiceSample-${{ matrix.os }}
path: java-api-examples/NonStreamingSpeechEnhancementVoxLockWithVoiceSample
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: StreamingSpeechEnhancementVoxLock-${{ matrix.os }}
path: java-api-examples/StreamingSpeechEnhancementVoxLock
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: StreamingSpeechEnhancementVoxLockWithVoiceSample-${{ matrix.os }}
path: java-api-examples/StreamingSpeechEnhancementVoxLockWithVoiceSample
if-no-files-found: warn
- name: Run java test (Non-Streaming TTS)
shell: bash
run: |
+5
View File
@@ -142,6 +142,11 @@ README-DEV.txt
*.jit
##clion
.idea
## Eclipse and other IDEs
.classpath
.project
.settings/
sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02
dict
*.npz
@@ -0,0 +1,57 @@
// Copyright 2026 Xiaomi Corporation
// This file shows how to use VoxLock speech enhancement models in sherpa-onnx.
//
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example does not provide a voice sample, so VoxLock takes the loudest
// 3 seconds in the first 15 seconds of the input as the speaker to keep. See
// NonStreamingSpeechEnhancementVoxLockWithVoiceSample.java if you have a voice
// sample of the speaker.
//
// Download models and sample test waves from:
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
import com.k2fsa.sherpa.onnx.*;
public class NonStreamingSpeechEnhancementVoxLock {
public static void main(String[] args) {
OfflineSpeechDenoiserModelConfig modelConfig =
OfflineSpeechDenoiserModelConfig.builder()
.setNumThreads(1)
.setDebug(true)
.setProvider("cpu")
.setVoxlock(
OfflineSpeechDenoiserVoxLockModelConfig.builder()
.setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx")
.setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx")
.build())
.build();
OfflineSpeechDenoiserConfig config =
OfflineSpeechDenoiserConfig.builder().setModel(modelConfig).build();
OfflineSpeechDenoiser speechDenoiser = new OfflineSpeechDenoiser(config);
WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav");
long begin = System.currentTimeMillis();
DenoisedAudio denoised = speechDenoiser.run(reader.getSamples(), reader.getSampleRate());
long elapsedMs = System.currentTimeMillis() - begin;
String outFilename = "voxlock-enhanced-16k.wav";
WaveWriter.write(outFilename, denoised.getSamples(), denoised.getSampleRate());
float elapsedSeconds = elapsedMs / 1000.0f;
float duration = reader.getSamples().length / (float) reader.getSampleRate();
System.out.printf("Saved to %s\n", outFilename);
System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds);
System.out.printf("Audio duration in seconds: %.3f\n", duration);
System.out.printf(
"RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration);
speechDenoiser.release();
}
}
@@ -0,0 +1,60 @@
// Copyright 2026 Xiaomi Corporation
// This file shows how to use VoxLock speech enhancement models in sherpa-onnx
// with a voice sample.
//
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example provides a voice sample of the speaker to keep, so VoxLock
// isolates that speaker from the first sample. The voice sample must have at
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
// loudest 3 seconds within its first 30 seconds. See
// NonStreamingSpeechEnhancementVoxLock.java if you don't have a voice sample.
//
// Download models and sample test waves from:
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
import com.k2fsa.sherpa.onnx.*;
public class NonStreamingSpeechEnhancementVoxLockWithVoiceSample {
public static void main(String[] args) {
OfflineSpeechDenoiserModelConfig modelConfig =
OfflineSpeechDenoiserModelConfig.builder()
.setNumThreads(1)
.setDebug(true)
.setProvider("cpu")
.setVoxlock(
OfflineSpeechDenoiserVoxLockModelConfig.builder()
.setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx")
.setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx")
.setVoiceSample("./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav")
.build())
.build();
OfflineSpeechDenoiserConfig config =
OfflineSpeechDenoiserConfig.builder().setModel(modelConfig).build();
OfflineSpeechDenoiser speechDenoiser = new OfflineSpeechDenoiser(config);
WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav");
long begin = System.currentTimeMillis();
DenoisedAudio denoised = speechDenoiser.run(reader.getSamples(), reader.getSampleRate());
long elapsedMs = System.currentTimeMillis() - begin;
String outFilename = "voxlock-enhanced-with-voice-sample-16k.wav";
WaveWriter.write(outFilename, denoised.getSamples(), denoised.getSampleRate());
float elapsedSeconds = elapsedMs / 1000.0f;
float duration = reader.getSamples().length / (float) reader.getSampleRate();
System.out.printf("Saved to %s\n", outFilename);
System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds);
System.out.printf("Audio duration in seconds: %.3f\n", duration);
System.out.printf(
"RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration);
speechDenoiser.release();
}
}
@@ -0,0 +1,85 @@
// Copyright 2026 Xiaomi Corporation
//
// This file shows how to use streaming VoxLock speech enhancement models in
// sherpa-onnx.
//
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example does not provide a voice sample, so VoxLock passes the audio
// through unchanged until it has heard 3 seconds of speech, takes that speech
// as the speaker to keep, and isolates from then on. See
// StreamingSpeechEnhancementVoxLockWithVoiceSample.java if you have a voice
// sample of the speaker.
//
// Download models and sample test waves from:
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
import com.k2fsa.sherpa.onnx.*;
public class StreamingSpeechEnhancementVoxLock {
private static void appendSamples(java.util.ArrayList<Float> dst, float[] src) {
for (float v : src) {
dst.add(v);
}
}
private static float[] toFloatArray(java.util.ArrayList<Float> src) {
float[] ans = new float[src.size()];
for (int i = 0; i != src.size(); ++i) {
ans[i] = src.get(i);
}
return ans;
}
public static void main(String[] args) {
OfflineSpeechDenoiserModelConfig modelConfig =
OfflineSpeechDenoiserModelConfig.builder()
.setNumThreads(1)
.setDebug(true)
.setProvider("cpu")
.setVoxlock(
OfflineSpeechDenoiserVoxLockModelConfig.builder()
.setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx")
.setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx")
.build())
.build();
OnlineSpeechDenoiserConfig config =
OnlineSpeechDenoiserConfig.builder().setModel(modelConfig).build();
OnlineSpeechDenoiser speechDenoiser = new OnlineSpeechDenoiser(config);
WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav");
int frameShift = speechDenoiser.getFrameShiftInSamples();
java.util.ArrayList<Float> output = new java.util.ArrayList<>();
float[] samples = reader.getSamples();
long begin = System.currentTimeMillis();
for (int start = 0; start < samples.length; start += frameShift) {
int end = Math.min(start + frameShift, samples.length);
float[] chunk = java.util.Arrays.copyOfRange(samples, start, end);
DenoisedAudio denoised = speechDenoiser.run(chunk, reader.getSampleRate());
appendSamples(output, denoised.getSamples());
}
DenoisedAudio denoised = speechDenoiser.flush();
appendSamples(output, denoised.getSamples());
long elapsedMs = System.currentTimeMillis() - begin;
String outFilename = "voxlock-enhanced-online-16k.wav";
WaveWriter.write(outFilename, toFloatArray(output), speechDenoiser.getSampleRate());
float elapsedSeconds = elapsedMs / 1000.0f;
float duration = samples.length / (float) reader.getSampleRate();
System.out.printf("Saved to %s\n", outFilename);
System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds);
System.out.printf("Audio duration in seconds: %.3f\n", duration);
System.out.printf(
"RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration);
speechDenoiser.release();
}
}
@@ -0,0 +1,86 @@
// Copyright 2026 Xiaomi Corporation
//
// This file shows how to use streaming VoxLock speech enhancement models in
// sherpa-onnx with a voice sample.
//
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example provides a voice sample of the speaker to keep, so VoxLock
// isolates that speaker from the first sample. The voice sample must have at
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
// loudest 3 seconds within its first 30 seconds. See
// StreamingSpeechEnhancementVoxLock.java if you don't have a voice sample.
//
// Download models and sample test waves from:
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
import com.k2fsa.sherpa.onnx.*;
public class StreamingSpeechEnhancementVoxLockWithVoiceSample {
private static void appendSamples(java.util.ArrayList<Float> dst, float[] src) {
for (float v : src) {
dst.add(v);
}
}
private static float[] toFloatArray(java.util.ArrayList<Float> src) {
float[] ans = new float[src.size()];
for (int i = 0; i != src.size(); ++i) {
ans[i] = src.get(i);
}
return ans;
}
public static void main(String[] args) {
OfflineSpeechDenoiserModelConfig modelConfig =
OfflineSpeechDenoiserModelConfig.builder()
.setNumThreads(1)
.setDebug(true)
.setProvider("cpu")
.setVoxlock(
OfflineSpeechDenoiserVoxLockModelConfig.builder()
.setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx")
.setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx")
.setVoiceSample("./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav")
.build())
.build();
OnlineSpeechDenoiserConfig config =
OnlineSpeechDenoiserConfig.builder().setModel(modelConfig).build();
OnlineSpeechDenoiser speechDenoiser = new OnlineSpeechDenoiser(config);
WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav");
int frameShift = speechDenoiser.getFrameShiftInSamples();
java.util.ArrayList<Float> output = new java.util.ArrayList<>();
float[] samples = reader.getSamples();
long begin = System.currentTimeMillis();
for (int start = 0; start < samples.length; start += frameShift) {
int end = Math.min(start + frameShift, samples.length);
float[] chunk = java.util.Arrays.copyOfRange(samples, start, end);
DenoisedAudio denoised = speechDenoiser.run(chunk, reader.getSampleRate());
appendSamples(output, denoised.getSamples());
}
DenoisedAudio denoised = speechDenoiser.flush();
appendSamples(output, denoised.getSamples());
long elapsedMs = System.currentTimeMillis() - begin;
String outFilename = "voxlock-enhanced-online-with-voice-sample-16k.wav";
WaveWriter.write(outFilename, toFloatArray(output), speechDenoiser.getSampleRate());
float elapsedSeconds = elapsedMs / 1000.0f;
float duration = samples.length / (float) reader.getSampleRate();
System.out.printf("Saved to %s\n", outFilename);
System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds);
System.out.printf("Audio duration in seconds: %.3f\n", duration);
System.out.printf(
"RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration);
speechDenoiser.release();
}
}
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -ex
source ./setup.sh
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
rm sherpa-onnx-voxlock-int8.tar.bz2
fi
java \
-Dsherpa_onnx.native.path=$PWD/../build/lib \
-cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \
NonStreamingSpeechEnhancementVoxLockWithVoiceSample.java
mkdir -p NonStreamingSpeechEnhancementVoxLockWithVoiceSample
mv voxlock-enhanced-with-voice-sample-16k.wav NonStreamingSpeechEnhancementVoxLockWithVoiceSample/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav NonStreamingSpeechEnhancementVoxLockWithVoiceSample/
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -ex
source ./setup.sh
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
rm sherpa-onnx-voxlock-int8.tar.bz2
fi
java \
-Dsherpa_onnx.native.path=$PWD/../build/lib \
-cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \
NonStreamingSpeechEnhancementVoxLock.java
mkdir -p NonStreamingSpeechEnhancementVoxLock
mv voxlock-enhanced-16k.wav NonStreamingSpeechEnhancementVoxLock/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav NonStreamingSpeechEnhancementVoxLock/
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -ex
source ./setup.sh
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
rm sherpa-onnx-voxlock-int8.tar.bz2
fi
java \
-Dsherpa_onnx.native.path=$PWD/../build/lib \
-cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \
StreamingSpeechEnhancementVoxLockWithVoiceSample.java
mkdir -p StreamingSpeechEnhancementVoxLockWithVoiceSample
mv voxlock-enhanced-online-with-voice-sample-16k.wav StreamingSpeechEnhancementVoxLockWithVoiceSample/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav StreamingSpeechEnhancementVoxLockWithVoiceSample/
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -ex
source ./setup.sh
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
rm sherpa-onnx-voxlock-int8.tar.bz2
fi
java \
-Dsherpa_onnx.native.path=$PWD/../build/lib \
-cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \
StreamingSpeechEnhancementVoxLock.java
mkdir -p StreamingSpeechEnhancementVoxLock
mv voxlock-enhanced-online-16k.wav StreamingSpeechEnhancementVoxLock/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav StreamingSpeechEnhancementVoxLock/
+106
View File
@@ -930,7 +930,113 @@ function testOfflineZipformerCtcHlg() {
java -Djava.library.path=../build/lib -jar $out_filename
}
function testOfflineSpeechDenoiserVoxLock() {
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
rm sherpa-onnx-voxlock-int8.tar.bz2
fi
out_filename=test_offline_speech_denoiser_voxlock.jar
kotlinc-jvm -include-runtime -d $out_filename \
test_offline_speech_denoiser_voxlock.kt \
OfflineSpeechDenoiser.kt \
DenoisedAudio.kt \
WaveReader.kt \
faked-asset-manager.kt \
faked-log.kt
ls -lh $out_filename
java -Djava.library.path=../build/lib -jar $out_filename
mkdir -p test_offline_speech_denoiser_voxlock
mv voxlock-enhanced-16k.wav test_offline_speech_denoiser_voxlock/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test_offline_speech_denoiser_voxlock/
}
function testOfflineSpeechDenoiserVoxLockWithVoiceSample() {
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
rm sherpa-onnx-voxlock-int8.tar.bz2
fi
out_filename=test_offline_speech_denoiser_voxlock_with_voice_sample.jar
kotlinc-jvm -include-runtime -d $out_filename \
test_offline_speech_denoiser_voxlock_with_voice_sample.kt \
OfflineSpeechDenoiser.kt \
DenoisedAudio.kt \
WaveReader.kt \
faked-asset-manager.kt \
faked-log.kt
ls -lh $out_filename
java -Djava.library.path=../build/lib -jar $out_filename
mkdir -p test_offline_speech_denoiser_voxlock_with_voice_sample
mv voxlock-enhanced-with-voice-sample-16k.wav test_offline_speech_denoiser_voxlock_with_voice_sample/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test_offline_speech_denoiser_voxlock_with_voice_sample/
}
function testOnlineSpeechDenoiserVoxLock() {
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
rm sherpa-onnx-voxlock-int8.tar.bz2
fi
out_filename=test_online_speech_denoiser_voxlock.jar
kotlinc-jvm -include-runtime -d $out_filename \
test_online_speech_denoiser_voxlock.kt \
OnlineSpeechDenoiser.kt \
OfflineSpeechDenoiser.kt \
DenoisedAudio.kt \
WaveReader.kt \
faked-asset-manager.kt \
faked-log.kt
ls -lh $out_filename
java -Djava.library.path=../build/lib -jar $out_filename
mkdir -p test_online_speech_denoiser_voxlock
mv voxlock-enhanced-online-16k.wav test_online_speech_denoiser_voxlock/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test_online_speech_denoiser_voxlock/
}
function testOnlineSpeechDenoiserVoxLockWithVoiceSample() {
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
rm sherpa-onnx-voxlock-int8.tar.bz2
fi
out_filename=test_online_speech_denoiser_voxlock_with_voice_sample.jar
kotlinc-jvm -include-runtime -d $out_filename \
test_online_speech_denoiser_voxlock_with_voice_sample.kt \
OnlineSpeechDenoiser.kt \
OfflineSpeechDenoiser.kt \
DenoisedAudio.kt \
WaveReader.kt \
faked-asset-manager.kt \
faked-log.kt
ls -lh $out_filename
java -Djava.library.path=../build/lib -jar $out_filename
mkdir -p test_online_speech_denoiser_voxlock_with_voice_sample
mv voxlock-enhanced-online-with-voice-sample-16k.wav test_online_speech_denoiser_voxlock_with_voice_sample/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test_online_speech_denoiser_voxlock_with_voice_sample/
}
testVersion
testOfflineSpeechDenoiserVoxLock
testOfflineSpeechDenoiserVoxLockWithVoiceSample
testOnlineSpeechDenoiserVoxLock
testOnlineSpeechDenoiserVoxLockWithVoiceSample
testPocketTts
testZipVoiceTtsLexicon
testSupertonicTts
@@ -0,0 +1,60 @@
package com.k2fsa.sherpa.onnx
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example does not provide a voice sample, so VoxLock takes the loudest
// 3 seconds in the first 15 seconds of the input as the speaker to keep. See
// test_offline_speech_denoiser_voxlock_with_voice_sample.kt if you have a
// voice sample of the speaker.
//
// Please download test files in this script from
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
fun main() {
test()
}
fun test() {
val denoiser = createOfflineSpeechDenoiser()
val waveFilename = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"
val waveData = WaveReader.readWaveFromFile(filename = waveFilename)
val begin = System.currentTimeMillis()
val denoised = denoiser.run(waveData.samples, waveData.sampleRate)
val elapsedMs = System.currentTimeMillis() - begin
denoised.save(filename = "./voxlock-enhanced-16k.wav")
val elapsedSeconds = elapsedMs / 1000.0f
val duration = waveData.samples.size / waveData.sampleRate.toFloat()
println("Saved to ./voxlock-enhanced-16k.wav")
println("Elapsed seconds: " + "%.3f".format(elapsedSeconds))
println("Audio duration in seconds: " + "%.3f".format(duration))
println(
"RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " +
"%.3f".format(elapsedSeconds / duration)
)
denoiser.release()
}
fun createOfflineSpeechDenoiser(): OfflineSpeechDenoiser {
val config = OfflineSpeechDenoiserConfig(
model = OfflineSpeechDenoiserModelConfig(
voxlock = OfflineSpeechDenoiserVoxLockModelConfig(
enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx",
chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
),
provider = "cpu",
numThreads = 1,
),
)
println(config)
return OfflineSpeechDenoiser(config = config)
}
@@ -0,0 +1,62 @@
package com.k2fsa.sherpa.onnx
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example provides a voice sample of the speaker to keep, so VoxLock
// isolates that speaker from the first sample. The voice sample must have at
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
// loudest 3 seconds within its first 30 seconds. See
// test_offline_speech_denoiser_voxlock.kt if you don't have a voice sample.
//
// Please download test files in this script from
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
fun main() {
test()
}
fun test() {
val denoiser = createOfflineSpeechDenoiser()
val waveFilename = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"
val waveData = WaveReader.readWaveFromFile(filename = waveFilename)
val begin = System.currentTimeMillis()
val denoised = denoiser.run(waveData.samples, waveData.sampleRate)
val elapsedMs = System.currentTimeMillis() - begin
denoised.save(filename = "./voxlock-enhanced-with-voice-sample-16k.wav")
val elapsedSeconds = elapsedMs / 1000.0f
val duration = waveData.samples.size / waveData.sampleRate.toFloat()
println("Saved to ./voxlock-enhanced-with-voice-sample-16k.wav")
println("Elapsed seconds: " + "%.3f".format(elapsedSeconds))
println("Audio duration in seconds: " + "%.3f".format(duration))
println(
"RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " +
"%.3f".format(elapsedSeconds / duration)
)
denoiser.release()
}
fun createOfflineSpeechDenoiser(): OfflineSpeechDenoiser {
val config = OfflineSpeechDenoiserConfig(
model = OfflineSpeechDenoiserModelConfig(
voxlock = OfflineSpeechDenoiserVoxLockModelConfig(
enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx",
chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
voiceSample = "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav",
),
provider = "cpu",
numThreads = 1,
),
)
println(config)
return OfflineSpeechDenoiser(config = config)
}
@@ -0,0 +1,70 @@
package com.k2fsa.sherpa.onnx
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example does not provide a voice sample, so VoxLock passes the audio
// through unchanged until it has heard 3 seconds of speech, takes that speech
// as the speaker to keep, and isolates from then on. See
// test_online_speech_denoiser_voxlock_with_voice_sample.kt if you have a voice
// sample of the speaker.
//
// Please download test files in this script from
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
fun main() {
test()
}
fun test() {
val denoiser = createOnlineSpeechDenoiser()
val waveData = WaveReader.readWaveFromFile("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav")
val output = mutableListOf<Float>()
val frameShift = denoiser.frameShiftInSamples
val begin = System.currentTimeMillis()
var start = 0
while (start < waveData.samples.size) {
val end = minOf(start + frameShift, waveData.samples.size)
val chunk = waveData.samples.copyOfRange(start, end)
val denoised = denoiser.run(chunk, waveData.sampleRate)
output.addAll(denoised.samples.asList())
start = end
}
output.addAll(denoiser.flush().samples.asList())
val elapsedMs = System.currentTimeMillis() - begin
DenoisedAudio(output.toFloatArray(), denoiser.sampleRate).save(
filename = "./voxlock-enhanced-online-16k.wav"
)
val elapsedSeconds = elapsedMs / 1000.0f
val duration = waveData.samples.size / waveData.sampleRate.toFloat()
println("Saved to ./voxlock-enhanced-online-16k.wav")
println("Elapsed seconds: " + "%.3f".format(elapsedSeconds))
println("Audio duration in seconds: " + "%.3f".format(duration))
println(
"RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " +
"%.3f".format(elapsedSeconds / duration)
)
denoiser.release()
}
fun createOnlineSpeechDenoiser(): OnlineSpeechDenoiser {
val config = OnlineSpeechDenoiserConfig(
model = OfflineSpeechDenoiserModelConfig(
voxlock = OfflineSpeechDenoiserVoxLockModelConfig(
enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx",
chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
),
provider = "cpu",
numThreads = 1,
),
)
return OnlineSpeechDenoiser(config = config)
}
@@ -0,0 +1,71 @@
package com.k2fsa.sherpa.onnx
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example provides a voice sample of the speaker to keep, so VoxLock
// isolates that speaker from the first sample. The voice sample must have at
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
// loudest 3 seconds within its first 30 seconds. See
// test_online_speech_denoiser_voxlock.kt if you don't have a voice sample.
//
// Please download test files in this script from
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
fun main() {
test()
}
fun test() {
val denoiser = createOnlineSpeechDenoiser()
val waveData = WaveReader.readWaveFromFile("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav")
val output = mutableListOf<Float>()
val frameShift = denoiser.frameShiftInSamples
val begin = System.currentTimeMillis()
var start = 0
while (start < waveData.samples.size) {
val end = minOf(start + frameShift, waveData.samples.size)
val chunk = waveData.samples.copyOfRange(start, end)
val denoised = denoiser.run(chunk, waveData.sampleRate)
output.addAll(denoised.samples.asList())
start = end
}
output.addAll(denoiser.flush().samples.asList())
val elapsedMs = System.currentTimeMillis() - begin
DenoisedAudio(output.toFloatArray(), denoiser.sampleRate).save(
filename = "./voxlock-enhanced-online-with-voice-sample-16k.wav"
)
val elapsedSeconds = elapsedMs / 1000.0f
val duration = waveData.samples.size / waveData.sampleRate.toFloat()
println("Saved to ./voxlock-enhanced-online-with-voice-sample-16k.wav")
println("Elapsed seconds: " + "%.3f".format(elapsedSeconds))
println("Audio duration in seconds: " + "%.3f".format(duration))
println(
"RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " +
"%.3f".format(elapsedSeconds / duration)
)
denoiser.release()
}
fun createOnlineSpeechDenoiser(): OnlineSpeechDenoiser {
val config = OnlineSpeechDenoiserConfig(
model = OfflineSpeechDenoiserModelConfig(
voxlock = OfflineSpeechDenoiserVoxLockModelConfig(
enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx",
chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
voiceSample = "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav",
),
provider = "cpu",
numThreads = 1,
),
)
return OnlineSpeechDenoiser(config = config)
}
@@ -4,6 +4,7 @@ package com.k2fsa.sherpa.onnx;
public class OfflineSpeechDenoiserModelConfig {
private final OfflineSpeechDenoiserGtcrnModelConfig gtcrn;
private final OfflineSpeechDenoiserDpdfNetModelConfig dpdfnet;
private final OfflineSpeechDenoiserVoxLockModelConfig voxlock;
private final int numThreads;
private final boolean debug;
private final String provider;
@@ -11,6 +12,7 @@ public class OfflineSpeechDenoiserModelConfig {
private OfflineSpeechDenoiserModelConfig(Builder builder) {
this.gtcrn = builder.gtcrn;
this.dpdfnet = builder.dpdfnet;
this.voxlock = builder.voxlock;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
@@ -23,6 +25,7 @@ public class OfflineSpeechDenoiserModelConfig {
public static class Builder {
private OfflineSpeechDenoiserGtcrnModelConfig gtcrn = OfflineSpeechDenoiserGtcrnModelConfig.builder().build();
private OfflineSpeechDenoiserDpdfNetModelConfig dpdfnet = OfflineSpeechDenoiserDpdfNetModelConfig.builder().build();
private OfflineSpeechDenoiserVoxLockModelConfig voxlock = OfflineSpeechDenoiserVoxLockModelConfig.builder().build();
private int numThreads = 1;
private boolean debug = true;
private String provider = "cpu";
@@ -41,6 +44,11 @@ public class OfflineSpeechDenoiserModelConfig {
return this;
}
public Builder setVoxlock(OfflineSpeechDenoiserVoxLockModelConfig voxlock) {
this.voxlock = voxlock;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
@@ -0,0 +1,80 @@
// Copyright 2026 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
/**
* VoxLock target-speaker voice isolation model config.
*
* <p>VoxLock keeps the voice of one speaker (the caller) and removes the other voices. The model is
* from https://github.com/arbazsiddiqui/voxlock
*/
public class OfflineSpeechDenoiserVoxLockModelConfig {
private final String enrollModel;
private final String chunkModel;
private final String voiceSample;
private OfflineSpeechDenoiserVoxLockModelConfig(Builder builder) {
this.enrollModel = builder.enrollModel;
this.chunkModel = builder.chunkModel;
this.voiceSample = builder.voiceSample;
}
public static Builder builder() {
return new Builder();
}
public String getEnrollModel() {
return enrollModel;
}
public String getChunkModel() {
return chunkModel;
}
public String getVoiceSample() {
return voiceSample;
}
public static class Builder {
private String enrollModel = "";
private String chunkModel = "";
private String voiceSample = "";
public OfflineSpeechDenoiserVoxLockModelConfig build() {
return new OfflineSpeechDenoiserVoxLockModelConfig(this);
}
/**
* Path to the VoxLock enroll model, e.g., enroll.onnx. It runs once on a voice sample of the
* speaker to keep.
*/
public Builder setEnrollModel(String enrollModel) {
this.enrollModel = enrollModel;
return this;
}
/**
* Path to the VoxLock chunk model, e.g., chunk.onnx or chunk_int8.onnx. It runs on each chunk
* of the audio.
*/
public Builder setChunkModel(String chunkModel) {
this.chunkModel = chunkModel;
return this;
}
/**
* Optional. Path to a wave file with at least 3 seconds of the caller's voice. VoxLock keeps
* this voice and removes the others. From a longer file, it uses the loudest 3 seconds within
* the first 30 seconds. A file that is shorter or cannot be read is ignored, as if it were
* empty.
*
* <p>If empty or ignored, the voice sample is taken from the input audio: the offline denoiser
* uses the loudest 3 seconds in the first 15 seconds of the input, and the online denoiser
* passes the audio through unchanged until 3 seconds of speech have been heard, then uses that
* speech.
*/
public Builder setVoiceSample(String voiceSample) {
this.voiceSample = voiceSample;
return this;
}
}
}
+13
View File
@@ -33,6 +33,19 @@ OfflineSpeechDenoiserModelConfig GetOfflineSpeechDenoiserModelConfig(
SHERPA_ONNX_JNI_READ_FLOAT(ans.dpdfnet.attenuation_limit_db,
attenuationLimitDb, dpdfnet_cls, dpdfnet);
fid = env->GetFieldID(
model_config_cls, "voxlock",
"Lcom/k2fsa/sherpa/onnx/OfflineSpeechDenoiserVoxLockModelConfig;");
jobject voxlock = env->GetObjectField(model, fid);
jclass voxlock_cls = env->GetObjectClass(voxlock);
SHERPA_ONNX_JNI_READ_STRING(ans.voxlock.enroll_model, enrollModel, voxlock_cls,
voxlock);
SHERPA_ONNX_JNI_READ_STRING(ans.voxlock.chunk_model, chunkModel, voxlock_cls,
voxlock);
SHERPA_ONNX_JNI_READ_STRING(ans.voxlock.voice_sample, voiceSample, voxlock_cls,
voxlock);
SHERPA_ONNX_JNI_READ_INT(ans.num_threads, numThreads, model_config_cls,
model);
SHERPA_ONNX_JNI_READ_BOOL(ans.debug, debug, model_config_cls, model);
@@ -11,9 +11,27 @@ data class OfflineSpeechDenoiserDpdfNetModelConfig(
var attenuationLimitDb: Float = 0.0f,
)
// VoxLock keeps the voice of one speaker (the caller) and removes the other
// voices. The model is from https://github.com/arbazsiddiqui/voxlock
data class OfflineSpeechDenoiserVoxLockModelConfig(
var enrollModel: String = "",
var chunkModel: String = "",
// Optional. A wave file with at least 3 seconds of the caller's voice.
// From a longer file, the loudest 3 seconds within its first 30 seconds
// are used. A file that is shorter or cannot be read is ignored, as if it
// were empty. If empty or ignored, the voice sample is taken from the
// input audio: the offline denoiser uses the loudest 3 seconds in the
// first 15 seconds of the input, and the online denoiser passes audio
// through unchanged until 3 seconds of speech have been heard, then uses
// that speech.
var voiceSample: String = "",
)
data class OfflineSpeechDenoiserModelConfig(
var gtcrn: OfflineSpeechDenoiserGtcrnModelConfig = OfflineSpeechDenoiserGtcrnModelConfig(),
var dpdfnet: OfflineSpeechDenoiserDpdfNetModelConfig = OfflineSpeechDenoiserDpdfNetModelConfig(),
var voxlock: OfflineSpeechDenoiserVoxLockModelConfig =
OfflineSpeechDenoiserVoxLockModelConfig(),
var numThreads: Int = 1,
var debug: Boolean = false,
var provider: String = "cpu",