diff --git a/.github/workflows/jni.yaml b/.github/workflows/jni.yaml index 142f8aed3..d3822e177 100644 --- a/.github/workflows/jni.yaml +++ b/.github/workflows/jni.yaml @@ -78,3 +78,29 @@ jobs: with: name: tts-files-${{ matrix.os }} path: kotlin-api-examples/test-*.wav + + - uses: actions/upload-artifact@v4 + with: + name: test-offline-speech-denoiser-voxlock-${{ matrix.os }} + # run.sh keeps the input and output wavs of each test in a folder + # named after the test file + path: kotlin-api-examples/test_offline_speech_denoiser_voxlock + if-no-files-found: warn + + - uses: actions/upload-artifact@v4 + with: + name: test-offline-speech-denoiser-voxlock-with-voice-sample-${{ matrix.os }} + path: kotlin-api-examples/test_offline_speech_denoiser_voxlock_with_voice_sample + if-no-files-found: warn + + - uses: actions/upload-artifact@v4 + with: + name: test-online-speech-denoiser-voxlock-${{ matrix.os }} + path: kotlin-api-examples/test_online_speech_denoiser_voxlock + if-no-files-found: warn + + - uses: actions/upload-artifact@v4 + with: + name: test-online-speech-denoiser-voxlock-with-voice-sample-${{ matrix.os }} + path: kotlin-api-examples/test_online_speech_denoiser_voxlock_with_voice_sample + if-no-files-found: warn diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml index 417a31195..d10591839 100644 --- a/.github/workflows/run-java-test.yaml +++ b/.github/workflows/run-java-test.yaml @@ -158,6 +158,41 @@ jobs: cd ./java-api-examples ./run-version-test.sh + - name: Run speech enhancement (VoxLock) + shell: bash + run: | + cd ./java-api-examples + ./run-non-streaming-speech-enhancement-voxlock.sh + ./run-non-streaming-speech-enhancement-voxlock-with-voice-sample.sh + ./run-streaming-speech-enhancement-voxlock.sh + ./run-streaming-speech-enhancement-voxlock-with-voice-sample.sh + + - uses: actions/upload-artifact@v4 + with: + name: NonStreamingSpeechEnhancementVoxLock-${{ matrix.os }} + # each run-*.sh keeps its input and output wavs in a folder named + # after the example + path: java-api-examples/NonStreamingSpeechEnhancementVoxLock + if-no-files-found: warn + + - uses: actions/upload-artifact@v4 + with: + name: NonStreamingSpeechEnhancementVoxLockWithVoiceSample-${{ matrix.os }} + path: java-api-examples/NonStreamingSpeechEnhancementVoxLockWithVoiceSample + if-no-files-found: warn + + - uses: actions/upload-artifact@v4 + with: + name: StreamingSpeechEnhancementVoxLock-${{ matrix.os }} + path: java-api-examples/StreamingSpeechEnhancementVoxLock + if-no-files-found: warn + + - uses: actions/upload-artifact@v4 + with: + name: StreamingSpeechEnhancementVoxLockWithVoiceSample-${{ matrix.os }} + path: java-api-examples/StreamingSpeechEnhancementVoxLockWithVoiceSample + if-no-files-found: warn + - name: Run java test (Non-Streaming TTS) shell: bash run: | diff --git a/.gitignore b/.gitignore index 4a1cbebb2..07a2c87da 100755 --- a/.gitignore +++ b/.gitignore @@ -142,6 +142,11 @@ README-DEV.txt *.jit ##clion .idea + +## Eclipse and other IDEs +.classpath +.project +.settings/ sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02 dict *.npz diff --git a/java-api-examples/NonStreamingSpeechEnhancementVoxLock.java b/java-api-examples/NonStreamingSpeechEnhancementVoxLock.java new file mode 100644 index 000000000..c0543b71c --- /dev/null +++ b/java-api-examples/NonStreamingSpeechEnhancementVoxLock.java @@ -0,0 +1,57 @@ +// Copyright 2026 Xiaomi Corporation + +// This file shows how to use VoxLock speech enhancement models in sherpa-onnx. +// +// VoxLock is a target-speaker voice isolation model. It keeps the voice of one +// speaker, e.g., the caller in a phone call, and removes the other voices. +// +// The model is from https://github.com/arbazsiddiqui/voxlock +// +// This example does not provide a voice sample, so VoxLock takes the loudest +// 3 seconds in the first 15 seconds of the input as the speaker to keep. See +// NonStreamingSpeechEnhancementVoxLockWithVoiceSample.java if you have a voice +// sample of the speaker. +// +// Download models and sample test waves from: +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models + +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingSpeechEnhancementVoxLock { + public static void main(String[] args) { + OfflineSpeechDenoiserModelConfig modelConfig = + OfflineSpeechDenoiserModelConfig.builder() + .setNumThreads(1) + .setDebug(true) + .setProvider("cpu") + .setVoxlock( + OfflineSpeechDenoiserVoxLockModelConfig.builder() + .setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx") + .setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx") + .build()) + .build(); + OfflineSpeechDenoiserConfig config = + OfflineSpeechDenoiserConfig.builder().setModel(modelConfig).build(); + + OfflineSpeechDenoiser speechDenoiser = new OfflineSpeechDenoiser(config); + + WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"); + + long begin = System.currentTimeMillis(); + DenoisedAudio denoised = speechDenoiser.run(reader.getSamples(), reader.getSampleRate()); + long elapsedMs = System.currentTimeMillis() - begin; + + String outFilename = "voxlock-enhanced-16k.wav"; + WaveWriter.write(outFilename, denoised.getSamples(), denoised.getSampleRate()); + + float elapsedSeconds = elapsedMs / 1000.0f; + float duration = reader.getSamples().length / (float) reader.getSampleRate(); + System.out.printf("Saved to %s\n", outFilename); + System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds); + System.out.printf("Audio duration in seconds: %.3f\n", duration); + System.out.printf( + "RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration); + + speechDenoiser.release(); + } +} diff --git a/java-api-examples/NonStreamingSpeechEnhancementVoxLockWithVoiceSample.java b/java-api-examples/NonStreamingSpeechEnhancementVoxLockWithVoiceSample.java new file mode 100644 index 000000000..9fd658644 --- /dev/null +++ b/java-api-examples/NonStreamingSpeechEnhancementVoxLockWithVoiceSample.java @@ -0,0 +1,60 @@ +// Copyright 2026 Xiaomi Corporation + +// This file shows how to use VoxLock speech enhancement models in sherpa-onnx +// with a voice sample. +// +// VoxLock is a target-speaker voice isolation model. It keeps the voice of one +// speaker, e.g., the caller in a phone call, and removes the other voices. +// +// The model is from https://github.com/arbazsiddiqui/voxlock +// +// This example provides a voice sample of the speaker to keep, so VoxLock +// isolates that speaker from the first sample. The voice sample must have at +// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the +// loudest 3 seconds within its first 30 seconds. See +// NonStreamingSpeechEnhancementVoxLock.java if you don't have a voice sample. +// +// Download models and sample test waves from: +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models + +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingSpeechEnhancementVoxLockWithVoiceSample { + public static void main(String[] args) { + OfflineSpeechDenoiserModelConfig modelConfig = + OfflineSpeechDenoiserModelConfig.builder() + .setNumThreads(1) + .setDebug(true) + .setProvider("cpu") + .setVoxlock( + OfflineSpeechDenoiserVoxLockModelConfig.builder() + .setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx") + .setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx") + .setVoiceSample("./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav") + .build()) + .build(); + OfflineSpeechDenoiserConfig config = + OfflineSpeechDenoiserConfig.builder().setModel(modelConfig).build(); + + OfflineSpeechDenoiser speechDenoiser = new OfflineSpeechDenoiser(config); + + WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"); + + long begin = System.currentTimeMillis(); + DenoisedAudio denoised = speechDenoiser.run(reader.getSamples(), reader.getSampleRate()); + long elapsedMs = System.currentTimeMillis() - begin; + + String outFilename = "voxlock-enhanced-with-voice-sample-16k.wav"; + WaveWriter.write(outFilename, denoised.getSamples(), denoised.getSampleRate()); + + float elapsedSeconds = elapsedMs / 1000.0f; + float duration = reader.getSamples().length / (float) reader.getSampleRate(); + System.out.printf("Saved to %s\n", outFilename); + System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds); + System.out.printf("Audio duration in seconds: %.3f\n", duration); + System.out.printf( + "RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration); + + speechDenoiser.release(); + } +} diff --git a/java-api-examples/StreamingSpeechEnhancementVoxLock.java b/java-api-examples/StreamingSpeechEnhancementVoxLock.java new file mode 100644 index 000000000..dcdb16a83 --- /dev/null +++ b/java-api-examples/StreamingSpeechEnhancementVoxLock.java @@ -0,0 +1,85 @@ +// Copyright 2026 Xiaomi Corporation +// +// This file shows how to use streaming VoxLock speech enhancement models in +// sherpa-onnx. +// +// VoxLock is a target-speaker voice isolation model. It keeps the voice of one +// speaker, e.g., the caller in a phone call, and removes the other voices. +// +// The model is from https://github.com/arbazsiddiqui/voxlock +// +// This example does not provide a voice sample, so VoxLock passes the audio +// through unchanged until it has heard 3 seconds of speech, takes that speech +// as the speaker to keep, and isolates from then on. See +// StreamingSpeechEnhancementVoxLockWithVoiceSample.java if you have a voice +// sample of the speaker. +// +// Download models and sample test waves from: +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models + +import com.k2fsa.sherpa.onnx.*; + +public class StreamingSpeechEnhancementVoxLock { + private static void appendSamples(java.util.ArrayList dst, float[] src) { + for (float v : src) { + dst.add(v); + } + } + + private static float[] toFloatArray(java.util.ArrayList src) { + float[] ans = new float[src.size()]; + for (int i = 0; i != src.size(); ++i) { + ans[i] = src.get(i); + } + return ans; + } + + public static void main(String[] args) { + OfflineSpeechDenoiserModelConfig modelConfig = + OfflineSpeechDenoiserModelConfig.builder() + .setNumThreads(1) + .setDebug(true) + .setProvider("cpu") + .setVoxlock( + OfflineSpeechDenoiserVoxLockModelConfig.builder() + .setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx") + .setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx") + .build()) + .build(); + + OnlineSpeechDenoiserConfig config = + OnlineSpeechDenoiserConfig.builder().setModel(modelConfig).build(); + + OnlineSpeechDenoiser speechDenoiser = new OnlineSpeechDenoiser(config); + + WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"); + int frameShift = speechDenoiser.getFrameShiftInSamples(); + java.util.ArrayList output = new java.util.ArrayList<>(); + + float[] samples = reader.getSamples(); + long begin = System.currentTimeMillis(); + for (int start = 0; start < samples.length; start += frameShift) { + int end = Math.min(start + frameShift, samples.length); + float[] chunk = java.util.Arrays.copyOfRange(samples, start, end); + DenoisedAudio denoised = speechDenoiser.run(chunk, reader.getSampleRate()); + appendSamples(output, denoised.getSamples()); + } + + DenoisedAudio denoised = speechDenoiser.flush(); + appendSamples(output, denoised.getSamples()); + long elapsedMs = System.currentTimeMillis() - begin; + + String outFilename = "voxlock-enhanced-online-16k.wav"; + WaveWriter.write(outFilename, toFloatArray(output), speechDenoiser.getSampleRate()); + + float elapsedSeconds = elapsedMs / 1000.0f; + float duration = samples.length / (float) reader.getSampleRate(); + System.out.printf("Saved to %s\n", outFilename); + System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds); + System.out.printf("Audio duration in seconds: %.3f\n", duration); + System.out.printf( + "RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration); + + speechDenoiser.release(); + } +} diff --git a/java-api-examples/StreamingSpeechEnhancementVoxLockWithVoiceSample.java b/java-api-examples/StreamingSpeechEnhancementVoxLockWithVoiceSample.java new file mode 100644 index 000000000..6625dfc55 --- /dev/null +++ b/java-api-examples/StreamingSpeechEnhancementVoxLockWithVoiceSample.java @@ -0,0 +1,86 @@ +// Copyright 2026 Xiaomi Corporation +// +// This file shows how to use streaming VoxLock speech enhancement models in +// sherpa-onnx with a voice sample. +// +// VoxLock is a target-speaker voice isolation model. It keeps the voice of one +// speaker, e.g., the caller in a phone call, and removes the other voices. +// +// The model is from https://github.com/arbazsiddiqui/voxlock +// +// This example provides a voice sample of the speaker to keep, so VoxLock +// isolates that speaker from the first sample. The voice sample must have at +// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the +// loudest 3 seconds within its first 30 seconds. See +// StreamingSpeechEnhancementVoxLock.java if you don't have a voice sample. +// +// Download models and sample test waves from: +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models + +import com.k2fsa.sherpa.onnx.*; + +public class StreamingSpeechEnhancementVoxLockWithVoiceSample { + private static void appendSamples(java.util.ArrayList dst, float[] src) { + for (float v : src) { + dst.add(v); + } + } + + private static float[] toFloatArray(java.util.ArrayList src) { + float[] ans = new float[src.size()]; + for (int i = 0; i != src.size(); ++i) { + ans[i] = src.get(i); + } + return ans; + } + + public static void main(String[] args) { + OfflineSpeechDenoiserModelConfig modelConfig = + OfflineSpeechDenoiserModelConfig.builder() + .setNumThreads(1) + .setDebug(true) + .setProvider("cpu") + .setVoxlock( + OfflineSpeechDenoiserVoxLockModelConfig.builder() + .setEnrollModel("./sherpa-onnx-voxlock-int8/enroll.onnx") + .setChunkModel("./sherpa-onnx-voxlock-int8/chunk_int8.onnx") + .setVoiceSample("./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav") + .build()) + .build(); + + OnlineSpeechDenoiserConfig config = + OnlineSpeechDenoiserConfig.builder().setModel(modelConfig).build(); + + OnlineSpeechDenoiser speechDenoiser = new OnlineSpeechDenoiser(config); + + WaveReader reader = new WaveReader("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"); + int frameShift = speechDenoiser.getFrameShiftInSamples(); + java.util.ArrayList output = new java.util.ArrayList<>(); + + float[] samples = reader.getSamples(); + long begin = System.currentTimeMillis(); + for (int start = 0; start < samples.length; start += frameShift) { + int end = Math.min(start + frameShift, samples.length); + float[] chunk = java.util.Arrays.copyOfRange(samples, start, end); + DenoisedAudio denoised = speechDenoiser.run(chunk, reader.getSampleRate()); + appendSamples(output, denoised.getSamples()); + } + + DenoisedAudio denoised = speechDenoiser.flush(); + appendSamples(output, denoised.getSamples()); + long elapsedMs = System.currentTimeMillis() - begin; + + String outFilename = "voxlock-enhanced-online-with-voice-sample-16k.wav"; + WaveWriter.write(outFilename, toFloatArray(output), speechDenoiser.getSampleRate()); + + float elapsedSeconds = elapsedMs / 1000.0f; + float duration = samples.length / (float) reader.getSampleRate(); + System.out.printf("Saved to %s\n", outFilename); + System.out.printf("Elapsed seconds: %.3f\n", elapsedSeconds); + System.out.printf("Audio duration in seconds: %.3f\n", duration); + System.out.printf( + "RTF: %.3f/%.3f = %.3f\n", elapsedSeconds, duration, elapsedSeconds / duration); + + speechDenoiser.release(); + } +} diff --git a/java-api-examples/run-non-streaming-speech-enhancement-voxlock-with-voice-sample.sh b/java-api-examples/run-non-streaming-speech-enhancement-voxlock-with-voice-sample.sh new file mode 100755 index 000000000..ae998c7aa --- /dev/null +++ b/java-api-examples/run-non-streaming-speech-enhancement-voxlock-with-voice-sample.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2 + tar xvf sherpa-onnx-voxlock-int8.tar.bz2 + rm sherpa-onnx-voxlock-int8.tar.bz2 +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + NonStreamingSpeechEnhancementVoxLockWithVoiceSample.java + +mkdir -p NonStreamingSpeechEnhancementVoxLockWithVoiceSample +mv voxlock-enhanced-with-voice-sample-16k.wav NonStreamingSpeechEnhancementVoxLockWithVoiceSample/ +cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav NonStreamingSpeechEnhancementVoxLockWithVoiceSample/ diff --git a/java-api-examples/run-non-streaming-speech-enhancement-voxlock.sh b/java-api-examples/run-non-streaming-speech-enhancement-voxlock.sh new file mode 100755 index 000000000..ca9df1fb6 --- /dev/null +++ b/java-api-examples/run-non-streaming-speech-enhancement-voxlock.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2 + tar xvf sherpa-onnx-voxlock-int8.tar.bz2 + rm sherpa-onnx-voxlock-int8.tar.bz2 +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + NonStreamingSpeechEnhancementVoxLock.java + +mkdir -p NonStreamingSpeechEnhancementVoxLock +mv voxlock-enhanced-16k.wav NonStreamingSpeechEnhancementVoxLock/ +cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav NonStreamingSpeechEnhancementVoxLock/ diff --git a/java-api-examples/run-streaming-speech-enhancement-voxlock-with-voice-sample.sh b/java-api-examples/run-streaming-speech-enhancement-voxlock-with-voice-sample.sh new file mode 100755 index 000000000..d754789cc --- /dev/null +++ b/java-api-examples/run-streaming-speech-enhancement-voxlock-with-voice-sample.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2 + tar xvf sherpa-onnx-voxlock-int8.tar.bz2 + rm sherpa-onnx-voxlock-int8.tar.bz2 +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + StreamingSpeechEnhancementVoxLockWithVoiceSample.java + +mkdir -p StreamingSpeechEnhancementVoxLockWithVoiceSample +mv voxlock-enhanced-online-with-voice-sample-16k.wav StreamingSpeechEnhancementVoxLockWithVoiceSample/ +cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav StreamingSpeechEnhancementVoxLockWithVoiceSample/ diff --git a/java-api-examples/run-streaming-speech-enhancement-voxlock.sh b/java-api-examples/run-streaming-speech-enhancement-voxlock.sh new file mode 100755 index 000000000..cc7ccfb72 --- /dev/null +++ b/java-api-examples/run-streaming-speech-enhancement-voxlock.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2 + tar xvf sherpa-onnx-voxlock-int8.tar.bz2 + rm sherpa-onnx-voxlock-int8.tar.bz2 +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + StreamingSpeechEnhancementVoxLock.java + +mkdir -p StreamingSpeechEnhancementVoxLock +mv voxlock-enhanced-online-16k.wav StreamingSpeechEnhancementVoxLock/ +cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav StreamingSpeechEnhancementVoxLock/ diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh index 26b82195c..c555865e2 100755 --- a/kotlin-api-examples/run.sh +++ b/kotlin-api-examples/run.sh @@ -930,7 +930,113 @@ function testOfflineZipformerCtcHlg() { java -Djava.library.path=../build/lib -jar $out_filename } +function testOfflineSpeechDenoiserVoxLock() { + if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2 + tar xvf sherpa-onnx-voxlock-int8.tar.bz2 + rm sherpa-onnx-voxlock-int8.tar.bz2 + fi + + out_filename=test_offline_speech_denoiser_voxlock.jar + kotlinc-jvm -include-runtime -d $out_filename \ + test_offline_speech_denoiser_voxlock.kt \ + OfflineSpeechDenoiser.kt \ + DenoisedAudio.kt \ + WaveReader.kt \ + faked-asset-manager.kt \ + faked-log.kt + + ls -lh $out_filename + + java -Djava.library.path=../build/lib -jar $out_filename + + mkdir -p test_offline_speech_denoiser_voxlock + mv voxlock-enhanced-16k.wav test_offline_speech_denoiser_voxlock/ + cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test_offline_speech_denoiser_voxlock/ +} + +function testOfflineSpeechDenoiserVoxLockWithVoiceSample() { + if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2 + tar xvf sherpa-onnx-voxlock-int8.tar.bz2 + rm sherpa-onnx-voxlock-int8.tar.bz2 + fi + + out_filename=test_offline_speech_denoiser_voxlock_with_voice_sample.jar + kotlinc-jvm -include-runtime -d $out_filename \ + test_offline_speech_denoiser_voxlock_with_voice_sample.kt \ + OfflineSpeechDenoiser.kt \ + DenoisedAudio.kt \ + WaveReader.kt \ + faked-asset-manager.kt \ + faked-log.kt + + ls -lh $out_filename + + java -Djava.library.path=../build/lib -jar $out_filename + + mkdir -p test_offline_speech_denoiser_voxlock_with_voice_sample + mv voxlock-enhanced-with-voice-sample-16k.wav test_offline_speech_denoiser_voxlock_with_voice_sample/ + cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test_offline_speech_denoiser_voxlock_with_voice_sample/ +} + +function testOnlineSpeechDenoiserVoxLock() { + if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2 + tar xvf sherpa-onnx-voxlock-int8.tar.bz2 + rm sherpa-onnx-voxlock-int8.tar.bz2 + fi + + out_filename=test_online_speech_denoiser_voxlock.jar + kotlinc-jvm -include-runtime -d $out_filename \ + test_online_speech_denoiser_voxlock.kt \ + OnlineSpeechDenoiser.kt \ + OfflineSpeechDenoiser.kt \ + DenoisedAudio.kt \ + WaveReader.kt \ + faked-asset-manager.kt \ + faked-log.kt + + ls -lh $out_filename + + java -Djava.library.path=../build/lib -jar $out_filename + + mkdir -p test_online_speech_denoiser_voxlock + mv voxlock-enhanced-online-16k.wav test_online_speech_denoiser_voxlock/ + cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test_online_speech_denoiser_voxlock/ +} + +function testOnlineSpeechDenoiserVoxLockWithVoiceSample() { + if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2 + tar xvf sherpa-onnx-voxlock-int8.tar.bz2 + rm sherpa-onnx-voxlock-int8.tar.bz2 + fi + + out_filename=test_online_speech_denoiser_voxlock_with_voice_sample.jar + kotlinc-jvm -include-runtime -d $out_filename \ + test_online_speech_denoiser_voxlock_with_voice_sample.kt \ + OnlineSpeechDenoiser.kt \ + OfflineSpeechDenoiser.kt \ + DenoisedAudio.kt \ + WaveReader.kt \ + faked-asset-manager.kt \ + faked-log.kt + + ls -lh $out_filename + + java -Djava.library.path=../build/lib -jar $out_filename + + mkdir -p test_online_speech_denoiser_voxlock_with_voice_sample + mv voxlock-enhanced-online-with-voice-sample-16k.wav test_online_speech_denoiser_voxlock_with_voice_sample/ + cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test_online_speech_denoiser_voxlock_with_voice_sample/ +} + testVersion +testOfflineSpeechDenoiserVoxLock +testOfflineSpeechDenoiserVoxLockWithVoiceSample +testOnlineSpeechDenoiserVoxLock +testOnlineSpeechDenoiserVoxLockWithVoiceSample testPocketTts testZipVoiceTtsLexicon testSupertonicTts diff --git a/kotlin-api-examples/test_offline_speech_denoiser_voxlock.kt b/kotlin-api-examples/test_offline_speech_denoiser_voxlock.kt new file mode 100644 index 000000000..27cbabe76 --- /dev/null +++ b/kotlin-api-examples/test_offline_speech_denoiser_voxlock.kt @@ -0,0 +1,60 @@ +package com.k2fsa.sherpa.onnx + +// VoxLock is a target-speaker voice isolation model. It keeps the voice of one +// speaker, e.g., the caller in a phone call, and removes the other voices. +// +// The model is from https://github.com/arbazsiddiqui/voxlock +// +// This example does not provide a voice sample, so VoxLock takes the loudest +// 3 seconds in the first 15 seconds of the input as the speaker to keep. See +// test_offline_speech_denoiser_voxlock_with_voice_sample.kt if you have a +// voice sample of the speaker. +// +// Please download test files in this script from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models + +fun main() { + test() +} + +fun test() { + val denoiser = createOfflineSpeechDenoiser() + + val waveFilename = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav" + val waveData = WaveReader.readWaveFromFile(filename = waveFilename) + + val begin = System.currentTimeMillis() + val denoised = denoiser.run(waveData.samples, waveData.sampleRate) + val elapsedMs = System.currentTimeMillis() - begin + + denoised.save(filename = "./voxlock-enhanced-16k.wav") + + val elapsedSeconds = elapsedMs / 1000.0f + val duration = waveData.samples.size / waveData.sampleRate.toFloat() + println("Saved to ./voxlock-enhanced-16k.wav") + println("Elapsed seconds: " + "%.3f".format(elapsedSeconds)) + println("Audio duration in seconds: " + "%.3f".format(duration)) + println( + "RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " + + "%.3f".format(elapsedSeconds / duration) + ) + + denoiser.release() +} + +fun createOfflineSpeechDenoiser(): OfflineSpeechDenoiser { + val config = OfflineSpeechDenoiserConfig( + model = OfflineSpeechDenoiserModelConfig( + voxlock = OfflineSpeechDenoiserVoxLockModelConfig( + enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx", + chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx", + ), + provider = "cpu", + numThreads = 1, + ), + ) + + println(config) + + return OfflineSpeechDenoiser(config = config) +} diff --git a/kotlin-api-examples/test_offline_speech_denoiser_voxlock_with_voice_sample.kt b/kotlin-api-examples/test_offline_speech_denoiser_voxlock_with_voice_sample.kt new file mode 100644 index 000000000..37df30493 --- /dev/null +++ b/kotlin-api-examples/test_offline_speech_denoiser_voxlock_with_voice_sample.kt @@ -0,0 +1,62 @@ +package com.k2fsa.sherpa.onnx + +// VoxLock is a target-speaker voice isolation model. It keeps the voice of one +// speaker, e.g., the caller in a phone call, and removes the other voices. +// +// The model is from https://github.com/arbazsiddiqui/voxlock +// +// This example provides a voice sample of the speaker to keep, so VoxLock +// isolates that speaker from the first sample. The voice sample must have at +// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the +// loudest 3 seconds within its first 30 seconds. See +// test_offline_speech_denoiser_voxlock.kt if you don't have a voice sample. +// +// Please download test files in this script from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models + +fun main() { + test() +} + +fun test() { + val denoiser = createOfflineSpeechDenoiser() + + val waveFilename = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav" + val waveData = WaveReader.readWaveFromFile(filename = waveFilename) + + val begin = System.currentTimeMillis() + val denoised = denoiser.run(waveData.samples, waveData.sampleRate) + val elapsedMs = System.currentTimeMillis() - begin + + denoised.save(filename = "./voxlock-enhanced-with-voice-sample-16k.wav") + + val elapsedSeconds = elapsedMs / 1000.0f + val duration = waveData.samples.size / waveData.sampleRate.toFloat() + println("Saved to ./voxlock-enhanced-with-voice-sample-16k.wav") + println("Elapsed seconds: " + "%.3f".format(elapsedSeconds)) + println("Audio duration in seconds: " + "%.3f".format(duration)) + println( + "RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " + + "%.3f".format(elapsedSeconds / duration) + ) + + denoiser.release() +} + +fun createOfflineSpeechDenoiser(): OfflineSpeechDenoiser { + val config = OfflineSpeechDenoiserConfig( + model = OfflineSpeechDenoiserModelConfig( + voxlock = OfflineSpeechDenoiserVoxLockModelConfig( + enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx", + chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx", + voiceSample = "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav", + ), + provider = "cpu", + numThreads = 1, + ), + ) + + println(config) + + return OfflineSpeechDenoiser(config = config) +} diff --git a/kotlin-api-examples/test_online_speech_denoiser_voxlock.kt b/kotlin-api-examples/test_online_speech_denoiser_voxlock.kt new file mode 100644 index 000000000..44f5e0c14 --- /dev/null +++ b/kotlin-api-examples/test_online_speech_denoiser_voxlock.kt @@ -0,0 +1,70 @@ +package com.k2fsa.sherpa.onnx + +// VoxLock is a target-speaker voice isolation model. It keeps the voice of one +// speaker, e.g., the caller in a phone call, and removes the other voices. +// +// The model is from https://github.com/arbazsiddiqui/voxlock +// +// This example does not provide a voice sample, so VoxLock passes the audio +// through unchanged until it has heard 3 seconds of speech, takes that speech +// as the speaker to keep, and isolates from then on. See +// test_online_speech_denoiser_voxlock_with_voice_sample.kt if you have a voice +// sample of the speaker. +// +// Please download test files in this script from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models + +fun main() { + test() +} + +fun test() { + val denoiser = createOnlineSpeechDenoiser() + val waveData = WaveReader.readWaveFromFile("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav") + val output = mutableListOf() + val frameShift = denoiser.frameShiftInSamples + + val begin = System.currentTimeMillis() + var start = 0 + while (start < waveData.samples.size) { + val end = minOf(start + frameShift, waveData.samples.size) + val chunk = waveData.samples.copyOfRange(start, end) + val denoised = denoiser.run(chunk, waveData.sampleRate) + output.addAll(denoised.samples.asList()) + start = end + } + + output.addAll(denoiser.flush().samples.asList()) + val elapsedMs = System.currentTimeMillis() - begin + + DenoisedAudio(output.toFloatArray(), denoiser.sampleRate).save( + filename = "./voxlock-enhanced-online-16k.wav" + ) + + val elapsedSeconds = elapsedMs / 1000.0f + val duration = waveData.samples.size / waveData.sampleRate.toFloat() + println("Saved to ./voxlock-enhanced-online-16k.wav") + println("Elapsed seconds: " + "%.3f".format(elapsedSeconds)) + println("Audio duration in seconds: " + "%.3f".format(duration)) + println( + "RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " + + "%.3f".format(elapsedSeconds / duration) + ) + + denoiser.release() +} + +fun createOnlineSpeechDenoiser(): OnlineSpeechDenoiser { + val config = OnlineSpeechDenoiserConfig( + model = OfflineSpeechDenoiserModelConfig( + voxlock = OfflineSpeechDenoiserVoxLockModelConfig( + enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx", + chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx", + ), + provider = "cpu", + numThreads = 1, + ), + ) + + return OnlineSpeechDenoiser(config = config) +} diff --git a/kotlin-api-examples/test_online_speech_denoiser_voxlock_with_voice_sample.kt b/kotlin-api-examples/test_online_speech_denoiser_voxlock_with_voice_sample.kt new file mode 100644 index 000000000..16f694425 --- /dev/null +++ b/kotlin-api-examples/test_online_speech_denoiser_voxlock_with_voice_sample.kt @@ -0,0 +1,71 @@ +package com.k2fsa.sherpa.onnx + +// VoxLock is a target-speaker voice isolation model. It keeps the voice of one +// speaker, e.g., the caller in a phone call, and removes the other voices. +// +// The model is from https://github.com/arbazsiddiqui/voxlock +// +// This example provides a voice sample of the speaker to keep, so VoxLock +// isolates that speaker from the first sample. The voice sample must have at +// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the +// loudest 3 seconds within its first 30 seconds. See +// test_online_speech_denoiser_voxlock.kt if you don't have a voice sample. +// +// Please download test files in this script from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models + +fun main() { + test() +} + +fun test() { + val denoiser = createOnlineSpeechDenoiser() + val waveData = WaveReader.readWaveFromFile("./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav") + val output = mutableListOf() + val frameShift = denoiser.frameShiftInSamples + + val begin = System.currentTimeMillis() + var start = 0 + while (start < waveData.samples.size) { + val end = minOf(start + frameShift, waveData.samples.size) + val chunk = waveData.samples.copyOfRange(start, end) + val denoised = denoiser.run(chunk, waveData.sampleRate) + output.addAll(denoised.samples.asList()) + start = end + } + + output.addAll(denoiser.flush().samples.asList()) + val elapsedMs = System.currentTimeMillis() - begin + + DenoisedAudio(output.toFloatArray(), denoiser.sampleRate).save( + filename = "./voxlock-enhanced-online-with-voice-sample-16k.wav" + ) + + val elapsedSeconds = elapsedMs / 1000.0f + val duration = waveData.samples.size / waveData.sampleRate.toFloat() + println("Saved to ./voxlock-enhanced-online-with-voice-sample-16k.wav") + println("Elapsed seconds: " + "%.3f".format(elapsedSeconds)) + println("Audio duration in seconds: " + "%.3f".format(duration)) + println( + "RTF: " + "%.3f".format(elapsedSeconds) + "/" + "%.3f".format(duration) + " = " + + "%.3f".format(elapsedSeconds / duration) + ) + + denoiser.release() +} + +fun createOnlineSpeechDenoiser(): OnlineSpeechDenoiser { + val config = OnlineSpeechDenoiserConfig( + model = OfflineSpeechDenoiserModelConfig( + voxlock = OfflineSpeechDenoiserVoxLockModelConfig( + enrollModel = "./sherpa-onnx-voxlock-int8/enroll.onnx", + chunkModel = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx", + voiceSample = "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav", + ), + provider = "cpu", + numThreads = 1, + ), + ) + + return OnlineSpeechDenoiser(config = config) +} diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeechDenoiserModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeechDenoiserModelConfig.java index 79a01728f..a45d67a9e 100644 --- a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeechDenoiserModelConfig.java +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeechDenoiserModelConfig.java @@ -4,6 +4,7 @@ package com.k2fsa.sherpa.onnx; public class OfflineSpeechDenoiserModelConfig { private final OfflineSpeechDenoiserGtcrnModelConfig gtcrn; private final OfflineSpeechDenoiserDpdfNetModelConfig dpdfnet; + private final OfflineSpeechDenoiserVoxLockModelConfig voxlock; private final int numThreads; private final boolean debug; private final String provider; @@ -11,6 +12,7 @@ public class OfflineSpeechDenoiserModelConfig { private OfflineSpeechDenoiserModelConfig(Builder builder) { this.gtcrn = builder.gtcrn; this.dpdfnet = builder.dpdfnet; + this.voxlock = builder.voxlock; this.numThreads = builder.numThreads; this.debug = builder.debug; this.provider = builder.provider; @@ -23,6 +25,7 @@ public class OfflineSpeechDenoiserModelConfig { public static class Builder { private OfflineSpeechDenoiserGtcrnModelConfig gtcrn = OfflineSpeechDenoiserGtcrnModelConfig.builder().build(); private OfflineSpeechDenoiserDpdfNetModelConfig dpdfnet = OfflineSpeechDenoiserDpdfNetModelConfig.builder().build(); + private OfflineSpeechDenoiserVoxLockModelConfig voxlock = OfflineSpeechDenoiserVoxLockModelConfig.builder().build(); private int numThreads = 1; private boolean debug = true; private String provider = "cpu"; @@ -41,6 +44,11 @@ public class OfflineSpeechDenoiserModelConfig { return this; } + public Builder setVoxlock(OfflineSpeechDenoiserVoxLockModelConfig voxlock) { + this.voxlock = voxlock; + return this; + } + public Builder setNumThreads(int numThreads) { this.numThreads = numThreads; return this; diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeechDenoiserVoxLockModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeechDenoiserVoxLockModelConfig.java new file mode 100644 index 000000000..5b87e6586 --- /dev/null +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeechDenoiserVoxLockModelConfig.java @@ -0,0 +1,80 @@ +// Copyright 2026 Xiaomi Corporation +package com.k2fsa.sherpa.onnx; + +/** + * VoxLock target-speaker voice isolation model config. + * + *

VoxLock keeps the voice of one speaker (the caller) and removes the other voices. The model is + * from https://github.com/arbazsiddiqui/voxlock + */ +public class OfflineSpeechDenoiserVoxLockModelConfig { + private final String enrollModel; + private final String chunkModel; + private final String voiceSample; + + private OfflineSpeechDenoiserVoxLockModelConfig(Builder builder) { + this.enrollModel = builder.enrollModel; + this.chunkModel = builder.chunkModel; + this.voiceSample = builder.voiceSample; + } + + public static Builder builder() { + return new Builder(); + } + + public String getEnrollModel() { + return enrollModel; + } + + public String getChunkModel() { + return chunkModel; + } + + public String getVoiceSample() { + return voiceSample; + } + + public static class Builder { + private String enrollModel = ""; + private String chunkModel = ""; + private String voiceSample = ""; + + public OfflineSpeechDenoiserVoxLockModelConfig build() { + return new OfflineSpeechDenoiserVoxLockModelConfig(this); + } + + /** + * Path to the VoxLock enroll model, e.g., enroll.onnx. It runs once on a voice sample of the + * speaker to keep. + */ + public Builder setEnrollModel(String enrollModel) { + this.enrollModel = enrollModel; + return this; + } + + /** + * Path to the VoxLock chunk model, e.g., chunk.onnx or chunk_int8.onnx. It runs on each chunk + * of the audio. + */ + public Builder setChunkModel(String chunkModel) { + this.chunkModel = chunkModel; + return this; + } + + /** + * Optional. Path to a wave file with at least 3 seconds of the caller's voice. VoxLock keeps + * this voice and removes the others. From a longer file, it uses the loudest 3 seconds within + * the first 30 seconds. A file that is shorter or cannot be read is ignored, as if it were + * empty. + * + *

If empty or ignored, the voice sample is taken from the input audio: the offline denoiser + * uses the loudest 3 seconds in the first 15 seconds of the input, and the online denoiser + * passes the audio through unchanged until 3 seconds of speech have been heard, then uses that + * speech. + */ + public Builder setVoiceSample(String voiceSample) { + this.voiceSample = voiceSample; + return this; + } + } +} diff --git a/sherpa-onnx/jni/speech-denoiser.cc b/sherpa-onnx/jni/speech-denoiser.cc index e48fd39e0..3c359bbf0 100644 --- a/sherpa-onnx/jni/speech-denoiser.cc +++ b/sherpa-onnx/jni/speech-denoiser.cc @@ -33,6 +33,19 @@ OfflineSpeechDenoiserModelConfig GetOfflineSpeechDenoiserModelConfig( SHERPA_ONNX_JNI_READ_FLOAT(ans.dpdfnet.attenuation_limit_db, attenuationLimitDb, dpdfnet_cls, dpdfnet); + fid = env->GetFieldID( + model_config_cls, "voxlock", + "Lcom/k2fsa/sherpa/onnx/OfflineSpeechDenoiserVoxLockModelConfig;"); + jobject voxlock = env->GetObjectField(model, fid); + jclass voxlock_cls = env->GetObjectClass(voxlock); + + SHERPA_ONNX_JNI_READ_STRING(ans.voxlock.enroll_model, enrollModel, voxlock_cls, + voxlock); + SHERPA_ONNX_JNI_READ_STRING(ans.voxlock.chunk_model, chunkModel, voxlock_cls, + voxlock); + SHERPA_ONNX_JNI_READ_STRING(ans.voxlock.voice_sample, voiceSample, voxlock_cls, + voxlock); + SHERPA_ONNX_JNI_READ_INT(ans.num_threads, numThreads, model_config_cls, model); SHERPA_ONNX_JNI_READ_BOOL(ans.debug, debug, model_config_cls, model); diff --git a/sherpa-onnx/kotlin-api/OfflineSpeechDenoiser.kt b/sherpa-onnx/kotlin-api/OfflineSpeechDenoiser.kt index faeeccc65..63855c880 100644 --- a/sherpa-onnx/kotlin-api/OfflineSpeechDenoiser.kt +++ b/sherpa-onnx/kotlin-api/OfflineSpeechDenoiser.kt @@ -11,9 +11,27 @@ data class OfflineSpeechDenoiserDpdfNetModelConfig( var attenuationLimitDb: Float = 0.0f, ) +// VoxLock keeps the voice of one speaker (the caller) and removes the other +// voices. The model is from https://github.com/arbazsiddiqui/voxlock +data class OfflineSpeechDenoiserVoxLockModelConfig( + var enrollModel: String = "", + var chunkModel: String = "", + // Optional. A wave file with at least 3 seconds of the caller's voice. + // From a longer file, the loudest 3 seconds within its first 30 seconds + // are used. A file that is shorter or cannot be read is ignored, as if it + // were empty. If empty or ignored, the voice sample is taken from the + // input audio: the offline denoiser uses the loudest 3 seconds in the + // first 15 seconds of the input, and the online denoiser passes audio + // through unchanged until 3 seconds of speech have been heard, then uses + // that speech. + var voiceSample: String = "", +) + data class OfflineSpeechDenoiserModelConfig( var gtcrn: OfflineSpeechDenoiserGtcrnModelConfig = OfflineSpeechDenoiserGtcrnModelConfig(), var dpdfnet: OfflineSpeechDenoiserDpdfNetModelConfig = OfflineSpeechDenoiserDpdfNetModelConfig(), + var voxlock: OfflineSpeechDenoiserVoxLockModelConfig = + OfflineSpeechDenoiserVoxLockModelConfig(), var numThreads: Int = 1, var debug: Boolean = false, var provider: String = "cpu",