Add JavaScript (WebAssembly) API for VoxLock (#4106)

This commit is contained in:
Fangjun Kuang authored and GitHub committed 2026-10-26 19:33:06 +08:00
1 parent cb2071f22f
commit 42e1e0d077
10 files changed
+434 -4

No files matched your search

+28
View File
@@ -13,6 +13,34 @@ ls -lh node_modules
echo "---test version---"
node ./test-version.js
echo "---test voxlock speech enhancement---"
download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
node ./test-offline-speech-enhancement-voxlock.js
mkdir -p test-offline-speech-enhancement-voxlock
mv voxlock-enhanced-16k.wav test-offline-speech-enhancement-voxlock/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test-offline-speech-enhancement-voxlock/
node ./test-offline-speech-enhancement-voxlock-with-voice-sample.js
mkdir -p test-offline-speech-enhancement-voxlock-with-voice-sample
mv voxlock-enhanced-with-voice-sample-16k.wav test-offline-speech-enhancement-voxlock-with-voice-sample/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test-offline-speech-enhancement-voxlock-with-voice-sample/
node ./test-online-speech-enhancement-voxlock.js
mkdir -p test-online-speech-enhancement-voxlock
mv voxlock-enhanced-online-16k.wav test-online-speech-enhancement-voxlock/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test-online-speech-enhancement-voxlock/
node ./test-online-speech-enhancement-voxlock-with-voice-sample.js
mkdir -p test-online-speech-enhancement-voxlock-with-voice-sample
mv voxlock-enhanced-online-with-voice-sample-16k.wav test-online-speech-enhancement-voxlock-with-voice-sample/
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test-online-speech-enhancement-voxlock-with-voice-sample/
ls -lh test-offline-speech-enhancement-voxlock test-offline-speech-enhancement-voxlock-with-voice-sample test-online-speech-enhancement-voxlock test-online-speech-enhancement-voxlock-with-voice-sample
rm -rf sherpa-onnx-voxlock-int8
# tts
download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2
+26 -1
View File
@@ -56,8 +56,33 @@ jobs:
./.github/scripts/test-nodejs-npm.sh
- uses: actions/upload-artifact@v4
with:
name: tts-waves-${{ matrix.os }}-${{ matrix.node-version }}
name: tts-waves-${{ matrix.os }}
# all TTS outputs: *-lexicon.wav, *-phonemize.wav, plus the
# own-frontend models (test-pocket-en.wav, test-supertonic-*.wav)
path: nodejs-examples/test-*.wav
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-offline-speech-enhancement-voxlock-${{ matrix.os }}
# this workflow runs the tests from nodejs-examples (export d=...)
path: nodejs-examples/test-offline-speech-enhancement-voxlock
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-offline-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
path: nodejs-examples/test-offline-speech-enhancement-voxlock-with-voice-sample
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-online-speech-enhancement-voxlock-${{ matrix.os }}
path: nodejs-examples/test-online-speech-enhancement-voxlock
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-online-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
path: nodejs-examples/test-online-speech-enhancement-voxlock-with-voice-sample
if-no-files-found: warn
+26 -1
View File
@@ -119,13 +119,38 @@ jobs:
./.github/scripts/test-nodejs-npm.sh
- uses: actions/upload-artifact@v4
with:
name: tts-waves-${{ matrix.os }}-${{ matrix.node-version }}
name: tts-waves-${{ matrix.os }}
# The examples are copied into scripts/nodejs and run from there
# (export d=scripts/nodejs), so their wavs land in scripts/nodejs,
# not in nodejs-examples.
path: scripts/nodejs/test-*.wav
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-offline-speech-enhancement-voxlock-${{ matrix.os }}
# the examples run from scripts/nodejs, see the comment above
path: scripts/nodejs/test-offline-speech-enhancement-voxlock
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-offline-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
path: scripts/nodejs/test-offline-speech-enhancement-voxlock-with-voice-sample
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-online-speech-enhancement-voxlock-${{ matrix.os }}
path: scripts/nodejs/test-online-speech-enhancement-voxlock
if-no-files-found: warn
- uses: actions/upload-artifact@v4
with:
name: test-online-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
path: scripts/nodejs/test-online-speech-enhancement-voxlock-with-voice-sample
if-no-files-found: warn
- name: Setup tmate session
if: false
# if: failure()
+13
View File
@@ -77,6 +77,19 @@ node ./test-offline-speech-enhancement-dpdfnet.js
node ./test-online-speech-enhancement-dpdfnet.js
```
VoxLock is a target-speaker voice isolation model. The model is from
`https://github.com/arbazsiddiqui/voxlock`. Each variant has an example with
and without a voice sample of the speaker to keep.
```bash
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
node ./test-offline-speech-enhancement-voxlock.js
node ./test-offline-speech-enhancement-voxlock-with-voice-sample.js
node ./test-online-speech-enhancement-voxlock.js
node ./test-online-speech-enhancement-voxlock-with-voice-sample.js
```
The following four example files are available:
```bash
@@ -0,0 +1,62 @@
// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang)
//
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example provides a voice sample of the speaker to keep, so VoxLock
// isolates that speaker from the first sample. The voice sample must have at
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
// loudest 3 seconds within its first 30 seconds. See
// ./test-offline-speech-enhancement-voxlock.js if you don't have a voice
// sample.
//
// Please download the model used in this file from
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
//
// Example:
//
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
//
// This script shows how to use speech enhancement API from sherpa-onnx.
const sherpa_onnx = require('sherpa-onnx');
function createOfflineSpeechDenoiser() {
const config = {
model: {
voxlock: {
enrollModel: './sherpa-onnx-voxlock-int8/enroll.onnx',
chunkModel: './sherpa-onnx-voxlock-int8/chunk_int8.onnx',
voiceSample: './sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav',
},
debug: 1,
},
};
return sherpa_onnx.createOfflineSpeechDenoiser(config);
}
const speech_denoiser = createOfflineSpeechDenoiser();
const waveFilename = './sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav';
const wave = sherpa_onnx.readWave(waveFilename);
const begin = Date.now();
const denoised = speech_denoiser.run(wave.samples, wave.sampleRate);
const elapsedSeconds = (Date.now() - begin) / 1000;
const outputFilename = './voxlock-enhanced-with-voice-sample-16k.wav';
sherpa_onnx.writeWave(outputFilename, denoised);
const duration = wave.samples.length / wave.sampleRate;
const rtf = elapsedSeconds / duration;
console.log(`Saved to ${outputFilename}`);
console.log(`Elapsed seconds: ${elapsedSeconds.toFixed(3)}`);
console.log(`Audio duration in seconds: ${duration.toFixed(3)}`);
console.log(
`RTF: ${elapsedSeconds.toFixed(3)}/${duration.toFixed(3)} = ${rtf.toFixed(3)}`);
speech_denoiser.free();
@@ -0,0 +1,59 @@
// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang)
//
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example does not provide a voice sample, so VoxLock takes the loudest
// 3 seconds in the first 15 seconds of the input as the speaker to keep. See
// ./test-offline-speech-enhancement-voxlock-with-voice-sample.js if you have a
// voice sample of the speaker.
//
// Please download the model used in this file from
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
//
// Example:
//
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
//
// This script shows how to use speech enhancement API from sherpa-onnx.
const sherpa_onnx = require('sherpa-onnx');
function createOfflineSpeechDenoiser() {
const config = {
model: {
voxlock: {
enrollModel: './sherpa-onnx-voxlock-int8/enroll.onnx',
chunkModel: './sherpa-onnx-voxlock-int8/chunk_int8.onnx',
},
debug: 1,
},
};
return sherpa_onnx.createOfflineSpeechDenoiser(config);
}
const speech_denoiser = createOfflineSpeechDenoiser();
const waveFilename = './sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav';
const wave = sherpa_onnx.readWave(waveFilename);
const begin = Date.now();
const denoised = speech_denoiser.run(wave.samples, wave.sampleRate);
const elapsedSeconds = (Date.now() - begin) / 1000;
const outputFilename = './voxlock-enhanced-16k.wav';
sherpa_onnx.writeWave(outputFilename, denoised);
const duration = wave.samples.length / wave.sampleRate;
const rtf = elapsedSeconds / duration;
console.log(`Saved to ${outputFilename}`);
console.log(`Elapsed seconds: ${elapsedSeconds.toFixed(3)}`);
console.log(`Audio duration in seconds: ${duration.toFixed(3)}`);
console.log(
`RTF: ${elapsedSeconds.toFixed(3)}/${duration.toFixed(3)} = ${rtf.toFixed(3)}`);
speech_denoiser.free();
@@ -0,0 +1,79 @@
// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang)
//
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example provides a voice sample of the speaker to keep, so VoxLock
// isolates that speaker from the first sample. The voice sample must have at
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
// loudest 3 seconds within its first 30 seconds. See
// ./test-online-speech-enhancement-voxlock.js if you don't have a voice
// sample.
//
// Please download the model used in this file from
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
//
// Example:
//
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
//
// This script shows how to use the streaming speech enhancement API from
// sherpa-onnx.
const sherpa_onnx = require('sherpa-onnx');
function createOnlineSpeechDenoiser() {
const config = {
model: {
voxlock: {
enrollModel: './sherpa-onnx-voxlock-int8/enroll.onnx',
chunkModel: './sherpa-onnx-voxlock-int8/chunk_int8.onnx',
voiceSample: './sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav',
},
debug: 1,
},
};
return sherpa_onnx.createOnlineSpeechDenoiser(config);
}
const speech_denoiser = createOnlineSpeechDenoiser();
const waveFilename = './sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav';
const wave = sherpa_onnx.readWave(waveFilename);
const frameShift = speech_denoiser.frameShiftInSamples;
const output = [];
const begin = Date.now();
let start = 0;
while (start < wave.samples.length) {
const end = Math.min(start + frameShift, wave.samples.length);
const chunk = wave.samples.slice(start, end);
const denoised = speech_denoiser.run(chunk, wave.sampleRate);
output.push(...denoised.samples);
start = end;
}
output.push(...speech_denoiser.flush().samples);
const elapsedSeconds = (Date.now() - begin) / 1000;
const outputFilename = './voxlock-enhanced-online-with-voice-sample-16k.wav';
sherpa_onnx.writeWave(outputFilename, {
samples: Float32Array.from(output),
sampleRate: speech_denoiser.sampleRate,
});
const duration = wave.samples.length / wave.sampleRate;
const rtf = elapsedSeconds / duration;
console.log(`Saved to ${outputFilename}`);
console.log(`Elapsed seconds: ${elapsedSeconds.toFixed(3)}`);
console.log(`Audio duration in seconds: ${duration.toFixed(3)}`);
console.log(
`RTF: ${elapsedSeconds.toFixed(3)}/${duration.toFixed(3)} = ${rtf.toFixed(3)}`);
speech_denoiser.free();
@@ -0,0 +1,77 @@
// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang)
//
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
// speaker, e.g., the caller in a phone call, and removes the other voices.
//
// The model is from https://github.com/arbazsiddiqui/voxlock
//
// This example does not provide a voice sample, so VoxLock passes the audio
// through unchanged until it has heard 3 seconds of speech, takes that speech
// as the speaker to keep, and isolates from then on. See
// ./test-online-speech-enhancement-voxlock-with-voice-sample.js if you have a
// voice sample of the speaker.
//
// Please download the model used in this file from
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
//
// Example:
//
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
//
// This script shows how to use the streaming speech enhancement API from
// sherpa-onnx.
const sherpa_onnx = require('sherpa-onnx');
function createOnlineSpeechDenoiser() {
const config = {
model: {
voxlock: {
enrollModel: './sherpa-onnx-voxlock-int8/enroll.onnx',
chunkModel: './sherpa-onnx-voxlock-int8/chunk_int8.onnx',
},
debug: 1,
},
};
return sherpa_onnx.createOnlineSpeechDenoiser(config);
}
const speech_denoiser = createOnlineSpeechDenoiser();
const waveFilename = './sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav';
const wave = sherpa_onnx.readWave(waveFilename);
const frameShift = speech_denoiser.frameShiftInSamples;
const output = [];
const begin = Date.now();
let start = 0;
while (start < wave.samples.length) {
const end = Math.min(start + frameShift, wave.samples.length);
const chunk = wave.samples.slice(start, end);
const denoised = speech_denoiser.run(chunk, wave.sampleRate);
output.push(...denoised.samples);
start = end;
}
output.push(...speech_denoiser.flush().samples);
const elapsedSeconds = (Date.now() - begin) / 1000;
const outputFilename = './voxlock-enhanced-online-16k.wav';
sherpa_onnx.writeWave(outputFilename, {
samples: Float32Array.from(output),
sampleRate: speech_denoiser.sampleRate,
});
const duration = wave.samples.length / wave.sampleRate;
const rtf = elapsedSeconds / duration;
console.log(`Saved to ${outputFilename}`);
console.log(`Elapsed seconds: ${elapsedSeconds.toFixed(3)}`);
console.log(`Audio duration in seconds: ${duration.toFixed(3)}`);
console.log(
`RTF: ${elapsedSeconds.toFixed(3)}/${duration.toFixed(3)} = ${rtf.toFixed(3)}`);
speech_denoiser.free();
@@ -15,6 +15,10 @@ function freeConfig(config, Module) {
freeConfig(config.dpdfnet, Module)
}
if ('voxlock' in config) {
freeConfig(config.voxlock, Module)
}
Module._free(config.ptr);
}
@@ -69,6 +73,43 @@ function initSherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig(config, Module) {
};
}
function initSherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(config, Module) {
const enrollModelLen = Module.lengthBytesUTF8(config.enrollModel || '') + 1;
const chunkModelLen = Module.lengthBytesUTF8(config.chunkModel || '') + 1;
const voiceSampleLen = Module.lengthBytesUTF8(config.voiceSample || '') + 1;
const n = enrollModelLen + chunkModelLen + voiceSampleLen;
const buffer = Module._malloc(n);
const len = 3 * 4; // 3 pointers
const ptr = Module._malloc(len);
let offset = 0;
Module.stringToUTF8(config.enrollModel || '', buffer + offset, enrollModelLen);
offset += enrollModelLen;
Module.stringToUTF8(config.chunkModel || '', buffer + offset, chunkModelLen);
offset += chunkModelLen;
Module.stringToUTF8(config.voiceSample || '', buffer + offset, voiceSampleLen);
offset = 0;
Module.setValue(ptr, buffer + offset, 'i8*');
offset += enrollModelLen;
Module.setValue(ptr + 4, buffer + offset, 'i8*');
offset += chunkModelLen;
Module.setValue(ptr + 8, buffer + offset, 'i8*');
return {
buffer: buffer,
ptr: ptr,
len: len,
};
}
function initSherpaOnnxOfflineSpeechDenoiserModelConfig(config, Module) {
if (!('gtcrn' in config)) {
config.gtcrn = {model: ''};
@@ -78,13 +119,20 @@ function initSherpaOnnxOfflineSpeechDenoiserModelConfig(config, Module) {
config.dpdfnet = {model: ''};
}
if (!('voxlock' in config)) {
config.voxlock = {enrollModel: '', chunkModel: '', voiceSample: ''};
}
const gtcrn =
initSherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig(config.gtcrn, Module);
const dpdfnet =
initSherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig(
config.dpdfnet, Module);
const voxlock =
initSherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(
config.voxlock, Module);
const len = gtcrn.len + 3 * 4 + dpdfnet.len;
const len = gtcrn.len + 3 * 4 + dpdfnet.len + voxlock.len;
const ptr = Module._malloc(len);
let offset = 0;
@@ -106,12 +154,16 @@ function initSherpaOnnxOfflineSpeechDenoiserModelConfig(config, Module) {
Module._CopyHeap(dpdfnet.ptr, dpdfnet.len, ptr + offset);
offset += dpdfnet.len;
Module._CopyHeap(voxlock.ptr, voxlock.len, ptr + offset);
offset += voxlock.len;
return {
buffer: buffer,
ptr: ptr,
len: len,
gtcrn: gtcrn,
dpdfnet: dpdfnet,
voxlock: voxlock,
};
}
@@ -120,6 +172,7 @@ function initSherpaOnnxOfflineSpeechDenoiserConfig(config, Module) {
config.model = {
gtcrn: {model: ''},
dpdfnet: {model: ''},
voxlock: {enrollModel: '', chunkModel: '', voiceSample: ''},
provider: 'cpu',
debug: 1,
numThreads: 1,
@@ -147,6 +200,7 @@ function initSherpaOnnxOnlineSpeechDenoiserConfig(config, Module) {
config.model = {
gtcrn: {model: ''},
dpdfnet: {model: ''},
voxlock: {enrollModel: '', chunkModel: '', voiceSample: ''},
provider: 'cpu',
debug: 1,
numThreads: 1,
@@ -18,10 +18,14 @@ static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig) == 1 * 4,
static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig) ==
2 * 4,
"");
static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig) ==
3 * 4,
"");
static_assert(
sizeof(SherpaOnnxOfflineSpeechDenoiserModelConfig) ==
sizeof(SherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig) +
sizeof(SherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig) + 3 * 4,
sizeof(SherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig) +
sizeof(SherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig) + 3 * 4,
"");
static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserConfig) ==
sizeof(SherpaOnnxOfflineSpeechDenoiserModelConfig),
@@ -31,9 +35,13 @@ void MyPrint(SherpaOnnxOfflineSpeechDenoiserConfig *config) {
auto model = &config->model;
auto gtcrn = &model->gtcrn;
auto dpdfnet = &model->dpdfnet;
auto voxlock = &model->voxlock;
fprintf(stdout, "----------offline speech denoiser model config----------\n");
fprintf(stdout, "gtcrn: %s\n", gtcrn->model);
fprintf(stdout, "dpdfnet: %s\n", dpdfnet->model);
fprintf(stdout, "voxlock enroll model: %s\n", voxlock->enroll_model);
fprintf(stdout, "voxlock chunk model: %s\n", voxlock->chunk_model);
fprintf(stdout, "voxlock voice sample: %s\n", voxlock->voice_sample);
fprintf(stdout, "num threads: %d\n", model->num_threads);
fprintf(stdout, "debug: %d\n", model->debug);
fprintf(stdout, "provider: %s\n", model->provider);