mirror of
https://github.com/k2-fsa/sherpa-onnx.git
synced 2026-11-10 21:11:03 +08:00
Add JavaScript (WebAssembly) API for VoxLock (#4106)
This commit is contained in:
1 parent
cb2071f22f
commit
42e1e0d077
10 files changed
+434
-4
No files matched your search
@@ -13,6 +13,34 @@ ls -lh node_modules
|
|||||||
echo "---test version---"
|
echo "---test version---"
|
||||||
node ./test-version.js
|
node ./test-version.js
|
||||||
|
|
||||||
|
echo "---test voxlock speech enhancement---"
|
||||||
|
|
||||||
|
download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
|
||||||
|
node ./test-offline-speech-enhancement-voxlock.js
|
||||||
|
mkdir -p test-offline-speech-enhancement-voxlock
|
||||||
|
mv voxlock-enhanced-16k.wav test-offline-speech-enhancement-voxlock/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test-offline-speech-enhancement-voxlock/
|
||||||
|
|
||||||
|
node ./test-offline-speech-enhancement-voxlock-with-voice-sample.js
|
||||||
|
mkdir -p test-offline-speech-enhancement-voxlock-with-voice-sample
|
||||||
|
mv voxlock-enhanced-with-voice-sample-16k.wav test-offline-speech-enhancement-voxlock-with-voice-sample/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test-offline-speech-enhancement-voxlock-with-voice-sample/
|
||||||
|
|
||||||
|
node ./test-online-speech-enhancement-voxlock.js
|
||||||
|
mkdir -p test-online-speech-enhancement-voxlock
|
||||||
|
mv voxlock-enhanced-online-16k.wav test-online-speech-enhancement-voxlock/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav test-online-speech-enhancement-voxlock/
|
||||||
|
|
||||||
|
node ./test-online-speech-enhancement-voxlock-with-voice-sample.js
|
||||||
|
mkdir -p test-online-speech-enhancement-voxlock-with-voice-sample
|
||||||
|
mv voxlock-enhanced-online-with-voice-sample-16k.wav test-online-speech-enhancement-voxlock-with-voice-sample/
|
||||||
|
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav test-online-speech-enhancement-voxlock-with-voice-sample/
|
||||||
|
|
||||||
|
ls -lh test-offline-speech-enhancement-voxlock test-offline-speech-enhancement-voxlock-with-voice-sample test-online-speech-enhancement-voxlock test-online-speech-enhancement-voxlock-with-voice-sample
|
||||||
|
|
||||||
|
rm -rf sherpa-onnx-voxlock-int8
|
||||||
|
|
||||||
# tts
|
# tts
|
||||||
|
|
||||||
download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2
|
download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2
|
||||||
|
|||||||
@@ -56,8 +56,33 @@ jobs:
|
|||||||
./.github/scripts/test-nodejs-npm.sh
|
./.github/scripts/test-nodejs-npm.sh
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: tts-waves-${{ matrix.os }}-${{ matrix.node-version }}
|
name: tts-waves-${{ matrix.os }}
|
||||||
# all TTS outputs: *-lexicon.wav, *-phonemize.wav, plus the
|
# all TTS outputs: *-lexicon.wav, *-phonemize.wav, plus the
|
||||||
# own-frontend models (test-pocket-en.wav, test-supertonic-*.wav)
|
# own-frontend models (test-pocket-en.wav, test-supertonic-*.wav)
|
||||||
path: nodejs-examples/test-*.wav
|
path: nodejs-examples/test-*.wav
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-offline-speech-enhancement-voxlock-${{ matrix.os }}
|
||||||
|
# this workflow runs the tests from nodejs-examples (export d=...)
|
||||||
|
path: nodejs-examples/test-offline-speech-enhancement-voxlock
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-offline-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
|
||||||
|
path: nodejs-examples/test-offline-speech-enhancement-voxlock-with-voice-sample
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-online-speech-enhancement-voxlock-${{ matrix.os }}
|
||||||
|
path: nodejs-examples/test-online-speech-enhancement-voxlock
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-online-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
|
||||||
|
path: nodejs-examples/test-online-speech-enhancement-voxlock-with-voice-sample
|
||||||
|
if-no-files-found: warn
|
||||||
@@ -119,13 +119,38 @@ jobs:
|
|||||||
./.github/scripts/test-nodejs-npm.sh
|
./.github/scripts/test-nodejs-npm.sh
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v4
|
||||||
with:
|
with:
|
||||||
name: tts-waves-${{ matrix.os }}-${{ matrix.node-version }}
|
name: tts-waves-${{ matrix.os }}
|
||||||
# The examples are copied into scripts/nodejs and run from there
|
# The examples are copied into scripts/nodejs and run from there
|
||||||
# (export d=scripts/nodejs), so their wavs land in scripts/nodejs,
|
# (export d=scripts/nodejs), so their wavs land in scripts/nodejs,
|
||||||
# not in nodejs-examples.
|
# not in nodejs-examples.
|
||||||
path: scripts/nodejs/test-*.wav
|
path: scripts/nodejs/test-*.wav
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-offline-speech-enhancement-voxlock-${{ matrix.os }}
|
||||||
|
# the examples run from scripts/nodejs, see the comment above
|
||||||
|
path: scripts/nodejs/test-offline-speech-enhancement-voxlock
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-offline-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
|
||||||
|
path: scripts/nodejs/test-offline-speech-enhancement-voxlock-with-voice-sample
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-online-speech-enhancement-voxlock-${{ matrix.os }}
|
||||||
|
path: scripts/nodejs/test-online-speech-enhancement-voxlock
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: test-online-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
|
||||||
|
path: scripts/nodejs/test-online-speech-enhancement-voxlock-with-voice-sample
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
- name: Setup tmate session
|
- name: Setup tmate session
|
||||||
if: false
|
if: false
|
||||||
# if: failure()
|
# if: failure()
|
||||||
|
|||||||
@@ -77,6 +77,19 @@ node ./test-offline-speech-enhancement-dpdfnet.js
|
|||||||
node ./test-online-speech-enhancement-dpdfnet.js
|
node ./test-online-speech-enhancement-dpdfnet.js
|
||||||
```
|
```
|
||||||
|
|
||||||
|
VoxLock is a target-speaker voice isolation model. The model is from
|
||||||
|
`https://github.com/arbazsiddiqui/voxlock`. Each variant has an example with
|
||||||
|
and without a voice sample of the speaker to keep.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
node ./test-offline-speech-enhancement-voxlock.js
|
||||||
|
node ./test-offline-speech-enhancement-voxlock-with-voice-sample.js
|
||||||
|
node ./test-online-speech-enhancement-voxlock.js
|
||||||
|
node ./test-online-speech-enhancement-voxlock-with-voice-sample.js
|
||||||
|
```
|
||||||
|
|
||||||
The following four example files are available:
|
The following four example files are available:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang)
|
||||||
|
//
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example provides a voice sample of the speaker to keep, so VoxLock
|
||||||
|
// isolates that speaker from the first sample. The voice sample must have at
|
||||||
|
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
|
||||||
|
// loudest 3 seconds within its first 30 seconds. See
|
||||||
|
// ./test-offline-speech-enhancement-voxlock.js if you don't have a voice
|
||||||
|
// sample.
|
||||||
|
//
|
||||||
|
// Please download the model used in this file from
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
//
|
||||||
|
// Example:
|
||||||
|
//
|
||||||
|
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
//
|
||||||
|
// This script shows how to use speech enhancement API from sherpa-onnx.
|
||||||
|
const sherpa_onnx = require('sherpa-onnx');
|
||||||
|
|
||||||
|
function createOfflineSpeechDenoiser() {
|
||||||
|
const config = {
|
||||||
|
model: {
|
||||||
|
voxlock: {
|
||||||
|
enrollModel: './sherpa-onnx-voxlock-int8/enroll.onnx',
|
||||||
|
chunkModel: './sherpa-onnx-voxlock-int8/chunk_int8.onnx',
|
||||||
|
voiceSample: './sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav',
|
||||||
|
},
|
||||||
|
debug: 1,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
return sherpa_onnx.createOfflineSpeechDenoiser(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
const speech_denoiser = createOfflineSpeechDenoiser();
|
||||||
|
|
||||||
|
const waveFilename = './sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav';
|
||||||
|
const wave = sherpa_onnx.readWave(waveFilename);
|
||||||
|
|
||||||
|
const begin = Date.now();
|
||||||
|
const denoised = speech_denoiser.run(wave.samples, wave.sampleRate);
|
||||||
|
const elapsedSeconds = (Date.now() - begin) / 1000;
|
||||||
|
|
||||||
|
const outputFilename = './voxlock-enhanced-with-voice-sample-16k.wav';
|
||||||
|
sherpa_onnx.writeWave(outputFilename, denoised);
|
||||||
|
|
||||||
|
const duration = wave.samples.length / wave.sampleRate;
|
||||||
|
const rtf = elapsedSeconds / duration;
|
||||||
|
|
||||||
|
console.log(`Saved to ${outputFilename}`);
|
||||||
|
console.log(`Elapsed seconds: ${elapsedSeconds.toFixed(3)}`);
|
||||||
|
console.log(`Audio duration in seconds: ${duration.toFixed(3)}`);
|
||||||
|
console.log(
|
||||||
|
`RTF: ${elapsedSeconds.toFixed(3)}/${duration.toFixed(3)} = ${rtf.toFixed(3)}`);
|
||||||
|
|
||||||
|
speech_denoiser.free();
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang)
|
||||||
|
//
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example does not provide a voice sample, so VoxLock takes the loudest
|
||||||
|
// 3 seconds in the first 15 seconds of the input as the speaker to keep. See
|
||||||
|
// ./test-offline-speech-enhancement-voxlock-with-voice-sample.js if you have a
|
||||||
|
// voice sample of the speaker.
|
||||||
|
//
|
||||||
|
// Please download the model used in this file from
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
//
|
||||||
|
// Example:
|
||||||
|
//
|
||||||
|
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
//
|
||||||
|
// This script shows how to use speech enhancement API from sherpa-onnx.
|
||||||
|
const sherpa_onnx = require('sherpa-onnx');
|
||||||
|
|
||||||
|
function createOfflineSpeechDenoiser() {
|
||||||
|
const config = {
|
||||||
|
model: {
|
||||||
|
voxlock: {
|
||||||
|
enrollModel: './sherpa-onnx-voxlock-int8/enroll.onnx',
|
||||||
|
chunkModel: './sherpa-onnx-voxlock-int8/chunk_int8.onnx',
|
||||||
|
},
|
||||||
|
debug: 1,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
return sherpa_onnx.createOfflineSpeechDenoiser(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
const speech_denoiser = createOfflineSpeechDenoiser();
|
||||||
|
|
||||||
|
const waveFilename = './sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav';
|
||||||
|
const wave = sherpa_onnx.readWave(waveFilename);
|
||||||
|
|
||||||
|
const begin = Date.now();
|
||||||
|
const denoised = speech_denoiser.run(wave.samples, wave.sampleRate);
|
||||||
|
const elapsedSeconds = (Date.now() - begin) / 1000;
|
||||||
|
|
||||||
|
const outputFilename = './voxlock-enhanced-16k.wav';
|
||||||
|
sherpa_onnx.writeWave(outputFilename, denoised);
|
||||||
|
|
||||||
|
const duration = wave.samples.length / wave.sampleRate;
|
||||||
|
const rtf = elapsedSeconds / duration;
|
||||||
|
|
||||||
|
console.log(`Saved to ${outputFilename}`);
|
||||||
|
console.log(`Elapsed seconds: ${elapsedSeconds.toFixed(3)}`);
|
||||||
|
console.log(`Audio duration in seconds: ${duration.toFixed(3)}`);
|
||||||
|
console.log(
|
||||||
|
`RTF: ${elapsedSeconds.toFixed(3)}/${duration.toFixed(3)} = ${rtf.toFixed(3)}`);
|
||||||
|
|
||||||
|
speech_denoiser.free();
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang)
|
||||||
|
//
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example provides a voice sample of the speaker to keep, so VoxLock
|
||||||
|
// isolates that speaker from the first sample. The voice sample must have at
|
||||||
|
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
|
||||||
|
// loudest 3 seconds within its first 30 seconds. See
|
||||||
|
// ./test-online-speech-enhancement-voxlock.js if you don't have a voice
|
||||||
|
// sample.
|
||||||
|
//
|
||||||
|
// Please download the model used in this file from
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
//
|
||||||
|
// Example:
|
||||||
|
//
|
||||||
|
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
//
|
||||||
|
// This script shows how to use the streaming speech enhancement API from
|
||||||
|
// sherpa-onnx.
|
||||||
|
const sherpa_onnx = require('sherpa-onnx');
|
||||||
|
|
||||||
|
function createOnlineSpeechDenoiser() {
|
||||||
|
const config = {
|
||||||
|
model: {
|
||||||
|
voxlock: {
|
||||||
|
enrollModel: './sherpa-onnx-voxlock-int8/enroll.onnx',
|
||||||
|
chunkModel: './sherpa-onnx-voxlock-int8/chunk_int8.onnx',
|
||||||
|
voiceSample: './sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav',
|
||||||
|
},
|
||||||
|
debug: 1,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
return sherpa_onnx.createOnlineSpeechDenoiser(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
const speech_denoiser = createOnlineSpeechDenoiser();
|
||||||
|
|
||||||
|
const waveFilename = './sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav';
|
||||||
|
const wave = sherpa_onnx.readWave(waveFilename);
|
||||||
|
const frameShift = speech_denoiser.frameShiftInSamples;
|
||||||
|
const output = [];
|
||||||
|
|
||||||
|
const begin = Date.now();
|
||||||
|
|
||||||
|
let start = 0;
|
||||||
|
while (start < wave.samples.length) {
|
||||||
|
const end = Math.min(start + frameShift, wave.samples.length);
|
||||||
|
const chunk = wave.samples.slice(start, end);
|
||||||
|
const denoised = speech_denoiser.run(chunk, wave.sampleRate);
|
||||||
|
output.push(...denoised.samples);
|
||||||
|
start = end;
|
||||||
|
}
|
||||||
|
|
||||||
|
output.push(...speech_denoiser.flush().samples);
|
||||||
|
|
||||||
|
const elapsedSeconds = (Date.now() - begin) / 1000;
|
||||||
|
|
||||||
|
const outputFilename = './voxlock-enhanced-online-with-voice-sample-16k.wav';
|
||||||
|
sherpa_onnx.writeWave(outputFilename, {
|
||||||
|
samples: Float32Array.from(output),
|
||||||
|
sampleRate: speech_denoiser.sampleRate,
|
||||||
|
});
|
||||||
|
|
||||||
|
const duration = wave.samples.length / wave.sampleRate;
|
||||||
|
const rtf = elapsedSeconds / duration;
|
||||||
|
|
||||||
|
console.log(`Saved to ${outputFilename}`);
|
||||||
|
console.log(`Elapsed seconds: ${elapsedSeconds.toFixed(3)}`);
|
||||||
|
console.log(`Audio duration in seconds: ${duration.toFixed(3)}`);
|
||||||
|
console.log(
|
||||||
|
`RTF: ${elapsedSeconds.toFixed(3)}/${duration.toFixed(3)} = ${rtf.toFixed(3)}`);
|
||||||
|
|
||||||
|
speech_denoiser.free();
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang)
|
||||||
|
//
|
||||||
|
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||||
|
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||||
|
//
|
||||||
|
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||||
|
//
|
||||||
|
// This example does not provide a voice sample, so VoxLock passes the audio
|
||||||
|
// through unchanged until it has heard 3 seconds of speech, takes that speech
|
||||||
|
// as the speaker to keep, and isolates from then on. See
|
||||||
|
// ./test-online-speech-enhancement-voxlock-with-voice-sample.js if you have a
|
||||||
|
// voice sample of the speaker.
|
||||||
|
//
|
||||||
|
// Please download the model used in this file from
|
||||||
|
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||||
|
//
|
||||||
|
// Example:
|
||||||
|
//
|
||||||
|
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||||
|
//
|
||||||
|
// This script shows how to use the streaming speech enhancement API from
|
||||||
|
// sherpa-onnx.
|
||||||
|
const sherpa_onnx = require('sherpa-onnx');
|
||||||
|
|
||||||
|
function createOnlineSpeechDenoiser() {
|
||||||
|
const config = {
|
||||||
|
model: {
|
||||||
|
voxlock: {
|
||||||
|
enrollModel: './sherpa-onnx-voxlock-int8/enroll.onnx',
|
||||||
|
chunkModel: './sherpa-onnx-voxlock-int8/chunk_int8.onnx',
|
||||||
|
},
|
||||||
|
debug: 1,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
return sherpa_onnx.createOnlineSpeechDenoiser(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
const speech_denoiser = createOnlineSpeechDenoiser();
|
||||||
|
|
||||||
|
const waveFilename = './sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav';
|
||||||
|
const wave = sherpa_onnx.readWave(waveFilename);
|
||||||
|
const frameShift = speech_denoiser.frameShiftInSamples;
|
||||||
|
const output = [];
|
||||||
|
|
||||||
|
const begin = Date.now();
|
||||||
|
|
||||||
|
let start = 0;
|
||||||
|
while (start < wave.samples.length) {
|
||||||
|
const end = Math.min(start + frameShift, wave.samples.length);
|
||||||
|
const chunk = wave.samples.slice(start, end);
|
||||||
|
const denoised = speech_denoiser.run(chunk, wave.sampleRate);
|
||||||
|
output.push(...denoised.samples);
|
||||||
|
start = end;
|
||||||
|
}
|
||||||
|
|
||||||
|
output.push(...speech_denoiser.flush().samples);
|
||||||
|
|
||||||
|
const elapsedSeconds = (Date.now() - begin) / 1000;
|
||||||
|
|
||||||
|
const outputFilename = './voxlock-enhanced-online-16k.wav';
|
||||||
|
sherpa_onnx.writeWave(outputFilename, {
|
||||||
|
samples: Float32Array.from(output),
|
||||||
|
sampleRate: speech_denoiser.sampleRate,
|
||||||
|
});
|
||||||
|
|
||||||
|
const duration = wave.samples.length / wave.sampleRate;
|
||||||
|
const rtf = elapsedSeconds / duration;
|
||||||
|
|
||||||
|
console.log(`Saved to ${outputFilename}`);
|
||||||
|
console.log(`Elapsed seconds: ${elapsedSeconds.toFixed(3)}`);
|
||||||
|
console.log(`Audio duration in seconds: ${duration.toFixed(3)}`);
|
||||||
|
console.log(
|
||||||
|
`RTF: ${elapsedSeconds.toFixed(3)}/${duration.toFixed(3)} = ${rtf.toFixed(3)}`);
|
||||||
|
|
||||||
|
speech_denoiser.free();
|
||||||
@@ -15,6 +15,10 @@ function freeConfig(config, Module) {
|
|||||||
freeConfig(config.dpdfnet, Module)
|
freeConfig(config.dpdfnet, Module)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ('voxlock' in config) {
|
||||||
|
freeConfig(config.voxlock, Module)
|
||||||
|
}
|
||||||
|
|
||||||
Module._free(config.ptr);
|
Module._free(config.ptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,6 +73,43 @@ function initSherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig(config, Module) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function initSherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(config, Module) {
|
||||||
|
const enrollModelLen = Module.lengthBytesUTF8(config.enrollModel || '') + 1;
|
||||||
|
const chunkModelLen = Module.lengthBytesUTF8(config.chunkModel || '') + 1;
|
||||||
|
const voiceSampleLen = Module.lengthBytesUTF8(config.voiceSample || '') + 1;
|
||||||
|
|
||||||
|
const n = enrollModelLen + chunkModelLen + voiceSampleLen;
|
||||||
|
|
||||||
|
const buffer = Module._malloc(n);
|
||||||
|
|
||||||
|
const len = 3 * 4; // 3 pointers
|
||||||
|
const ptr = Module._malloc(len);
|
||||||
|
|
||||||
|
let offset = 0;
|
||||||
|
Module.stringToUTF8(config.enrollModel || '', buffer + offset, enrollModelLen);
|
||||||
|
offset += enrollModelLen;
|
||||||
|
|
||||||
|
Module.stringToUTF8(config.chunkModel || '', buffer + offset, chunkModelLen);
|
||||||
|
offset += chunkModelLen;
|
||||||
|
|
||||||
|
Module.stringToUTF8(config.voiceSample || '', buffer + offset, voiceSampleLen);
|
||||||
|
|
||||||
|
offset = 0;
|
||||||
|
Module.setValue(ptr, buffer + offset, 'i8*');
|
||||||
|
offset += enrollModelLen;
|
||||||
|
|
||||||
|
Module.setValue(ptr + 4, buffer + offset, 'i8*');
|
||||||
|
offset += chunkModelLen;
|
||||||
|
|
||||||
|
Module.setValue(ptr + 8, buffer + offset, 'i8*');
|
||||||
|
|
||||||
|
return {
|
||||||
|
buffer: buffer,
|
||||||
|
ptr: ptr,
|
||||||
|
len: len,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function initSherpaOnnxOfflineSpeechDenoiserModelConfig(config, Module) {
|
function initSherpaOnnxOfflineSpeechDenoiserModelConfig(config, Module) {
|
||||||
if (!('gtcrn' in config)) {
|
if (!('gtcrn' in config)) {
|
||||||
config.gtcrn = {model: ''};
|
config.gtcrn = {model: ''};
|
||||||
@@ -78,13 +119,20 @@ function initSherpaOnnxOfflineSpeechDenoiserModelConfig(config, Module) {
|
|||||||
config.dpdfnet = {model: ''};
|
config.dpdfnet = {model: ''};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!('voxlock' in config)) {
|
||||||
|
config.voxlock = {enrollModel: '', chunkModel: '', voiceSample: ''};
|
||||||
|
}
|
||||||
|
|
||||||
const gtcrn =
|
const gtcrn =
|
||||||
initSherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig(config.gtcrn, Module);
|
initSherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig(config.gtcrn, Module);
|
||||||
const dpdfnet =
|
const dpdfnet =
|
||||||
initSherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig(
|
initSherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig(
|
||||||
config.dpdfnet, Module);
|
config.dpdfnet, Module);
|
||||||
|
const voxlock =
|
||||||
|
initSherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(
|
||||||
|
config.voxlock, Module);
|
||||||
|
|
||||||
const len = gtcrn.len + 3 * 4 + dpdfnet.len;
|
const len = gtcrn.len + 3 * 4 + dpdfnet.len + voxlock.len;
|
||||||
const ptr = Module._malloc(len);
|
const ptr = Module._malloc(len);
|
||||||
|
|
||||||
let offset = 0;
|
let offset = 0;
|
||||||
@@ -106,12 +154,16 @@ function initSherpaOnnxOfflineSpeechDenoiserModelConfig(config, Module) {
|
|||||||
Module._CopyHeap(dpdfnet.ptr, dpdfnet.len, ptr + offset);
|
Module._CopyHeap(dpdfnet.ptr, dpdfnet.len, ptr + offset);
|
||||||
offset += dpdfnet.len;
|
offset += dpdfnet.len;
|
||||||
|
|
||||||
|
Module._CopyHeap(voxlock.ptr, voxlock.len, ptr + offset);
|
||||||
|
offset += voxlock.len;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
buffer: buffer,
|
buffer: buffer,
|
||||||
ptr: ptr,
|
ptr: ptr,
|
||||||
len: len,
|
len: len,
|
||||||
gtcrn: gtcrn,
|
gtcrn: gtcrn,
|
||||||
dpdfnet: dpdfnet,
|
dpdfnet: dpdfnet,
|
||||||
|
voxlock: voxlock,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,6 +172,7 @@ function initSherpaOnnxOfflineSpeechDenoiserConfig(config, Module) {
|
|||||||
config.model = {
|
config.model = {
|
||||||
gtcrn: {model: ''},
|
gtcrn: {model: ''},
|
||||||
dpdfnet: {model: ''},
|
dpdfnet: {model: ''},
|
||||||
|
voxlock: {enrollModel: '', chunkModel: '', voiceSample: ''},
|
||||||
provider: 'cpu',
|
provider: 'cpu',
|
||||||
debug: 1,
|
debug: 1,
|
||||||
numThreads: 1,
|
numThreads: 1,
|
||||||
@@ -147,6 +200,7 @@ function initSherpaOnnxOnlineSpeechDenoiserConfig(config, Module) {
|
|||||||
config.model = {
|
config.model = {
|
||||||
gtcrn: {model: ''},
|
gtcrn: {model: ''},
|
||||||
dpdfnet: {model: ''},
|
dpdfnet: {model: ''},
|
||||||
|
voxlock: {enrollModel: '', chunkModel: '', voiceSample: ''},
|
||||||
provider: 'cpu',
|
provider: 'cpu',
|
||||||
debug: 1,
|
debug: 1,
|
||||||
numThreads: 1,
|
numThreads: 1,
|
||||||
|
|||||||
@@ -18,10 +18,14 @@ static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig) == 1 * 4,
|
|||||||
static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig) ==
|
static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig) ==
|
||||||
2 * 4,
|
2 * 4,
|
||||||
"");
|
"");
|
||||||
|
static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig) ==
|
||||||
|
3 * 4,
|
||||||
|
"");
|
||||||
static_assert(
|
static_assert(
|
||||||
sizeof(SherpaOnnxOfflineSpeechDenoiserModelConfig) ==
|
sizeof(SherpaOnnxOfflineSpeechDenoiserModelConfig) ==
|
||||||
sizeof(SherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig) +
|
sizeof(SherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig) +
|
||||||
sizeof(SherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig) + 3 * 4,
|
sizeof(SherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig) +
|
||||||
|
sizeof(SherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig) + 3 * 4,
|
||||||
"");
|
"");
|
||||||
static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserConfig) ==
|
static_assert(sizeof(SherpaOnnxOfflineSpeechDenoiserConfig) ==
|
||||||
sizeof(SherpaOnnxOfflineSpeechDenoiserModelConfig),
|
sizeof(SherpaOnnxOfflineSpeechDenoiserModelConfig),
|
||||||
@@ -31,9 +35,13 @@ void MyPrint(SherpaOnnxOfflineSpeechDenoiserConfig *config) {
|
|||||||
auto model = &config->model;
|
auto model = &config->model;
|
||||||
auto gtcrn = &model->gtcrn;
|
auto gtcrn = &model->gtcrn;
|
||||||
auto dpdfnet = &model->dpdfnet;
|
auto dpdfnet = &model->dpdfnet;
|
||||||
|
auto voxlock = &model->voxlock;
|
||||||
fprintf(stdout, "----------offline speech denoiser model config----------\n");
|
fprintf(stdout, "----------offline speech denoiser model config----------\n");
|
||||||
fprintf(stdout, "gtcrn: %s\n", gtcrn->model);
|
fprintf(stdout, "gtcrn: %s\n", gtcrn->model);
|
||||||
fprintf(stdout, "dpdfnet: %s\n", dpdfnet->model);
|
fprintf(stdout, "dpdfnet: %s\n", dpdfnet->model);
|
||||||
|
fprintf(stdout, "voxlock enroll model: %s\n", voxlock->enroll_model);
|
||||||
|
fprintf(stdout, "voxlock chunk model: %s\n", voxlock->chunk_model);
|
||||||
|
fprintf(stdout, "voxlock voice sample: %s\n", voxlock->voice_sample);
|
||||||
fprintf(stdout, "num threads: %d\n", model->num_threads);
|
fprintf(stdout, "num threads: %d\n", model->num_threads);
|
||||||
fprintf(stdout, "debug: %d\n", model->debug);
|
fprintf(stdout, "debug: %d\n", model->debug);
|
||||||
fprintf(stdout, "provider: %s\n", model->provider);
|
fprintf(stdout, "provider: %s\n", model->provider);
|
||||||
|
|||||||
Reference in new issue
Block a user