mirror of
https://github.com/k2-fsa/sherpa-onnx.git
synced 2026-11-10 21:11:03 +08:00
Add Python API for VoxLock (#4102)
This commit is contained in:
1 parent
5a26343944
commit
5f97192eb9
11 files changed
+602
-12
No files matched your search
@@ -14,6 +14,34 @@ source "$SCRIPT_DIR/common.sh"
|
||||
log "test version"
|
||||
python3 ./python-api-examples/version-test.py
|
||||
|
||||
log "test offline speech enhancement (VoxLock)"
|
||||
|
||||
download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
|
||||
python3 ./python-api-examples/offline-speech-enhancement-voxlock.py
|
||||
mkdir -p offline-speech-enhancement-voxlock
|
||||
mv voxlock-enhanced-16k.wav offline-speech-enhancement-voxlock/
|
||||
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav offline-speech-enhancement-voxlock/
|
||||
|
||||
python3 ./python-api-examples/offline-speech-enhancement-voxlock-with-voice-sample.py
|
||||
mkdir -p offline-speech-enhancement-voxlock-with-voice-sample
|
||||
mv voxlock-enhanced-with-voice-sample-16k.wav offline-speech-enhancement-voxlock-with-voice-sample/
|
||||
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav offline-speech-enhancement-voxlock-with-voice-sample/
|
||||
|
||||
python3 ./python-api-examples/online-speech-enhancement-voxlock.py
|
||||
mkdir -p online-speech-enhancement-voxlock
|
||||
mv voxlock-enhanced-online-16k.wav online-speech-enhancement-voxlock/
|
||||
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav online-speech-enhancement-voxlock/
|
||||
|
||||
python3 ./python-api-examples/online-speech-enhancement-voxlock-with-voice-sample.py
|
||||
mkdir -p online-speech-enhancement-voxlock-with-voice-sample
|
||||
mv voxlock-enhanced-online-with-voice-sample-16k.wav online-speech-enhancement-voxlock-with-voice-sample/
|
||||
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav online-speech-enhancement-voxlock-with-voice-sample/
|
||||
|
||||
ls -lh offline-speech-enhancement-voxlock offline-speech-enhancement-voxlock-with-voice-sample online-speech-enhancement-voxlock online-speech-enhancement-voxlock-with-voice-sample
|
||||
|
||||
rm -rf sherpa-onnx-voxlock-int8
|
||||
|
||||
log "test Cohere Transcribe"
|
||||
|
||||
download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-cohere-transcribe-14-lang-int8-2026-04-01.tar.bz2
|
||||
|
||||
@@ -228,7 +228,7 @@ jobs:
|
||||
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
|
||||
fi
|
||||
cmake --version
|
||||
export SHERPA_ONNX_CMAKE_ARGS="-G Ninja -DCMAKE_BUILD_TYPE=Release"
|
||||
export SHERPA_ONNX_CMAKE_ARGS="-G Ninja -DCMAKE_BUILD_TYPE=Release -DSHERPA_ONNX_ENABLE_BINARY=OFF"
|
||||
export SHERPA_ONNX_MAKE_ARGS="-j 6"
|
||||
|
||||
python3 setup.py bdist_wheel
|
||||
@@ -255,6 +255,7 @@ jobs:
|
||||
export CMAKE_CXX_COMPILER_LAUNCHER=ccache
|
||||
export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
|
||||
cmake --version
|
||||
export SHERPA_ONNX_CMAKE_ARGS="-DSHERPA_ONNX_ENABLE_BINARY=OFF"
|
||||
|
||||
python3 -m pip install .
|
||||
|
||||
@@ -266,6 +267,7 @@ jobs:
|
||||
export CMAKE_C_COMPILER_LAUNCHER=sccache
|
||||
export CMAKE_CXX_COMPILER_LAUNCHER=sccache
|
||||
fi
|
||||
export SHERPA_ONNX_CMAKE_ARGS="-DSHERPA_ONNX_ENABLE_BINARY=OFF"
|
||||
|
||||
python3 -m pip install .
|
||||
|
||||
@@ -291,10 +293,6 @@ jobs:
|
||||
unzip ../*.whl
|
||||
readelf -d sherpa_onnx/lib/_sherpa_onnx*.so
|
||||
|
||||
echo "----"
|
||||
|
||||
readelf -d sherpa_onnx-*.data/data/bin/sherpa-onnx
|
||||
|
||||
- name: Show dependencies - patched (Linux)
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
@@ -305,10 +303,6 @@ jobs:
|
||||
unzip ../*.whl
|
||||
readelf -d sherpa_onnx/lib/_sherpa_onnx*.so
|
||||
|
||||
echo "----"
|
||||
|
||||
readelf -d sherpa_onnx-*.data/data/bin/sherpa-onnx
|
||||
|
||||
- name: Test sherpa-onnx (Linux)
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
@@ -346,3 +340,29 @@ jobs:
|
||||
with:
|
||||
name: tts-generated-test-files-${{ matrix.os }}-${{ matrix.python-version }}
|
||||
path: tts
|
||||
|
||||
# The VoxLock examples run first in test-python.sh; use if: always() so
|
||||
# their input/output wavs are uploaded even when a later test fails.
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: offline-speech-enhancement-voxlock-${{ matrix.os }}-${{ matrix.python-version }}
|
||||
path: offline-speech-enhancement-voxlock
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: offline-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}-${{ matrix.python-version }}
|
||||
path: offline-speech-enhancement-voxlock-with-voice-sample
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: online-speech-enhancement-voxlock-${{ matrix.os }}-${{ matrix.python-version }}
|
||||
path: online-speech-enhancement-voxlock
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: online-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}-${{ matrix.python-version }}
|
||||
path: online-speech-enhancement-voxlock-with-voice-sample
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
This file shows how to use the speech enhancement API with VoxLock and a voice
|
||||
sample.
|
||||
|
||||
VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||
speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||
|
||||
The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
|
||||
This script provides a voice sample of the speaker to keep, so VoxLock isolates
|
||||
that speaker from the first sample. The voice sample must have at least 3
|
||||
seconds of the speaker's voice; from a longer file, VoxLock uses the loudest
|
||||
3 seconds within its first 30 seconds.
|
||||
|
||||
If you don't have a voice sample, please see
|
||||
./offline-speech-enhancement-voxlock.py instead.
|
||||
|
||||
Please download files used in this script from
|
||||
https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||
|
||||
Example:
|
||||
|
||||
wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
"""
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
import soundfile as sf
|
||||
|
||||
|
||||
def create_speech_denoiser():
|
||||
enroll_model = "./sherpa-onnx-voxlock-int8/enroll.onnx"
|
||||
chunk_model = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx"
|
||||
voice_sample = "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav"
|
||||
if not Path(enroll_model).is_file() or not Path(chunk_model).is_file():
|
||||
raise ValueError(
|
||||
"Please first download the models from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
if not Path(voice_sample).is_file():
|
||||
raise ValueError(
|
||||
f"{voice_sample} does not exist. You can download it from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
|
||||
config = sherpa_onnx.OfflineSpeechDenoiserConfig(
|
||||
model=sherpa_onnx.OfflineSpeechDenoiserModelConfig(
|
||||
voxlock=sherpa_onnx.OfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enroll_model=enroll_model,
|
||||
chunk_model=chunk_model,
|
||||
voice_sample=voice_sample,
|
||||
),
|
||||
debug=False,
|
||||
num_threads=1,
|
||||
provider="cpu",
|
||||
)
|
||||
)
|
||||
if not config.validate():
|
||||
print(config)
|
||||
raise ValueError("Errors in config. Please check previous error logs")
|
||||
return sherpa_onnx.OfflineSpeechDenoiser(config)
|
||||
|
||||
|
||||
def load_audio(filename: str) -> Tuple[np.ndarray, int]:
|
||||
data, sample_rate = sf.read(
|
||||
filename,
|
||||
always_2d=True,
|
||||
dtype="float32",
|
||||
)
|
||||
data = data[:, 0] # use only the first channel
|
||||
samples = np.ascontiguousarray(data)
|
||||
return samples, sample_rate
|
||||
|
||||
|
||||
def main():
|
||||
sd = create_speech_denoiser()
|
||||
test_wave = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"
|
||||
if not Path(test_wave).is_file():
|
||||
raise ValueError(
|
||||
f"{test_wave} does not exist. You can download it from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
|
||||
samples, sample_rate = load_audio(test_wave)
|
||||
|
||||
start = time.time()
|
||||
denoised = sd(samples, sample_rate)
|
||||
end = time.time()
|
||||
|
||||
elapsed_seconds = end - start
|
||||
audio_duration = len(samples) / sample_rate
|
||||
real_time_factor = elapsed_seconds / audio_duration
|
||||
|
||||
sf.write(
|
||||
"./voxlock-enhanced-with-voice-sample-16k.wav",
|
||||
denoised.samples,
|
||||
denoised.sample_rate,
|
||||
)
|
||||
print("Saved to ./voxlock-enhanced-with-voice-sample-16k.wav")
|
||||
print(f"Elapsed seconds: {elapsed_seconds:.3f}")
|
||||
print(f"Audio duration in seconds: {audio_duration:.3f}")
|
||||
print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {real_time_factor:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
This file shows how to use the speech enhancement API with VoxLock.
|
||||
|
||||
VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||
speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||
|
||||
The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
|
||||
This script does not provide a voice sample, so VoxLock takes the loudest
|
||||
3 seconds in the first 15 seconds of the input as the speaker to keep.
|
||||
|
||||
If you have a voice sample of the speaker, please see
|
||||
./offline-speech-enhancement-voxlock-with-voice-sample.py instead.
|
||||
|
||||
Please download files used in this script from
|
||||
https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||
|
||||
Example:
|
||||
|
||||
wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
"""
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
import soundfile as sf
|
||||
|
||||
|
||||
def create_speech_denoiser():
|
||||
enroll_model = "./sherpa-onnx-voxlock-int8/enroll.onnx"
|
||||
chunk_model = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx"
|
||||
if not Path(enroll_model).is_file() or not Path(chunk_model).is_file():
|
||||
raise ValueError(
|
||||
"Please first download the models from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
|
||||
config = sherpa_onnx.OfflineSpeechDenoiserConfig(
|
||||
model=sherpa_onnx.OfflineSpeechDenoiserModelConfig(
|
||||
voxlock=sherpa_onnx.OfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enroll_model=enroll_model,
|
||||
chunk_model=chunk_model,
|
||||
voice_sample="",
|
||||
),
|
||||
debug=False,
|
||||
num_threads=1,
|
||||
provider="cpu",
|
||||
)
|
||||
)
|
||||
if not config.validate():
|
||||
print(config)
|
||||
raise ValueError("Errors in config. Please check previous error logs")
|
||||
return sherpa_onnx.OfflineSpeechDenoiser(config)
|
||||
|
||||
|
||||
def load_audio(filename: str) -> Tuple[np.ndarray, int]:
|
||||
data, sample_rate = sf.read(
|
||||
filename,
|
||||
always_2d=True,
|
||||
dtype="float32",
|
||||
)
|
||||
data = data[:, 0] # use only the first channel
|
||||
samples = np.ascontiguousarray(data)
|
||||
return samples, sample_rate
|
||||
|
||||
|
||||
def main():
|
||||
sd = create_speech_denoiser()
|
||||
test_wave = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"
|
||||
if not Path(test_wave).is_file():
|
||||
raise ValueError(
|
||||
f"{test_wave} does not exist. You can download it from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
|
||||
samples, sample_rate = load_audio(test_wave)
|
||||
|
||||
start = time.time()
|
||||
denoised = sd(samples, sample_rate)
|
||||
end = time.time()
|
||||
|
||||
elapsed_seconds = end - start
|
||||
audio_duration = len(samples) / sample_rate
|
||||
real_time_factor = elapsed_seconds / audio_duration
|
||||
|
||||
sf.write("./voxlock-enhanced-16k.wav", denoised.samples, denoised.sample_rate)
|
||||
print("Saved to ./voxlock-enhanced-16k.wav")
|
||||
print(f"Elapsed seconds: {elapsed_seconds:.3f}")
|
||||
print(f"Audio duration in seconds: {audio_duration:.3f}")
|
||||
print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {real_time_factor:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
This file shows how to use the online speech enhancement API with VoxLock and a
|
||||
voice sample.
|
||||
|
||||
VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||
speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||
|
||||
The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
|
||||
This script provides a voice sample of the speaker to keep, so VoxLock isolates
|
||||
that speaker from the first sample. The voice sample must have at least 3
|
||||
seconds of the speaker's voice; from a longer file, VoxLock uses the loudest
|
||||
3 seconds within its first 30 seconds.
|
||||
|
||||
If you don't have a voice sample, please see
|
||||
./online-speech-enhancement-voxlock.py instead.
|
||||
|
||||
Please download files used in this script from
|
||||
https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||
|
||||
Example:
|
||||
|
||||
wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
"""
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
import soundfile as sf
|
||||
|
||||
|
||||
def create_speech_denoiser():
|
||||
enroll_model = "./sherpa-onnx-voxlock-int8/enroll.onnx"
|
||||
chunk_model = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx"
|
||||
voice_sample = "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav"
|
||||
if not Path(enroll_model).is_file() or not Path(chunk_model).is_file():
|
||||
raise ValueError(
|
||||
"Please first download the models from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
if not Path(voice_sample).is_file():
|
||||
raise ValueError(
|
||||
f"{voice_sample} does not exist. You can download it from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
|
||||
config = sherpa_onnx.OnlineSpeechDenoiserConfig(
|
||||
model=sherpa_onnx.OfflineSpeechDenoiserModelConfig(
|
||||
voxlock=sherpa_onnx.OfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enroll_model=enroll_model,
|
||||
chunk_model=chunk_model,
|
||||
voice_sample=voice_sample,
|
||||
),
|
||||
debug=False,
|
||||
num_threads=1,
|
||||
provider="cpu",
|
||||
)
|
||||
)
|
||||
|
||||
if not config.validate():
|
||||
print(config)
|
||||
raise ValueError("Errors in config. Please check previous error logs")
|
||||
|
||||
return sherpa_onnx.OnlineSpeechDenoiser(config)
|
||||
|
||||
|
||||
def load_audio(filename: str):
|
||||
data, sample_rate = sf.read(filename, always_2d=True, dtype="float32")
|
||||
samples = np.ascontiguousarray(data[:, 0])
|
||||
return samples, sample_rate
|
||||
|
||||
|
||||
def main():
|
||||
sd = create_speech_denoiser()
|
||||
test_wave = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"
|
||||
if not Path(test_wave).is_file():
|
||||
raise ValueError(
|
||||
f"{test_wave} does not exist. You can download it from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
|
||||
samples, sample_rate = load_audio(test_wave)
|
||||
frame_shift = sd.frame_shift_in_samples
|
||||
output = []
|
||||
|
||||
start = time.time()
|
||||
for offset in range(0, len(samples), frame_shift):
|
||||
chunk = samples[offset : offset + frame_shift]
|
||||
denoised = sd(chunk, sample_rate)
|
||||
output.append(np.asarray(denoised.samples, dtype=np.float32))
|
||||
|
||||
output.append(np.asarray(sd.flush().samples, dtype=np.float32))
|
||||
end = time.time()
|
||||
|
||||
elapsed_seconds = end - start
|
||||
audio_duration = len(samples) / sample_rate
|
||||
real_time_factor = elapsed_seconds / audio_duration
|
||||
|
||||
enhanced = np.concatenate(output) if output else np.empty(0, dtype=np.float32)
|
||||
|
||||
sf.write(
|
||||
"./voxlock-enhanced-online-with-voice-sample-16k.wav",
|
||||
enhanced,
|
||||
sd.sample_rate,
|
||||
)
|
||||
print("Saved to ./voxlock-enhanced-online-with-voice-sample-16k.wav")
|
||||
print(f"Elapsed seconds: {elapsed_seconds:.3f}")
|
||||
print(f"Audio duration in seconds: {audio_duration:.3f}")
|
||||
print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {real_time_factor:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
This file shows how to use the online speech enhancement API with VoxLock.
|
||||
|
||||
VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||
speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||
|
||||
The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
|
||||
This script does not provide a voice sample, so VoxLock passes the audio
|
||||
through unchanged until it has heard 3 seconds of speech, takes that speech
|
||||
as the speaker to keep, and isolates from then on.
|
||||
|
||||
If you have a voice sample of the speaker, please see
|
||||
./online-speech-enhancement-voxlock-with-voice-sample.py instead.
|
||||
|
||||
Please download files used in this script from
|
||||
https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||
|
||||
Example:
|
||||
|
||||
wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
"""
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
import soundfile as sf
|
||||
|
||||
|
||||
def create_speech_denoiser():
|
||||
enroll_model = "./sherpa-onnx-voxlock-int8/enroll.onnx"
|
||||
chunk_model = "./sherpa-onnx-voxlock-int8/chunk_int8.onnx"
|
||||
if not Path(enroll_model).is_file() or not Path(chunk_model).is_file():
|
||||
raise ValueError(
|
||||
"Please first download the models from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
|
||||
config = sherpa_onnx.OnlineSpeechDenoiserConfig(
|
||||
model=sherpa_onnx.OfflineSpeechDenoiserModelConfig(
|
||||
voxlock=sherpa_onnx.OfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enroll_model=enroll_model,
|
||||
chunk_model=chunk_model,
|
||||
voice_sample="",
|
||||
),
|
||||
debug=False,
|
||||
num_threads=1,
|
||||
provider="cpu",
|
||||
)
|
||||
)
|
||||
|
||||
if not config.validate():
|
||||
print(config)
|
||||
raise ValueError("Errors in config. Please check previous error logs")
|
||||
|
||||
return sherpa_onnx.OnlineSpeechDenoiser(config)
|
||||
|
||||
|
||||
def load_audio(filename: str):
|
||||
data, sample_rate = sf.read(filename, always_2d=True, dtype="float32")
|
||||
samples = np.ascontiguousarray(data[:, 0])
|
||||
return samples, sample_rate
|
||||
|
||||
|
||||
def main():
|
||||
sd = create_speech_denoiser()
|
||||
test_wave = "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav"
|
||||
if not Path(test_wave).is_file():
|
||||
raise ValueError(
|
||||
f"{test_wave} does not exist. You can download it from "
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models"
|
||||
)
|
||||
|
||||
samples, sample_rate = load_audio(test_wave)
|
||||
frame_shift = sd.frame_shift_in_samples
|
||||
output = []
|
||||
|
||||
start = time.time()
|
||||
for offset in range(0, len(samples), frame_shift):
|
||||
chunk = samples[offset : offset + frame_shift]
|
||||
denoised = sd(chunk, sample_rate)
|
||||
output.append(np.asarray(denoised.samples, dtype=np.float32))
|
||||
|
||||
output.append(np.asarray(sd.flush().samples, dtype=np.float32))
|
||||
end = time.time()
|
||||
|
||||
elapsed_seconds = end - start
|
||||
audio_duration = len(samples) / sample_rate
|
||||
real_time_factor = elapsed_seconds / audio_duration
|
||||
|
||||
enhanced = np.concatenate(output) if output else np.empty(0, dtype=np.float32)
|
||||
|
||||
sf.write("./voxlock-enhanced-online-16k.wav", enhanced, sd.sample_rate)
|
||||
print("Saved to ./voxlock-enhanced-online-16k.wav")
|
||||
print(f"Elapsed seconds: {elapsed_seconds:.3f}")
|
||||
print(f"Audio duration in seconds: {audio_duration:.3f}")
|
||||
print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {real_time_factor:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -35,6 +35,7 @@ set(srcs
|
||||
offline-speech-denoiser-dpdfnet-model-config.cc
|
||||
offline-speech-denoiser-gtcrn-model-config.cc
|
||||
offline-speech-denoiser-model-config.cc
|
||||
offline-speech-denoiser-voxlock-model-config.cc
|
||||
offline-speech-denoiser.cc
|
||||
offline-stream.cc
|
||||
offline-tdnn-model-config.cc
|
||||
|
||||
@@ -9,25 +9,40 @@
|
||||
#include "sherpa-onnx/csrc/offline-speech-denoiser-model-config.h"
|
||||
#include "sherpa-onnx/python/csrc/offline-speech-denoiser-dpdfnet-model-config.h"
|
||||
#include "sherpa-onnx/python/csrc/offline-speech-denoiser-gtcrn-model-config.h"
|
||||
#include "sherpa-onnx/python/csrc/offline-speech-denoiser-voxlock-model-config.h"
|
||||
|
||||
namespace sherpa_onnx {
|
||||
|
||||
void PybindOfflineSpeechDenoiserModelConfig(py::module *m) {
|
||||
PybindOfflineSpeechDenoiserDpdfNetModelConfig(m);
|
||||
PybindOfflineSpeechDenoiserGtcrnModelConfig(m);
|
||||
PybindOfflineSpeechDenoiserVoxLockModelConfig(m);
|
||||
|
||||
using PyClass = OfflineSpeechDenoiserModelConfig;
|
||||
py::class_<PyClass>(*m, "OfflineSpeechDenoiserModelConfig")
|
||||
.def(py::init<>())
|
||||
.def(py::init<const OfflineSpeechDenoiserGtcrnModelConfig &,
|
||||
const OfflineSpeechDenoiserDpdfNetModelConfig &, int32_t,
|
||||
bool, const std::string &>(),
|
||||
.def(py::init([](const OfflineSpeechDenoiserGtcrnModelConfig >crn,
|
||||
const OfflineSpeechDenoiserDpdfNetModelConfig &dpdfnet,
|
||||
const OfflineSpeechDenoiserVoxLockModelConfig &voxlock,
|
||||
int32_t num_threads, bool debug,
|
||||
const std::string &provider) {
|
||||
PyClass ans;
|
||||
ans.gtcrn = gtcrn;
|
||||
ans.dpdfnet = dpdfnet;
|
||||
ans.voxlock = voxlock;
|
||||
ans.num_threads = num_threads;
|
||||
ans.debug = debug;
|
||||
ans.provider = provider;
|
||||
return ans;
|
||||
}),
|
||||
py::arg("gtcrn") = OfflineSpeechDenoiserGtcrnModelConfig{},
|
||||
py::arg("dpdfnet") = OfflineSpeechDenoiserDpdfNetModelConfig{},
|
||||
py::arg("voxlock") = OfflineSpeechDenoiserVoxLockModelConfig{},
|
||||
py::arg("num_threads") = 1, py::arg("debug") = false,
|
||||
py::arg("provider") = "cpu")
|
||||
.def_readwrite("gtcrn", &PyClass::gtcrn)
|
||||
.def_readwrite("dpdfnet", &PyClass::dpdfnet)
|
||||
.def_readwrite("voxlock", &PyClass::voxlock)
|
||||
.def_readwrite("num_threads", &PyClass::num_threads)
|
||||
.def_readwrite("debug", &PyClass::debug)
|
||||
.def_readwrite("provider", &PyClass::provider)
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
// sherpa-onnx/python/csrc/offline-speech-denoiser-voxlock-model-config.cc
|
||||
//
|
||||
// Copyright (c) 2026 Xiaomi Corporation
|
||||
|
||||
#include "sherpa-onnx/python/csrc/offline-speech-denoiser-voxlock-model-config.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "sherpa-onnx/csrc/offline-speech-denoiser-voxlock-model-config.h"
|
||||
|
||||
namespace sherpa_onnx {
|
||||
|
||||
static constexpr const char *kVoxLockModelConfigDoc = R"doc(
|
||||
Configuration for the VoxLock speech denoiser.
|
||||
|
||||
VoxLock keeps the voice of one speaker (the caller) and removes the other
|
||||
voices.
|
||||
|
||||
The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
)doc";
|
||||
|
||||
static constexpr const char *kVoxLockEnrollModelDoc = R"doc(
|
||||
Path to the VoxLock enroll model, e.g., ``enroll.onnx``.
|
||||
|
||||
It runs once on a voice sample of the speaker to keep. Download the models
|
||||
from
|
||||
https://github.com/arbazsiddiqui/voxlock/releases/tag/sherpa-onnx-models
|
||||
)doc";
|
||||
|
||||
static constexpr const char *kVoxLockChunkModelDoc = R"doc(
|
||||
Path to the VoxLock chunk model, e.g., ``chunk.onnx`` or
|
||||
``chunk_int8.onnx``.
|
||||
|
||||
It runs on each chunk of the audio.
|
||||
)doc";
|
||||
|
||||
static constexpr const char *kVoxLockVoiceSampleDoc = R"doc(
|
||||
Optional. Path to a wave file with at least 3 seconds of the caller's
|
||||
voice. VoxLock keeps this voice and removes the others. From a longer
|
||||
file, it uses the loudest 3 seconds within the first 30 seconds. A file
|
||||
that is shorter or cannot be read is ignored, as if it were empty.
|
||||
|
||||
If empty or ignored, the voice sample is taken from the input audio:
|
||||
|
||||
- Offline denoiser: the loudest 3 seconds in the first 15 seconds of
|
||||
the input are used.
|
||||
- Online denoiser: the audio is passed through unchanged until 3
|
||||
seconds of speech have been heard; that speech is used as the voice
|
||||
sample.
|
||||
|
||||
Defaults to ``""``.
|
||||
)doc";
|
||||
|
||||
void PybindOfflineSpeechDenoiserVoxLockModelConfig(py::module *m) {
|
||||
using PyClass = OfflineSpeechDenoiserVoxLockModelConfig;
|
||||
py::class_<PyClass>(*m, "OfflineSpeechDenoiserVoxLockModelConfig",
|
||||
kVoxLockModelConfigDoc)
|
||||
.def(py::init<const std::string &, const std::string &,
|
||||
const std::string &>(),
|
||||
py::arg("enroll_model") = "", py::arg("chunk_model") = "",
|
||||
py::arg("voice_sample") = "")
|
||||
.def_readwrite("enroll_model", &PyClass::enroll_model,
|
||||
kVoxLockEnrollModelDoc)
|
||||
.def_readwrite("chunk_model", &PyClass::chunk_model,
|
||||
kVoxLockChunkModelDoc)
|
||||
.def_readwrite("voice_sample", &PyClass::voice_sample,
|
||||
kVoxLockVoiceSampleDoc)
|
||||
.def("validate", &PyClass::Validate)
|
||||
.def("__str__", &PyClass::ToString);
|
||||
}
|
||||
|
||||
} // namespace sherpa_onnx
|
||||
@@ -0,0 +1,16 @@
|
||||
// sherpa-onnx/python/csrc/offline-speech-denoiser-voxlock-model-config.h
|
||||
//
|
||||
// Copyright (c) 2026 Xiaomi Corporation
|
||||
|
||||
#ifndef SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SPEECH_DENOISER_VOXLOCK_MODEL_CONFIG_H_
|
||||
#define SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SPEECH_DENOISER_VOXLOCK_MODEL_CONFIG_H_
|
||||
|
||||
#include "sherpa-onnx/python/csrc/sherpa-onnx.h"
|
||||
|
||||
namespace sherpa_onnx {
|
||||
|
||||
void PybindOfflineSpeechDenoiserVoxLockModelConfig(py::module *m);
|
||||
|
||||
}
|
||||
|
||||
#endif // SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SPEECH_DENOISER_VOXLOCK_MODEL_CONFIG_H_
|
||||
@@ -47,6 +47,7 @@ from sherpa_onnx.lib._sherpa_onnx import (
|
||||
OfflineSpeechDenoiserDpdfNetModelConfig,
|
||||
OfflineSpeechDenoiserGtcrnModelConfig,
|
||||
OfflineSpeechDenoiserModelConfig,
|
||||
OfflineSpeechDenoiserVoxLockModelConfig,
|
||||
OfflineStream,
|
||||
OfflineTdnnModelConfig,
|
||||
OfflineTransducerModelConfig,
|
||||
|
||||
Reference in new issue
Block a user