mirror of
https://github.com/k2-fsa/sherpa-onnx.git
synced 2026-11-10 21:11:03 +08:00
Add Swift API for VoxLock (#4109)
This commit is contained in:
1 parent
b17952620d
commit
37ce81d5a0
11 files changed
+582
-1
No files matched your search
@@ -10,6 +10,11 @@ ls -lh
|
||||
|
||||
./run-test-version.sh
|
||||
|
||||
./run-speech-enhancement-voxlock.sh
|
||||
./run-speech-enhancement-voxlock-with-voice-sample.sh
|
||||
./run-online-speech-enhancement-voxlock.sh
|
||||
./run-online-speech-enhancement-voxlock-with-voice-sample.sh
|
||||
|
||||
./run-tts-pocket-en.sh
|
||||
ls -lh
|
||||
rm -rf sherpa-onnx-pocket-*
|
||||
|
||||
@@ -84,3 +84,28 @@ jobs:
|
||||
swift-api-examples/test-*-lexicon.wav
|
||||
swift-api-examples/generated-*.wav
|
||||
if-no-files-found: warn
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: speech-enhancement-voxlock-${{ matrix.os }}
|
||||
# each run-*.sh keeps the input and output wavs in wav/<example>
|
||||
path: swift-api-examples/wav/speech-enhancement-voxlock
|
||||
if-no-files-found: warn
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
|
||||
path: swift-api-examples/wav/speech-enhancement-voxlock-with-voice-sample
|
||||
if-no-files-found: warn
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: online-speech-enhancement-voxlock-${{ matrix.os }}
|
||||
path: swift-api-examples/wav/online-speech-enhancement-voxlock
|
||||
if-no-files-found: warn
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: online-speech-enhancement-voxlock-with-voice-sample-${{ matrix.os }}
|
||||
path: swift-api-examples/wav/online-speech-enhancement-voxlock-with-voice-sample
|
||||
if-no-files-found: warn
|
||||
@@ -2076,11 +2076,43 @@ public func sherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig(
|
||||
attenuation_limit_db: attenuationLimitDb)
|
||||
}
|
||||
|
||||
/// Return an instance of SherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig.
|
||||
///
|
||||
/// VoxLock keeps the voice of one speaker (the caller) and removes the other
|
||||
/// voices. The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - enrollModel: Path to the VoxLock enroll model, e.g., enroll.onnx. It
|
||||
/// runs once on a voice sample of the speaker to keep.
|
||||
/// - chunkModel: Path to the VoxLock chunk model, e.g., chunk.onnx or
|
||||
/// chunk_int8.onnx. It runs on each chunk of the audio.
|
||||
/// - voiceSample: Optional. Path to a wave file with at least 3 seconds of
|
||||
/// the caller's voice. VoxLock keeps this voice and removes the others.
|
||||
/// From a longer file, it uses the loudest 3 seconds within the first 30
|
||||
/// seconds. A file that is shorter or cannot be read is ignored, as if it
|
||||
/// were empty. If empty or ignored, the voice sample is taken from the
|
||||
/// input audio: the offline denoiser uses the loudest 3 seconds in the
|
||||
/// first 15 seconds of the input, and the online denoiser passes the audio
|
||||
/// through unchanged until 3 seconds of speech have been heard, then uses
|
||||
/// that speech.
|
||||
public func sherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enrollModel: String = "",
|
||||
chunkModel: String = "",
|
||||
voiceSample: String = ""
|
||||
) -> SherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig {
|
||||
return SherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enroll_model: toCPointer(enrollModel),
|
||||
chunk_model: toCPointer(chunkModel),
|
||||
voice_sample: toCPointer(voiceSample))
|
||||
}
|
||||
|
||||
public func sherpaOnnxOfflineSpeechDenoiserModelConfig(
|
||||
gtcrn: SherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig =
|
||||
sherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig(),
|
||||
dpdfnet: SherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig =
|
||||
sherpaOnnxOfflineSpeechDenoiserDpdfNetModelConfig(),
|
||||
voxlock: SherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig =
|
||||
sherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(),
|
||||
numThreads: Int = 1,
|
||||
provider: String = "cpu",
|
||||
debug: Int = 0
|
||||
@@ -2090,7 +2122,8 @@ public func sherpaOnnxOfflineSpeechDenoiserModelConfig(
|
||||
num_threads: Int32(numThreads),
|
||||
debug: Int32(debug),
|
||||
provider: toCPointer(provider),
|
||||
dpdfnet: dpdfnet
|
||||
dpdfnet: dpdfnet,
|
||||
voxlock: voxlock
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
// Copyright (c) 2026 Xiaomi Corporation
|
||||
//
|
||||
// This file shows how to use the online speech enhancement API with VoxLock
|
||||
// models and a voice sample.
|
||||
//
|
||||
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||
//
|
||||
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
//
|
||||
// This example provides a voice sample of the speaker to keep, so VoxLock
|
||||
// isolates that speaker from the first sample. The voice sample must have at
|
||||
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
|
||||
// loudest 3 seconds within its first 30 seconds. See
|
||||
// ./online-speech-enhancement-voxlock.swift if you don't have a voice sample.
|
||||
//
|
||||
// Please refer to
|
||||
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||
// to download files used in this script
|
||||
//
|
||||
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
|
||||
import AVFoundation
|
||||
|
||||
extension AudioBuffer {
|
||||
func array() -> [Float] {
|
||||
return Array(UnsafeBufferPointer(self))
|
||||
}
|
||||
}
|
||||
|
||||
extension AVAudioPCMBuffer {
|
||||
func array() -> [Float] {
|
||||
return self.audioBufferList.pointee.mBuffers.array()
|
||||
}
|
||||
}
|
||||
|
||||
func run() {
|
||||
var config = sherpaOnnxOnlineSpeechDenoiserConfig(
|
||||
model: sherpaOnnxOfflineSpeechDenoiserModelConfig(
|
||||
voxlock: sherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enrollModel: "./sherpa-onnx-voxlock-int8/enroll.onnx",
|
||||
chunkModel: "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
|
||||
voiceSample: "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav")))
|
||||
|
||||
let sd = SherpaOnnxOnlineSpeechDenoiserWrapper(config: &config)
|
||||
|
||||
let fileURL: NSURL =
|
||||
NSURL(fileURLWithPath: "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav")
|
||||
let audioFile = try! AVAudioFile(forReading: fileURL as URL)
|
||||
|
||||
let audioFormat = audioFile.processingFormat
|
||||
assert(audioFormat.sampleRate == 16000)
|
||||
assert(audioFormat.channelCount == 1)
|
||||
assert(audioFormat.commonFormat == AVAudioCommonFormat.pcmFormatFloat32)
|
||||
|
||||
let audioFrameCount = UInt32(audioFile.length)
|
||||
let audioFileBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: audioFrameCount)
|
||||
|
||||
try! audioFile.read(into: audioFileBuffer!)
|
||||
let samples: [Float]! = audioFileBuffer?.array()
|
||||
|
||||
var enhanced: [Float] = []
|
||||
let frameShift = sd.frameShiftInSamples
|
||||
|
||||
let begin = Date()
|
||||
var start = 0
|
||||
while start < samples.count {
|
||||
let end = min(start + frameShift, samples.count)
|
||||
let audio = sd.run(samples: Array(samples[start..<end]), sampleRate: Int(audioFormat.sampleRate))
|
||||
enhanced.append(contentsOf: audio.samples)
|
||||
start = end
|
||||
}
|
||||
|
||||
enhanced.append(contentsOf: sd.flush().samples)
|
||||
let elapsedSeconds = Date().timeIntervalSince(begin)
|
||||
|
||||
let filename = "voxlock-enhanced-online-with-voice-sample-16k.wav"
|
||||
_ = enhanced.withUnsafeBufferPointer { p in
|
||||
SherpaOnnxWriteWave(
|
||||
p.baseAddress,
|
||||
Int32(enhanced.count),
|
||||
Int32(sd.sampleRate),
|
||||
toCPointer(filename))
|
||||
}
|
||||
print("\nSaved to:\(filename)")
|
||||
|
||||
let duration = Double(samples.count) / audioFormat.sampleRate
|
||||
print(String(format: "Elapsed seconds: %.3f", elapsedSeconds))
|
||||
print(String(format: "Audio duration in seconds: %.3f", duration))
|
||||
print(String(format: "RTF: %.3f/%.3f = %.3f", elapsedSeconds, duration, elapsedSeconds / duration))
|
||||
}
|
||||
|
||||
@main
|
||||
struct App {
|
||||
static func main() {
|
||||
run()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// Copyright (c) 2026 Xiaomi Corporation
|
||||
//
|
||||
// This file shows how to use the online speech enhancement API with VoxLock
|
||||
// models.
|
||||
//
|
||||
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||
//
|
||||
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
//
|
||||
// This example does not provide a voice sample, so VoxLock passes the audio
|
||||
// through unchanged until it has heard 3 seconds of speech, takes that speech
|
||||
// as the speaker to keep, and isolates from then on. See
|
||||
// ./online-speech-enhancement-voxlock-with-voice-sample.swift if you have a
|
||||
// voice sample of the speaker.
|
||||
//
|
||||
// Please refer to
|
||||
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||
// to download files used in this script
|
||||
//
|
||||
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
|
||||
import AVFoundation
|
||||
|
||||
extension AudioBuffer {
|
||||
func array() -> [Float] {
|
||||
return Array(UnsafeBufferPointer(self))
|
||||
}
|
||||
}
|
||||
|
||||
extension AVAudioPCMBuffer {
|
||||
func array() -> [Float] {
|
||||
return self.audioBufferList.pointee.mBuffers.array()
|
||||
}
|
||||
}
|
||||
|
||||
func run() {
|
||||
var config = sherpaOnnxOnlineSpeechDenoiserConfig(
|
||||
model: sherpaOnnxOfflineSpeechDenoiserModelConfig(
|
||||
voxlock: sherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enrollModel: "./sherpa-onnx-voxlock-int8/enroll.onnx",
|
||||
chunkModel: "./sherpa-onnx-voxlock-int8/chunk_int8.onnx")))
|
||||
|
||||
let sd = SherpaOnnxOnlineSpeechDenoiserWrapper(config: &config)
|
||||
|
||||
let fileURL: NSURL =
|
||||
NSURL(fileURLWithPath: "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav")
|
||||
let audioFile = try! AVAudioFile(forReading: fileURL as URL)
|
||||
|
||||
let audioFormat = audioFile.processingFormat
|
||||
assert(audioFormat.sampleRate == 16000)
|
||||
assert(audioFormat.channelCount == 1)
|
||||
assert(audioFormat.commonFormat == AVAudioCommonFormat.pcmFormatFloat32)
|
||||
|
||||
let audioFrameCount = UInt32(audioFile.length)
|
||||
let audioFileBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: audioFrameCount)
|
||||
|
||||
try! audioFile.read(into: audioFileBuffer!)
|
||||
let samples: [Float]! = audioFileBuffer?.array()
|
||||
|
||||
var enhanced: [Float] = []
|
||||
let frameShift = sd.frameShiftInSamples
|
||||
|
||||
let begin = Date()
|
||||
var start = 0
|
||||
while start < samples.count {
|
||||
let end = min(start + frameShift, samples.count)
|
||||
let audio = sd.run(samples: Array(samples[start..<end]), sampleRate: Int(audioFormat.sampleRate))
|
||||
enhanced.append(contentsOf: audio.samples)
|
||||
start = end
|
||||
}
|
||||
|
||||
enhanced.append(contentsOf: sd.flush().samples)
|
||||
let elapsedSeconds = Date().timeIntervalSince(begin)
|
||||
|
||||
let filename = "voxlock-enhanced-online-16k.wav"
|
||||
_ = enhanced.withUnsafeBufferPointer { p in
|
||||
SherpaOnnxWriteWave(
|
||||
p.baseAddress,
|
||||
Int32(enhanced.count),
|
||||
Int32(sd.sampleRate),
|
||||
toCPointer(filename))
|
||||
}
|
||||
print("\nSaved to:\(filename)")
|
||||
|
||||
let duration = Double(samples.count) / audioFormat.sampleRate
|
||||
print(String(format: "Elapsed seconds: %.3f", elapsedSeconds))
|
||||
print(String(format: "Audio duration in seconds: %.3f", duration))
|
||||
print(String(format: "RTF: %.3f/%.3f = %.3f", elapsedSeconds, duration, elapsedSeconds / duration))
|
||||
}
|
||||
|
||||
@main
|
||||
struct App {
|
||||
static func main() {
|
||||
run()
|
||||
}
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -ex
|
||||
|
||||
if [ ! -d ../build-macos ]; then
|
||||
echo "Please run ../build-macos.sh first!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||
fi
|
||||
|
||||
if [ ! -e ./online-speech-enhancement-voxlock-with-voice-sample ]; then
|
||||
# Note: We use -lc++ to link against libc++ instead of libstdc++
|
||||
swiftc \
|
||||
-lc++ \
|
||||
-I ../build-macos/install/include \
|
||||
-import-objc-header ./SherpaOnnx-Bridging-Header.h \
|
||||
./online-speech-enhancement-voxlock-with-voice-sample.swift ./SherpaOnnx.swift \
|
||||
-L ../build-macos/install/lib/ \
|
||||
-l sherpa-onnx-c-api \
|
||||
-l onnxruntime \
|
||||
-o online-speech-enhancement-voxlock-with-voice-sample
|
||||
|
||||
strip online-speech-enhancement-voxlock-with-voice-sample
|
||||
else
|
||||
echo "./online-speech-enhancement-voxlock-with-voice-sample exists - skip building"
|
||||
fi
|
||||
|
||||
export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH
|
||||
./online-speech-enhancement-voxlock-with-voice-sample
|
||||
|
||||
mkdir -p wav/online-speech-enhancement-voxlock-with-voice-sample
|
||||
mv voxlock-enhanced-online-with-voice-sample-16k.wav wav/online-speech-enhancement-voxlock-with-voice-sample/
|
||||
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav wav/online-speech-enhancement-voxlock-with-voice-sample/
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -ex
|
||||
|
||||
if [ ! -d ../build-macos ]; then
|
||||
echo "Please run ../build-macos.sh first!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||
fi
|
||||
|
||||
if [ ! -e ./online-speech-enhancement-voxlock ]; then
|
||||
# Note: We use -lc++ to link against libc++ instead of libstdc++
|
||||
swiftc \
|
||||
-lc++ \
|
||||
-I ../build-macos/install/include \
|
||||
-import-objc-header ./SherpaOnnx-Bridging-Header.h \
|
||||
./online-speech-enhancement-voxlock.swift ./SherpaOnnx.swift \
|
||||
-L ../build-macos/install/lib/ \
|
||||
-l sherpa-onnx-c-api \
|
||||
-l onnxruntime \
|
||||
-o online-speech-enhancement-voxlock
|
||||
|
||||
strip online-speech-enhancement-voxlock
|
||||
else
|
||||
echo "./online-speech-enhancement-voxlock exists - skip building"
|
||||
fi
|
||||
|
||||
export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH
|
||||
./online-speech-enhancement-voxlock
|
||||
|
||||
mkdir -p wav/online-speech-enhancement-voxlock
|
||||
mv voxlock-enhanced-online-16k.wav wav/online-speech-enhancement-voxlock/
|
||||
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav wav/online-speech-enhancement-voxlock/
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -ex
|
||||
|
||||
if [ ! -d ../build-macos ]; then
|
||||
echo "Please run ../build-macos.sh first!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||
fi
|
||||
|
||||
if [ ! -e ./speech-enhancement-voxlock-with-voice-sample ]; then
|
||||
# Note: We use -lc++ to link against libc++ instead of libstdc++
|
||||
swiftc \
|
||||
-lc++ \
|
||||
-I ../build-macos/install/include \
|
||||
-import-objc-header ./SherpaOnnx-Bridging-Header.h \
|
||||
./speech-enhancement-voxlock-with-voice-sample.swift ./SherpaOnnx.swift \
|
||||
-L ../build-macos/install/lib/ \
|
||||
-l sherpa-onnx-c-api \
|
||||
-l onnxruntime \
|
||||
-o speech-enhancement-voxlock-with-voice-sample
|
||||
|
||||
strip speech-enhancement-voxlock-with-voice-sample
|
||||
else
|
||||
echo "./speech-enhancement-voxlock-with-voice-sample exists - skip building"
|
||||
fi
|
||||
|
||||
export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH
|
||||
./speech-enhancement-voxlock-with-voice-sample
|
||||
|
||||
mkdir -p wav/speech-enhancement-voxlock-with-voice-sample
|
||||
mv voxlock-enhanced-with-voice-sample-16k.wav wav/speech-enhancement-voxlock-with-voice-sample/
|
||||
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav wav/speech-enhancement-voxlock-with-voice-sample/
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -ex
|
||||
|
||||
if [ ! -d ../build-macos ]; then
|
||||
echo "Please run ../build-macos.sh first!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d ./sherpa-onnx-voxlock-int8 ]; then
|
||||
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
rm sherpa-onnx-voxlock-int8.tar.bz2
|
||||
fi
|
||||
|
||||
if [ ! -e ./speech-enhancement-voxlock ]; then
|
||||
# Note: We use -lc++ to link against libc++ instead of libstdc++
|
||||
swiftc \
|
||||
-lc++ \
|
||||
-I ../build-macos/install/include \
|
||||
-import-objc-header ./SherpaOnnx-Bridging-Header.h \
|
||||
./speech-enhancement-voxlock.swift ./SherpaOnnx.swift \
|
||||
-L ../build-macos/install/lib/ \
|
||||
-l sherpa-onnx-c-api \
|
||||
-l onnxruntime \
|
||||
-o speech-enhancement-voxlock
|
||||
|
||||
strip speech-enhancement-voxlock
|
||||
else
|
||||
echo "./speech-enhancement-voxlock exists - skip building"
|
||||
fi
|
||||
|
||||
export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH
|
||||
./speech-enhancement-voxlock
|
||||
|
||||
mkdir -p wav/speech-enhancement-voxlock
|
||||
mv voxlock-enhanced-16k.wav wav/speech-enhancement-voxlock/
|
||||
cp sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav wav/speech-enhancement-voxlock/
|
||||
@@ -0,0 +1,86 @@
|
||||
// Copyright (c) 2026 Xiaomi Corporation
|
||||
//
|
||||
// This file shows how to use the speech enhancement API with VoxLock models
|
||||
// and a voice sample.
|
||||
//
|
||||
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||
//
|
||||
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
//
|
||||
// This example provides a voice sample of the speaker to keep, so VoxLock
|
||||
// isolates that speaker from the first sample. The voice sample must have at
|
||||
// least 3 seconds of the speaker's voice; from a longer file, VoxLock uses the
|
||||
// loudest 3 seconds within its first 30 seconds. See
|
||||
// ./speech-enhancement-voxlock.swift if you don't have a voice sample.
|
||||
//
|
||||
// Please refer to
|
||||
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||
// to download files used in this script
|
||||
//
|
||||
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
|
||||
import AVFoundation
|
||||
|
||||
extension AudioBuffer {
|
||||
func array() -> [Float] {
|
||||
return Array(UnsafeBufferPointer(self))
|
||||
}
|
||||
}
|
||||
|
||||
extension AVAudioPCMBuffer {
|
||||
func array() -> [Float] {
|
||||
return self.audioBufferList.pointee.mBuffers.array()
|
||||
}
|
||||
}
|
||||
|
||||
func run() {
|
||||
var config = sherpaOnnxOfflineSpeechDenoiserConfig(
|
||||
model: sherpaOnnxOfflineSpeechDenoiserModelConfig(
|
||||
voxlock: sherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enrollModel: "./sherpa-onnx-voxlock-int8/enroll.onnx",
|
||||
chunkModel: "./sherpa-onnx-voxlock-int8/chunk_int8.onnx",
|
||||
voiceSample: "./sherpa-onnx-voxlock-int8/test_wavs/en-voice-sample-f.wav")))
|
||||
|
||||
let sd = SherpaOnnxOfflineSpeechDenoiserWrapper(config: &config)
|
||||
|
||||
let fileURL: NSURL =
|
||||
NSURL(fileURLWithPath: "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav")
|
||||
let audioFile = try! AVAudioFile(forReading: fileURL as URL)
|
||||
|
||||
let audioFormat = audioFile.processingFormat
|
||||
assert(audioFormat.sampleRate == 16000)
|
||||
assert(audioFormat.channelCount == 1)
|
||||
assert(audioFormat.commonFormat == AVAudioCommonFormat.pcmFormatFloat32)
|
||||
|
||||
let audioFrameCount = UInt32(audioFile.length)
|
||||
let audioFileBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: audioFrameCount)
|
||||
|
||||
try! audioFile.read(into: audioFileBuffer!)
|
||||
let array: [Float]! = audioFileBuffer?.array()
|
||||
|
||||
let begin = Date()
|
||||
let audio = sd.run(samples: array, sampleRate: Int(audioFormat.sampleRate))
|
||||
let elapsedSeconds = Date().timeIntervalSince(begin)
|
||||
|
||||
let filename = "voxlock-enhanced-with-voice-sample-16k.wav"
|
||||
let ok = audio.save(filename: filename)
|
||||
if ok == 1 {
|
||||
print("\nSaved to:\(filename)")
|
||||
} else {
|
||||
print("Failed to save to \(filename)")
|
||||
}
|
||||
|
||||
let duration = Double(array.count) / audioFormat.sampleRate
|
||||
print(String(format: "Elapsed seconds: %.3f", elapsedSeconds))
|
||||
print(String(format: "Audio duration in seconds: %.3f", duration))
|
||||
print(String(format: "RTF: %.3f/%.3f = %.3f", elapsedSeconds, duration, elapsedSeconds / duration))
|
||||
}
|
||||
|
||||
@main
|
||||
struct App {
|
||||
static func main() {
|
||||
run()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright (c) 2026 Xiaomi Corporation
|
||||
//
|
||||
// This file shows how to use the speech enhancement API with VoxLock models.
|
||||
//
|
||||
// VoxLock is a target-speaker voice isolation model. It keeps the voice of one
|
||||
// speaker, e.g., the caller in a phone call, and removes the other voices.
|
||||
//
|
||||
// The model is from https://github.com/arbazsiddiqui/voxlock
|
||||
//
|
||||
// This example does not provide a voice sample, so VoxLock takes the loudest
|
||||
// 3 seconds in the first 15 seconds of the input as the speaker to keep. See
|
||||
// ./speech-enhancement-voxlock-with-voice-sample.swift if you have a voice
|
||||
// sample of the speaker.
|
||||
//
|
||||
// Please refer to
|
||||
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speech-enhancement-models
|
||||
// to download files used in this script
|
||||
//
|
||||
// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/sherpa-onnx-voxlock-int8.tar.bz2
|
||||
// tar xvf sherpa-onnx-voxlock-int8.tar.bz2
|
||||
|
||||
import AVFoundation
|
||||
|
||||
extension AudioBuffer {
|
||||
func array() -> [Float] {
|
||||
return Array(UnsafeBufferPointer(self))
|
||||
}
|
||||
}
|
||||
|
||||
extension AVAudioPCMBuffer {
|
||||
func array() -> [Float] {
|
||||
return self.audioBufferList.pointee.mBuffers.array()
|
||||
}
|
||||
}
|
||||
|
||||
func run() {
|
||||
var config = sherpaOnnxOfflineSpeechDenoiserConfig(
|
||||
model: sherpaOnnxOfflineSpeechDenoiserModelConfig(
|
||||
voxlock: sherpaOnnxOfflineSpeechDenoiserVoxLockModelConfig(
|
||||
enrollModel: "./sherpa-onnx-voxlock-int8/enroll.onnx",
|
||||
chunkModel: "./sherpa-onnx-voxlock-int8/chunk_int8.onnx")))
|
||||
|
||||
let sd = SherpaOnnxOfflineSpeechDenoiserWrapper(config: &config)
|
||||
|
||||
let fileURL: NSURL =
|
||||
NSURL(fileURLWithPath: "./sherpa-onnx-voxlock-int8/test_wavs/en-combined.wav")
|
||||
let audioFile = try! AVAudioFile(forReading: fileURL as URL)
|
||||
|
||||
let audioFormat = audioFile.processingFormat
|
||||
assert(audioFormat.sampleRate == 16000)
|
||||
assert(audioFormat.channelCount == 1)
|
||||
assert(audioFormat.commonFormat == AVAudioCommonFormat.pcmFormatFloat32)
|
||||
|
||||
let audioFrameCount = UInt32(audioFile.length)
|
||||
let audioFileBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: audioFrameCount)
|
||||
|
||||
try! audioFile.read(into: audioFileBuffer!)
|
||||
let array: [Float]! = audioFileBuffer?.array()
|
||||
|
||||
let begin = Date()
|
||||
let audio = sd.run(samples: array, sampleRate: Int(audioFormat.sampleRate))
|
||||
let elapsedSeconds = Date().timeIntervalSince(begin)
|
||||
|
||||
let filename = "voxlock-enhanced-16k.wav"
|
||||
let ok = audio.save(filename: filename)
|
||||
if ok == 1 {
|
||||
print("\nSaved to:\(filename)")
|
||||
} else {
|
||||
print("Failed to save to \(filename)")
|
||||
}
|
||||
|
||||
let duration = Double(array.count) / audioFormat.sampleRate
|
||||
print(String(format: "Elapsed seconds: %.3f", elapsedSeconds))
|
||||
print(String(format: "Audio duration in seconds: %.3f", duration))
|
||||
print(String(format: "RTF: %.3f/%.3f = %.3f", elapsedSeconds, duration, elapsedSeconds / duration))
|
||||
}
|
||||
|
||||
@main
|
||||
struct App {
|
||||
static func main() {
|
||||
run()
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user