diff --git a/.github/scripts/common.sh b/.github/scripts/common.sh index 6e7a7ed28..38e015d98 100755 --- a/.github/scripts/common.sh +++ b/.github/scripts/common.sh @@ -9,6 +9,49 @@ log() { echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" } +# --- Extracted-model cleanup --------------------------------------------- +# Test scripts download large ASR models. Each script normally ends with an +# `rm -rf $repo`, but if the script aborts early (e.g. `set -e` when a test +# fails, or a download/extract error) that line never runs and the model is +# left on disk. Across the many test steps in a single CI job this fills the +# runner's disk. To avoid that, download_and_extract registers every extracted +# top-level path here and they are all removed when the script exits, whether +# it succeeded or failed. +_sherpa_cleanup_dirs=() +_sherpa_cleanup_saved_exit_trap="" +_sherpa_cleanup_installed=0 + +_sherpa_cleanup_on_exit() { + local d + for d in ${_sherpa_cleanup_dirs[@]+"${_sherpa_cleanup_dirs[@]}"}; do + if [ -n "$d" ] && [ -e "$d" ]; then + log "Cleaning up extracted model: $d" + rm -rf "$d" + fi + done + # Run any EXIT trap that was installed before ours (e.g. test-rust.sh's + # diagnostic trap) so we never clobber another script's cleanup. + if [ -n "$_sherpa_cleanup_saved_exit_trap" ]; then + eval "$_sherpa_cleanup_saved_exit_trap" + fi +} + +_sherpa_register_cleanup() { + _sherpa_cleanup_dirs+=("$1") + if [ "$_sherpa_cleanup_installed" -eq 0 ]; then + _sherpa_cleanup_installed=1 + # Save whatever EXIT trap is currently installed so ours can chain to it. + local saved + saved="$(trap -p EXIT)" + if [ -n "$saved" ]; then + # saved looks like: trap -- 'some command' EXIT + _sherpa_cleanup_saved_exit_trap="${saved#trap -- \'}" + _sherpa_cleanup_saved_exit_trap="${_sherpa_cleanup_saved_exit_trap%\' EXIT}" + fi + trap _sherpa_cleanup_on_exit EXIT + fi +} + # Download a file with retry logic and validation # Validates that downloaded files are not empty and not HTML error pages # Retries with random sleep (1-10 seconds) between attempts @@ -101,11 +144,21 @@ download_and_extract() { fi # Extract based on file extension + # Compute the top-level name the archive extracts to (release model + # archives are named after the directory they contain) so we can clean it + # up automatically when the script exits. + local extracted="$filename" + extracted="${extracted%.tar.bz2}"; extracted="${extracted%.tar.bz}" + extracted="${extracted%.tbz2}"; extracted="${extracted%.tbz}" + extracted="${extracted%.tar.gz}"; extracted="${extracted%.tgz}" + extracted="${extracted%.tar.xz}"; extracted="${extracted%.txz}" + extracted="${extracted%.zip}" case "$filename" in *.tar.bz2|*.tar.bz|*.tbz2|*.tbz) if tar jxvf "$filename" -C "$output_dir" 2>/dev/null; then log "Successfully extracted: $filename" rm -f "$filename" + _sherpa_register_cleanup "$output_dir/$extracted" return 0 fi ;; @@ -113,6 +166,7 @@ download_and_extract() { if tar zxvf "$filename" -C "$output_dir" 2>/dev/null; then log "Successfully extracted: $filename" rm -f "$filename" + _sherpa_register_cleanup "$output_dir/$extracted" return 0 fi ;; @@ -120,6 +174,7 @@ download_and_extract() { if tar Jxvf "$filename" -C "$output_dir" 2>/dev/null; then log "Successfully extracted: $filename" rm -f "$filename" + _sherpa_register_cleanup "$output_dir/$extracted" return 0 fi ;; @@ -127,6 +182,7 @@ download_and_extract() { if unzip -q "$filename" -d "$output_dir" 2>/dev/null; then log "Successfully extracted: $filename" rm -f "$filename" + _sherpa_register_cleanup "$output_dir/$extracted" return 0 fi ;; diff --git a/.github/scripts/flutter/generate_streaming_asr_matrix.py b/.github/scripts/flutter/generate_streaming_asr_matrix.py new file mode 100755 index 000000000..a70e84c74 --- /dev/null +++ b/.github/scripts/flutter/generate_streaming_asr_matrix.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# Copyright 2026 Xiaomi Corp. (authors: Fangjun Kuang) +# +# Generate build matrix for test-flutter-asr.yaml. +# Each job builds flutter-examples/streaming_asr with one streaming ASR model +# selected (`final type = N;` in lib/streaming_asr.dart). + +import json + +# (model_index, model_dir_name, tarball filename, rule_fsts, post_cmd) +# +# model_dir_name is both the extracted directory in assets/ and the name that +# goes into the released zip (flutter-streaming_asr--.zip). +# post_cmd runs inside assets/ after extraction (downloads the optional rule +# fst and strips files the app does not need). +STREAMING_ASR_MODELS = [ + ( + 0, + "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", + "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2", + "itn_zh_number.fst", + """ + if [ ! -f itn_zh_number.fst ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst + fi + pushd sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 + rm -fv encoder-epoch-99-avg-1.onnx + rm -fv decoder-epoch-99-avg-1.int8.onnx + rm -fv joiner-epoch-99-avg-1.int8.onnx + rm -fv *.sh bpe.model README.md .gitattributes *state* + rm -rfv test_wavs + ls -lh + popd + """, + ), + ( + 1, + "sherpa-onnx-streaming-zipformer-en-2023-06-26", + "sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2", + "", + """ + pushd sherpa-onnx-streaming-zipformer-en-2023-06-26 + rm -fv encoder-epoch-99-avg-1-chunk-16-left-128.onnx + rm -fv decoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx + rm -fv joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx + rm -fv *.sh bpe.model README.md .gitattributes *state* + rm -rfv test_wavs + ls -lh + popd + """, + ), + ( + 2, + "icefall-asr-zipformer-streaming-wenetspeech-20230615", + "icefall-asr-zipformer-streaming-wenetspeech-20230615.tar.bz2", + "itn_zh_number.fst", + """ + if [ ! -f itn_zh_number.fst ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst + fi + pushd icefall-asr-zipformer-streaming-wenetspeech-20230615 + rm -fv exp/encoder-epoch-12-avg-4-chunk-16-left-128.onnx + rm -fv exp/decoder-epoch-12-avg-4-chunk-16-left-128.int8.onnx + rm -fv exp/joiner-epoch-12-avg-4-chunk-16-left-128.int8.onnx + rm -fv *.sh bpe.model README.md .gitattributes *state* + rm -rfv test_wavs + ls -lh + popd + """, + ), + ( + 3, + "sherpa-onnx-streaming-zipformer-fr-2023-04-14", + "sherpa-onnx-streaming-zipformer-fr-2023-04-14.tar.bz2", + "", + """ + pushd sherpa-onnx-streaming-zipformer-fr-2023-04-14 + rm -fv encoder-epoch-12-avg-5.onnx + rm -fv decoder-epoch-12-avg-5.int8.onnx + rm -fv joiner-epoch-12-avg-5.int8.onnx + rm -fv *.sh bpe.model README.md .gitattributes *state* + rm -rfv test_wavs + ls -lh + popd + """, + ), + ( + 4, + "sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05", + "sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05.tar.bz2", + "", + """ + pushd sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05 + rm -fv encoder.onnx joiner.onnx decoder.int8.onnx + rm -fv bpe.model test_onnx.py README.md .gitattributes *state* + rm -rfv test_wavs + ls -lh + popd + """, + ), + ( + 5, + "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11", + "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11.tar.bz2", + "", + """ + pushd sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11 + rm -fv encoder.onnx decoder.onnx joiner.onnx + rm -fv *.py README.md .gitattributes *state* + rm -rfv test_wavs + ls -lh + popd + """, + ), + ( + 6, + "sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05", + "sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05.tar.bz2", + "", + """ + pushd sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05 + rm -fv encoder.onnx joiner.onnx decoder.int8.onnx + rm -fv bpe.model test_onnx.py README.md .gitattributes *state* + rm -rfv test_wavs + ls -lh + popd + """, + ), + ( + 7, + "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11", + "sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11.tar.bz2", + "", + """ + pushd sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11 + rm -fv encoder.onnx decoder.onnx joiner.onnx + rm -fv *.py README.md .gitattributes *state* + rm -rfv test_wavs + ls -lh + popd + """, + ), +] + +ANDROID_ABIS = [ + { + "abi": "arm64-v8a", + "target_platform": "android-arm64", + "build_script": "build-android-arm64-v8a.sh", + "build_dir": "build-android-arm64-v8a", + "plugin": "sherpa_onnx_android_arm64", + "plugin_key": "sherpa_onnx_android_arm64", + }, + { + "abi": "armeabi-v7a", + "target_platform": "android-arm", + "build_script": "build-android-armv7-eabi.sh", + "build_dir": "build-android-armv7-eabi", + "plugin": "sherpa_onnx_android_armeabi", + "plugin_key": "sherpa_onnx_android_armeabi", + }, + { + "abi": "x86_64", + "target_platform": "android-x64", + "build_script": "build-android-x86-64.sh", + "build_dir": "build-android-x86-64", + "plugin": "sherpa_onnx_android_x86_64", + "plugin_key": "sherpa_onnx_android_x86_64", + }, +] + + +def make_entry(idx, name, tarball, rule_fsts, post_cmd): + return { + "demo": "streaming_asr", + "asr_model_index": idx, + "asr_model_name": name, + "asr_tarball": tarball, + "asr_rule_fsts": rule_fsts, + "asr_post_cmd": post_cmd, + } + + +def main(): + desktop_entries = [] + android_entries = [] + for idx, name, tarball, rule_fsts, post_cmd in STREAMING_ASR_MODELS: + desktop_entries.append(make_entry(idx, name, tarball, rule_fsts, post_cmd)) + for abi_cfg in ANDROID_ABIS: + entry = make_entry(idx, name, tarball, rule_fsts, post_cmd) + entry.update(abi_cfg) + android_entries.append(entry) + + print(json.dumps({ + "desktop": {"include": desktop_entries}, + "android": {"include": android_entries}, + })) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/flutter/generate_tts_matrix.py b/.github/scripts/flutter/generate_tts_matrix.py new file mode 100755 index 000000000..97f87a59d --- /dev/null +++ b/.github/scripts/flutter/generate_tts_matrix.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +# Copyright 2026 Xiaomi Corp. (authors: Fangjun Kuang) +# +# Generate build matrix for test-flutter-tts.yaml. +# Each job builds flutter-examples/tts with one TTS model selected +# (selectedModelIndex in lib/model_config.dart). + +import json + +# (model_index, model_dir_name, tarball filename, extra_cmd, cleanup_cmd) +# +# model_dir_name is both the extracted directory in assets/ and the name that +# goes into the released zip (flutter-tts--.zip). +# extra_cmd/cleanup_cmd run inside the model's assets/ staging directory. +TTS_MODELS = [ + ( + 0, + "vits-piper-en_US-amy-low", + "vits-piper-en_US-amy-low.tar.bz2", + "", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 1, + "vits-piper-zh_CN-xiao_ya-medium", + "vits-piper-zh_CN-xiao_ya-medium.tar.bz2", + "", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 2, + "vits-piper-en_US-libritts_r-medium", + "vits-piper-en_US-libritts_r-medium.tar.bz2", + "", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 3, + "vits-inflect-en-nano-v2", + "vits-inflect-en-nano-v2.tar.bz2", + "", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 4, + "kokoro-int8-en-v0_19", + "kokoro-int8-en-v0_19.tar.bz2", + "", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 5, + "kokoro-multi-lang-v1_0", + "kokoro-multi-lang-v1_0.tar.bz2", + "", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 6, + "matcha-icefall-en_US-ljspeech", + "matcha-icefall-en_US-ljspeech.tar.bz2", + "curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 7, + "matcha-icefall-zh-en", + "matcha-icefall-zh-en.tar.bz2", + "curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-16khz-univ.onnx", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 8, + "kitten-nano-en-v0_1-fp16", + "kitten-nano-en-v0_1-fp16.tar.bz2", + "", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 9, + "sherpa-onnx-pocket-tts-int8-2026-01-26", + "sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2", + "", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), + ( + 10, + "sherpa-onnx-supertonic-3-tts-int8-2026-05-11", + "sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2", + "", + """ + rm -rf test_wavs espeak-ng-data + rm -fv README.md LICENSE *.py + ls -lh + """, + ), +] + +ANDROID_ABIS = [ + { + "abi": "arm64-v8a", + "target_platform": "android-arm64", + "build_script": "build-android-arm64-v8a.sh", + "build_dir": "build-android-arm64-v8a", + "plugin": "sherpa_onnx_android_arm64", + "plugin_key": "sherpa_onnx_android_arm64", + }, + { + "abi": "armeabi-v7a", + "target_platform": "android-arm", + "build_script": "build-android-armv7-eabi.sh", + "build_dir": "build-android-armv7-eabi", + "plugin": "sherpa_onnx_android_armeabi", + "plugin_key": "sherpa_onnx_android_armeabi", + }, + { + "abi": "x86_64", + "target_platform": "android-x64", + "build_script": "build-android-x86-64.sh", + "build_dir": "build-android-x86-64", + "plugin": "sherpa_onnx_android_x86_64", + "plugin_key": "sherpa_onnx_android_x86_64", + }, +] + + +def make_entry(idx, name, tarball, extra_cmd, cleanup_cmd): + return { + "demo": "tts", + "tts_model_index": idx, + "tts_model_name": name, + "tts_tarball": tarball, + "tts_extra_cmd": extra_cmd, + "tts_cleanup_cmd": cleanup_cmd, + } + + +def main(): + desktop_entries = [] + android_entries = [] + for idx, name, tarball, extra_cmd, cleanup_cmd in TTS_MODELS: + desktop_entries.append(make_entry(idx, name, tarball, extra_cmd, cleanup_cmd)) + for abi_cfg in ANDROID_ABIS: + entry = make_entry(idx, name, tarball, extra_cmd, cleanup_cmd) + entry.update(abi_cfg) + android_entries.append(entry) + + print(json.dumps({ + "desktop": {"include": desktop_entries}, + "android": {"include": android_entries}, + })) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/flutter/generate_vad_asr_matrix.py b/.github/scripts/flutter/generate_vad_asr_matrix.py old mode 100644 new mode 100755 diff --git a/.github/scripts/go/non-streaming-tts-phonemize.sh b/.github/scripts/go/non-streaming-tts-phonemize.sh new file mode 100755 index 000000000..6956662e4 --- /dev/null +++ b/.github/scripts/go/non-streaming-tts-phonemize.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -ex +cd go-api-examples/non-streaming-tts-phonemize +go mod tidy + +# On Windows, piper-phonemize's native DLLs live inside the +# piper-phonemize-go-windows module (the module cache) and are not on the +# loader's search path. Copy them next to the built binary so they can be +# found at runtime. (Linux/macOS find the lib via the normal mechanisms.) +if [[ "$OSTYPE" == msys* || "$OSTYPE" == cygwin* || "$OSTYPE" == win32 ]]; then + case "$(go env GOARCH)" in + amd64) triple=x86_64-pc-windows-gnu ;; + 386) triple=i686-pc-windows-gnu ;; + arm64) triple=aarch64-pc-windows-gnu ;; + esac + pp_dir=$(go list -m -f '{{.Dir}}' github.com/csukuangfj/piper-phonemize-go-windows) + cp -v "$pp_dir/lib/$triple"/*.dll . +fi + +go build +./run-vits-piper-phonemize.sh +./run-kitten-en-phonemize.sh +./run-kokoro-en-phonemize.sh +./run-matcha-en-phonemize.sh diff --git a/.github/scripts/go/offline-zipformer-ctc-hlg.sh b/.github/scripts/go/offline-zipformer-ctc-hlg.sh new file mode 100755 index 000000000..3ae89d81d --- /dev/null +++ b/.github/scripts/go/offline-zipformer-ctc-hlg.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -ex +cd go-api-examples/offline-zipformer-ctc-hlg +go mod tidy +go build +./run.sh diff --git a/.github/scripts/test-dart.sh b/.github/scripts/test-dart.sh index 15c433374..ff1c57cd7 100755 --- a/.github/scripts/test-dart.sh +++ b/.github/scripts/test-dart.sh @@ -10,6 +10,48 @@ pushd version ./run.sh popd +pushd tts + +echo '----------tts----------' +./run-pocket-en.sh +./run-kitten-en-lexicon.sh +./run-kitten-en-phonemize.sh +./run-supertonic-en.sh +./run-kokoro-zh-en-lexicon.sh +./run-kokoro-en-lexicon.sh +./run-kokoro-en-phonemize.sh +./run-matcha-zh-lexicon.sh +./run-matcha-en-lexicon.sh +./run-matcha-en-phonemize.sh +./run-zipvoice-zh-en-lexicon.sh +ls -lh *.wav +rm -rf matcha-icefall-* +rm -rf sherpa-onnx-zipvoice-* +rm -f lexicon-en-us.txt +rm *.onnx + +echo '----------piper tts----------' +./run-piper-lexicon.sh +./run-piper-phonemize.sh +rm -f lexicon-en-us.txt +rm -rf vits-piper-* + +echo '----------german tts----------' +./run-vits-de-phonemize.sh +rm -rf vits-piper-* + +echo '----------coqui tts----------' +./run-coqui.sh +rm -rf vits-coqui-* + +echo '----------zh tts----------' +./run-vits-zh-lexicon.sh +rm -rf sherpa-onnx-* + +ls -lh *.wav + +popd # tts + pushd non-streaming-asr echo '----------Moonshine v2----------' @@ -24,6 +66,10 @@ echo '----------Zipformer CTC----------' ./run-zipformer-ctc.sh rm -rf sherpa-onnx-* +echo '----------Zipformer CTC HLG----------' +./run-zipformer-ctc-hlg.sh +rm -rf sherpa-onnx-* + echo '----------Cohere Transcribe----------' ./run-cohere-transcribe.sh rm -rf sherpa-onnx-* @@ -122,38 +168,6 @@ echo "streaming speech enhancement with dpdfnet models" ls -lh popd -pushd tts - -echo '----------tts----------' -./run-pocket-en.sh -./run-kitten-en.sh -./run-supertonic-en.sh -./run-kokoro-zh-en.sh -./run-kokoro-en.sh -./run-matcha-zh.sh -./run-matcha-en.sh -./run-zipvoice-zh-en.sh -ls -lh *.wav -rm -rf matcha-icefall-* -rm -rf sherpa-onnx-zipvoice-* -rm *.onnx - -echo '----------piper tts----------' -./run-piper.sh -rm -rf vits-piper-* - -echo '----------coqui tts----------' -./run-coqui.sh -rm -rf vits-coqui-* - -echo '----------zh tts----------' -./run-vits-zh.sh -rm -rf sherpa-onnx-* - -ls -lh *.wav - -popd # tts - pushd spoken-language-identification ./run-whisper.sh popd diff --git a/.github/scripts/test-dot-net.sh b/.github/scripts/test-dot-net.sh index 07f5ad7b3..dc6fcf1de 100755 --- a/.github/scripts/test-dot-net.sh +++ b/.github/scripts/test-dot-net.sh @@ -5,14 +5,88 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/common.sh" cd dotnet-examples/ -cd ./non-streaming-cohere-transcribe-decode-files +cd ./version-test +./run.sh +ls -lh + + +cd ../kitten-tts-lexicon +./run-kitten-lexicon.sh +ls -lh +rm -rf kitten-nano-en-v0_1-fp16 +rm -f lexicon-en-us.txt + +cd ../kitten-tts-phonemize +./run-kitten-phonemize.sh +ls -lh +rm -rf kitten-nano-en-v0_1-fp16 + +cd ../kokoro-tts-lexicon +./run-kokoro-lexicon.sh +ls -lh +rm -f lexicon-en-us.txt + +cd ../kokoro-tts-phonemize +./run-kokoro-phonemize.sh +ls -lh + +cd ../offline-tts-lexicon +./run-matcha-zh-lexicon.sh +ls -lh *.wav +./run-matcha-en-lexicon.sh +ls -lh *.wav +./run-aishell3-lexicon.sh +ls -lh *.wav +./run-piper-lexicon.sh +ls -lh *.wav +./run-hf-fanchen-lexicon.sh +ls -lh *.wav +ls -lh +rm -f lexicon-en-us.txt + +cd ../offline-tts-phonemize +./run-piper-phonemize.sh +ls -lh *.wav +./run-matcha-en-phonemize.sh +ls -lh *.wav +ls -lh + +cd ../pocket-tts-zero-shot +./run.sh +ls -lh +rm -rf sherpa-onnx-pocket-* + +cd ../supertonic-tts +./run.sh +ls -lh +rm -rf sherpa-onnx-supertonic-* + +cd ../zipvoice-tts-lexicon +./run-lexicon.sh +ls -lh +rm -rf sherpa-onnx-zipvoice-* +rm -f vocos_24khz.onnx lexicon-en-us.txt + +pushd ../.. + +mkdir tts + +cp -v dotnet-examples/kokoro-tts-lexicon/*.wav ./tts +cp -v dotnet-examples/kokoro-tts-phonemize/*.wav ./tts +cp -v dotnet-examples/kitten-tts-lexicon/*.wav ./tts +cp -v dotnet-examples/kitten-tts-phonemize/*.wav ./tts +cp -v dotnet-examples/offline-tts-lexicon/*.wav ./tts +cp -v dotnet-examples/offline-tts-phonemize/*.wav ./tts +cp -v dotnet-examples/supertonic-tts/*.wav ./tts +cp -v dotnet-examples/zipvoice-tts-lexicon/*.wav ./tts +popd + +cd ../non-streaming-cohere-transcribe-decode-files ./run.sh ls -lh rm -rf sherpa-onnx-cohere-transcribe-* -cd .. - -cd ./vad-non-streaming-qwen3-asr +cd ../vad-non-streaming-qwen3-asr ./run-ten-vad.sh rm -fv *.onnx @@ -36,11 +110,6 @@ cd ../source-separation-uvr ls -lh rm -f UVR-MDX-NET-Voc_FT.onnx -cd ../supertonic-tts -./run.sh -ls -lh -rm -rf sherpa-onnx-supertonic-* - cd ../non-streaming-moonshine-v2-decode-files ./run.sh rm -rf sherpa-onnx-moonshine-* @@ -101,17 +170,6 @@ rm -rf sherpa-onnx-* ./run-tdnn-yesno.sh rm -rf sherpa-onnx-* -cd ../pocket-tts-zero-shot -./run.sh -ls -lh -rm -rf sherpa-onnx-pocket-* - -cd ../zipvoice-tts -./run.sh -ls -lh -rm -rf sherpa-onnx-zipvoice-* -rm -f vocos_24khz.onnx - cd ../vad-non-streaming-funasr-nano ./run-ten-vad.sh rm -fv *.onnx @@ -124,20 +182,12 @@ cd ../non-streaming-funasr-nano-decode-files ls -lh rm -rf sherpa-onnx-funasr-* -cd ../version-test -./run.sh -ls -lh cd ../offline-audio-tagging ./run.sh ls -lh rm -rf sherpa-onnx-* -cd ../kitten-tts -./run-kitten.sh -ls -lh -rm -rf kitten-nano-en-v0_1-fp16 - cd ../vad-non-streaming-asr-paraformer ./run-ten-vad.sh rm -fv *.onnx @@ -151,8 +201,6 @@ cd ../non-streaming-canary-decode-files ls -lh rm -rf sherpa-onnx-nemo-* - - cd ../speech-enhancement-gtcrn ./run.sh ls -lh @@ -169,33 +217,6 @@ cd ../streaming-speech-enhancement-dpdfnet ./run.sh ls -lh -cd ../kokoro-tts -./run-kokoro.sh -ls -lh - -cd ../offline-tts -./run-matcha-zh.sh -ls -lh *.wav -./run-matcha-en.sh -ls -lh *.wav -./run-aishell3.sh -ls -lh *.wav -./run-piper.sh -ls -lh *.wav -./run-hf-fanchen.sh -ls -lh *.wav -ls -lh - -pushd ../.. - -mkdir tts - -cp -v dotnet-examples/kokoro-tts/*.wav ./tts -cp -v dotnet-examples/offline-tts/*.wav ./tts -cp -v dotnet-examples/supertonic-tts/*.wav ./tts -cp -v dotnet-examples/zipvoice-tts/*.wav ./tts -popd - cd ../offline-speaker-diarization ./run.sh rm -rfv *.onnx @@ -236,6 +257,10 @@ cd ../streaming-hlg-decoding/ ./run.sh rm -rf sherpa-onnx-* +cd ../offline-zipformer-ctc-hlg/ +./run.sh +rm -rf sherpa-onnx-* + cd ../spoken-language-identification ./run.sh rm -rf sherpa-onnx-* diff --git a/.github/scripts/test-nodejs-addon-npm.sh b/.github/scripts/test-nodejs-addon-npm.sh index 1ef3f6153..c4d6d2540 100755 --- a/.github/scripts/test-nodejs-addon-npm.sh +++ b/.github/scripts/test-nodejs-addon-npm.sh @@ -11,6 +11,128 @@ arch=$(node -p "require('os').arch()") platform=$(node -p "require('os').platform()") node_version=$(node -p "process.versions.node.split('.')[0]") +echo "----------Version----------" +node ./test_version.js + +echo "----------PocketTTS----------" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 + +node ./test_tts_non_streaming_pocket_en.js +node ./test_tts_non_streaming_pocket_en_async.js + +rm -rf sherpa-onnx-pocket-tts-int8-2026-01-26 + +echo "----------ZipVoice----------" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 + +download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx + +node ./test_tts_non_streaming_zipvoice_zh_en_lexicon.js +node ./test_tts_non_streaming_zipvoice_zh_en_lexicon_async.js + +rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia +rm -f vocos_24khz.onnx + +echo "----------KittenTTS----------" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 + +# shared English lexicon for the lexicon-based examples +if [ ! -f ./lexicon-en-us.txt ]; then + download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +node ./test_tts_non_streaming_kitten_en_lexicon.js +node ./test_tts_non_streaming_kitten_en_lexicon_sync.js +node ./test_tts_non_streaming_kitten_en_phonemize.js +node ./test_tts_non_streaming_kitten_en_phonemize_sync.js + +rm -rf kitten-nano-en-v0_1-fp16 + +echo "----------SupertonicTTS----------" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 + +node ./test_tts_non_streaming_supertonic_en.js +node ./test_tts_non_streaming_supertonic_en_async.js + +rm -rf sherpa-onnx-supertonic-3-tts-int8-2026-05-11 + +echo "----------tts----------" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 + +node ./test_tts_non_streaming_kokoro_zh_en_lexicon.js +node ./test_tts_non_streaming_kokoro_zh_en_lexicon_async.js +ls -lh *.wav +rm -rf kokoro-multi-lang-v1_0 + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + +node ./test_tts_non_streaming_kokoro_en_lexicon.js +node ./test_tts_non_streaming_kokoro_en_lexicon_async.js +node ./test_tts_non_streaming_kokoro_en_phonemize.js +node ./test_tts_non_streaming_kokoro_en_phonemize_async.js +ls -lh *.wav +rm -rf kokoro-en-v0_19 + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + +node ./test_tts_non_streaming_matcha_icefall_en_lexicon.js +node ./test_tts_non_streaming_matcha_icefall_en_lexicon_async.js +node ./test_tts_non_streaming_matcha_icefall_en_phonemize.js +node ./test_tts_non_streaming_matcha_icefall_en_phonemize_async.js +rm vocos-22khz-univ.onnx +rm -rf matcha-icefall-en_US-ljspeech + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + +node ./test_tts_non_streaming_matcha_icefall_zh_lexicon.js +node ./test_tts_non_streaming_matcha_icefall_zh_lexicon_async.js +rm vocos-22khz-univ.onnx +rm -rf matcha-icefall-zh-baker +ls -lh *.wav + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_GB-cori-medium.tar.bz2 + +node ./test_tts_non_streaming_vits_piper_en_lexicon.js +node ./test_tts_non_streaming_vits_piper_en_lexicon_async.js +node ./test_tts_async_callback_stress_lexicon.js ./vits-piper-en_GB-cori-medium 30 +node ./test_tts_non_streaming_vits_piper_en_phonemize.js +node ./test_tts_non_streaming_vits_piper_en_phonemize_async.js +node ./test_tts_async_callback_stress_phonemize.js ./vits-piper-en_GB-cori-medium 30 + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-glados-low.tar.bz2 + +node ./test_tts_non_streaming_vits_piper_de_phonemize.js +node ./test_tts_non_streaming_vits_piper_de_phonemize_async.js + +rm -rf vits-piper-de_DE-glados-low +rm -f lexicon-en-us.txt +rm -rf vits-piper-en_GB-cori-medium + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-coqui-de-css10.tar.bz2 + +node ./test_tts_non_streaming_vits_coqui_de.js +node ./test_tts_non_streaming_vits_coqui_de_async.js +rm -rf vits-coqui-de-css10 + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2 + +node ./test_tts_non_streaming_vits_zh_ll_lexicon.js +node ./test_tts_non_streaming_vits_zh_ll_lexicon_async.js +rm -rf sherpa-onnx-vits-zh-ll + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 + +node ./test_tts_non_streaming_vits_zh_aishell3_lexicon.js +node ./test_tts_non_streaming_vits_zh_aishell3_lexicon_async.js +rm -rf vits-icefall-zh-aishell3 + echo "----------Cohere Transcribe----------" download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-cohere-transcribe-14-lang-int8-2026-04-01.tar.bz2 @@ -42,26 +164,6 @@ node ./test_asr_non_streaming_fire_red_asr_ctc_async.js rm -rf sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25 -echo "----------PocketTTS----------" - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 - -node ./test_tts_non_streaming_pocket_en.js -node ./test_tts_non_streaming_pocket_en_async.js - -rm -rf sherpa-onnx-pocket-tts-int8-2026-01-26 - -echo "----------ZipVoice----------" - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 - -download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx - -node ./test_tts_non_streaming_zipvoice_zh_en.js -node ./test_tts_non_streaming_zipvoice_zh_en_async.js - -rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia -rm -f vocos_24khz.onnx echo "----------non-streaming ASR FunASR Nano----------" @@ -102,23 +204,6 @@ node ./test_asr_streaming_t_one_ctc.js rm -rf sherpa-onnx-streaming-t-one-russian-2025-09-08 -echo "----------KittenTTS----------" - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 - -node ./test_tts_non_streaming_kitten_en.js -node ./test_tts_non_streaming_kitten_en_sync.js - -rm -rf kitten-nano-en-v0_1-fp16 - -echo "----------SupertonicTTS----------" - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 - -node ./test_tts_non_streaming_supertonic_en.js -node ./test_tts_non_streaming_supertonic_en_async.js - -rm -rf sherpa-onnx-supertonic-3-tts-int8-2026-05-11 echo "----------non-streaming ASR NeMo Canary----------" @@ -134,6 +219,12 @@ download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr node ./test_asr_non_streaming_zipformer_ctc.js rm -rf sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03 +echo "----------non-streaming ASR Zipformer CTC HLG----------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + +node ./test_asr_non_streaming_zipformer_ctc_hlg.js +rm -rf sherpa-onnx-zipformer-ctc-en-2023-10-02 + echo "----------non-streaming ASR NeMo parakeet tdt----------" download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8.tar.bz2 @@ -244,63 +335,6 @@ if [[ $arch != "ia32" && $platform != "win32" ]]; then rm -rf sherpa-onnx-paraformer-zh-2023-09-14 fi -echo "----------tts----------" - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 - -node ./test_tts_non_streaming_kokoro_zh_en.js -node ./test_tts_non_streaming_kokoro_zh_en_async.js -ls -lh *.wav -rm -rf kokoro-multi-lang-v1_0 - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 - -node ./test_tts_non_streaming_kokoro_en.js -node ./test_tts_non_streaming_kokoro_en_async.js -ls -lh *.wav -rm -rf kokoro-en-v0_19 - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 -download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx - -node ./test_tts_non_streaming_matcha_icefall_en.js -node ./test_tts_non_streaming_matcha_icefall_en_async.js -rm vocos-22khz-univ.onnx -rm -rf matcha-icefall-en_US-ljspeech - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 -download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx - -node ./test_tts_non_streaming_matcha_icefall_zh.js -node ./test_tts_non_streaming_matcha_icefall_zh_async.js -rm vocos-22khz-univ.onnx -rm -rf matcha-icefall-zh-baker -ls -lh *.wav - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_GB-cori-medium.tar.bz2 - -node ./test_tts_non_streaming_vits_piper_en.js -node ./test_tts_non_streaming_vits_piper_en_async.js -node ./test_tts_async_callback_stress.js ./vits-piper-en_GB-cori-medium 30 -rm -rf vits-piper-en_GB-cori-medium - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-coqui-de-css10.tar.bz2 - -node ./test_tts_non_streaming_vits_coqui_de.js -node ./test_tts_non_streaming_vits_coqui_de_async.js -rm -rf vits-coqui-de-css10 - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2 - -node ./test_tts_non_streaming_vits_zh_ll.js -node ./test_tts_non_streaming_vits_zh_ll_async.js -rm -rf sherpa-onnx-vits-zh-ll - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 - -node ./test_tts_non_streaming_vits_zh_aishell3.js -node ./test_tts_non_streaming_vits_zh_aishell3_async.js -rm -rf vits-icefall-zh-aishell3 echo "----------keyword spotting----------" diff --git a/.github/scripts/test-nodejs-npm.sh b/.github/scripts/test-nodejs-npm.sh index 9c7fa9e04..490a4f015 100755 --- a/.github/scripts/test-nodejs-npm.sh +++ b/.github/scripts/test-nodejs-npm.sh @@ -13,6 +13,95 @@ ls -lh node_modules echo "---test version---" node ./test-version.js +# tts + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 + +# shared English lexicon for the lexicon-based examples +if [ ! -f ./lexicon-en-us.txt ]; then + download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +node ./test-offline-tts-kitten-en-lexicon.js +node ./test-offline-tts-kitten-en-phonemize.js +ls -lh *.wav +rm -rf kitten-nano-en-v0_1-fp16 + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 + +node ./test-offline-tts-supertonic.js +ls -lh *.wav +rm -rf sherpa-onnx-supertonic-3-tts-int8-2026-05-11 + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 + +download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx + +node ./test-offline-tts-zipvoice-zh-en-lexicon.js +ls -lh *.wav +rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia +rm -f vocos_24khz.onnx + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 + +node ./test-offline-tts-kokoro-zh-en-lexicon.js +ls -lh *.wav +rm -rf kokoro-multi-lang-v1_0 + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + +node ./test-offline-tts-kokoro-en-lexicon.js +node ./test-offline-tts-kokoro-en-phonemize.js +rm -rf kokoro-en-v0_19 + +ls -lh + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + +download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + +node ./test-offline-tts-matcha-zh-lexicon.js + +rm -rf matcha-icefall-zh-baker +rm vocos-22khz-univ.onnx + + +echo "---" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + +download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + +node ./test-offline-tts-matcha-en-lexicon.js +node ./test-offline-tts-matcha-en-phonemize.js + +rm -rf matcha-icefall-en_US-ljspeech +rm vocos-22khz-univ.onnx + +echo "---" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 +node ./test-offline-tts-vits-en-lexicon.js +node ./test-offline-tts-vits-en-phonemize.js +rm -rf vits-piper-en_US-amy-low* + +echo "---" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-glados-low.tar.bz2 +node ./test-offline-tts-vits-de-phonemize.js +rm -rf vits-piper-de_DE-glados-low* + +echo "---" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 +node ./test-offline-tts-vits-zh-lexicon.js +rm -rf vits-icefall-zh-aishell3* + +rm -f lexicon-en-us.txt + +ls -lh *.wav + + if false; then # disable it for now since it fails, possible due to not using a recent version of onnxruntime echo "---test Qwen3 ASR---" @@ -83,18 +172,6 @@ node ./test-online-t-one-ctc.js rm -rf sherpa-onnx-streaming-t-one-russian-2025-09-08 -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 - -node ./test-offline-tts-kitten-en.js -ls -lh *.wav -rm -rf kitten-nano-en-v0_1-fp16 - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 - -node ./test-offline-tts-supertonic.js -ls -lh *.wav -rm -rf sherpa-onnx-supertonic-3-tts-int8-2026-05-11 - # online asr download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2 node ./test-online-paraformer.js @@ -140,6 +217,12 @@ download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr node ./test-offline-zipformer-ctc.js rm -rf sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03 +# asr with offline zipformer ctc + HLG graph via ctcFstDecoderConfig +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + +node ./test-offline-zipformer-ctc-hlg.js +rm -rf sherpa-onnx-zipformer-ctc-en-2023-10-02 + # asr with offline dolphin ctc download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02.tar.bz2 node ./test-offline-dolphin-ctc.js @@ -159,65 +242,6 @@ rm dpdfnet_baseline.onnx rm -fv inp_16k.wav rm -fv enhanced*.wav -# offline tts - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 - -download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx - -node ./test-offline-tts-zipvoice-zh-en.js -ls -lh *.wav -rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia -rm -f vocos_24khz.onnx - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 - -node ./test-offline-tts-kokoro-zh-en.js -ls -lh *.wav -rm -rf kokoro-multi-lang-v1_0 - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 - -node ./test-offline-tts-kokoro-en.js -rm -rf kokoro-en-v0_19 - -ls -lh - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 - -download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx - -node ./test-offline-tts-matcha-zh.js - -rm -rf matcha-icefall-zh-baker -rm vocos-22khz-univ.onnx - - -echo "---" - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 - -download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx - -node ./test-offline-tts-matcha-en.js - -rm -rf matcha-icefall-en_US-ljspeech -rm vocos-22khz-univ.onnx - -echo "---" - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 -node ./test-offline-tts-vits-en.js -rm -rf vits-piper-en_US-amy-low* - -echo "---" - -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 -node ./test-offline-tts-vits-zh.js -rm -rf vits-icefall-zh-aishell3* - -ls -lh *.wav - echo '-----speaker diarization----------' download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 diff --git a/.github/scripts/test-offline-tts.sh b/.github/scripts/test-offline-tts.sh index 142771633..607e69970 100755 --- a/.github/scripts/test-offline-tts.sh +++ b/.github/scripts/test-offline-tts.sh @@ -45,6 +45,7 @@ log "------------------------------------------------------------" log "kokoro-en-v0_19" log "------------------------------------------------------------" download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt # mapping of sid to voice name # 0->af, 1->af_bella, 2->af_nicole, 3->af_sarah, 4->af_sky, 5->am_adam @@ -56,33 +57,40 @@ for sid in $(seq 0 10); do --kokoro-model=./kokoro-en-v0_19/model.onnx \ --kokoro-voices=./kokoro-en-v0_19/voices.bin \ --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ - --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --kokoro-lexicon=./lexicon-en-us.txt \ --num-threads=2 \ --sid=$sid \ --output-filename="./tts/kokoro-$sid.wav" \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be a statesman, a businessman, an official, or a scholar." done rm -rf kokoro-en-v0_19 +rm -f lexicon-en-us.txt -log "------------------------------------------------------------" -log "matcha-tts-fa_en-musa" -log "------------------------------------------------------------" -download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-tts-fa_en-musa.tar.bz2 +# Disabled: matcha-tts-fa_en-musa ships only espeak-ng-data, no lexicon.txt. +# espeak-ng support was removed in v2.0.0, so --matcha-data-dir is now ignored +# and this model has no usable text frontend. Re-enable it once a Persian +# lexicon is published, or once the CLI can accept phoneme_codepoints. +# +# log "------------------------------------------------------------" +# log "matcha-tts-fa_en-musa" +# log "------------------------------------------------------------" +# download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-tts-fa_en-musa.tar.bz2 +# +# download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +# +# +# $EXE \ +# --matcha-acoustic-model=./matcha-tts-fa_en-musa/model.onnx \ +# --matcha-vocoder=./vocos-22khz-univ.onnx \ +# --matcha-tokens=./matcha-tts-fa_en-musa/tokens.txt \ +# --matcha-data-dir=./matcha-tts-fa_en-musa/espeak-ng-data \ +# --output-filename=./tts/test-matcha-fa-en-musa.wav \ +# --num-threads=2 \ +# "How are you doing today? این یک نمونه ی تست فارسی است. This is a test." +# +# rm -rf matcha-tts-fa_en-musa +# rm vocos-22khz-univ.onnx -download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx - - -$EXE \ - --matcha-acoustic-model=./matcha-tts-fa_en-musa/model.onnx \ - --matcha-vocoder=./vocos-22khz-univ.onnx \ - --matcha-tokens=./matcha-tts-fa_en-musa/tokens.txt \ - --matcha-data-dir=./matcha-tts-fa_en-musa/espeak-ng-data \ - --output-filename=./tts/test-matcha-fa-en-musa.wav \ - --num-threads=2 \ - "How are you doing today? این یک نمونه ی تست فارسی است. This is a test." - -rm -rf matcha-tts-fa_en-musa -rm vocos-22khz-univ.onnx ls -lh tts/*.wav log "------------------------------------------------------------" @@ -91,13 +99,14 @@ log "------------------------------------------------------------" download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt $EXE \ --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ --matcha-vocoder=./vocos-22khz-univ.onnx \ --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ - --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --matcha-lexicon=./lexicon-en-us.txt \ --num-threads=2 \ --output-filename=./tts/matcha-ljspeech-1.wav \ --debug=1 \ @@ -105,6 +114,7 @@ $EXE \ rm vocos-22khz-univ.onnx rm -rf matcha-icefall-en_US-ljspeech +rm -f lexicon-en-us.txt ls -lh tts/*.wav log "------------------------------------------------------------" @@ -141,17 +151,19 @@ log "------------------------------------------------------------" log "vits-piper-en_US-amy-low" log "------------------------------------------------------------" download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 +download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt $EXE \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --debug=1 \ --output-filename=./tts/amy.wav \ "“Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.” The sun shone bleakly in the sky, its meager light struggling to penetrate the thick foliage of the forest. Birds sang their songs up in the crowns of the trees, fluttering from one branch to the other. A blanket of total tranquility lied over the forest. The peace was only broken by the steady gallop of the horses of the soldiers who were traveling to their upcoming knighting the morrow at Camelot, and rowdy conversation. “Finally we will get what we deserve,” “It’s been about time,” Perceval agreed. “We’ve been risking our arses for the past two years. It’s the least they could give us.” Merlin remained ostensibly silent, refusing to join the verbal parade of self-aggrandizing his fellow soldiers have engaged in. He found it difficult to happy about anything, when even if they had won the war, he had lost everything else in the process." file ./tts/amy.wav rm -rf vits-piper-en_US-amy-low +rm -f lexicon-en-us.txt log "------------------------------------------------------------" log "vits-ljs test" diff --git a/.github/scripts/test-online-transducer.sh b/.github/scripts/test-online-transducer.sh index f6fab1db8..ccdb627e6 100755 --- a/.github/scripts/test-online-transducer.sh +++ b/.github/scripts/test-online-transducer.sh @@ -21,351 +21,20 @@ log "Run NeMo transducer (English)" log "------------------------------------------------------------" repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms.tar.bz2 download_and_extract $repo_url + +log "Start testing ${repo_url}" + repo=sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms -log "Start testing ${repo_url}" - -waves=( -$repo/test_wavs/0.wav -$repo/test_wavs/1.wav -$repo/test_wavs/8k.wav -) - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder.onnx \ - --decoder=$repo/decoder.onnx \ - --joiner=$repo/joiner.onnx \ - --num-threads=2 \ - $wave -done - +# decode-file-c-api hard-codes these model paths and ignores any command-line +# arguments (its main() takes none). The generic sherpa-onnx CLI needs them. +# Passing the paths works for both: the CLI uses them, the C example ignores +# them. time $EXE \ --tokens=$repo/tokens.txt \ --encoder=$repo/encoder.onnx \ --decoder=$repo/decoder.onnx \ --joiner=$repo/joiner.onnx \ - --num-threads=2 \ - $repo/test_wavs/0.wav \ - $repo/test_wavs/1.wav \ - $repo/test_wavs/8k.wav - -log "Test NeMo transducer with modified_beam_search (no hotwords)" - -time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder.onnx \ - --decoder=$repo/decoder.onnx \ - --joiner=$repo/joiner.onnx \ - --num-threads=2 \ - --decoding-method=modified_beam_search \ $repo/test_wavs/0.wav -log "Test NeMo transducer with modified_beam_search and hotwords" - -# The model does not ship a bpe.vocab; derive one from tokens.txt so that -# hotwords can be given as words. Equal scores make the encoder use the -# longest match. -awk '{print $1 "\t-1.0"}' $repo/tokens.txt > $repo/bpe.vocab - -cat > $repo/hotwords.txt << EOF -the -and -that -EOF - -time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder.onnx \ - --decoder=$repo/decoder.onnx \ - --joiner=$repo/joiner.onnx \ - --num-threads=2 \ - --decoding-method=modified_beam_search \ - --modeling-unit=bpe \ - --bpe-vocab=$repo/bpe.vocab \ - --hotwords-file=$repo/hotwords.txt \ - --hotwords-score=1.5 \ - $repo/test_wavs/0.wav > $repo/hotwords.log 2>&1 - -cat $repo/hotwords.log - -case "$EXE" in - *decode-file-c-api*) - # prints only the text - ;; - *) - # the matched hotwords must carry the hotwords score in context_scores - grep -q '"context_scores": \[[^]]*1.500000' $repo/hotwords.log - ;; -esac - -rm -rf $repo - -log "------------------------------------------------------------" -log "Run LSTM transducer (English)" -log "------------------------------------------------------------" - -repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-lstm-en-2023-02-17.tar.bz2 -download_and_extract $repo_url -repo=sherpa-onnx-lstm-en-2023-02-17 - -log "Start testing ${repo_url}" - -waves=( -$repo/test_wavs/0.wav -$repo/test_wavs/1.wav -$repo/test_wavs/8k.wav -) - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.onnx \ - --num-threads=2 \ - $wave -done - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.int8.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.int8.onnx \ - --num-threads=2 \ - $wave -done - -rm -rf $repo - -log "------------------------------------------------------------" -log "Run LSTM transducer (Chinese)" -log "------------------------------------------------------------" - -repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-lstm-zh-2023-02-20.tar.bz2 -download_and_extract $repo_url -repo=sherpa-onnx-lstm-zh-2023-02-20 - -log "Start testing ${repo_url}" - -waves=( -$repo/test_wavs/0.wav -$repo/test_wavs/1.wav -$repo/test_wavs/8k.wav -) - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-11-avg-1.onnx \ - --decoder=$repo/decoder-epoch-11-avg-1.onnx \ - --joiner=$repo/joiner-epoch-11-avg-1.onnx \ - --num-threads=2 \ - $wave -done - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-11-avg-1.int8.onnx \ - --decoder=$repo/decoder-epoch-11-avg-1.onnx \ - --joiner=$repo/joiner-epoch-11-avg-1.int8.onnx \ - --num-threads=2 \ - $wave -done - -rm -rf $repo - -log "------------------------------------------------------------" -log "Run streaming Zipformer transducer (English)" -log "------------------------------------------------------------" - -repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-2023-02-21.tar.bz2 -download_and_extract $repo_url -repo=sherpa-onnx-streaming-zipformer-en-2023-02-21 - -log "Start testing ${repo_url}" - -waves=( -$repo/test_wavs/0.wav -$repo/test_wavs/1.wav -$repo/test_wavs/8k.wav -) - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.onnx \ - --num-threads=2 \ - $wave -done - -# test int8 -# -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.int8.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.int8.onnx \ - --num-threads=2 \ - $wave -done - -lm_repo_url=https://huggingface.co/vsd-vector/icefall-librispeech-rnn-lm -log "Download pre-trained RNN-LM model from ${lm_repo_url}" -GIT_LFS_SKIP_SMUDGE=1 git clone $lm_repo_url -lm_repo=$(basename $lm_repo_url) -pushd $lm_repo -git lfs pull --include "with-state-epoch-99-avg-1.onnx" -popd - -bigram_repo_url=https://huggingface.co/vsd-vector/librispeech_bigram_sherpa-onnx-zipformer-large-en-2023-06-26 -log "Download bi-gram LM from ${bigram_repo_url}" -GIT_LFS_SKIP_SMUDGE=1 git clone $bigram_repo_url -bigramlm_repo=$(basename $bigram_repo_url) -pushd $bigramlm_repo -git lfs pull --include "2gram.fst" -popd - -log "Start testing LODR" - -waves=( -$repo/test_wavs/0.wav -$repo/test_wavs/1.wav -$repo/test_wavs/8k.wav -) - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.onnx \ - --num-threads=2 \ - --decoding_method="modified_beam_search" \ - --lm=$lm_repo/with-state-epoch-99-avg-1.onnx \ - --lodr-fst=$bigramlm_repo/2gram.fst \ - --lodr-scale=-0.5 \ - $wave -done - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.onnx \ - --num-threads=2 \ - --decoding_method="modified_beam_search" \ - --lm=$lm_repo/with-state-epoch-99-avg-1.onnx \ - --lodr-fst=$bigramlm_repo/2gram.fst \ - --lodr-scale=-0.5 \ - --lm-shallow-fusion=true \ - $wave -done - -rm -rf $repo $bigramlm_repo $lm_repo - -log "------------------------------------------------------------" -log "Run streaming Zipformer transducer (Bilingual, Chinese + English)" -log "------------------------------------------------------------" - -repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 -download_and_extract $repo_url -repo=sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 - -log "Start testing ${repo_url}" - -waves=( -$repo/test_wavs/0.wav -$repo/test_wavs/1.wav -$repo/test_wavs/2.wav -$repo/test_wavs/3.wav -$repo/test_wavs/8k.wav -) - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.onnx \ - --num-threads=2 \ - $wave -done - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.int8.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.int8.onnx \ - --num-threads=2 \ - $wave -done - -# Decode a URL -if [ $EXE == "sherpa-onnx-ffmpeg" ]; then - time $EXE \ - $repo/tokens.txt \ - $repo/encoder-epoch-99-avg-1.onnx \ - $repo/decoder-epoch-99-avg-1.onnx \ - $repo/joiner-epoch-99-avg-1.onnx \ - https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/resolve/main/test_wavs/4.wav \ - 2 -fi - -if [ $EXE == "sherpa-onnx-ffmpeg" ]; then - time $EXE \ - $repo/tokens.txt \ - $repo/encoder-epoch-99-avg-1.int8.onnx \ - $repo/decoder-epoch-99-avg-1.onnx \ - $repo/joiner-epoch-99-avg-1.int8.onnx \ - https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/resolve/main/test_wavs/4.wav \ - 2 -fi - -rm -rf $repo - -log "------------------------------------------------------------" -log "Run streaming Conformer transducer (English)" -log "------------------------------------------------------------" - -repo_url=https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-conformer-en-2023-05-09.tar.bz2 -download_and_extract $repo_url -repo=sherpa-onnx-streaming-conformer-en-2023-05-09 - -log "Start testing ${repo_url}" - -waves=( -$repo/test_wavs/0.wav -$repo/test_wavs/1.wav -$repo/test_wavs/2.wav -) - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.onnx \ - --num-threads=2 \ - $wave -done - -for wave in ${waves[@]}; do - time $EXE \ - --tokens=$repo/tokens.txt \ - --encoder=$repo/encoder-epoch-99-avg-1.int8.onnx \ - --decoder=$repo/decoder-epoch-99-avg-1.onnx \ - --joiner=$repo/joiner-epoch-99-avg-1.int8.onnx \ - --num-threads=2 \ - $wave -done - rm -rf $repo diff --git a/.github/scripts/test-python-tts-frontends.sh b/.github/scripts/test-python-tts-frontends.sh new file mode 100755 index 000000000..76d065e51 --- /dev/null +++ b/.github/scripts/test-python-tts-frontends.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# +# Tests every python-api-examples/test-offline-tts-*.py example. +# +# These cover the two text-frontend paths that replaced the removed espeak-ng +# support: +# +# * *-lexicon.py -- pass a lexicon file, no external phonemizer needed +# * *-phonemize.py -- phonemize outside sherpa-onnx (piper_phonemize, +# pypinyin, misaki) and pass phoneme_codepoints +# * *-tokens.py -- same as *-phonemize.py but pass token strings via +# GenerationConfig.tokens (sherpa-onnx adds the +# bos/pad/eos markers itself) +# +# Requires: piper_phonemize, pypinyin, misaki[zh], kaldifst, soundfile +# +# Note: misaki keeps its Chinese dependencies behind the "zh" extra, so +# "misaki[zh]" (not bare "misaki") is required for `from misaki import zh`. +set -e +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" +log() { + # This function is from espnet + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} + +export GIT_CLONE_PROTECTION_ACTIVE=false + +echo "EXE is $EXE" +echo "PATH: $PATH" + +python3 -m pip install --upgrade soundfile +python3 -m pip install --upgrade pypinyin 'misaki[zh]' kaldifst +python3 -m pip install --upgrade piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +# Import exactly what the examples import -- bare "import misaki" does NOT +# exercise misaki.zh, which is where the extras-gated deps (cn2an, jieba, +# pypinyin-dict) are needed. +python3 -c " +import kaldifst +import numpy +import soundfile +import piper_phonemize +from pypinyin import Style, lazy_pinyin +from pypinyin.contrib.tone_convert import to_finals_tone3, to_initials +from misaki import zh +print('phonemizer deps OK') +" + +# test waves are saved in ./tts +mkdir -p ./tts + +LEXICON=https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +VOCOS_22=https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +VOCOS_16=https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-16khz-univ.onnx +VOCOS_24=https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx + +run() { + # run + local ex=$1 + local wav=$2 + log "Run $ex" + python3 ./python-api-examples/$ex + mv -v "$wav" ./tts/ +} + +log "------------------------------------------------------------" +log "coqui (character frontend)" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-coqui-de-css10.tar.bz2 +run test-offline-tts-coqui-phonemize.py test-coqui-phonemize.wav +rm -rf vits-coqui-de-css10 + +log "------------------------------------------------------------" +log "inflect (vits-ext)" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-inflect-en-nano-v2.tar.bz2 +download $LEXICON +run test-offline-tts-inflect-lexicon.py test-inflect-lexicon.wav +run test-offline-tts-inflect-phonemize.py test-inflect-phonemize.wav +rm -rf vits-inflect-en-nano-v2 lexicon-en-us.txt + +log "------------------------------------------------------------" +log "kitten" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-micro-en-v0_8.tar.bz2 +download $LEXICON +run test-offline-tts-kitten-lexicon.py test-kitten-lexicon.wav +run test-offline-tts-kitten-phonemize.py test-kitten-phonemize.wav +rm -rf kitten-micro-en-v0_8 lexicon-en-us.txt + +log "------------------------------------------------------------" +log "kokoro en" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +download $LEXICON +run test-offline-tts-kokoro-en-lexicon.py test-kokoro-en-lexicon.wav +run test-offline-tts-kokoro-en-phonemize.py test-kokoro-en-phonemize.wav +rm -rf kokoro-en-v0_19 lexicon-en-us.txt + +log "------------------------------------------------------------" +log "kokoro multi-lang" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 +run test-offline-tts-kokoro-multilang-lexicon.py test-kokoro-multilang-lexicon.wav +run test-offline-tts-kokoro-multilang-phonemize.py test-kokoro-multilang-phonemize.wav +rm -rf kokoro-multi-lang-v1_0 + +log "------------------------------------------------------------" +log "matcha en" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +download $VOCOS_22 +download $LEXICON +run test-offline-tts-matcha-en-lexicon.py test-matcha-en-lexicon.wav +run test-offline-tts-matcha-en-phonemize.py test-matcha-en-phonemize.wav +rm -rf matcha-icefall-en_US-ljspeech lexicon-en-us.txt + +log "------------------------------------------------------------" +log "matcha zh" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +run test-offline-tts-matcha-zh-lexicon.py test-matcha-zh-lexicon.wav +run test-offline-tts-matcha-zh-phonemize.py test-matcha-zh-phonemize.wav +rm -rf matcha-icefall-zh-baker + +log "------------------------------------------------------------" +log "matcha zh-en" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-en.tar.bz2 +download $VOCOS_16 +download $LEXICON +run test-offline-tts-matcha-zh-en-lexicon.py test-matcha-zh-en-lexicon.wav +run test-offline-tts-matcha-zh-en-phonemize.py test-matcha-zh-en-phonemize.wav +rm -rf matcha-icefall-zh-en vocos-16khz-univ.onnx lexicon-en-us.txt + +log "------------------------------------------------------------" +log "piper (vits)" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 +download $LEXICON +run test-offline-tts-piper-lexicon.py test-piper-lexicon.wav +run test-offline-tts-piper-phonemize.py test-piper-phonemize.wav +run test-offline-tts-piper-tokens.py test-piper-tokens.wav +rm -rf vits-piper-en_US-amy-low lexicon-en-us.txt + +log "------------------------------------------------------------" +log "zipvoice" +log "------------------------------------------------------------" +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 +download $VOCOS_24 +download $LEXICON +run test-offline-tts-zipvoice-lexicon.py test-zipvoice-lexicon.wav +run test-offline-tts-zipvoice-phonemize.py test-zipvoice-phonemize.wav +rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia vocos_24khz.onnx lexicon-en-us.txt + +# the 22 kHz vocoder is shared by matcha-en and matcha-zh +rm -f vocos-22khz-univ.onnx + +log "All test-offline-tts examples passed" +ls -lh ./tts/ diff --git a/.github/scripts/test-python.sh b/.github/scripts/test-python.sh index 83291b478..b82e29b62 100755 --- a/.github/scripts/test-python.sh +++ b/.github/scripts/test-python.sh @@ -87,12 +87,15 @@ download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx +download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/zipvoice-tts.py cp generated-zipvoice-zh-en-python.wav tts/ rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia rm -f vocos_24khz.onnx +rm -f lexicon-en-us.txt log "test Google MedASR" download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2 @@ -347,6 +350,19 @@ python3 ./python-api-examples/online-zipformer-ctc-hlg-decode-file.py \ rm -rf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18 +log "test offline zipformer ctc HLG decoding" + +download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + +python3 ./python-api-examples/offline-zipformer-ctc-hlg-decode-files.py \ + --tokens ./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt \ + --model ./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx \ + --graph ./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst \ + ./sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav + +rm -rf sherpa-onnx-zipformer-ctc-en-2023-10-02 + + mkdir -p /tmp/icefall-models dir=/tmp/icefall-models @@ -428,19 +444,21 @@ mkdir -p ./tts log "test kitten tts" download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 +download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt python3 ./python-api-examples/offline-tts.py \ --debug=1 \ --kitten-model=./kitten-nano-en-v0_1-fp16/model.fp16.onnx \ --kitten-voices=./kitten-nano-en-v0_1-fp16/voices.bin \ --kitten-tokens=./kitten-nano-en-v0_1-fp16/tokens.txt \ - --kitten-data-dir=./kitten-nano-en-v0_1-fp16/espeak-ng-data \ + --kitten-lexicon=./lexicon-en-us.txt \ --num-threads=2 \ --sid=0 \ --output-filename="./tts/kitten-0.wav" \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." rm -rf kitten-nano-en-v0_1-fp16 +rm -f lexicon-en-us.txt log "kokoro-multi-lang-v1_0 test" @@ -451,7 +469,6 @@ python3 ./python-api-examples/offline-tts.py \ --kokoro-model=./kokoro-multi-lang-v1_0/model.onnx \ --kokoro-voices=./kokoro-multi-lang-v1_0/voices.bin \ --kokoro-tokens=./kokoro-multi-lang-v1_0/tokens.txt \ - --kokoro-data-dir=./kokoro-multi-lang-v1_0/espeak-ng-data \ --kokoro-lexicon=./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt \ --num-threads=2 \ --sid=18 \ @@ -463,37 +480,41 @@ rm -rf kokoro-multi-lang-v1_0 log "kokoro-en-v0_19 test" download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 +download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt python3 ./python-api-examples/offline-tts.py \ --debug=1 \ --kokoro-model=./kokoro-en-v0_19/model.onnx \ --kokoro-voices=./kokoro-en-v0_19/voices.bin \ --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ - --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --kokoro-lexicon=./lexicon-en-us.txt \ --num-threads=2 \ --sid=10 \ --output-filename="./tts/kokoro-10.wav" \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be a statesman, a businessman, an official, or a scholar." rm -rf kokoro-en-v0_19 +rm -f lexicon-en-us.txt log "matcha-ljspeech-en test" download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 download https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt python3 ./python-api-examples/offline-tts.py \ --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ --matcha-vocoder=./vocos-22khz-univ.onnx \ --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ - --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --matcha-lexicon=./lexicon-en-us.txt \ --output-filename=./tts/test-matcha-ljspeech-en.wav \ --num-threads=2 \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." rm vocos-22khz-univ.onnx rm -rf matcha-icefall-en_US-ljspeech +rm -f lexicon-en-us.txt log "matcha-baker-zh test" diff --git a/.github/scripts/test-rust.sh b/.github/scripts/test-rust.sh index 96629d283..910e86556 100755 --- a/.github/scripts/test-rust.sh +++ b/.github/scripts/test-rust.sh @@ -9,6 +9,43 @@ trap 'bash ../.github/scripts/show-rust-binary-info.sh --all || true' EXIT bash ./run-version.sh +bash ./run-vits-en-lexicon.sh +bash ./run-vits-en-phonemize.sh +rm -rf vits-piper-en_US-amy-low +rm -f lexicon-en-us.txt + +bash ./run-vits-de-phonemize.sh +rm -rf vits-piper-de_DE-glados-high + +bash ./run-matcha-tts-en-lexicon.sh +bash ./run-matcha-tts-en-phonemize.sh +bash ./run-matcha-tts-zh-lexicon.sh +rm -rf matcha-icefall-en_US-ljspeech matcha-icefall-zh-baker +rm -f vocos-22khz-univ.onnx lexicon-en-us.txt + +bash ./run-kokoro-tts-en-lexicon.sh +bash ./run-kokoro-tts-en-phonemize.sh +rm -rf kokoro-en-v0_19 +rm -f lexicon-en-us.txt + +bash ./run-kokoro-tts-zh-en-lexicon.sh +rm -rf kokoro-multi-lang-v1_0 + +bash ./run-kitten-tts-en-lexicon.sh +bash ./run-kitten-tts-en-phonemize.sh +rm -rf kitten-nano-en-v0_1-fp16 +rm -f lexicon-en-us.txt + +bash ./run-pocket-tts.sh +rm -rf sherpa-onnx-pocket-* + +bash ./run-supertonic-tts.sh +rm -rf sherpa-onnx-supertonic-* + +bash ./run-zipvoice-tts-lexicon.sh +rm -rf sherpa-onnx-zipvoice-* +rm -f vocos_24khz.onnx lexicon-en-us.txt + bash ./run-qwen3-asr.sh rm -rf sherpa-onnx-qwen3-* @@ -33,36 +70,6 @@ bash ./run-offline-speaker-diarization.sh rm -rf sherpa-onnx-pyannote-segmentation-3-0 rm -f 3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx 0-four-speakers-zh.wav -bash ./run-vits-en.sh -rm -rf vits-piper-en_US-amy-low - -bash ./run-vits-de.sh -rm -rf vits-piper-de_DE-glados-high - -bash ./run-matcha-tts-en.sh -bash ./run-matcha-tts-zh.sh -rm -rf matcha-icefall-en_US-ljspeech matcha-icefall-zh-baker -rm -f vocos-22khz-univ.onnx - -bash ./run-kokoro-tts-en.sh -rm -rf kokoro-en-v0_19 - -bash ./run-kokoro-tts-zh-en.sh -rm -rf kokoro-multi-lang-v1_0 - -bash ./run-kitten-tts-en.sh -rm -rf kitten-nano-en-v0_1-fp16 - -bash ./run-pocket-tts.sh -rm -rf sherpa-onnx-pocket-* - -bash ./run-supertonic-tts.sh -rm -rf sherpa-onnx-supertonic-* - -bash ./run-zipvoice-tts.sh -rm -rf sherpa-onnx-zipvoice-* -rm -f vocos_24khz.onnx - bash ./run-online-punctuation.sh rm -rf sherpa-onnx-online-punct-* @@ -91,11 +98,15 @@ bash ./run-nemo-parakeet-en.sh bash ./run-zipformer-vi.sh bash ./run-zipformer-zh-en.sh bash ./run-zipformer-en.sh +bash ./run-zipformer-ctc-hlg.sh +rm -rf sherpa-onnx-zipformer-ctc-en-2023-10-02 bash ./run-sense-voice.sh bash ./run-streaming-zipformer-en.sh bash ./run-streaming-zipformer-zh-en.sh +bash ./run-streaming-zipformer-ctc-hlg.sh +rm -rf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18 bash ./run-offline-speech-enhancement-gtcrn.sh bash ./run-offline-speech-enhancement-dpdfnet.sh diff --git a/.github/scripts/test-swift.sh b/.github/scripts/test-swift.sh index 98f044707..cabec3b06 100755 --- a/.github/scripts/test-swift.sh +++ b/.github/scripts/test-swift.sh @@ -8,6 +8,46 @@ echo "pwd: $PWD" cd swift-api-examples ls -lh +./run-test-version.sh + +./run-tts-pocket-en.sh +ls -lh +rm -rf sherpa-onnx-pocket-* + +./run-tts-supertonic-en.sh +ls -lh +rm -rf sherpa-onnx-supertonic-* + +./run-tts-kitten-en-lexicon.sh +ls -lh +rm -rf kitten-* + +./run-tts-vits-lexicon.sh +ls -lh +rm -rf vits-piper-* + +./run-tts-kokoro-zh-en-lexicon.sh +ls -lh +rm -rf kokoro-multi-* + +./run-tts-kokoro-en-lexicon.sh +ls -lh +rm -rf kokoro-en-* + +./run-tts-matcha-zh-lexicon.sh +ls -lh +rm -rf matcha-icefall-* + +./run-tts-matcha-en-lexicon.sh +ls -lh +rm -rf matcha-icefall-* + +./run-tts-zipvoice-lexicon.sh +ls -lh +rm -rf sherpa-onnx-zipvoice-* +rm -f vocos_24khz.onnx +rm -f lexicon-en-us.txt + ./run-cohere-transcribe.sh rm -rf sherpa-onnx-cohere-transcribe-* @@ -23,22 +63,12 @@ rm -f qi-feng-le-zh.wav ./run-qwen3-asr.sh rm -rf sherpa-onnx-qwen3-* -./run-test-version.sh - ./run-moonshine-v2-asr.sh rm -rf sherpa-onnx-moonshine-* ./run-fire-red-asr-ctc.sh rm -rf sherpa-onnx-fire-red-* -./run-tts-pocket-en.sh -ls -lh -rm -rf sherpa-onnx-pocket-* - -./run-tts-supertonic-en.sh -ls -lh -rm -rf sherpa-onnx-supertonic-* - ./run-medasr-ctc-asr.sh rm -rf sherpa-onnx-medasr-* @@ -52,11 +82,7 @@ rm -rf sherpa-onnx-omnilingual-* rm -rf sherpa-onnx-streaming-* ./run-compute-speaker-embeddings.sh -rm -fv *.wav *.onnx - -./run-tts-kitten-en.sh -ls -lh -rm -rf kitten-* +rm -f wespeaker_zh_cnceleb_resnet34.onnx fangjun-sr-1.wav fangjun-sr-2.wav leijun-sr-1.wav ./run-wenet-ctc-asr.sh rm -rf sherpa-onnx-* @@ -64,6 +90,9 @@ rm -rf sherpa-onnx-* ./run-zipformer-ctc-asr.sh rm -rf sherpa-onnx-zipformer-* +./run-zipformer-ctc-hlg.sh +rm -rf sherpa-onnx-zipformer-* + ./run-decode-file-sense-voice-with-hr.sh rm -rf sherpa-onnx-sense-voice-* rm -rf dict lexicon.txt replace.fst test-hr.wav @@ -80,35 +109,9 @@ ls -lh *.wav ./run-fire-red-asr.sh rm -rf sherpa-onnx-fire-red-asr-* -./run-tts-vits.sh -ls -lh -rm -rf vits-piper-* - -./run-tts-kokoro-zh-en.sh -ls -lh -rm -rf kokoro-multi-* - -./run-tts-kokoro-en.sh -ls -lh -rm -rf kokoro-en-* - -./run-tts-matcha-zh.sh -ls -lh -rm -rf matcha-icefall-* - -./run-tts-matcha-en.sh -ls -lh -rm -rf matcha-icefall-* - -./run-tts-zipvoice.sh -ls -lh -rm -rf sherpa-onnx-zipvoice-* -rm -f vocos_24khz.onnx - ./run-speaker-diarization.sh -rm -rf *.onnx +rm -f 3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx 0-four-speakers-zh.wav rm -rf sherpa-onnx-pyannote-segmentation-3-0 -rm -fv *.wav ./run-add-punctuations.sh rm ./add-punctuations diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml index 5137c137e..03c147f7a 100644 --- a/.github/workflows/aarch64-linux-gnu-shared.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -88,8 +88,6 @@ jobs: cp -v bin/zipformer-ctc-simulate-streaming-alsa-cxx-api install/bin rm -rf install/lib/pkgconfig - rm -fv install/lib/cargs.h - rm -fv install/lib/libcargs.so - name: Build sherpa-onnx if: matrix.gpu == 'OFF' @@ -139,8 +137,6 @@ jobs: cp -v bin/zipformer-ctc-simulate-streaming-alsa-cxx-api install/bin rm -rf install/lib/pkgconfig - rm -fv install/lib/cargs.h - rm -fv install/lib/libcargs.so ' - name: Display system info @@ -202,7 +198,6 @@ jobs: lib_dst=${dst}-lib mkdir -p ${lib_dst}/lib cp -av build/install/lib/lib*.so* ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/libcargs.so ${lib_dst}/lib/libcargs.so.* tree ${lib_dst} tar cjvf ${lib_dst}.tar.bz2 ${lib_dst} @@ -220,7 +215,7 @@ jobs: file: sherpa-onnx-*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.32 + tag: v1.13.8 - name: Release pre-compiled binaries and libs for linux aarch64 if: github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml index 3dae6329c..15c655989 100644 --- a/.github/workflows/aarch64-linux-gnu-static.yaml +++ b/.github/workflows/aarch64-linux-gnu-static.yaml @@ -105,7 +105,6 @@ jobs: ls -lh install/lib rm -rf install/lib/pkgconfig - rm -fv install/lib/cargs.h ' - name: Copy files @@ -131,7 +130,6 @@ jobs: lib_dst=${dst}-lib mkdir -p ${lib_dst}/lib cp -av build/install/lib/lib*.a ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/libcargs.a tree ${lib_dst} tar cjvf ${lib_dst}.tar.bz2 ${lib_dst} @@ -149,7 +147,7 @@ jobs: file: sherpa-onnx-*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.32 + tag: v1.13.8 - name: Release pre-compiled binaries and libs for linux aarch64 if: github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') diff --git a/.github/workflows/android-rknn.yaml b/.github/workflows/android-rknn.yaml index 3a1fbb812..3dfeece7c 100644 --- a/.github/workflows/android-rknn.yaml +++ b/.github/workflows/android-rknn.yaml @@ -139,7 +139,7 @@ jobs: file: sherpa-onnx-*-android-rknn.tar.bz2 # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.12.17 + # tag: v1.13.8 build-android-aar-rknn: needs: [build-android-rknn-libs] @@ -274,7 +274,7 @@ jobs: file: ./*.aar # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.12.17 + # tag: v1.13.8 - name: Release android aar if: github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') diff --git a/.github/workflows/android-static.yaml b/.github/workflows/android-static.yaml index 98786c626..2a4479007 100644 --- a/.github/workflows/android-static.yaml +++ b/.github/workflows/android-static.yaml @@ -171,7 +171,7 @@ jobs: file: sherpa-onnx-*-android*.tar.bz2 # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.12.17 + # tag: v1.13.8 build-android-aar-static: needs: [build-android-static-libs] @@ -306,4 +306,4 @@ jobs: file: ./*.aar # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.12.17 + # tag: v1.13.8 diff --git a/.github/workflows/arm-linux-gnueabihf.yaml b/.github/workflows/arm-linux-gnueabihf.yaml index be91f9dc4..84d16b01c 100644 --- a/.github/workflows/arm-linux-gnueabihf.yaml +++ b/.github/workflows/arm-linux-gnueabihf.yaml @@ -177,4 +177,4 @@ jobs: file: sherpa-onnx-*linux-arm-gnueabihf*.tar.bz2 # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.12.0 + # tag: v1.13.8 diff --git a/.github/workflows/axcl-linux-aarch64.yaml b/.github/workflows/axcl-linux-aarch64.yaml index fad566683..7e7012c26 100644 --- a/.github/workflows/axcl-linux-aarch64.yaml +++ b/.github/workflows/axcl-linux-aarch64.yaml @@ -112,8 +112,6 @@ jobs: make -j4 install rm -rf install/lib/pkgconfig - rm -fv install/lib/cargs.h - rm -fv install/lib/libcargs.so ' - name: Display system info @@ -234,4 +232,4 @@ jobs: file: sherpa-onnx-*linux-aarch64*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.19 + tag: v1.13.8 diff --git a/.github/workflows/axera-linux-aarch64.yaml b/.github/workflows/axera-linux-aarch64.yaml index 373b7d2a4..0b61f6ef3 100644 --- a/.github/workflows/axera-linux-aarch64.yaml +++ b/.github/workflows/axera-linux-aarch64.yaml @@ -123,8 +123,6 @@ jobs: make -j4 install rm -rf install/lib/pkgconfig - rm -fv install/lib/cargs.h - rm -fv install/lib/libcargs.so ' - name: Display system info @@ -247,4 +245,4 @@ jobs: file: sherpa-onnx-*linux-aarch64*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.19 + tag: v1.13.8 diff --git a/.github/workflows/build-wheels-aarch64.yaml b/.github/workflows/build-wheels-aarch64.yaml index d5c72ddb1..776b05392 100644 --- a/.github/workflows/build-wheels-aarch64.yaml +++ b/.github/workflows/build-wheels-aarch64.yaml @@ -104,8 +104,6 @@ jobs: echo "----" ls -lh install/lib - rm -fv install/lib/libcargs.so - echo "----" ls -lh install/bin diff --git a/.github/workflows/build-wheels-android-aarch64.yaml b/.github/workflows/build-wheels-android-aarch64.yaml index f853aae0e..b3383178f 100644 --- a/.github/workflows/build-wheels-android-aarch64.yaml +++ b/.github/workflows/build-wheels-android-aarch64.yaml @@ -71,10 +71,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -106,7 +102,6 @@ jobs: cp -v ./install/lib/lib*.so ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib mkdir -p ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api cp -v ./install/include/sherpa-onnx/c-api/*.h ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api - rm -fv ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib/libcargs.so rm -fv ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib/libonnxruntime4j_jni.so pushd ../scripts/wheel/sherpa-onnx-core diff --git a/.github/workflows/build-wheels-android-arm.yaml b/.github/workflows/build-wheels-android-arm.yaml index d8538973a..a3b876e2b 100644 --- a/.github/workflows/build-wheels-android-arm.yaml +++ b/.github/workflows/build-wheels-android-arm.yaml @@ -80,10 +80,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -115,7 +111,6 @@ jobs: cp -v ./install/lib/lib*.so ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib mkdir -p ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api cp -v ./install/include/sherpa-onnx/c-api/*.h ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api - rm -fv ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib/libcargs.so rm -fv ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib/libonnxruntime4j_jni.so pushd ../scripts/wheel/sherpa-onnx-core diff --git a/.github/workflows/build-wheels-android-i686.yaml b/.github/workflows/build-wheels-android-i686.yaml index 3b38d8acf..1ddb17a07 100644 --- a/.github/workflows/build-wheels-android-i686.yaml +++ b/.github/workflows/build-wheels-android-i686.yaml @@ -80,10 +80,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -115,7 +111,6 @@ jobs: cp -v ./install/lib/lib*.so ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib mkdir -p ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api cp -v ./install/include/sherpa-onnx/c-api/*.h ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api - rm -fv ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib/libcargs.so rm -fv ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib/libonnxruntime4j_jni.so pushd ../scripts/wheel/sherpa-onnx-core diff --git a/.github/workflows/build-wheels-android-x86_64.yaml b/.github/workflows/build-wheels-android-x86_64.yaml index 78c00c630..4c8c01fdf 100644 --- a/.github/workflows/build-wheels-android-x86_64.yaml +++ b/.github/workflows/build-wheels-android-x86_64.yaml @@ -80,10 +80,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -115,7 +111,6 @@ jobs: cp -v ./install/lib/lib*.so ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib mkdir -p ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api cp -v ./install/include/sherpa-onnx/c-api/*.h ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/include/sherpa-onnx/c-api - rm -fv ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib/libcargs.so rm -fv ../scripts/wheel/sherpa-onnx-core/sherpa_onnx/lib/libonnxruntime4j_jni.so pushd ../scripts/wheel/sherpa-onnx-core diff --git a/.github/workflows/build-wheels-armv7l.yaml b/.github/workflows/build-wheels-armv7l.yaml index 32a5c6b81..2ae2ac934 100644 --- a/.github/workflows/build-wheels-armv7l.yaml +++ b/.github/workflows/build-wheels-armv7l.yaml @@ -114,8 +114,6 @@ jobs: file install/lib/* - rm -fv install/lib/libcargs.so - echo "----" ls -lh install/bin diff --git a/.github/workflows/build-wheels-linux.yaml b/.github/workflows/build-wheels-linux.yaml index 112db20a9..95c1e8fbe 100644 --- a/.github/workflows/build-wheels-linux.yaml +++ b/.github/workflows/build-wheels-linux.yaml @@ -114,8 +114,6 @@ jobs: echo "----" ls -lh install/lib - rm -fv install/lib/libcargs.so - echo "----" ls -lh install/bin diff --git a/.github/workflows/build-wheels-macos-arm64.yaml b/.github/workflows/build-wheels-macos-arm64.yaml index 2731e6cbf..fabac707d 100644 --- a/.github/workflows/build-wheels-macos-arm64.yaml +++ b/.github/workflows/build-wheels-macos-arm64.yaml @@ -85,11 +85,6 @@ jobs: ls -lh bin file ./bin/sherpa-onnx - - rm -fv ./install/include/cargs.h - rm -fv ./install/lib/cargs.h - rm -fv ./install/lib/libcargs.dylib - rm -fv ./install/lib/libcargs.a rm -rfv ./install/lib/pkgconfig - name: Copy files diff --git a/.github/workflows/build-wheels-macos-universal2.yaml b/.github/workflows/build-wheels-macos-universal2.yaml index 3b1ee72da..6e9499b90 100644 --- a/.github/workflows/build-wheels-macos-universal2.yaml +++ b/.github/workflows/build-wheels-macos-universal2.yaml @@ -85,11 +85,6 @@ jobs: ls -lh bin file ./bin/sherpa-onnx - - rm -fv ./install/include/cargs.h - rm -fv ./install/lib/cargs.h - rm -fv ./install/lib/libcargs.dylib - rm -fv ./install/lib/libcargs.a rm -rfv ./install/lib/pkgconfig - name: Copy files diff --git a/.github/workflows/build-wheels-macos-x64.yaml b/.github/workflows/build-wheels-macos-x64.yaml index 88bf8e091..d829b4c26 100644 --- a/.github/workflows/build-wheels-macos-x64.yaml +++ b/.github/workflows/build-wheels-macos-x64.yaml @@ -86,11 +86,6 @@ jobs: ls -lh bin file ./bin/sherpa-onnx - - rm -fv ./install/include/cargs.h - rm -fv ./install/lib/cargs.h - rm -fv ./install/lib/libcargs.dylib - rm -fv ./install/lib/libcargs.a rm -rfv ./install/lib/pkgconfig - name: Copy files diff --git a/.github/workflows/build-xcframework.yaml b/.github/workflows/build-xcframework.yaml index 2f9d19a61..e3e50c454 100644 --- a/.github/workflows/build-xcframework.yaml +++ b/.github/workflows/build-xcframework.yaml @@ -513,15 +513,13 @@ jobs: # Restore root Package.swift for static test (already patched with local paths) # Patch ios-swiftui/ios-swift xcodeproj files to use local package find ios-swiftui ios-swift -name "project.pbxproj" -exec sed -i '' \ - -e 's|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|' \ - -e 's|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|' \ - -e '/requirement = {/,/};/d' \ + -e '/\/\* XCRemoteSwiftPackageReference "sherpa-onnx" \*\/ = {/,/^\t\t};/{s|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|;s|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|;/requirement = {/,/};/d;}' \ {} + # Ensure we're using the static product in spm-examples # (Root Package.swift already has static binary targets patched to local paths) - for app in SherpaOnnxAsr SherpaOnnx2Pass SherpaOnnxTts; do + for app in SherpaOnnxAsr SherpaOnnx2Pass SherpaOnnxTtsLexicon SherpaOnnxTtsPhonemize; do echo "=== Building ios-swiftui/$app (static, local SPM) ===" cd ios-swiftui/$app xcodebuild -project $app.xcodeproj \ @@ -560,7 +558,7 @@ jobs: open('Package.swift', 'w').write(text) " - for app in SherpaOnnxAsr SherpaOnnx2Pass SherpaOnnxTts; do + for app in SherpaOnnxAsr SherpaOnnx2Pass SherpaOnnxTtsLexicon SherpaOnnxTtsPhonemize; do echo "=== Building ios-swiftui/$app (shared, local SPM) ===" cd ios-swiftui/$app xcodebuild -project $app.xcodeproj \ diff --git a/.github/workflows/c-api.yaml b/.github/workflows/c-api.yaml index 369ad3a00..9f3769b59 100644 --- a/.github/workflows/c-api.yaml +++ b/.github/workflows/c-api.yaml @@ -1003,6 +1003,8 @@ jobs: curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH @@ -1011,6 +1013,7 @@ jobs: rm $name rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia rm -f vocos_24khz.onnx + rm -f lexicon-en-us.txt ls -lh ./generated-zipvoice-zh-en-c.wav @@ -1035,6 +1038,8 @@ jobs: tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH @@ -1042,6 +1047,7 @@ jobs: rm $name rm -rf kitten-nano-en-v0_1-fp16 + rm -f lexicon-en-us.txt - uses: actions/upload-artifact@v4 with: @@ -1082,6 +1088,8 @@ jobs: tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH @@ -1089,6 +1097,7 @@ jobs: rm ./kokoro-tts-en-c-api rm -rf kokoro-en-* + rm -f lexicon-en-us.txt - uses: actions/upload-artifact@v4 with: @@ -1134,6 +1143,8 @@ jobs: curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + export LD_LIBRARY_PATH=$PWD/build/install/lib:$LD_LIBRARY_PATH export DYLD_LIBRARY_PATH=$PWD/build/install/lib:$DYLD_LIBRARY_PATH @@ -1142,6 +1153,7 @@ jobs: rm ./matcha-tts-en-c-api rm -rf matcha-icefall-* rm vocos-22khz-univ.onnx + rm -f lexicon-en-us.txt - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/cxx-api.yaml b/.github/workflows/cxx-api.yaml index 9c945b693..a79169134 100644 --- a/.github/workflows/cxx-api.yaml +++ b/.github/workflows/cxx-api.yaml @@ -243,6 +243,35 @@ jobs: rm -rf sherpa-onnx-zipformer-ctc-en-* rm -v ./$name + - name: Test streaming Zipformer CTC with an HLG graph + shell: bash + run: | + name=streaming-hlg-decode-file-cxx-api + g++ -std=c++17 -o $name ./cxx-api-examples/$name.cc \ + -I ./build/install/include \ + -L ./build/install/lib/ \ + -l sherpa-onnx-cxx-api \ + -l sherpa-onnx-c-api \ + -l onnxruntime + + ls -lh $name + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 + tar xvf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 + rm sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 + + echo "---" + + # A clean exit proves nothing here: without a graph the decoder + # still succeeds, it just decodes freely. The word ids show up only + # when the FST decoder ran. + ./$name > /tmp/hlg-streaming.txt 2>&1 + cat /tmp/hlg-streaming.txt + grep -q '"words": \[[0-9]' /tmp/hlg-streaming.txt + + rm -rf sherpa-onnx-streaming-zipformer-ctc-* + rm -v ./$name + - name: Test FunASR Nano shell: bash run: | @@ -818,12 +847,15 @@ jobs: curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + echo "---" ./$name rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia rm -f vocos_24khz.onnx + rm -f lexicon-en-us.txt rm -v ./$name ls -lh ./generated-zipvoice-zh-en-cxx.wav @@ -850,11 +882,14 @@ jobs: tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + echo "---" ./$name rm -rf kitten-nano-en-v0_1-fp16 + rm -f lexicon-en-us.txt rm -v ./$name - uses: actions/upload-artifact@v4 @@ -895,10 +930,13 @@ jobs: tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./kokoro-tts-en-cxx-api rm kokoro-tts-en-cxx-api rm -rf kokoro-en-* + rm -f lexicon-en-us.txt - uses: actions/upload-artifact@v4 with: @@ -943,11 +981,14 @@ jobs: curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./matcha-tts-en-cxx-api rm matcha-tts-en-cxx-api rm -rf matcha-icefall-* rm vocos-22khz-univ.onnx + rm -f lexicon-en-us.txt - uses: actions/upload-artifact@v4 with: @@ -971,9 +1012,12 @@ jobs: tar xf vits-piper-en_US-lessac-medium.tar.bz2 rm vits-piper-en_US-lessac-medium.tar.bz2 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./$name rm -rf vits-piper-en_US-lessac-medium + rm -f lexicon-en-us.txt rm -v ./$name ls -lh ./generated-piper-en-cxx.wav diff --git a/.github/workflows/dot-net.yaml b/.github/workflows/dot-net.yaml index 0ef0c489a..9162c1f61 100644 --- a/.github/workflows/dot-net.yaml +++ b/.github/workflows/dot-net.yaml @@ -54,7 +54,6 @@ jobs: -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ .. diff --git a/.github/workflows/flutter-android.yaml b/.github/workflows/flutter-android.yaml deleted file mode 100644 index 1b7d04185..000000000 --- a/.github/workflows/flutter-android.yaml +++ /dev/null @@ -1,526 +0,0 @@ -name: flutter-android - -on: - push: - branches: - - flutter - - apk - tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' - - workflow_dispatch: - -concurrency: - group: flutter-android-${{ github.ref }} - cancel-in-progress: true - -jobs: - asr: - if: false - name: asr ${{ matrix.index }}/${{ matrix.total }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - total: ["3"] - index: ["0", "1", "2"] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '17' - - - name: Check Java version - run: | - java -version - echo $JAVA_HOME - - - name: Set JAVA_HOME for Gradle - run: echo "JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV - - - name: Check Java version - run: | - java -version - echo $JAVA_HOME - - - name: Update version - shell: bash - run: | - ./new-release.sh - git diff . - - - name: Display NDK HOME - shell: bash - run: | - echo "ANDROID_NDK_LATEST_HOME: ${ANDROID_NDK_LATEST_HOME}" - ls -lh ${ANDROID_NDK_LATEST_HOME} - - - name: Setup build tool version variable - shell: bash - run: | - echo "---" - ls -lh /usr/local/lib/android/ - echo "---" - - ls -lh /usr/local/lib/android/sdk - echo "---" - - ls -lh /usr/local/lib/android/sdk/build-tools - echo "---" - - BUILD_TOOL_VERSION=$(ls /usr/local/lib/android/sdk/build-tools/ | tail -n 1) - echo "BUILD_TOOL_VERSION=$BUILD_TOOL_VERSION" >> $GITHUB_ENV - echo "Last build tool version is: $BUILD_TOOL_VERSION" - - - name: Install Python dependencies - shell: bash - run: | - python3 -m pip install --upgrade jinja2 iso639-lang - - - name: Install deps - shell: bash - run: | - sudo apt-get update -y - sudo apt-get install -y build-essential jq git cmake - sudo apt-get install -y curl - - - name: Setup Flutter SDK - uses: flutter-actions/setup-flutter@v4 - with: - channel: stable - version: 3.29.0 - cache: true - - - name: Install ninja - shell: bash - run: | - sudo apt-get install -y ninja-build - - - name: Display ninja version - shell: bash - run: | - ninja --version - ninja --help || true - which ninja - - - name: Display PWD - shell: bash - run: | - echo "pwd: $PWD" - ls -lh - - - name: Display machine info - shell: bash - run: | - uname -a - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - - git config --global --add safe.directory /__t/flutter-Linux-*/flutter || true - - flutter --version - - dart --version - flutter doctor - - - name: Install libgtk-3-dev - shell: bash - run: | - sudo apt install -y libgtk-3-dev tree clang pkg-config - - - name: Accept Android licenses - run: yes | $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager --licenses - - - name: Install Android SDK Components - run: | - $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager "platforms;android-35" "build-tools;35.0.0" - - - name: Install NDK 27 - run: | - $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager "ndk;27.0.12077973" - - - name: Display flutter info (2) - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - cd .. - - - name: Build flutter - shell: bash - run: | - cd scripts/flutter - - total=${{ matrix.total }} - index=${{ matrix.index }} - - ./generate-streaming-asr.py --total $total --index $index - - chmod +x *.sh - ./build-android-streaming-asr.sh - - cd ../../ - - ls -lh *.apk - - - name: Display generated files - shell: bash - run: | - ls -lh *.apk - - mkdir apks - - mv -v *.apk ./apks - - # https://github.com/marketplace/actions/sign-android-release - - uses: r0adkll/sign-android-release@v1 - name: Sign app APK - with: - releaseDirectory: ./apks - signingKeyBase64: ${{ secrets.ANDROID_SIGNING_KEY }} - alias: ${{ secrets.ANDROID_SIGNING_KEY_ALIAS }} - keyStorePassword: ${{ secrets.ANDROID_SIGNING_KEY_STORE_PASSWORD }} - env: - BUILD_TOOLS_VERSION: ${{ env.BUILD_TOOL_VERSION }} - - - name: Display APK after signing - shell: bash - run: | - ls -lh ./apks/ - du -h -d1 . - - - name: Rename APK after signing - shell: bash - run: | - cd apks - rm -fv signingKey.jks - rm -fv *.apk.idsig - rm -fv *-aligned.apk - - all_apks=$(ls -1 *-signed.apk) - echo "----" - echo $all_apks - echo "----" - for apk in ${all_apks[@]}; do - n=$(echo $apk | sed -e s/-signed//) - mv -v $apk $n - done - - cd .. - - ls -lh ./apks/ - du -h -d1 . - - - name: Display APK after rename - shell: bash - run: | - ls -lh ./apks/ - du -h -d1 . - - - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' || github.repository_owner == 'csu-fangjun') && ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') || contains(github.ref, 'refs/tags/')) - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - uses: nick-fields/retry@v3 - with: - max_attempts: 20 - timeout_seconds: 200 - shell: bash - command: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" - - rm -rf huggingface - export GIT_LFS_SKIP_SMUDGE=1 - export GIT_CLONE_PROTECTION_ACTIVE=false - - SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - - git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter huggingface - cd huggingface - git fetch - git pull - git merge -m "merge remote" --ff origin main - - dst=flutter/asr/android/$SHERPA_ONNX_VERSION - mkdir -p $dst - - cp -v ../apks/*.apk $dst - - git status - git lfs track "*.apk" - git add . - git commit -m "add more files" - git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter main - - tts: - if: false - name: tts ${{ matrix.index }}/${{ matrix.total }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - total: ["15"] - index: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14"] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '17' - - - name: Check Java version - run: | - java -version - echo $JAVA_HOME - - - name: Set JAVA_HOME for Gradle - run: echo "JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV - - - name: Check Java version - run: | - java -version - echo $JAVA_HOME - - - name: Update version - shell: bash - run: | - ./new-release.sh - git diff . - - - name: Display NDK HOME - shell: bash - run: | - echo "ANDROID_NDK_LATEST_HOME: ${ANDROID_NDK_LATEST_HOME}" - ls -lh ${ANDROID_NDK_LATEST_HOME} - - - name: Setup build tool version variable - shell: bash - run: | - echo "---" - ls -lh /usr/local/lib/android/ - echo "---" - - ls -lh /usr/local/lib/android/sdk - echo "---" - - ls -lh /usr/local/lib/android/sdk/build-tools - echo "---" - - BUILD_TOOL_VERSION=$(ls /usr/local/lib/android/sdk/build-tools/ | tail -n 1) - echo "BUILD_TOOL_VERSION=$BUILD_TOOL_VERSION" >> $GITHUB_ENV - echo "Last build tool version is: $BUILD_TOOL_VERSION" - - - name: Install Python dependencies - shell: bash - run: | - python3 -m pip install --upgrade jinja2 iso639-lang - - - name: Install deps - shell: bash - run: | - sudo apt-get update -y - sudo apt-get install -y build-essential jq git cmake - sudo apt-get install -y curl - - - name: Setup Flutter SDK - uses: flutter-actions/setup-flutter@v4 - with: - channel: stable - version: 3.29.0 - cache: true - - - name: Install ninja - shell: bash - run: | - sudo apt-get install -y ninja-build - - - name: Display ninja version - shell: bash - run: | - ninja --version - ninja --help || true - which ninja - - - name: Display PWD - shell: bash - run: | - echo "pwd: $PWD" - ls -lh - - - name: Display machine info - shell: bash - run: | - uname -a - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - - git config --global --add safe.directory /__t/flutter-Linux-*/flutter || true - - flutter --version - - dart --version - flutter doctor - - - name: Install libgtk-3-dev - shell: bash - run: | - sudo apt install -y libgtk-3-dev tree clang pkg-config - - - name: Accept Android licenses - run: yes | $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager --licenses - - - name: Install Android SDK Components - run: | - $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager "platforms;android-35" "build-tools;35.0.0" - - - name: Display flutter info (2) - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - cd .. - - - name: Build flutter - shell: bash - run: | - cd scripts/flutter - - total=${{ matrix.total }} - index=${{ matrix.index }} - - ./generate-tts.py --total $total --index $index - - chmod +x *.sh - ./build-android-tts.sh - - cd ../../ - - ls -lh *.apk - - - name: Display generated files - shell: bash - run: | - ls -lh *.apk - - mkdir apks - - mv -v *.apk ./apks - - # https://github.com/marketplace/actions/sign-android-release - - uses: r0adkll/sign-android-release@v1 - name: Sign app APK - with: - releaseDirectory: ./apks - signingKeyBase64: ${{ secrets.ANDROID_SIGNING_KEY }} - alias: ${{ secrets.ANDROID_SIGNING_KEY_ALIAS }} - keyStorePassword: ${{ secrets.ANDROID_SIGNING_KEY_STORE_PASSWORD }} - env: - BUILD_TOOLS_VERSION: ${{ env.BUILD_TOOL_VERSION }} - - - name: Display APK after signing - shell: bash - run: | - ls -lh ./apks/ - du -h -d1 . - - - name: Rename APK after signing - shell: bash - run: | - cd apks - rm -fv signingKey.jks - rm -fv *.apk.idsig - rm -fv *-aligned.apk - - all_apks=$(ls -1 *-signed.apk) - echo "----" - echo $all_apks - echo "----" - for apk in ${all_apks[@]}; do - n=$(echo $apk | sed -e s/-signed//) - mv -v $apk $n - done - - cd .. - - ls -lh ./apks/ - du -h -d1 . - - - name: Display APK after rename - shell: bash - run: | - ls -lh ./apks/ - du -h -d1 . - - - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' || github.repository_owner == 'csu-fangjun') && ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') || contains(github.ref, 'refs/tags/')) - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - uses: nick-fields/retry@v3 - with: - max_attempts: 20 - timeout_seconds: 200 - shell: bash - command: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" - - rm -rf huggingface - export GIT_LFS_SKIP_SMUDGE=1 - export GIT_CLONE_PROTECTION_ACTIVE=false - - SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - - git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter huggingface - cd huggingface - git fetch - git pull - git merge -m "merge remote" --ff origin main - - dst=flutter/tts/android/$SHERPA_ONNX_VERSION - mkdir -p $dst - - cp -v ../apks/*.apk $dst - - git status - git lfs track "*.apk" - git add . - git commit -m "add more files" - git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter main diff --git a/.github/workflows/flutter-linux.yaml b/.github/workflows/flutter-linux.yaml deleted file mode 100644 index aeb7734ce..000000000 --- a/.github/workflows/flutter-linux.yaml +++ /dev/null @@ -1,290 +0,0 @@ -name: flutter-linux - -on: - push: - branches: - - flutter - - apk - tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' - - workflow_dispatch: - -concurrency: - group: flutter-linux-${{ github.ref }} - cancel-in-progress: true - -# See https://github.com/actions/checkout/issues/1590#issuecomment-2207052044 -# and -# https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/ -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - -jobs: - asr: - if: false - name: asr ${{ matrix.arch }} ${{ matrix.index }}/${{ matrix.total }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - arch: [x86_64] - total: ["3"] - index: ["0", "1", "2"] - - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Update version - shell: bash - run: | - ./new-release.sh - git diff . - - - name: Install Python dependencies - shell: bash - run: | - python3 -m pip install --upgrade jinja2 - - - name: Setup Flutter SDK - uses: flutter-actions/setup-flutter@v4 - with: - channel: stable - version: 3.24.3 - cache: true - - - name: Display PWD - shell: bash - run: | - echo "pwd: $PWD" - ls -lh - - - name: Display machine info - shell: bash - run: | - uname -a - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - - name: Install libgtk-3-dev - shell: bash - run: | - sudo apt install -y libgtk-3-dev tree clang pkg-config - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - - name: Build flutter - shell: bash - run: | - export arch=${{ matrix.arch }} - cd scripts/flutter - - total=${{ matrix.total }} - index=${{ matrix.index }} - - ./generate-streaming-asr.py --total $total --index $index - - chmod +x *.sh - ./build-linux-streaming-asr.sh - cd ../../ - ls -lh *.tar.bz2 - - - name: Display generated files - shell: bash - run: | - ls -lh *.tar.bz2 - - - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' || github.repository_owner == 'csu-fangjun') && ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') || contains(github.ref, 'refs/tags/')) - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - uses: nick-fields/retry@v3 - with: - max_attempts: 20 - timeout_seconds: 200 - shell: bash - command: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" - - rm -rf huggingface - export GIT_LFS_SKIP_SMUDGE=1 - export GIT_CLONE_PROTECTION_ACTIVE=false - - SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - - git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter huggingface - cd huggingface - git fetch - git pull - git merge -m "merge remote" --ff origin main - - dst=flutter/asr/linux/$SHERPA_ONNX_VERSION - mkdir -p $dst - - cp -v ../*.tar.bz2 $dst - - git status - git lfs track "*.bz2" - git add . - git commit -m "add more files" - git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter main - - tts: - if: false - name: tts ${{ matrix.arch }} ${{ matrix.index }}/${{ matrix.total }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - arch: [x86_64] - total: ["20"] - index: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"] - - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Update version - shell: bash - run: | - ./new-release.sh - git diff . - - - name: Install Python dependencies - shell: bash - run: | - python3 -m pip install --upgrade jinja2 - - - name: Setup Flutter SDK - uses: flutter-actions/setup-flutter@v4 - with: - channel: stable - version: 3.24.3 - cache: true - - - name: Display PWD - shell: bash - run: | - echo "pwd: $PWD" - ls -lh - - - name: Display machine info - shell: bash - run: | - uname -a - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - - name: Install libgtk-3-dev - shell: bash - run: | - sudo apt install -y libgtk-3-dev tree clang pkg-config - - - name: Install deps - shell: bash - run: | - sudo apt-get update -y - sudo apt-get install -y build-essential jq git python3-pip - sudo apt-get install -y curl - sudo apt-get install -y libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libunwind-dev - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - - name: Build flutter - shell: bash - run: | - export arch=${{ matrix.arch }} - cd scripts/flutter - - total=${{ matrix.total }} - index=${{ matrix.index }} - - ./generate-tts.py --total $total --index $index - - chmod +x *.sh - ./build-linux-tts.sh - cd ../../ - ls -lh *.tar.bz2 - - - name: Display generated files - shell: bash - run: | - ls -lh *.tar.bz2 - - - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' || github.repository_owner == 'csu-fangjun') && ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') || contains(github.ref, 'refs/tags/')) - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - uses: nick-fields/retry@v3 - with: - max_attempts: 20 - timeout_seconds: 200 - shell: bash - command: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" - - rm -rf huggingface - export GIT_LFS_SKIP_SMUDGE=1 - export GIT_CLONE_PROTECTION_ACTIVE=false - - SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - - git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter huggingface - cd huggingface - git fetch - git pull - git merge -m "merge remote" --ff origin main - - dst=flutter/tts/linux/$SHERPA_ONNX_VERSION - mkdir -p $dst - - cp -v ../*.tar.bz2 $dst - - git status - git lfs track "*.bz2" - git add . - git commit -m "add more files" - git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter main diff --git a/.github/workflows/flutter-macos.yaml b/.github/workflows/flutter-macos.yaml deleted file mode 100644 index 424240807..000000000 --- a/.github/workflows/flutter-macos.yaml +++ /dev/null @@ -1,242 +0,0 @@ -name: flutter-macos - -on: - push: - branches: - - flutter - - apk - tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' - - workflow_dispatch: - -concurrency: - group: flutter-macos-${{ github.ref }} - cancel-in-progress: true - -jobs: - asr: - if: false - name: asr ${{ matrix.arch }}/${{ matrix.total }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macos-latest] - arch: [x86_64, arm64] - total: ["3"] - index: ["0", "1", "2"] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Update version - shell: bash - run: | - ./new-release.sh - git diff . - - - name: Install Python dependencies - shell: bash - run: | - python3 -m pip install --break-system-packages --upgrade jinja2 - - - name: Setup Flutter SDK - uses: flutter-actions/setup-flutter@v4 - with: - channel: stable - version: 3.24.3 - cache: true - - - name: Display PWD - shell: bash - run: | - echo "pwd: $PWD" - ls -lh - - - name: Display machine info - shell: bash - run: | - uname -a - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - - name: Build flutter - shell: bash - run: | - export arch=${{ matrix.arch }} - cd scripts/flutter - - total=${{ matrix.total }} - index=${{ matrix.index }} - - ./generate-streaming-asr.py --total $total --index $index - - chmod +x *.sh - ./build-macos-streaming-asr.sh - cd ../../ - ls -lh *.tar.bz2 - - - name: Display generated files - shell: bash - run: | - ls -lh *.tar.bz2 - - - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' || github.repository_owner == 'csu-fangjun') && ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') || contains(github.ref, 'refs/tags/')) - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - uses: nick-fields/retry@v3 - with: - max_attempts: 20 - timeout_seconds: 200 - shell: bash - command: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" - - rm -rf huggingface - export GIT_LFS_SKIP_SMUDGE=1 - export GIT_CLONE_PROTECTION_ACTIVE=false - - SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - - git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter huggingface - cd huggingface - git fetch - git pull - git merge -m "merge remote" --ff origin main - - dst=flutter/asr/macos/$SHERPA_ONNX_VERSION - mkdir -p $dst - - cp -v ../*.tar.bz2 $dst - - git status - git lfs track "*.bz2" - git add . - git commit -m "add more files" - git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter main - - tts: - if: false - name: tts ${{ matrix.arch }} ${{ matrix.index }}/${{ matrix.total }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macos-latest] - arch: [x86_64, arm64] - total: ["10"] - index: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Update version - shell: bash - run: | - ./new-release.sh - git diff . - - - name: Install Python dependencies - shell: bash - run: | - python3 -m pip install --break-system-packages --upgrade jinja2 iso639-lang - - - name: Setup Flutter SDK - uses: flutter-actions/setup-flutter@v4 - with: - channel: stable - version: 3.24.3 - cache: true - - - name: Display PWD - shell: bash - run: | - echo "pwd: $PWD" - ls -lh - - - name: Display machine info - shell: bash - run: | - uname -a - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - - name: Build flutter - shell: bash - run: | - export arch=${{ matrix.arch }} - cd scripts/flutter - - total=${{ matrix.total }} - index=${{ matrix.index }} - - ./generate-tts.py --total $total --index $index - - chmod +x *.sh - ./build-macos-tts.sh - cd ../../ - ls -lh *.tar.bz2 - - - name: Display generated files - shell: bash - run: | - ls -lh *.tar.bz2 - - - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' || github.repository_owner == 'csu-fangjun') && ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') || contains(github.ref, 'refs/tags/')) - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - uses: nick-fields/retry@v3 - with: - max_attempts: 20 - timeout_seconds: 200 - shell: bash - command: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" - - rm -rf huggingface - export GIT_LFS_SKIP_SMUDGE=1 - export GIT_CLONE_PROTECTION_ACTIVE=false - - SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - - git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter huggingface - cd huggingface - git fetch - git pull - git merge -m "merge remote" --ff origin main - dst=flutter/tts/macos/$SHERPA_ONNX_VERSION - mkdir -p $dst - cp -v ../*.tar.bz2 $dst - - git status - git lfs track "*.bz2" - git add . - git commit -m "add more files" - git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter main diff --git a/.github/workflows/flutter-windows-x64.yaml b/.github/workflows/flutter-windows-x64.yaml deleted file mode 100644 index 712e73a95..000000000 --- a/.github/workflows/flutter-windows-x64.yaml +++ /dev/null @@ -1,227 +0,0 @@ -name: flutter-windows-x64 - -on: - push: - branches: - - flutter - - apk - tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' - - workflow_dispatch: - -concurrency: - group: flutter-windows-x64-${{ github.ref }} - cancel-in-progress: true - -jobs: - asr: - if: false - name: asr ${{ matrix.index }}/${{ matrix.total }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [windows-2022] - total: ["3"] - index: ["0", "1", "2"] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Update version - shell: bash - run: | - ./new-release.sh - git diff . - - - name: Install Python dependencies - shell: bash - run: | - python3 -m pip install --upgrade jinja2 - - - name: Setup Flutter SDK - uses: flutter-actions/setup-flutter@v4 - with: - channel: stable - version: 3.24.3 - cache: true - - - name: Display PWD - shell: bash - run: | - echo "pwd: $PWD" - ls -lh - - - name: Display machine info - shell: bash - run: | - uname -a - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - - name: Build flutter - shell: bash - run: | - cd scripts/flutter - - total=${{ matrix.total }} - index=${{ matrix.index }} - - ./generate-streaming-asr.py --total $total --index $index - - chmod +x *.sh - ./build-windows-streaming-asr.sh - cd ../../ - ls -lh *.tar.bz2 - - - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' || github.repository_owner == 'csu-fangjun') && ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') || contains(github.ref, 'refs/tags/')) - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - uses: nick-fields/retry@v3 - with: - max_attempts: 20 - timeout_seconds: 200 - shell: bash - command: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" - - rm -rf huggingface - export GIT_LFS_SKIP_SMUDGE=1 - export GIT_CLONE_PROTECTION_ACTIVE=false - - SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - - git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter huggingface - cd huggingface - git fetch - git pull - git merge -m "merge remote" --ff origin main - - dst=flutter/asr/windows/$SHERPA_ONNX_VERSION - mkdir -p $dst - cp -v ../*.tar.bz2 $dst - - git status - git lfs track "*.bz2" - git add . - git commit -m "add more files" - git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter main - - tts: - if: false - name: tts ${{ matrix.index }}/${{ matrix.total }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [windows-2022] - total: ["20"] - index: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12", "13", "14", "15", "16", "17", "18", "19"] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Update version - shell: bash - run: | - ./new-release.sh - git diff . - - - name: Install Python dependencies - shell: bash - run: | - python3 -m pip install --upgrade jinja2 iso639-lang - - - name: Setup Flutter SDK - uses: flutter-actions/setup-flutter@v3 - with: - channel: stable - version: latest - - - name: Display PWD - shell: bash - run: | - echo "pwd: $PWD" - ls -lh - - - name: Display machine info - shell: bash - run: | - uname -a - - - name: Display flutter info - shell: bash - run: | - which flutter - which dart - - flutter --version - dart --version - flutter doctor - - - name: Build flutter - shell: bash - run: | - cd scripts/flutter - - total=${{ matrix.total }} - index=${{ matrix.index }} - - ./generate-tts.py --total $total --index $index - - chmod +x *.sh - ./build-windows-tts.sh - cd ../../ - ls -lh *.tar.bz2 - - - name: Publish to huggingface - if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' || github.repository_owner == 'csu-fangjun') && ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') || contains(github.ref, 'refs/tags/')) - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - uses: nick-fields/retry@v3 - with: - max_attempts: 20 - timeout_seconds: 200 - shell: bash - command: | - git config --global user.email "csukuangfj@gmail.com" - git config --global user.name "Fangjun Kuang" - - rm -rf huggingface - export GIT_LFS_SKIP_SMUDGE=1 - export GIT_CLONE_PROTECTION_ACTIVE=false - - SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) - echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" - - git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter huggingface - cd huggingface - git fetch - git pull - git merge -m "merge remote" --ff origin main - - dst=flutter/tts/windows/$SHERPA_ONNX_VERSION - mkdir -p $dst - cp -v ../*.tar.bz2 $dst - - git status - git lfs track "*.bz2" - git add . - git commit -m "add more files" - git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/sherpa-onnx-flutter main diff --git a/.github/workflows/har.yaml b/.github/workflows/har.yaml index 29cbb8fa6..d61f5a72a 100644 --- a/.github/workflows/har.yaml +++ b/.github/workflows/har.yaml @@ -182,7 +182,7 @@ jobs: file: ./*.har # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.12.30 + # tag: v1.13.8 - name: Publish to huggingface if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') diff --git a/.github/workflows/harmony-os.yaml b/.github/workflows/harmony-os.yaml index d493dfb9b..fbe462860 100644 --- a/.github/workflows/harmony-os.yaml +++ b/.github/workflows/harmony-os.yaml @@ -162,4 +162,4 @@ jobs: file: ./*.tar.bz2 # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.10.23 + # tag: v1.13.8 diff --git a/.github/workflows/ios.yaml b/.github/workflows/ios.yaml index 792a30eb3..3fc174b6c 100644 --- a/.github/workflows/ios.yaml +++ b/.github/workflows/ios.yaml @@ -39,9 +39,7 @@ jobs: run: | set -euo pipefail find ios-swiftui ios-swift -name "project.pbxproj" -exec sed -i '' \ - -e 's|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|' \ - -e 's|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|' \ - -e '/requirement = {/,/};/d' \ + -e '/\/\* XCRemoteSwiftPackageReference "sherpa-onnx" \*\/ = {/,/^\t\t};/{s|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|;s|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|;/requirement = {/,/};/d;}' \ {} + git diff cat ios-swiftui/SherpaOnnxAsr/SherpaOnnxAsr.xcodeproj/project.pbxproj @@ -97,9 +95,7 @@ jobs: run: | set -euo pipefail find ios-swiftui ios-swift -name "project.pbxproj" -exec sed -i '' \ - -e 's|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|' \ - -e 's|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|' \ - -e '/requirement = {/,/};/d' \ + -e '/\/\* XCRemoteSwiftPackageReference "sherpa-onnx" \*\/ = {/,/^\t\t};/{s|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|;s|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|;/requirement = {/,/};/d;}' \ {} + git diff cat ios-swiftui/SherpaOnnx2Pass/SherpaOnnx2Pass.xcodeproj/project.pbxproj @@ -155,9 +151,7 @@ jobs: run: | set -euo pipefail find ios-swiftui ios-swift -name "project.pbxproj" -exec sed -i '' \ - -e 's|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|' \ - -e 's|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|' \ - -e '/requirement = {/,/};/d' \ + -e '/\/\* XCRemoteSwiftPackageReference "sherpa-onnx" \*\/ = {/,/^\t\t};/{s|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|;s|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|;/requirement = {/,/};/d;}' \ {} + git diff cat ios-swiftui/SherpaOnnxLangID/SherpaOnnxLangID.xcodeproj/project.pbxproj @@ -213,9 +207,7 @@ jobs: run: | set -euo pipefail find ios-swiftui ios-swift -name "project.pbxproj" -exec sed -i '' \ - -e 's|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|' \ - -e 's|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|' \ - -e '/requirement = {/,/};/d' \ + -e '/\/\* XCRemoteSwiftPackageReference "sherpa-onnx" \*\/ = {/,/^\t\t};/{s|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|;s|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|;/requirement = {/,/};/d;}' \ {} + git diff cat ios-swiftui/SherpaOnnxSubtitle/SherpaOnnxSubtitle.xcodeproj/project.pbxproj @@ -271,20 +263,18 @@ jobs: run: | set -euo pipefail find ios-swiftui ios-swift -name "project.pbxproj" -exec sed -i '' \ - -e 's|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|' \ - -e 's|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|' \ - -e '/requirement = {/,/};/d' \ + -e '/\/\* XCRemoteSwiftPackageReference "sherpa-onnx" \*\/ = {/,/^\t\t};/{s|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|;s|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|;/requirement = {/,/};/d;}' \ {} + git diff - cat ios-swiftui/SherpaOnnxTts/SherpaOnnxTts.xcodeproj/project.pbxproj + cat ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon.xcodeproj/project.pbxproj - - name: Build SherpaOnnxTts (local) + - name: Build SherpaOnnxTtsLexicon (local) shell: bash run: | set -euo pipefail - cd ios-swiftui/SherpaOnnxTts - xcodebuild -project SherpaOnnxTts.xcodeproj \ - -scheme SherpaOnnxTts \ + cd ios-swiftui/SherpaOnnxTtsLexicon + xcodebuild -project SherpaOnnxTtsLexicon.xcodeproj \ + -scheme SherpaOnnxTtsLexicon \ -destination 'generic/platform=iOS Simulator' \ -skipPackagePluginValidation \ build @@ -301,13 +291,69 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Build SherpaOnnxTts + - name: Build SherpaOnnxTtsLexicon shell: bash run: | set -euo pipefail - cd ios-swiftui/SherpaOnnxTts - xcodebuild -project SherpaOnnxTts.xcodeproj \ - -scheme SherpaOnnxTts \ + cd ios-swiftui/SherpaOnnxTtsLexicon + xcodebuild -project SherpaOnnxTtsLexicon.xcodeproj \ + -scheme SherpaOnnxTtsLexicon \ + -destination 'generic/platform=iOS Simulator' \ + -skipPackagePluginValidation \ + build + + sherpa-onnx-tts-phonemize-local: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, macos-15-intel] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Patch to local package + shell: bash + run: | + set -euo pipefail + find ios-swiftui ios-swift -name "project.pbxproj" -exec sed -i '' \ + -e '/\/\* XCRemoteSwiftPackageReference "sherpa-onnx" \*\/ = {/,/^\t\t};/{s|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|;s|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|;/requirement = {/,/};/d;}' \ + {} + + git diff + cat ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.pbxproj + + - name: Build SherpaOnnxTtsPhonemize (local) + shell: bash + run: | + set -euo pipefail + cd ios-swiftui/SherpaOnnxTtsPhonemize + xcodebuild -project SherpaOnnxTtsPhonemize.xcodeproj \ + -scheme SherpaOnnxTtsPhonemize \ + -destination 'generic/platform=iOS Simulator' \ + -skipPackagePluginValidation \ + build + + sherpa-onnx-tts-phonemize: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, macos-15-intel] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Build SherpaOnnxTtsPhonemize + shell: bash + run: | + set -euo pipefail + cd ios-swiftui/SherpaOnnxTtsPhonemize + xcodebuild -project SherpaOnnxTtsPhonemize.xcodeproj \ + -scheme SherpaOnnxTtsPhonemize \ -destination 'generic/platform=iOS Simulator' \ -skipPackagePluginValidation \ build @@ -329,9 +375,7 @@ jobs: run: | set -euo pipefail find ios-swiftui ios-swift -name "project.pbxproj" -exec sed -i '' \ - -e 's|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|' \ - -e 's|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|' \ - -e '/requirement = {/,/};/d' \ + -e '/\/\* XCRemoteSwiftPackageReference "sherpa-onnx" \*\/ = {/,/^\t\t};/{s|isa = XCRemoteSwiftPackageReference|isa = XCLocalSwiftPackageReference|;s|repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"|relativePath = ../../|;/requirement = {/,/};/d;}' \ {} + git diff cat ios-swift/SherpaOnnxAsr/SherpaOnnxAsr.xcodeproj/project.pbxproj diff --git a/.github/workflows/jar.yaml b/.github/workflows/jar.yaml index cdd12840f..10dc8fda6 100644 --- a/.github/workflows/jar.yaml +++ b/.github/workflows/jar.yaml @@ -315,6 +315,8 @@ jobs: tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + java \ -cp "../sherpa-onnx/java-api/target/sherpa-onnx-jvm-$SHERPA_ONNX_VERSION.jar${SEP}../sherpa-onnx/java-api/$native_jar" \ - NonStreamingTtsKittenEn.java + NonStreamingTtsKittenEnLexicon.java diff --git a/.github/workflows/linux-gpu.yaml b/.github/workflows/linux-gpu.yaml index a97c5a13a..8f6cc303a 100644 --- a/.github/workflows/linux-gpu.yaml +++ b/.github/workflows/linux-gpu.yaml @@ -136,10 +136,6 @@ jobs: file build/bin/sherpa-onnx ls -lh build/bin/sherpa-onnx readelf -d build/bin/sherpa-onnx - - rm -fv build/install/include/cargs.h - rm -fv build/install/lib/cargs.h - rm -fv build/install/lib/libcargs.so rm -rfv build/install/lib/pkgconfig strings build/install/lib/*.so | grep "^GLIBC_" diff --git a/.github/workflows/linux-jni-aarch64.yaml b/.github/workflows/linux-jni-aarch64.yaml index 8e8f662a3..b2bb34e8b 100644 --- a/.github/workflows/linux-jni-aarch64.yaml +++ b/.github/workflows/linux-jni-aarch64.yaml @@ -103,9 +103,6 @@ jobs: ls -lh lib rm -rf ./install/lib/pkgconfig rm -rf ./install/lib/share - rm -rf ./install/lib/cargs.h - rm -rf ./install/include/cargs.h - rm -rf ./install/lib/libcargs.so rm -rf ./install/lib/libsherpa-onnx-c-api.so echo "----" diff --git a/.github/workflows/linux-jni.yaml b/.github/workflows/linux-jni.yaml index 133ff9340..8716761be 100644 --- a/.github/workflows/linux-jni.yaml +++ b/.github/workflows/linux-jni.yaml @@ -137,9 +137,6 @@ jobs: ls -lh bin rm -rf ./install/lib/pkgconfig rm -rf ./install/lib/share - rm -rf ./install/lib/cargs.h - rm -rf ./install/include/cargs.h - rm -rf ./install/lib/libcargs.so rm -rf ./install/lib/libsherpa-onnx-c-api.so rm -rf ./install/lib/libsherpa-onnx-cxx-api.so diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 9a3e43d97..048fe0543 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -184,10 +184,6 @@ jobs: file build/bin/sherpa-onnx ls -lh build/bin/sherpa-onnx readelf -d build/bin/sherpa-onnx - - rm -fv build/install/include/cargs.h - rm -fv build/install/lib/cargs.h - rm -fv build/install/lib/libcargs.so rm -rfv build/install/lib/pkgconfig - uses: actions/upload-artifact@v4 @@ -230,10 +226,8 @@ jobs: mkdir -p ${lib_dst}/lib if [[ ${{ matrix.shared_lib }} == ON ]]; then cp -av build/install/lib/lib*.so* ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/libcargs.so ${lib_dst}/lib/libcargs.so.* else cp -av build/install/lib/lib*.a ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/libcargs.a fi tree ${lib_dst} tar cjvf ${lib_dst}.tar.bz2 ${lib_dst} @@ -249,7 +243,7 @@ jobs: file: sherpa-onnx-*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.31 + tag: v1.13.8 - name: Release pre-compiled binaries and libs for linux x64 if: github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.build_type == 'Release' diff --git a/.github/workflows/macos-jni.yaml b/.github/workflows/macos-jni.yaml index daf71c68f..b2cce9821 100644 --- a/.github/workflows/macos-jni.yaml +++ b/.github/workflows/macos-jni.yaml @@ -79,9 +79,6 @@ jobs: rm -rf ./install/lib/pkgconfig rm -rf ./install/lib/share - rm -rf ./install/lib/cargs.h - rm -rf ./install/include/cargs.h - rm -rf ./install/lib/libcargs.dylib - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 5c8501eac..fcfc2867e 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -126,11 +126,6 @@ jobs: ls -lh bin file ./bin/sherpa-onnx - - rm -fv ./install/include/cargs.h - rm -fv ./install/lib/cargs.h - rm -fv ./install/lib/libcargs.dylib - rm -fv ./install/lib/libcargs.a rm -rfv ./install/lib/pkgconfig - name: Display dependencies of sherpa-onnx for macos @@ -198,10 +193,8 @@ jobs: mkdir -p ${lib_dst}/lib if [[ ${{ matrix.lib_type }} == shared ]]; then cp -av build/install/lib/lib*.dylib* ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/libcargs.dylib ${lib_dst}/lib/libcargs*.dylib else cp -av build/install/lib/lib*.a ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/libcargs.a fi tree ${lib_dst} sign_macos_tree "$lib_dst" @@ -236,7 +229,7 @@ jobs: file: sherpa-onnx-*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.31 + tag: v1.13.8 - name: Release pre-compiled binaries and libs for macOS if: github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.build_type == 'Release' diff --git a/.github/workflows/mfc.yaml b/.github/workflows/mfc.yaml index 5ada46021..1a8ac980b 100644 --- a/.github/workflows/mfc.yaml +++ b/.github/workflows/mfc.yaml @@ -127,7 +127,7 @@ jobs: file: ./mfc-examples/${{ matrix.arch }}/Release/sherpa-onnx-streaming-*.exe # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.12.32 + # tag: v1.13.8 - name: Release pre-compiled binaries and libs for Windows ${{ matrix.arch }} if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') @@ -138,7 +138,7 @@ jobs: file: ./mfc-examples/${{ matrix.arch }}/Release/sherpa-onnx-non-streaming-*.exe # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.12.32 + # tag: v1.13.8 - name: Release pre-compiled binaries and libs for Windows ${{ matrix.arch }} if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') @@ -149,4 +149,4 @@ jobs: file: ./mfc-examples/${{ matrix.arch }}/sherpa-onnx-non-streaming-*.exe # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.12.32 + # tag: v1.13.8 diff --git a/.github/workflows/nightly-wheel-arm.yaml b/.github/workflows/nightly-wheel-arm.yaml index 32cb357c8..3e8400a5d 100644 --- a/.github/workflows/nightly-wheel-arm.yaml +++ b/.github/workflows/nightly-wheel-arm.yaml @@ -1,15 +1,6 @@ name: nightly-wheel-arm on: - schedule: - # minute (0-59) - # hour (0-23) - # day of the month (1-31) - # month (1-12) - # day of the week (0-6) - # nightly build at 23:50 UTC time every day - - cron: "50 23 * * *" - workflow_dispatch: concurrency: diff --git a/.github/workflows/npm-addon-win-x64.yaml b/.github/workflows/npm-addon-win-x64.yaml index 2b2b847c2..46384e252 100644 --- a/.github/workflows/npm-addon-win-x64.yaml +++ b/.github/workflows/npm-addon-win-x64.yaml @@ -61,7 +61,6 @@ jobs: -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF \ .. diff --git a/.github/workflows/npm-addon-win-x86.yaml b/.github/workflows/npm-addon-win-x86.yaml index e6426c7f9..b2ffebaec 100644 --- a/.github/workflows/npm-addon-win-x86.yaml +++ b/.github/workflows/npm-addon-win-x86.yaml @@ -78,7 +78,6 @@ jobs: -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF \ .. diff --git a/.github/workflows/pascal.yaml b/.github/workflows/pascal.yaml index 707966195..63df4d77d 100644 --- a/.github/workflows/pascal.yaml +++ b/.github/workflows/pascal.yaml @@ -152,6 +152,9 @@ jobs: ./run-cohere-transcribe.sh rm -rf sherpa-onnx-cohere-transcribe-* echo "---" + ./run-zipformer-ctc-hlg.sh + rm -rf sherpa-onnx-zipformer-ctc-* + echo "---" popd - name: Run Speech Enhancement test @@ -190,6 +193,7 @@ jobs: ./run-piper.sh rm -rf vits-piper-* + rm -f lexicon-en-us.txt rm piper ls -lh echo "---" @@ -202,6 +206,7 @@ jobs: ./run-kokoro-en.sh rm -rf kokoro-en-* + rm -f lexicon-en-us.txt rm kokoro-en ls -lh echo "---" @@ -214,6 +219,7 @@ jobs: ./run-matcha-en.sh rm -rf matcha-icefall-* + rm -f lexicon-en-us.txt rm matcha-en ls -lh echo "---" @@ -227,6 +233,7 @@ jobs: ./run-zipvoice-zh-en.sh rm -rf sherpa-onnx-zipvoice-* rm -f vocos_24khz.onnx + rm -f lexicon-en-us.txt rm zipvoice-zh-en ls -lh echo "---" diff --git a/.github/workflows/pkg-config.yaml b/.github/workflows/pkg-config.yaml index 6276ad51f..9cf0c1aad 100644 --- a/.github/workflows/pkg-config.yaml +++ b/.github/workflows/pkg-config.yaml @@ -106,8 +106,6 @@ jobs: gcc -o decode-file-c-api $(pkg-config --cflags sherpa-onnx) ./decode-file-c-api.c $(pkg-config --libs sherpa-onnx) - ./decode-file-c-api --help - - name: Build C API example (tts) if: matrix.tts == 'ON' shell: bash @@ -119,7 +117,15 @@ jobs: gcc -o offline-tts-c-api $(pkg-config --cflags sherpa-onnx) ./offline-tts-c-api.c $(pkg-config --libs sherpa-onnx) - ./offline-tts-c-api --help + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-ljs.tar.bz2 + tar xf vits-ljs.tar.bz2 + rm vits-ljs.tar.bz2 + + ./offline-tts-c-api + + mkdir -p ../tts + mv generated-offline-tts-c.wav ../tts/ + rm -rf vits-ljs - name: Test online transducer (C API) shell: bash diff --git a/.github/workflows/release-dart-package.yaml b/.github/workflows/release-dart-package.yaml index 5cedcf5cd..22b0a5ad8 100644 --- a/.github/workflows/release-dart-package.yaml +++ b/.github/workflows/release-dart-package.yaml @@ -464,7 +464,6 @@ jobs: -D CMAKE_INSTALL_PREFIX=./install \ -D SHERPA_ONNX_ENABLE_JNI=OFF \ -D SHERPA_ONNX_ENABLE_BINARY=OFF \ - -D BUILD_ESPEAK_NG_EXE=OFF \ .. - name: Build sherpa-onnx diff --git a/.github/workflows/riscv64-linux.yaml b/.github/workflows/riscv64-linux.yaml index ccffd8cd0..717ec4f23 100644 --- a/.github/workflows/riscv64-linux.yaml +++ b/.github/workflows/riscv64-linux.yaml @@ -214,7 +214,7 @@ jobs: file: sherpa-onnx-*linux-riscv64*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.11 + tag: v1.13.8 - name: Release pre-compiled binaries and libs for riscv64 linux ${{ matrix.lib_type }} if: github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') @@ -277,10 +277,11 @@ jobs: source .github/scripts/common.sh download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-lessac-medium.tar.bz2 + download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt qemu-riscv64 ./build-riscv64-linux-gnu/bin/sherpa-onnx-offline-tts \ --vits-model=./vits-piper-en_US-lessac-medium/en_US-lessac-medium.onnx \ - --vits-data-dir=./vits-piper-en_US-lessac-medium/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --vits-tokens=./vits-piper-en_US-lessac-medium/tokens.txt \ --output-filename=./liliana-piper-en_US-lessac-medium.wav \ 'liliana, the most beautiful and lovely assistant of our team!' diff --git a/.github/workflows/riscv64-spacemit-linux.yaml b/.github/workflows/riscv64-spacemit-linux.yaml index 61eb99a29..f3595b9fb 100755 --- a/.github/workflows/riscv64-spacemit-linux.yaml +++ b/.github/workflows/riscv64-spacemit-linux.yaml @@ -218,7 +218,7 @@ jobs: file: sherpa-onnx-*linux-riscv64*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.11 + tag: v1.13.8 - name: Release pre-compiled binaries and libs for riscv64 linux ${{ matrix.lib_type }} if: github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') @@ -288,10 +288,12 @@ jobs: tar xf vits-piper-en_US-lessac-medium.tar.bz2 rm vits-piper-en_US-lessac-medium.tar.bz2 + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + qemu-riscv64 ${QEMU_ARGS} ./build-riscv64-linux-gnu-spacemit/bin/sherpa-onnx-offline-tts \ --provider=spacemit \ --vits-model=./vits-piper-en_US-lessac-medium/en_US-lessac-medium.onnx \ - --vits-data-dir=./vits-piper-en_US-lessac-medium/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --vits-tokens=./vits-piper-en_US-lessac-medium/tokens.txt \ --output-filename=./liliana-piper-en_US-lessac-medium.wav \ 'liliana, the most beautiful and lovely assistant of our team!' diff --git a/.github/workflows/rknn-linux-aarch64.yaml b/.github/workflows/rknn-linux-aarch64.yaml index 24dcc9cef..25ea07558 100644 --- a/.github/workflows/rknn-linux-aarch64.yaml +++ b/.github/workflows/rknn-linux-aarch64.yaml @@ -125,8 +125,6 @@ jobs: make -j4 install rm -rf install/lib/pkgconfig - rm -fv install/lib/cargs.h - rm -fv install/lib/libcargs.so ' - name: Display system info @@ -253,7 +251,7 @@ jobs: file: sherpa-onnx-*linux-aarch64*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.13 + tag: v1.13.8 - name: Test offline Moonshine if: matrix.build_type != 'Debug' && !startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml index dae9280dc..a071ee2a0 100644 --- a/.github/workflows/run-java-test.yaml +++ b/.github/workflows/run-java-test.yaml @@ -129,7 +129,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_JNI=ON \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -154,6 +153,63 @@ jobs: cd ./java-api-examples ./run-version-test.sh + - name: Run java test (Non-Streaming TTS) + shell: bash + run: | + cd ./java-api-examples + + ./run-pocket-tts.sh + ./run-zipvoice-tts-lexicon.sh + ./run-supertonic-tts.sh + ./run-non-streaming-tts-kitten-en-lexicon.sh + ./run-non-streaming-tts-kokoro-zh-en-lexicon.sh + ./run-non-streaming-tts-kokoro-en-lexicon.sh + ./run-non-streaming-tts-matcha-zh-lexicon.sh + ./run-non-streaming-tts-matcha-en-lexicon.sh + ls -lh + + rm -rf sherpa-onnx-pocket-tts-* + rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia + rm -rf sherpa-onnx-supertonic-* + rm -rf kitten-nano-en-* + rm -rf kokoro-multi-* + rm -rf kokoro-en-* + + rm -rf matcha-icefall-* + rm vocos-22khz-univ.onnx + rm vocos_24khz.onnx + rm -f lexicon-en-us.txt + + ./run-non-streaming-tts-piper-en-lexicon.sh + ./run-non-streaming-tts-piper-en-with-callback-lexicon.sh + rm -rf vits-piper-* + rm -f lexicon-en-us.txt + + ./run-non-streaming-tts-coqui-de.sh + rm -rf vits-coqui-* + + ./run-non-streaming-tts-vits-zh-lexicon.sh + rm -rf vits-zh-* + + ./run-non-streaming-tts-piper-en-phonemize.sh + ./run-non-streaming-tts-piper-en-with-callback-phonemize.sh + ./run-non-streaming-tts-kitten-en-phonemize.sh + ./run-non-streaming-tts-kokoro-en-phonemize.sh + ./run-non-streaming-tts-matcha-en-phonemize.sh + ./run-non-streaming-tts-vits-de-phonemize.sh + ls -lh + + rm -rf vits-piper-* kitten-nano-en-* kokoro-en-* matcha-icefall-* + rm -f vocos-22khz-univ.onnx + rm -f piper-phonemize-*.jar + + - uses: actions/upload-artifact@v4 + with: + name: tts-wav-files-${{ matrix.os }} + path: | + java-api-examples/tts-*.wav + java-api-examples/generated-*.wav + - name: Run java test (speaker diarization) shell: bash run: | @@ -187,6 +243,9 @@ jobs: ./run-non-streaming-decode-file-zipformer-ctc.sh rm -rf sherpa-onnx-zipformer-ctc-* + ./run-non-streaming-zipformer-ctc-hlg.sh + rm -rf sherpa-onnx-zipformer-ctc-* + ./run-non-streaming-decode-file-dolphin-ctc.sh rm -rf sherpa-onnx-dolphin-* @@ -406,46 +465,6 @@ jobs: ./run-streaming-decode-file-transducer.sh rm -rf sherpa-onnx-streaming-* - - name: Run java test (Non-Streaming TTS) - shell: bash - run: | - cd ./java-api-examples - - ./run-pocket-tts.sh - ./run-zipvoice-tts.sh - ./run-supertonic-tts.sh - ./run-non-streaming-tts-kitten-en.sh - ./run-non-streaming-tts-kokoro-zh-en.sh - ./run-non-streaming-tts-kokoro-en.sh - ./run-non-streaming-tts-matcha-zh.sh - ./run-non-streaming-tts-matcha-en.sh - ls -lh - - rm -rf sherpa-onnx-pocket-tts-* - rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia - rm -rf sherpa-onnx-supertonic-* - rm -rf kitten-nano-en-* - rm -rf kokoro-multi-* - rm -rf kokoro-en-* - - rm -rf matcha-icefall-* - rm vocos-22khz-univ.onnx - rm vocos_24khz.onnx - - ./run-non-streaming-tts-piper-en.sh - rm -rf vits-piper-* - - ./run-non-streaming-tts-coqui-de.sh - rm -rf vits-coqui-* - - ./run-non-streaming-tts-vits-zh.sh - rm -rf vits-zh-* - - - uses: actions/upload-artifact@v4 - with: - name: tts-wav-files-${{ matrix.os }} - path: java-api-examples/*.wav - run_maven_test: runs-on: ${{ matrix.os }} name: maven-${{ matrix.os }} ${{ matrix.arch }} diff --git a/.github/workflows/run-python-test.yaml b/.github/workflows/run-python-test.yaml index ea1df45f2..7146b3d92 100644 --- a/.github/workflows/run-python-test.yaml +++ b/.github/workflows/run-python-test.yaml @@ -9,6 +9,7 @@ on: paths: - '.github/workflows/run-python-test.yaml' - '.github/scripts/test-python.sh' + - '.github/scripts/test-python-tts-frontends.sh' - '.github/scripts/test-speaker-recognition-python.sh' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -19,6 +20,7 @@ on: paths: - '.github/workflows/run-python-test.yaml' - '.github/scripts/test-python.sh' + - '.github/scripts/test-python-tts-frontends.sh' - '.github/scripts/test-speaker-recognition-python.sh' - 'cmake/**' - 'sherpa-onnx/csrc/*' @@ -158,8 +160,13 @@ jobs: - name: Install Python dependencies shell: bash run: | - python3 -m pip install --upgrade pip numpy pypinyin soundfile + python3 -m pip install --upgrade pip numpy pypinyin soundfile python3 -m pip install wheel twine setuptools + # External phonemizers used by the test-offline-tts-*-phonemize.py + # examples. piper_phonemize ships espeak-ng inside the wheel; it is + # a pip dependency, not a build dependency of sherpa-onnx. + python3 -m pip install --upgrade 'misaki[zh]' kaldifst + python3 -m pip install --upgrade piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html - name: Install librosa (Linux/macOS) if: runner.os != 'Windows' @@ -301,6 +308,7 @@ jobs: echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" .github/scripts/test-python.sh + .github/scripts/test-python-tts-frontends.sh .github/scripts/test-speaker-recognition-python.sh - name: Test sherpa-onnx (macOS/Windows) @@ -309,6 +317,7 @@ jobs: run: | export OS=${{ matrix.os }} .github/scripts/test-python.sh + .github/scripts/test-python-tts-frontends.sh .github/scripts/test-speaker-recognition-python.sh - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/sanitizer.yaml b/.github/workflows/sanitizer.yaml index 6b13557d7..2f9fe9d6c 100644 --- a/.github/workflows/sanitizer.yaml +++ b/.github/workflows/sanitizer.yaml @@ -3,15 +3,6 @@ name: sanitizer on: workflow_dispatch: - schedule: - # minute (0-59) - # hour (0-23) - # day of the month (1-31) - # month (1-12) - # day of the week (0-6) - # nightly build at 22:50 UTC time every day - - cron: "50 22 * * *" - concurrency: group: sanitizer-${{ github.ref }} cancel-in-progress: true @@ -54,6 +45,7 @@ jobs: cmake \ -DSHERPA_ONNX_ENABLE_PYTHON=ON \ + -DPYTHON_EXECUTABLE="$(command -v python3)" \ -DSHERPA_ONNX_ENABLE_TESTS=ON \ -DSHERPA_ONNX_ENABLE_JNI=ON \ -DSHERPA_ONNX_ENABLE_SANITIZER=ON \ @@ -76,6 +68,12 @@ jobs: file ./bin/sherpa-onnx + - name: Run unit tests + shell: bash + run: | + cd build + ctest --output-on-failure + - name: Display dependencies of sherpa-onnx for macos shell: bash run: | diff --git a/.github/workflows/swift.yaml b/.github/workflows/swift.yaml index 0bc22386d..b03d04bcf 100644 --- a/.github/workflows/swift.yaml +++ b/.github/workflows/swift.yaml @@ -76,3 +76,11 @@ jobs: shell: bash run: | .github/scripts/test-swift.sh + + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ matrix.os }} + path: | + swift-api-examples/test-*-lexicon.wav + swift-api-examples/generated-*.wav + if-no-files-found: warn diff --git a/.github/workflows/termux.yaml b/.github/workflows/termux.yaml index ceb6f2f1f..7f5b2ea7b 100644 --- a/.github/workflows/termux.yaml +++ b/.github/workflows/termux.yaml @@ -85,10 +85,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -168,11 +164,7 @@ jobs: cp -a build/install/bin $dst/ cp -a build/install/lib $dst/ cp -a build/install/include $dst/ - - rm -fv $dst/lib/cargs.h - rm -fv $dst/lib/libcargs.so rm -fv $dst/lib/libonnxruntime4j_jni.so - rm -fv $dst/include/cargs.h echo "strip" strip $dst/bin/* @@ -315,10 +307,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -532,10 +520,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -615,11 +599,7 @@ jobs: cp -a build/install/bin $dst/ cp -a build/install/lib $dst/ cp -a build/install/include $dst/ - - rm -fv $dst/lib/cargs.h - rm -fv $dst/lib/libcargs.so rm -fv $dst/lib/libonnxruntime4j_jni.so - rm -fv $dst/include/cargs.h echo "strip" strip $dst/bin/* @@ -771,10 +751,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -989,10 +965,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -1072,11 +1044,7 @@ jobs: cp -a build/install/bin $dst/ cp -a build/install/lib $dst/ cp -a build/install/include $dst/ - - rm -fv $dst/lib/cargs.h - rm -fv $dst/lib/libcargs.so rm -fv $dst/lib/libonnxruntime4j_jni.so - rm -fv $dst/include/cargs.h echo "strip" strip $dst/bin/* @@ -1235,10 +1203,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -1318,11 +1282,7 @@ jobs: cp -a build/install/bin $dst/ cp -a build/install/lib $dst/ cp -a build/install/include $dst/ - - rm -fv $dst/lib/cargs.h - rm -fv $dst/lib/libcargs.so rm -fv $dst/lib/libonnxruntime4j_jni.so - rm -fv $dst/include/cargs.h echo "strip" strip $dst/bin/* @@ -1466,10 +1426,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -1678,10 +1634,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=ON \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ diff --git a/.github/workflows/test-dart-package.yaml b/.github/workflows/test-dart-package.yaml index 5c267769c..d96376017 100644 --- a/.github/workflows/test-dart-package.yaml +++ b/.github/workflows/test-dart-package.yaml @@ -5,15 +5,6 @@ on: branches: - dart-xcframework-2 - schedule: - # minute (0-59) - # hour (0-23) - # day of the month (1-31) - # month (1-12) - # day of the week (0-6) - # nightly build at 15:50 UTC time every day - - cron: "50 15 * * *" - workflow_dispatch: concurrency: @@ -82,6 +73,12 @@ jobs: run: | .github/scripts/test-dart.sh + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ github.job }} + path: dart-api-examples/tts/*.wav + if-no-files-found: warn + # Test using xcframework built from source (SPM approach) test_dart_package_xcframework: if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' @@ -164,3 +161,9 @@ jobs: cp scripts/dart/sherpa-onnx-pubspec.yaml flutter/sherpa_onnx/pubspec.yaml .github/scripts/test-dart.sh + + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ github.job }} + path: dart-api-examples/tts/*.wav + if-no-files-found: warn diff --git a/.github/workflows/test-dart.yaml b/.github/workflows/test-dart.yaml index 40c318484..3aa5e34ba 100644 --- a/.github/workflows/test-dart.yaml +++ b/.github/workflows/test-dart.yaml @@ -124,7 +124,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -183,3 +182,9 @@ jobs: cp scripts/dart/sherpa-onnx-pubspec.yaml flutter/sherpa_onnx/pubspec.yaml .github/scripts/test-dart.sh + + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ matrix.os }} + path: dart-api-examples/tts/*.wav + if-no-files-found: warn diff --git a/.github/workflows/test-dot-net-nuget.yaml b/.github/workflows/test-dot-net-nuget.yaml index 49c1924c2..7596b090e 100644 --- a/.github/workflows/test-dot-net-nuget.yaml +++ b/.github/workflows/test-dot-net-nuget.yaml @@ -3,15 +3,6 @@ name: test-dot-net-nuget on: workflow_dispatch: - schedule: - # minute (0-59) - # hour (0-23) - # day of the month (1-31) - # month (1-12) - # day of the week (0-6) - # nightly build at 23:50 UTC time every day - - cron: "50 23 * * *" - concurrency: group: test-dot-net-nuget-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/test-dot-net.yaml b/.github/workflows/test-dot-net.yaml index 1bdaecb6d..6b50086b1 100644 --- a/.github/workflows/test-dot-net.yaml +++ b/.github/workflows/test-dot-net.yaml @@ -62,7 +62,6 @@ jobs: -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ .. diff --git a/.github/workflows/test-flutter-asr.yaml b/.github/workflows/test-flutter-asr.yaml new file mode 100644 index 000000000..a2cd80362 --- /dev/null +++ b/.github/workflows/test-flutter-asr.yaml @@ -0,0 +1,603 @@ +name: test-flutter-asr + +on: + push: + branches: + - master + + workflow_dispatch: + +concurrency: + group: test-flutter-asr-${{ github.ref }} + cancel-in-progress: true + +jobs: + generate_matrix: + runs-on: ubuntu-latest + outputs: + desktop: ${{ steps.set-matrix.outputs.desktop }} + android: ${{ steps.set-matrix.outputs.android }} + steps: + - uses: actions/checkout@v4 + - name: Generating build matrix + id: set-matrix + shell: bash + run: | + JSON=$(python3 .github/scripts/flutter/generate_streaming_asr_matrix.py) + DESKTOP=$(echo "$JSON" | python3 -c "import json,sys; d=json.load(sys.stdin); print(json.dumps(d['desktop']))") + ANDROID=$(echo "$JSON" | python3 -c "import json,sys; d=json.load(sys.stdin); print(json.dumps(d['android']))") + echo "desktop=$DESKTOP" >> $GITHUB_OUTPUT + echo "android=$ANDROID" >> $GITHUB_OUTPUT + + macos: + name: macOS ${{ matrix.demo }} ${{ matrix.asr_model_name }} + needs: [generate_matrix] + runs-on: macos-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate_matrix.outputs.desktop) }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update version + shell: bash + run: | + ./new-release.sh + git diff . + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: macos-flutter-punct-${{ matrix.demo }} + + - name: Setup Flutter SDK + uses: flutter-actions/setup-flutter@v4 + with: + channel: stable + version: 3.44.8 + + - name: Display flutter info + shell: bash + run: | + which flutter + which dart + flutter --version + dart --version + flutter doctor + + - name: Cache xcframework + id: cache-xcframework + uses: actions/cache@v4 + with: + path: build-macos-shared-sherpa-with-static-onnxruntime/sherpa-onnx.xcframework + key: macos-xcframework-${{ hashFiles('build-macos-shared-sherpa-with-static-onnxruntime.sh', 'CMakeLists.txt') }} + restore-keys: | + macos-xcframework- + + - name: Build xcframework from source + if: steps.cache-xcframework.outputs.cache-hit != 'true' + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + ./build-macos-shared-sherpa-with-static-onnxruntime.sh + + - name: Copy xcframework + shell: bash + run: | + cp -av build-macos-shared-sherpa-with-static-onnxruntime/sherpa-onnx.xcframework flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos/sherpa-onnx.xcframework + ls -lh flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos/sherpa-onnx.xcframework/ + + - name: Use local sherpa_onnx_macos + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's| sherpa_onnx_macos: 1.13.8| # sherpa_onnx_macos: 1.13.8\n sherpa_onnx_macos:\n path: ../sherpa_onnx_macos|' pubspec.yaml + rm -f pubspec.yaml.bak + cat pubspec.yaml + + - name: Remove sherpa_onnx_web dependency + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's|^ sherpa_onnx_web:.*| # sherpa_onnx_web: 1.13.8|' pubspec.yaml + rm -f pubspec.yaml.bak + + + - name: Download model + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/assets + rm -fv .gitkeep + curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${{ matrix.asr_tarball }} + tar xf ${{ matrix.asr_tarball }} + rm ${{ matrix.asr_tarball }} + ${{ matrix.asr_post_cmd }} + cd .. + sed -i.bak "s| - assets/$| - assets/\n - assets/${{ matrix.asr_model_name }}/|g" ./pubspec.yaml + rm -f pubspec.yaml.bak + + - name: Set model type + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + sed -i.bak "s/final type = .*;$/final type = ${{ matrix.asr_model_index }};/g" lib/streaming_asr.dart + rm -f lib/streaming_asr.dart.bak + if [ -n "${{ matrix.asr_rule_fsts }}" ]; then + sed -i.bak "s|ruleFsts: ''|ruleFsts: await copyAssetFile('assets/${{ matrix.asr_rule_fsts }}')|g" lib/streaming_asr.dart + rm -f lib/streaming_asr.dart.bak + fi + grep 'final type\|ruleFsts' lib/streaming_asr.dart + + - name: Copy LICENSE for sherpa_onnx_macos podspec + shell: bash + run: | + cp LICENSE flutter/sherpa_onnx_macos/LICENSE + + - name: Build Flutter app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + + sed -i.bak 's|sherpa_onnx: 1.13.8|sherpa_onnx:\n path: ../../flutter/sherpa_onnx|' pubspec.yaml + rm -f pubspec.yaml.bak + + flutter config --no-enable-swift-package-manager + flutter pub get + cd macos && pod install && cd .. + flutter build macos + + ls -lh build/macos/Build/Products/Release/ + + - name: Ad-hoc sign the app + shell: bash + run: | + app_name=$(echo ${{ matrix.demo }} | tr '-' '_') + codesign --force --deep -s - flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/${app_name}.app + xattr -cr flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/${app_name}.app + + - name: Zip app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/ + app_name=$(echo ${{ matrix.demo }} | tr '-' '_') + # Rename the bundle so the extracted app carries the model name. + mv ${app_name}.app ${{ matrix.demo }}-${{ matrix.asr_model_name }}.app + zip -r -y flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-macos.zip ${{ matrix.demo }}-${{ matrix.asr_model_name }}.app/ + ls -lh flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-macos.zip + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-macos.zip + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: flutter + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-macos + path: flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-macos.zip + + linux: + name: Linux ${{ matrix.demo }} ${{ matrix.asr_model_name }} + needs: [generate_matrix] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate_matrix.outputs.desktop) }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update version + shell: bash + run: | + ./new-release.sh + git diff . + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: linux-flutter-punct-${{ matrix.demo }} + + - name: Install dependencies + shell: bash + run: | + sudo apt-get update + # libgstreamer*: required by audioplayers_linux (the tts demo plays + # the generated audio). + sudo apt-get install -y clang cmake ninja-build pkg-config libgtk-3-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev + + - name: Setup Flutter SDK + uses: flutter-actions/setup-flutter@v4 + with: + channel: stable + version: latest + + - name: Display flutter info + shell: bash + run: | + which flutter + which dart + flutter --version + dart --version + flutter doctor + + - name: Build sherpa-onnx from source + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + mkdir build + cd build + cmake \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ + -DSHERPA_ONNX_ENABLE_BINARY=OFF \ + -DCMAKE_INSTALL_PREFIX=./install \ + .. + cmake --build . --target install --config Release + + - name: Copy libs + shell: bash + run: | + if [[ $(uname -m) == "aarch64" ]]; then + arch=aarch64 + else + arch=x64 + fi + cp -fv build/install/lib/lib* ./flutter/sherpa_onnx_linux/linux/$arch + ls -lh ./flutter/sherpa_onnx_linux/linux/$arch + + - name: Use local sherpa_onnx_linux + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's| sherpa_onnx_linux: 1.13.8| # sherpa_onnx_linux: 1.13.8\n sherpa_onnx_linux:\n path: ../sherpa_onnx_linux|' pubspec.yaml + rm -f pubspec.yaml.bak + cat pubspec.yaml + + - name: Remove sherpa_onnx_web dependency + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's|^ sherpa_onnx_web:.*| # sherpa_onnx_web: 1.13.8|' pubspec.yaml + rm -f pubspec.yaml.bak + + + - name: Download model + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/assets + rm -fv .gitkeep + curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${{ matrix.asr_tarball }} + tar xf ${{ matrix.asr_tarball }} + rm ${{ matrix.asr_tarball }} + ${{ matrix.asr_post_cmd }} + cd .. + sed -i.bak "s| - assets/$| - assets/\n - assets/${{ matrix.asr_model_name }}/|g" ./pubspec.yaml + rm -f pubspec.yaml.bak + + - name: Set model type + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + sed -i.bak "s/final type = .*;$/final type = ${{ matrix.asr_model_index }};/g" lib/streaming_asr.dart + rm -f lib/streaming_asr.dart.bak + if [ -n "${{ matrix.asr_rule_fsts }}" ]; then + sed -i.bak "s|ruleFsts: ''|ruleFsts: await copyAssetFile('assets/${{ matrix.asr_rule_fsts }}')|g" lib/streaming_asr.dart + rm -f lib/streaming_asr.dart.bak + fi + grep 'final type\|ruleFsts' lib/streaming_asr.dart + + - name: Build Flutter app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + + sed -i.bak 's|sherpa_onnx: 1.13.8|sherpa_onnx:\n path: ../../flutter/sherpa_onnx|' pubspec.yaml + rm -f pubspec.yaml.bak + + flutter pub get + flutter build linux + + ls -lh build/linux/*/release/bundle/ + + - name: Zip app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/build/linux/*/release/ + # Rename the bundle dir so the extracted folder carries the model name. + mv bundle ${{ matrix.demo }}-${{ matrix.asr_model_name }} + zip -r -y flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-linux-x64.zip ${{ matrix.demo }}-${{ matrix.asr_model_name }}/ + ls -lh flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-linux-x64.zip + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./flutter-examples/${{ matrix.demo }}/build/linux/*/release/flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-linux-x64.zip + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: flutter + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-linux-x64 + path: flutter-examples/${{ matrix.demo }}/build/linux/*/release/flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-linux-x64.zip + + windows: + name: Windows ${{ matrix.demo }} ${{ matrix.asr_model_name }} + needs: [generate_matrix] + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate_matrix.outputs.desktop) }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update version + shell: bash + run: | + ./new-release.sh + git diff . + + - name: Setup Flutter SDK + uses: flutter-actions/setup-flutter@v4 + with: + channel: stable + version: latest + + - name: Display flutter info + shell: bash + run: | + which flutter + which dart + flutter --version + dart --version + flutter doctor + + - name: Build sherpa-onnx from source + shell: bash + run: | + mkdir build + cd build + cmake \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ + -DSHERPA_ONNX_ENABLE_BINARY=OFF \ + -DCMAKE_INSTALL_PREFIX=./install \ + .. + cmake --build . --target install --config Release + + - name: Copy libs + shell: bash + run: | + cp -fv build/install/lib/*.dll ./flutter/sherpa_onnx_windows/windows/x64/ + ls -lh ./flutter/sherpa_onnx_windows/windows/x64/ + + - name: Use local sherpa_onnx_windows + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's| sherpa_onnx_windows: 1.13.8| # sherpa_onnx_windows: 1.13.8\n sherpa_onnx_windows:\n path: ../sherpa_onnx_windows|' pubspec.yaml + rm -f pubspec.yaml.bak + cat pubspec.yaml + + - name: Remove sherpa_onnx_web dependency + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's|^ sherpa_onnx_web:.*| # sherpa_onnx_web: 1.13.8|' pubspec.yaml + rm -f pubspec.yaml.bak + + - name: Download model + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/assets + rm -fv .gitkeep + curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${{ matrix.asr_tarball }} + tar xf ${{ matrix.asr_tarball }} + rm ${{ matrix.asr_tarball }} + ${{ matrix.asr_post_cmd }} + cd .. + sed -i.bak "s| - assets/$| - assets/\n - assets/${{ matrix.asr_model_name }}/|g" ./pubspec.yaml + rm -f pubspec.yaml.bak + + - name: Set model type + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + sed -i.bak "s/final type = .*;$/final type = ${{ matrix.asr_model_index }};/g" lib/streaming_asr.dart + rm -f lib/streaming_asr.dart.bak + if [ -n "${{ matrix.asr_rule_fsts }}" ]; then + sed -i.bak "s|ruleFsts: ''|ruleFsts: await copyAssetFile('assets/${{ matrix.asr_rule_fsts }}')|g" lib/streaming_asr.dart + rm -f lib/streaming_asr.dart.bak + fi + grep 'final type\|ruleFsts' lib/streaming_asr.dart + + - name: Build Flutter app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + + sed -i.bak 's|sherpa_onnx: 1.13.8|sherpa_onnx:\n path: ../../flutter/sherpa_onnx|' pubspec.yaml + rm -f pubspec.yaml.bak + + flutter pub get + flutter build windows + + ls -lh build/windows/x64/runner/Release/ + + - name: Zip app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/build/windows/x64/runner/ + # Rename the folder so the extracted app carries the model name. + mv Release ${{ matrix.demo }}-${{ matrix.asr_model_name }} + powershell -Command "Compress-Archive -Path ${{ matrix.demo }}-${{ matrix.asr_model_name }} -DestinationPath flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-win-x64.zip" + ls -lh flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-win-x64.zip + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./flutter-examples/${{ matrix.demo }}/build/windows/x64/runner/flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-win-x64.zip + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: flutter + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-win-x64 + path: flutter-examples/${{ matrix.demo }}/build/windows/x64/runner/flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-win-x64.zip + + android: + name: Android (${{ matrix.demo }}, ${{ matrix.asr_model_name }}, ${{ matrix.abi }}) + needs: [generate_matrix] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate_matrix.outputs.android) }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update version + shell: bash + run: | + ./new-release.sh + git diff . + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: android-flutter-punct-${{ matrix.demo }}-${{ matrix.abi }} + + - name: Setup Flutter SDK + uses: flutter-actions/setup-flutter@v4 + with: + channel: stable + version: latest + + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Display flutter info + shell: bash + run: | + which flutter + which dart + flutter --version + dart --version + flutter doctor + + - name: Build ${{ matrix.abi }} from source + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export SHERPA_ONNX_ENABLE_C_API=ON + export SHERPA_ONNX_ENABLE_JNI=OFF + export SHERPA_ONNX_ENABLE_BINARY=OFF + ./${{ matrix.build_script }} + + - name: Copy libs + shell: bash + run: | + cp -fv ${{ matrix.build_dir }}/install/lib/lib*.so flutter/${{ matrix.plugin }}/android/src/main/jniLibs/${{ matrix.abi }}/ + ls -lh flutter/${{ matrix.plugin }}/android/src/main/jniLibs/${{ matrix.abi }}/ + + - name: Use local ${{ matrix.plugin }} + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's| ${{ matrix.plugin_key }}: 1.13.8| # ${{ matrix.plugin_key }}: 1.13.8\n ${{ matrix.plugin_key }}:\n path: ../${{ matrix.plugin }}|' pubspec.yaml + rm -f pubspec.yaml.bak + cat pubspec.yaml + + + - name: Download model + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/assets + rm -fv .gitkeep + curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${{ matrix.asr_tarball }} + tar xf ${{ matrix.asr_tarball }} + rm ${{ matrix.asr_tarball }} + ${{ matrix.asr_post_cmd }} + cd .. + sed -i.bak "s| - assets/$| - assets/\n - assets/${{ matrix.asr_model_name }}/|g" ./pubspec.yaml + rm -f pubspec.yaml.bak + + - name: Set model type + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + sed -i.bak "s/final type = .*;$/final type = ${{ matrix.asr_model_index }};/g" lib/streaming_asr.dart + rm -f lib/streaming_asr.dart.bak + if [ -n "${{ matrix.asr_rule_fsts }}" ]; then + sed -i.bak "s|ruleFsts: ''|ruleFsts: await copyAssetFile('assets/${{ matrix.asr_rule_fsts }}')|g" lib/streaming_asr.dart + rm -f lib/streaming_asr.dart.bak + fi + grep 'final type\|ruleFsts' lib/streaming_asr.dart + + - name: Build Flutter APK + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + + sed -i.bak 's|sherpa_onnx: 1.13.8|sherpa_onnx:\n path: ../../flutter/sherpa_onnx|' pubspec.yaml + rm -f pubspec.yaml.bak + + flutter pub get + flutter build apk --split-per-abi --target-platform ${{ matrix.target_platform }} + + ls -lh build/app/outputs/flutter-apk/ + + - name: Rename APK + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/build/app/outputs/flutter-apk/ + mv app-*-release.apk flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-${{ matrix.abi }}.apk + ls -lh *.apk + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./flutter-examples/${{ matrix.demo }}/build/app/outputs/flutter-apk/flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-${{ matrix.abi }}.apk + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: flutter + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-android-${{ matrix.abi }} + path: flutter-examples/${{ matrix.demo }}/build/app/outputs/flutter-apk/flutter-${{ matrix.demo }}-${{ matrix.asr_model_name }}-${{ matrix.abi }}.apk + diff --git a/.github/workflows/test-flutter-punct.yaml b/.github/workflows/test-flutter-punct.yaml index 5775053e1..a0c0157ac 100644 --- a/.github/workflows/test-flutter-punct.yaml +++ b/.github/workflows/test-flutter-punct.yaml @@ -214,7 +214,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -344,7 +343,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. diff --git a/.github/workflows/test-flutter-tts.yaml b/.github/workflows/test-flutter-tts.yaml new file mode 100644 index 000000000..c8c744ae8 --- /dev/null +++ b/.github/workflows/test-flutter-tts.yaml @@ -0,0 +1,702 @@ +name: test-flutter-tts + +on: + push: + branches: + - master + + workflow_dispatch: + +concurrency: + group: test-flutter-tts-${{ github.ref }} + cancel-in-progress: true + +jobs: + generate_matrix: + runs-on: ubuntu-latest + outputs: + desktop: ${{ steps.set-matrix.outputs.desktop }} + android: ${{ steps.set-matrix.outputs.android }} + steps: + - uses: actions/checkout@v4 + - name: Generating build matrix + id: set-matrix + shell: bash + run: | + JSON=$(python3 .github/scripts/flutter/generate_tts_matrix.py) + DESKTOP=$(echo "$JSON" | python3 -c "import json,sys; d=json.load(sys.stdin); print(json.dumps(d['desktop']))") + ANDROID=$(echo "$JSON" | python3 -c "import json,sys; d=json.load(sys.stdin); print(json.dumps(d['android']))") + echo "desktop=$DESKTOP" >> $GITHUB_OUTPUT + echo "android=$ANDROID" >> $GITHUB_OUTPUT + + macos: + name: macOS ${{ matrix.demo }} ${{ matrix.tts_model_name }} + needs: [generate_matrix] + runs-on: macos-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate_matrix.outputs.desktop) }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update version + shell: bash + run: | + ./new-release.sh + git diff . + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: macos-flutter-punct-${{ matrix.demo }} + + - name: Setup Flutter SDK + uses: flutter-actions/setup-flutter@v4 + with: + channel: stable + version: 3.44.8 + + - name: Display flutter info + shell: bash + run: | + which flutter + which dart + flutter --version + dart --version + flutter doctor + + - name: Cache xcframework + id: cache-xcframework + uses: actions/cache@v4 + with: + path: build-macos-shared-sherpa-with-static-onnxruntime/sherpa-onnx.xcframework + key: macos-xcframework-${{ hashFiles('build-macos-shared-sherpa-with-static-onnxruntime.sh', 'CMakeLists.txt') }} + restore-keys: | + macos-xcframework- + + - name: Build xcframework from source + if: steps.cache-xcframework.outputs.cache-hit != 'true' + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + ./build-macos-shared-sherpa-with-static-onnxruntime.sh + + - name: Copy xcframework + shell: bash + run: | + cp -av build-macos-shared-sherpa-with-static-onnxruntime/sherpa-onnx.xcframework flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos/sherpa-onnx.xcframework + ls -lh flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos/sherpa-onnx.xcframework/ + + - name: Use local sherpa_onnx_macos + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's| sherpa_onnx_macos: 1.13.8| # sherpa_onnx_macos: 1.13.8\n sherpa_onnx_macos:\n path: ../sherpa_onnx_macos|' pubspec.yaml + rm -f pubspec.yaml.bak + cat pubspec.yaml + + - name: Remove sherpa_onnx_web dependency + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's|^ sherpa_onnx_web:.*| # sherpa_onnx_web: 1.13.8|' pubspec.yaml + rm -f pubspec.yaml.bak + + + - name: Download model + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/assets + rm -fv .gitkeep + curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/${{ matrix.tts_tarball }} + tar xf ${{ matrix.tts_tarball }} + rm ${{ matrix.tts_tarball }} + ${{ matrix.tts_extra_cmd }} + ${{ matrix.tts_cleanup_cmd }} + cd .. + ./generate-asset-list.py + + - name: Set selectedModelIndex + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + sed -i.bak "s|const int selectedModelIndex = [0-9]*;|const int selectedModelIndex = ${{ matrix.tts_model_index }};|" lib/model_config.dart + rm -f lib/model_config.dart.bak + grep 'selectedModelIndex' lib/model_config.dart + + - name: Copy LICENSE for sherpa_onnx_macos podspec + shell: bash + run: | + cp LICENSE flutter/sherpa_onnx_macos/LICENSE + + - name: Build Flutter app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + + sed -i.bak 's|sherpa_onnx: 1.13.8|sherpa_onnx:\n path: ../../flutter/sherpa_onnx|' pubspec.yaml + rm -f pubspec.yaml.bak + + flutter config --no-enable-swift-package-manager + flutter pub get + cd macos && pod install && cd .. + flutter build macos + + ls -lh build/macos/Build/Products/Release/ + + - name: Ad-hoc sign the app + shell: bash + run: | + app_name=$(echo ${{ matrix.demo }} | tr '-' '_') + codesign --force --deep -s - flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/${app_name}.app + xattr -cr flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/${app_name}.app + + - name: Zip app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/ + app_name=$(echo ${{ matrix.demo }} | tr '-' '_') + # Rename the bundle so the extracted app carries the model name. + mv ${app_name}.app tts-${{ matrix.tts_model_name }}.app + zip -r -y flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-macos.zip tts-${{ matrix.tts_model_name }}.app/ + ls -lh flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-macos.zip + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-macos.zip + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: flutter + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-macos + path: flutter-examples/${{ matrix.demo }}/build/macos/Build/Products/Release/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-macos.zip + + linux: + name: Linux ${{ matrix.demo }} ${{ matrix.tts_model_name }} + needs: [generate_matrix] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate_matrix.outputs.desktop) }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update version + shell: bash + run: | + ./new-release.sh + git diff . + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: linux-flutter-punct-${{ matrix.demo }} + + - name: Install dependencies + shell: bash + run: | + sudo apt-get update + # libgstreamer*: required by audioplayers_linux (the tts demo plays + # the generated audio). + sudo apt-get install -y clang cmake ninja-build pkg-config libgtk-3-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev + + - name: Setup Flutter SDK + uses: flutter-actions/setup-flutter@v4 + with: + channel: stable + version: latest + + - name: Display flutter info + shell: bash + run: | + which flutter + which dart + flutter --version + dart --version + flutter doctor + + - name: Build sherpa-onnx from source + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + mkdir build + cd build + cmake \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ + -DSHERPA_ONNX_ENABLE_BINARY=OFF \ + -DCMAKE_INSTALL_PREFIX=./install \ + .. + cmake --build . --target install --config Release + + - name: Copy libs + shell: bash + run: | + if [[ $(uname -m) == "aarch64" ]]; then + arch=aarch64 + else + arch=x64 + fi + cp -fv build/install/lib/lib* ./flutter/sherpa_onnx_linux/linux/$arch + ls -lh ./flutter/sherpa_onnx_linux/linux/$arch + + - name: Use local sherpa_onnx_linux + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's| sherpa_onnx_linux: 1.13.8| # sherpa_onnx_linux: 1.13.8\n sherpa_onnx_linux:\n path: ../sherpa_onnx_linux|' pubspec.yaml + rm -f pubspec.yaml.bak + cat pubspec.yaml + + - name: Remove sherpa_onnx_web dependency + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's|^ sherpa_onnx_web:.*| # sherpa_onnx_web: 1.13.8|' pubspec.yaml + rm -f pubspec.yaml.bak + + + - name: Download model + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/assets + rm -fv .gitkeep + curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/${{ matrix.tts_tarball }} + tar xf ${{ matrix.tts_tarball }} + rm ${{ matrix.tts_tarball }} + ${{ matrix.tts_extra_cmd }} + ${{ matrix.tts_cleanup_cmd }} + cd .. + ./generate-asset-list.py + + - name: Set selectedModelIndex + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + sed -i.bak "s|const int selectedModelIndex = [0-9]*;|const int selectedModelIndex = ${{ matrix.tts_model_index }};|" lib/model_config.dart + rm -f lib/model_config.dart.bak + grep 'selectedModelIndex' lib/model_config.dart + + - name: Build Flutter app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + + sed -i.bak 's|sherpa_onnx: 1.13.8|sherpa_onnx:\n path: ../../flutter/sherpa_onnx|' pubspec.yaml + rm -f pubspec.yaml.bak + + flutter pub get + flutter build linux + + ls -lh build/linux/*/release/bundle/ + + - name: Zip app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/build/linux/*/release/ + # Rename the bundle dir so the extracted folder carries the model name. + mv bundle tts-${{ matrix.tts_model_name }} + zip -r -y flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-linux-x64.zip tts-${{ matrix.tts_model_name }}/ + ls -lh flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-linux-x64.zip + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./flutter-examples/${{ matrix.demo }}/build/linux/*/release/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-linux-x64.zip + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: flutter + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-linux-x64 + path: flutter-examples/${{ matrix.demo }}/build/linux/*/release/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-linux-x64.zip + + windows: + name: Windows ${{ matrix.demo }} ${{ matrix.tts_model_name }} + needs: [generate_matrix] + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate_matrix.outputs.desktop) }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update version + shell: bash + run: | + ./new-release.sh + git diff . + + - name: Setup Flutter SDK + uses: flutter-actions/setup-flutter@v4 + with: + channel: stable + version: latest + + - name: Display flutter info + shell: bash + run: | + which flutter + which dart + flutter --version + dart --version + flutter doctor + + - name: Build sherpa-onnx from source + shell: bash + run: | + mkdir build + cd build + cmake \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ + -DSHERPA_ONNX_ENABLE_BINARY=OFF \ + -DCMAKE_INSTALL_PREFIX=./install \ + .. + cmake --build . --target install --config Release + + - name: Copy libs + shell: bash + run: | + cp -fv build/install/lib/*.dll ./flutter/sherpa_onnx_windows/windows/x64/ + ls -lh ./flutter/sherpa_onnx_windows/windows/x64/ + + - name: Use local sherpa_onnx_windows + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's| sherpa_onnx_windows: 1.13.8| # sherpa_onnx_windows: 1.13.8\n sherpa_onnx_windows:\n path: ../sherpa_onnx_windows|' pubspec.yaml + rm -f pubspec.yaml.bak + cat pubspec.yaml + + - name: Remove sherpa_onnx_web dependency + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's|^ sherpa_onnx_web:.*| # sherpa_onnx_web: 1.13.8|' pubspec.yaml + rm -f pubspec.yaml.bak + + - name: Download model + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/assets + rm -fv .gitkeep + curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/${{ matrix.tts_tarball }} + tar xf ${{ matrix.tts_tarball }} + rm ${{ matrix.tts_tarball }} + ${{ matrix.tts_extra_cmd }} + ${{ matrix.tts_cleanup_cmd }} + cd .. + ./generate-asset-list.py + + - name: Set selectedModelIndex + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + sed -i.bak "s|const int selectedModelIndex = [0-9]*;|const int selectedModelIndex = ${{ matrix.tts_model_index }};|" lib/model_config.dart + rm -f lib/model_config.dart.bak + grep 'selectedModelIndex' lib/model_config.dart + + - name: Build Flutter app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + + sed -i.bak 's|sherpa_onnx: 1.13.8|sherpa_onnx:\n path: ../../flutter/sherpa_onnx|' pubspec.yaml + rm -f pubspec.yaml.bak + + flutter pub get + flutter build windows + + ls -lh build/windows/x64/runner/Release/ + + - name: Zip app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/build/windows/x64/runner/ + # Rename the folder so the extracted app carries the model name. + mv Release tts-${{ matrix.tts_model_name }} + powershell -Command "Compress-Archive -Path tts-${{ matrix.tts_model_name }} -DestinationPath flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-win-x64.zip" + ls -lh flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-win-x64.zip + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./flutter-examples/${{ matrix.demo }}/build/windows/x64/runner/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-win-x64.zip + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: flutter + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-win-x64 + path: flutter-examples/${{ matrix.demo }}/build/windows/x64/runner/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-win-x64.zip + + android: + name: Android (${{ matrix.demo }}, ${{ matrix.tts_model_name }}, ${{ matrix.abi }}) + needs: [generate_matrix] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate_matrix.outputs.android) }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update version + shell: bash + run: | + ./new-release.sh + git diff . + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: android-flutter-punct-${{ matrix.demo }}-${{ matrix.abi }} + + - name: Setup Flutter SDK + uses: flutter-actions/setup-flutter@v4 + with: + channel: stable + version: latest + + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Display flutter info + shell: bash + run: | + which flutter + which dart + flutter --version + dart --version + flutter doctor + + - name: Build ${{ matrix.abi }} from source + shell: bash + run: | + export CMAKE_CXX_COMPILER_LAUNCHER=ccache + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + export SHERPA_ONNX_ENABLE_C_API=ON + export SHERPA_ONNX_ENABLE_JNI=OFF + export SHERPA_ONNX_ENABLE_BINARY=OFF + ./${{ matrix.build_script }} + + - name: Copy libs + shell: bash + run: | + cp -fv ${{ matrix.build_dir }}/install/lib/lib*.so flutter/${{ matrix.plugin }}/android/src/main/jniLibs/${{ matrix.abi }}/ + ls -lh flutter/${{ matrix.plugin }}/android/src/main/jniLibs/${{ matrix.abi }}/ + + - name: Use local ${{ matrix.plugin }} + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's| ${{ matrix.plugin_key }}: 1.13.8| # ${{ matrix.plugin_key }}: 1.13.8\n ${{ matrix.plugin_key }}:\n path: ../${{ matrix.plugin }}|' pubspec.yaml + rm -f pubspec.yaml.bak + cat pubspec.yaml + + + - name: Download model + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/assets + rm -fv .gitkeep + curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/${{ matrix.tts_tarball }} + tar xf ${{ matrix.tts_tarball }} + rm ${{ matrix.tts_tarball }} + ${{ matrix.tts_extra_cmd }} + ${{ matrix.tts_cleanup_cmd }} + cd .. + ./generate-asset-list.py + + - name: Set selectedModelIndex + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + sed -i.bak "s|const int selectedModelIndex = [0-9]*;|const int selectedModelIndex = ${{ matrix.tts_model_index }};|" lib/model_config.dart + rm -f lib/model_config.dart.bak + grep 'selectedModelIndex' lib/model_config.dart + + - name: Build Flutter APK + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + + sed -i.bak 's|sherpa_onnx: 1.13.8|sherpa_onnx:\n path: ../../flutter/sherpa_onnx|' pubspec.yaml + rm -f pubspec.yaml.bak + + flutter pub get + flutter build apk --split-per-abi --target-platform ${{ matrix.target_platform }} + + ls -lh build/app/outputs/flutter-apk/ + + - name: Rename APK + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/build/app/outputs/flutter-apk/ + mv app-*-release.apk flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-${{ matrix.abi }}.apk + ls -lh *.apk + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./flutter-examples/${{ matrix.demo }}/build/app/outputs/flutter-apk/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-${{ matrix.abi }}.apk + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: flutter + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-android-${{ matrix.abi }} + path: flutter-examples/${{ matrix.demo }}/build/app/outputs/flutter-apk/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-${{ matrix.abi }}.apk + + web: + name: Web ${{ matrix.demo }} ${{ matrix.tts_model_name }} + needs: [generate_matrix] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate_matrix.outputs.desktop) }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Update version + shell: bash + run: | + ./new-release.sh + git diff . + + - name: Install Emscripten + uses: mymindstorm/setup-emsdk@v14 + with: + version: 4.0.23 + + - name: Setup Flutter SDK + uses: flutter-actions/setup-flutter@v4 + with: + channel: stable + version: latest + + - name: Display flutter info + shell: bash + run: | + which flutter + which dart + flutter --version + dart --version + flutter doctor + + - name: Build WASM module + shell: bash + run: | + ./build-wasm-simd-web.sh + + - name: Copy WASM assets to web plugin + shell: bash + run: | + ./build-flutter-web-wasm.sh + + - name: Download model + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/assets + rm -fv .gitkeep + curl -sSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/${{ matrix.tts_tarball }} + tar xf ${{ matrix.tts_tarball }} + rm ${{ matrix.tts_tarball }} + ${{ matrix.tts_extra_cmd }} + ${{ matrix.tts_cleanup_cmd }} + cd .. + ./generate-asset-list.py + + - name: Set selectedModelIndex + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + sed -i.bak "s|const int selectedModelIndex = [0-9]*;|const int selectedModelIndex = ${{ matrix.tts_model_index }};|" lib/model_config.dart + rm -f lib/model_config.dart.bak + grep 'selectedModelIndex' lib/model_config.dart + + + - name: Use local sherpa_onnx_web + shell: bash + run: | + cd flutter/sherpa_onnx + sed -i.bak 's|^ sherpa_onnx_web:.*| # sherpa_onnx_web: 1.13.8\n sherpa_onnx_web:\n path: ../sherpa_onnx_web|' pubspec.yaml + rm -f pubspec.yaml.bak + + - name: Build Flutter web app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }} + + sed -i.bak 's|sherpa_onnx: 1.13.8|sherpa_onnx:\n path: ../../flutter/sherpa_onnx|' pubspec.yaml + rm -f pubspec.yaml.bak + + flutter pub get + flutter build web + + ls -lh build/web/ + + - name: Zip web app + shell: bash + run: | + cd flutter-examples/${{ matrix.demo }}/build/ + # Rename the folder so the extracted app carries the model name. + mv web tts-${{ matrix.tts_model_name }} + zip -r -y flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-web.zip tts-${{ matrix.tts_model_name }}/ + ls -lh flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-web.zip + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + overwrite: true + file: ./flutter-examples/${{ matrix.demo }}/build/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-web.zip + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: flutter + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-web + path: flutter-examples/${{ matrix.demo }}/build/flutter-${{ matrix.demo }}-${{ matrix.tts_model_name }}-web.zip diff --git a/.github/workflows/test-flutter-vad-asr.yaml b/.github/workflows/test-flutter-vad-asr.yaml index 857106a98..68670c5d6 100644 --- a/.github/workflows/test-flutter-vad-asr.yaml +++ b/.github/workflows/test-flutter-vad-asr.yaml @@ -260,7 +260,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -409,7 +408,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. diff --git a/.github/workflows/test-flutter-vad.yaml b/.github/workflows/test-flutter-vad.yaml index c63205a3a..9634981fa 100644 --- a/.github/workflows/test-flutter-vad.yaml +++ b/.github/workflows/test-flutter-vad.yaml @@ -222,7 +222,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -349,7 +348,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. diff --git a/.github/workflows/test-flutter.yaml b/.github/workflows/test-flutter.yaml index 5ad7c5da1..6acb12fcb 100644 --- a/.github/workflows/test-flutter.yaml +++ b/.github/workflows/test-flutter.yaml @@ -426,7 +426,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -508,7 +507,6 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. diff --git a/.github/workflows/test-go-package.yaml b/.github/workflows/test-go-package.yaml index 8eb02cf01..3436f00d6 100644 --- a/.github/workflows/test-go-package.yaml +++ b/.github/workflows/test-go-package.yaml @@ -4,14 +4,6 @@ on: push: branches: - fix-go-2 - schedule: - # minute (0-59) - # hour (0-23) - # day of the month (1-31) - # month (1-12) - # day of the week (0-6) - # nightly build at 15:50 UTC time every day - - cron: "50 15 * * *" workflow_dispatch: @@ -44,7 +36,7 @@ jobs: - os: macos-latest arch: arm64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: macos-latest arch: arm64 index: 4 @@ -73,7 +65,7 @@ jobs: - os: macos-15-intel arch: amd64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: macos-15-intel arch: amd64 index: 4 @@ -102,7 +94,7 @@ jobs: - os: ubuntu-latest arch: amd64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: ubuntu-latest arch: amd64 index: 4 @@ -131,7 +123,7 @@ jobs: - os: ubuntu-22.04-arm arch: arm64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: ubuntu-22.04-arm arch: arm64 index: 4 @@ -160,7 +152,7 @@ jobs: - os: windows-2022 arch: x64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: windows-2022 arch: x64 index: 4 @@ -189,7 +181,7 @@ jobs: - os: windows-2022 arch: x86 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: windows-2022 arch: x86 index: 4 @@ -218,7 +210,7 @@ jobs: - os: windows-11-arm arch: arm64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: windows-11-arm arch: arm64 index: 4 @@ -239,7 +231,12 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '>=1.17' + # windows/arm64 needs Go >= 1.27: Go 1.26 cgo passes the MinGW-only + # -mthreads flag, which clang rejects when it targets + # aarch64-pc-windows-msvc (the ABI used to link the MSVC-built + # sherpa-onnx-c-api.dll). Go 1.27 passes -pthread instead. + # Other platforms keep the flexible spec. + go-version: ${{ matrix.os == 'windows-11-arm' && '1.27.x' || '>=1.17' }} - name: Set up MinGW for x64 if: matrix.os == 'windows-2022' && matrix.arch == 'x64' @@ -313,7 +310,16 @@ jobs: done - uses: actions/upload-artifact@v4 - if: contains(matrix.tests, 'non-streaming-tts') || contains(matrix.tests, 'zero-shot-zipvoice-tts') + if: contains(matrix.tests, 'non-streaming-tts') || contains(matrix.tests, 'zero-shot-zipvoice-tts') || contains(matrix.tests, 'zero-shot-pocket-tts') || contains(matrix.tests, 'supertonic-tts') with: name: tts-waves-${{ matrix.os }}-${{ matrix.arch }} - path: tts-waves + path: | + go-api-examples/non-streaming-tts/*.wav + go-api-examples/non-streaming-tts-phonemize/*.wav + go-api-examples/offline-tts-play/*.wav + go-api-examples/zero-shot-zipvoice-tts/*.wav + go-api-examples/zero-shot-zipvoice-tts-play/*.wav + go-api-examples/zero-shot-pocket-tts/*.wav + go-api-examples/zero-shot-pocket-tts-play/*.wav + go-api-examples/supertonic-tts/*.wav + if-no-files-found: warn diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index 754351055..8e24e10a1 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -72,7 +72,12 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '>=1.17' + # windows/arm64 needs Go >= 1.27: Go 1.26 cgo passes the MinGW-only + # -mthreads flag, which clang rejects when it targets + # aarch64-pc-windows-msvc (the ABI used to link the MSVC-built + # sherpa-onnx-c-api.dll). Go 1.27 passes -pthread instead. + # Other platforms keep the flexible spec. + go-version: ${{ matrix.os == 'windows-11-arm' && '1.27.x' || '>=1.17' }} - name: Set GOARCH for x86 if: matrix.os == 'windows-2022' && matrix.arch == 'x86' @@ -238,7 +243,7 @@ jobs: - os: macos-latest arch: arm64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: macos-latest arch: arm64 index: 4 @@ -250,7 +255,7 @@ jobs: - os: macos-latest arch: arm64 index: 6 - tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding" + tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding offline-zipformer-ctc-hlg" - os: macos-latest arch: arm64 index: 7 @@ -275,7 +280,7 @@ jobs: - os: macos-15-intel arch: amd64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: macos-15-intel arch: amd64 index: 4 @@ -287,7 +292,7 @@ jobs: - os: macos-15-intel arch: amd64 index: 6 - tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding" + tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding offline-zipformer-ctc-hlg" - os: macos-15-intel arch: amd64 index: 7 @@ -312,7 +317,7 @@ jobs: - os: ubuntu-latest arch: amd64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: ubuntu-latest arch: amd64 index: 4 @@ -324,7 +329,7 @@ jobs: - os: ubuntu-latest arch: amd64 index: 6 - tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding" + tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding offline-zipformer-ctc-hlg" - os: ubuntu-latest arch: amd64 index: 7 @@ -349,7 +354,7 @@ jobs: - os: ubuntu-22.04-arm arch: arm64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: ubuntu-22.04-arm arch: arm64 index: 4 @@ -361,7 +366,7 @@ jobs: - os: ubuntu-22.04-arm arch: arm64 index: 6 - tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding" + tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding offline-zipformer-ctc-hlg" - os: ubuntu-22.04-arm arch: arm64 index: 7 @@ -386,7 +391,7 @@ jobs: - os: windows-2022 arch: x64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: windows-2022 arch: x64 index: 4 @@ -398,7 +403,7 @@ jobs: - os: windows-2022 arch: x64 index: 6 - tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding" + tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding offline-zipformer-ctc-hlg" - os: windows-2022 arch: x64 index: 7 @@ -423,7 +428,7 @@ jobs: - os: windows-2022 arch: x86 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: windows-2022 arch: x86 index: 4 @@ -435,7 +440,7 @@ jobs: - os: windows-2022 arch: x86 index: 6 - tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding" + tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding offline-zipformer-ctc-hlg" - os: windows-2022 arch: x86 index: 7 @@ -460,7 +465,7 @@ jobs: - os: windows-11-arm arch: arm64 index: 3 - tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts streaming-decode-files" + tests: "non-streaming-medasr-ctc-decode-files non-streaming-omnilingual-asr-ctc-decode-files non-streaming-tts non-streaming-tts-phonemize streaming-decode-files" - os: windows-11-arm arch: arm64 index: 4 @@ -472,7 +477,7 @@ jobs: - os: windows-11-arm arch: arm64 index: 6 - tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding" + tests: "add-punctuation-online non-streaming-speaker-diarization streaming-hlg-decoding offline-zipformer-ctc-hlg" - os: windows-11-arm arch: arm64 index: 7 @@ -487,7 +492,12 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '>=1.17' + # windows/arm64 needs Go >= 1.27: Go 1.26 cgo passes the MinGW-only + # -mthreads flag, which clang rejects when it targets + # aarch64-pc-windows-msvc (the ABI used to link the MSVC-built + # sherpa-onnx-c-api.dll). Go 1.27 passes -pthread instead. + # Other platforms keep the flexible spec. + go-version: ${{ matrix.os == 'windows-11-arm' && '1.27.x' || '>=1.17' }} - name: Set up MinGW for x86 if: matrix.os == 'windows-2022' && matrix.arch == 'x86' @@ -537,3 +547,18 @@ jobs: .github/scripts/go/${test}.sh echo "=== ${test} passed ===" done + + - uses: actions/upload-artifact@v4 + if: contains(matrix.tests, 'non-streaming-tts') || contains(matrix.tests, 'zero-shot-zipvoice-tts') || contains(matrix.tests, 'zero-shot-pocket-tts') || contains(matrix.tests, 'supertonic-tts') + with: + name: tts-waves-${{ matrix.os }}-${{ matrix.arch }} + path: | + go-api-examples/non-streaming-tts/*.wav + go-api-examples/non-streaming-tts-phonemize/*.wav + go-api-examples/offline-tts-play/*.wav + go-api-examples/zero-shot-zipvoice-tts/*.wav + go-api-examples/zero-shot-zipvoice-tts-play/*.wav + go-api-examples/zero-shot-pocket-tts/*.wav + go-api-examples/zero-shot-pocket-tts-play/*.wav + go-api-examples/supertonic-tts/*.wav + if-no-files-found: warn diff --git a/.github/workflows/test-nodejs-addon-api.yaml b/.github/workflows/test-nodejs-addon-api.yaml index 190f9b5fd..c838bd45e 100644 --- a/.github/workflows/test-nodejs-addon-api.yaml +++ b/.github/workflows/test-nodejs-addon-api.yaml @@ -168,4 +168,21 @@ jobs: cp -v scripts/node-addon-api/lib/*.js $d/ cp -v ./build/install/lib/lib* $d/ + # piper-phonemize-node (and the other example deps) come from npm + (cd $d && npm install) + + # ...but the tests must run against the addon we just built: npm + # install also brings in sherpa-onnx-node with a prebuilt + # sherpa-onnx- binary, and addon.js looks in + # node_modules/ before ./sherpa-onnx.node. Drop those so the + # freshly built ./sherpa-onnx.node copied above is used. + rm -rf $d/node_modules/sherpa-onnx-* + .github/scripts/test-nodejs-addon-npm.sh + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ matrix.os }}-${{ matrix.node-version }} + path: | + nodejs-addon-examples/test-*-lexicon.wav + nodejs-addon-examples/test-*-phonemize.wav + if-no-files-found: warn diff --git a/.github/workflows/test-nodejs-addon-npm.yaml b/.github/workflows/test-nodejs-addon-npm.yaml index 6da0c0019..81c1abc5c 100644 --- a/.github/workflows/test-nodejs-addon-npm.yaml +++ b/.github/workflows/test-nodejs-addon-npm.yaml @@ -75,3 +75,10 @@ jobs: cd ../ .github/scripts/test-nodejs-addon-npm.sh + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ matrix.os }}-${{ matrix.node-version }} + path: | + nodejs-addon-examples/test-*-lexicon.wav + nodejs-addon-examples/test-*-phonemize.wav + if-no-files-found: warn diff --git a/.github/workflows/test-nodejs-npm.yaml b/.github/workflows/test-nodejs-npm.yaml index 5b6562063..218f40cc2 100644 --- a/.github/workflows/test-nodejs-npm.yaml +++ b/.github/workflows/test-nodejs-npm.yaml @@ -3,15 +3,6 @@ name: test-nodejs-npm on: workflow_dispatch: - schedule: - # minute (0-59) - # hour (0-23) - # day of the month (1-31) - # month (1-12) - # day of the week (0-6) - # nightly build at 23:50 UTC time every day - - cron: "50 23 * * *" - concurrency: group: test-nodejs-npm-${{ github.ref }} cancel-in-progress: true @@ -63,3 +54,10 @@ jobs: export d=nodejs-examples ./.github/scripts/test-nodejs-npm.sh + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ matrix.os }}-${{ matrix.node-version }} + # all TTS outputs: *-lexicon.wav, *-phonemize.wav, plus the + # own-frontend models (test-pocket-en.wav, test-supertonic-*.wav) + path: nodejs-examples/test-*.wav + if-no-files-found: warn diff --git a/.github/workflows/test-nodejs.yaml b/.github/workflows/test-nodejs.yaml index 3bc863167..364624dcb 100644 --- a/.github/workflows/test-nodejs.yaml +++ b/.github/workflows/test-nodejs.yaml @@ -113,9 +113,18 @@ jobs: pushd $d npm install npm install wav + npm install piper-phonemize popd ./.github/scripts/test-nodejs-npm.sh + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ matrix.os }}-${{ matrix.node-version }} + # The examples are copied into scripts/nodejs and run from there + # (export d=scripts/nodejs), so their wavs land in scripts/nodejs, + # not in nodejs-examples. + path: scripts/nodejs/test-*.wav + if-no-files-found: warn - name: Setup tmate session if: false diff --git a/.github/workflows/test-onnxruntime-version.yaml b/.github/workflows/test-onnxruntime-version.yaml index 3d84d849a..55b6dac45 100644 --- a/.github/workflows/test-onnxruntime-version.yaml +++ b/.github/workflows/test-onnxruntime-version.yaml @@ -94,11 +94,6 @@ jobs: ls -lh bin file ./bin/sherpa-onnx - - rm -fv ./install/include/cargs.h - rm -fv ./install/lib/cargs.h - rm -fv ./install/lib/libcargs.dylib - rm -fv ./install/lib/libcargs.a rm -rfv ./install/lib/pkgconfig - name: Display dependencies of sherpa-onnx for macos diff --git a/.github/workflows/test-pip-install.yaml b/.github/workflows/test-pip-install.yaml index bb73ca0a1..c7edce710 100644 --- a/.github/workflows/test-pip-install.yaml +++ b/.github/workflows/test-pip-install.yaml @@ -3,15 +3,7 @@ name: test-pip-install on: push: branches: - - test-pip-install - schedule: - # minute (0-59) - # hour (0-23) - # day of the month (1-31) - # month (1-12) - # day of the week (0-6) - # nightly build at 23:50 UTC time every day - - cron: "50 23 * * *" + - master workflow_dispatch: concurrency: diff --git a/.github/workflows/test-piper-phonemize.yaml b/.github/workflows/test-piper-phonemize.yaml deleted file mode 100644 index 7a4fa99ff..000000000 --- a/.github/workflows/test-piper-phonemize.yaml +++ /dev/null @@ -1,91 +0,0 @@ -name: test-piper-phonemize -on: - push: - branches: - - master - paths: - - '.github/workflows/test-piper-phonemize.yaml' - - 'cmake/**' - - 'sherpa-onnx/csrc/*' - - workflow_dispatch: - -concurrency: - group: test-piper-phonemize-${{ github.ref }} - cancel-in-progress: true - -jobs: - test_piper_phonemize: - name: ${{ matrix.os }} ${{ matrix.build_type }} ${{ matrix.shared_lib }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-2022] - build_type: [Release, Debug] - shared_lib: [ON, OFF] - exclude: - - os: windows-2022 - build_type: Debug - shared_lib: OFF - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Update version - shell: bash - run: | - ./new-release.sh - git diff . - - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ matrix.os }}-${{ matrix.build_type }}-shared-${{ matrix.shared_lib }} - - - name: Configure CMake - shell: bash - run: | - export CMAKE_CXX_COMPILER_LAUNCHER=ccache - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - cmake --version - - mkdir build - cd build - cmake -DSHERPA_ONNX_ENABLE_EPSEAK_NG_EXE=ON -DBUILD_ESPEAK_NG_EXE=ON -DCMAKE_VERBOSE_MAKEFILE=ON -D SHERPA_ONNX_ENABLE_TESTS=ON -D CMAKE_BUILD_TYPE=${{ matrix.build_type }} -D BUILD_SHARED_LIBS=${{ matrix.shared_lib }} -DCMAKE_INSTALL_PREFIX=./install .. - - - name: Build - shell: bash - run: | - export CMAKE_CXX_COMPILER_LAUNCHER=ccache - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - cmake --version - - cd build - cmake --build . --target install --config ${{ matrix.build_type }} - - - name: run test - if: matrix.os != 'windows-2022' - shell: bash - run: | - cd build - - ls -lh install/ - ls -lh install/share - ls -lh install/share/espeak-ng-data/ - - ./bin/piper-phonemize-test - - - name: run test - if: matrix.os == 'windows-2022' - shell: bash - run: | - cd build - - ls -lh install/ - ls -lh install/share - ls -lh install/share/espeak-ng-data/ - - ./bin/${{ matrix.build_type }}/piper-phonemize-test diff --git a/.github/workflows/test-rust-package.yaml b/.github/workflows/test-rust-package.yaml index f7d51d2be..24a671011 100644 --- a/.github/workflows/test-rust-package.yaml +++ b/.github/workflows/test-rust-package.yaml @@ -156,6 +156,12 @@ jobs: run: | ./.github/scripts/test-rust-shared.sh + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ matrix.os }}-${{ matrix.lib_setup }} + path: rust-api-examples/generated-*-rust.wav + if-no-files-found: warn + termux: name: Termux ${{ matrix.arch }} (${{ matrix.lib_setup }}) runs-on: ${{ matrix.runner }} @@ -255,7 +261,6 @@ jobs: --volume ${{ github.workspace }}/:/k2-fsa/sherpa-onnx \ --volume /tmp/sherpa-onnx-android-archive:/archive \ --volume /tmp/sherpa-onnx-android-lib:/lib \ - -e LIB_SETUP \ -e SHERPA_ONNX_LIB_DIR \ -e SHERPA_ONNX_ARCHIVE_DIR \ termux/termux-docker:${{ matrix.arch }} \ @@ -276,27 +281,28 @@ jobs: cd /k2-fsa/sherpa-onnx - # Detect and set library paths based on available directories + # Detect and set library paths based on available directories. + # + # NOTE: the termux/termux-docker image does NOT pass -e environment + # variables into the container (verified: they arrive unset), so we + # must detect the library location by path here instead of + # receiving it from the host. + # + # This matters because sherpa-onnx-sys's build.rs silently falls + # back to downloading the *released* prebuilt libraries from GitHub + # when SHERPA_ONNX_LIB_DIR/SHERPA_ONNX_ARCHIVE_DIR are unset. For + # branches under test those prebuilt libs have older C struct + # layouts than the local checkout, so the examples crash with a + # wild-pointer SIGSEGV. Watch the "[build.rs]" cargo:warning lines + # in the log to confirm which library directory was picked up. echo "=== Detecting library setup ===" - - # Detect Android ABI based on architecture ARCH=$(uname -m) case "$ARCH" in - aarch64|arm64) - ABI="arm64-v8a" - ;; - armv7*|armv8l) - ABI="armeabi-v7a" - ;; - x86_64) - ABI="x86_64" - ;; - i*86) - ABI="x86" - ;; - *) - ABI="arm64-v8a" # default - ;; + aarch64|arm64) ABI="arm64-v8a" ;; + armv7*|armv8l) ABI="armeabi-v7a" ;; + x86_64) ABI="x86_64" ;; + i*86) ABI="x86" ;; + *) ABI="arm64-v8a" ;; esac echo "Detected architecture: $ARCH -> ABI: $ABI" @@ -304,12 +310,14 @@ jobs: echo "Found pre-extracted libs at /lib/jniLibs/$ABI" export SHERPA_ONNX_LIB_DIR=/lib/jniLibs/$ABI fi - if ls /archive/*.tar.bz2 &>/dev/null; then + if ls /archive/*.tar.bz2 >/dev/null 2>&1; then echo "Found archive at /archive" export SHERPA_ONNX_ARCHIVE_DIR=/archive fi - # Add sherpa-onnx library path if available + echo "SHERPA_ONNX_LIB_DIR=${SHERPA_ONNX_LIB_DIR:-unset}" + echo "SHERPA_ONNX_ARCHIVE_DIR=${SHERPA_ONNX_ARCHIVE_DIR:-unset}" + if [ -n "$SHERPA_ONNX_LIB_DIR" ]; then export LD_LIBRARY_PATH=$SHERPA_ONNX_LIB_DIR:$LD_LIBRARY_PATH fi @@ -335,3 +343,9 @@ jobs: # Run all tests (Android only supports shared linking) bash .github/scripts/test-rust-shared.sh ' + + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-termux-${{ matrix.arch }}-${{ matrix.lib_setup }} + path: rust-api-examples/generated-*-rust.wav + if-no-files-found: warn diff --git a/.github/workflows/test-rust.yaml b/.github/workflows/test-rust.yaml index c38b19bf3..d450da062 100644 --- a/.github/workflows/test-rust.yaml +++ b/.github/workflows/test-rust.yaml @@ -33,12 +33,10 @@ jobs: with: key: ${{ matrix.os }}-rust-${{ matrix.link_mode }} - - name: Cache installed libs - id: cache-libs - uses: actions/cache@v4 - with: - path: build/install - key: rust-libs-${{ matrix.os }}-${{ matrix.link_mode }}-${{ hashFiles('CMakeLists.txt') }} + # NOTE: caching build/install is disabled on purpose. The old cache key + # only hashed CMakeLists.txt, so a struct/ABI change in c-api.h could + # restore prebuilt libs that disagree with the Rust FFI structs and + # crash at runtime. The libs are rebuilt from source every run. - name: Update version shell: bash @@ -53,7 +51,6 @@ jobs: arch: ${{ matrix.os == 'windows-11-arm' && 'amd64_arm64' || 'x64' }} - name: Configure sherpa-onnx - if: steps.cache-libs.outputs.cache-hit != 'true' shell: bash run: | export CMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -96,7 +93,6 @@ jobs: fi - name: Build sherpa-onnx - if: steps.cache-libs.outputs.cache-hit != 'true' shell: bash run: | export CMAKE_CXX_COMPILER_LAUNCHER=ccache @@ -262,6 +258,12 @@ jobs: ./.github/scripts/test-rust-static.sh fi + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-${{ matrix.os }}-${{ matrix.link_mode }}-${{ matrix.lib_setup }} + path: rust-api-examples/generated-*-rust.wav + if-no-files-found: warn + termux: name: Termux ${{ matrix.arch }} (${{ matrix.lib_setup }}) runs-on: ${{ matrix.runner }} @@ -345,10 +347,6 @@ jobs: -DSHERPA_ONNX_ENABLE_TTS=ON \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=ON \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ -DSHERPA_ONNX_ENABLE_TESTS=OFF \ -DSHERPA_ONNX_ENABLE_CHECK=OFF \ @@ -375,17 +373,62 @@ jobs: echo 'sherpa-onnx = { path = "../sherpa-onnx/rust/sherpa-onnx" }' >> rust-api-examples/.cargo/config.toml echo 'sherpa-onnx-sys = { path = "../sherpa-onnx/rust/sherpa-onnx-sys" }' >> rust-api-examples/.cargo/config.toml + - name: Prepare directories and stage libs (host) + shell: bash + env: + LIB_SETUP: ${{ matrix.lib_setup }} + run: | + # Do all directory creation and file staging on the HOST: + # * the Termux container cannot always create directories (read-only + # bits of the filesystem / different uid than the build container); + # * files created by the build container may not be writable by the + # test container, hence chmod -R 777. + # The container then only *detects* what the host prepared (see the + # "Detecting library setup" comment in the Run Rust tests step). + # + # sudo is required: build/ was written by the build container under a + # different uid, so the runner user cannot chmod it or create + # directories inside it ("Operation not permitted"/"Permission denied"). + sudo chmod -R 777 "${{ github.workspace }}" || true + mkdir -p "${{ github.workspace }}/rust-api-examples/target" + chmod -R 777 "${{ github.workspace }}/rust-api-examples/target" || true + + if [[ "$LIB_SETUP" == "archive_dir" ]]; then + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d "\"" -f 2) + # build.rs expects the archive to contain jniLibs/{abi}/ at its root + sudo mkdir -p build/install/jniLibs/arm64-v8a + sudo cp -v build/install/lib/*.so build/install/jniLibs/arm64-v8a/ + sudo chmod -R 777 build/install/jniLibs + mkdir -p /tmp/sherpa-onnx-archive + rm -f /tmp/sherpa-onnx-archive/sherpa-onnx-v${SHERPA_ONNX_VERSION}-android.tar.bz2 + tar cjf /tmp/sherpa-onnx-archive/sherpa-onnx-v${SHERPA_ONNX_VERSION}-android.tar.bz2 -C build/install jniLibs + chmod -R 777 /tmp/sherpa-onnx-archive + ls -lh /tmp/sherpa-onnx-archive/ + fi + - name: Run Rust tests in Termux shell: bash env: LIB_SETUP: ${{ matrix.lib_setup }} run: | + # NOTE: the termux/termux-docker image does NOT pass -e environment + # variables into the container (verified: both -e LIB_SETUP and + # -e SHERPA_ONNX_LIB_DIR arrived unset). So the library location is + # detected by path *inside* the container instead. + # + # This matters because sherpa-onnx-sys's build.rs silently falls + # back to downloading the *released* prebuilt libraries from GitHub + # when SHERPA_ONNX_LIB_DIR/SHERPA_ONNX_ARCHIVE_DIR are unset. Those + # prebuilt libs have older C struct layouts than this checkout, so + # the examples crash with a wild-pointer SIGSEGV. See the + # "[build.rs]" cargo:warning lines in the log to confirm which + # library directory was picked up. docker run --rm \ --volume ${{ github.workspace }}/:/k2-fsa/sherpa-onnx \ - -e LIB_SETUP \ + --volume /tmp/sherpa-onnx-archive:/tmp/sherpa-onnx-archive \ termux/termux-docker:${{ matrix.arch }} \ bash -c ' - set -e + set -ex apt update -y apt install -y libc++ clang git @@ -400,23 +443,21 @@ jobs: cd /k2-fsa/sherpa-onnx - # Set environment variables based on lib_setup mode - if [ "$LIB_SETUP" = "lib_dir" ]; then + # Set the library location by detecting which one the host prepared + if [ -d /k2-fsa/sherpa-onnx/build/install/lib ]; then export SHERPA_ONNX_LIB_DIR=/k2-fsa/sherpa-onnx/build/install/lib - elif [ "$LIB_SETUP" = "archive_dir" ]; then - # Create an archive with the exact name build.rs expects for Android - SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d "\"" -f 2) - mkdir -p /tmp/sherpa-onnx-archive - cd build/install - # Android archive name: sherpa-onnx-v{version}-android.tar.bz2 - # But we need jniLibs/{abi}/ structure, so create that - mkdir -p jniLibs/arm64-v8a - cp lib/*.so jniLibs/arm64-v8a/ - tar cjf /tmp/sherpa-onnx-archive/sherpa-onnx-v${SHERPA_ONNX_VERSION}-android.tar.bz2 jniLibs/ - cd /k2-fsa/sherpa-onnx + elif ls /tmp/sherpa-onnx-archive/*.tar.bz2 >/dev/null 2>&1; then export SHERPA_ONNX_ARCHIVE_DIR=/tmp/sherpa-onnx-archive fi + echo "SHERPA_ONNX_LIB_DIR=${SHERPA_ONNX_LIB_DIR:-unset}" + echo "SHERPA_ONNX_ARCHIVE_DIR=${SHERPA_ONNX_ARCHIVE_DIR:-unset}" # Run all tests (Android only supports shared linking) bash .github/scripts/test-rust-shared.sh ' + + - uses: actions/upload-artifact@v4 + with: + name: tts-waves-termux-${{ matrix.arch }}-${{ matrix.lib_setup }} + path: rust-api-examples/generated-*-rust.wav + if-no-files-found: warn diff --git a/.github/workflows/test-tauri-hello-world.yaml b/.github/workflows/test-tauri-hello-world.yaml index 874619588..ea84e7580 100644 --- a/.github/workflows/test-tauri-hello-world.yaml +++ b/.github/workflows/test-tauri-hello-world.yaml @@ -67,7 +67,6 @@ jobs: -DBUILD_SHARED_LIBS=$build_shared \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -395,7 +394,6 @@ jobs: -DBUILD_SHARED_LIBS=$build_shared \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -550,7 +548,6 @@ jobs: -DBUILD_SHARED_LIBS=$build_shared \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -560,7 +557,6 @@ jobs: -DBUILD_SHARED_LIBS=$build_shared \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. diff --git a/.github/workflows/test-uv-install.yaml b/.github/workflows/test-uv-install.yaml index b6bab5c23..3797fdef9 100644 --- a/.github/workflows/test-uv-install.yaml +++ b/.github/workflows/test-uv-install.yaml @@ -3,15 +3,7 @@ name: test-uv-install on: push: branches: - - test-uv-install - schedule: - # minute (0-59) - # hour (0-23) - # day of the month (1-31) - # month (1-12) - # day of the week (0-6) - # nightly build at 23:50 UTC time every day - - cron: "50 23 * * *" + - master workflow_dispatch: concurrency: @@ -178,4 +170,4 @@ jobs: uv run sherpa-onnx-offline-websocket-server --help uv run sherpa-onnx-online-websocket-server --help - uv run sherpa-onnx-online-websocket-client --help \ No newline at end of file + uv run sherpa-onnx-online-websocket-client --help diff --git a/.github/workflows/wasm-simd-hf-space-speech-enhancement-gtcrn.yaml b/.github/workflows/wasm-simd-hf-space-speech-enhancement-gtcrn.yaml index 3fecebe9a..71af4fb4d 100644 --- a/.github/workflows/wasm-simd-hf-space-speech-enhancement-gtcrn.yaml +++ b/.github/workflows/wasm-simd-hf-space-speech-enhancement-gtcrn.yaml @@ -86,7 +86,7 @@ jobs: file: ./*.tar.bz2 # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.10.46 + # tag: v1.13.8 - name: Release if: github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') diff --git a/.github/workflows/wasm-simd-hf-space-vad-asr.yaml b/.github/workflows/wasm-simd-hf-space-vad-asr.yaml index c900f2474..480ee4fc9 100644 --- a/.github/workflows/wasm-simd-hf-space-vad-asr.yaml +++ b/.github/workflows/wasm-simd-hf-space-vad-asr.yaml @@ -93,7 +93,7 @@ jobs: file: ./*.tar.bz2 # repo_name: k2-fsa/sherpa-onnx # repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - # tag: v1.10.23 + # tag: v1.13.8 - name: Upload wasm files uses: actions/upload-artifact@v4 diff --git a/.github/workflows/windows-arm64-jni.yaml b/.github/workflows/windows-arm64-jni.yaml index 65b0bea66..29e444392 100644 --- a/.github/workflows/windows-arm64-jni.yaml +++ b/.github/workflows/windows-arm64-jni.yaml @@ -50,7 +50,6 @@ jobs: -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DSHERPA_ONNX_ENABLE_C_API=OFF \ diff --git a/.github/workflows/windows-arm64.yaml b/.github/workflows/windows-arm64.yaml index 4c4c27a24..451d833f6 100644 --- a/.github/workflows/windows-arm64.yaml +++ b/.github/workflows/windows-arm64.yaml @@ -73,7 +73,6 @@ jobs: -D SHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -D BUILD_SHARED_LIBS=${{ matrix.shared_lib }} \ -D CMAKE_INSTALL_PREFIX=./install \ - -D BUILD_ESPEAK_NG_EXE=OFF \ .. - name: Check 1 @@ -198,8 +197,6 @@ jobs: fi rm -rf $dst/lib/pkgconfig - rm -f $dst/lib/cargs.dll $dst/lib/cargs.lib $dst/lib/cargs.h - rm -f $dst/include/cargs.h ls -lh $dst/bin/ echo "---" @@ -212,10 +209,8 @@ jobs: if [[ $shared_lib == "ON" ]]; then cp -av build/install/lib/*.dll ${lib_dst}/lib/ cp -av build/install/lib/*.lib ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/cargs.dll ${lib_dst}/lib/cargs.lib else cp -av build/install/lib/*.lib ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/cargs.lib fi ls -lh ${lib_dst}/lib/ tar cjvf ${lib_dst}.tar.bz2 ${lib_dst} @@ -224,6 +219,15 @@ jobs: ls -lh *.tar.bz2 + # The PDB debug symbols (~200 MB each) are already inside the release + # tarballs above. The loose copies in build/ and build/install are only + # needed for debugging a crash locally, not for running CI tests, so + # drop them here to free several GB for the publish/test steps below. + rm -f build/bin/${{ matrix.build_type }}/*.pdb + rm -f build/install/bin/*.pdb + echo "Freed PDB files. Disk now:" + df -h . + # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') @@ -267,7 +271,7 @@ jobs: file: sherpa-onnx-*-win-arm64*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.28 + tag: v1.13.8 - name: Release pre-compiled binaries and libs for Windows arm64 if: github.repository_owner == 'k2-fsa'&& github.event_name == 'push' && contains(github.ref, 'refs/tags/') @@ -277,6 +281,28 @@ jobs: overwrite: true file: sherpa-onnx-*-win-arm64*.tar.bz2 + # The release artifacts, the huggingface clone and the RelWithDebInfo/Debug + # PDB files are no longer needed once we get here (the Release steps above + # are tag-only and are skipped for branch builds). Windows arm64 runners + # have limited disk, and the tests below each download large ASR models, + # so free that space first or the model extraction fails with "no space + # left on device". Only the test binaries in build/bin/ and + # the shared libs they load are kept. + - name: Free disk space before tests + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + shell: bash + run: | + echo "Disk space before cleanup:" + df -h . + rm -rf huggingface + rm -rf sherpa-onnx-*-win-arm64*/ + rm -f sherpa-onnx-*-win-arm64*.tar.bz2 + rm -f build/bin/${{ matrix.build_type }}/*.pdb + rm -f build/install/bin/*.pdb + rm -rf build/install/share build/install/lib/pkgconfig + echo "Disk space after cleanup:" + df -h . + - name: Test offline Cohere Transcribe for windows arm64 if: ${{ !startsWith(github.ref, 'refs/tags/') }} shell: bash diff --git a/.github/workflows/windows-x64-jni.yaml b/.github/workflows/windows-x64-jni.yaml index fbcd5bfb3..dad33a113 100644 --- a/.github/workflows/windows-x64-jni.yaml +++ b/.github/workflows/windows-x64-jni.yaml @@ -50,7 +50,6 @@ jobs: -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DSHERPA_ONNX_ENABLE_C_API=OFF \ diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index 33220bcb8..439fbca4d 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -73,7 +73,6 @@ jobs: -D SHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -D BUILD_SHARED_LIBS=${{ matrix.shared_lib }} \ -D CMAKE_INSTALL_PREFIX=./install \ - -D BUILD_ESPEAK_NG_EXE=OFF \ .. - name: Check 1 @@ -198,8 +197,6 @@ jobs: fi rm -rf $dst/lib/pkgconfig - rm -f $dst/lib/cargs.dll $dst/lib/cargs.lib $dst/lib/cargs.h - rm -f $dst/include/cargs.h ls -lh $dst/bin/ echo "---" @@ -212,10 +209,8 @@ jobs: if [[ $shared_lib == "ON" ]]; then cp -av build/install/lib/*.dll ${lib_dst}/lib/ cp -av build/install/lib/*.lib ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/cargs.dll ${lib_dst}/lib/cargs.lib else cp -av build/install/lib/*.lib ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/cargs.lib fi ls -lh ${lib_dst}/lib/ tar cjvf ${lib_dst}.tar.bz2 ${lib_dst} @@ -224,6 +219,15 @@ jobs: ls -lh *.tar.bz2 + # The PDB debug symbols (~200 MB each) are already inside the release + # tarballs above. The loose copies in build/ and build/install are only + # needed for debugging a crash locally, not for running CI tests, so + # drop them here to free several GB for the publish/test steps below. + rm -f build/bin/${{ matrix.build_type }}/*.pdb + rm -f build/install/bin/*.pdb + echo "Freed PDB files. Disk now:" + df -h . + # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') @@ -267,7 +271,7 @@ jobs: file: sherpa-onnx-*-win-x64*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.31 + tag: v1.13.8 - name: Release pre-compiled binaries and libs for Windows x64 if: github.repository_owner == 'k2-fsa'&& github.event_name == 'push' && contains(github.ref, 'refs/tags/') diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index 88d826192..3597111e5 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -68,7 +68,6 @@ jobs: -D SHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -D BUILD_SHARED_LIBS=${{ matrix.shared_lib }} \ -D CMAKE_INSTALL_PREFIX=./install \ - -D BUILD_ESPEAK_NG_EXE=OFF \ .. - name: Check 1 @@ -190,8 +189,6 @@ jobs: fi rm -rf $dst/lib/pkgconfig - rm -f $dst/lib/cargs.dll $dst/lib/cargs.lib $dst/lib/cargs.h - rm -f $dst/include/cargs.h ls -lh $dst/bin/ echo "---" @@ -204,10 +201,8 @@ jobs: if [[ $shared_lib == "ON" ]]; then cp -av build/install/lib/*.dll ${lib_dst}/lib/ cp -av build/install/lib/*.lib ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/cargs.dll ${lib_dst}/lib/cargs.lib else cp -av build/install/lib/*.lib ${lib_dst}/lib/ - rm -f ${lib_dst}/lib/cargs.lib fi ls -lh ${lib_dst}/lib/ tar cjvf ${lib_dst}.tar.bz2 ${lib_dst} @@ -216,6 +211,15 @@ jobs: ls -lh *.tar.bz2 + # The PDB debug symbols (~200 MB each) are already inside the release + # tarballs above. The loose copies in build/ and build/install are only + # needed for debugging a crash locally, not for running CI tests, so + # drop them here to free several GB for the publish/test steps below. + rm -f build/bin/${{ matrix.build_type }}/*.pdb + rm -f build/install/bin/*.pdb + echo "Freed PDB files. Disk now:" + df -h . + # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') @@ -259,7 +263,7 @@ jobs: file: sherpa-onnx-*-win-x86*.tar.bz2 repo_name: k2-fsa/sherpa-onnx repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} - tag: v1.12.28 + tag: v1.13.8 - name: Release pre-compiled binaries and libs for Windows x86 if: github.repository_owner == 'k2-fsa'&& github.event_name == 'push' && contains(github.ref, 'refs/tags/') diff --git a/.gitignore b/.gitignore index 5cab6eee0..d39911cc8 100755 --- a/.gitignore +++ b/.gitignore @@ -192,9 +192,15 @@ sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25 sherpa-onnx-spleeter-2stems-fp16 sherpa-onnx-cohere-transcribe-14-lang-int8-2026-04-01 espeak-ng-data +# downloaded piper-phonemize wasm bundle for the TTS web demo; see +# wasm/tts/assets/README.md +wasm/tts/piper-phonemize/ sherpa-onnx-nemo* kokoro-int8-en-v0_19 vits-inflect-*-v2 assets-2 swiftpm .sherpa-onnx-xcframework-version +lexicon*.txt +tts.json +*.bin diff --git a/CMakeLists.txt b/CMakeLists.txt index 03ed01e97..c3943d670 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -566,13 +566,6 @@ endif() include(json) -if(SHERPA_ONNX_ENABLE_TTS) - include(espeak-ng-for-piper) - set(ESPEAK_NG_DIR ${espeak_ng_SOURCE_DIR}) - message(STATUS "ESPEAK_NG_DIR: ${ESPEAK_NG_DIR}") - include(piper-phonemize) -endif() - if(SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION) include(hclust-cpp) endif() @@ -607,7 +600,6 @@ endif() add_subdirectory(sherpa-onnx) if(SHERPA_ONNX_ENABLE_C_API AND SHERPA_ONNX_ENABLE_BINARY AND SHERPA_ONNX_BUILD_C_API_EXAMPLES) - set(SHERPA_ONNX_PKG_WITH_CARGS "-lcargs") add_subdirectory(c-api-examples) add_subdirectory(cxx-api-examples) endif() diff --git a/Package.swift b/Package.swift index 426180bb7..403c1c268 100644 --- a/Package.swift +++ b/Package.swift @@ -88,15 +88,15 @@ let package = Package( "spoken-language-identification.swift", "streaming-hlg-decode-file.swift", "test-version.swift", - "tts-kitten-en.swift", - "tts-kokoro-en.swift", - "tts-kokoro-zh-en.swift", - "tts-matcha-en.swift", - "tts-matcha-zh.swift", + "tts-kitten-en-lexicon.swift", + "tts-kokoro-en-lexicon.swift", + "tts-kokoro-zh-en-lexicon.swift", + "tts-matcha-en-lexicon.swift", + "tts-matcha-zh-lexicon.swift", "tts-pocket-en.swift", "tts-supertonic-en.swift", - "tts-vits.swift", - "tts-zipvoice.swift", + "tts-vits-lexicon.swift", + "tts-zipvoice-lexicon.swift", "wenet-ctc-asr.swift", "zipformer-ctc-asr.swift", "run-add-punctuations-online.sh", @@ -128,15 +128,15 @@ let package = Package( "run-spoken-language-identification.sh", "run-streaming-hlg-decode-file.sh", "run-test-version.sh", - "run-tts-kitten-en.sh", - "run-tts-kokoro-en.sh", - "run-tts-kokoro-zh-en.sh", - "run-tts-matcha-en.sh", - "run-tts-matcha-zh.sh", + "run-tts-kitten-en-lexicon.sh", + "run-tts-kokoro-en-lexicon.sh", + "run-tts-kokoro-zh-en-lexicon.sh", + "run-tts-matcha-en-lexicon.sh", + "run-tts-matcha-zh-lexicon.sh", "run-tts-pocket-en.sh", "run-tts-supertonic-en.sh", - "run-tts-vits.sh", - "run-tts-zipvoice.sh", + "run-tts-vits-lexicon.sh", + "run-tts-zipvoice-lexicon.sh", "run-wenet-ctc-asr.sh", "run-zipformer-ctc-asr.sh", ], diff --git a/android/SherpaOnnxTts/app/build.gradle b/android/SherpaOnnxTts/app/build.gradle index 32b894174..d6982642d 100644 --- a/android/SherpaOnnxTts/app/build.gradle +++ b/android/SherpaOnnxTts/app/build.gradle @@ -38,6 +38,8 @@ dependencies { implementation 'androidx.core:core-ktx:1.7.0' implementation 'com.google.android.material:material:1.9.0' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' + // piper-phonemize provides the text frontend for models without a lexicon + implementation 'com.github.csukuangfj.piper-phonemize:piper-phonemize:v1.5.8' testImplementation 'junit:junit:4.13.2' androidTestImplementation 'androidx.test.ext:junit:1.1.5' androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1' diff --git a/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt index 66d673695..5490f2082 100644 --- a/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt +++ b/android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -16,9 +16,9 @@ import android.widget.Toast import androidx.activity.result.contract.ActivityResultContracts import androidx.appcompat.app.AppCompatActivity import androidx.core.content.FileProvider +import com.github.csukuangfj.piper.phonemize.piperPhonemizeInitialize +import com.github.csukuangfj.piper.phonemize.piperPhonemizeText import java.io.File -import java.io.FileOutputStream -import java.io.IOException const val TAG = "sherpa-onnx" @@ -36,6 +36,7 @@ class MainActivity : AppCompatActivity() { private var mediaPlayer: MediaPlayer? = null private var isSupertonic: Boolean = false private var supertonicLang: String = "en" + private var usePhonemize: Boolean = false private val saveLauncher = registerForActivityResult(ActivityResultContracts.CreateDocument("audio/wav")) { uri -> if (uri != null) { @@ -166,8 +167,17 @@ class MainActivity : AppCompatActivity() { if (isSupertonic) { genConfig.extra = mapOf("lang" to supertonicLang) } + var genText = textStr + if (usePhonemize) { + // Phonemize outside of sherpa-onnx with piper-phonemize and + // pass the phoneme codepoints. The espeak-ng voice is taken + // from the model metadata. + genConfig.phonemeCodepoints = phonemize(textStr) + // The text has already been phonemized; pass an empty string. + genText = "" + } val audio = tts.generateWithConfigAndCallback( - text = textStr, + text = genText, config = genConfig, callback = this::callback ) @@ -255,7 +265,7 @@ class MainActivity : AppCompatActivity() { var ruleFsts: String? var ruleFars: String? var lexicon: String? - var dataDir: String? + var usePhonemize = false var assets: AssetManager? = application.assets var isKitten = false var isSupertonic = false @@ -290,7 +300,6 @@ class MainActivity : AppCompatActivity() { ruleFsts = null ruleFars = null lexicon = null - dataDir = null // Example 1: // modelDir = "vits-vctk" @@ -302,7 +311,17 @@ class MainActivity : AppCompatActivity() { // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 // modelDir = "vits-piper-en_US-amy-low" // modelName = "en_US-amy-low.onnx" - // dataDir = "vits-piper-en_US-amy-low/espeak-ng-data" + // + // Choose exactly one of the two text frontends below (not both): + // + // usePhonemize = true // option 1: phonemize the text with piper-phonemize; no extra files needed + // lexicon = "lexicon-en-us.txt" // option 2: lexicon file inside the model directory. + // Download it from https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + // + // Neither option uses the espeak-ng-data directory shipped in the model + // package; delete it to make the APK smaller. + // + // lang = "eng" // Example 3: // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 @@ -343,21 +362,20 @@ class MainActivity : AppCompatActivity() { // modelDir = "matcha-icefall-en_US-ljspeech" // acousticModelName = "model-steps-3.onnx" // vocoder = "vocos-22khz-univ.onnx" - // dataDir = "matcha-icefall-en_US-ljspeech/espeak-ng-data" + // usePhonemize = true // or lexicon = "lexicon-en-us.txt", see Example 2 // Example 9 // kokoro-en-v0_19 // modelDir = "kokoro-en-v0_19" // modelName = "model.onnx" // voices = "voices.bin" - // dataDir = "kokoro-en-v0_19/espeak-ng-data" + // usePhonemize = true // or lexicon = "lexicon-en-us.txt", see Example 2 // Example 10 // kokoro-multi-lang-v1_0 // modelDir = "kokoro-multi-lang-v1_0" // modelName = "model.onnx" // voices = "voices.bin" - // dataDir = "kokoro-multi-lang-v1_0/espeak-ng-data" // lexicon = "kokoro-multi-lang-v1_0/lexicon-us-en.txt,kokoro-multi-lang-v1_0/lexicon-zh.txt" // ruleFsts = "$modelDir/phone-zh.fst,$modelDir/date-zh.fst,$modelDir/number-zh.fst" @@ -366,8 +384,8 @@ class MainActivity : AppCompatActivity() { // modelDir = "kitten-nano-en-v0_1-fp16" // modelName = "model.fp16.onnx" // voices = "voices.bin" - // dataDir = "kokoro-multi-lang-v1_0/espeak-ng-data" // isKitten = true + // usePhonemize = true // or lexicon = "lexicon-en-us.txt", see Example 2 // Example 12 // matcha-icefall-zh-en @@ -375,7 +393,6 @@ class MainActivity : AppCompatActivity() { // modelDir = "matcha-icefall-zh-en" // acousticModelName = "model-steps-3.onnx" // vocoder = "vocos-16khz-univ.onnx" // Vocoder should be downloaded separately; place in the **root directory of your resources folder**, not under modelDir. - // dataDir = "matcha-icefall-zh-en/espeak-ng-data" // lexicon = "lexicon.txt" // Example 13 @@ -394,10 +411,12 @@ class MainActivity : AppCompatActivity() { this.isSupertonic = isSupertonic this.supertonicLang = supertonicLang + this.usePhonemize = usePhonemize - if (dataDir != null) { - val newDir = copyDataDir(dataDir!!) - dataDir = "$newDir/$dataDir" + if (usePhonemize) { + // The espeak-ng-data embedded in piper-phonemize only feeds the + // phonemizer; sherpa-onnx itself does not use espeak-ng anymore. + piperPhonemizeInitialize(applicationContext) } val config = getOfflineTtsConfig( @@ -407,7 +426,7 @@ class MainActivity : AppCompatActivity() { vocoder = vocoder ?: "", voices = voices ?: "", lexicon = lexicon ?: "", - dataDir = dataDir ?: "", + dataDir = "", dictDir = "", ruleFsts = ruleFsts ?: "", ruleFars = ruleFars ?: "", @@ -425,53 +444,19 @@ class MainActivity : AppCompatActivity() { tts = OfflineTts(assetManager = assets, config = config) } - - private fun copyDataDir(dataDir: String): String { - Log.i(TAG, "data dir is $dataDir") - copyAssets(dataDir) - - val newDataDir = application.getExternalFilesDir(null)!!.absolutePath - Log.i(TAG, "newDataDir: $newDataDir") - return newDataDir - } - - private fun copyAssets(path: String) { - val assets: Array? - try { - assets = application.assets.list(path) - if (assets!!.isEmpty()) { - copyFile(path) - } else { - val fullPath = "${application.getExternalFilesDir(null)}/$path" - val dir = File(fullPath) - dir.mkdirs() - for (asset in assets.iterator()) { - val p: String = if (path == "") "" else path + "/" - copyAssets(p + asset) - } - } - } catch (ex: IOException) { - Log.e(TAG, "Failed to copy $path. $ex") + private fun phonemize(text: String): Array { + var voice = tts.lang() + if (voice.isEmpty()) { + Log.w(TAG, "The model has no language information. Falling back to en-us.") + voice = "en-us" } - } - - private fun copyFile(filename: String) { + Log.i(TAG, "Phonemizer language: $voice") + val result = piperPhonemizeText(text, voice) + ?: throw RuntimeException("Failed to phonemize the text") try { - val istream = application.assets.open(filename) - val newFilename = application.getExternalFilesDir(null).toString() + "/" + filename - val ostream = FileOutputStream(newFilename) - // Log.i(TAG, "Copying $filename to $newFilename") - val buffer = ByteArray(1024) - var read = 0 - while (read != -1) { - ostream.write(buffer, 0, read) - read = istream.read(buffer) - } - istream.close() - ostream.flush() - ostream.close() - } catch (ex: Exception) { - Log.e(TAG, "Failed to copy $filename, $ex") + return Array(result.numSentences) { i -> result.getPhonemes(i) } + } finally { + result.close() } } } diff --git a/android/SherpaOnnxTts/build.gradle b/android/SherpaOnnxTts/build.gradle index 253697423..771199534 100644 --- a/android/SherpaOnnxTts/build.gradle +++ b/android/SherpaOnnxTts/build.gradle @@ -2,5 +2,5 @@ plugins { id 'com.android.application' version '7.3.1' apply false id 'com.android.library' version '7.3.1' apply false - id 'org.jetbrains.kotlin.android' version '1.7.20' apply false + id 'org.jetbrains.kotlin.android' version '1.9.0' apply false } \ No newline at end of file diff --git a/android/SherpaOnnxTts/settings.gradle b/android/SherpaOnnxTts/settings.gradle index 7c107adbc..b7ad94350 100644 --- a/android/SherpaOnnxTts/settings.gradle +++ b/android/SherpaOnnxTts/settings.gradle @@ -10,6 +10,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() + maven { url 'https://jitpack.io' } } } rootProject.name = "SherpaOnnxTts" diff --git a/android/SherpaOnnxTtsEngine/app/build.gradle.kts b/android/SherpaOnnxTtsEngine/app/build.gradle.kts index 23fdf8f75..94eb067b8 100644 --- a/android/SherpaOnnxTtsEngine/app/build.gradle.kts +++ b/android/SherpaOnnxTtsEngine/app/build.gradle.kts @@ -61,6 +61,8 @@ dependencies { implementation("androidx.compose.material3:material3") implementation("androidx.appcompat:appcompat:1.6.1") implementation("com.google.android.material:material:1.9.0") + // piper-phonemize provides the text frontend for models without a lexicon + implementation("com.github.csukuangfj.piper-phonemize:piper-phonemize:v1.5.8") testImplementation("junit:junit:4.13.2") androidTestImplementation("androidx.test.ext:junit:1.1.5") androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt index fa8bba219..4b4042eb4 100644 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt @@ -249,9 +249,19 @@ class MainActivity : ComponentActivity() { if (TtsEngine.isSupertonic) { genConfig.extra = mapOf("lang" to TtsEngine.supertonicLang) } + var genText = testText + if (TtsEngine.usePhonemize) { + // Phonemize outside of sherpa-onnx with + // piper-phonemize and pass the phoneme + // codepoints. + genConfig.phonemeCodepoints = + TtsEngine.phonemize(testText) + // The text has already been phonemized. + genText = "" + } val audio = TtsEngine.tts!!.generateWithConfigAndCallback( - text = testText, + text = genText, config = genConfig, callback = ::callback, ) diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt index 9fe347190..6d63cea07 100644 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt @@ -7,11 +7,10 @@ import android.util.Log import androidx.compose.runtime.MutableState import androidx.compose.runtime.mutableFloatStateOf import androidx.compose.runtime.mutableIntStateOf +import com.github.csukuangfj.piper.phonemize.piperPhonemizeInitialize +import com.github.csukuangfj.piper.phonemize.piperPhonemizeText import com.k2fsa.sherpa.onnx.OfflineTts import com.k2fsa.sherpa.onnx.getOfflineTtsConfig -import java.io.File -import java.io.FileOutputStream -import java.io.IOException const val MIN_TTS_SPEED = 0.1f const val MAX_TTS_SPEED = 5.0f @@ -56,7 +55,7 @@ object TtsEngine { private var ruleFsts: String? = null private var ruleFars: String? = null private var lexicon: String? = null - private var dataDir: String? = null + var usePhonemize = false private var assets: AssetManager? = null private var isKitten = false var isSupertonic = false @@ -90,7 +89,6 @@ object TtsEngine { ruleFsts = null ruleFars = null lexicon = null - dataDir = null lang = null lang2 = null @@ -108,7 +106,16 @@ object TtsEngine { // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 // modelDir = "vits-piper-en_US-amy-low" // modelName = "en_US-amy-low.onnx" - // dataDir = "vits-piper-en_US-amy-low/espeak-ng-data" + // + // Choose exactly one of the two text frontends below (not both): + // + // usePhonemize = true // option 1: phonemize the text with piper-phonemize; no extra files needed + // lexicon = "lexicon-en-us.txt" // option 2: lexicon file inside the model directory. + // Download it from https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + // + // Neither option uses the espeak-ng-data directory shipped in the model + // package; delete it to make the APK smaller. + // // lang = "eng" // Example 3: @@ -128,7 +135,7 @@ object TtsEngine { // Example 5: // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-coqui-de-css10.tar.bz2 - // This model does not need lexicon or dataDir + // This model does not need lexicon or piper-phonemize // modelDir = "vits-coqui-de-css10" // modelName = "model.onnx" // lang = "deu" @@ -157,7 +164,7 @@ object TtsEngine { // modelDir = "matcha-icefall-en_US-ljspeech" // acousticModelName = "model-steps-3.onnx" // vocoder = "vocos-22khz-univ.onnx" - // dataDir = "matcha-icefall-en_US-ljspeech/espeak-ng-data" + // usePhonemize = true // or lexicon = "lexicon-en-us.txt", see Example 2 // lang = "eng" // Example 9 @@ -165,7 +172,7 @@ object TtsEngine { // modelDir = "kokoro-en-v0_19" // modelName = "model.onnx" // voices = "voices.bin" - // dataDir = "kokoro-en-v0_19/espeak-ng-data" + // usePhonemize = true // or lexicon = "lexicon-en-us.txt", see Example 2 // lang = "eng" // Example 10 @@ -173,7 +180,6 @@ object TtsEngine { // modelDir = "kokoro-multi-lang-v1_0" // modelName = "model.onnx" // voices = "voices.bin" - // dataDir = "kokoro-multi-lang-v1_0/espeak-ng-data" // lexicon = "kokoro-multi-lang-v1_0/lexicon-us-en.txt,kokoro-multi-lang-v1_0/lexicon-zh.txt" // lang = "eng" // lang2 = "zho" @@ -187,7 +193,7 @@ object TtsEngine { // modelDir = "kitten-nano-en-v0_1-fp16" // modelName = "model.fp16.onnx" // voices = "voices.bin" - // dataDir = "kitten-nano-en-v0_1-fp16/espeak-ng-data" + // usePhonemize = true // or lexicon = "lexicon-en-us.txt", see Example 2 // lang = "eng" // isKitten = true @@ -197,7 +203,6 @@ object TtsEngine { // modelDir = "matcha-icefall-zh-en" // acousticModelName = "model-steps-3.onnx" // vocoder = "vocos-16khz-univ.onnx" - // dataDir = "matcha-icefall-zh-en/espeak-ng-data" // lexicon = "lexicon.txt" // lang = "zho" @@ -226,9 +231,10 @@ object TtsEngine { private fun initTts(context: Context) { assets = context.assets - if (dataDir != null) { - val newDir = copyDataDir(context, dataDir!!) - dataDir = "$newDir/$dataDir" + if (usePhonemize) { + // The espeak-ng-data embedded in piper-phonemize only feeds the + // phonemizer; sherpa-onnx itself does not use espeak-ng anymore. + piperPhonemizeInitialize(context) } val config = getOfflineTtsConfig( @@ -238,7 +244,7 @@ object TtsEngine { vocoder = vocoder ?: "", voices = voices ?: "", lexicon = lexicon ?: "", - dataDir = dataDir ?: "", + dataDir = "", dictDir = "", ruleFsts = ruleFsts ?: "", ruleFars = ruleFars ?: "", @@ -260,52 +266,19 @@ object TtsEngine { } - private fun copyDataDir(context: Context, dataDir: String): String { - Log.i(TAG, "data dir is $dataDir") - copyAssets(context, dataDir) - - val newDataDir = context.getExternalFilesDir(null)!!.absolutePath - Log.i(TAG, "newDataDir: $newDataDir") - return newDataDir - } - - private fun copyAssets(context: Context, path: String) { - val assets: Array? - try { - assets = context.assets.list(path) - if (assets!!.isEmpty()) { - copyFile(context, path) - } else { - val fullPath = "${context.getExternalFilesDir(null)}/$path" - val dir = File(fullPath) - dir.mkdirs() - for (asset in assets.iterator()) { - val p: String = if (path == "") "" else "$path/" - copyAssets(context, p + asset) - } - } - } catch (ex: IOException) { - Log.e(TAG, "Failed to copy $path. $ex") + fun phonemize(text: String): Array { + var voice = tts?.lang() ?: "" + if (voice.isEmpty()) { + Log.w(TAG, "The model has no language information. Falling back to en-us.") + voice = "en-us" } - } - - private fun copyFile(context: Context, filename: String) { + Log.i(TAG, "Phonemizer language: $voice") + val result = piperPhonemizeText(text, voice) + ?: throw RuntimeException("Failed to phonemize the text") try { - val istream = context.assets.open(filename) - val newFilename = context.getExternalFilesDir(null).toString() + "/" + filename - val ostream = FileOutputStream(newFilename) - // Log.i(TAG, "Copying $filename to $newFilename") - val buffer = ByteArray(1024) - var read = 0 - while (read != -1) { - ostream.write(buffer, 0, read) - read = istream.read(buffer) - } - istream.close() - ostream.flush() - ostream.close() - } catch (ex: Exception) { - Log.e(TAG, "Failed to copy $filename, $ex") + return Array(result.numSentences) { i -> result.getPhonemes(i) } + } finally { + result.close() } } } diff --git a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt index 05ef5d3d6..5b56a6655 100644 --- a/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt +++ b/android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt @@ -158,9 +158,19 @@ class TtsService : TextToSpeechService() { } Log.i(TAG, "text: $text") + val genConfig = GenerationConfig(sid = TtsEngine.speakerId, speed = engineSpeed) + var genText = text + if (TtsEngine.usePhonemize) { + // Phonemize outside of sherpa-onnx with piper-phonemize and pass + // the phoneme codepoints. The espeak-ng voice is taken from the + // model metadata. + genConfig.phonemeCodepoints = TtsEngine.phonemize(text) + // The text has already been phonemized; pass an empty string. + genText = "" + } tts.generateWithConfigAndCallback( - text = text, - config = GenerationConfig(sid = TtsEngine.speakerId, speed = engineSpeed), + text = genText, + config = genConfig, callback = ttsCallback, ) diff --git a/android/SherpaOnnxTtsEngine/settings.gradle.kts b/android/SherpaOnnxTtsEngine/settings.gradle.kts index df556b808..40a42ba8c 100644 --- a/android/SherpaOnnxTtsEngine/settings.gradle.kts +++ b/android/SherpaOnnxTtsEngine/settings.gradle.kts @@ -10,6 +10,7 @@ dependencyResolutionManagement { repositories { google() mavenCentral() + maven { url = uri("https://jitpack.io") } } } diff --git a/build-aarch64-linux-gnu.sh b/build-aarch64-linux-gnu.sh index 233119256..3f2d6501a 100755 --- a/build-aarch64-linux-gnu.sh +++ b/build-aarch64-linux-gnu.sh @@ -98,10 +98,6 @@ fi cmake \ -DALSA_INCLUDE_DIR=$PWD/alsa-lib/include \ -DALSA_LIBRARY=$PWD/alsa-lib/src/.libs/libasound.so \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DSHERPA_ONNX_ENABLE_GPU=$SHERPA_ONNX_ENABLE_GPU \ diff --git a/build-android-arm64-v8a.sh b/build-android-arm64-v8a.sh index 8e05154fd..abef747a4 100755 --- a/build-android-arm64-v8a.sh +++ b/build-android-arm64-v8a.sh @@ -173,10 +173,6 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ diff --git a/build-android-armv7-eabi.sh b/build-android-armv7-eabi.sh index 58c396ebf..ee3b327e7 100755 --- a/build-android-armv7-eabi.sh +++ b/build-android-armv7-eabi.sh @@ -166,10 +166,6 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ diff --git a/build-android-x86-64.sh b/build-android-x86-64.sh index afea4a229..1d6f57e59 100755 --- a/build-android-x86-64.sh +++ b/build-android-x86-64.sh @@ -149,10 +149,6 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ diff --git a/build-android-x86.sh b/build-android-x86.sh index f9c3e47ac..f5f951ab1 100755 --- a/build-android-x86.sh +++ b/build-android-x86.sh @@ -135,10 +135,6 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ diff --git a/build-arm-linux-gnueabihf.sh b/build-arm-linux-gnueabihf.sh index 0d4efc17f..cc8ffde16 100755 --- a/build-arm-linux-gnueabihf.sh +++ b/build-arm-linux-gnueabihf.sh @@ -40,10 +40,6 @@ if [[ x"$BUILD_SHARED_LIBS" == x"" ]]; then fi cmake \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ diff --git a/build-axcl-linux-aarch64.sh b/build-axcl-linux-aarch64.sh index e3e7e5635..1f2267f6a 100755 --- a/build-axcl-linux-aarch64.sh +++ b/build-axcl-linux-aarch64.sh @@ -99,10 +99,6 @@ fi cmake \ -DALSA_INCLUDE_DIR=$PWD/alsa-lib/include \ -DALSA_LIBRARY=$PWD/alsa-lib/src/.libs/libasound.so \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DSHERPA_ONNX_ENABLE_GPU=OFF \ diff --git a/build-axera-linux-aarch64.sh b/build-axera-linux-aarch64.sh index a67167128..4fd1a7913 100755 --- a/build-axera-linux-aarch64.sh +++ b/build-axera-linux-aarch64.sh @@ -183,10 +183,6 @@ fi cmake \ -DALSA_INCLUDE_DIR=$PWD/alsa-lib/include \ -DALSA_LIBRARY=$PWD/alsa-lib/src/.libs/libasound.so \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DSHERPA_ONNX_ENABLE_GPU=OFF \ diff --git a/build-ios-no-tts.sh b/build-ios-no-tts.sh index 660b861aa..ac548c2a9 100755 --- a/build-ios-no-tts.sh +++ b/build-ios-no-tts.sh @@ -35,10 +35,6 @@ echo "SHERPA_ONNXRUNTIME_INCLUDE_DIR $SHERPA_ONNXRUNTIME_INCLUDE_DIR" cmake \ -DSHERPA_ONNX_ENABLE_TTS=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATOR64 \ @@ -64,10 +60,6 @@ echo "Building for simulator (arm64)" cmake \ -DSHERPA_ONNX_ENABLE_TTS=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATORARM64 \ @@ -98,10 +90,6 @@ export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$PWD/ios-onnxruntime/onnxruntime.xcframewo cmake \ -DSHERPA_ONNX_ENABLE_TTS=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=OS64 \ diff --git a/build-ios-shared-sherpa-with-static-onnxruntime.sh b/build-ios-shared-sherpa-with-static-onnxruntime.sh index e5c6f6d5a..4dde8b38e 100755 --- a/build-ios-shared-sherpa-with-static-onnxruntime.sh +++ b/build-ios-shared-sherpa-with-static-onnxruntime.sh @@ -56,10 +56,6 @@ echo "Building for simulator (x86_64)" if [[ ! -f build/simulator_x86_64/install/lib/libsherpa-onnx-c-api.dylib ]]; then cmake \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. -D CMAKE_VERBOSE_MAKEFILE=$CMAKE_VERBOSE_MAKEFILE \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATOR64 \ @@ -89,10 +85,6 @@ echo "Building for simulator (arm64)" if [[ ! -f build/simulator_arm64/install/lib/libsherpa-onnx-c-api.dylib ]]; then cmake \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. -D CMAKE_VERBOSE_MAKEFILE=$CMAKE_VERBOSE_MAKEFILE \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATORARM64 \ @@ -125,10 +117,6 @@ if [[ ! -f build/os64/install/lib/libsherpa-onnx-c-api.dylib ]]; then cmake \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. -D CMAKE_VERBOSE_MAKEFILE=$CMAKE_VERBOSE_MAKEFILE \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=OS64 \ diff --git a/build-ios-shared.sh b/build-ios-shared.sh index 810980104..aada35143 100755 --- a/build-ios-shared.sh +++ b/build-ios-shared.sh @@ -40,10 +40,6 @@ echo "SHERPA_ONNXRUNTIME_INCLUDE_DIR $SHERPA_ONNXRUNTIME_INCLUDE_DIR" if [[ ! -f build/simulator_x86_64/install/lib/libsherpa-onnx-c-api.dylib ]]; then cmake \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. -D CMAKE_VERBOSE_MAKEFILE=$CMAKE_VERBOSE_MAKEFILE \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATOR64 \ @@ -73,10 +69,6 @@ echo "Building for simulator (arm64)" if [[ ! -f build/simulator_arm64/install/lib/libsherpa-onnx-c-api.dylib ]]; then cmake \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. -D CMAKE_VERBOSE_MAKEFILE=$CMAKE_VERBOSE_MAKEFILE \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATORARM64 \ @@ -109,10 +101,6 @@ if [[ ! -f build/os64/install/lib/libsherpa-onnx-c-api.dylib ]]; then cmake \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. -D CMAKE_VERBOSE_MAKEFILE=$CMAKE_VERBOSE_MAKEFILE \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=OS64 \ diff --git a/build-ios.sh b/build-ios.sh index d61207403..c113d37e4 100755 --- a/build-ios.sh +++ b/build-ios.sh @@ -35,10 +35,6 @@ echo "SHERPA_ONNXRUNTIME_INCLUDE_DIR $SHERPA_ONNXRUNTIME_INCLUDE_DIR" # cmake \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATOR64 \ @@ -63,10 +59,6 @@ cmake --build build/simulator_x86_64 -j 4 echo "Building for simulator (arm64)" cmake \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=SIMULATORARM64 \ @@ -95,10 +87,6 @@ export SHERPA_ONNXRUNTIME_LIB_DIR=$PWD/ios-onnxruntime/onnxruntime.xcframework/i export SHERPA_ONNXRUNTIME_INCLUDE_DIR=$PWD/ios-onnxruntime/onnxruntime.xcframework/ios-arm64/onnxruntime.framework/Headers cmake \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -S .. \ -DCMAKE_TOOLCHAIN_FILE=./toolchains/ios.toolchain.cmake \ -DPLATFORM=OS64 \ @@ -128,8 +116,7 @@ echo "Generate xcframework" mkdir -p "build/simulator/lib" for f in libkaldi-native-fbank-core.a libkissfft-float.a libsherpa-onnx-c-api.a libsherpa-onnx-core.a \ libsherpa-onnx-fstfar.a libssentencepiece_core.a \ - libsherpa-onnx-fst.a libsherpa-onnx-kaldifst-core.a libkaldi-decoder-core.a \ - libucd.a libpiper_phonemize.a libespeak-ng.a; do + libsherpa-onnx-fst.a libsherpa-onnx-kaldifst-core.a libkaldi-decoder-core.a; do lipo -create build/simulator_arm64/lib/${f} \ build/simulator_x86_64/lib/${f} \ -output build/simulator/lib/${f} @@ -146,9 +133,6 @@ libtool -static -o build/simulator/libsherpa-onnx.a \ build/simulator/lib/libsherpa-onnx-fst.a \ build/simulator/lib/libsherpa-onnx-kaldifst-core.a \ build/simulator/lib/libkaldi-decoder-core.a \ - build/simulator/lib/libucd.a \ - build/simulator/lib/libpiper_phonemize.a \ - build/simulator/lib/libespeak-ng.a \ build/simulator/lib/libssentencepiece_core.a libtool -static -o build/os64/libsherpa-onnx.a \ @@ -160,9 +144,6 @@ libtool -static -o build/os64/libsherpa-onnx.a \ build/os64/lib/libsherpa-onnx-fst.a \ build/os64/lib/libsherpa-onnx-kaldifst-core.a \ build/os64/lib/libkaldi-decoder-core.a \ - build/os64/lib/libucd.a \ - build/os64/lib/libpiper_phonemize.a \ - build/os64/lib/libespeak-ng.a \ build/os64/lib/libssentencepiece_core.a # Rename to match the shared library naming convention diff --git a/build-macos-shared-sherpa-with-static-onnxruntime.sh b/build-macos-shared-sherpa-with-static-onnxruntime.sh index 967d521a1..f0a65306f 100755 --- a/build-macos-shared-sherpa-with-static-onnxruntime.sh +++ b/build-macos-shared-sherpa-with-static-onnxruntime.sh @@ -51,7 +51,6 @@ cmake \ make -j4 make install -rm -fv ./install/include/cargs.h echo "Verifying onnxruntime is NOT a dynamic dependency:" otool -L ./install/lib/libsherpa-onnx-c-api.dylib diff --git a/build-macos-shared.sh b/build-macos-shared.sh index f42f09cb8..c62024bd1 100755 --- a/build-macos-shared.sh +++ b/build-macos-shared.sh @@ -24,7 +24,6 @@ cmake \ make -j4 make install -rm -fv ./install/include/cargs.h # Create a framework bundle (like onnxruntime does) so SPM can resolve the module FRAMEWORK_DIR=SherpaOnnxC.framework diff --git a/build-macos.sh b/build-macos.sh index 1d5a00866..84cf9b746 100755 --- a/build-macos.sh +++ b/build-macos.sh @@ -24,7 +24,6 @@ cmake \ make -j4 make install -rm -fv ./install/include/cargs.h libtool -static -o ./install/lib/libsherpa-onnx.a \ ./install/lib/libsherpa-onnx-c-api.a \ @@ -35,9 +34,6 @@ libtool -static -o ./install/lib/libsherpa-onnx.a \ ./install/lib/libsherpa-onnx-fst.a \ ./install/lib/libsherpa-onnx-kaldifst-core.a \ ./install/lib/libkaldi-decoder-core.a \ - ./install/lib/libucd.a \ - ./install/lib/libpiper_phonemize.a \ - ./install/lib/libespeak-ng.a \ ./install/lib/libssentencepiece_core.a # Rename to match the expected library name for linking with -l sherpa-onnx-c-api diff --git a/build-ohos-arm64-v8a.sh b/build-ohos-arm64-v8a.sh index 4e0ecbb29..2698071c1 100755 --- a/build-ohos-arm64-v8a.sh +++ b/build-ohos-arm64-v8a.sh @@ -112,10 +112,6 @@ cmake \ -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ diff --git a/build-ohos-armeabi-v7a.sh b/build-ohos-armeabi-v7a.sh index e0a2ac883..080d5c393 100755 --- a/build-ohos-armeabi-v7a.sh +++ b/build-ohos-armeabi-v7a.sh @@ -102,10 +102,6 @@ cmake \ -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ diff --git a/build-ohos-x86-64.sh b/build-ohos-x86-64.sh index 9584edafc..cdb8f78cd 100755 --- a/build-ohos-x86-64.sh +++ b/build-ohos-x86-64.sh @@ -112,10 +112,6 @@ cmake \ -DSHERPA_ONNX_ENABLE_TTS=$SHERPA_ONNX_ENABLE_TTS \ -DSHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION=$SHERPA_ONNX_ENABLE_SPEAKER_DIARIZATION \ -DSHERPA_ONNX_ENABLE_BINARY=$SHERPA_ONNX_ENABLE_BINARY \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ diff --git a/build-riscv64-linux-gnu-spacemit.sh b/build-riscv64-linux-gnu-spacemit.sh index 7573bafb6..12823c25e 100755 --- a/build-riscv64-linux-gnu-spacemit.sh +++ b/build-riscv64-linux-gnu-spacemit.sh @@ -47,10 +47,6 @@ if [[ x"$BUILD_SHARED_LIBS" == x"" ]]; then fi cmake \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ diff --git a/build-riscv64-linux-gnu.sh b/build-riscv64-linux-gnu.sh index 91e893c11..f38198e9d 100755 --- a/build-riscv64-linux-gnu.sh +++ b/build-riscv64-linux-gnu.sh @@ -48,10 +48,6 @@ if [[ x"$BUILD_SHARED_LIBS" == x"" ]]; then fi cmake \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \ diff --git a/build-rknn-linux-aarch64.sh b/build-rknn-linux-aarch64.sh index d34a8aa5e..db8ff6989 100755 --- a/build-rknn-linux-aarch64.sh +++ b/build-rknn-linux-aarch64.sh @@ -74,10 +74,6 @@ fi cmake \ -DALSA_INCLUDE_DIR=$PWD/alsa-lib/include \ -DALSA_LIBRARY=$PWD/alsa-lib/src/.libs/libasound.so \ - -DBUILD_PIPER_PHONMIZE_EXE=OFF \ - -DBUILD_PIPER_PHONMIZE_TESTS=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ - -DBUILD_ESPEAK_NG_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ -DCMAKE_BUILD_TYPE=Release \ -DSHERPA_ONNX_ENABLE_GPU=OFF \ diff --git a/c-api-examples/CMakeLists.txt b/c-api-examples/CMakeLists.txt index 96e1a71cb..85ab36268 100644 --- a/c-api-examples/CMakeLists.txt +++ b/c-api-examples/CMakeLists.txt @@ -1,11 +1,9 @@ -include(cargs) - include_directories(${PROJECT_SOURCE_DIR}) add_executable(version-c-api version-c-api.c) target_link_libraries(version-c-api sherpa-onnx-c-api) add_executable(decode-file-c-api decode-file-c-api.c) -target_link_libraries(decode-file-c-api sherpa-onnx-c-api cargs) +target_link_libraries(decode-file-c-api sherpa-onnx-c-api) add_executable(kws-c-api kws-c-api.c) target_link_libraries(kws-c-api sherpa-onnx-c-api) @@ -32,7 +30,7 @@ target_link_libraries(online-speech-enhancement-dpdfnet-c-api sherpa-onnx-c-api) if(SHERPA_ONNX_ENABLE_TTS) add_executable(offline-tts-c-api offline-tts-c-api.c) - target_link_libraries(offline-tts-c-api sherpa-onnx-c-api cargs) + target_link_libraries(offline-tts-c-api sherpa-onnx-c-api) add_executable(matcha-tts-zh-c-api matcha-tts-zh-c-api.c) target_link_libraries(matcha-tts-zh-c-api sherpa-onnx-c-api) diff --git a/c-api-examples/Makefile b/c-api-examples/Makefile index cbaaf63c0..b74a2715b 100644 --- a/c-api-examples/Makefile +++ b/c-api-examples/Makefile @@ -1,10 +1,10 @@ CUR_DIR :=$(shell pwd) -CFLAGS := -I ../ -I ../build/_deps/cargs-src/include/ +CFLAGS := -I ../ LDFLAGS := -L ../build/lib LDFLAGS += -L ../build/_deps/onnxruntime-src/lib -LDFLAGS += -lsherpa-onnx-c-api -lsherpa-onnx-core -lkaldi-decoder-core -lsherpa-onnx-kaldifst-core -lsherpa-onnx-fstfar -lsherpa-onnx-fst -lkaldi-native-fbank-core -lkissfft-float -lpiper_phonemize -lespeak-ng -lucd -lcargs -lonnxruntime +LDFLAGS += -lsherpa-onnx-c-api -lsherpa-onnx-core -lkaldi-decoder-core -lsherpa-onnx-kaldifst-core -lsherpa-onnx-fstfar -lsherpa-onnx-fst -lkaldi-native-fbank-core -lkissfft-float -lonnxruntime LDFLAGS += -framework Foundation LDFLAGS += -lc++ LDFLAGS += -Wl,-rpath,${CUR_DIR}/../build/lib diff --git a/c-api-examples/asr-microphone-example/CMakeLists.txt b/c-api-examples/asr-microphone-example/CMakeLists.txt index 1c486bb3b..e416d5b10 100644 --- a/c-api-examples/asr-microphone-example/CMakeLists.txt +++ b/c-api-examples/asr-microphone-example/CMakeLists.txt @@ -1,6 +1,6 @@ add_executable(c-api-alsa c-api-alsa.cc alsa.cc) -target_link_libraries(c-api-alsa sherpa-onnx-c-api cargs) +target_link_libraries(c-api-alsa sherpa-onnx-c-api) if(DEFINED ENV{SHERPA_ONNX_ALSA_LIB_DIR}) target_link_libraries(c-api-alsa -L$ENV{SHERPA_ONNX_ALSA_LIB_DIR} -lasound) diff --git a/c-api-examples/asr-microphone-example/c-api-alsa.cc b/c-api-examples/asr-microphone-example/c-api-alsa.cc index 654f0ace5..ad7f30230 100644 --- a/c-api-examples/asr-microphone-example/c-api-alsa.cc +++ b/c-api-examples/asr-microphone-example/c-api-alsa.cc @@ -1,6 +1,24 @@ // c-api-examples/asr-microphone-example/c-api-alsa.cc // Copyright (c) 2022-2024 Xiaomi Corporation +// This file shows how to use sherpa-onnx C API +// for real-time speech recognition from a microphone. +// +// It runs only on Linux. +// +// clang-format off +/* +Usage + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms.tar.bz2 +tar xf sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms.tar.bz2 +rm sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms.tar.bz2 + +./c-api-alsa + + */ +// clang-format on + #include #include #include @@ -13,100 +31,8 @@ #include #include "c-api-examples/asr-microphone-example/alsa.h" - -// NOTE: You don't need to use cargs.h in your own project. -// We use it in this file to parse commandline arguments -#include "cargs.h" // NOLINT #include "sherpa-onnx/c-api/c-api.h" -static struct cag_option options[] = { - {/*.identifier =*/'h', - /*.access_letters =*/"h", - /*.access_name =*/"help", - /*.value_name =*/"help", - /*.description =*/"Show help"}, - {/*.identifier =*/'t', - /*.access_letters =*/NULL, - /*.access_name =*/"tokens", - /*.value_name =*/"tokens", - /*.description =*/"Tokens file"}, - {/*.identifier =*/'e', - /*.access_letters =*/NULL, - /*.access_name =*/"encoder", - /*.value_name =*/"encoder", - /*.description =*/"Encoder ONNX file"}, - {/*.identifier =*/'d', - /*.access_letters =*/NULL, - /*.access_name =*/"decoder", - /*.value_name =*/"decoder", - /*.description =*/"Decoder ONNX file"}, - {/*.identifier =*/'j', - /*.access_letters =*/NULL, - /*.access_name =*/"joiner", - /*.value_name =*/"joiner", - /*.description =*/"Joiner ONNX file"}, - {/*.identifier =*/'n', - /*.access_letters =*/NULL, - /*.access_name =*/"num-threads", - /*.value_name =*/"num-threads", - /*.description =*/"Number of threads"}, - {/*.identifier =*/'p', - /*.access_letters =*/NULL, - /*.access_name =*/"provider", - /*.value_name =*/"provider", - /*.description =*/"Provider: cpu (default), cuda, coreml"}, - {/*.identifier =*/'m', - /*.access_letters =*/NULL, - /*.access_name =*/"decoding-method", - /*.value_name =*/"decoding-method", - /*.description =*/ - "Decoding method: greedy_search (default), modified_beam_search"}, - {/*.identifier =*/'f', - /*.access_letters =*/NULL, - /*.access_name =*/"hotwords-file", - /*.value_name =*/"hotwords-file", - /*.description =*/ - "The file containing hotwords, one words/phrases per line, and for each " - "phrase the bpe/cjkchar are separated by a space. For example: ▁HE LL O " - "▁WORLD, 你 好 世 界"}, - {/*.identifier =*/'s', - /*.access_letters =*/NULL, - /*.access_name =*/"hotwords-score", - /*.value_name =*/"hotwords-score", - /*.description =*/ - "The bonus score for each token in hotwords. Used only when " - "decoding_method is modified_beam_search"}, -}; - -const char *kUsage = - R"( -Usage: - ./bin/c-api-alsa \ - --tokens=/path/to/tokens.txt \ - --encoder=/path/to/encoder.onnx \ - --decoder=/path/to/decoder.onnx \ - --joiner=/path/to/decoder.onnx \ - device_name - -The device name specifies which microphone to use in case there are several -on your system. You can use - - arecord -l - -to find all available microphones on your computer. For instance, if it outputs - -**** List of CAPTURE Hardware Devices **** -card 3: UACDemoV10 [UACDemoV1.0], device 0: USB Audio [USB Audio] - Subdevices: 1/1 - Subdevice #0: subdevice #0 - -and if you want to select card 3 and device 0 on that card, please use: - - plughw:3,0 - -as the device_name. -)"; - bool stop = false; static void Handler(int sig) { @@ -114,11 +40,31 @@ static void Handler(int sig) { fprintf(stderr, "\nCaught Ctrl + C. Exiting...\n"); } -int32_t main(int32_t argc, char *argv[]) { - if (argc < 6) { - fprintf(stderr, "%s\n", kUsage); - exit(0); - } +int32_t main() { + // Recording device to use. Run + // + // arecord -l + // + // to list all available microphones. For instance, if it prints + // + // **** List of CAPTURE Hardware Devices **** + // card 3: UACDemoV10 [UACDemoV1.0], device 0: USB Audio [USB Audio] + // + // and you want card 3, device 0, then use "plughw:3,0" below. + const char *device_name = "default"; + + const char *tokens = + "./sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms/" + "tokens.txt"; + const char *encoder = + "./sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms/" + "encoder.onnx"; + const char *decoder = + "./sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms/" + "decoder.onnx"; + const char *joiner = + "./sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms/" + "joiner.onnx"; signal(SIGINT, Handler); @@ -128,6 +74,10 @@ int32_t main(int32_t argc, char *argv[]) { config.model_config.debug = 0; config.model_config.num_threads = 1; config.model_config.provider = "cpu"; + config.model_config.tokens = tokens; + config.model_config.transducer.encoder = encoder; + config.model_config.transducer.decoder = decoder; + config.model_config.transducer.joiner = joiner; config.decoding_method = "greedy_search"; @@ -141,54 +91,6 @@ int32_t main(int32_t argc, char *argv[]) { config.rule2_min_trailing_silence = 1.2; config.rule3_min_utterance_length = 300; - cag_option_context context; - char identifier; - const char *value; - - cag_option_prepare(&context, options, CAG_ARRAY_SIZE(options), argc, argv); - - while (cag_option_fetch(&context)) { - identifier = cag_option_get(&context); - value = cag_option_get_value(&context); - switch (identifier) { - case 't': - config.model_config.tokens = value; - break; - case 'e': - config.model_config.transducer.encoder = value; - break; - case 'd': - config.model_config.transducer.decoder = value; - break; - case 'j': - config.model_config.transducer.joiner = value; - break; - case 'n': - config.model_config.num_threads = atoi(value); - break; - case 'p': - config.model_config.provider = value; - break; - case 'm': - config.decoding_method = value; - break; - case 'f': - config.hotwords_file = value; - break; - case 's': - config.hotwords_score = atof(value); - break; - case 'h': { - fprintf(stderr, "%s\n", kUsage); - exit(0); - break; - } - default: - // do nothing as config already has valid default values - break; - } - } - const SherpaOnnxOnlineRecognizer *recognizer = SherpaOnnxCreateOnlineRecognizer(&config); const SherpaOnnxOnlineStream *stream = @@ -197,7 +99,6 @@ int32_t main(int32_t argc, char *argv[]) { const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); int32_t segment_id = 0; - const char *device_name = argv[context.index]; sherpa_onnx::Alsa alsa(device_name); fprintf(stderr, "Use recording device: %s\n", device_name); fprintf(stderr, diff --git a/c-api-examples/decode-file-c-api.c b/c-api-examples/decode-file-c-api.c index 8b74bbff1..a33221e6d 100644 --- a/c-api-examples/decode-file-c-api.c +++ b/c-api-examples/decode-file-c-api.c @@ -4,115 +4,53 @@ // This file shows how to use sherpa-onnx C API // to decode a file. +// +// clang-format off +/* +Usage + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms.tar.bz2 +tar xf sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms.tar.bz2 +rm sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms.tar.bz2 + +./decode-file-c-api + + */ +// clang-format on #include #include #include -#include "cargs.h" #include "sherpa-onnx/c-api/c-api.h" -static struct cag_option options[] = { - {.identifier = 'h', - .access_letters = "h", - .access_name = "help", - .description = "Show help"}, - {.identifier = 't', - .access_letters = NULL, - .access_name = "tokens", - .value_name = "tokens", - .description = "Tokens file"}, - {.identifier = 'e', - .access_letters = NULL, - .access_name = "encoder", - .value_name = "encoder", - .description = "Encoder ONNX file"}, - {.identifier = 'd', - .access_letters = NULL, - .access_name = "decoder", - .value_name = "decoder", - .description = "Decoder ONNX file"}, - {.identifier = 'j', - .access_letters = NULL, - .access_name = "joiner", - .value_name = "joiner", - .description = "Joiner ONNX file"}, - {.identifier = 'n', - .access_letters = NULL, - .access_name = "num-threads", - .value_name = "num-threads", - .description = "Number of threads"}, - {.identifier = 'p', - .access_letters = NULL, - .access_name = "provider", - .value_name = "provider", - .description = "Provider: cpu (default), cuda, coreml"}, - {.identifier = 'm', - .access_letters = NULL, - .access_name = "decoding-method", - .value_name = "decoding-method", - .description = - "Decoding method: greedy_search (default), modified_beam_search"}, - {.identifier = 'f', - .access_letters = NULL, - .access_name = "hotwords-file", - .value_name = "hotwords-file", - .description = "The file containing hotwords, one words/phrases per line, " - "and for each phrase the bpe/cjkchar are separated by a " - "space. For example: ▁HE LL O ▁WORLD, 你 好 世 界"}, - {.identifier = 's', - .access_letters = NULL, - .access_name = "hotwords-score", - .value_name = "hotwords-score", - .description = "The bonus score for each token in hotwords. Used only " - "when decoding_method is modified_beam_search"}, - {.identifier = 'u', - .access_letters = NULL, - .access_name = "modeling-unit", - .value_name = "modeling-unit", - .description = "Modeling unit of the hotwords: cjkchar (default), bpe, " - "cjkchar+bpe"}, - {.identifier = 'b', - .access_letters = NULL, - .access_name = "bpe-vocab", - .value_name = "bpe-vocab", - .description = "The bpe.vocab file. Used only when modeling-unit is bpe " - "or cjkchar+bpe"}, -}; - -const char *kUsage = - "\n" - "Usage:\n " - " ./bin/decode-file-c-api \\\n" - " --tokens=/path/to/tokens.txt \\\n" - " --encoder=/path/to/encoder.onnx \\\n" - " --decoder=/path/to/decoder.onnx \\\n" - " --joiner=/path/to/joiner.onnx \\\n" - " --provider=cpu \\\n" - " /path/to/foo.wav\n" - "\n\n" - "Default num_threads is 1.\n" - "Valid decoding_method: greedy_search (default), modified_beam_search\n\n" - "Valid provider: cpu (default), cuda, coreml\n\n" - "Please refer to \n" - "https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/" - "index.html\n" - "for a list of pre-trained models to download.\n" - "\n" - "Note that this file supports only streaming transducer models.\n"; - -int32_t main(int32_t argc, char *argv[]) { - if (argc < 6) { - fprintf(stderr, "%s\n", kUsage); - exit(0); - } +int32_t main() { + const char* wav_filename = + "./sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms/" + "test_wavs/0.wav"; + const char* tokens = + "./sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms/" + "tokens.txt"; + const char* encoder = + "./sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms/" + "encoder.onnx"; + const char* decoder = + "./sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms/" + "decoder.onnx"; + const char* joiner = + "./sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-80ms/" + "joiner.onnx"; SherpaOnnxOnlineRecognizerConfig config; memset(&config, 0, sizeof(config)); config.model_config.debug = 0; - config.model_config.num_threads = 1; + config.model_config.num_threads = 2; config.model_config.provider = "cpu"; + config.model_config.tokens = tokens; + config.model_config.transducer.encoder = encoder; + config.model_config.transducer.decoder = decoder; + config.model_config.transducer.joiner = joiner; config.decoding_method = "greedy_search"; @@ -126,75 +64,19 @@ int32_t main(int32_t argc, char *argv[]) { config.rule2_min_trailing_silence = 1.2; config.rule3_min_utterance_length = 300; - cag_option_context context; - char identifier; - const char *value; - - cag_option_prepare(&context, options, CAG_ARRAY_SIZE(options), argc, argv); - - while (cag_option_fetch(&context)) { - identifier = cag_option_get(&context); - value = cag_option_get_value(&context); - switch (identifier) { - case 't': - config.model_config.tokens = value; - break; - case 'e': - config.model_config.transducer.encoder = value; - break; - case 'd': - config.model_config.transducer.decoder = value; - break; - case 'j': - config.model_config.transducer.joiner = value; - break; - case 'n': - config.model_config.num_threads = atoi(value); - break; - case 'p': - config.model_config.provider = value; - break; - case 'm': - config.decoding_method = value; - break; - case 'f': - config.hotwords_file = value; - break; - case 's': - config.hotwords_score = atof(value); - break; - case 'u': - config.model_config.modeling_unit = value; - break; - case 'b': - config.model_config.bpe_vocab = value; - break; - case 'h': { - fprintf(stderr, "%s\n", kUsage); - exit(0); - break; - } - default: - // do nothing as config already has valid default values - break; - } - } - - const SherpaOnnxOnlineRecognizer *recognizer = + const SherpaOnnxOnlineRecognizer* recognizer = SherpaOnnxCreateOnlineRecognizer(&config); - const SherpaOnnxOnlineStream *stream = + const SherpaOnnxOnlineStream* stream = SherpaOnnxCreateOnlineStream(recognizer); - const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50); + const SherpaOnnxDisplay* display = SherpaOnnxCreateDisplay(50); int32_t segment_id = 0; - const char *wav_filename = argv[context.index]; - const SherpaOnnxWave *wave = SherpaOnnxReadWave(wav_filename); + const SherpaOnnxWave* wave = SherpaOnnxReadWave(wav_filename); if (wave == NULL) { fprintf(stderr, "Failed to read %s\n", wav_filename); return -1; } - // simulate streaming #define N 3200 // 0.2 s. Sample rate is fixed to 16 kHz @@ -215,7 +97,7 @@ int32_t main(int32_t argc, char *argv[]) { SherpaOnnxDecodeOnlineStream(recognizer, stream); } - const SherpaOnnxOnlineRecognizerResult *r = + const SherpaOnnxOnlineRecognizerResult* r = SherpaOnnxGetOnlineStreamResult(recognizer, stream); if (strlen(r->text)) { @@ -244,7 +126,7 @@ int32_t main(int32_t argc, char *argv[]) { SherpaOnnxDecodeOnlineStream(recognizer, stream); } - const SherpaOnnxOnlineRecognizerResult *r = + const SherpaOnnxOnlineRecognizerResult* r = SherpaOnnxGetOnlineStreamResult(recognizer, stream); if (strlen(r->text)) { diff --git a/c-api-examples/kitten-tts-en-c-api.c b/c-api-examples/kitten-tts-en-c-api.c index c15e75761..23d48f0c6 100644 --- a/c-api-examples/kitten-tts-en-c-api.c +++ b/c-api-examples/kitten-tts-en-c-api.c @@ -13,6 +13,8 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-n tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./kitten-tts-en-c-api */ @@ -38,7 +40,7 @@ int32_t main(int32_t argc, char *argv[]) { config.model.kitten.model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; config.model.kitten.voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; config.model.kitten.tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; - config.model.kitten.data_dir = "./kitten-nano-en-v0_1-fp16/espeak-ng-data"; + config.model.kitten.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 2; diff --git a/c-api-examples/kokoro-tts-en-c-api.c b/c-api-examples/kokoro-tts-en-c-api.c index a626c082f..76d4bb0d2 100644 --- a/c-api-examples/kokoro-tts-en-c-api.c +++ b/c-api-examples/kokoro-tts-en-c-api.c @@ -14,6 +14,8 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-e tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./kokoro-tts-en-c-api */ @@ -39,7 +41,7 @@ int32_t main(int32_t argc, char *argv[]) { config.model.kokoro.model = "./kokoro-en-v0_19/model.onnx"; config.model.kokoro.voices = "./kokoro-en-v0_19/voices.bin"; config.model.kokoro.tokens = "./kokoro-en-v0_19/tokens.txt"; - config.model.kokoro.data_dir = "./kokoro-en-v0_19/espeak-ng-data"; + config.model.kokoro.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 2; diff --git a/c-api-examples/kokoro-tts-zh-en-c-api.c b/c-api-examples/kokoro-tts-zh-en-c-api.c index c2b1a2260..6f1932dfa 100644 --- a/c-api-examples/kokoro-tts-zh-en-c-api.c +++ b/c-api-examples/kokoro-tts-zh-en-c-api.c @@ -39,8 +39,6 @@ int32_t main(int32_t argc, char *argv[]) { config.model.kokoro.model = "./kokoro-multi-lang-v1_0/model.onnx"; config.model.kokoro.voices = "./kokoro-multi-lang-v1_0/voices.bin"; config.model.kokoro.tokens = "./kokoro-multi-lang-v1_0/tokens.txt"; - config.model.kokoro.data_dir = "./kokoro-multi-lang-v1_0/espeak-ng-data"; - config.model.kokoro.dict_dir = "./kokoro-multi-lang-v1_0/dict"; config.model.kokoro.lexicon = "./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/" "lexicon-zh.txt"; diff --git a/c-api-examples/matcha-tts-en-c-api.c b/c-api-examples/matcha-tts-en-c-api.c index c79ec5b12..1d2214625 100644 --- a/c-api-examples/matcha-tts-en-c-api.c +++ b/c-api-examples/matcha-tts-en-c-api.c @@ -15,6 +15,8 @@ rm matcha-icefall-en_US-ljspeech.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./matcha-tts-en-c-api */ @@ -44,8 +46,7 @@ int32_t main(int32_t argc, char *argv[]) { config.model.matcha.tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt"; - config.model.matcha.data_dir = - "./matcha-icefall-en_US-ljspeech/espeak-ng-data"; + config.model.matcha.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 1; diff --git a/c-api-examples/matcha-tts-zh-c-api.c b/c-api-examples/matcha-tts-zh-c-api.c index adbc0d0aa..af308982a 100644 --- a/c-api-examples/matcha-tts-zh-c-api.c +++ b/c-api-examples/matcha-tts-zh-c-api.c @@ -42,7 +42,6 @@ int32_t main(int32_t argc, char *argv[]) { config.model.matcha.vocoder = "./vocos-22khz-univ.onnx"; config.model.matcha.lexicon = "./matcha-icefall-zh-baker/lexicon.txt"; config.model.matcha.tokens = "./matcha-icefall-zh-baker/tokens.txt"; - config.model.matcha.dict_dir = "./matcha-icefall-zh-baker/dict"; config.model.num_threads = 1; // If you don't want to see debug messages, please set it to 0 diff --git a/c-api-examples/offline-tts-c-api.c b/c-api-examples/offline-tts-c-api.c index 62d2013d9..69179381f 100644 --- a/c-api-examples/offline-tts-c-api.c +++ b/c-api-examples/offline-tts-c-api.c @@ -3,241 +3,65 @@ // Copyright (c) 2023 Xiaomi Corporation // This file shows how to use sherpa-onnx C API -// to convert text to speech using an offline model. +// for offline text-to-speech. +// +// clang-format off +/* +Usage + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-ljs.tar.bz2 +tar xf vits-ljs.tar.bz2 +rm vits-ljs.tar.bz2 + +./offline-tts-c-api + + */ +// clang-format on #include #include #include -#include "cargs.h" #include "sherpa-onnx/c-api/c-api.h" -static struct cag_option options[] = { - {.identifier = 'h', - .access_letters = "h", - .access_name = "help", - .description = "Show help"}, - {.access_name = "vits-model", - .value_name = "/path/to/xxx.onnx", - .identifier = '0', - .description = "Path to VITS model"}, - {.access_name = "vits-lexicon", - .value_name = "/path/to/lexicon.txt", - .identifier = '1', - .description = "Path to lexicon.txt for VITS models"}, - {.access_name = "vits-tokens", - .value_name = "/path/to/tokens.txt", - .identifier = '2', - .description = "Path to tokens.txt for VITS models"}, - {.access_name = "vits-noise-scale", - .value_name = "0.667", - .identifier = '3', - .description = "noise_scale for VITS models"}, - {.access_name = "vits-noise-scale-w", - .value_name = "0.8", - .identifier = '4', - .description = "noise_scale_w for VITS models"}, - {.access_name = "vits-length-scale", - .value_name = "1.0", - .identifier = '5', - .description = - "length_scale for VITS models. Default to 1. You can tune it " - "to change the speech speed. small -> faster; large -> slower. "}, - {.access_name = "num-threads", - .value_name = "1", - .identifier = '6', - .description = "Number of threads"}, - {.access_name = "provider", - .value_name = "cpu", - .identifier = '7', - .description = "Provider: cpu (default), cuda, coreml"}, - {.access_name = "debug", - .value_name = "0", - .identifier = '8', - .description = "1 to show debug messages while loading the model"}, - {.access_name = "sid", - .value_name = "0", - .identifier = '9', - .description = "Speaker ID. Default to 0. Note it is not used for " - "single-speaker models."}, - {.access_name = "output-filename", - .value_name = "./generated.wav", - .identifier = 'a', - .description = - "Filename to save the generated audio. Default to ./generated.wav"}, - - {.access_name = "tts-rule-fsts", - .value_name = "/path/to/rule.fst", - .identifier = 'b', - .description = "It not empty, it contains a list of rule FST filenames." - "Multiple filenames are separated by a comma and they are " - "applied from left to right. An example value: " - "rule1.fst,rule2,fst,rule3.fst"}, - - {.access_name = "max-num-sentences", - .value_name = "2", - .identifier = 'c', - .description = "Maximum number of sentences that we process at a time. " - "This is to avoid OOM for very long input text. " - "If you set it to -1, then we process all sentences in a " - "single batch."}, - - {.access_name = "vits-data-dir", - .value_name = "/path/to/espeak-ng-data", - .identifier = 'd', - .description = - "Path to espeak-ng-data. If it is given, --vits-lexicon is ignored"}, - -}; - -static void ShowUsage() { - const char *kUsageMessage = - "Offline text-to-speech with sherpa-onnx C API" - "\n" - "./offline-tts-c-api \\\n" - " --vits-model=/path/to/model.onnx \\\n" - " --vits-lexicon=/path/to/lexicon.txt \\\n" - " --vits-tokens=/path/to/tokens.txt \\\n" - " --sid=0 \\\n" - " --output-filename=./generated.wav \\\n" - " 'some text within single quotes on linux/macos or use double quotes on " - "windows'\n" - "\n" - "It will generate a file ./generated.wav as specified by " - "--output-filename.\n" - "\n" - "You can download a test model from\n" - "https://huggingface.co/csukuangfj/vits-ljs\n" - "\n" - "For instance, you can use:\n" - "wget " - "https://huggingface.co/csukuangfj/vits-ljs/resolve/main/vits-ljs.onnx\n" - "wget " - "https://huggingface.co/csukuangfj/vits-ljs/resolve/main/lexicon.txt\n" - "wget " - "https://huggingface.co/csukuangfj/vits-ljs/resolve/main/tokens.txt\n" - "\n" - "./offline-tts-c-api \\\n" - " --vits-model=./vits-ljs.onnx \\\n" - " --vits-lexicon=./lexicon.txt \\\n" - " --vits-tokens=./tokens.txt \\\n" - " --sid=0 \\\n" - " --output-filename=./generated.wav \\\n" - " 'liliana, the most beautiful and lovely assistant of our team!'\n" - "\n" - "Please see\n" - "https://k2-fsa.github.io/sherpa/onnx/tts/index.html\n" - "or details.\n\n"; - - fprintf(stderr, "%s", kUsageMessage); - cag_option_print(options, CAG_ARRAY_SIZE(options), stderr); - exit(0); -} - -int32_t main(int32_t argc, char *argv[]) { - cag_option_context context; - char identifier; - const char *value; - - cag_option_prepare(&context, options, CAG_ARRAY_SIZE(options), argc, argv); +int32_t main() { + const char *model = "./vits-ljs/vits-ljs.onnx"; + const char *lexicon = "./vits-ljs/lexicon.txt"; + const char *tokens = "./vits-ljs/tokens.txt"; + const char *filename = "./generated-offline-tts-c.wav"; + const char *text = + "Today as always, men fall into two groups: slaves and free men. " + "Whoever does not have two-thirds of his day for himself, is a slave, " + "whatever he may be: a statesman, a businessman, an official, or a " + "scholar."; SherpaOnnxOfflineTtsConfig config; memset(&config, 0, sizeof(config)); - - int32_t sid = 0; - const char *filename = strdup("./generated.wav"); - const char *text; - - while (cag_option_fetch(&context)) { - identifier = cag_option_get(&context); - value = cag_option_get_value(&context); - switch (identifier) { - case '0': - config.model.vits.model = value; - break; - case '1': - config.model.vits.lexicon = value; - break; - case '2': - config.model.vits.tokens = value; - break; - case '3': - config.model.vits.noise_scale = atof(value); - break; - case '4': - config.model.vits.noise_scale_w = atof(value); - break; - case '5': - config.model.vits.length_scale = atof(value); - break; - case '6': - config.model.num_threads = atoi(value); - break; - case '7': - config.model.provider = value; - break; - case '8': - config.model.debug = atoi(value); - break; - case '9': - sid = atoi(value); - break; - case 'a': - free((void *)filename); - filename = strdup(value); - break; - case 'b': - config.rule_fsts = value; - break; - case 'c': - config.max_num_sentences = atoi(value); - break; - case 'd': - config.model.vits.data_dir = value; - break; - case '?': - fprintf(stderr, "Unknown option\n"); - // fall through - case 'h': - // fall through - default: - ShowUsage(); - } - } - fprintf(stderr, "here\n"); - - if (!config.model.vits.model) { - fprintf(stderr, "Please provide --vits-model\n"); - ShowUsage(); - } - - if (!config.model.vits.tokens) { - fprintf(stderr, "Please provide --vits-tokens\n"); - ShowUsage(); - } - - if (!config.model.vits.data_dir && !config.model.vits.lexicon) { - fprintf(stderr, "Please provide --vits-data-dir or --vits-lexicon\n"); - ShowUsage(); - } - - // the last arg is the text - text = argv[argc - 1]; - if (text[0] == '-') { - fprintf(stderr, "\n***Please input your text!***\n\n"); - fprintf(stderr, "\n---------------Usage---------------\n\n"); - ShowUsage(); - } + config.model.vits.model = model; + config.model.vits.lexicon = lexicon; + config.model.vits.tokens = tokens; + config.model.num_threads = 1; + config.model.provider = "cpu"; + config.model.debug = 0; const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); + if (tts == NULL) { + fprintf(stderr, "Please check your config!\n"); + return -1; + } SherpaOnnxGenerationConfig cfg = {0}; cfg.silence_scale = 0.2f; - cfg.sid = sid; + cfg.sid = 0; cfg.speed = 1.0f; const SherpaOnnxGeneratedAudio *audio = SherpaOnnxOfflineTtsGenerateWithConfig(tts, text, &cfg, NULL, NULL); + if (audio == NULL) { + fprintf(stderr, "Failed to generate audio!\n"); + SherpaOnnxDestroyOfflineTts(tts); + return -1; + } SherpaOnnxWriteWave(audio->samples, audio->n, audio->sample_rate, filename); @@ -245,10 +69,8 @@ int32_t main(int32_t argc, char *argv[]) { SherpaOnnxDestroyOfflineTts(tts); fprintf(stderr, "Input text is: %s\n", text); - fprintf(stderr, "Speaker ID is: %d\n", sid); + fprintf(stderr, "Speaker ID is: %d\n", cfg.sid); fprintf(stderr, "Saved to: %s\n", filename); - free((void *)filename); - return 0; } diff --git a/c-api-examples/zipvoice-tts-zh-en-c-api.c b/c-api-examples/zipvoice-tts-zh-en-c-api.c index fb31f730b..928ab8a81 100644 --- a/c-api-examples/zipvoice-tts-zh-en-c-api.c +++ b/c-api-examples/zipvoice-tts-zh-en-c-api.c @@ -15,6 +15,8 @@ rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./zipvoice-tts-zh-en-c-api */ // clang-format on @@ -38,10 +40,9 @@ int32_t main(int32_t argc, char *argv[]) { "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx"; config.model.zipvoice.decoder = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx"; - config.model.zipvoice.data_dir = - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data"; config.model.zipvoice.lexicon = - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt"; + "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt," + "./lexicon-en-us.txt"; config.model.zipvoice.tokens = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt"; config.model.zipvoice.vocoder = "./vocos_24khz.onnx"; diff --git a/cmake/cargs.cmake b/cmake/cargs.cmake deleted file mode 100644 index d7c605508..000000000 --- a/cmake/cargs.cmake +++ /dev/null @@ -1,50 +0,0 @@ -function(download_cargs) - include(FetchContent) - - set(cargs_URL "https://github.com/likle/cargs/archive/refs/tags/v1.0.3.tar.gz") - set(cargs_URL2 "https://hf-mirror.com/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/cargs-1.0.3.tar.gz") - set(cargs_HASH "SHA256=ddba25bd35e9c6c75bc706c126001b8ce8e084d40ef37050e6aa6963e836eb8b") - - # If you don't have access to the Internet, - # please pre-download cargs - set(possible_file_locations - $ENV{HOME}/Downloads/cargs-1.0.3.tar.gz - ${CMAKE_SOURCE_DIR}/cargs-1.0.3.tar.gz - ${CMAKE_BINARY_DIR}/cargs-1.0.3.tar.gz - /tmp/cargs-1.0.3.tar.gz - /star-fj/fangjun/download/github/cargs-1.0.3.tar.gz - ) - - foreach(f IN LISTS possible_file_locations) - if(EXISTS ${f}) - set(cargs_URL "${f}") - file(TO_CMAKE_PATH "${cargs_URL}" cargs_URL) - message(STATUS "Found local downloaded cargs: ${cargs_URL}") - set(cargs_URL2) - break() - endif() - endforeach() - - FetchContent_Declare(cargs - URL - ${cargs_URL} - ${cargs_URL2} - URL_HASH - ${cargs_HASH} - ) - - FetchContent_GetProperties(cargs) - if(NOT cargs_POPULATED) - message(STATUS "Downloading cargs ${cargs_URL}") - FetchContent_Populate(cargs) - endif() - message(STATUS "cargs is downloaded to ${cargs_SOURCE_DIR}") - add_subdirectory(${cargs_SOURCE_DIR} ${cargs_BINARY_DIR} EXCLUDE_FROM_ALL) - - install(TARGETS cargs DESTINATION lib) - install(FILES ${cargs_SOURCE_DIR}/include/cargs.h - DESTINATION include - ) -endfunction() - -download_cargs() diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index f56c70e3b..ab8ec7c2a 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -154,10 +154,6 @@ class BuildExtension(build_ext): if not need_split_package(): extra_cmake_args += f" -DCMAKE_INSTALL_PREFIX={install_dir} " extra_cmake_args += " -DBUILD_SHARED_LIBS=ON " - extra_cmake_args += " -DBUILD_PIPER_PHONMIZE_EXE=OFF " - extra_cmake_args += " -DBUILD_PIPER_PHONMIZE_TESTS=OFF " - extra_cmake_args += " -DBUILD_ESPEAK_NG_EXE=OFF " - extra_cmake_args += " -DBUILD_ESPEAK_NG_TESTS=OFF " if not need_split_package(): extra_cmake_args += " -DSHERPA_ONNX_ENABLE_C_API=ON " diff --git a/cmake/download-all-deps.py b/cmake/download-all-deps.py index efca9c4ca..ca386a536 100755 --- a/cmake/download-all-deps.py +++ b/cmake/download-all-deps.py @@ -175,16 +175,13 @@ def discover_common_deps(cmake_dir): # These cmake files contain platform-independent deps dep_files = [ "asio.cmake", - "cargs.cmake", "eigen.cmake", - "espeak-ng-for-piper.cmake", "googletest.cmake", "hclust-cpp.cmake", "json.cmake", "kaldi-decoder.cmake", "kaldi-native-fbank.cmake", "openfst.cmake", - "piper-phonemize.cmake", "portaudio.cmake", "pybind11.cmake", "simple-sentencepiece.cmake", @@ -414,7 +411,7 @@ def build_canonical_filename_lookup(cmake_dir): """Build a mapping from URL basename to canonical local filename. Scans cmake files for $ENV{HOME}/Downloads/ entries, which - define the expected local filenames (e.g., "cargs-1.0.3.tar.gz" even + define the expected local filenames (e.g., "asio-1.28.0.tar.gz" even when the URL basename is just "v1.0.3.tar.gz"). """ lookup = {} @@ -426,7 +423,7 @@ def build_canonical_filename_lookup(cmake_dir): for m in dl_pattern.finditer(text): local_name = m.group(1) # Also index by stripped version (without project prefix) - # so URLs like "v1.0.3.tar.gz" can map to "cargs-1.0.3.tar.gz" + # so URLs like "v1.28.0.tar.gz" can map to "asio-1.28.0.tar.gz" lookup[local_name] = local_name return lookup diff --git a/cmake/espeak-ng-for-piper.cmake b/cmake/espeak-ng-for-piper.cmake deleted file mode 100644 index 6f6dbd0b2..000000000 --- a/cmake/espeak-ng-for-piper.cmake +++ /dev/null @@ -1,131 +0,0 @@ -function(download_espeak_ng_for_piper) - include(FetchContent) - - set(espeak_ng_URL "https://github.com/csukuangfj/espeak-ng/archive/ed530aa113046142eb5115cf2fc9157854d0ffe1.zip") - set(espeak_ng_HASH "SHA256=e4e262cbe34f7fe21f91f1ba3397f2728e1f30eafbae7853f2b753a9ed13f0dd") - - set(BUILD_ESPEAK_NG_TESTS OFF CACHE BOOL "" FORCE) - set(USE_ASYNC OFF CACHE BOOL "" FORCE) - set(USE_MBROLA OFF CACHE BOOL "" FORCE) - set(USE_LIBSONIC OFF CACHE BOOL "" FORCE) - set(USE_LIBPCAUDIO OFF CACHE BOOL "" FORCE) - set(USE_KLATT OFF CACHE BOOL "" FORCE) - set(USE_SPEECHPLAYER OFF CACHE BOOL "" FORCE) - set(EXTRA_cmn ON CACHE BOOL "" FORCE) - set(EXTRA_ru ON CACHE BOOL "" FORCE) - if (NOT SHERPA_ONNX_ENABLE_EPSEAK_NG_EXE) - set(BUILD_ESPEAK_NG_EXE OFF CACHE BOOL "" FORCE) - endif() - - # If you don't have access to the Internet, - # please pre-download kaldi-decoder - set(possible_file_locations - $ENV{HOME}/Downloads/espeak-ng-ed530aa113046142eb5115cf2fc9157854d0ffe1.zip - ${CMAKE_SOURCE_DIR}/espeak-ng-ed530aa113046142eb5115cf2fc9157854d0ffe1.zip - ${CMAKE_BINARY_DIR}/espeak-ng-ed530aa113046142eb5115cf2fc9157854d0ffe1.zip - /tmp/espeak-ng-ed530aa113046142eb5115cf2fc9157854d0ffe1.zip - /star-fj/fangjun/download/github/espeak-ng-ed530aa113046142eb5115cf2fc9157854d0ffe1.zip - ) - - foreach(f IN LISTS possible_file_locations) - if(EXISTS ${f}) - set(espeak_ng_URL "${f}") - file(TO_CMAKE_PATH "${espeak_ng_URL}" espeak_ng_URL) - message(STATUS "Found local downloaded espeak-ng: ${espeak_ng_URL}") - break() - endif() - endforeach() - - FetchContent_Declare(espeak_ng - URL - ${espeak_ng_URL} - URL_HASH ${espeak_ng_HASH} - ) - - FetchContent_GetProperties(espeak_ng) - if(NOT espeak_ng_POPULATED) - message(STATUS "Downloading espeak-ng from ${espeak_ng_URL}") - FetchContent_Populate(espeak_ng) - endif() - message(STATUS "espeak-ng is downloaded to ${espeak_ng_SOURCE_DIR}") - message(STATUS "espeak-ng binary dir is ${espeak_ng_BINARY_DIR}") - - if(BUILD_SHARED_LIBS) - set(_build_shared_libs_bak ${BUILD_SHARED_LIBS}) - set(BUILD_SHARED_LIBS OFF) - endif() - - add_subdirectory(${espeak_ng_SOURCE_DIR} ${espeak_ng_BINARY_DIR}) - - if(_build_shared_libs_bak) - set_target_properties(espeak-ng - PROPERTIES - POSITION_INDEPENDENT_CODE ON - C_VISIBILITY_PRESET hidden - CXX_VISIBILITY_PRESET hidden - ) - set(BUILD_SHARED_LIBS ON) - endif() - - set(espeak_ng_SOURCE_DIR ${espeak_ng_SOURCE_DIR} PARENT_SCOPE) - - if(WIN32 AND MSVC) - target_compile_options(ucd PUBLIC - /wd4309 - ) - - target_compile_options(espeak-ng PUBLIC - /wd4005 - /wd4018 - /wd4067 - /wd4068 - /wd4090 - /wd4101 - /wd4244 - /wd4267 - /wd4996 - ) - - if(TARGET espeak-ng-bin) - target_compile_options(espeak-ng-bin PRIVATE - /wd4244 - /wd4024 - /wd4047 - /wd4067 - /wd4267 - /wd4996 - ) - endif() - endif() - - if(UNIX AND NOT APPLE) - target_compile_options(espeak-ng PRIVATE - -Wno-unused-result - -Wno-format-overflow - -Wno-format-truncation - -Wno-uninitialized - -Wno-format - ) - - if(TARGET espeak-ng-bin) - target_compile_options(espeak-ng-bin PRIVATE - -Wno-unused-result - ) - endif() - endif() - - target_include_directories(espeak-ng - INTERFACE - ${espeak_ng_SOURCE_DIR}/src/include - ${espeak_ng_SOURCE_DIR}/src/ucd-tools/src/include - ) - - if(NOT BUILD_SHARED_LIBS) - install(TARGETS - espeak-ng - ucd - DESTINATION lib) - endif() -endfunction() - -download_espeak_ng_for_piper() diff --git a/cmake/piper-phonemize.cmake b/cmake/piper-phonemize.cmake deleted file mode 100644 index 129b3a1d6..000000000 --- a/cmake/piper-phonemize.cmake +++ /dev/null @@ -1,75 +0,0 @@ -function(download_piper_phonemize) - include(FetchContent) - - set(piper_phonemize_URL "https://github.com/csukuangfj/piper-phonemize/archive/f3ff95afc03640bc1399e113e83361192a2fafb4.zip") - set(piper_phonemize_HASH "SHA256=d9cca4e2bdc7d6dd8dffb96a4668283dbd3f77a9c194a3e530c1e8eba9406a5d") - - # If you don't have access to the Internet, - # please pre-download kaldi-decoder - set(possible_file_locations - $ENV{HOME}/Downloads/piper-phonemize-f3ff95afc03640bc1399e113e83361192a2fafb4.zip - ${CMAKE_SOURCE_DIR}/piper-phonemize-f3ff95afc03640bc1399e113e83361192a2fafb4.zip - ${CMAKE_BINARY_DIR}/piper-phonemize-f3ff95afc03640bc1399e113e83361192a2fafb4.zip - /tmp/piper-phonemize-f3ff95afc03640bc1399e113e83361192a2fafb4.zip - /star-fj/fangjun/download/github/piper-phonemize-f3ff95afc03640bc1399e113e83361192a2fafb4.zip - ) - - foreach(f IN LISTS possible_file_locations) - if(EXISTS ${f}) - set(piper_phonemize_URL "${f}") - file(TO_CMAKE_PATH "${piper_phonemize_URL}" piper_phonemize_URL) - message(STATUS "Found local downloaded espeak-ng: ${piper_phonemize_URL}") - break() - endif() - endforeach() - - FetchContent_Declare(piper_phonemize - URL - ${piper_phonemize_URL} - URL_HASH ${piper_phonemize_HASH} - ) - - FetchContent_GetProperties(piper_phonemize) - if(NOT piper_phonemize_POPULATED) - message(STATUS "Downloading piper-phonemize from ${piper_phonemize_URL}") - FetchContent_Populate(piper_phonemize) - endif() - message(STATUS "piper-phonemize is downloaded to ${piper_phonemize_SOURCE_DIR}") - message(STATUS "piper-phonemize binary dir is ${piper_phonemize_BINARY_DIR}") - - if(BUILD_SHARED_LIBS) - set(_build_shared_libs_bak ${BUILD_SHARED_LIBS}) - set(BUILD_SHARED_LIBS OFF) - endif() - - add_subdirectory(${piper_phonemize_SOURCE_DIR} ${piper_phonemize_BINARY_DIR} EXCLUDE_FROM_ALL) - - if(_build_shared_libs_bak) - set_target_properties(piper_phonemize - PROPERTIES - POSITION_INDEPENDENT_CODE ON - C_VISIBILITY_PRESET hidden - CXX_VISIBILITY_PRESET hidden - ) - set(BUILD_SHARED_LIBS ON) - endif() - - if(WIN32 AND MSVC) - target_compile_options(piper_phonemize PUBLIC - /wd4309 - ) - endif() - - target_include_directories(piper_phonemize - INTERFACE - ${piper_phonemize_SOURCE_DIR}/src/include - ) - - if(NOT BUILD_SHARED_LIBS) - install(TARGETS - piper_phonemize - DESTINATION lib) - endif() -endfunction() - -download_piper_phonemize() diff --git a/cmake/sherpa-onnx-shared.pc.in b/cmake/sherpa-onnx-shared.pc.in index 724a5393e..3690d13f5 100644 --- a/cmake/sherpa-onnx-shared.pc.in +++ b/cmake/sherpa-onnx-shared.pc.in @@ -19,7 +19,4 @@ URL: https://github.com/k2-fsa/sherpa-onnx Version: @SHERPA_ONNX_VERSION@ Cflags: -I"${includedir}" -# Note: -lcargs is required only for the following file -# https://github.com/k2-fsa/sherpa-onnx/blob/master/c-api-examples/decode-file-c-api.c -# We add it here so that users don't need to specify -lcargs when compiling decode-file-c-api.c -Libs: -L"${libdir}" -lsherpa-onnx-cxx-api -lsherpa-onnx-c-api -lonnxruntime -Wl,-rpath,${libdir} @SHERPA_ONNX_PKG_WITH_CARGS@ @SHERPA_ONNX_PKG_CONFIG_EXTRA_LIBS@ +Libs: -L"${libdir}" -lsherpa-onnx-cxx-api -lsherpa-onnx-c-api -lonnxruntime -Wl,-rpath,${libdir} @SHERPA_ONNX_PKG_CONFIG_EXTRA_LIBS@ diff --git a/cmake/sherpa-onnx-static-no-tts.pc.in b/cmake/sherpa-onnx-static-no-tts.pc.in index 4fab62cdd..e478ba71d 100644 --- a/cmake/sherpa-onnx-static-no-tts.pc.in +++ b/cmake/sherpa-onnx-static-no-tts.pc.in @@ -19,7 +19,4 @@ URL: https://github.com/k2-fsa/sherpa-onnx Version: @SHERPA_ONNX_VERSION@ Cflags: -I"${includedir}" -# Note: -lcargs is required only for the following file -# https://github.com/k2-fsa/sherpa-onnx/blob/master/c-api-examples/decode-file-c-api.c -# We add it here so that users don't need to specify -lcargs when compiling decode-file-c-api.c -Libs: -L"${libdir}" -lsherpa-onnx-c-api -lsherpa-onnx-core -lkaldi-decoder-core -lsherpa-onnx-kaldifst-core -lsherpa-onnx-fstfar -lsherpa-onnx-fst -lkaldi-native-fbank-core -lkissfft-float -lonnxruntime -lssentencepiece_core -Wl,-rpath,${libdir} @SHERPA_ONNX_PKG_WITH_CARGS@ @SHERPA_ONNX_PKG_CONFIG_EXTRA_LIBS@ +Libs: -L"${libdir}" -lsherpa-onnx-c-api -lsherpa-onnx-core -lkaldi-decoder-core -lsherpa-onnx-kaldifst-core -lsherpa-onnx-fstfar -lsherpa-onnx-fst -lkaldi-native-fbank-core -lkissfft-float -lonnxruntime -lssentencepiece_core -Wl,-rpath,${libdir} @SHERPA_ONNX_PKG_CONFIG_EXTRA_LIBS@ diff --git a/cmake/sherpa-onnx-static.pc.in b/cmake/sherpa-onnx-static.pc.in index 42beb084a..1416f8082 100644 --- a/cmake/sherpa-onnx-static.pc.in +++ b/cmake/sherpa-onnx-static.pc.in @@ -19,7 +19,4 @@ URL: https://github.com/k2-fsa/sherpa-onnx Version: @SHERPA_ONNX_VERSION@ Cflags: -I"${includedir}" -# Note: -lcargs is required only for the following file -# https://github.com/k2-fsa/sherpa-onnx/blob/master/c-api-examples/decode-file-c-api.c -# We add it here so that users don't need to specify -lcargs when compiling decode-file-c-api.c -Libs: -L"${libdir}" -lsherpa-onnx-c-api -lsherpa-onnx-core -lkaldi-decoder-core -lsherpa-onnx-kaldifst-core -lsherpa-onnx-fstfar -lsherpa-onnx-fst -lkaldi-native-fbank-core -lkissfft-float -lpiper_phonemize -lespeak-ng -lucd -lonnxruntime -lssentencepiece_core -Wl,-rpath,${libdir} @SHERPA_ONNX_PKG_WITH_CARGS@ @SHERPA_ONNX_PKG_CONFIG_EXTRA_LIBS@ +Libs: -L"${libdir}" -lsherpa-onnx-c-api -lsherpa-onnx-core -lkaldi-decoder-core -lsherpa-onnx-kaldifst-core -lsherpa-onnx-fstfar -lsherpa-onnx-fst -lkaldi-native-fbank-core -lkissfft-float -lonnxruntime -lssentencepiece_core -Wl,-rpath,${libdir} @SHERPA_ONNX_PKG_CONFIG_EXTRA_LIBS@ diff --git a/cmake/simple-sentencepiece.cmake b/cmake/simple-sentencepiece.cmake index 14a2a1790..49bdf574c 100644 --- a/cmake/simple-sentencepiece.cmake +++ b/cmake/simple-sentencepiece.cmake @@ -1,3 +1,19 @@ +# Write ${content} to ${path} only when it differs from what is on disk. +# The patches below run on every configure; a plain file(WRITE) would update +# the header's mtime each time and make/ssentencepiece (plus every dependent +# in sherpa-onnx-core) would recompile on every build. +function(write_file_if_different path content) + string(SHA256 _want "${content}") + if(EXISTS "${path}") + file(READ "${path}" _existing) + string(SHA256 _have "${_existing}") + if(_have STREQUAL _want) + return() + endif() + endif() + file(WRITE "${path}" "${content}") +endfunction() + function(download_simple_sentencepiece) include(FetchContent) @@ -47,7 +63,7 @@ function(download_simple_sentencepiece) if(SHERPA_ONNX_ENABLE_WASM) # Replace threadpool.h with a WASM-safe stub that provides a no-op ThreadPool. set(_tp_header "${simple-sentencepiece_SOURCE_DIR}/ssentencepiece/csrc/threadpool.h") - file(WRITE "${_tp_header}" [=[ + write_file_if_different("${_tp_header}" [=[ // WASM-safe ThreadPool stub (std::thread not available). #ifndef THREAD_POOL_H #define THREAD_POOL_H @@ -83,7 +99,7 @@ class ThreadPool { # 1. Replace threadpool.h with an empty stub. set(_tp_header "${simple-sentencepiece_SOURCE_DIR}/ssentencepiece/csrc/threadpool.h") - file(WRITE "${_tp_header}" [=[ + write_file_if_different("${_tp_header}" [=[ // Empty stub — threading removed for Android/Termux. #ifndef THREAD_POOL_H #define THREAD_POOL_H @@ -99,7 +115,7 @@ class ThreadPool { string(REPLACE "pool_ = std::make_unique(num_threads);\n Build(vocab_path);" "Build(vocab_path);" _sph_content "${_sph_content}") string(REPLACE "pool_ = std::make_unique(num_threads);" "" _sph_content "${_sph_content}") string(REPLACE "std::unique_ptr pool_;" "" _sph_content "${_sph_content}") - file(WRITE "${_sph}" "${_sph_content}") + write_file_if_different("${_sph}" "${_sph_content}") # 3. Patch ssentencepiece.cc: replace threaded Encode/Decode with synchronous loops. set(_spc "${simple-sentencepiece_SOURCE_DIR}/ssentencepiece/csrc/ssentencepiece.cc") @@ -162,7 +178,7 @@ class ThreadPool { return res;]==] _spc_content "${_spc_content}") - file(WRITE "${_spc}" "${_spc_content}") + write_file_if_different("${_spc}" "${_spc_content}") message(STATUS "Patched ssentencepiece for Android: all threading removed") endif() diff --git a/cxx-api-examples/kitten-tts-en-cxx-api.cc b/cxx-api-examples/kitten-tts-en-cxx-api.cc index 6c0a586bc..c41cce292 100644 --- a/cxx-api-examples/kitten-tts-en-cxx-api.cc +++ b/cxx-api-examples/kitten-tts-en-cxx-api.cc @@ -13,6 +13,8 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-n tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./kitten-tts-en-cxx-api */ @@ -39,7 +41,7 @@ int32_t main(int32_t argc, char *argv[]) { config.model.kitten.model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; config.model.kitten.voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; config.model.kitten.tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; - config.model.kitten.data_dir = "./kitten-nano-en-v0_1-fp16/espeak-ng-data"; + config.model.kitten.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 2; diff --git a/cxx-api-examples/kokoro-tts-en-cxx-api.cc b/cxx-api-examples/kokoro-tts-en-cxx-api.cc index fcdaac6fc..0be314472 100644 --- a/cxx-api-examples/kokoro-tts-en-cxx-api.cc +++ b/cxx-api-examples/kokoro-tts-en-cxx-api.cc @@ -13,6 +13,8 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-e tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./kokoro-tts-en-cxx-api */ @@ -39,7 +41,7 @@ int32_t main(int32_t argc, char *argv[]) { config.model.kokoro.model = "./kokoro-en-v0_19/model.onnx"; config.model.kokoro.voices = "./kokoro-en-v0_19/voices.bin"; config.model.kokoro.tokens = "./kokoro-en-v0_19/tokens.txt"; - config.model.kokoro.data_dir = "./kokoro-en-v0_19/espeak-ng-data"; + config.model.kokoro.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 2; diff --git a/cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc b/cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc index 9a5b38f0a..c991a22e0 100644 --- a/cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc +++ b/cxx-api-examples/kokoro-tts-zh-en-cxx-api.cc @@ -39,8 +39,6 @@ int32_t main(int32_t argc, char *argv[]) { config.model.kokoro.model = "./kokoro-multi-lang-v1_0/model.onnx"; config.model.kokoro.voices = "./kokoro-multi-lang-v1_0/voices.bin"; config.model.kokoro.tokens = "./kokoro-multi-lang-v1_0/tokens.txt"; - config.model.kokoro.data_dir = "./kokoro-multi-lang-v1_0/espeak-ng-data"; - config.model.kokoro.dict_dir = "./kokoro-multi-lang-v1_0/dict"; config.model.kokoro.lexicon = "./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/" "lexicon-zh.txt"; diff --git a/cxx-api-examples/matcha-tts-en-cxx-api.cc b/cxx-api-examples/matcha-tts-en-cxx-api.cc index 79e810a7e..89947086e 100644 --- a/cxx-api-examples/matcha-tts-en-cxx-api.cc +++ b/cxx-api-examples/matcha-tts-en-cxx-api.cc @@ -15,6 +15,8 @@ rm matcha-icefall-en_US-ljspeech.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./matcha-tts-en-cxx-api */ @@ -45,8 +47,7 @@ int32_t main(int32_t argc, char *argv[]) { config.model.matcha.tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt"; - config.model.matcha.data_dir = - "./matcha-icefall-en_US-ljspeech/espeak-ng-data"; + config.model.matcha.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 1; diff --git a/cxx-api-examples/matcha-tts-zh-cxx-api.cc b/cxx-api-examples/matcha-tts-zh-cxx-api.cc index 38b300b22..fb085bc34 100644 --- a/cxx-api-examples/matcha-tts-zh-cxx-api.cc +++ b/cxx-api-examples/matcha-tts-zh-cxx-api.cc @@ -42,7 +42,6 @@ int32_t main(int32_t argc, char *argv[]) { config.model.matcha.vocoder = "./vocos-22khz-univ.onnx"; config.model.matcha.lexicon = "./matcha-icefall-zh-baker/lexicon.txt"; config.model.matcha.tokens = "./matcha-icefall-zh-baker/tokens.txt"; - config.model.matcha.dict_dir = "./matcha-icefall-zh-baker/dict"; config.model.num_threads = 1; // If you don't want to see debug messages, please set it to 0 diff --git a/cxx-api-examples/offline-tts-piper-cxx-api.cc b/cxx-api-examples/offline-tts-piper-cxx-api.cc index 4ba26eed1..6db9d7f97 100644 --- a/cxx-api-examples/offline-tts-piper-cxx-api.cc +++ b/cxx-api-examples/offline-tts-piper-cxx-api.cc @@ -13,6 +13,8 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-pip tar xf vits-piper-en_US-lessac-medium.tar.bz2 rm vits-piper-en_US-lessac-medium.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./offline-tts-piper-cxx-api */ @@ -39,8 +41,7 @@ int32_t main(int32_t argc, char *argv[]) { config.model.vits.model = "./vits-piper-en_US-lessac-medium/en_US-lessac-medium.onnx"; config.model.vits.tokens = "./vits-piper-en_US-lessac-medium/tokens.txt"; - config.model.vits.data_dir = - "./vits-piper-en_US-lessac-medium/espeak-ng-data"; + config.model.vits.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 1; diff --git a/cxx-api-examples/streaming-hlg-decode-file-cxx-api.cc b/cxx-api-examples/streaming-hlg-decode-file-cxx-api.cc new file mode 100644 index 000000000..a576c0c97 --- /dev/null +++ b/cxx-api-examples/streaming-hlg-decode-file-cxx-api.cc @@ -0,0 +1,128 @@ +// cxx-api-examples/streaming-hlg-decode-file-cxx-api.cc +// +// Copyright (c) 2026 Xiaomi Corporation +// +// This file demonstrates how to decode a streaming CTC model with an HLG +// decoding graph using sherpa-onnx's C++ API. +// +// The HLG graph is passed to the C API through ctc_fst_decoder_config. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 +// tar xvf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 +// rm sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 +// +// clang-format on + +#include // NOLINT +#include +#include +#include +#include + +#include "sherpa-onnx/c-api/cxx-api.h" + +int32_t main() { + using namespace sherpa_onnx::cxx; // NOLINT + OnlineRecognizerConfig config; + + std::string dir = "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18"; + + config.model_config.zipformer2_ctc.model = + dir + "/ctc-epoch-30-avg-3-chunk-16-left-128.int8.onnx"; + config.model_config.tokens = dir + "/tokens.txt"; + config.model_config.num_threads = 1; + config.model_config.debug = 0; + + // This is what the example exercises: the graph and max_active reach the C + // API through ctc_fst_decoder_config. + config.ctc_fst_decoder_config.graph = dir + "/HLG.fst"; + config.ctc_fst_decoder_config.max_active = 3000; + + std::cout << "Loading model\n"; + OnlineRecognizer recognizer = OnlineRecognizer::Create(config); + if (!recognizer.Get()) { + std::cerr << "Please check your config\n"; + return -1; + } + std::cout << "Loading model done\n"; + + std::string wave_filename = dir + "/test_wavs/8k.wav"; + Wave wave = ReadWave(wave_filename); + if (wave.samples.empty()) { + std::cerr << "Failed to read: '" << wave_filename << "'\n"; + return -1; + } + + std::cout << "Start recognition\n"; + const auto begin = std::chrono::steady_clock::now(); + + OnlineStream stream = recognizer.CreateStream(); + + // simulate streaming. You can choose an arbitrary N + const int32_t N = 3200; + int32_t segment_id = 0; + + int32_t k = 0; + while (k < static_cast(wave.samples.size())) { + int32_t start = k; + int32_t end = (start + N > static_cast(wave.samples.size())) + ? wave.samples.size() + : (start + N); + k += N; + + stream.AcceptWaveform(wave.sample_rate, wave.samples.data() + start, + end - start); + while (recognizer.IsReady(&stream)) { + recognizer.Decode(&stream); + } + + OnlineRecognizerResult result = recognizer.GetResult(&stream); + if (!result.text.empty()) { + std::cout << "Segment " << segment_id << ": " << result.text << "\n"; + } + + if (recognizer.IsEndpoint(&stream)) { + if (!result.text.empty()) { + ++segment_id; + } + recognizer.Reset(&stream); + } + } + + // add some tail padding + std::vector tail_paddings( + static_cast(wave.sample_rate * 0.3), 0.0f); + stream.AcceptWaveform(wave.sample_rate, tail_paddings.data(), + tail_paddings.size()); + + stream.InputFinished(); + while (recognizer.IsReady(&stream)) { + recognizer.Decode(&stream); + } + + OnlineRecognizerResult result = recognizer.GetResult(&stream); + if (!result.text.empty()) { + std::cout << "Segment " << segment_id << ": " << result.text << "\n"; + } + + // The graph's output labels are in the json field, under "words". + std::cout << "json: " << result.json << "\n"; + + const auto end = std::chrono::steady_clock::now(); + const float elapsed_seconds = + std::chrono::duration_cast(end - begin) + .count() / + 1000.; + float duration = wave.samples.size() / static_cast(wave.sample_rate); + float rtf = elapsed_seconds / duration; + + printf("Number of threads: %d\n", config.model_config.num_threads); + printf("Duration: %.3fs\n", duration); + printf("Elapsed seconds: %.3fs\n", elapsed_seconds); + printf("(Real time factor) RTF = %.3f / %.3f = %.3f\n", elapsed_seconds, + duration, rtf); + + return 0; +} diff --git a/cxx-api-examples/zipvoice-tts-zh-en-cxx-api.cc b/cxx-api-examples/zipvoice-tts-zh-en-cxx-api.cc index 3525cd4b2..a9eeb9d0d 100644 --- a/cxx-api-examples/zipvoice-tts-zh-en-cxx-api.cc +++ b/cxx-api-examples/zipvoice-tts-zh-en-cxx-api.cc @@ -15,6 +15,8 @@ rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./zipvoice-tts-zh-en-cxx-api */ // clang-format on @@ -42,10 +44,9 @@ int32_t main(int32_t argc, char *argv[]) { "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx"; config.model.zipvoice.decoder = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx"; - config.model.zipvoice.data_dir = - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data"; config.model.zipvoice.lexicon = - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt"; + "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt," + "./lexicon-en-us.txt"; config.model.zipvoice.tokens = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt"; config.model.zipvoice.vocoder = "./vocos_24khz.onnx"; diff --git a/dart-api-examples/non-streaming-asr/bin/zipformer-ctc-hlg.dart b/dart-api-examples/non-streaming-asr/bin/zipformer-ctc-hlg.dart new file mode 100644 index 000000000..0a90a50eb --- /dev/null +++ b/dart-api-examples/non-streaming-asr/bin/zipformer-ctc-hlg.dart @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file shows how to use a non-streaming Zipformer CTC model with an HLG +// decoding graph via sherpa-onnx's Dart API. +// +// The HLG graph is passed to the C API through ctcFstDecoderConfig. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// +// dart run bin/zipformer-ctc-hlg.dart \ +// --model ./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx \ +// --tokens ./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt \ +// --graph ./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst \ +// --input-wav ./sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav +// +// clang-format on +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +void main(List arguments) async { + await sherpa_onnx.initBindingsAsync(); + + final parser = ArgParser() + ..addOption('model', help: 'Path to the Zipformer CTC model') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption('graph', help: 'Path to H.fst, HL.fst or HLG.fst') + ..addOption('input-wav', help: 'Path to input.wav to transcribe'); + + final res = parser.parse(arguments); + if (res['model'] == null || + res['tokens'] == null || + res['graph'] == null || + res['input-wav'] == null) { + print(parser.usage); + exit(1); + } + + final model = res['model'] as String; + final tokens = res['tokens'] as String; + final graph = res['graph'] as String; + final inputWav = res['input-wav'] as String; + + final zipformerCtc = sherpa_onnx.OfflineZipformerCtcModelConfig(model: model); + + final modelConfig = sherpa_onnx.OfflineModelConfig( + zipformerCtc: zipformerCtc, + tokens: tokens, + debug: true, + numThreads: 1, + ); + + // This is what the example exercises: the graph and maxActive reach the C + // API through ctcFstDecoderConfig. + final config = sherpa_onnx.OfflineRecognizerConfig( + model: modelConfig, + ctcFstDecoderConfig: sherpa_onnx.OfflineCtcFstDecoderConfig( + graph: graph, + maxActive: 3000, + ), + ); + final recognizer = sherpa_onnx.OfflineRecognizer(config); + + final waveData = sherpa_onnx.readWave(inputWav); + final stream = recognizer.createStream(); + + stream.acceptWaveform( + samples: waveData.samples, sampleRate: waveData.sampleRate); + recognizer.decode(stream); + + final result = recognizer.getResult(stream); + print(result.text); + print(result); + + stream.free(); + recognizer.free(); +} diff --git a/dart-api-examples/non-streaming-asr/run-zipformer-ctc-hlg.sh b/dart-api-examples/non-streaming-asr/run-zipformer-ctc-hlg.sh new file mode 100755 index 000000000..22c7022c2 --- /dev/null +++ b/dart-api-examples/non-streaming-asr/run-zipformer-ctc-hlg.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +if [ ! -f ./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + + tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +fi + +dart run \ + ./bin/zipformer-ctc-hlg.dart \ + --model ./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx \ + --tokens ./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt \ + --graph ./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst \ + --input-wav ./sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav diff --git a/dart-api-examples/tts/README.md b/dart-api-examples/tts/README.md index 7f5c03c6b..f50bf4a4f 100644 --- a/dart-api-examples/tts/README.md +++ b/dart-api-examples/tts/README.md @@ -4,7 +4,7 @@ This folder contains examples for text to speech with Dart API. | File | Description| |------|------------| -|[./bin/piper.dart](./bin/piper.dart)| Use a Piper tts model for text to speech. See [./run-piper.sh](./run-piper.sh)| +|[./bin/piper-lexicon.dart](./bin/piper-lexicon.dart)| Use a Piper tts model for text to speech. See [./run-piper-lexicon.sh](./run-piper-lexicon.sh)| |[./bin/coqui.dart](./bin/coqui.dart)| Use a Coqui tts model for text to speech. See [./run-coqui.sh](./run-coqui.sh)| -|[./bin/zh.dart](./bin/zh.dart)| Use a Chinese VITS tts model for text to speech. See [./run-zh.sh](./run-zh.sh)| -|[./bin/zipvoice-zh-en.dart](./bin/zipvoice-zh-en.dart)| Use a ZipVoice Chinese/English zero-shot TTS model. See [./run-zipvoice-zh-en.sh](./run-zipvoice-zh-en.sh)| +|[./bin/vits-zh-lexicon.dart](./bin/vits-zh-lexicon.dart)| Use a Chinese VITS tts model for text to speech. See [./run-vits-zh-lexicon.sh](./run-vits-zh-lexicon.sh)| +|[./bin/zipvoice-zh-en-lexicon.dart](./bin/zipvoice-zh-en-lexicon.dart)| Use a ZipVoice Chinese/English zero-shot TTS model. See [./run-zipvoice-zh-en-lexicon.sh](./run-zipvoice-zh-en-lexicon.sh)| diff --git a/dart-api-examples/tts/bin/coqui.dart b/dart-api-examples/tts/bin/coqui.dart index 97a5a60ab..7a2f1d4a5 100644 --- a/dart-api-examples/tts/bin/coqui.dart +++ b/dart-api-examples/tts/bin/coqui.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); @@ -51,7 +49,7 @@ void main(List arguments) async { ); final config = sherpa_onnx.OfflineTtsConfig( model: modelConfig, - maxNumSenetences: 1, + maxNumSentences: 1, ); final tts = sherpa_onnx.OfflineTts(config); diff --git a/dart-api-examples/tts/bin/kitten-en.dart b/dart-api-examples/tts/bin/kitten-en-lexicon.dart similarity index 88% rename from dart-api-examples/tts/bin/kitten-en.dart rename to dart-api-examples/tts/bin/kitten-en-lexicon.dart index d450768f8..d0052fa62 100644 --- a/dart-api-examples/tts/bin/kitten-en.dart +++ b/dart-api-examples/tts/bin/kitten-en-lexicon.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); @@ -13,11 +11,7 @@ void main(List arguments) async { ..addOption('model', help: 'Path to the onnx model') ..addOption('voices', help: 'Path to the voices.bin') ..addOption('tokens', help: 'Path to tokens.txt') - ..addOption( - 'data-dir', - help: 'Path to espeak-ng-data directory', - defaultsTo: '', - ) + ..addOption('lexicon', help: 'Path to lexicon-en-us.txt') ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') ..addOption('rule-fars', help: 'Path to rule fars', defaultsTo: '') ..addOption('text', help: 'Text to generate TTS for') @@ -32,7 +26,7 @@ void main(List arguments) async { if (res['model'] == null || res['voices'] == null || res['tokens'] == null || - res['data-dir'] == null || + res['lexicon'] == null || res['output-wav'] == null || res['text'] == null) { print(parser.usage); @@ -41,7 +35,7 @@ void main(List arguments) async { final model = res['model'] as String; final voices = res['voices'] as String; final tokens = res['tokens'] as String; - final dataDir = res['data-dir'] as String; + final lexicon = res['lexicon'] as String; final ruleFsts = res['rule-fsts'] as String; final ruleFars = res['rule-fars'] as String; final text = res['text'] as String; @@ -57,7 +51,7 @@ void main(List arguments) async { model: model, voices: voices, tokens: tokens, - dataDir: dataDir, + lexicon: lexicon, ); final modelConfig = sherpa_onnx.OfflineTtsModelConfig( @@ -67,7 +61,7 @@ void main(List arguments) async { ); final config = sherpa_onnx.OfflineTtsConfig( model: modelConfig, - maxNumSenetences: 1, + maxNumSentences: 1, ruleFsts: ruleFsts, ruleFars: ruleFars, ); diff --git a/dart-api-examples/tts/bin/kitten-en-phonemize.dart b/dart-api-examples/tts/bin/kitten-en-phonemize.dart new file mode 100644 index 000000000..493b08e94 --- /dev/null +++ b/dart-api-examples/tts/bin/kitten-en-phonemize.dart @@ -0,0 +1,111 @@ +// Copyright (c) 2025 Xiaomi Corporation +// +// Phonemize approach: the text is converted to phoneme code points with the +// piper_phonemize package and passed to sherpa-onnx. No lexicon file and no +// espeak-ng-data for sherpa-onnx are needed. See ./kitten-en-lexicon.dart for +// the lexicon-based twin. +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:args/args.dart'; +import 'package:piper_phonemize/piper_phonemize.dart' as piper; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +void main(List arguments) async { + await sherpa_onnx.initBindingsAsync(); + + final parser = ArgParser() + ..addOption('model', help: 'Path to the onnx model') + ..addOption('voices', help: 'Path to the voices.bin') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') + ..addOption('rule-fars', help: 'Path to rule fars', defaultsTo: '') + ..addOption('text', help: 'Text to generate TTS for') + ..addOption('output-wav', help: 'Filename to save the generated audio') + ..addOption('speed', help: 'Speech speed', defaultsTo: '1.0') + ..addOption( + 'sid', + help: 'Speaker ID to select. Used only for multi-speaker TTS', + defaultsTo: '0', + ); + final res = parser.parse(arguments); + if (res['model'] == null || + res['voices'] == null || + res['tokens'] == null || + res['output-wav'] == null || + res['text'] == null) { + print(parser.usage); + exit(1); + } + final model = res['model'] as String; + final voices = res['voices'] as String; + final tokens = res['tokens'] as String; + final ruleFsts = res['rule-fsts'] as String; + final ruleFars = res['rule-fars'] as String; + final text = res['text'] as String; + final outputWav = res['output-wav'] as String; + var speed = double.tryParse(res['speed'] as String) ?? 1.0; + final sid = int.tryParse(res['sid'] as String) ?? 0; + + if (speed == 0) { + speed = 1.0; + } + + await piper.initBindingsAsync(); + + // The espeak-ng-data below feeds only the piper phonemizer; sherpa-onnx + // ignores it. Use the copy shipped inside the model package. + final phonemizerData = '${File(model).parent.path}/espeak-ng-data'; + final ret = piper.PiperPhonemize.initialize(phonemizerData); + if (ret < 0) { + print('Failed to initialize the piper phonemizer at $phonemizerData'); + exit(1); + } + + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + final sentences = piper.PiperPhonemize.phonemize(text, voice: 'en-us') + .where((sentence) => sentence.isNotEmpty) + .toList(); + + final kitten = sherpa_onnx.OfflineTtsKittenModelConfig( + model: model, + voices: voices, + tokens: tokens, + ); + + final modelConfig = sherpa_onnx.OfflineTtsModelConfig( + kitten: kitten, + numThreads: 1, + debug: true, + ); + final config = sherpa_onnx.OfflineTtsConfig( + model: modelConfig, + maxNumSentences: 1, + ruleFsts: ruleFsts, + ruleFars: ruleFars, + ); + + final tts = sherpa_onnx.OfflineTts(config); + final genConfig = sherpa_onnx.OfflineTtsGenerationConfig( + sid: sid, + speed: speed, + silenceScale: 0.2, + phonemeCodepoints: sentences, + ); + final audio = tts.generateWithConfig( + text: '', + config: genConfig, + onProgress: (Float32List samples, double progress) { + print('${samples.length} samples received'); + return 1; + }); + tts.free(); + + sherpa_onnx.writeWave( + filename: outputWav, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + print('Saved to $outputWav'); +} diff --git a/dart-api-examples/tts/bin/kokoro-zh-en.dart b/dart-api-examples/tts/bin/kokoro-en-lexicon.dart similarity index 86% rename from dart-api-examples/tts/bin/kokoro-zh-en.dart rename to dart-api-examples/tts/bin/kokoro-en-lexicon.dart index 6808956c7..2e1269cae 100644 --- a/dart-api-examples/tts/bin/kokoro-zh-en.dart +++ b/dart-api-examples/tts/bin/kokoro-en-lexicon.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); @@ -13,16 +11,7 @@ void main(List arguments) async { ..addOption('model', help: 'Path to the onnx model') ..addOption('voices', help: 'Path to the voices.bin') ..addOption('tokens', help: 'Path to tokens.txt') - ..addOption( - 'data-dir', - help: 'Path to espeak-ng-data directory', - defaultsTo: '', - ) - ..addOption( - 'lexicon', - help: 'Path to lexicon files', - defaultsTo: '', - ) + ..addOption('lexicon', help: 'Path to lexicon-en-us.txt') ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') ..addOption('rule-fars', help: 'Path to rule fars', defaultsTo: '') ..addOption('text', help: 'Text to generate TTS for') @@ -37,7 +26,6 @@ void main(List arguments) async { if (res['model'] == null || res['voices'] == null || res['tokens'] == null || - res['data-dir'] == null || res['lexicon'] == null || res['output-wav'] == null || res['text'] == null) { @@ -47,7 +35,6 @@ void main(List arguments) async { final model = res['model'] as String; final voices = res['voices'] as String; final tokens = res['tokens'] as String; - final dataDir = res['data-dir'] as String; final lexicon = res['lexicon'] as String; final ruleFsts = res['rule-fsts'] as String; final ruleFars = res['rule-fars'] as String; @@ -64,7 +51,6 @@ void main(List arguments) async { model: model, voices: voices, tokens: tokens, - dataDir: dataDir, lexicon: lexicon, ); @@ -75,7 +61,7 @@ void main(List arguments) async { ); final config = sherpa_onnx.OfflineTtsConfig( model: modelConfig, - maxNumSenetences: 1, + maxNumSentences: 1, ruleFsts: ruleFsts, ruleFars: ruleFars, ); diff --git a/dart-api-examples/tts/bin/kokoro-en-phonemize.dart b/dart-api-examples/tts/bin/kokoro-en-phonemize.dart new file mode 100644 index 000000000..c10e7e433 --- /dev/null +++ b/dart-api-examples/tts/bin/kokoro-en-phonemize.dart @@ -0,0 +1,111 @@ +// Copyright (c) 2025 Xiaomi Corporation +// +// Phonemize approach: the text is converted to phoneme code points with the +// piper_phonemize package and passed to sherpa-onnx. No lexicon file and no +// espeak-ng-data for sherpa-onnx are needed. See ./kokoro-en-lexicon.dart for +// the lexicon-based twin. +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:args/args.dart'; +import 'package:piper_phonemize/piper_phonemize.dart' as piper; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +void main(List arguments) async { + await sherpa_onnx.initBindingsAsync(); + + final parser = ArgParser() + ..addOption('model', help: 'Path to the onnx model') + ..addOption('voices', help: 'Path to the voices.bin') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') + ..addOption('rule-fars', help: 'Path to rule fars', defaultsTo: '') + ..addOption('text', help: 'Text to generate TTS for') + ..addOption('output-wav', help: 'Filename to save the generated audio') + ..addOption('speed', help: 'Speech speed', defaultsTo: '1.0') + ..addOption( + 'sid', + help: 'Speaker ID to select. Used only for multi-speaker TTS', + defaultsTo: '0', + ); + final res = parser.parse(arguments); + if (res['model'] == null || + res['voices'] == null || + res['tokens'] == null || + res['output-wav'] == null || + res['text'] == null) { + print(parser.usage); + exit(1); + } + final model = res['model'] as String; + final voices = res['voices'] as String; + final tokens = res['tokens'] as String; + final ruleFsts = res['rule-fsts'] as String; + final ruleFars = res['rule-fars'] as String; + final text = res['text'] as String; + final outputWav = res['output-wav'] as String; + var speed = double.tryParse(res['speed'] as String) ?? 1.0; + final sid = int.tryParse(res['sid'] as String) ?? 0; + + if (speed == 0) { + speed = 1.0; + } + + await piper.initBindingsAsync(); + + // The espeak-ng-data below feeds only the piper phonemizer; sherpa-onnx + // ignores it. Use the copy shipped inside the model package. + final phonemizerData = '${File(model).parent.path}/espeak-ng-data'; + final ret = piper.PiperPhonemize.initialize(phonemizerData); + if (ret < 0) { + print('Failed to initialize the piper phonemizer at $phonemizerData'); + exit(1); + } + + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + final sentences = piper.PiperPhonemize.phonemize(text, voice: 'en-us') + .where((sentence) => sentence.isNotEmpty) + .toList(); + + final kokoro = sherpa_onnx.OfflineTtsKokoroModelConfig( + model: model, + voices: voices, + tokens: tokens, + ); + + final modelConfig = sherpa_onnx.OfflineTtsModelConfig( + kokoro: kokoro, + numThreads: 1, + debug: true, + ); + final config = sherpa_onnx.OfflineTtsConfig( + model: modelConfig, + maxNumSentences: 1, + ruleFsts: ruleFsts, + ruleFars: ruleFars, + ); + + final tts = sherpa_onnx.OfflineTts(config); + final genConfig = sherpa_onnx.OfflineTtsGenerationConfig( + sid: sid, + speed: speed, + silenceScale: 0.2, + phonemeCodepoints: sentences, + ); + final audio = tts.generateWithConfig( + text: '', + config: genConfig, + onProgress: (Float32List samples, double progress) { + print('${samples.length} samples received'); + return 1; + }); + tts.free(); + + sherpa_onnx.writeWave( + filename: outputWav, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + print('Saved to $outputWav'); +} diff --git a/dart-api-examples/tts/bin/kokoro-en.dart b/dart-api-examples/tts/bin/kokoro-zh-en-lexicon.dart similarity index 90% rename from dart-api-examples/tts/bin/kokoro-en.dart rename to dart-api-examples/tts/bin/kokoro-zh-en-lexicon.dart index 5f9d5faa2..eafb50e29 100644 --- a/dart-api-examples/tts/bin/kokoro-en.dart +++ b/dart-api-examples/tts/bin/kokoro-zh-en-lexicon.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); @@ -14,8 +12,8 @@ void main(List arguments) async { ..addOption('voices', help: 'Path to the voices.bin') ..addOption('tokens', help: 'Path to tokens.txt') ..addOption( - 'data-dir', - help: 'Path to espeak-ng-data directory', + 'lexicon', + help: 'Path to lexicon files', defaultsTo: '', ) ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') @@ -32,7 +30,7 @@ void main(List arguments) async { if (res['model'] == null || res['voices'] == null || res['tokens'] == null || - res['data-dir'] == null || + res['lexicon'] == null || res['output-wav'] == null || res['text'] == null) { print(parser.usage); @@ -41,7 +39,7 @@ void main(List arguments) async { final model = res['model'] as String; final voices = res['voices'] as String; final tokens = res['tokens'] as String; - final dataDir = res['data-dir'] as String; + final lexicon = res['lexicon'] as String; final ruleFsts = res['rule-fsts'] as String; final ruleFars = res['rule-fars'] as String; final text = res['text'] as String; @@ -57,7 +55,7 @@ void main(List arguments) async { model: model, voices: voices, tokens: tokens, - dataDir: dataDir, + lexicon: lexicon, ); final modelConfig = sherpa_onnx.OfflineTtsModelConfig( @@ -67,7 +65,7 @@ void main(List arguments) async { ); final config = sherpa_onnx.OfflineTtsConfig( model: modelConfig, - maxNumSenetences: 1, + maxNumSentences: 1, ruleFsts: ruleFsts, ruleFars: ruleFars, ); diff --git a/dart-api-examples/tts/bin/matcha-en.dart b/dart-api-examples/tts/bin/matcha-en-lexicon.dart similarity index 89% rename from dart-api-examples/tts/bin/matcha-en.dart rename to dart-api-examples/tts/bin/matcha-en-lexicon.dart index 58c778a91..9523beead 100644 --- a/dart-api-examples/tts/bin/matcha-en.dart +++ b/dart-api-examples/tts/bin/matcha-en-lexicon.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); @@ -13,11 +11,7 @@ void main(List arguments) async { ..addOption('acoustic-model', help: 'Path to the acoustic model') ..addOption('vocoder', help: 'Path to the vocoder model') ..addOption('tokens', help: 'Path to tokens.txt') - ..addOption( - 'data-dir', - help: 'Path to espeak-ng-data directory', - defaultsTo: '', - ) + ..addOption('lexicon', help: 'Path to lexicon-en-us.txt') ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') ..addOption('rule-fars', help: 'Path to rule fars', defaultsTo: '') ..addOption('text', help: 'Text to generate TTS for') @@ -32,7 +26,7 @@ void main(List arguments) async { if (res['acoustic-model'] == null || res['vocoder'] == null || res['tokens'] == null || - res['data-dir'] == null || + res['lexicon'] == null || res['output-wav'] == null || res['text'] == null) { print(parser.usage); @@ -41,7 +35,7 @@ void main(List arguments) async { final acousticModel = res['acoustic-model'] as String; final vocoder = res['vocoder'] as String; final tokens = res['tokens'] as String; - final dataDir = res['data-dir'] as String; + final lexicon = res['lexicon'] as String; final ruleFsts = res['rule-fsts'] as String; final ruleFars = res['rule-fars'] as String; final text = res['text'] as String; @@ -57,7 +51,7 @@ void main(List arguments) async { acousticModel: acousticModel, vocoder: vocoder, tokens: tokens, - dataDir: dataDir, + lexicon: lexicon, ); final modelConfig = sherpa_onnx.OfflineTtsModelConfig( @@ -67,7 +61,7 @@ void main(List arguments) async { ); final config = sherpa_onnx.OfflineTtsConfig( model: modelConfig, - maxNumSenetences: 1, + maxNumSentences: 1, ruleFsts: ruleFsts, ruleFars: ruleFars, ); diff --git a/dart-api-examples/tts/bin/matcha-en-phonemize.dart b/dart-api-examples/tts/bin/matcha-en-phonemize.dart new file mode 100644 index 000000000..156bceac0 --- /dev/null +++ b/dart-api-examples/tts/bin/matcha-en-phonemize.dart @@ -0,0 +1,111 @@ +// Copyright (c) 2025 Xiaomi Corporation +// +// Phonemize approach: the text is converted to phoneme code points with the +// piper_phonemize package and passed to sherpa-onnx. No lexicon file and no +// espeak-ng-data for sherpa-onnx are needed. See ./matcha-en-lexicon.dart for +// the lexicon-based twin. +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:args/args.dart'; +import 'package:piper_phonemize/piper_phonemize.dart' as piper; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +void main(List arguments) async { + await sherpa_onnx.initBindingsAsync(); + + final parser = ArgParser() + ..addOption('acoustic-model', help: 'Path to the acoustic model') + ..addOption('vocoder', help: 'Path to the vocoder model') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption('rule-fsts', help: 'Path to rule fsts', defaultsTo: '') + ..addOption('rule-fars', help: 'Path to rule fars', defaultsTo: '') + ..addOption('text', help: 'Text to generate TTS for') + ..addOption('output-wav', help: 'Filename to save the generated audio') + ..addOption('speed', help: 'Speech speed', defaultsTo: '1.0') + ..addOption( + 'sid', + help: 'Speaker ID to select. Used only for multi-speaker TTS', + defaultsTo: '0', + ); + final res = parser.parse(arguments); + if (res['acoustic-model'] == null || + res['vocoder'] == null || + res['tokens'] == null || + res['output-wav'] == null || + res['text'] == null) { + print(parser.usage); + exit(1); + } + final acousticModel = res['acoustic-model'] as String; + final vocoder = res['vocoder'] as String; + final tokens = res['tokens'] as String; + final ruleFsts = res['rule-fsts'] as String; + final ruleFars = res['rule-fars'] as String; + final text = res['text'] as String; + final outputWav = res['output-wav'] as String; + var speed = double.tryParse(res['speed'] as String) ?? 1.0; + final sid = int.tryParse(res['sid'] as String) ?? 0; + + if (speed == 0) { + speed = 1.0; + } + + await piper.initBindingsAsync(); + + // The espeak-ng-data below feeds only the piper phonemizer; sherpa-onnx + // ignores it. Use the copy shipped inside the model package. + final phonemizerData = '${File(acousticModel).parent.path}/espeak-ng-data'; + final ret = piper.PiperPhonemize.initialize(phonemizerData); + if (ret < 0) { + print('Failed to initialize the piper phonemizer at $phonemizerData'); + exit(1); + } + + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + final sentences = piper.PiperPhonemize.phonemize(text, voice: 'en-us') + .where((sentence) => sentence.isNotEmpty) + .toList(); + + final matcha = sherpa_onnx.OfflineTtsMatchaModelConfig( + acousticModel: acousticModel, + vocoder: vocoder, + tokens: tokens, + ); + + final modelConfig = sherpa_onnx.OfflineTtsModelConfig( + matcha: matcha, + numThreads: 1, + debug: true, + ); + final config = sherpa_onnx.OfflineTtsConfig( + model: modelConfig, + maxNumSentences: 1, + ruleFsts: ruleFsts, + ruleFars: ruleFars, + ); + + final tts = sherpa_onnx.OfflineTts(config); + final genConfig = sherpa_onnx.OfflineTtsGenerationConfig( + sid: sid, + speed: speed, + silenceScale: 0.2, + phonemeCodepoints: sentences, + ); + final audio = tts.generateWithConfig( + text: '', + config: genConfig, + onProgress: (Float32List samples, double progress) { + print('${samples.length} samples received'); + return 1; + }); + tts.free(); + + sherpa_onnx.writeWave( + filename: outputWav, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + print('Saved to $outputWav'); +} diff --git a/dart-api-examples/tts/bin/matcha-zh.dart b/dart-api-examples/tts/bin/matcha-zh-lexicon.dart similarity index 96% rename from dart-api-examples/tts/bin/matcha-zh.dart rename to dart-api-examples/tts/bin/matcha-zh-lexicon.dart index 8b2f19c41..de207d5a7 100644 --- a/dart-api-examples/tts/bin/matcha-zh.dart +++ b/dart-api-examples/tts/bin/matcha-zh-lexicon.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); @@ -63,7 +61,7 @@ void main(List arguments) async { ); final config = sherpa_onnx.OfflineTtsConfig( model: modelConfig, - maxNumSenetences: 1, + maxNumSentences: 1, ruleFsts: ruleFsts, ruleFars: ruleFars, ); diff --git a/dart-api-examples/tts/bin/piper.dart b/dart-api-examples/tts/bin/piper-lexicon.dart similarity index 89% rename from dart-api-examples/tts/bin/piper.dart rename to dart-api-examples/tts/bin/piper-lexicon.dart index ef142cdf7..db854f6c0 100644 --- a/dart-api-examples/tts/bin/piper.dart +++ b/dart-api-examples/tts/bin/piper-lexicon.dart @@ -5,15 +5,13 @@ import 'dart:typed_data'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); final parser = ArgParser() ..addOption('model', help: 'Path to the ONNX model') ..addOption('tokens', help: 'Path to tokens.txt') - ..addOption('data-dir', help: 'Path to espeak-ng-data directory') + ..addOption('lexicon', help: 'Path to lexicon-en-us.txt') ..addOption('text', help: 'Text to generate TTS for') ..addOption('output-wav', help: 'Filename to save the generated audio') ..addOption('speed', help: 'Speech speed', defaultsTo: '1.0') @@ -25,7 +23,7 @@ void main(List arguments) async { final res = parser.parse(arguments); if (res['model'] == null || res['tokens'] == null || - res['data-dir'] == null || + res['lexicon'] == null || res['output-wav'] == null || res['text'] == null) { print(parser.usage); @@ -33,7 +31,7 @@ void main(List arguments) async { } final model = res['model'] as String; final tokens = res['tokens'] as String; - final dataDir = res['data-dir'] as String; + final lexicon = res['lexicon'] as String; final text = res['text'] as String; final outputWav = res['output-wav'] as String; var speed = double.tryParse(res['speed'] as String) ?? 1.0; @@ -46,7 +44,7 @@ void main(List arguments) async { final vits = sherpa_onnx.OfflineTtsVitsModelConfig( model: model, tokens: tokens, - dataDir: dataDir, + lexicon: lexicon, ); final modelConfig = sherpa_onnx.OfflineTtsModelConfig( @@ -56,7 +54,7 @@ void main(List arguments) async { ); final config = sherpa_onnx.OfflineTtsConfig( model: modelConfig, - maxNumSenetences: 1, + maxNumSentences: 1, ); final tts = sherpa_onnx.OfflineTts(config); diff --git a/dart-api-examples/tts/bin/piper-phonemize.dart b/dart-api-examples/tts/bin/piper-phonemize.dart new file mode 100644 index 000000000..8b8221d44 --- /dev/null +++ b/dart-api-examples/tts/bin/piper-phonemize.dart @@ -0,0 +1,105 @@ +// Copyright (c) 2024 Xiaomi Corporation +// +// Phonemize approach: the text is converted to phoneme code points with the +// piper_phonemize package and passed to sherpa-onnx. No lexicon file and no +// espeak-ng-data for sherpa-onnx are needed. See ./piper-lexicon.dart for the +// lexicon-based twin. +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:args/args.dart'; +import 'package:piper_phonemize/piper_phonemize.dart' as piper; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +void main(List arguments) async { + await sherpa_onnx.initBindingsAsync(); + + final parser = ArgParser() + ..addOption('model', help: 'Path to the ONNX model') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption('text', help: 'Text to generate TTS for') + ..addOption('output-wav', help: 'Filename to save the generated audio') + ..addOption('speed', help: 'Speech speed', defaultsTo: '1.0') + ..addOption( + 'sid', + help: 'Speaker ID to select. Used only for multi-speaker TTS', + defaultsTo: '0', + ); + final res = parser.parse(arguments); + if (res['model'] == null || + res['tokens'] == null || + res['output-wav'] == null || + res['text'] == null) { + print(parser.usage); + exit(1); + } + final model = res['model'] as String; + final tokens = res['tokens'] as String; + final text = res['text'] as String; + final outputWav = res['output-wav'] as String; + var speed = double.tryParse(res['speed'] as String) ?? 1.0; + final sid = int.tryParse(res['sid'] as String) ?? 0; + + if (speed == 0) { + speed = 1.0; + } + + await piper.initBindingsAsync(); + + // The espeak-ng-data below feeds only the piper phonemizer; sherpa-onnx + // ignores it. Use the copy shipped inside the model package. + final phonemizerData = '${File(model).parent.path}/espeak-ng-data'; + final ret = piper.PiperPhonemize.initialize(phonemizerData); + if (ret < 0) { + print('Failed to initialize the piper phonemizer at $phonemizerData'); + exit(1); + } + + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + final sentences = piper.PiperPhonemize.phonemize(text, voice: 'en-us') + .where((sentence) => sentence.isNotEmpty) + .toList(); + + final vits = sherpa_onnx.OfflineTtsVitsModelConfig( + model: model, + tokens: tokens, + ); + + final modelConfig = sherpa_onnx.OfflineTtsModelConfig( + vits: vits, + numThreads: 1, + debug: true, + ); + final config = sherpa_onnx.OfflineTtsConfig( + model: modelConfig, + maxNumSentences: 1, + ); + + final tts = sherpa_onnx.OfflineTts(config); + final genConfig = sherpa_onnx.OfflineTtsGenerationConfig( + sid: sid, + speed: speed, + silenceScale: 0.2, + phonemeCodepoints: sentences, + ); + final audio = tts.generateWithConfig( + text: '', + config: genConfig, + onProgress: (Float32List samples, double progress) { + print('${samples.length} samples received'); + // You can play samples in a separate thread/isolate + + // 1 means to continue + // 0 means to stop + return 1; + }); + tts.free(); + + sherpa_onnx.writeWave( + filename: outputWav, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + print('Saved to $outputWav'); +} diff --git a/dart-api-examples/tts/bin/pocket-en.dart b/dart-api-examples/tts/bin/pocket-en.dart index f1db5048b..8c32200e5 100644 --- a/dart-api-examples/tts/bin/pocket-en.dart +++ b/dart-api-examples/tts/bin/pocket-en.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); diff --git a/dart-api-examples/tts/bin/supertonic-en.dart b/dart-api-examples/tts/bin/supertonic-en.dart index 4af6706bd..e81d497f4 100644 --- a/dart-api-examples/tts/bin/supertonic-en.dart +++ b/dart-api-examples/tts/bin/supertonic-en.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); diff --git a/dart-api-examples/tts/bin/vits-de-phonemize.dart b/dart-api-examples/tts/bin/vits-de-phonemize.dart new file mode 100644 index 000000000..4886ca9e5 --- /dev/null +++ b/dart-api-examples/tts/bin/vits-de-phonemize.dart @@ -0,0 +1,100 @@ +// Copyright (c) 2024 Xiaomi Corporation +// +// Phonemize approach for a German VITS (piper) model. The text is converted +// to phoneme code points with the piper_phonemize package and passed to +// sherpa-onnx. There is no German lexicon, so this model is phonemize-only. +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:args/args.dart'; +import 'package:piper_phonemize/piper_phonemize.dart' as piper; +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +void main(List arguments) async { + await sherpa_onnx.initBindingsAsync(); + + final parser = ArgParser() + ..addOption('model', help: 'Path to the ONNX model') + ..addOption('tokens', help: 'Path to tokens.txt') + ..addOption('text', help: 'Text to generate TTS for') + ..addOption('output-wav', help: 'Filename to save the generated audio') + ..addOption('speed', help: 'Speech speed', defaultsTo: '1.0') + ..addOption( + 'sid', + help: 'Speaker ID to select. Used only for multi-speaker TTS', + defaultsTo: '0', + ); + final res = parser.parse(arguments); + if (res['model'] == null || + res['tokens'] == null || + res['output-wav'] == null || + res['text'] == null) { + print(parser.usage); + exit(1); + } + final model = res['model'] as String; + final tokens = res['tokens'] as String; + final text = res['text'] as String; + final outputWav = res['output-wav'] as String; + var speed = double.tryParse(res['speed'] as String) ?? 1.0; + final sid = int.tryParse(res['sid'] as String) ?? 0; + + if (speed == 0) { + speed = 1.0; + } + + await piper.initBindingsAsync(); + + // The espeak-ng-data below feeds only the piper phonemizer; sherpa-onnx + // ignores it. Use the copy shipped inside the model package. + final phonemizerData = '${File(model).parent.path}/espeak-ng-data'; + final ret = piper.PiperPhonemize.initialize(phonemizerData); + if (ret < 0) { + print('Failed to initialize the piper phonemizer at $phonemizerData'); + exit(1); + } + + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + final sentences = piper.PiperPhonemize.phonemize(text, voice: 'de') + .where((sentence) => sentence.isNotEmpty) + .toList(); + + final vits = sherpa_onnx.OfflineTtsVitsModelConfig( + model: model, + tokens: tokens, + ); + + final modelConfig = sherpa_onnx.OfflineTtsModelConfig( + vits: vits, + numThreads: 1, + debug: true, + ); + final config = sherpa_onnx.OfflineTtsConfig( + model: modelConfig, + maxNumSentences: 1, + ); + + final tts = sherpa_onnx.OfflineTts(config); + final genConfig = sherpa_onnx.OfflineTtsGenerationConfig( + sid: sid, + speed: speed, + silenceScale: 0.2, + phonemeCodepoints: sentences, + ); + final audio = tts.generateWithConfig( + text: '', + config: genConfig, + onProgress: (Float32List samples, double progress) { + print('${samples.length} samples received'); + return 1; + }); + tts.free(); + + sherpa_onnx.writeWave( + filename: outputWav, + samples: audio.samples, + sampleRate: audio.sampleRate, + ); + print('Saved to $outputWav'); +} diff --git a/dart-api-examples/tts/bin/vits-zh.dart b/dart-api-examples/tts/bin/vits-zh-lexicon.dart similarity index 96% rename from dart-api-examples/tts/bin/vits-zh.dart rename to dart-api-examples/tts/bin/vits-zh-lexicon.dart index 5af564f09..4f7e91f9b 100644 --- a/dart-api-examples/tts/bin/vits-zh.dart +++ b/dart-api-examples/tts/bin/vits-zh-lexicon.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); @@ -59,7 +57,7 @@ void main(List arguments) async { ); final config = sherpa_onnx.OfflineTtsConfig( model: modelConfig, - maxNumSenetences: 1, + maxNumSentences: 1, ruleFsts: ruleFsts, ruleFars: ruleFars, ); diff --git a/dart-api-examples/tts/bin/zipvoice-zh-en.dart b/dart-api-examples/tts/bin/zipvoice-zh-en-lexicon.dart similarity index 93% rename from dart-api-examples/tts/bin/zipvoice-zh-en.dart rename to dart-api-examples/tts/bin/zipvoice-zh-en-lexicon.dart index 5d45c444a..e7809bd29 100644 --- a/dart-api-examples/tts/bin/zipvoice-zh-en.dart +++ b/dart-api-examples/tts/bin/zipvoice-zh-en-lexicon.dart @@ -4,8 +4,6 @@ import 'dart:io'; import 'package:args/args.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; - void main(List arguments) async { await sherpa_onnx.initBindingsAsync(); @@ -14,7 +12,6 @@ void main(List arguments) async { ..addOption('encoder', help: 'Path to the encoder model') ..addOption('decoder', help: 'Path to the decoder model') ..addOption('vocoder', help: 'Path to the vocoder model') - ..addOption('data-dir', help: 'Path to espeak-ng-data directory') ..addOption('lexicon', help: 'Path to lexicon.txt') ..addOption('reference-audio', help: 'Path to reference audio (wav)') ..addOption('reference-text', help: 'Reference text for zero-shot TTS') @@ -32,7 +29,6 @@ void main(List arguments) async { res['encoder'] == null || res['decoder'] == null || res['vocoder'] == null || - res['data-dir'] == null || res['lexicon'] == null || res['reference-audio'] == null || res['reference-text'] == null || @@ -46,7 +42,6 @@ void main(List arguments) async { final encoder = res['encoder'] as String; final decoder = res['decoder'] as String; final vocoder = res['vocoder'] as String; - final dataDir = res['data-dir'] as String; final lexicon = res['lexicon'] as String; final referenceAudioPath = res['reference-audio'] as String; final referenceText = res['reference-text'] as String; @@ -59,7 +54,6 @@ void main(List arguments) async { encoder: encoder, decoder: decoder, vocoder: vocoder, - dataDir: dataDir, lexicon: lexicon, ); diff --git a/dart-api-examples/tts/pubspec.yaml b/dart-api-examples/tts/pubspec.yaml index 643e8e1b1..d1058f161 100644 --- a/dart-api-examples/tts/pubspec.yaml +++ b/dart-api-examples/tts/pubspec.yaml @@ -1,6 +1,7 @@ name: tts description: A sample command-line application. version: 1.0.0 +publish_to: 'none' # repository: https://github.com/my_org/my_repo environment: @@ -8,7 +9,11 @@ environment: # Add regular dependencies here. dependencies: - sherpa_onnx: 1.13.8 + sherpa_onnx: + path: ../../flutter/sherpa_onnx + + piper_phonemize: ^1.5.8 + path: ^1.9.0 args: ^2.5.0 diff --git a/dart-api-examples/tts/run-kitten-en-lexicon.sh b/dart-api-examples/tts/run-kitten-en-lexicon.sh new file mode 100755 index 000000000..b4a714709 --- /dev/null +++ b/dart-api-examples/tts/run-kitten-en-lexicon.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kitten.html +# to download more models +if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 + tar xf kitten-nano-en-v0_1-fp16.tar.bz2 + rm kitten-nano-en-v0_1-fp16.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +dart run \ + ./bin/kitten-en-lexicon.dart \ + --model ./kitten-nano-en-v0_1-fp16/model.fp16.onnx \ + --voices ./kitten-nano-en-v0_1-fp16/voices.bin \ + --tokens ./kitten-nano-en-v0_1-fp16/tokens.txt \ + --lexicon ./lexicon-en-us.txt \ + --sid 0 \ + --speed 1.0 \ + --output-wav kitten-en-0-lexicon.wav \ + --text "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." + +ls -lh *.wav diff --git a/dart-api-examples/tts/run-kitten-en.sh b/dart-api-examples/tts/run-kitten-en-phonemize.sh similarity index 86% rename from dart-api-examples/tts/run-kitten-en.sh rename to dart-api-examples/tts/run-kitten-en-phonemize.sh index 4795c2a3d..51e798775 100755 --- a/dart-api-examples/tts/run-kitten-en.sh +++ b/dart-api-examples/tts/run-kitten-en-phonemize.sh @@ -14,14 +14,13 @@ if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then fi dart run \ - ./bin/kitten-en.dart \ + ./bin/kitten-en-phonemize.dart \ --model ./kitten-nano-en-v0_1-fp16/model.fp16.onnx \ --voices ./kitten-nano-en-v0_1-fp16/voices.bin \ --tokens ./kitten-nano-en-v0_1-fp16/tokens.txt \ - --data-dir ./kitten-nano-en-v0_1-fp16/espeak-ng-data \ --sid 0 \ --speed 1.0 \ - --output-wav kitten-en-0.wav \ + --output-wav kitten-en-0-phonemize.wav \ --text "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." ls -lh *.wav diff --git a/dart-api-examples/tts/run-kokoro-en-lexicon.sh b/dart-api-examples/tts/run-kokoro-en-lexicon.sh new file mode 100755 index 000000000..d146dd6ee --- /dev/null +++ b/dart-api-examples/tts/run-kokoro-en-lexicon.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html +# to download more models +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +dart run \ + ./bin/kokoro-en-lexicon.dart \ + --model ./kokoro-en-v0_19/model.onnx \ + --voices ./kokoro-en-v0_19/voices.bin \ + --tokens ./kokoro-en-v0_19/tokens.txt \ + --lexicon ./lexicon-en-us.txt \ + --sid 9 \ + --speed 1.0 \ + --output-wav kokoro-en-9-lexicon.wav \ + --text "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." + +ls -lh *.wav diff --git a/dart-api-examples/tts/run-kokoro-en.sh b/dart-api-examples/tts/run-kokoro-en-phonemize.sh similarity index 86% rename from dart-api-examples/tts/run-kokoro-en.sh rename to dart-api-examples/tts/run-kokoro-en-phonemize.sh index 78e21a286..d8a3794a6 100755 --- a/dart-api-examples/tts/run-kokoro-en.sh +++ b/dart-api-examples/tts/run-kokoro-en-phonemize.sh @@ -14,14 +14,13 @@ if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then fi dart run \ - ./bin/kokoro-en.dart \ + ./bin/kokoro-en-phonemize.dart \ --model ./kokoro-en-v0_19/model.onnx \ --voices ./kokoro-en-v0_19/voices.bin \ --tokens ./kokoro-en-v0_19/tokens.txt \ - --data-dir ./kokoro-en-v0_19/espeak-ng-data \ --sid 9 \ --speed 1.0 \ - --output-wav kokoro-en-9.wav \ + --output-wav kokoro-en-9-phonemize.wav \ --text "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." ls -lh *.wav diff --git a/dart-api-examples/tts/run-kokoro-zh-en.sh b/dart-api-examples/tts/run-kokoro-zh-en-lexicon.sh similarity index 87% rename from dart-api-examples/tts/run-kokoro-zh-en.sh rename to dart-api-examples/tts/run-kokoro-zh-en-lexicon.sh index ae4f89aa9..39b22e0a3 100755 --- a/dart-api-examples/tts/run-kokoro-zh-en.sh +++ b/dart-api-examples/tts/run-kokoro-zh-en-lexicon.sh @@ -14,15 +14,14 @@ if [ ! -f ./kokoro-multi-lang-v1_0/model.onnx ]; then fi dart run \ - ./bin/kokoro-zh-en.dart \ + ./bin/kokoro-zh-en-lexicon.dart \ --model ./kokoro-multi-lang-v1_0/model.onnx \ --voices ./kokoro-multi-lang-v1_0/voices.bin \ --tokens ./kokoro-multi-lang-v1_0/tokens.txt \ - --data-dir ./kokoro-multi-lang-v1_0/espeak-ng-data \ --lexicon ./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt \ --sid 45 \ --speed 1.0 \ - --output-wav kokoro-zh-en-45.wav \ + --output-wav kokoro-zh-en-45-lexicon.wav \ --text "中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？" ls -lh *.wav diff --git a/dart-api-examples/tts/run-matcha-en-lexicon.sh b/dart-api-examples/tts/run-matcha-en-lexicon.sh new file mode 100755 index 000000000..6b39222e0 --- /dev/null +++ b/dart-api-examples/tts/run-matcha-en-lexicon.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./vocos-22khz-univ.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +dart run \ + ./bin/matcha-en-lexicon.dart \ + --acoustic-model ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --vocoder ./vocos-22khz-univ.onnx \ + --tokens ./matcha-icefall-en_US-ljspeech/tokens.txt \ + --lexicon ./lexicon-en-us.txt \ + --sid 0 \ + --speed 1.0 \ + --output-wav matcha-en-1-lexicon.wav \ + --text "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." \ + +ls -lh *.wav diff --git a/dart-api-examples/tts/run-matcha-en.sh b/dart-api-examples/tts/run-matcha-en-phonemize.sh similarity index 90% rename from dart-api-examples/tts/run-matcha-en.sh rename to dart-api-examples/tts/run-matcha-en-phonemize.sh index 57560168f..d5b4f964c 100755 --- a/dart-api-examples/tts/run-matcha-en.sh +++ b/dart-api-examples/tts/run-matcha-en-phonemize.sh @@ -19,14 +19,13 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then fi dart run \ - ./bin/matcha-en.dart \ + ./bin/matcha-en-phonemize.dart \ --acoustic-model ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ --vocoder ./vocos-22khz-univ.onnx \ --tokens ./matcha-icefall-en_US-ljspeech/tokens.txt \ - --data-dir ./matcha-icefall-en_US-ljspeech/espeak-ng-data \ --sid 0 \ --speed 1.0 \ - --output-wav matcha-en-1.wav \ + --output-wav matcha-en-1-phonemize.wav \ --text "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." \ ls -lh *.wav diff --git a/dart-api-examples/tts/run-matcha-zh.sh b/dart-api-examples/tts/run-matcha-zh-lexicon.sh similarity index 91% rename from dart-api-examples/tts/run-matcha-zh.sh rename to dart-api-examples/tts/run-matcha-zh-lexicon.sh index 3edf54048..e5573562a 100755 --- a/dart-api-examples/tts/run-matcha-zh.sh +++ b/dart-api-examples/tts/run-matcha-zh-lexicon.sh @@ -18,7 +18,7 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then fi dart run \ - ./bin/matcha-zh.dart \ + ./bin/matcha-zh-lexicon.dart \ --acoustic-model ./matcha-icefall-zh-baker/model-steps-3.onnx \ --vocoder ./vocos-22khz-univ.onnx \ --lexicon ./matcha-icefall-zh-baker/lexicon.txt \ @@ -26,18 +26,18 @@ dart run \ --rule-fsts ./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ --sid 0 \ --speed 1.0 \ - --output-wav matcha-zh-1.wav \ + --output-wav matcha-zh-1-lexicon.wav \ --text "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" \ dart run \ - ./bin/matcha-zh.dart \ + ./bin/matcha-zh-lexicon.dart \ --acoustic-model ./matcha-icefall-zh-baker/model-steps-3.onnx \ --vocoder ./vocos-22khz-univ.onnx \ --lexicon ./matcha-icefall-zh-baker/lexicon.txt \ --tokens ./matcha-icefall-zh-baker/tokens.txt \ --sid 0 \ --speed 1.0 \ - --output-wav matcha-zh-2.wav \ + --output-wav matcha-zh-2-lexicon.wav \ --text "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔." \ ls -lh *.wav diff --git a/dart-api-examples/tts/run-piper-lexicon.sh b/dart-api-examples/tts/run-piper-lexicon.sh new file mode 100755 index 000000000..c397665f4 --- /dev/null +++ b/dart-api-examples/tts/run-piper-lexicon.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + + +# Please visit +# https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +# to download more models + +if [[ ! -f ./vits-piper-en_US-libritts_r-medium/tokens.txt ]]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-libritts_r-medium.tar.bz2 + tar xf vits-piper-en_US-libritts_r-medium.tar.bz2 + rm vits-piper-en_US-libritts_r-medium.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +dart run \ + ./bin/piper-lexicon.dart \ + --model ./vits-piper-en_US-libritts_r-medium/en_US-libritts_r-medium.onnx \ + --tokens ./vits-piper-en_US-libritts_r-medium/tokens.txt \ + --lexicon ./lexicon-en-us.txt \ + --sid 351 \ + --speed 1.0 \ + --text 'How are you doing? This is a speech to text example, using next generation kaldi with piper.' \ + --output-wav piper-351-lexicon.wav + +ls -lh *.wav diff --git a/dart-api-examples/tts/run-piper.sh b/dart-api-examples/tts/run-piper-phonemize.sh similarity index 86% rename from dart-api-examples/tts/run-piper.sh rename to dart-api-examples/tts/run-piper-phonemize.sh index ce8d42c70..1a6a38fcc 100755 --- a/dart-api-examples/tts/run-piper.sh +++ b/dart-api-examples/tts/run-piper-phonemize.sh @@ -16,13 +16,12 @@ if [[ ! -f ./vits-piper-en_US-libritts_r-medium/tokens.txt ]]; then fi dart run \ - ./bin/piper.dart \ + ./bin/piper-phonemize.dart \ --model ./vits-piper-en_US-libritts_r-medium/en_US-libritts_r-medium.onnx \ --tokens ./vits-piper-en_US-libritts_r-medium/tokens.txt \ - --data-dir ./vits-piper-en_US-libritts_r-medium/espeak-ng-data \ --sid 351 \ --speed 1.0 \ --text 'How are you doing? This is a speech to text example, using next generation kaldi with piper.' \ - --output-wav piper-351.wav + --output-wav piper-351-phonemize.wav ls -lh *.wav diff --git a/dart-api-examples/tts/run-vits-de-phonemize.sh b/dart-api-examples/tts/run-vits-de-phonemize.sh new file mode 100755 index 000000000..62af2a5f5 --- /dev/null +++ b/dart-api-examples/tts/run-vits-de-phonemize.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -ex + +dart pub get + + +# Please visit +# https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +# to download more models + +if [[ ! -f ./vits-piper-de_DE-glados-low/tokens.txt ]]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-glados-low.tar.bz2 + tar xf vits-piper-de_DE-glados-low.tar.bz2 + rm vits-piper-de_DE-glados-low.tar.bz2 +fi + +dart run \ + ./bin/vits-de-phonemize.dart \ + --model ./vits-piper-de_DE-glados-low/de_DE-glados-low.onnx \ + --tokens ./vits-piper-de_DE-glados-low/tokens.txt \ + --sid 0 \ + --speed 1.0 \ + --text 'Wie geht es dir? Das ist ein Text-zu-Sprache-Beispiel mit next generation Kaldi.' \ + --output-wav vits-de-0-phonemize.wav + +ls -lh *.wav diff --git a/dart-api-examples/tts/run-vits-zh.sh b/dart-api-examples/tts/run-vits-zh-lexicon.sh similarity index 89% rename from dart-api-examples/tts/run-vits-zh.sh rename to dart-api-examples/tts/run-vits-zh-lexicon.sh index d61202137..526188999 100755 --- a/dart-api-examples/tts/run-vits-zh.sh +++ b/dart-api-examples/tts/run-vits-zh-lexicon.sh @@ -16,17 +16,17 @@ if [[ ! -f ./sherpa-onnx-vits-zh-ll/tokens.txt ]]; then fi dart run \ - ./bin/vits-zh.dart \ + ./bin/vits-zh-lexicon.dart \ --model ./sherpa-onnx-vits-zh-ll/model.onnx \ --lexicon ./sherpa-onnx-vits-zh-ll/lexicon.txt \ --tokens ./sherpa-onnx-vits-zh-ll/tokens.txt \ --sid 2 \ --speed 1.0 \ --text '当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。' \ - --output-wav vits-zh-jieba-2.wav + --output-wav vits-zh-jieba-2-lexicon.wav dart run \ - ./bin/vits-zh.dart \ + ./bin/vits-zh-lexicon.dart \ --model ./sherpa-onnx-vits-zh-ll/model.onnx \ --lexicon ./sherpa-onnx-vits-zh-ll/lexicon.txt \ --tokens ./sherpa-onnx-vits-zh-ll/tokens.txt \ @@ -34,6 +34,6 @@ dart run \ --sid 3 \ --speed 1.0 \ --text '今天是2024年6月15号，13点23分。如果有困难，请拨打110或者18920240511。123456块钱。' \ - --output-wav vits-zh-jieba-3.wav + --output-wav vits-zh-jieba-3-lexicon.wav ls -lh *.wav diff --git a/dart-api-examples/tts/run-zipvoice-zh-en.sh b/dart-api-examples/tts/run-zipvoice-zh-en-lexicon.sh similarity index 90% rename from dart-api-examples/tts/run-zipvoice-zh-en.sh rename to dart-api-examples/tts/run-zipvoice-zh-en-lexicon.sh index e5dec9cc0..e309c4470 100755 --- a/dart-api-examples/tts/run-zipvoice-zh-en.sh +++ b/dart-api-examples/tts/run-zipvoice-zh-en-lexicon.sh @@ -18,17 +18,16 @@ if [ ! -f ./vocos_24khz.onnx ]; then fi dart run \ - ./bin/zipvoice-zh-en.dart \ + ./bin/zipvoice-zh-en-lexicon.dart \ --tokens ./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt \ --encoder ./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx \ --decoder ./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx \ --vocoder ./vocos_24khz.onnx \ - --data-dir ./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data \ --lexicon ./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt \ --reference-audio ./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/test_wavs/leijun-1.wav \ --reference-text "那还是三十六年前, 一九八七年. 我呢考上了武汉大学的计算机系." \ --num-steps 4 \ - --output-wav zipvoice-zh-en-0.wav \ + --output-wav zipvoice-zh-en-0-lexicon.wav \ --text "小米的价值观是真诚, 热爱. 真诚，就是不欺人也不自欺. 热爱, 就是全心投入并享受其中." ls -lh *.wav diff --git a/dotnet-examples/README.md b/dotnet-examples/README.md index bb0d7643b..8fd386afa 100644 --- a/dotnet-examples/README.md +++ b/dotnet-examples/README.md @@ -17,14 +17,20 @@ for details. It shows how to use the online speech denoiser API with GTCRN models. - [./streaming-speech-enhancement-dpdfnet](./streaming-speech-enhancement-dpdfnet) It shows how to use the online speech denoiser API with DPDFNet models. -- [./zipvoice-tts](./zipvoice-tts) It shows how to use ZipVoice for - Chinese/English zero-shot text-to-speech. -- [./zipvoice-tts-play](./zipvoice-tts-play) It shows how to use ZipVoice for - Chinese/English zero-shot text-to-speech with playback. +- [./zipvoice-tts-lexicon](./zipvoice-tts-lexicon) It shows how to use ZipVoice for + Chinese/English zero-shot text-to-speech with the lexicon text frontend. +- [./zipvoice-tts-play-lexicon](./zipvoice-tts-play-lexicon) It shows how to use + ZipVoice for Chinese/English zero-shot text-to-speech with playback. + +The TTS examples come in two text-frontend flavors: +- `*-lexicon` — a lexicon file, no external phonemizer +- `*-phonemize` — the text is phonemized outside of sherpa-onnx with + [piper-phonemize](https://www.nuget.org/packages/com.github.csukuangfj.piper.phonemize) + (see `./offline-tts-phonemize`, `./kitten-tts-phonemize`, `./kokoro-tts-phonemize`) ```bash -dotnet new console -n offline-tts-play -dotnet sln ./sherpa-onnx.sln add ./offline-tts-play +dotnet new console -n offline-tts-play-lexicon +dotnet sln ./sherpa-onnx.sln add ./offline-tts-play-lexicon ``` ```bash diff --git a/dotnet-examples/kitten-tts/Program.cs b/dotnet-examples/kitten-tts-lexicon/Program.cs similarity index 92% rename from dotnet-examples/kitten-tts/Program.cs rename to dotnet-examples/kitten-tts-lexicon/Program.cs index 0a8cae7a6..56e5b0acb 100644 --- a/dotnet-examples/kitten-tts/Program.cs +++ b/dotnet-examples/kitten-tts-lexicon/Program.cs @@ -1,81 +1,81 @@ -// Copyright (c) 2025 Xiaomi Corporation -// -// This file shows how to use a non-streaming KittenTTS model -// for text-to-speech -// Please refer to -// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html -// and -// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models -// to download pre-trained models -using SherpaOnnx; -using System.Runtime.InteropServices; - -class KittenTtsDemo -{ - static void Main(string[] args) - { - - TestEn(); - } - - static void TestEn() - { - var config = new OfflineTtsConfig(); - config.Model.Kitten.Model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; - config.Model.Kitten.Voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; - config.Model.Kitten.Tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; - config.Model.Kitten.DataDir = "./kitten-nano-en-v0_1-fp16/espeak-ng-data"; - - config.Model.NumThreads = 2; - config.Model.Debug = 1; - config.Model.Provider = "cpu"; - - var tts = new OfflineTts(config); - var speed = 1.0f; - var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + - "does not have two-thirds of his day for himself, is a slave, whatever " + - "he may be: a statesman, a businessman, an official, or a scholar. " + - "Friends fell out often because life was changing so fast. The easiest " + - "thing in the world was to lose touch with someone."; - - // mapping of sid to voice name - // 0->expr-voice-2-m, 1->expr-voice-2-f, 2->expr-voice-3-m - // 3->expr-voice-3-f, 4->expr-voice-4-m, 5->expr-voice-4-f - // 6->expr-voice-5-m, 7->expr-voice-5-f - var sid = 0; - +// Copyright (c) 2025 Xiaomi Corporation +// +// This file shows how to use a non-streaming KittenTTS model +// for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +using SherpaOnnx; +using System.Runtime.InteropServices; + +class KittenTtsDemo +{ + static void Main(string[] args) + { + + TestEn(); + } + + static void TestEn() + { + var config = new OfflineTtsConfig(); + config.Model.Kitten.Model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; + config.Model.Kitten.Voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; + config.Model.Kitten.Tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; + config.Model.Kitten.Lexicon = "./lexicon-en-us.txt"; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + // mapping of sid to voice name + // 0->expr-voice-2-m, 1->expr-voice-2-f, 2->expr-voice-3-m + // 3->expr-voice-3-f, 4->expr-voice-4-m, 5->expr-voice-4-f + // 6->expr-voice-5-m, 7->expr-voice-5-f + var sid = 0; + OfflineTtsGenerationConfig genConfig = new OfflineTtsGenerationConfig(); genConfig.Sid = sid; genConfig.Speed = speed; var MyCallback = (IntPtr samples, int n, float progress, IntPtr arg) => - { - float[] data = new float[n]; - Marshal.Copy(samples, data, 0, n); - // You can process samples here, e.g., play them. - // See ../kitten-tts-play for how to play them - Console.WriteLine($"Progress {progress*100}%"); - - // 1 means to keep generating - // 0 means to stop generating - return 1; - }; - + { + float[] data = new float[n]; + Marshal.Copy(samples, data, 0, n); + // You can process samples here, e.g., play them. + // See ../kitten-tts-play for how to play them + Console.WriteLine($"Progress {progress*100}%"); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + }; + var callback = new OfflineTtsCallbackProgressWithArg(MyCallback); var audio = tts.GenerateWithConfig(text, genConfig, callback); - - var outputFilename = "./generated-kitten-en.wav"; - var ok = audio.SaveToWaveFile(outputFilename); - - if (ok) - { - Console.WriteLine($"Wrote to {outputFilename} succeeded!"); - } - else - { - Console.WriteLine($"Failed to write {outputFilename}"); - } - } -} - + + var outputFilename = "./generated-kitten-en-lexicon.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {outputFilename}"); + } + } +} + diff --git a/dotnet-examples/kitten-tts/kitten-tts.csproj b/dotnet-examples/kitten-tts-lexicon/kitten-tts-lexicon.csproj similarity index 100% rename from dotnet-examples/kitten-tts/kitten-tts.csproj rename to dotnet-examples/kitten-tts-lexicon/kitten-tts-lexicon.csproj diff --git a/rust-api-examples/run-kitten-tts-en.sh b/dotnet-examples/kitten-tts-lexicon/run-kitten-lexicon.sh similarity index 65% rename from rust-api-examples/run-kitten-tts-en.sh rename to dotnet-examples/kitten-tts-lexicon/run-kitten-lexicon.sh index 5ef675f02..6a2646542 100755 --- a/rust-api-examples/run-kitten-tts-en.sh +++ b/dotnet-examples/kitten-tts-lexicon/run-kitten-lexicon.sh @@ -7,4 +7,9 @@ if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then rm kitten-nano-en-v0_1-fp16.tar.bz2 fi -cargo run --example kitten_tts_en + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +dotnet run diff --git a/dotnet-examples/kitten-tts-phonemize/Program.cs b/dotnet-examples/kitten-tts-phonemize/Program.cs new file mode 100644 index 000000000..f323433c7 --- /dev/null +++ b/dotnet-examples/kitten-tts-phonemize/Program.cs @@ -0,0 +1,139 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file shows how to use Kitten TTS with the piper-phonemize text +// frontend. The text is phonemized outside of sherpa-onnx with the +// com.github.csukuangfj.piper.phonemize NuGet package and the resulting +// phoneme codepoints are passed via OfflineTtsGenerationConfig.PhonemeCodepoints. +// No lexicon file is needed. +// +// The C# counterpart of +// https://github.com/k2-fsa/sherpa-onnx/blob/master/python-api-examples/test-offline-tts-kitten-phonemize.py +using System; +using System.Runtime.InteropServices; +using PiperPhonemize; +using SherpaOnnx; + +class KittenTtsPhonemizeDemo +{ + static void Main(string[] args) + { + Console.WriteLine($"piper-phonemize version: {PiperPhonemizeApi.GetVersionStr()}"); + + string dataDir = FindEspeakNgData(); + if (dataDir == null) + { + Console.Error.WriteLine("ERROR: espeak-ng-data not found."); + Environment.Exit(1); + } + Console.WriteLine($"espeak-ng-data: {dataDir}"); + + // Note: espeak-ng-data is only used by the piper-phonemize phonemizer. + // sherpa-onnx itself does not use espeak-ng anymore. + int sampleRate = PiperPhonemizeApi.Initialize(dataDir); + if (sampleRate < 0) + { + Console.Error.WriteLine("Failed to initialize piper-phonemize"); + Environment.Exit(1); + } + Console.WriteLine($"Sample rate: {sampleRate}"); + + var config = new OfflineTtsConfig(); + config.Model.Kitten.Model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; + config.Model.Kitten.Voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; + config.Model.Kitten.Tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; + config.Model.Kitten.LengthScale = 1.0F; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + // mapping of sid to voice name + // 0->expr-voice-2-m, 1->expr-voice-2-f, 2->expr-voice-3-m + // 3->expr-voice-3-f, 4->expr-voice-4-m, 5->expr-voice-4-f + // 6->expr-voice-5-m, 7->expr-voice-5-f + var sid = 0; + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + string lang = tts.Lang; + if (string.IsNullOrEmpty(lang)) + { + Console.Error.WriteLine("Warning: the model has no language information. Falling back to en-us."); + lang = "en-us"; + } + Console.WriteLine($"Phonemizer language: {lang}"); + + int[][] phonemeCodepoints; + using (var result = PiperPhonemizeApi.Text(text, lang)) + { + if (result == null) + { + Console.Error.WriteLine("Phonemization failed."); + Environment.Exit(1); + } + + phonemeCodepoints = new int[result.NumSentences][]; + for (int i = 0; i < result.NumSentences; i++) + { + string ipa = result.GetPhonemesAsString(i) ?? ""; + int[] cps = new int[ipa.Length]; + for (int j = 0; j < ipa.Length; j++) + cps[j] = ipa[j]; + phonemeCodepoints[i] = cps; + } + } + Console.WriteLine($"Phonemized sentences: {phonemeCodepoints.Length}"); + + OfflineTtsGenerationConfig genConfig = new OfflineTtsGenerationConfig(); + genConfig.Sid = sid; + genConfig.Speed = speed; + genConfig.PhonemeCodepoints = phonemeCodepoints; + + // The text has already been phonemized; pass an empty string here. + var audio = tts.GenerateWithConfig("", genConfig, null); + + var outputFilename = "./generated-kitten-en-phonemize.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to save {outputFilename}"); + } + } + + // The piper-phonemize NuGet package copies espeak-ng-data next to the + // executable (see its .targets file). espeak_Initialize requires a real + // directory: an empty path falls back to the system default + // (/usr/share/espeak-ng-data), which may not exist. + static string FindEspeakNgData() + { + string exeDir = AppDomain.CurrentDomain.BaseDirectory; + + string[] candidates = new[] + { + Path.Combine(exeDir, "espeak-ng-data"), + Path.Combine(exeDir, "..", "espeak-ng-data"), + Path.Combine(exeDir, "..", "..", "..", "espeak-ng-data"), + "espeak-ng-data", + }; + + foreach (var candidate in candidates) + { + if (Directory.Exists(candidate) && File.Exists(Path.Combine(candidate, "phontab"))) + return Path.GetFullPath(candidate); + } + + return null; + } +} diff --git a/dotnet-examples/kitten-tts-phonemize/kitten-tts-phonemize.csproj b/dotnet-examples/kitten-tts-phonemize/kitten-tts-phonemize.csproj new file mode 100644 index 000000000..5f626f7c5 --- /dev/null +++ b/dotnet-examples/kitten-tts-phonemize/kitten-tts-phonemize.csproj @@ -0,0 +1,16 @@ + + + + Exe + net8.0 + kitten_tts_phonemize + enable + enable + + + + + + + + diff --git a/dotnet-examples/kitten-tts-play/run-kitten.sh b/dotnet-examples/kitten-tts-phonemize/run-kitten-phonemize.sh similarity index 99% rename from dotnet-examples/kitten-tts-play/run-kitten.sh rename to dotnet-examples/kitten-tts-phonemize/run-kitten-phonemize.sh index 72a912fff..a1da2cd74 100755 --- a/dotnet-examples/kitten-tts-play/run-kitten.sh +++ b/dotnet-examples/kitten-tts-phonemize/run-kitten-phonemize.sh @@ -7,5 +7,4 @@ if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then rm kitten-nano-en-v0_1-fp16.tar.bz2 fi - dotnet run diff --git a/dotnet-examples/kitten-tts-play/Program.cs b/dotnet-examples/kitten-tts-play-lexicon/Program.cs similarity index 95% rename from dotnet-examples/kitten-tts-play/Program.cs rename to dotnet-examples/kitten-tts-play-lexicon/Program.cs index 26871c8c8..c43ce3f16 100644 --- a/dotnet-examples/kitten-tts-play/Program.cs +++ b/dotnet-examples/kitten-tts-play-lexicon/Program.cs @@ -1,40 +1,40 @@ -// Copyright (c) 2025 Xiaomi Corporation -// -// This file shows how to use a non-streaming Kitten TTS model -// for text-to-speech -// Please refer to -// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html -// and -// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models -// to download pre-trained models -using PortAudioSharp; -using SherpaOnnx; -using System.Collections.Concurrent; -using System.Runtime.InteropServices; - -class KittenTtsPlayDemo -{ - static void Main(string[] args) - { - var config = new OfflineTtsConfig(); - config.Model.Kitten.Model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; - config.Model.Kitten.Voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; - config.Model.Kitten.Tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; - config.Model.Kitten.DataDir = "./kitten-nano-en-v0_1-fp16/espeak-ng-data"; - - config.Model.NumThreads = 2; - config.Model.Debug = 1; - config.Model.Provider = "cpu"; - - var tts = new OfflineTts(config); - var speed = 1.0f; - var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + - "does not have two-thirds of his day for himself, is a slave, whatever " + - "he may be: a statesman, a businessman, an official, or a scholar. " + - "Friends fell out often because life was changing so fast. The easiest " + - "thing in the world was to lose touch with someone."; - - // mapping of sid to voice name +// Copyright (c) 2025 Xiaomi Corporation +// +// This file shows how to use a non-streaming Kitten TTS model +// for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +using PortAudioSharp; +using SherpaOnnx; +using System.Collections.Concurrent; +using System.Runtime.InteropServices; + +class KittenTtsPlayDemo +{ + static void Main(string[] args) + { + var config = new OfflineTtsConfig(); + config.Model.Kitten.Model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; + config.Model.Kitten.Voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; + config.Model.Kitten.Tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; + config.Model.Kitten.Lexicon = "./lexicon-en-us.txt"; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + // mapping of sid to voice name // 0->expr-voice-2-m, 1->expr-voice-2-f, 2->expr-voice-3-m // 3->expr-voice-3-f, 4->expr-voice-4-m, 5->expr-voice-4-f // 6->expr-voice-5-m, 7->expr-voice-5-f @@ -42,152 +42,152 @@ class KittenTtsPlayDemo OfflineTtsGenerationConfig genConfig = new OfflineTtsGenerationConfig(); genConfig.Sid = sid; genConfig.Speed = speed; - - - Console.WriteLine(PortAudio.VersionInfo.versionText); - PortAudio.Initialize(); - Console.WriteLine($"Number of devices: {PortAudio.DeviceCount}"); - - for (int i = 0; i != PortAudio.DeviceCount; ++i) - { - Console.WriteLine($" Device {i}"); - DeviceInfo deviceInfo = PortAudio.GetDeviceInfo(i); - Console.WriteLine($" Name: {deviceInfo.name}"); - Console.WriteLine($" Max output channels: {deviceInfo.maxOutputChannels}"); - Console.WriteLine($" Default sample rate: {deviceInfo.defaultSampleRate}"); - } - int deviceIndex = PortAudio.DefaultOutputDevice; - if (deviceIndex == PortAudio.NoDevice) - { - Console.WriteLine("No default output device found. Please use ../offline-tts instead"); - Environment.Exit(1); - } - - var info = PortAudio.GetDeviceInfo(deviceIndex); - Console.WriteLine(); - Console.WriteLine($"Use output default device {deviceIndex} ({info.name})"); - - var param = new StreamParameters(); - param.device = deviceIndex; - param.channelCount = 1; - param.sampleFormat = SampleFormat.Float32; - param.suggestedLatency = info.defaultLowOutputLatency; - param.hostApiSpecificStreamInfo = IntPtr.Zero; - - // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview - var dataItems = new BlockingCollection(); - + + + Console.WriteLine(PortAudio.VersionInfo.versionText); + PortAudio.Initialize(); + Console.WriteLine($"Number of devices: {PortAudio.DeviceCount}"); + + for (int i = 0; i != PortAudio.DeviceCount; ++i) + { + Console.WriteLine($" Device {i}"); + DeviceInfo deviceInfo = PortAudio.GetDeviceInfo(i); + Console.WriteLine($" Name: {deviceInfo.name}"); + Console.WriteLine($" Max output channels: {deviceInfo.maxOutputChannels}"); + Console.WriteLine($" Default sample rate: {deviceInfo.defaultSampleRate}"); + } + int deviceIndex = PortAudio.DefaultOutputDevice; + if (deviceIndex == PortAudio.NoDevice) + { + Console.WriteLine("No default output device found. Please use ../offline-tts instead"); + Environment.Exit(1); + } + + var info = PortAudio.GetDeviceInfo(deviceIndex); + Console.WriteLine(); + Console.WriteLine($"Use output default device {deviceIndex} ({info.name})"); + + var param = new StreamParameters(); + param.device = deviceIndex; + param.channelCount = 1; + param.sampleFormat = SampleFormat.Float32; + param.suggestedLatency = info.defaultLowOutputLatency; + param.hostApiSpecificStreamInfo = IntPtr.Zero; + + // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview + var dataItems = new BlockingCollection(); + var myCallback = (IntPtr samples, int n, float progress, IntPtr arg) => - { - Console.WriteLine($"Progress {progress*100}%"); - - float[] data = new float[n]; - - Marshal.Copy(samples, data, 0, n); - - dataItems.Add(data); - - // 1 means to keep generating - // 0 means to stop generating - return 1; - }; - - var playFinished = false; - - float[]? lastSampleArray = null; - int lastIndex = 0; // not played - - PortAudioSharp.Stream.Callback playCallback = (IntPtr input, IntPtr output, - UInt32 frameCount, - ref StreamCallbackTimeInfo timeInfo, - StreamCallbackFlags statusFlags, - IntPtr userData - ) => - { - if (dataItems.IsCompleted && lastSampleArray == null && lastIndex == 0) - { - Console.WriteLine($"Finished playing"); - playFinished = true; - return StreamCallbackResult.Complete; - } - - int expected = Convert.ToInt32(frameCount); - int i = 0; - - while ((lastSampleArray != null || dataItems.Count != 0) && (i < expected)) - { - int needed = expected - i; - - if (lastSampleArray != null) - { - int remaining = lastSampleArray.Length - lastIndex; - if (remaining >= needed) - { - float[] this_block = lastSampleArray.Skip(lastIndex).Take(needed).ToArray(); - lastIndex += needed; - if (lastIndex == lastSampleArray.Length) - { - lastSampleArray = null; - lastIndex = 0; - } - - Marshal.Copy(this_block, 0, IntPtr.Add(output, i * sizeof(float)), needed); - return StreamCallbackResult.Continue; - } - - float[] this_block2 = lastSampleArray.Skip(lastIndex).Take(remaining).ToArray(); - lastIndex = 0; - lastSampleArray = null; - - Marshal.Copy(this_block2, 0, IntPtr.Add(output, i * sizeof(float)), remaining); - i += remaining; - continue; - } - - if (dataItems.Count != 0) - { - lastSampleArray = dataItems.Take(); - lastIndex = 0; - } - } - - if (i < expected) - { - int sizeInBytes = (expected - i) * 4; - Marshal.Copy(new byte[sizeInBytes], 0, IntPtr.Add(output, i * sizeof(float)), sizeInBytes); - } - - return StreamCallbackResult.Continue; - }; - - PortAudioSharp.Stream stream = new PortAudioSharp.Stream(inParams: null, outParams: param, sampleRate: tts.SampleRate, - framesPerBuffer: 0, - streamFlags: StreamFlags.ClipOff, - callback: playCallback, - userData: IntPtr.Zero - ); - - stream.Start(); - + { + Console.WriteLine($"Progress {progress*100}%"); + + float[] data = new float[n]; + + Marshal.Copy(samples, data, 0, n); + + dataItems.Add(data); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + }; + + var playFinished = false; + + float[]? lastSampleArray = null; + int lastIndex = 0; // not played + + PortAudioSharp.Stream.Callback playCallback = (IntPtr input, IntPtr output, + UInt32 frameCount, + ref StreamCallbackTimeInfo timeInfo, + StreamCallbackFlags statusFlags, + IntPtr userData + ) => + { + if (dataItems.IsCompleted && lastSampleArray == null && lastIndex == 0) + { + Console.WriteLine($"Finished playing"); + playFinished = true; + return StreamCallbackResult.Complete; + } + + int expected = Convert.ToInt32(frameCount); + int i = 0; + + while ((lastSampleArray != null || dataItems.Count != 0) && (i < expected)) + { + int needed = expected - i; + + if (lastSampleArray != null) + { + int remaining = lastSampleArray.Length - lastIndex; + if (remaining >= needed) + { + float[] this_block = lastSampleArray.Skip(lastIndex).Take(needed).ToArray(); + lastIndex += needed; + if (lastIndex == lastSampleArray.Length) + { + lastSampleArray = null; + lastIndex = 0; + } + + Marshal.Copy(this_block, 0, IntPtr.Add(output, i * sizeof(float)), needed); + return StreamCallbackResult.Continue; + } + + float[] this_block2 = lastSampleArray.Skip(lastIndex).Take(remaining).ToArray(); + lastIndex = 0; + lastSampleArray = null; + + Marshal.Copy(this_block2, 0, IntPtr.Add(output, i * sizeof(float)), remaining); + i += remaining; + continue; + } + + if (dataItems.Count != 0) + { + lastSampleArray = dataItems.Take(); + lastIndex = 0; + } + } + + if (i < expected) + { + int sizeInBytes = (expected - i) * 4; + Marshal.Copy(new byte[sizeInBytes], 0, IntPtr.Add(output, i * sizeof(float)), sizeInBytes); + } + + return StreamCallbackResult.Continue; + }; + + PortAudioSharp.Stream stream = new PortAudioSharp.Stream(inParams: null, outParams: param, sampleRate: tts.SampleRate, + framesPerBuffer: 0, + streamFlags: StreamFlags.ClipOff, + callback: playCallback, + userData: IntPtr.Zero + ); + + stream.Start(); + var callback = new OfflineTtsCallbackProgressWithArg(myCallback); var audio = tts.GenerateWithConfig(text, genConfig, callback); - var outputFilename = "./generated-kitten-0.wav"; - var ok = audio.SaveToWaveFile(outputFilename); - - if (ok) - { - Console.WriteLine($"Wrote to {outputFilename} succeeded!"); - } - else - { - Console.WriteLine($"Failed to write {outputFilename}"); - } - dataItems.CompleteAdding(); - - while (!playFinished) - { - Thread.Sleep(100); // 100ms - } - } -} + var outputFilename = "./generated-kitten-0-lexicon.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {outputFilename}"); + } + dataItems.CompleteAdding(); + + while (!playFinished) + { + Thread.Sleep(100); // 100ms + } + } +} diff --git a/dotnet-examples/kitten-tts-play/kitten-tts-play.csproj b/dotnet-examples/kitten-tts-play-lexicon/kitten-tts-play-lexicon.csproj similarity index 100% rename from dotnet-examples/kitten-tts-play/kitten-tts-play.csproj rename to dotnet-examples/kitten-tts-play-lexicon/kitten-tts-play-lexicon.csproj diff --git a/dotnet-examples/kitten-tts-play-lexicon/run-kitten-lexicon.sh b/dotnet-examples/kitten-tts-play-lexicon/run-kitten-lexicon.sh new file mode 100755 index 000000000..6a2646542 --- /dev/null +++ b/dotnet-examples/kitten-tts-play-lexicon/run-kitten-lexicon.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 + tar xf kitten-nano-en-v0_1-fp16.tar.bz2 + rm kitten-nano-en-v0_1-fp16.tar.bz2 +fi + + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +dotnet run diff --git a/dotnet-examples/kokoro-tts/Program.cs b/dotnet-examples/kokoro-tts-lexicon/Program.cs similarity index 91% rename from dotnet-examples/kokoro-tts/Program.cs rename to dotnet-examples/kokoro-tts-lexicon/Program.cs index ee56762c4..50506a262 100644 --- a/dotnet-examples/kokoro-tts/Program.cs +++ b/dotnet-examples/kokoro-tts-lexicon/Program.cs @@ -1,136 +1,135 @@ -// Copyright (c) 2025 Xiaomi Corporation -// -// This file shows how to use a non-streaming Kokoro TTS model -// for text-to-speech -// Please refer to -// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html -// and -// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models -// to download pre-trained models -using SherpaOnnx; -using System.Runtime.InteropServices; - -class KokoroTtsDemo -{ - static void Main(string[] args) - { - - TestZhEn(); - TestEn(); - } - - static void TestZhEn() - { - var config = new OfflineTtsConfig(); - config.Model.Kokoro.Model = "./kokoro-multi-lang-v1_0/model.onnx"; - config.Model.Kokoro.Voices = "./kokoro-multi-lang-v1_0/voices.bin"; - config.Model.Kokoro.Tokens = "./kokoro-multi-lang-v1_0/tokens.txt"; - config.Model.Kokoro.DataDir = "./kokoro-multi-lang-v1_0/espeak-ng-data"; - config.Model.Kokoro.Lexicon = "./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt"; - - config.Model.NumThreads = 2; - config.Model.Debug = 1; - config.Model.Provider = "cpu"; - - var tts = new OfflineTts(config); - var speed = 1.0f; - var text = "中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？"; - - var sid = 50; - +// Copyright (c) 2025 Xiaomi Corporation +// +// This file shows how to use a non-streaming Kokoro TTS model +// for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +using SherpaOnnx; +using System.Runtime.InteropServices; + +class KokoroTtsDemo +{ + static void Main(string[] args) + { + + TestZhEn(); + TestEn(); + } + + static void TestZhEn() + { + var config = new OfflineTtsConfig(); + config.Model.Kokoro.Model = "./kokoro-multi-lang-v1_0/model.onnx"; + config.Model.Kokoro.Voices = "./kokoro-multi-lang-v1_0/voices.bin"; + config.Model.Kokoro.Tokens = "./kokoro-multi-lang-v1_0/tokens.txt"; + config.Model.Kokoro.Lexicon = "./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt"; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？"; + + var sid = 50; + OfflineTtsGenerationConfig genConfig = new OfflineTtsGenerationConfig(); genConfig.Sid = sid; genConfig.Speed = speed; genConfig.SilenceScale = 0.2f; var MyCallback = (IntPtr samples, int n, float progress, IntPtr arg) => - { - float[] data = new float[n]; - Marshal.Copy(samples, data, 0, n); - // You can process samples here, e.g., play them. - // See ../kokoro-tts-playback for how to play them - Console.WriteLine($"Progress {progress*100}%"); - - // 1 means to keep generating - // 0 means to stop generating - return 1; - }; - + { + float[] data = new float[n]; + Marshal.Copy(samples, data, 0, n); + // You can process samples here, e.g., play them. + // See ../kokoro-tts-playback for how to play them + Console.WriteLine($"Progress {progress*100}%"); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + }; + var callback = new OfflineTtsCallbackProgressWithArg(MyCallback); var audio = tts.GenerateWithConfig(text, genConfig, callback); - - var outputFilename = "./generated-kokoro-zh-en.wav"; - var ok = audio.SaveToWaveFile(outputFilename); - - if (ok) - { - Console.WriteLine($"Wrote to {outputFilename} succeeded!"); - } - else - { - Console.WriteLine($"Failed to write {outputFilename}"); - } - } - - static void TestEn() - { - var config = new OfflineTtsConfig(); - config.Model.Kokoro.Model = "./kokoro-en-v0_19/model.onnx"; - config.Model.Kokoro.Voices = "./kokoro-en-v0_19/voices.bin"; - config.Model.Kokoro.Tokens = "./kokoro-en-v0_19/tokens.txt"; - config.Model.Kokoro.DataDir = "./kokoro-en-v0_19/espeak-ng-data"; - - config.Model.NumThreads = 2; - config.Model.Debug = 1; - config.Model.Provider = "cpu"; - - var tts = new OfflineTts(config); - var speed = 1.0f; - var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + - "does not have two-thirds of his day for himself, is a slave, whatever " + - "he may be: a statesman, a businessman, an official, or a scholar. " + - "Friends fell out often because life was changing so fast. The easiest " + - "thing in the world was to lose touch with someone."; - - // mapping of sid to voice name - // 0->af, 1->af_bella, 2->af_nicole, 3->af_sarah, 4->af_sky, 5->am_adam - // 6->am_michael, 7->bf_emma, 8->bf_isabella, 9->bm_george, 10->bm_lewis - var sid = 0; - + + var outputFilename = "./generated-kokoro-zh-en-lexicon.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {outputFilename}"); + } + } + + static void TestEn() + { + var config = new OfflineTtsConfig(); + config.Model.Kokoro.Model = "./kokoro-en-v0_19/model.onnx"; + config.Model.Kokoro.Voices = "./kokoro-en-v0_19/voices.bin"; + config.Model.Kokoro.Tokens = "./kokoro-en-v0_19/tokens.txt"; + config.Model.Kokoro.Lexicon = "./lexicon-en-us.txt"; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + // mapping of sid to voice name + // 0->af, 1->af_bella, 2->af_nicole, 3->af_sarah, 4->af_sky, 5->am_adam + // 6->am_michael, 7->bf_emma, 8->bf_isabella, 9->bm_george, 10->bm_lewis + var sid = 0; + OfflineTtsGenerationConfig genConfig = new OfflineTtsGenerationConfig(); genConfig.Sid = sid; genConfig.Speed = speed; genConfig.SilenceScale = 0.2f; var MyCallback = (IntPtr samples, int n, float progress, IntPtr arg) => - { - float[] data = new float[n]; - Marshal.Copy(samples, data, 0, n); - // You can process samples here, e.g., play them. - // See ../kokoro-tts-playback for how to play them - Console.WriteLine($"Progress {progress*100}%"); - - // 1 means to keep generating - // 0 means to stop generating - return 1; - }; - + { + float[] data = new float[n]; + Marshal.Copy(samples, data, 0, n); + // You can process samples here, e.g., play them. + // See ../kokoro-tts-playback for how to play them + Console.WriteLine($"Progress {progress*100}%"); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + }; + var callback = new OfflineTtsCallbackProgressWithArg(MyCallback); var audio = tts.GenerateWithConfig(text, genConfig, callback); - - var outputFilename = "./generated-kokoro-en.wav"; - var ok = audio.SaveToWaveFile(outputFilename); - - if (ok) - { - Console.WriteLine($"Wrote to {outputFilename} succeeded!"); - } - else - { - Console.WriteLine($"Failed to write {outputFilename}"); - } - } -} - + + var outputFilename = "./generated-kokoro-en-lexicon.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {outputFilename}"); + } + } +} + diff --git a/dotnet-examples/kokoro-tts/kokoro-tts.csproj b/dotnet-examples/kokoro-tts-lexicon/kokoro-tts-lexicon.csproj similarity index 100% rename from dotnet-examples/kokoro-tts/kokoro-tts.csproj rename to dotnet-examples/kokoro-tts-lexicon/kokoro-tts-lexicon.csproj diff --git a/dotnet-examples/kokoro-tts/run-kokoro.sh b/dotnet-examples/kokoro-tts-lexicon/run-kokoro-lexicon.sh similarity index 78% rename from dotnet-examples/kokoro-tts/run-kokoro.sh rename to dotnet-examples/kokoro-tts-lexicon/run-kokoro-lexicon.sh index 117a2fc1b..993aa100f 100755 --- a/dotnet-examples/kokoro-tts/run-kokoro.sh +++ b/dotnet-examples/kokoro-tts-lexicon/run-kokoro-lexicon.sh @@ -13,4 +13,8 @@ if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then rm kokoro-en-v0_19.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + dotnet run diff --git a/dotnet-examples/kokoro-tts-phonemize/Program.cs b/dotnet-examples/kokoro-tts-phonemize/Program.cs new file mode 100644 index 000000000..15d3503ba --- /dev/null +++ b/dotnet-examples/kokoro-tts-phonemize/Program.cs @@ -0,0 +1,132 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file shows how to use Kokoro TTS (English) with the piper-phonemize +// text frontend. The text is phonemized outside of sherpa-onnx with the +// com.github.csukuangfj.piper.phonemize NuGet package and the resulting +// phoneme codepoints are passed via OfflineTtsGenerationConfig.PhonemeCodepoints. +// No lexicon file is needed. +// +// The C# counterpart of +// https://github.com/k2-fsa/sherpa-onnx/blob/master/python-api-examples/test-offline-tts-kokoro-en-phonemize.py +using System; +using System.Runtime.InteropServices; +using PiperPhonemize; +using SherpaOnnx; + +class KokoroTtsPhonemizeDemo +{ + static void Main(string[] args) + { + Console.WriteLine($"piper-phonemize version: {PiperPhonemizeApi.GetVersionStr()}"); + + string dataDir = FindEspeakNgData(); + if (dataDir == null) + { + Console.Error.WriteLine("ERROR: espeak-ng-data not found."); + Environment.Exit(1); + } + Console.WriteLine($"espeak-ng-data: {dataDir}"); + + // Note: espeak-ng-data is only used by the piper-phonemize phonemizer. + // sherpa-onnx itself does not use espeak-ng anymore. + int sampleRate = PiperPhonemizeApi.Initialize(dataDir); + if (sampleRate < 0) + { + Console.Error.WriteLine("Failed to initialize piper-phonemize"); + Environment.Exit(1); + } + Console.WriteLine($"Sample rate: {sampleRate}"); + + var config = new OfflineTtsConfig(); + config.Model.Kokoro.Model = "./kokoro-en-v0_19/model.onnx"; + config.Model.Kokoro.Voices = "./kokoro-en-v0_19/voices.bin"; + config.Model.Kokoro.Tokens = "./kokoro-en-v0_19/tokens.txt"; + config.Model.Kokoro.LengthScale = 1.0F; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + string lang = tts.Lang; + if (string.IsNullOrEmpty(lang)) + { + Console.Error.WriteLine("Warning: the model has no language information. Falling back to en-us."); + lang = "en-us"; + } + Console.WriteLine($"Phonemizer language: {lang}"); + + int[][] phonemeCodepoints; + using (var result = PiperPhonemizeApi.Text(text, lang)) + { + if (result == null) + { + Console.Error.WriteLine("Phonemization failed."); + Environment.Exit(1); + } + + phonemeCodepoints = new int[result.NumSentences][]; + for (int i = 0; i < result.NumSentences; i++) + { + string ipa = result.GetPhonemesAsString(i) ?? ""; + int[] cps = new int[ipa.Length]; + for (int j = 0; j < ipa.Length; j++) + cps[j] = ipa[j]; + phonemeCodepoints[i] = cps; + } + } + Console.WriteLine($"Phonemized sentences: {phonemeCodepoints.Length}"); + + OfflineTtsGenerationConfig genConfig = new OfflineTtsGenerationConfig(); + genConfig.Speed = speed; + genConfig.PhonemeCodepoints = phonemeCodepoints; + + // The text has already been phonemized; pass an empty string here. + var audio = tts.GenerateWithConfig("", genConfig, null); + + var outputFilename = "./generated-kokoro-en-phonemize.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to save {outputFilename}"); + } + } + + // The piper-phonemize NuGet package copies espeak-ng-data next to the + // executable (see its .targets file). espeak_Initialize requires a real + // directory: an empty path falls back to the system default + // (/usr/share/espeak-ng-data), which may not exist. + static string FindEspeakNgData() + { + string exeDir = AppDomain.CurrentDomain.BaseDirectory; + + string[] candidates = new[] + { + Path.Combine(exeDir, "espeak-ng-data"), + Path.Combine(exeDir, "..", "espeak-ng-data"), + Path.Combine(exeDir, "..", "..", "..", "espeak-ng-data"), + "espeak-ng-data", + }; + + foreach (var candidate in candidates) + { + if (Directory.Exists(candidate) && File.Exists(Path.Combine(candidate, "phontab"))) + return Path.GetFullPath(candidate); + } + + return null; + } +} diff --git a/dotnet-examples/kokoro-tts-phonemize/kokoro-tts-phonemize.csproj b/dotnet-examples/kokoro-tts-phonemize/kokoro-tts-phonemize.csproj new file mode 100644 index 000000000..a1ea7cdad --- /dev/null +++ b/dotnet-examples/kokoro-tts-phonemize/kokoro-tts-phonemize.csproj @@ -0,0 +1,16 @@ + + + + Exe + net8.0 + kokoro_tts_phonemize + enable + enable + + + + + + + + diff --git a/dotnet-examples/kokoro-tts-play/run-kokoro-en.sh b/dotnet-examples/kokoro-tts-phonemize/run-kokoro-phonemize.sh similarity index 100% rename from dotnet-examples/kokoro-tts-play/run-kokoro-en.sh rename to dotnet-examples/kokoro-tts-phonemize/run-kokoro-phonemize.sh diff --git a/dotnet-examples/kokoro-tts-play/Program.cs b/dotnet-examples/kokoro-tts-play-lexicon/Program.cs similarity index 95% rename from dotnet-examples/kokoro-tts-play/Program.cs rename to dotnet-examples/kokoro-tts-play-lexicon/Program.cs index d968de193..a12ad857e 100644 --- a/dotnet-examples/kokoro-tts-play/Program.cs +++ b/dotnet-examples/kokoro-tts-play-lexicon/Program.cs @@ -1,40 +1,40 @@ -// Copyright (c) 2025 Xiaomi Corporation -// -// This file shows how to use a non-streaming Kokoro TTS model -// for text-to-speech -// Please refer to -// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html -// and -// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models -// to download pre-trained models -using PortAudioSharp; -using SherpaOnnx; -using System.Collections.Concurrent; -using System.Runtime.InteropServices; - -class KokoroTtsPlayDemo -{ - static void Main(string[] args) - { - var config = new OfflineTtsConfig(); - config.Model.Kokoro.Model = "./kokoro-en-v0_19/model.onnx"; - config.Model.Kokoro.Voices = "./kokoro-en-v0_19/voices.bin"; - config.Model.Kokoro.Tokens = "./kokoro-en-v0_19/tokens.txt"; - config.Model.Kokoro.DataDir = "./kokoro-en-v0_19/espeak-ng-data"; - - config.Model.NumThreads = 2; - config.Model.Debug = 1; - config.Model.Provider = "cpu"; - - var tts = new OfflineTts(config); - var speed = 1.0f; - var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + - "does not have two-thirds of his day for himself, is a slave, whatever " + - "he may be: a statesman, a businessman, an official, or a scholar. " + - "Friends fell out often because life was changing so fast. The easiest " + - "thing in the world was to lose touch with someone."; - - // mapping of sid to voice name +// Copyright (c) 2025 Xiaomi Corporation +// +// This file shows how to use a non-streaming Kokoro TTS model +// for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +using PortAudioSharp; +using SherpaOnnx; +using System.Collections.Concurrent; +using System.Runtime.InteropServices; + +class KokoroTtsPlayDemo +{ + static void Main(string[] args) + { + var config = new OfflineTtsConfig(); + config.Model.Kokoro.Model = "./kokoro-en-v0_19/model.onnx"; + config.Model.Kokoro.Voices = "./kokoro-en-v0_19/voices.bin"; + config.Model.Kokoro.Tokens = "./kokoro-en-v0_19/tokens.txt"; + config.Model.Kokoro.Lexicon = "./lexicon-en-us.txt"; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + // mapping of sid to voice name // 0->af, 1->af_bella, 2->af_nicole, 3->af_sarah, 4->af_sky, 5->am_adam // 6->am_michael, 7->bf_emma, 8->bf_isabella, 9->bm_george, 10->bm_lewis var sid = 0; @@ -42,152 +42,152 @@ class KokoroTtsPlayDemo genConfig.Sid = sid; genConfig.Speed = speed; genConfig.SilenceScale = 0.2f; - - - Console.WriteLine(PortAudio.VersionInfo.versionText); - PortAudio.Initialize(); - Console.WriteLine($"Number of devices: {PortAudio.DeviceCount}"); - - for (int i = 0; i != PortAudio.DeviceCount; ++i) - { - Console.WriteLine($" Device {i}"); - DeviceInfo deviceInfo = PortAudio.GetDeviceInfo(i); - Console.WriteLine($" Name: {deviceInfo.name}"); - Console.WriteLine($" Max output channels: {deviceInfo.maxOutputChannels}"); - Console.WriteLine($" Default sample rate: {deviceInfo.defaultSampleRate}"); - } - int deviceIndex = PortAudio.DefaultOutputDevice; - if (deviceIndex == PortAudio.NoDevice) - { - Console.WriteLine("No default output device found. Please use ../offline-tts instead"); - Environment.Exit(1); - } - - var info = PortAudio.GetDeviceInfo(deviceIndex); - Console.WriteLine(); - Console.WriteLine($"Use output default device {deviceIndex} ({info.name})"); - - var param = new StreamParameters(); - param.device = deviceIndex; - param.channelCount = 1; - param.sampleFormat = SampleFormat.Float32; - param.suggestedLatency = info.defaultLowOutputLatency; - param.hostApiSpecificStreamInfo = IntPtr.Zero; - - // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview - var dataItems = new BlockingCollection(); - + + + Console.WriteLine(PortAudio.VersionInfo.versionText); + PortAudio.Initialize(); + Console.WriteLine($"Number of devices: {PortAudio.DeviceCount}"); + + for (int i = 0; i != PortAudio.DeviceCount; ++i) + { + Console.WriteLine($" Device {i}"); + DeviceInfo deviceInfo = PortAudio.GetDeviceInfo(i); + Console.WriteLine($" Name: {deviceInfo.name}"); + Console.WriteLine($" Max output channels: {deviceInfo.maxOutputChannels}"); + Console.WriteLine($" Default sample rate: {deviceInfo.defaultSampleRate}"); + } + int deviceIndex = PortAudio.DefaultOutputDevice; + if (deviceIndex == PortAudio.NoDevice) + { + Console.WriteLine("No default output device found. Please use ../offline-tts instead"); + Environment.Exit(1); + } + + var info = PortAudio.GetDeviceInfo(deviceIndex); + Console.WriteLine(); + Console.WriteLine($"Use output default device {deviceIndex} ({info.name})"); + + var param = new StreamParameters(); + param.device = deviceIndex; + param.channelCount = 1; + param.sampleFormat = SampleFormat.Float32; + param.suggestedLatency = info.defaultLowOutputLatency; + param.hostApiSpecificStreamInfo = IntPtr.Zero; + + // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview + var dataItems = new BlockingCollection(); + var MyCallback = (IntPtr samples, int n, float progress, IntPtr arg) => - { - Console.WriteLine($"Progress {progress*100}%"); - - float[] data = new float[n]; - - Marshal.Copy(samples, data, 0, n); - - dataItems.Add(data); - - // 1 means to keep generating - // 0 means to stop generating - return 1; - }; - - var playFinished = false; - - float[]? lastSampleArray = null; - int lastIndex = 0; // not played - - PortAudioSharp.Stream.Callback playCallback = (IntPtr input, IntPtr output, - UInt32 frameCount, - ref StreamCallbackTimeInfo timeInfo, - StreamCallbackFlags statusFlags, - IntPtr userData - ) => - { - if (dataItems.IsCompleted && lastSampleArray == null && lastIndex == 0) - { - Console.WriteLine($"Finished playing"); - playFinished = true; - return StreamCallbackResult.Complete; - } - - int expected = Convert.ToInt32(frameCount); - int i = 0; - - while ((lastSampleArray != null || dataItems.Count != 0) && (i < expected)) - { - int needed = expected - i; - - if (lastSampleArray != null) - { - int remaining = lastSampleArray.Length - lastIndex; - if (remaining >= needed) - { - float[] this_block = lastSampleArray.Skip(lastIndex).Take(needed).ToArray(); - lastIndex += needed; - if (lastIndex == lastSampleArray.Length) - { - lastSampleArray = null; - lastIndex = 0; - } - - Marshal.Copy(this_block, 0, IntPtr.Add(output, i * sizeof(float)), needed); - return StreamCallbackResult.Continue; - } - - float[] this_block2 = lastSampleArray.Skip(lastIndex).Take(remaining).ToArray(); - lastIndex = 0; - lastSampleArray = null; - - Marshal.Copy(this_block2, 0, IntPtr.Add(output, i * sizeof(float)), remaining); - i += remaining; - continue; - } - - if (dataItems.Count != 0) - { - lastSampleArray = dataItems.Take(); - lastIndex = 0; - } - } - - if (i < expected) - { - int sizeInBytes = (expected - i) * 4; - Marshal.Copy(new byte[sizeInBytes], 0, IntPtr.Add(output, i * sizeof(float)), sizeInBytes); - } - - return StreamCallbackResult.Continue; - }; - - PortAudioSharp.Stream stream = new PortAudioSharp.Stream(inParams: null, outParams: param, sampleRate: tts.SampleRate, - framesPerBuffer: 0, - streamFlags: StreamFlags.ClipOff, - callback: playCallback, - userData: IntPtr.Zero - ); - - stream.Start(); - + { + Console.WriteLine($"Progress {progress*100}%"); + + float[] data = new float[n]; + + Marshal.Copy(samples, data, 0, n); + + dataItems.Add(data); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + }; + + var playFinished = false; + + float[]? lastSampleArray = null; + int lastIndex = 0; // not played + + PortAudioSharp.Stream.Callback playCallback = (IntPtr input, IntPtr output, + UInt32 frameCount, + ref StreamCallbackTimeInfo timeInfo, + StreamCallbackFlags statusFlags, + IntPtr userData + ) => + { + if (dataItems.IsCompleted && lastSampleArray == null && lastIndex == 0) + { + Console.WriteLine($"Finished playing"); + playFinished = true; + return StreamCallbackResult.Complete; + } + + int expected = Convert.ToInt32(frameCount); + int i = 0; + + while ((lastSampleArray != null || dataItems.Count != 0) && (i < expected)) + { + int needed = expected - i; + + if (lastSampleArray != null) + { + int remaining = lastSampleArray.Length - lastIndex; + if (remaining >= needed) + { + float[] this_block = lastSampleArray.Skip(lastIndex).Take(needed).ToArray(); + lastIndex += needed; + if (lastIndex == lastSampleArray.Length) + { + lastSampleArray = null; + lastIndex = 0; + } + + Marshal.Copy(this_block, 0, IntPtr.Add(output, i * sizeof(float)), needed); + return StreamCallbackResult.Continue; + } + + float[] this_block2 = lastSampleArray.Skip(lastIndex).Take(remaining).ToArray(); + lastIndex = 0; + lastSampleArray = null; + + Marshal.Copy(this_block2, 0, IntPtr.Add(output, i * sizeof(float)), remaining); + i += remaining; + continue; + } + + if (dataItems.Count != 0) + { + lastSampleArray = dataItems.Take(); + lastIndex = 0; + } + } + + if (i < expected) + { + int sizeInBytes = (expected - i) * 4; + Marshal.Copy(new byte[sizeInBytes], 0, IntPtr.Add(output, i * sizeof(float)), sizeInBytes); + } + + return StreamCallbackResult.Continue; + }; + + PortAudioSharp.Stream stream = new PortAudioSharp.Stream(inParams: null, outParams: param, sampleRate: tts.SampleRate, + framesPerBuffer: 0, + streamFlags: StreamFlags.ClipOff, + callback: playCallback, + userData: IntPtr.Zero + ); + + stream.Start(); + var callback = new OfflineTtsCallbackProgressWithArg(MyCallback); var audio = tts.GenerateWithConfig(text, genConfig, callback); - var outputFilename = "./generated-kokoro-0.wav"; - var ok = audio.SaveToWaveFile(outputFilename); - - if (ok) - { - Console.WriteLine($"Wrote to {outputFilename} succeeded!"); - } - else - { - Console.WriteLine($"Failed to write {outputFilename}"); - } - dataItems.CompleteAdding(); - - while (!playFinished) - { - Thread.Sleep(100); // 100ms - } - } -} + var outputFilename = "./generated-kokoro-0-lexicon.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {outputFilename}"); + } + dataItems.CompleteAdding(); + + while (!playFinished) + { + Thread.Sleep(100); // 100ms + } + } +} diff --git a/dotnet-examples/kokoro-tts-play/kokoro-tts-play.csproj b/dotnet-examples/kokoro-tts-play-lexicon/kokoro-tts-play-lexicon.csproj similarity index 100% rename from dotnet-examples/kokoro-tts-play/kokoro-tts-play.csproj rename to dotnet-examples/kokoro-tts-play-lexicon/kokoro-tts-play-lexicon.csproj diff --git a/dotnet-examples/kokoro-tts-play-lexicon/run-kokoro-en-lexicon.sh b/dotnet-examples/kokoro-tts-play-lexicon/run-kokoro-en-lexicon.sh new file mode 100755 index 000000000..c8a78612a --- /dev/null +++ b/dotnet-examples/kokoro-tts-play-lexicon/run-kokoro-en-lexicon.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +dotnet run diff --git a/dotnet-examples/offline-tts/Program.cs b/dotnet-examples/offline-tts-lexicon/Program.cs similarity index 89% rename from dotnet-examples/offline-tts/Program.cs rename to dotnet-examples/offline-tts-lexicon/Program.cs index 810615ee0..76cc53129 100644 --- a/dotnet-examples/offline-tts/Program.cs +++ b/dotnet-examples/offline-tts-lexicon/Program.cs @@ -1,184 +1,177 @@ -// Copyright (c) 2024 Xiaomi Corporation -// -// This file shows how to use a non-streaming TTS model for text-to-speech -// Please refer to -// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html -// and -// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models -// to download pre-trained models -using CommandLine; -using CommandLine.Text; -using SherpaOnnx; - -class OfflineTtsDemo -{ - class Options - { - [Option("tts-rule-fsts", Required = false, Default = "", HelpText = "path to rule.fst")] - public string RuleFsts { get; set; } = string.Empty; - - [Option("tts-rule-fars", Required = false, Default = "", HelpText = "path to rule.far")] - public string RuleFars { get; set; } = string.Empty; - - [Option("data-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for espeak-ng.")] - public string DataDir { get; set; } = string.Empty; - - [Option("length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] - public float LengthScale { get; set; } = 1; - - [Option("noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS or Matcha models")] - public float NoiseScale { get; set; } = 0.667F; - - [Option("vits-noise-scale-w", Required = false, Default = 0.8F, HelpText = "noise_scale_w for VITS models")] - public float NoiseScaleW { get; set; } = 0.8F; - - [Option("lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] - public string Lexicon { get; set; } = string.Empty; - - [Option("tokens", Required = true, Default = "", HelpText = "Path to tokens.txt")] - public string Tokens { get; set; } = string.Empty; - - [Option("tts-max-num-sentences", Required = false, Default = 1, HelpText = "Maximum number of sentences that we process at a time.")] - public int MaxNumSentences { get; set; } = 1; - - [Option(Required = false, Default = 0, HelpText = "1 to show debug messages.")] - public int Debug { get; set; } = 0; - - [Option("vits-model", Required = false, HelpText = "Path to VITS model")] - public string Model { get; set; } = string.Empty; - - [Option("matcha-acoustic-model", Required = false, HelpText = "Path to the acoustic model of Matcha")] - public string AcousticModel { get; set; } = ""; - - [Option("matcha-vocoder", Required = false, HelpText = "Path to the vocoder model of Matcha")] - public string Vocoder { get; set; } = ""; - - [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")] - public int SpeakerId { get; set; } = 0; - - [Option("text", Required = true, HelpText = "Text to synthesize")] - public string Text { get; set; } = string.Empty; - - [Option("output-filename", Required = true, Default = "./generated.wav", HelpText = "Path to save the generated audio")] - public string OutputFilename { get; set; } = "./generated.wav"; - } - - static void Main(string[] args) - { - var parser = new Parser(with => with.HelpWriter = null); - var parserResult = parser.ParseArguments(args); - - parserResult - .WithParsed(options => Run(options)) - .WithNotParsed(errs => DisplayHelp(parserResult, errs)); - } - - private static void DisplayHelp(ParserResult result, IEnumerable errs) - { - var usage = @" -# matcha-icefall-zh-baker - -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 -tar xvf matcha-icefall-zh-baker.tar.bz2 -rm matcha-icefall-zh-baker.tar.bz2 - -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx - -dotnet run \ - --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ - --matcha-vocoder=./vocos-22khz-univ.onnx \ - --lexicon=./matcha-icefall-zh-baker/lexicon.txt \ - --tokens=./matcha-icefall-zh-baker/tokens.txt \ - --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ - --debug=1 \ - --output-filename=./matcha-zh.wav \ - --text='某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。' - -# matcha-icefall-en_US-ljspeech - -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 -tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 -rm matcha-icefall-en_US-ljspeech.tar.bz2 - -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx - -dotnet run \ - --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ - --matcha-vocoder=./vocos-22khz-univ.onnx \ - --tokens=./matcha-icefall-zh-baker/tokens.txt \ - --data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ - --debug=1 \ - --output-filename=./matcha-zh.wav \ - --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' - -# vits-aishell3 - -curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 -tar xvf vits-icefall-zh-aishell3.tar.bz2 - -dotnet run \ - --vits-model=./vits-icefall-zh-aishell3/model.onnx \ - --tokens=./vits-icefall-zh-aishell3/tokens.txt \ - --lexicon=./vits-icefall-zh-aishell3/lexicon.txt \ - --tts-rule-fsts=./vits-icefall-zh-aishell3/phone.fst,./vits-icefall-zh-aishell3/date.fst,./vits-icefall-zh-aishell3/number.fst \ - --tts-rule-fars=./vits-icefall-zh-aishell3/rule.far \ - --sid=66 \ - --debug=1 \ - --output-filename=./aishell3-66.wav \ - --text=这是一个语音合成测试 - -# Piper models - -wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 -tar xf vits-piper-en_US-amy-low.tar.bz2 - -dotnet run \ - --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ - --tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ - --debug=1 \ - --output-filename=./amy.wav \ - --text='This is a text to speech application in dotnet with Next Generation Kaldi' - -Please refer to -https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html -to download more models. -"; - - var helpText = HelpText.AutoBuild(result, h => - { - h.AdditionalNewLineAfterOption = false; - h.Heading = usage; - h.Copyright = "Copyright (c) 2024 Xiaomi Corporation"; - return HelpText.DefaultParsingErrorsHandler(result, h); - }, e => e); - Console.WriteLine(helpText); - } - - private static void Run(Options options) - { - var config = new OfflineTtsConfig(); - config.Model.Vits.Model = options.Model; - config.Model.Vits.Lexicon = options.Lexicon; - config.Model.Vits.Tokens = options.Tokens; - config.Model.Vits.DataDir = options.DataDir; - config.Model.Vits.NoiseScale = options.NoiseScale; - config.Model.Vits.NoiseScaleW = options.NoiseScaleW; - config.Model.Vits.LengthScale = options.LengthScale; - - config.Model.Matcha.AcousticModel = options.AcousticModel; - config.Model.Matcha.Vocoder = options.Vocoder; - config.Model.Matcha.Lexicon = options.Lexicon; - config.Model.Matcha.Tokens = options.Tokens; - config.Model.Matcha.DataDir = options.DataDir; - config.Model.Matcha.NoiseScale = options.NoiseScale; - config.Model.Matcha.LengthScale = options.LengthScale; - - config.Model.NumThreads = 1; - config.Model.Debug = options.Debug; - config.Model.Provider = "cpu"; - config.RuleFsts = options.RuleFsts; - config.RuleFars = options.RuleFars; +// Copyright (c) 2024 Xiaomi Corporation +// +// This file shows how to use a non-streaming TTS model for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +using CommandLine; +using CommandLine.Text; +using SherpaOnnx; + +class OfflineTtsDemo +{ + class Options + { + [Option("tts-rule-fsts", Required = false, Default = "", HelpText = "path to rule.fst")] + public string RuleFsts { get; set; } = string.Empty; + + [Option("tts-rule-fars", Required = false, Default = "", HelpText = "path to rule.far")] + public string RuleFars { get; set; } = string.Empty; + + [Option("length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] + public float LengthScale { get; set; } = 1; + + [Option("noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS or Matcha models")] + public float NoiseScale { get; set; } = 0.667F; + + [Option("vits-noise-scale-w", Required = false, Default = 0.8F, HelpText = "noise_scale_w for VITS models")] + public float NoiseScaleW { get; set; } = 0.8F; + + [Option("lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] + public string Lexicon { get; set; } = string.Empty; + + [Option("tokens", Required = true, Default = "", HelpText = "Path to tokens.txt")] + public string Tokens { get; set; } = string.Empty; + + [Option("tts-max-num-sentences", Required = false, Default = 1, HelpText = "Maximum number of sentences that we process at a time.")] + public int MaxNumSentences { get; set; } = 1; + + [Option(Required = false, Default = 0, HelpText = "1 to show debug messages.")] + public int Debug { get; set; } = 0; + + [Option("vits-model", Required = false, HelpText = "Path to VITS model")] + public string Model { get; set; } = string.Empty; + + [Option("matcha-acoustic-model", Required = false, HelpText = "Path to the acoustic model of Matcha")] + public string AcousticModel { get; set; } = ""; + + [Option("matcha-vocoder", Required = false, HelpText = "Path to the vocoder model of Matcha")] + public string Vocoder { get; set; } = ""; + + [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")] + public int SpeakerId { get; set; } = 0; + + [Option("text", Required = true, HelpText = "Text to synthesize")] + public string Text { get; set; } = string.Empty; + + [Option("output-filename", Required = true, Default = "./generated-lexicon.wav", HelpText = "Path to save the generated audio")] + public string OutputFilename { get; set; } = "./generated-lexicon.wav"; + } + + static void Main(string[] args) + { + var parser = new Parser(with => with.HelpWriter = null); + var parserResult = parser.ParseArguments(args); + + parserResult + .WithParsed(options => Run(options)) + .WithNotParsed(errs => DisplayHelp(parserResult, errs)); + } + + private static void DisplayHelp(ParserResult result, IEnumerable errs) + { + var usage = @" +# matcha-icefall-zh-baker + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./vocos-22khz-univ.onnx \ + --lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --debug=1 \ + --output-filename=./matcha-zh-lexicon.wav \ + --text='某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。' + +# matcha-icefall-en_US-ljspeech + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./vocos-22khz-univ.onnx \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --debug=1 \ + --output-filename=./matcha-zh-lexicon.wav \ + --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' + +# vits-aishell3 + +curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 +tar xvf vits-icefall-zh-aishell3.tar.bz2 + +dotnet run \ + --vits-model=./vits-icefall-zh-aishell3/model.onnx \ + --tokens=./vits-icefall-zh-aishell3/tokens.txt \ + --lexicon=./vits-icefall-zh-aishell3/lexicon.txt \ + --tts-rule-fsts=./vits-icefall-zh-aishell3/phone.fst,./vits-icefall-zh-aishell3/date.fst,./vits-icefall-zh-aishell3/number.fst \ + --tts-rule-fars=./vits-icefall-zh-aishell3/rule.far \ + --sid=66 \ + --debug=1 \ + --output-filename=./aishell3-66-lexicon.wav \ + --text=这是一个语音合成测试 + +# Piper models + +wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 +tar xf vits-piper-en_US-amy-low.tar.bz2 + +dotnet run \ + --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ + --tokens=./vits-piper-en_US-amy-low/tokens.txt \ + --debug=1 \ + --output-filename=./amy-lexicon.wav \ + --text='This is a text to speech application in dotnet with Next Generation Kaldi' + +Please refer to +https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html +to download more models. +"; + + var helpText = HelpText.AutoBuild(result, h => + { + h.AdditionalNewLineAfterOption = false; + h.Heading = usage; + h.Copyright = "Copyright (c) 2024 Xiaomi Corporation"; + return HelpText.DefaultParsingErrorsHandler(result, h); + }, e => e); + Console.WriteLine(helpText); + } + + private static void Run(Options options) + { + var config = new OfflineTtsConfig(); + config.Model.Vits.Model = options.Model; + config.Model.Vits.Lexicon = options.Lexicon; + config.Model.Vits.Tokens = options.Tokens; + config.Model.Vits.NoiseScale = options.NoiseScale; + config.Model.Vits.NoiseScaleW = options.NoiseScaleW; + config.Model.Vits.LengthScale = options.LengthScale; + + config.Model.Matcha.AcousticModel = options.AcousticModel; + config.Model.Matcha.Vocoder = options.Vocoder; + config.Model.Matcha.Lexicon = options.Lexicon; + config.Model.Matcha.Tokens = options.Tokens; + config.Model.Matcha.NoiseScale = options.NoiseScale; + config.Model.Matcha.LengthScale = options.LengthScale; + + config.Model.NumThreads = 1; + config.Model.Debug = options.Debug; + config.Model.Provider = "cpu"; + config.RuleFsts = options.RuleFsts; + config.RuleFars = options.RuleFars; config.MaxNumSentences = options.MaxNumSentences; var tts = new OfflineTts(config); @@ -190,14 +183,14 @@ to download more models. genConfig.SilenceScale = 0.2f; var audio = tts.GenerateWithConfig(options.Text, genConfig, null); var ok = audio.SaveToWaveFile(options.OutputFilename); - - if (ok) - { - Console.WriteLine($"Wrote to {options.OutputFilename} succeeded!"); - } - else - { - Console.WriteLine($"Failed to write {options.OutputFilename}"); - } - } -} + + if (ok) + { + Console.WriteLine($"Wrote to {options.OutputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {options.OutputFilename}"); + } + } +} diff --git a/dotnet-examples/offline-tts/offline-tts.csproj b/dotnet-examples/offline-tts-lexicon/offline-tts-lexicon.csproj similarity index 100% rename from dotnet-examples/offline-tts/offline-tts.csproj rename to dotnet-examples/offline-tts-lexicon/offline-tts-lexicon.csproj diff --git a/dotnet-examples/offline-tts/run-aishell3.sh b/dotnet-examples/offline-tts-lexicon/run-aishell3-lexicon.sh similarity index 92% rename from dotnet-examples/offline-tts/run-aishell3.sh rename to dotnet-examples/offline-tts-lexicon/run-aishell3-lexicon.sh index 9a54df349..137a029db 100755 --- a/dotnet-examples/offline-tts/run-aishell3.sh +++ b/dotnet-examples/offline-tts-lexicon/run-aishell3-lexicon.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash set -ex if [ ! -f ./vits-zh-aishell3/vits-aishell3.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 @@ -14,5 +14,5 @@ dotnet run \ --tts-rule-fars=./vits-icefall-zh-aishell3/rule.far \ --sid=66 \ --debug=1 \ - --output-filename=./aishell3-66.wav \ + --output-filename=./aishell3-66-lexicon.wav \ --text="这是一个语音合成测试, 写于公元 2024 年 1 月 28 号, 23点27分，星期天。长沙长大，去过长白山和长安街。行行出状元。行行，银行行长，行业。" diff --git a/dotnet-examples/offline-tts-play/run-hf-fanchen.sh b/dotnet-examples/offline-tts-lexicon/run-hf-fanchen-lexicon.sh similarity index 92% rename from dotnet-examples/offline-tts-play/run-hf-fanchen.sh rename to dotnet-examples/offline-tts-lexicon/run-hf-fanchen-lexicon.sh index 02dff9643..9ac975577 100755 --- a/dotnet-examples/offline-tts-play/run-hf-fanchen.sh +++ b/dotnet-examples/offline-tts-lexicon/run-hf-fanchen-lexicon.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash set -ex if [ ! -f ./vits-zh-hf-fanchen-C/vits-zh-hf-fanchen-C.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-hf-fanchen-C.tar.bz2 @@ -13,5 +13,5 @@ dotnet run \ --tts-rule-fsts=./vits-zh-hf-fanchen-C/phone.fst,./vits-zh-hf-fanchen-C/date.fst,./vits-zh-hf-fanchen-C/number.fst \ --sid=100 \ --debug=1 \ - --output-filename=./fanchen-100.wav \ + --output-filename=./fanchen-100-lexicon.wav \ --text="这是一个语音合成测试, 写于公元2024年4月26号, 11点05分，星期5。小米的使命是，始终坚持做'感动人心、价格厚道'的好产品，让全球每个人都能享受科技带来的美好生活。" diff --git a/dotnet-examples/offline-tts/run-matcha-en.sh b/dotnet-examples/offline-tts-lexicon/run-matcha-en-lexicon.sh similarity index 82% rename from dotnet-examples/offline-tts/run-matcha-en.sh rename to dotnet-examples/offline-tts-lexicon/run-matcha-en-lexicon.sh index 4a40f5bdf..bdaee7c8b 100755 --- a/dotnet-examples/offline-tts/run-matcha-en.sh +++ b/dotnet-examples/offline-tts-lexicon/run-matcha-en-lexicon.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash set -ex @@ -16,11 +16,15 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + dotnet run \ --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ --matcha-vocoder=./vocos-22khz-univ.onnx \ --tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ - --data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --lexicon=./lexicon-en-us.txt \ --debug=1 \ - --output-filename=./matcha-en.wav \ + --output-filename=./matcha-en-lexicon.wav \ --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' diff --git a/dotnet-examples/offline-tts/run-matcha-zh.sh b/dotnet-examples/offline-tts-lexicon/run-matcha-zh-lexicon.sh similarity index 94% rename from dotnet-examples/offline-tts/run-matcha-zh.sh rename to dotnet-examples/offline-tts-lexicon/run-matcha-zh-lexicon.sh index 36a8ddd62..225b71129 100755 --- a/dotnet-examples/offline-tts/run-matcha-zh.sh +++ b/dotnet-examples/offline-tts-lexicon/run-matcha-zh-lexicon.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash set -ex # please visit @@ -22,5 +22,5 @@ dotnet run \ --tokens=./matcha-icefall-zh-baker/tokens.txt \ --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ --debug=1 \ - --output-filename=./matcha-zh.wav \ + --output-filename=./matcha-zh-lexicon.wav \ --text="某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" diff --git a/dotnet-examples/offline-tts/run-piper.sh b/dotnet-examples/offline-tts-lexicon/run-piper-lexicon.sh similarity index 71% rename from dotnet-examples/offline-tts/run-piper.sh rename to dotnet-examples/offline-tts-lexicon/run-piper-lexicon.sh index 273799bb3..af9af612c 100755 --- a/dotnet-examples/offline-tts/run-piper.sh +++ b/dotnet-examples/offline-tts-lexicon/run-piper-lexicon.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash set -ex if [ ! -f ./vits-piper-en_US-amy-low/en_US-amy-low.onnx ]; then @@ -8,11 +8,15 @@ if [ ! -f ./vits-piper-en_US-amy-low/en_US-amy-low.onnx ]; then rm vits-piper-en_US-amy-low.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + dotnet run \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ --tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --lexicon=./lexicon-en-us.txt \ --debug=1 \ - --output-filename=./amy.wav \ + --output-filename=./amy-lexicon.wav \ --text="This is a text to speech application in dotnet with Next Generation Kaldi" diff --git a/dotnet-examples/offline-tts-phonemize/Program.cs b/dotnet-examples/offline-tts-phonemize/Program.cs new file mode 100644 index 000000000..909979fd7 --- /dev/null +++ b/dotnet-examples/offline-tts-phonemize/Program.cs @@ -0,0 +1,192 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file shows how to use a non-streaming TTS model for text-to-speech +// with the piper-phonemize text frontend. +// +// The text is phonemized outside of sherpa-onnx with the +// com.github.csukuangfj.piper.phonemize NuGet package and the resulting +// phoneme codepoints are passed via OfflineTtsGenerationConfig.PhonemeCodepoints. +// No lexicon file is needed. +// +// The C# counterpart of +// https://github.com/k2-fsa/sherpa-onnx/blob/master/python-api-examples/test-offline-tts-piper-phonemize.py +// and test-offline-tts-matcha-en-phonemize.py +using CommandLine; +using CommandLine.Text; +using PiperPhonemize; +using SherpaOnnx; + +class OfflineTtsPhonemizeDemo +{ + class Options + { + [Option("length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] + public float LengthScale { get; set; } = 1; + + [Option("noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS or Matcha models")] + public float NoiseScale { get; set; } = 0.667F; + + [Option("vits-noise-scale-w", Required = false, Default = 0.8F, HelpText = "noise_scale_w for VITS models")] + public float NoiseScaleW { get; set; } = 0.8F; + + [Option("tokens", Required = true, Default = "", HelpText = "Path to tokens.txt")] + public string Tokens { get; set; } = string.Empty; + + [Option(Required = false, Default = 0, HelpText = "1 to show debug messages.")] + public int Debug { get; set; } = 0; + + [Option("vits-model", Required = false, HelpText = "Path to VITS model")] + public string Model { get; set; } = string.Empty; + + [Option("matcha-acoustic-model", Required = false, HelpText = "Path to the acoustic model of Matcha")] + public string AcousticModel { get; set; } = ""; + + [Option("matcha-vocoder", Required = false, HelpText = "Path to the vocoder model of Matcha")] + public string Vocoder { get; set; } = ""; + + [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")] + public int SpeakerId { get; set; } = 0; + + [Option("text", Required = true, HelpText = "Text to synthesize")] + public string Text { get; set; } = string.Empty; + + [Option("lang", Required = false, Default = "", + HelpText = "espeak-ng voice for the phonemizer, e.g., en-us. Empty means the language of the model")] + public string Lang { get; set; } = string.Empty; + + [Option("output-filename", Required = false, Default = "./generated-dotnet-phonemize.wav", + HelpText = "Path to save the generated audio")] + public string OutputFilename { get; set; } = "./generated-dotnet-phonemize.wav"; + } + + static void Main(string[] args) + { + var parser = new Parser(with => + with.HelpWriter = null); + var parserResult = parser.ParseArguments(args); + parserResult.WithParsed(Run) + .WithNotParsed(errs => Console.WriteLine(HelpText.AutoBuild(parserResult, h => h, e => e))); + } + + private static void Run(Options options) + { + Console.WriteLine($"piper-phonemize version: {PiperPhonemizeApi.GetVersionStr()}"); + + string dataDir = FindEspeakNgData(); + if (dataDir == null) + { + Console.Error.WriteLine("ERROR: espeak-ng-data not found."); + Environment.Exit(1); + } + Console.WriteLine($"espeak-ng-data: {dataDir}"); + + // Note: espeak-ng-data is only used by the piper-phonemize phonemizer. + // sherpa-onnx itself does not use espeak-ng anymore. + int sampleRate = PiperPhonemizeApi.Initialize(dataDir); + if (sampleRate < 0) + { + Console.Error.WriteLine("Failed to initialize piper-phonemize"); + Environment.Exit(1); + } + Console.WriteLine($"Sample rate: {sampleRate}"); + + var config = new OfflineTtsConfig(); + config.Model.Vits.Model = options.Model; + config.Model.Vits.Tokens = options.Tokens; + config.Model.Vits.NoiseScale = options.NoiseScale; + config.Model.Vits.NoiseScaleW = options.NoiseScaleW; + config.Model.Vits.LengthScale = options.LengthScale; + + config.Model.Matcha.AcousticModel = options.AcousticModel; + config.Model.Matcha.Vocoder = options.Vocoder; + config.Model.Matcha.Tokens = options.Tokens; + config.Model.Matcha.NoiseScale = options.NoiseScale; + config.Model.Matcha.LengthScale = options.LengthScale; + + config.Model.NumThreads = 1; + config.Model.Debug = options.Debug; + config.Model.Provider = "cpu"; + + var tts = new OfflineTts(config); + var speed = 1.0f / options.LengthScale; + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + string lang = options.Lang; + if (string.IsNullOrEmpty(lang)) + { + lang = tts.Lang; + if (string.IsNullOrEmpty(lang)) + { + Console.Error.WriteLine("Warning: the model has no language information. Falling back to en-us."); + lang = "en-us"; + } + } + Console.WriteLine($"Phonemizer language: {lang}"); + + int[][] phonemeCodepoints; + using (var result = PiperPhonemizeApi.Text(options.Text, lang)) + { + if (result == null) + { + Console.Error.WriteLine("Phonemization failed."); + Environment.Exit(1); + } + + phonemeCodepoints = new int[result.NumSentences][]; + for (int i = 0; i < result.NumSentences; i++) + { + string ipa = result.GetPhonemesAsString(i) ?? ""; + int[] cps = new int[ipa.Length]; + for (int j = 0; j < ipa.Length; j++) + cps[j] = ipa[j]; + phonemeCodepoints[i] = cps; + } + } + Console.WriteLine($"Phonemized sentences: {phonemeCodepoints.Length}"); + + OfflineTtsGenerationConfig genConfig = new OfflineTtsGenerationConfig(); + genConfig.Sid = options.SpeakerId; + genConfig.Speed = speed; + genConfig.PhonemeCodepoints = phonemeCodepoints; + + // The text has already been phonemized; pass an empty string here. + var audio = tts.GenerateWithConfig("", genConfig, null); + + var ok = audio.SaveToWaveFile(options.OutputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {options.OutputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to save {options.OutputFilename}"); + } + } + + // The piper-phonemize NuGet package copies espeak-ng-data next to the + // executable (see its .targets file). espeak_Initialize requires a real + // directory: an empty path falls back to the system default + // (/usr/share/espeak-ng-data), which may not exist. + static string FindEspeakNgData() + { + string exeDir = AppDomain.CurrentDomain.BaseDirectory; + + string[] candidates = new[] + { + Path.Combine(exeDir, "espeak-ng-data"), + Path.Combine(exeDir, "..", "espeak-ng-data"), + Path.Combine(exeDir, "..", "..", "..", "espeak-ng-data"), + "espeak-ng-data", + }; + + foreach (var candidate in candidates) + { + if (Directory.Exists(candidate) && File.Exists(Path.Combine(candidate, "phontab"))) + return Path.GetFullPath(candidate); + } + + return null; + } +} diff --git a/dotnet-examples/offline-tts-phonemize/offline-tts-phonemize.csproj b/dotnet-examples/offline-tts-phonemize/offline-tts-phonemize.csproj new file mode 100644 index 000000000..c91f6b5c4 --- /dev/null +++ b/dotnet-examples/offline-tts-phonemize/offline-tts-phonemize.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + offline_tts_phonemize + enable + enable + + + + + + + + + diff --git a/dotnet-examples/offline-tts-phonemize/run-matcha-en-phonemize.sh b/dotnet-examples/offline-tts-phonemize/run-matcha-en-phonemize.sh new file mode 100755 index 000000000..f6216f3c2 --- /dev/null +++ b/dotnet-examples/offline-tts-phonemize/run-matcha-en-phonemize.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ex +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -OL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./vocos-22khz-univ.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +fi + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./vocos-22khz-univ.onnx \ + --tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ + --debug=1 \ + --output-filename=./matcha-en-phonemize.wav \ + --text="Today as always, men fall into two groups: slaves and free men." diff --git a/dotnet-examples/offline-tts-play/run-piper.sh b/dotnet-examples/offline-tts-phonemize/run-piper-phonemize.sh similarity index 85% rename from dotnet-examples/offline-tts-play/run-piper.sh rename to dotnet-examples/offline-tts-phonemize/run-piper-phonemize.sh index 1a4d10806..dcf25ad78 100755 --- a/dotnet-examples/offline-tts-play/run-piper.sh +++ b/dotnet-examples/offline-tts-phonemize/run-piper-phonemize.sh @@ -10,8 +10,6 @@ fi dotnet run \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ --tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ --debug=1 \ - --output-filename=./amy.wav \ + --output-filename=./amy-phonemize.wav \ --text="This is a text to speech application in dotnet with Next Generation Kaldi" - diff --git a/dotnet-examples/offline-tts-play/.gitignore b/dotnet-examples/offline-tts-play-lexicon/.gitignore similarity index 100% rename from dotnet-examples/offline-tts-play/.gitignore rename to dotnet-examples/offline-tts-play-lexicon/.gitignore diff --git a/dotnet-examples/offline-tts-play/Program.cs b/dotnet-examples/offline-tts-play-lexicon/Program.cs similarity index 92% rename from dotnet-examples/offline-tts-play/Program.cs rename to dotnet-examples/offline-tts-play-lexicon/Program.cs index dc693cfd4..de5364865 100644 --- a/dotnet-examples/offline-tts-play/Program.cs +++ b/dotnet-examples/offline-tts-play-lexicon/Program.cs @@ -1,190 +1,183 @@ -// Copyright (c) 2024 Xiaomi Corporation -// -// This file shows how to use a non-streaming TTS model for text-to-speech -// Please refer to -// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html -// and -// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models -// to download pre-trained models -// -// Note that you need a speaker to run this file since it will play -// the generated audio back as it is being generated. - -using CommandLine; -using CommandLine.Text; -using PortAudioSharp; -using SherpaOnnx; -using System.Collections.Concurrent; -using System.Runtime.InteropServices; - -class OfflineTtsPlayDemo -{ - class Options - { - [Option("tts-rule-fsts", Required = false, Default = "", HelpText = "path to rule.fst")] - public string RuleFsts { get; set; } = string.Empty; - - [Option("tts-rule-fars", Required = false, Default = "", HelpText = "path to rule.far")] - public string RuleFars { get; set; } = string.Empty; - - [Option("data-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for espeak-ng.")] - public string DataDir { get; set; } = string.Empty; - - [Option("length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] - public float LengthScale { get; set; } = 1; - - [Option("noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS or Matcha models")] - public float NoiseScale { get; set; } = 0.667F; - - [Option("vits-noise-scale-w", Required = false, Default = 0.8F, HelpText = "noise_scale_w for VITS models")] - public float NoiseScaleW { get; set; } = 0.8F; - - [Option("lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] - public string Lexicon { get; set; } = string.Empty; - - [Option("tokens", Required = true, Default = "", HelpText = "Path to tokens.txt")] - public string Tokens { get; set; } = string.Empty; - - [Option("tts-max-num-sentences", Required = false, Default = 1, HelpText = "Maximum number of sentences that we process at a time.")] - public int MaxNumSentences { get; set; } = 1; - - [Option(Required = false, Default = 0, HelpText = "1 to show debug messages.")] - public int Debug { get; set; } = 0; - - [Option("vits-model", Required = false, HelpText = "Path to VITS model")] - public string Model { get; set; } = string.Empty; - - [Option("matcha-acoustic-model", Required = false, HelpText = "Path to the acoustic model of Matcha")] - public string AcousticModel { get; set; } = ""; - - [Option("matcha-vocoder", Required = false, HelpText = "Path to the vocoder model of Matcha")] - public string Vocoder { get; set; } = ""; - - [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")] - public int SpeakerId { get; set; } = 0; - - [Option("text", Required = true, HelpText = "Text to synthesize")] - public string Text { get; set; } = string.Empty; - - [Option("output-filename", Required = true, Default = "./generated.wav", HelpText = "Path to save the generated audio")] - public string OutputFilename { get; set; } = "./generated.wav"; - } - - static void Main(string[] args) - { - var parser = new CommandLine.Parser(with => with.HelpWriter = null); - var parserResult = parser.ParseArguments(args); - - parserResult - .WithParsed(options => Run(options)) - .WithNotParsed(errs => DisplayHelp(parserResult, errs)); - } - - private static void DisplayHelp(ParserResult result, IEnumerable errs) - { - string usage = @" -# matcha-icefall-zh-baker - -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 -tar xvf matcha-icefall-zh-baker.tar.bz2 -rm matcha-icefall-zh-baker.tar.bz2 - -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx - -dotnet run \ - --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ - --matcha-vocoder=./vocos-22khz-univ.onnx \ - --lexicon=./matcha-icefall-zh-baker/lexicon.txt \ - --tokens=./matcha-icefall-zh-baker/tokens.txt \ - --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ - --debug=1 \ - --output-filename=./matcha-zh.wav \ - --text='某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。' - -# matcha-icefall-en_US-ljspeech - -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 -tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 -rm matcha-icefall-en_US-ljspeech.tar.bz2 - -wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx - -dotnet run \ - --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ - --matcha-vocoder=./vocos-22khz-univ.onnx \ - --tokens=./matcha-icefall-zh-baker/tokens.txt \ - --data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ - --debug=1 \ - --output-filename=./matcha-zh.wav \ - --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' - -# vits-aishell3 - -wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2 -tar xf vits-zh-aishell3.tar.bz2 - -dotnet run \ - --vits-model=./vits-zh-aishell3/vits-aishell3.onnx \ - --tokens=./vits-zh-aishell3/tokens.txt \ - --lexicon=./vits-zh-aishell3/lexicon.txt \ - --tts-rule-fsts=./vits-zh-aishell3/rule.fst \ - --sid=66 \ - --debug=1 \ - --output-filename=./aishell3-66.wav \ - --text=这是一个语音合成测试 - -# Piper models - -wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 -tar xf vits-piper-en_US-amy-low.tar.bz2 - -dotnet run \ - --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ - ---tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ - --debug=1 \ - --output-filename=./amy.wav \ - --text='This is a text to speech application in dotnet with Next Generation Kaldi' - -Please refer to -https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html -to download more models. -"; - - var helpText = HelpText.AutoBuild(result, h => - { - h.AdditionalNewLineAfterOption = false; - h.Heading = usage; - h.Copyright = "Copyright (c) 2024 Xiaomi Corporation"; - return HelpText.DefaultParsingErrorsHandler(result, h); - }, e => e); - Console.WriteLine(helpText); - } - - private static void Run(Options options) - { - var config = new OfflineTtsConfig(); - - config.Model.Vits.Model = options.Model; - config.Model.Vits.Lexicon = options.Lexicon; - config.Model.Vits.Tokens = options.Tokens; - config.Model.Vits.DataDir = options.DataDir; - config.Model.Vits.NoiseScale = options.NoiseScale; - config.Model.Vits.NoiseScaleW = options.NoiseScaleW; - config.Model.Vits.LengthScale = options.LengthScale; - - config.Model.Matcha.AcousticModel = options.AcousticModel; - config.Model.Matcha.Vocoder = options.Vocoder; - config.Model.Matcha.Lexicon = options.Lexicon; - config.Model.Matcha.Tokens = options.Tokens; - config.Model.Matcha.DataDir = options.DataDir; - config.Model.Matcha.NoiseScale = options.NoiseScale; - config.Model.Matcha.LengthScale = options.LengthScale; - - config.Model.NumThreads = 1; - config.Model.Debug = options.Debug; - config.Model.Provider = "cpu"; - config.RuleFsts = options.RuleFsts; +// Copyright (c) 2024 Xiaomi Corporation +// +// This file shows how to use a non-streaming TTS model for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +// +// Note that you need a speaker to run this file since it will play +// the generated audio back as it is being generated. + +using CommandLine; +using CommandLine.Text; +using PortAudioSharp; +using SherpaOnnx; +using System.Collections.Concurrent; +using System.Runtime.InteropServices; + +class OfflineTtsPlayDemo +{ + class Options + { + [Option("tts-rule-fsts", Required = false, Default = "", HelpText = "path to rule.fst")] + public string RuleFsts { get; set; } = string.Empty; + + [Option("tts-rule-fars", Required = false, Default = "", HelpText = "path to rule.far")] + public string RuleFars { get; set; } = string.Empty; + + [Option("length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")] + public float LengthScale { get; set; } = 1; + + [Option("noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS or Matcha models")] + public float NoiseScale { get; set; } = 0.667F; + + [Option("vits-noise-scale-w", Required = false, Default = 0.8F, HelpText = "noise_scale_w for VITS models")] + public float NoiseScaleW { get; set; } = 0.8F; + + [Option("lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")] + public string Lexicon { get; set; } = string.Empty; + + [Option("tokens", Required = true, Default = "", HelpText = "Path to tokens.txt")] + public string Tokens { get; set; } = string.Empty; + + [Option("tts-max-num-sentences", Required = false, Default = 1, HelpText = "Maximum number of sentences that we process at a time.")] + public int MaxNumSentences { get; set; } = 1; + + [Option(Required = false, Default = 0, HelpText = "1 to show debug messages.")] + public int Debug { get; set; } = 0; + + [Option("vits-model", Required = false, HelpText = "Path to VITS model")] + public string Model { get; set; } = string.Empty; + + [Option("matcha-acoustic-model", Required = false, HelpText = "Path to the acoustic model of Matcha")] + public string AcousticModel { get; set; } = ""; + + [Option("matcha-vocoder", Required = false, HelpText = "Path to the vocoder model of Matcha")] + public string Vocoder { get; set; } = ""; + + [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")] + public int SpeakerId { get; set; } = 0; + + [Option("text", Required = true, HelpText = "Text to synthesize")] + public string Text { get; set; } = string.Empty; + + [Option("output-filename", Required = true, Default = "./generated-lexicon.wav", HelpText = "Path to save the generated audio")] + public string OutputFilename { get; set; } = "./generated-lexicon.wav"; + } + + static void Main(string[] args) + { + var parser = new CommandLine.Parser(with => with.HelpWriter = null); + var parserResult = parser.ParseArguments(args); + + parserResult + .WithParsed(options => Run(options)) + .WithNotParsed(errs => DisplayHelp(parserResult, errs)); + } + + private static void DisplayHelp(ParserResult result, IEnumerable errs) + { + string usage = @" +# matcha-icefall-zh-baker + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 +tar xvf matcha-icefall-zh-baker.tar.bz2 +rm matcha-icefall-zh-baker.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-zh-baker/model-steps-3.onnx \ + --matcha-vocoder=./vocos-22khz-univ.onnx \ + --lexicon=./matcha-icefall-zh-baker/lexicon.txt \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ + --debug=1 \ + --output-filename=./matcha-zh-lexicon.wav \ + --text='某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。' + +# matcha-icefall-en_US-ljspeech + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 +tar xvf matcha-icefall-en_US-ljspeech.tar.bz2 +rm matcha-icefall-en_US-ljspeech.tar.bz2 + +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + +dotnet run \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./vocos-22khz-univ.onnx \ + --tokens=./matcha-icefall-zh-baker/tokens.txt \ + --debug=1 \ + --output-filename=./matcha-zh-lexicon.wav \ + --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' + +# vits-aishell3 + +wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2 +tar xf vits-zh-aishell3.tar.bz2 + +dotnet run \ + --vits-model=./vits-zh-aishell3/vits-aishell3.onnx \ + --tokens=./vits-zh-aishell3/tokens.txt \ + --lexicon=./vits-zh-aishell3/lexicon.txt \ + --tts-rule-fsts=./vits-zh-aishell3/rule.fst \ + --sid=66 \ + --debug=1 \ + --output-filename=./aishell3-66-lexicon.wav \ + --text=这是一个语音合成测试 + +# Piper models + +wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 +tar xf vits-piper-en_US-amy-low.tar.bz2 + +dotnet run \ + --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ + ---tokens=./vits-piper-en_US-amy-low/tokens.txt \ + --debug=1 \ + --output-filename=./amy-lexicon.wav \ + --text='This is a text to speech application in dotnet with Next Generation Kaldi' + +Please refer to +https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html +to download more models. +"; + + var helpText = HelpText.AutoBuild(result, h => + { + h.AdditionalNewLineAfterOption = false; + h.Heading = usage; + h.Copyright = "Copyright (c) 2024 Xiaomi Corporation"; + return HelpText.DefaultParsingErrorsHandler(result, h); + }, e => e); + Console.WriteLine(helpText); + } + + private static void Run(Options options) + { + var config = new OfflineTtsConfig(); + + config.Model.Vits.Model = options.Model; + config.Model.Vits.Lexicon = options.Lexicon; + config.Model.Vits.Tokens = options.Tokens; + config.Model.Vits.NoiseScale = options.NoiseScale; + config.Model.Vits.NoiseScaleW = options.NoiseScaleW; + config.Model.Vits.LengthScale = options.LengthScale; + + config.Model.Matcha.AcousticModel = options.AcousticModel; + config.Model.Matcha.Vocoder = options.Vocoder; + config.Model.Matcha.Lexicon = options.Lexicon; + config.Model.Matcha.Tokens = options.Tokens; + config.Model.Matcha.NoiseScale = options.NoiseScale; + config.Model.Matcha.LengthScale = options.LengthScale; + + config.Model.NumThreads = 1; + config.Model.Debug = options.Debug; + config.Model.Provider = "cpu"; + config.RuleFsts = options.RuleFsts; config.MaxNumSentences = options.MaxNumSentences; var tts = new OfflineTts(config); @@ -196,146 +189,146 @@ to download more models. genConfig.SilenceScale = 0.2f; Console.WriteLine(PortAudio.VersionInfo.versionText); - PortAudio.Initialize(); - Console.WriteLine($"Number of devices: {PortAudio.DeviceCount}"); - - for (int i = 0; i != PortAudio.DeviceCount; ++i) - { - Console.WriteLine($" Device {i}"); - DeviceInfo deviceInfo = PortAudio.GetDeviceInfo(i); - Console.WriteLine($" Name: {deviceInfo.name}"); - Console.WriteLine($" Max output channels: {deviceInfo.maxOutputChannels}"); - Console.WriteLine($" Default sample rate: {deviceInfo.defaultSampleRate}"); - } - int deviceIndex = PortAudio.DefaultOutputDevice; - if (deviceIndex == PortAudio.NoDevice) - { - Console.WriteLine("No default output device found. Please use ../offline-tts instead"); - Environment.Exit(1); - } - - var info = PortAudio.GetDeviceInfo(deviceIndex); - Console.WriteLine(); - Console.WriteLine($"Use output default device {deviceIndex} ({info.name})"); - - var param = new StreamParameters(); - param.device = deviceIndex; - param.channelCount = 1; - param.sampleFormat = SampleFormat.Float32; - param.suggestedLatency = info.defaultLowOutputLatency; - param.hostApiSpecificStreamInfo = IntPtr.Zero; - - // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview - var dataItems = new BlockingCollection(); - + PortAudio.Initialize(); + Console.WriteLine($"Number of devices: {PortAudio.DeviceCount}"); + + for (int i = 0; i != PortAudio.DeviceCount; ++i) + { + Console.WriteLine($" Device {i}"); + DeviceInfo deviceInfo = PortAudio.GetDeviceInfo(i); + Console.WriteLine($" Name: {deviceInfo.name}"); + Console.WriteLine($" Max output channels: {deviceInfo.maxOutputChannels}"); + Console.WriteLine($" Default sample rate: {deviceInfo.defaultSampleRate}"); + } + int deviceIndex = PortAudio.DefaultOutputDevice; + if (deviceIndex == PortAudio.NoDevice) + { + Console.WriteLine("No default output device found. Please use ../offline-tts instead"); + Environment.Exit(1); + } + + var info = PortAudio.GetDeviceInfo(deviceIndex); + Console.WriteLine(); + Console.WriteLine($"Use output default device {deviceIndex} ({info.name})"); + + var param = new StreamParameters(); + param.device = deviceIndex; + param.channelCount = 1; + param.sampleFormat = SampleFormat.Float32; + param.suggestedLatency = info.defaultLowOutputLatency; + param.hostApiSpecificStreamInfo = IntPtr.Zero; + + // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview + var dataItems = new BlockingCollection(); + var myCallback = (IntPtr samples, int n, float progress, IntPtr arg) => { float[] data = new float[n]; Marshal.Copy(samples, data, 0, n); - - dataItems.Add(data); - - // 1 means to keep generating - // 0 means to stop generating - return 1; - }; - - var playFinished = false; - - float[]? lastSampleArray = null; - int lastIndex = 0; // not played - - PortAudioSharp.Stream.Callback playCallback = (IntPtr input, IntPtr output, - UInt32 frameCount, - ref StreamCallbackTimeInfo timeInfo, - StreamCallbackFlags statusFlags, - IntPtr userData - ) => - { - if (dataItems.IsCompleted && lastSampleArray == null && lastIndex == 0) - { - Console.WriteLine($"Finished playing"); - playFinished = true; - return StreamCallbackResult.Complete; - } - - int expected = Convert.ToInt32(frameCount); - int i = 0; - - while ((lastSampleArray != null || dataItems.Count != 0) && (i < expected)) - { - int needed = expected - i; - - if (lastSampleArray != null) - { - int remaining = lastSampleArray.Length - lastIndex; - if (remaining >= needed) - { - float[] this_block = lastSampleArray.Skip(lastIndex).Take(needed).ToArray(); - lastIndex += needed; - if (lastIndex == lastSampleArray.Length) - { - lastSampleArray = null; - lastIndex = 0; - } - - Marshal.Copy(this_block, 0, IntPtr.Add(output, i * sizeof(float)), needed); - return StreamCallbackResult.Continue; - } - - float[] this_block2 = lastSampleArray.Skip(lastIndex).Take(remaining).ToArray(); - lastIndex = 0; - lastSampleArray = null; - - Marshal.Copy(this_block2, 0, IntPtr.Add(output, i * sizeof(float)), remaining); - i += remaining; - continue; - } - - if (dataItems.Count != 0) - { - lastSampleArray = dataItems.Take(); - lastIndex = 0; - } - } - - if (i < expected) - { - int sizeInBytes = (expected - i) * 4; - Marshal.Copy(new byte[sizeInBytes], 0, IntPtr.Add(output, i * sizeof(float)), sizeInBytes); - } - - return StreamCallbackResult.Continue; - }; - - PortAudioSharp.Stream stream = new PortAudioSharp.Stream(inParams: null, outParams: param, sampleRate: tts.SampleRate, - framesPerBuffer: 0, - streamFlags: StreamFlags.ClipOff, - callback: playCallback, - userData: IntPtr.Zero - ); - - stream.Start(); - + + dataItems.Add(data); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + }; + + var playFinished = false; + + float[]? lastSampleArray = null; + int lastIndex = 0; // not played + + PortAudioSharp.Stream.Callback playCallback = (IntPtr input, IntPtr output, + UInt32 frameCount, + ref StreamCallbackTimeInfo timeInfo, + StreamCallbackFlags statusFlags, + IntPtr userData + ) => + { + if (dataItems.IsCompleted && lastSampleArray == null && lastIndex == 0) + { + Console.WriteLine($"Finished playing"); + playFinished = true; + return StreamCallbackResult.Complete; + } + + int expected = Convert.ToInt32(frameCount); + int i = 0; + + while ((lastSampleArray != null || dataItems.Count != 0) && (i < expected)) + { + int needed = expected - i; + + if (lastSampleArray != null) + { + int remaining = lastSampleArray.Length - lastIndex; + if (remaining >= needed) + { + float[] this_block = lastSampleArray.Skip(lastIndex).Take(needed).ToArray(); + lastIndex += needed; + if (lastIndex == lastSampleArray.Length) + { + lastSampleArray = null; + lastIndex = 0; + } + + Marshal.Copy(this_block, 0, IntPtr.Add(output, i * sizeof(float)), needed); + return StreamCallbackResult.Continue; + } + + float[] this_block2 = lastSampleArray.Skip(lastIndex).Take(remaining).ToArray(); + lastIndex = 0; + lastSampleArray = null; + + Marshal.Copy(this_block2, 0, IntPtr.Add(output, i * sizeof(float)), remaining); + i += remaining; + continue; + } + + if (dataItems.Count != 0) + { + lastSampleArray = dataItems.Take(); + lastIndex = 0; + } + } + + if (i < expected) + { + int sizeInBytes = (expected - i) * 4; + Marshal.Copy(new byte[sizeInBytes], 0, IntPtr.Add(output, i * sizeof(float)), sizeInBytes); + } + + return StreamCallbackResult.Continue; + }; + + PortAudioSharp.Stream stream = new PortAudioSharp.Stream(inParams: null, outParams: param, sampleRate: tts.SampleRate, + framesPerBuffer: 0, + streamFlags: StreamFlags.ClipOff, + callback: playCallback, + userData: IntPtr.Zero + ); + + stream.Start(); + var callback = new OfflineTtsCallbackProgressWithArg(myCallback); var audio = tts.GenerateWithConfig(options.Text, genConfig, callback); var ok = audio.SaveToWaveFile(options.OutputFilename); - - if (ok) - { - Console.WriteLine($"Wrote to {options.OutputFilename} succeeded!"); - } - else - { - Console.WriteLine($"Failed to write {options.OutputFilename}"); - } - dataItems.CompleteAdding(); - - while (!playFinished) - { - Thread.Sleep(100); // 100ms - } - } -} + + if (ok) + { + Console.WriteLine($"Wrote to {options.OutputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {options.OutputFilename}"); + } + dataItems.CompleteAdding(); + + while (!playFinished) + { + Thread.Sleep(100); // 100ms + } + } +} diff --git a/dotnet-examples/offline-tts-play/offline-tts-play.csproj b/dotnet-examples/offline-tts-play-lexicon/offline-tts-play-lexicon.csproj similarity index 100% rename from dotnet-examples/offline-tts-play/offline-tts-play.csproj rename to dotnet-examples/offline-tts-play-lexicon/offline-tts-play-lexicon.csproj diff --git a/dotnet-examples/offline-tts/run-hf-fanchen.sh b/dotnet-examples/offline-tts-play-lexicon/run-hf-fanchen-lexicon.sh similarity index 92% rename from dotnet-examples/offline-tts/run-hf-fanchen.sh rename to dotnet-examples/offline-tts-play-lexicon/run-hf-fanchen-lexicon.sh index 02dff9643..9ac975577 100755 --- a/dotnet-examples/offline-tts/run-hf-fanchen.sh +++ b/dotnet-examples/offline-tts-play-lexicon/run-hf-fanchen-lexicon.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash set -ex if [ ! -f ./vits-zh-hf-fanchen-C/vits-zh-hf-fanchen-C.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-hf-fanchen-C.tar.bz2 @@ -13,5 +13,5 @@ dotnet run \ --tts-rule-fsts=./vits-zh-hf-fanchen-C/phone.fst,./vits-zh-hf-fanchen-C/date.fst,./vits-zh-hf-fanchen-C/number.fst \ --sid=100 \ --debug=1 \ - --output-filename=./fanchen-100.wav \ + --output-filename=./fanchen-100-lexicon.wav \ --text="这是一个语音合成测试, 写于公元2024年4月26号, 11点05分，星期5。小米的使命是，始终坚持做'感动人心、价格厚道'的好产品，让全球每个人都能享受科技带来的美好生活。" diff --git a/dotnet-examples/offline-tts-play/run-matcha-en.sh b/dotnet-examples/offline-tts-play-lexicon/run-matcha-en-lexicon.sh similarity index 82% rename from dotnet-examples/offline-tts-play/run-matcha-en.sh rename to dotnet-examples/offline-tts-play-lexicon/run-matcha-en-lexicon.sh index 4a40f5bdf..bdaee7c8b 100755 --- a/dotnet-examples/offline-tts-play/run-matcha-en.sh +++ b/dotnet-examples/offline-tts-play-lexicon/run-matcha-en-lexicon.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash set -ex @@ -16,11 +16,15 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + dotnet run \ --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ --matcha-vocoder=./vocos-22khz-univ.onnx \ --tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ - --data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --lexicon=./lexicon-en-us.txt \ --debug=1 \ - --output-filename=./matcha-en.wav \ + --output-filename=./matcha-en-lexicon.wav \ --text='Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.' diff --git a/dotnet-examples/offline-tts-play/run-matcha-zh.sh b/dotnet-examples/offline-tts-play-lexicon/run-matcha-zh-lexicon.sh similarity index 94% rename from dotnet-examples/offline-tts-play/run-matcha-zh.sh rename to dotnet-examples/offline-tts-play-lexicon/run-matcha-zh-lexicon.sh index 36a8ddd62..225b71129 100755 --- a/dotnet-examples/offline-tts-play/run-matcha-zh.sh +++ b/dotnet-examples/offline-tts-play-lexicon/run-matcha-zh-lexicon.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash set -ex # please visit @@ -22,5 +22,5 @@ dotnet run \ --tokens=./matcha-icefall-zh-baker/tokens.txt \ --tts-rule-fsts=./matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst \ --debug=1 \ - --output-filename=./matcha-zh.wav \ + --output-filename=./matcha-zh-lexicon.wav \ --text="某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。" diff --git a/dotnet-examples/offline-tts-play-lexicon/run-piper-lexicon.sh b/dotnet-examples/offline-tts-play-lexicon/run-piper-lexicon.sh new file mode 100755 index 000000000..db93dffde --- /dev/null +++ b/dotnet-examples/offline-tts-play-lexicon/run-piper-lexicon.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -ex +if [ ! -f ./vits-piper-en_US-amy-low/en_US-amy-low.onnx ]; then + curl -OL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + tar xf vits-piper-en_US-amy-low.tar.bz2 + rm vits-piper-en_US-amy-low.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +dotnet run \ + --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ + --tokens=./vits-piper-en_US-amy-low/tokens.txt \ + --lexicon=./lexicon-en-us.txt \ + --debug=1 \ + --output-filename=./amy-lexicon.wav \ + --text="This is a text to speech application in dotnet with Next Generation Kaldi" + diff --git a/dotnet-examples/offline-zipformer-ctc-hlg/Program.cs b/dotnet-examples/offline-zipformer-ctc-hlg/Program.cs new file mode 100644 index 000000000..f81d0bd99 --- /dev/null +++ b/dotnet-examples/offline-zipformer-ctc-hlg/Program.cs @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file shows how to use a non-streaming zipformer CTC model with an HLG +// decoding graph via sherpa-onnx's .NET API. +// +// The HLG graph is passed to the C API through CtcFstDecoderConfig. +// +// 1. Download the model for testing +// +// curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// +// 2. Now run it +// +// dotnet run -c Release + +using SherpaOnnx; + +class OfflineZipformerCtcHlgDemo +{ + static void Main(string[] args) + { + var config = new OfflineRecognizerConfig(); + config.FeatConfig.SampleRate = 16000; + config.FeatConfig.FeatureDim = 80; + config.ModelConfig.ZipformerCtc.Model = "./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx"; + config.ModelConfig.Tokens = "./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt"; + config.ModelConfig.Provider = "cpu"; + config.ModelConfig.NumThreads = 1; + config.ModelConfig.Debug = 0; + + // This is what the example exercises: the graph and max_active reach the C + // API through CtcFstDecoderConfig. + config.CtcFstDecoderConfig.Graph = "./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst"; + config.CtcFstDecoderConfig.MaxActive = 3000; + + var recognizer = new OfflineRecognizer(config); + + var filename = "./sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav"; + var waveReader = new WaveReader(filename); + var s = recognizer.CreateStream(); + s.AcceptWaveform(waveReader.SampleRate, waveReader.Samples); + + recognizer.Decode(s); + + var r = s.Result; + Console.WriteLine("--------------------"); + Console.WriteLine(filename); + Console.WriteLine("text: {0}", r.Text); + Console.WriteLine("tokens: [{0}]", string.Join(", ", r.Tokens)); + Console.WriteLine("--------------------"); + } +} diff --git a/dotnet-examples/offline-zipformer-ctc-hlg/offline-zipformer-ctc-hlg.csproj b/dotnet-examples/offline-zipformer-ctc-hlg/offline-zipformer-ctc-hlg.csproj new file mode 100644 index 000000000..3d7bb856e --- /dev/null +++ b/dotnet-examples/offline-zipformer-ctc-hlg/offline-zipformer-ctc-hlg.csproj @@ -0,0 +1,15 @@ + + + + Exe + net8.0 + offline_zipformer_ctc_hlg + enable + enable + + + + + + + diff --git a/dotnet-examples/offline-zipformer-ctc-hlg/run.sh b/dotnet-examples/offline-zipformer-ctc-hlg/run.sh new file mode 100755 index 000000000..441d84ff6 --- /dev/null +++ b/dotnet-examples/offline-zipformer-ctc-hlg/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -f ./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +fi + +dotnet run -c Release diff --git a/dotnet-examples/sherpa-onnx.sln b/dotnet-examples/sherpa-onnx.sln index 5a4b8e312..991ddf4d6 100644 --- a/dotnet-examples/sherpa-onnx.sln +++ b/dotnet-examples/sherpa-onnx.sln @@ -1,253 +1,274 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.31903.59 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "online-decode-files", "online-decode-files\online-decode-files.csproj", "{45307474-BECB-4ABE-9388-D01D55A1A9BE}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-decode-files", "offline-decode-files\offline-decode-files.csproj", "{2DAB152C-9E24-47A0-9DB0-781297ECE458}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "speech-recognition-from-microphone", "speech-recognition-from-microphone\speech-recognition-from-microphone.csproj", "{FE4EA1FF-062A-46B3-B78D-C828FED7B82E}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-tts", "offline-tts\offline-tts.csproj", "{72196886-7143-4043-96E2-BCACEC6C79EB}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-tts-play", "offline-tts-play\offline-tts-play.csproj", "{40781464-5948-462B-BA4B-98932711513F}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spoken-language-identification", "spoken-language-identification\spoken-language-identification.csproj", "{3D7CF3D6-AC45-4D50-9619-5687B1443E94}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "streaming-hlg-decoding", "streaming-hlg-decoding\streaming-hlg-decoding.csproj", "{C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "speaker-identification", "speaker-identification\speaker-identification.csproj", "{2B1B140E-A92F-426B-B0DF-5D916B67304F}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-punctuation", "offline-punctuation\offline-punctuation.csproj", "{42D85582-BB63-4259-A4EA-837D66AC078B}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vad-non-streaming-asr-paraformer", "vad-non-streaming-asr-paraformer\vad-non-streaming-asr-paraformer.csproj", "{8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Common", "Common\Common.csproj", "{401E963F-E25A-43CE-987D-8DB2D4715756}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "keyword-spotting-from-files", "keyword-spotting-from-files\keyword-spotting-from-files.csproj", "{A87EDD31-D654-4C9F-AED7-F6F2825659BD}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "keyword-spotting-from-microphone", "keyword-spotting-from-microphone\keyword-spotting-from-microphone.csproj", "{AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-speaker-diarization", "offline-speaker-diarization\offline-speaker-diarization.csproj", "{D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kokoro-tts", "kokoro-tts\kokoro-tts.csproj", "{9C0ABE6C-1F54-42B5-804E-C3FED6668F52}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kokoro-tts-play", "kokoro-tts-play\kokoro-tts-play.csproj", "{EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "speech-enhancement-gtcrn", "speech-enhancement-gtcrn\speech-enhancement-gtcrn.csproj", "{DF2569C6-6011-4716-9538-F9E9069E00EB}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "speech-enhancement-dpdfnet", "speech-enhancement-dpdfnet\speech-enhancement-dpdfnet.csproj", "{016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "version-test", "version-test\version-test.csproj", "{E57711E5-6546-4BA0-B627-79C94F415BC5}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-canary-decode-files", "non-streaming-canary-decode-files\non-streaming-canary-decode-files.csproj", "{925779DB-4429-4366-87C3-B14DD44AE1D4}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kitten-tts", "kitten-tts\kitten-tts.csproj", "{E5AB574B-9E31-45D4-9B75-1C1892241E41}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kitten-tts-play", "kitten-tts-play\kitten-tts-play.csproj", "{D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "offline-audio-tagging", "offline-audio-tagging\offline-audio-tagging.csproj", "{0EBE2CE5-8940-4472-8A38-6A0E976E678F}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-funasr-nano-decode-files", "non-streaming-funasr-nano-decode-files\non-streaming-funasr-nano-decode-files.csproj", "{32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "vad-non-streaming-funasr-nano", "vad-non-streaming-funasr-nano\vad-non-streaming-funasr-nano.csproj", "{32C8C12B-D7DB-455E-B35C-945A745520CC}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "pocket-tts-zero-shot", "pocket-tts-zero-shot\pocket-tts-zero-shot.csproj", "{9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "pocket-tts-zero-shot-play", "pocket-tts-zero-shot-play\pocket-tts-zero-shot-play.csproj", "{0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-moonshine-v2-decode-files", "non-streaming-moonshine-v2-decode-files\non-streaming-moonshine-v2-decode-files.csproj", "{C9E5A6D3-02F4-46DE-808B-5163348F45B3}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "supertonic-tts", "supertonic-tts\supertonic-tts.csproj", "{A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "streaming-speech-enhancement-gtcrn", "streaming-speech-enhancement-gtcrn\streaming-speech-enhancement-gtcrn.csproj", "{5B87496C-EF81-4232-A448-6308F8E5A18C}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "streaming-speech-enhancement-dpdfnet", "streaming-speech-enhancement-dpdfnet\streaming-speech-enhancement-dpdfnet.csproj", "{8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "zipvoice-tts", "zipvoice-tts\zipvoice-tts.csproj", "{BBC69A08-01A7-4F89-938F-F0D551AD3F6C}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "zipvoice-tts-play", "zipvoice-tts-play\zipvoice-tts-play.csproj", "{84A37E18-095E-42A6-93CC-C27CD90B8478}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "online-punctuation", "online-punctuation\online-punctuation.csproj", "{210F6E27-E7FC-4185-BE4B-9F6703ECDA29}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "source-separation-spleeter", "source-separation-spleeter\source-separation-spleeter.csproj", "{A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "source-separation-uvr", "source-separation-uvr\source-separation-uvr.csproj", "{D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-qwen3-asr-decode-files", "non-streaming-qwen3-asr-decode-files\non-streaming-qwen3-asr-decode-files.csproj", "{B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "vad-non-streaming-qwen3-asr", "vad-non-streaming-qwen3-asr\vad-non-streaming-qwen3-asr.csproj", "{E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-cohere-transcribe-decode-files", "non-streaming-cohere-transcribe-decode-files\non-streaming-cohere-transcribe-decode-files.csproj", "{32B1BF23-74BB-46BB-BBDD-EC9C79A69448}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {45307474-BECB-4ABE-9388-D01D55A1A9BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {45307474-BECB-4ABE-9388-D01D55A1A9BE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {45307474-BECB-4ABE-9388-D01D55A1A9BE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {45307474-BECB-4ABE-9388-D01D55A1A9BE}.Release|Any CPU.Build.0 = Release|Any CPU - {2DAB152C-9E24-47A0-9DB0-781297ECE458}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2DAB152C-9E24-47A0-9DB0-781297ECE458}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2DAB152C-9E24-47A0-9DB0-781297ECE458}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2DAB152C-9E24-47A0-9DB0-781297ECE458}.Release|Any CPU.Build.0 = Release|Any CPU - {FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Release|Any CPU.Build.0 = Release|Any CPU - {72196886-7143-4043-96E2-BCACEC6C79EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {72196886-7143-4043-96E2-BCACEC6C79EB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {72196886-7143-4043-96E2-BCACEC6C79EB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {72196886-7143-4043-96E2-BCACEC6C79EB}.Release|Any CPU.Build.0 = Release|Any CPU - {40781464-5948-462B-BA4B-98932711513F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {40781464-5948-462B-BA4B-98932711513F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {40781464-5948-462B-BA4B-98932711513F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {40781464-5948-462B-BA4B-98932711513F}.Release|Any CPU.Build.0 = Release|Any CPU - {3D7CF3D6-AC45-4D50-9619-5687B1443E94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3D7CF3D6-AC45-4D50-9619-5687B1443E94}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3D7CF3D6-AC45-4D50-9619-5687B1443E94}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3D7CF3D6-AC45-4D50-9619-5687B1443E94}.Release|Any CPU.Build.0 = Release|Any CPU - {C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}.Release|Any CPU.Build.0 = Release|Any CPU - {2B1B140E-A92F-426B-B0DF-5D916B67304F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2B1B140E-A92F-426B-B0DF-5D916B67304F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2B1B140E-A92F-426B-B0DF-5D916B67304F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2B1B140E-A92F-426B-B0DF-5D916B67304F}.Release|Any CPU.Build.0 = Release|Any CPU - {42D85582-BB63-4259-A4EA-837D66AC078B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {42D85582-BB63-4259-A4EA-837D66AC078B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {42D85582-BB63-4259-A4EA-837D66AC078B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {42D85582-BB63-4259-A4EA-837D66AC078B}.Release|Any CPU.Build.0 = Release|Any CPU - {8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}.Release|Any CPU.Build.0 = Release|Any CPU - {401E963F-E25A-43CE-987D-8DB2D4715756}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {401E963F-E25A-43CE-987D-8DB2D4715756}.Debug|Any CPU.Build.0 = Debug|Any CPU - {401E963F-E25A-43CE-987D-8DB2D4715756}.Release|Any CPU.ActiveCfg = Release|Any CPU - {401E963F-E25A-43CE-987D-8DB2D4715756}.Release|Any CPU.Build.0 = Release|Any CPU - {A87EDD31-D654-4C9F-AED7-F6F2825659BD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A87EDD31-D654-4C9F-AED7-F6F2825659BD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A87EDD31-D654-4C9F-AED7-F6F2825659BD}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A87EDD31-D654-4C9F-AED7-F6F2825659BD}.Release|Any CPU.Build.0 = Release|Any CPU - {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Release|Any CPU.Build.0 = Release|Any CPU - {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Release|Any CPU.Build.0 = Release|Any CPU - {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Release|Any CPU.Build.0 = Release|Any CPU - {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Release|Any CPU.Build.0 = Release|Any CPU - {DF2569C6-6011-4716-9538-F9E9069E00EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DF2569C6-6011-4716-9538-F9E9069E00EB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DF2569C6-6011-4716-9538-F9E9069E00EB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DF2569C6-6011-4716-9538-F9E9069E00EB}.Release|Any CPU.Build.0 = Release|Any CPU - {016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}.Debug|Any CPU.Build.0 = Debug|Any CPU - {016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}.Release|Any CPU.ActiveCfg = Release|Any CPU - {016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}.Release|Any CPU.Build.0 = Release|Any CPU - {E57711E5-6546-4BA0-B627-79C94F415BC5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E57711E5-6546-4BA0-B627-79C94F415BC5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E57711E5-6546-4BA0-B627-79C94F415BC5}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E57711E5-6546-4BA0-B627-79C94F415BC5}.Release|Any CPU.Build.0 = Release|Any CPU - {925779DB-4429-4366-87C3-B14DD44AE1D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {925779DB-4429-4366-87C3-B14DD44AE1D4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {925779DB-4429-4366-87C3-B14DD44AE1D4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {925779DB-4429-4366-87C3-B14DD44AE1D4}.Release|Any CPU.Build.0 = Release|Any CPU - {E5AB574B-9E31-45D4-9B75-1C1892241E41}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E5AB574B-9E31-45D4-9B75-1C1892241E41}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E5AB574B-9E31-45D4-9B75-1C1892241E41}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E5AB574B-9E31-45D4-9B75-1C1892241E41}.Release|Any CPU.Build.0 = Release|Any CPU - {D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}.Release|Any CPU.Build.0 = Release|Any CPU - {0EBE2CE5-8940-4472-8A38-6A0E976E678F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0EBE2CE5-8940-4472-8A38-6A0E976E678F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0EBE2CE5-8940-4472-8A38-6A0E976E678F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0EBE2CE5-8940-4472-8A38-6A0E976E678F}.Release|Any CPU.Build.0 = Release|Any CPU - {32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}.Release|Any CPU.Build.0 = Release|Any CPU - {32C8C12B-D7DB-455E-B35C-945A745520CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {32C8C12B-D7DB-455E-B35C-945A745520CC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {32C8C12B-D7DB-455E-B35C-945A745520CC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {32C8C12B-D7DB-455E-B35C-945A745520CC}.Release|Any CPU.Build.0 = Release|Any CPU - {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Release|Any CPU.Build.0 = Release|Any CPU - {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Release|Any CPU.Build.0 = Release|Any CPU - {C9E5A6D3-02F4-46DE-808B-5163348F45B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C9E5A6D3-02F4-46DE-808B-5163348F45B3}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C9E5A6D3-02F4-46DE-808B-5163348F45B3}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C9E5A6D3-02F4-46DE-808B-5163348F45B3}.Release|Any CPU.Build.0 = Release|Any CPU - {A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}.Release|Any CPU.Build.0 = Release|Any CPU - {5B87496C-EF81-4232-A448-6308F8E5A18C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5B87496C-EF81-4232-A448-6308F8E5A18C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5B87496C-EF81-4232-A448-6308F8E5A18C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5B87496C-EF81-4232-A448-6308F8E5A18C}.Release|Any CPU.Build.0 = Release|Any CPU - {8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}.Release|Any CPU.Build.0 = Release|Any CPU - {BBC69A08-01A7-4F89-938F-F0D551AD3F6C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {BBC69A08-01A7-4F89-938F-F0D551AD3F6C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {BBC69A08-01A7-4F89-938F-F0D551AD3F6C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {BBC69A08-01A7-4F89-938F-F0D551AD3F6C}.Release|Any CPU.Build.0 = Release|Any CPU - {84A37E18-095E-42A6-93CC-C27CD90B8478}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {84A37E18-095E-42A6-93CC-C27CD90B8478}.Debug|Any CPU.Build.0 = Debug|Any CPU - {84A37E18-095E-42A6-93CC-C27CD90B8478}.Release|Any CPU.ActiveCfg = Release|Any CPU - {84A37E18-095E-42A6-93CC-C27CD90B8478}.Release|Any CPU.Build.0 = Release|Any CPU - {210F6E27-E7FC-4185-BE4B-9F6703ECDA29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {210F6E27-E7FC-4185-BE4B-9F6703ECDA29}.Debug|Any CPU.Build.0 = Debug|Any CPU - {210F6E27-E7FC-4185-BE4B-9F6703ECDA29}.Release|Any CPU.ActiveCfg = Release|Any CPU - {210F6E27-E7FC-4185-BE4B-9F6703ECDA29}.Release|Any CPU.Build.0 = Release|Any CPU - {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}.Release|Any CPU.Build.0 = Release|Any CPU - {D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}.Release|Any CPU.Build.0 = Release|Any CPU - {B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}.Release|Any CPU.Build.0 = Release|Any CPU - {E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}.Release|Any CPU.Build.0 = Release|Any CPU - {32B1BF23-74BB-46BB-BBDD-EC9C79A69448}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {32B1BF23-74BB-46BB-BBDD-EC9C79A69448}.Debug|Any CPU.Build.0 = Debug|Any CPU - {32B1BF23-74BB-46BB-BBDD-EC9C79A69448}.Release|Any CPU.ActiveCfg = Release|Any CPU - {32B1BF23-74BB-46BB-BBDD-EC9C79A69448}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {07A6023C-0A37-4F82-A29F-896A3A338EAC} - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "online-decode-files", "online-decode-files\online-decode-files.csproj", "{45307474-BECB-4ABE-9388-D01D55A1A9BE}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-decode-files", "offline-decode-files\offline-decode-files.csproj", "{2DAB152C-9E24-47A0-9DB0-781297ECE458}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "speech-recognition-from-microphone", "speech-recognition-from-microphone\speech-recognition-from-microphone.csproj", "{FE4EA1FF-062A-46B3-B78D-C828FED7B82E}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-tts-lexicon", "offline-tts-lexicon\offline-tts-lexicon.csproj", "{72196886-7143-4043-96E2-BCACEC6C79EB}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-tts-play-lexicon", "offline-tts-play-lexicon\offline-tts-play-lexicon.csproj", "{40781464-5948-462B-BA4B-98932711513F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spoken-language-identification", "spoken-language-identification\spoken-language-identification.csproj", "{3D7CF3D6-AC45-4D50-9619-5687B1443E94}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "streaming-hlg-decoding", "streaming-hlg-decoding\streaming-hlg-decoding.csproj", "{C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "speaker-identification", "speaker-identification\speaker-identification.csproj", "{2B1B140E-A92F-426B-B0DF-5D916B67304F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-punctuation", "offline-punctuation\offline-punctuation.csproj", "{42D85582-BB63-4259-A4EA-837D66AC078B}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vad-non-streaming-asr-paraformer", "vad-non-streaming-asr-paraformer\vad-non-streaming-asr-paraformer.csproj", "{8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Common", "Common\Common.csproj", "{401E963F-E25A-43CE-987D-8DB2D4715756}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "keyword-spotting-from-files", "keyword-spotting-from-files\keyword-spotting-from-files.csproj", "{A87EDD31-D654-4C9F-AED7-F6F2825659BD}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "keyword-spotting-from-microphone", "keyword-spotting-from-microphone\keyword-spotting-from-microphone.csproj", "{AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "offline-speaker-diarization", "offline-speaker-diarization\offline-speaker-diarization.csproj", "{D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kokoro-tts-lexicon", "kokoro-tts-lexicon\kokoro-tts-lexicon.csproj", "{9C0ABE6C-1F54-42B5-804E-C3FED6668F52}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kokoro-tts-play-lexicon", "kokoro-tts-play-lexicon\kokoro-tts-play-lexicon.csproj", "{EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "speech-enhancement-gtcrn", "speech-enhancement-gtcrn\speech-enhancement-gtcrn.csproj", "{DF2569C6-6011-4716-9538-F9E9069E00EB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "speech-enhancement-dpdfnet", "speech-enhancement-dpdfnet\speech-enhancement-dpdfnet.csproj", "{016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "version-test", "version-test\version-test.csproj", "{E57711E5-6546-4BA0-B627-79C94F415BC5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-canary-decode-files", "non-streaming-canary-decode-files\non-streaming-canary-decode-files.csproj", "{925779DB-4429-4366-87C3-B14DD44AE1D4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kitten-tts-lexicon", "kitten-tts-lexicon\kitten-tts-lexicon.csproj", "{E5AB574B-9E31-45D4-9B75-1C1892241E41}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kitten-tts-play-lexicon", "kitten-tts-play-lexicon\kitten-tts-play-lexicon.csproj", "{D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "offline-audio-tagging", "offline-audio-tagging\offline-audio-tagging.csproj", "{0EBE2CE5-8940-4472-8A38-6A0E976E678F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-funasr-nano-decode-files", "non-streaming-funasr-nano-decode-files\non-streaming-funasr-nano-decode-files.csproj", "{32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "vad-non-streaming-funasr-nano", "vad-non-streaming-funasr-nano\vad-non-streaming-funasr-nano.csproj", "{32C8C12B-D7DB-455E-B35C-945A745520CC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "pocket-tts-zero-shot", "pocket-tts-zero-shot\pocket-tts-zero-shot.csproj", "{9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "pocket-tts-zero-shot-play", "pocket-tts-zero-shot-play\pocket-tts-zero-shot-play.csproj", "{0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-moonshine-v2-decode-files", "non-streaming-moonshine-v2-decode-files\non-streaming-moonshine-v2-decode-files.csproj", "{C9E5A6D3-02F4-46DE-808B-5163348F45B3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "supertonic-tts", "supertonic-tts\supertonic-tts.csproj", "{A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "streaming-speech-enhancement-gtcrn", "streaming-speech-enhancement-gtcrn\streaming-speech-enhancement-gtcrn.csproj", "{5B87496C-EF81-4232-A448-6308F8E5A18C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "streaming-speech-enhancement-dpdfnet", "streaming-speech-enhancement-dpdfnet\streaming-speech-enhancement-dpdfnet.csproj", "{8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "zipvoice-tts-lexicon", "zipvoice-tts-lexicon\zipvoice-tts-lexicon.csproj", "{BBC69A08-01A7-4F89-938F-F0D551AD3F6C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "zipvoice-tts-play-lexicon", "zipvoice-tts-play-lexicon\zipvoice-tts-play-lexicon.csproj", "{84A37E18-095E-42A6-93CC-C27CD90B8478}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "online-punctuation", "online-punctuation\online-punctuation.csproj", "{210F6E27-E7FC-4185-BE4B-9F6703ECDA29}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "source-separation-spleeter", "source-separation-spleeter\source-separation-spleeter.csproj", "{A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "source-separation-uvr", "source-separation-uvr\source-separation-uvr.csproj", "{D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-qwen3-asr-decode-files", "non-streaming-qwen3-asr-decode-files\non-streaming-qwen3-asr-decode-files.csproj", "{B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "vad-non-streaming-qwen3-asr", "vad-non-streaming-qwen3-asr\vad-non-streaming-qwen3-asr.csproj", "{E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-cohere-transcribe-decode-files", "non-streaming-cohere-transcribe-decode-files\non-streaming-cohere-transcribe-decode-files.csproj", "{32B1BF23-74BB-46BB-BBDD-EC9C79A69448}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "offline-tts-phonemize", "offline-tts-phonemize\offline-tts-phonemize.csproj", "{075B72D7-D8BB-468F-AEE2-E449E34B7D0C}" +EndProject + +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kitten-tts-phonemize", "kitten-tts-phonemize\kitten-tts-phonemize.csproj", "{1DC1A4A8-9AC6-4057-9A5D-954D2F81672C}" +EndProject + +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "kokoro-tts-phonemize", "kokoro-tts-phonemize\kokoro-tts-phonemize.csproj", "{E5B2E5FB-4043-4BF3-9B05-CB67DB149BE8}" +EndProject + +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {075B72D7-D8BB-468F-AEE2-E449E34B7D0C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {075B72D7-D8BB-468F-AEE2-E449E34B7D0C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {075B72D7-D8BB-468F-AEE2-E449E34B7D0C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {075B72D7-D8BB-468F-AEE2-E449E34B7D0C}.Release|Any CPU.Build.0 = Release|Any CPU + {1DC1A4A8-9AC6-4057-9A5D-954D2F81672C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1DC1A4A8-9AC6-4057-9A5D-954D2F81672C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1DC1A4A8-9AC6-4057-9A5D-954D2F81672C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1DC1A4A8-9AC6-4057-9A5D-954D2F81672C}.Release|Any CPU.Build.0 = Release|Any CPU + {E5B2E5FB-4043-4BF3-9B05-CB67DB149BE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5B2E5FB-4043-4BF3-9B05-CB67DB149BE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5B2E5FB-4043-4BF3-9B05-CB67DB149BE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5B2E5FB-4043-4BF3-9B05-CB67DB149BE8}.Release|Any CPU.Build.0 = Release|Any CPU + {45307474-BECB-4ABE-9388-D01D55A1A9BE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {45307474-BECB-4ABE-9388-D01D55A1A9BE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {45307474-BECB-4ABE-9388-D01D55A1A9BE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {45307474-BECB-4ABE-9388-D01D55A1A9BE}.Release|Any CPU.Build.0 = Release|Any CPU + {2DAB152C-9E24-47A0-9DB0-781297ECE458}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2DAB152C-9E24-47A0-9DB0-781297ECE458}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2DAB152C-9E24-47A0-9DB0-781297ECE458}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2DAB152C-9E24-47A0-9DB0-781297ECE458}.Release|Any CPU.Build.0 = Release|Any CPU + {FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Release|Any CPU.Build.0 = Release|Any CPU + {72196886-7143-4043-96E2-BCACEC6C79EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {72196886-7143-4043-96E2-BCACEC6C79EB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {72196886-7143-4043-96E2-BCACEC6C79EB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {72196886-7143-4043-96E2-BCACEC6C79EB}.Release|Any CPU.Build.0 = Release|Any CPU + {40781464-5948-462B-BA4B-98932711513F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {40781464-5948-462B-BA4B-98932711513F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {40781464-5948-462B-BA4B-98932711513F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {40781464-5948-462B-BA4B-98932711513F}.Release|Any CPU.Build.0 = Release|Any CPU + {3D7CF3D6-AC45-4D50-9619-5687B1443E94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3D7CF3D6-AC45-4D50-9619-5687B1443E94}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3D7CF3D6-AC45-4D50-9619-5687B1443E94}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3D7CF3D6-AC45-4D50-9619-5687B1443E94}.Release|Any CPU.Build.0 = Release|Any CPU + {C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4A368A5-FCA0-419D-97C9-C8CE0B08EB99}.Release|Any CPU.Build.0 = Release|Any CPU + {2B1B140E-A92F-426B-B0DF-5D916B67304F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2B1B140E-A92F-426B-B0DF-5D916B67304F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2B1B140E-A92F-426B-B0DF-5D916B67304F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2B1B140E-A92F-426B-B0DF-5D916B67304F}.Release|Any CPU.Build.0 = Release|Any CPU + {42D85582-BB63-4259-A4EA-837D66AC078B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {42D85582-BB63-4259-A4EA-837D66AC078B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {42D85582-BB63-4259-A4EA-837D66AC078B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {42D85582-BB63-4259-A4EA-837D66AC078B}.Release|Any CPU.Build.0 = Release|Any CPU + {8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8CD6B7E5-F59F-47B3-BB87-2B2E3678924D}.Release|Any CPU.Build.0 = Release|Any CPU + {401E963F-E25A-43CE-987D-8DB2D4715756}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {401E963F-E25A-43CE-987D-8DB2D4715756}.Debug|Any CPU.Build.0 = Debug|Any CPU + {401E963F-E25A-43CE-987D-8DB2D4715756}.Release|Any CPU.ActiveCfg = Release|Any CPU + {401E963F-E25A-43CE-987D-8DB2D4715756}.Release|Any CPU.Build.0 = Release|Any CPU + {A87EDD31-D654-4C9F-AED7-F6F2825659BD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A87EDD31-D654-4C9F-AED7-F6F2825659BD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A87EDD31-D654-4C9F-AED7-F6F2825659BD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A87EDD31-D654-4C9F-AED7-F6F2825659BD}.Release|Any CPU.Build.0 = Release|Any CPU + {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AEE0ED2B-C86F-4952-863C-EAD3219CB4EC}.Release|Any CPU.Build.0 = Release|Any CPU + {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D3A1FF28-A77D-429D-AEAC-2BA77CA682BC}.Release|Any CPU.Build.0 = Release|Any CPU + {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C0ABE6C-1F54-42B5-804E-C3FED6668F52}.Release|Any CPU.Build.0 = Release|Any CPU + {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EC0BCEAB-1B4E-4129-82CE-9880426AFA0B}.Release|Any CPU.Build.0 = Release|Any CPU + {DF2569C6-6011-4716-9538-F9E9069E00EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DF2569C6-6011-4716-9538-F9E9069E00EB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DF2569C6-6011-4716-9538-F9E9069E00EB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DF2569C6-6011-4716-9538-F9E9069E00EB}.Release|Any CPU.Build.0 = Release|Any CPU + {016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}.Debug|Any CPU.Build.0 = Debug|Any CPU + {016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}.Release|Any CPU.ActiveCfg = Release|Any CPU + {016E5D0E-6D79-4AF6-B2C6-F0E091D78C00}.Release|Any CPU.Build.0 = Release|Any CPU + {E57711E5-6546-4BA0-B627-79C94F415BC5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E57711E5-6546-4BA0-B627-79C94F415BC5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E57711E5-6546-4BA0-B627-79C94F415BC5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E57711E5-6546-4BA0-B627-79C94F415BC5}.Release|Any CPU.Build.0 = Release|Any CPU + {925779DB-4429-4366-87C3-B14DD44AE1D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {925779DB-4429-4366-87C3-B14DD44AE1D4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {925779DB-4429-4366-87C3-B14DD44AE1D4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {925779DB-4429-4366-87C3-B14DD44AE1D4}.Release|Any CPU.Build.0 = Release|Any CPU + {E5AB574B-9E31-45D4-9B75-1C1892241E41}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5AB574B-9E31-45D4-9B75-1C1892241E41}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5AB574B-9E31-45D4-9B75-1C1892241E41}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5AB574B-9E31-45D4-9B75-1C1892241E41}.Release|Any CPU.Build.0 = Release|Any CPU + {D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D60A8A84-D6D3-4B79-A18A-1817BEBD35B9}.Release|Any CPU.Build.0 = Release|Any CPU + {0EBE2CE5-8940-4472-8A38-6A0E976E678F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0EBE2CE5-8940-4472-8A38-6A0E976E678F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0EBE2CE5-8940-4472-8A38-6A0E976E678F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0EBE2CE5-8940-4472-8A38-6A0E976E678F}.Release|Any CPU.Build.0 = Release|Any CPU + {32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {32F7534B-117E-4D1D-BAED-A1D1A6C6A62C}.Release|Any CPU.Build.0 = Release|Any CPU + {32C8C12B-D7DB-455E-B35C-945A745520CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {32C8C12B-D7DB-455E-B35C-945A745520CC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {32C8C12B-D7DB-455E-B35C-945A745520CC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {32C8C12B-D7DB-455E-B35C-945A745520CC}.Release|Any CPU.Build.0 = Release|Any CPU + {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Release|Any CPU.Build.0 = Release|Any CPU + {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Release|Any CPU.Build.0 = Release|Any CPU + {C9E5A6D3-02F4-46DE-808B-5163348F45B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C9E5A6D3-02F4-46DE-808B-5163348F45B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C9E5A6D3-02F4-46DE-808B-5163348F45B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C9E5A6D3-02F4-46DE-808B-5163348F45B3}.Release|Any CPU.Build.0 = Release|Any CPU + {A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A3B7C4D1-E5F6-4A8B-9C0D-1E2F3A4B5C6D}.Release|Any CPU.Build.0 = Release|Any CPU + {5B87496C-EF81-4232-A448-6308F8E5A18C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5B87496C-EF81-4232-A448-6308F8E5A18C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5B87496C-EF81-4232-A448-6308F8E5A18C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5B87496C-EF81-4232-A448-6308F8E5A18C}.Release|Any CPU.Build.0 = Release|Any CPU + {8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8CD66C3E-3AE3-43AA-8FDA-DD5BA456F2EC}.Release|Any CPU.Build.0 = Release|Any CPU + {BBC69A08-01A7-4F89-938F-F0D551AD3F6C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BBC69A08-01A7-4F89-938F-F0D551AD3F6C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BBC69A08-01A7-4F89-938F-F0D551AD3F6C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BBC69A08-01A7-4F89-938F-F0D551AD3F6C}.Release|Any CPU.Build.0 = Release|Any CPU + {84A37E18-095E-42A6-93CC-C27CD90B8478}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84A37E18-095E-42A6-93CC-C27CD90B8478}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84A37E18-095E-42A6-93CC-C27CD90B8478}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84A37E18-095E-42A6-93CC-C27CD90B8478}.Release|Any CPU.Build.0 = Release|Any CPU + {210F6E27-E7FC-4185-BE4B-9F6703ECDA29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {210F6E27-E7FC-4185-BE4B-9F6703ECDA29}.Debug|Any CPU.Build.0 = Debug|Any CPU + {210F6E27-E7FC-4185-BE4B-9F6703ECDA29}.Release|Any CPU.ActiveCfg = Release|Any CPU + {210F6E27-E7FC-4185-BE4B-9F6703ECDA29}.Release|Any CPU.Build.0 = Release|Any CPU + {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A1B2C3D4-E5F6-4A7B-8C9D-0E1F2A3B4C5D}.Release|Any CPU.Build.0 = Release|Any CPU + {D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D5E6F7A8-B9C0-4D1E-2F3A-4B5C6D7E8F9A}.Release|Any CPU.Build.0 = Release|Any CPU + {B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B1C2D3E4-F5A6-4B7C-8D9E-0F1A2B3C4D5E}.Release|Any CPU.Build.0 = Release|Any CPU + {E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E5F6A7B8-C9D0-4E1F-2A3B-4C5D6E7F8A9B}.Release|Any CPU.Build.0 = Release|Any CPU + {32B1BF23-74BB-46BB-BBDD-EC9C79A69448}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {32B1BF23-74BB-46BB-BBDD-EC9C79A69448}.Debug|Any CPU.Build.0 = Debug|Any CPU + {32B1BF23-74BB-46BB-BBDD-EC9C79A69448}.Release|Any CPU.ActiveCfg = Release|Any CPU + {32B1BF23-74BB-46BB-BBDD-EC9C79A69448}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {07A6023C-0A37-4F82-A29F-896A3A338EAC} + EndGlobalSection +EndGlobal diff --git a/dotnet-examples/zipvoice-tts/Program.cs b/dotnet-examples/zipvoice-tts-lexicon/Program.cs similarity index 90% rename from dotnet-examples/zipvoice-tts/Program.cs rename to dotnet-examples/zipvoice-tts-lexicon/Program.cs index aa41da63a..88b591bc9 100644 --- a/dotnet-examples/zipvoice-tts/Program.cs +++ b/dotnet-examples/zipvoice-tts-lexicon/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2026 Xiaomi Corporation +// Copyright (c) 2026 Xiaomi Corporation // // This file shows how to use a non-streaming ZipVoice model // for zero-shot text-to-speech. @@ -24,8 +24,7 @@ class ZipVoiceTtsDemo config.Model.ZipVoice.Encoder = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx"; config.Model.ZipVoice.Decoder = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx"; config.Model.ZipVoice.Vocoder = "./vocos_24khz.onnx"; - config.Model.ZipVoice.DataDir = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data"; - config.Model.ZipVoice.Lexicon = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt"; + config.Model.ZipVoice.Lexicon = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt"; config.Model.NumThreads = 2; config.Model.Debug = 1; @@ -59,7 +58,7 @@ class ZipVoiceTtsDemo var audio = tts.GenerateWithConfig(text, genConfig, callback); - var outputFilename = "./generated-zipvoice-zh-en.wav"; + var outputFilename = "./generated-zipvoice-zh-en-lexicon.wav"; var ok = audio.SaveToWaveFile(outputFilename); if (ok) diff --git a/dotnet-examples/zipvoice-tts-play/run.sh b/dotnet-examples/zipvoice-tts-lexicon/run-lexicon.sh similarity index 79% rename from dotnet-examples/zipvoice-tts-play/run.sh rename to dotnet-examples/zipvoice-tts-lexicon/run-lexicon.sh index ac0d024bf..508cf5eea 100755 --- a/dotnet-examples/zipvoice-tts-play/run.sh +++ b/dotnet-examples/zipvoice-tts-lexicon/run-lexicon.sh @@ -11,4 +11,8 @@ if [ ! -f ./vocos_24khz.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + dotnet run diff --git a/dotnet-examples/zipvoice-tts/zipvoice-tts.csproj b/dotnet-examples/zipvoice-tts-lexicon/zipvoice-tts-lexicon.csproj similarity index 100% rename from dotnet-examples/zipvoice-tts/zipvoice-tts.csproj rename to dotnet-examples/zipvoice-tts-lexicon/zipvoice-tts-lexicon.csproj diff --git a/dotnet-examples/zipvoice-tts-play/Program.cs b/dotnet-examples/zipvoice-tts-play-lexicon/Program.cs similarity index 96% rename from dotnet-examples/zipvoice-tts-play/Program.cs rename to dotnet-examples/zipvoice-tts-play-lexicon/Program.cs index 668f93b75..339c0e075 100644 --- a/dotnet-examples/zipvoice-tts-play/Program.cs +++ b/dotnet-examples/zipvoice-tts-play-lexicon/Program.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2026 Xiaomi Corporation +// Copyright (c) 2026 Xiaomi Corporation // // This file shows how to use a non-streaming ZipVoice model // for zero-shot text-to-speech with playback. @@ -26,8 +26,7 @@ class ZipVoiceTtsDemo config.Model.ZipVoice.Encoder = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx"; config.Model.ZipVoice.Decoder = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx"; config.Model.ZipVoice.Vocoder = "./vocos_24khz.onnx"; - config.Model.ZipVoice.DataDir = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data"; - config.Model.ZipVoice.Lexicon = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt"; + config.Model.ZipVoice.Lexicon = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt"; config.Model.NumThreads = 2; config.Model.Debug = 1; @@ -171,7 +170,7 @@ class ZipVoiceTtsDemo var callback = new OfflineTtsCallbackProgressWithArg(myCallback); var audio = tts.GenerateWithConfig(text, genConfig, callback); - var outputFilename = "./generated-zipvoice-zh-en-play.wav"; + var outputFilename = "./generated-zipvoice-zh-en-play-lexicon.wav"; var ok = audio.SaveToWaveFile(outputFilename); if (ok) diff --git a/dotnet-examples/zipvoice-tts/run.sh b/dotnet-examples/zipvoice-tts-play-lexicon/run-lexicon.sh similarity index 79% rename from dotnet-examples/zipvoice-tts/run.sh rename to dotnet-examples/zipvoice-tts-play-lexicon/run-lexicon.sh index ac0d024bf..508cf5eea 100755 --- a/dotnet-examples/zipvoice-tts/run.sh +++ b/dotnet-examples/zipvoice-tts-play-lexicon/run-lexicon.sh @@ -11,4 +11,8 @@ if [ ! -f ./vocos_24khz.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + dotnet run diff --git a/dotnet-examples/zipvoice-tts-play/zipvoice-tts-play.csproj b/dotnet-examples/zipvoice-tts-play-lexicon/zipvoice-tts-play-lexicon.csproj similarity index 100% rename from dotnet-examples/zipvoice-tts-play/zipvoice-tts-play.csproj rename to dotnet-examples/zipvoice-tts-play-lexicon/zipvoice-tts-play-lexicon.csproj diff --git a/flutter-examples/streaming_asr/android/app/build.gradle b/flutter-examples/streaming_asr/android/app/build.gradle index 853746a18..222ae93a2 100644 --- a/flutter-examples/streaming_asr/android/app/build.gradle +++ b/flutter-examples/streaming_asr/android/app/build.gradle @@ -25,7 +25,7 @@ if (flutterVersionName == null) { android { namespace = "com.k2fsa.sherpa.onnx.streaming_asr" - compileSdk = 36 + compileSdk = flutter.compileSdkVersion ndkVersion = "27.0.12077973" compileOptions { @@ -49,7 +49,7 @@ android { // You can update the following values to match your application needs. // For more information, see: https://docs.flutter.dev/deployment/android#reviewing-the-gradle-build-configuration. minSdk = 23 - targetSdk = 36 + targetSdk = flutter.targetSdkVersion versionCode = flutterVersionCode.toInteger() versionName = flutterVersionName } diff --git a/flutter-examples/streaming_asr/android/gradle/wrapper/gradle-wrapper.properties b/flutter-examples/streaming_asr/android/gradle/wrapper/gradle-wrapper.properties index efdcc4ace..e4ef43fb9 100644 --- a/flutter-examples/streaming_asr/android/gradle/wrapper/gradle-wrapper.properties +++ b/flutter-examples/streaming_asr/android/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-all.zip diff --git a/flutter-examples/streaming_asr/android/settings.gradle b/flutter-examples/streaming_asr/android/settings.gradle index 7b603ca2f..ff03f22be 100644 --- a/flutter-examples/streaming_asr/android/settings.gradle +++ b/flutter-examples/streaming_asr/android/settings.gradle @@ -18,8 +18,8 @@ pluginManagement { plugins { id "dev.flutter.flutter-plugin-loader" version "1.0.0" - id "com.android.application" version "8.9.1" apply false - id "org.jetbrains.kotlin.android" version "1.9.24" apply false + id "com.android.application" version "8.11.1" apply false + id "org.jetbrains.kotlin.android" version "2.2.20" apply false } include ":app" diff --git a/flutter-examples/streaming_asr/lib/online_model.dart b/flutter-examples/streaming_asr/lib/online_model.dart index 35084049a..410c65d57 100644 --- a/flutter-examples/streaming_asr/lib/online_model.dart +++ b/flutter-examples/streaming_asr/lib/online_model.dart @@ -62,6 +62,56 @@ Future getOnlineModelConfig( tokens: await copyAssetFile('$modelDir/tokens.txt'), modelType: 'zipformer', ); + case 4: + final modelDir = + 'assets/sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05'; + return sherpa_onnx.OnlineModelConfig( + transducer: sherpa_onnx.OnlineTransducerModelConfig( + encoder: await copyAssetFile('$modelDir/encoder.int8.onnx'), + decoder: await copyAssetFile('$modelDir/decoder.onnx'), + joiner: await copyAssetFile('$modelDir/joiner.int8.onnx'), + ), + tokens: await copyAssetFile('$modelDir/tokens.txt'), + modelType: 'zipformer2', + ); + case 5: + final modelDir = + 'assets/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11'; + return sherpa_onnx.OnlineModelConfig( + transducer: sherpa_onnx.OnlineTransducerModelConfig( + encoder: await copyAssetFile('$modelDir/encoder.int8.onnx'), + decoder: await copyAssetFile('$modelDir/decoder.int8.onnx'), + joiner: await copyAssetFile('$modelDir/joiner.int8.onnx'), + ), + tokens: await copyAssetFile('$modelDir/tokens.txt'), + // Like the CLI (see run-nemo.sh): leave the model type empty so it is + // auto-detected from the onnx files. + modelType: '', + ); + case 6: + final modelDir = + 'assets/sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05'; + return sherpa_onnx.OnlineModelConfig( + transducer: sherpa_onnx.OnlineTransducerModelConfig( + encoder: await copyAssetFile('$modelDir/encoder.int8.onnx'), + decoder: await copyAssetFile('$modelDir/decoder.onnx'), + joiner: await copyAssetFile('$modelDir/joiner.int8.onnx'), + ), + tokens: await copyAssetFile('$modelDir/tokens.txt'), + modelType: 'zipformer2', + ); + case 7: + final modelDir = + 'assets/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11'; + return sherpa_onnx.OnlineModelConfig( + transducer: sherpa_onnx.OnlineTransducerModelConfig( + encoder: await copyAssetFile('$modelDir/encoder.int8.onnx'), + decoder: await copyAssetFile('$modelDir/decoder.int8.onnx'), + joiner: await copyAssetFile('$modelDir/joiner.int8.onnx'), + ), + tokens: await copyAssetFile('$modelDir/tokens.txt'), + modelType: '', + ); default: throw ArgumentError('Unsupported type: $type'); } diff --git a/flutter-examples/tts/android/app/build.gradle b/flutter-examples/tts/android/app/build.gradle index 00b238311..7c71e8df8 100644 --- a/flutter-examples/tts/android/app/build.gradle +++ b/flutter-examples/tts/android/app/build.gradle @@ -25,7 +25,7 @@ if (flutterVersionName == null) { android { namespace = "com.k2fsa.sherpa.onnx.tts" - compileSdk = 35 + compileSdk = flutter.compileSdkVersion ndkVersion = flutter.ndkVersion compileOptions { @@ -49,7 +49,7 @@ android { // You can update the following values to match your application needs. // For more information, see: https://docs.flutter.dev/deployment/android#reviewing-the-gradle-build-configuration. minSdk = flutter.minSdkVersion - targetSdk = 35 + targetSdk = flutter.targetSdkVersion versionCode = flutterVersionCode.toInteger() versionName = flutterVersionName } diff --git a/flutter-examples/tts/android/gradle/wrapper/gradle-wrapper.properties b/flutter-examples/tts/android/gradle/wrapper/gradle-wrapper.properties index 81a4301fc..e4ef43fb9 100644 --- a/flutter-examples/tts/android/gradle/wrapper/gradle-wrapper.properties +++ b/flutter-examples/tts/android/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-all.zip diff --git a/flutter-examples/tts/android/settings.gradle b/flutter-examples/tts/android/settings.gradle index d9a6b79fc..ff03f22be 100644 --- a/flutter-examples/tts/android/settings.gradle +++ b/flutter-examples/tts/android/settings.gradle @@ -18,8 +18,8 @@ pluginManagement { plugins { id "dev.flutter.flutter-plugin-loader" version "1.0.0" - id "com.android.application" version "8.7.0" apply false - id "org.jetbrains.kotlin.android" version "1.9.24" apply false + id "com.android.application" version "8.11.1" apply false + id "org.jetbrains.kotlin.android" version "2.2.20" apply false } include ":app" diff --git a/flutter-examples/tts/lib/model.dart b/flutter-examples/tts/lib/model.dart index c9524a0ac..3b36264dd 100644 --- a/flutter-examples/tts/lib/model.dart +++ b/flutter-examples/tts/lib/model.dart @@ -27,9 +27,8 @@ Future prepareModelConfig() async { model: sherpa_onnx.OfflineTtsModelConfig( vits: sherpa_onnx.OfflineTtsVitsModelConfig( model: _abs(d, m.vits.model), - lexicon: _abs(d, m.vits.lexicon), + lexicon: _absMulti(d, m.vits.lexicon), tokens: _abs(d, m.vits.tokens), - dataDir: _abs(d, m.vits.dataDir), noiseScale: m.vits.noiseScale, noiseScaleW: m.vits.noiseScaleW, lengthScale: m.vits.lengthScale, @@ -38,7 +37,6 @@ Future prepareModelConfig() async { model: _abs(d, m.kokoro.model), voices: _abs(d, m.kokoro.voices), tokens: _abs(d, m.kokoro.tokens), - dataDir: _abs(d, m.kokoro.dataDir), lexicon: _absMulti(d, m.kokoro.lexicon), lang: m.kokoro.lang, lengthScale: m.kokoro.lengthScale, @@ -47,15 +45,14 @@ Future prepareModelConfig() async { model: _abs(d, m.kitten.model), voices: _abs(d, m.kitten.voices), tokens: _abs(d, m.kitten.tokens), - dataDir: _abs(d, m.kitten.dataDir), + lexicon: _absMulti(d, m.kitten.lexicon), lengthScale: m.kitten.lengthScale, ), matcha: sherpa_onnx.OfflineTtsMatchaModelConfig( acousticModel: _abs(d, m.matcha.acousticModel), vocoder: _abs(d, m.matcha.vocoder), tokens: _abs(d, m.matcha.tokens), - dataDir: _abs(d, m.matcha.dataDir), - lexicon: _abs(d, m.matcha.lexicon), + lexicon: _absMulti(d, m.matcha.lexicon), noiseScale: m.matcha.noiseScale, lengthScale: m.matcha.lengthScale, ), @@ -83,8 +80,7 @@ Future prepareModelConfig() async { encoder: _abs(d, m.zipvoice.encoder), decoder: _abs(d, m.zipvoice.decoder), vocoder: _abs(d, m.zipvoice.vocoder), - dataDir: _abs(d, m.zipvoice.dataDir), - lexicon: _abs(d, m.zipvoice.lexicon), + lexicon: _absMulti(d, m.zipvoice.lexicon), featScale: m.zipvoice.featScale, tShift: m.zipvoice.tShift, targetRms: m.zipvoice.targetRms, @@ -96,7 +92,7 @@ Future prepareModelConfig() async { ), ruleFsts: _absMulti(d, cfg.ruleFsts), ruleFars: _absMulti(d, cfg.ruleFars), - maxNumSenetences: cfg.maxNumSenetences, + maxNumSentences: cfg.maxNumSentences, ); } diff --git a/flutter-examples/tts/lib/model_config.dart b/flutter-examples/tts/lib/model_config.dart index 913e77c1b..20d25326b 100644 --- a/flutter-examples/tts/lib/model_config.dart +++ b/flutter-examples/tts/lib/model_config.dart @@ -4,6 +4,29 @@ // import 'package:sherpa_onnx/sherpa_onnx.dart'; +// Choose exactly one of the two text frontends below (not both): +// +// usePhonemize = true // option 1 (default): the text is phonemized with the +// piper_phonemize package and passed to sherpa-onnx; no extra files needed. +// usePhonemize = false // option 2: lexicon — download lexicon-en-us.txt and +// put it next to the model files: +// https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +// +// Neither option uses the espeak-ng-data directory shipped in the model +// package; you can leave it out to keep the download small. +// +// Only the English piper-family models (indices 0, 2, 3, 4, 6, 8) can use +// either frontend. The Chinese/multi-lingual models always use their own +// lexicon; pocket (9) and supertonic (10) bring their own frontend. +const bool usePhonemize = true; + +/// Models that accept either text frontend (English piper-family). +const Set phonemizeCapableModels = {0, 2, 3, 4, 6, 8}; + +/// Whether the selected model phonemizes its input with piper_phonemize. +final bool selectedModelUsesPhonemize = + usePhonemize && phonemizeCapableModels.contains(selectedModelIndex); + // Change the index below to select a different model. /// Select which TTS model to use (0-10). const int selectedModelIndex = 0; @@ -41,12 +64,12 @@ final OfflineTtsConfig selectedTtsConfig = switch (selectedModelIndex) { vits: OfflineTtsVitsModelConfig( model: 'vits-piper-en_US-amy-low/en_US-amy-low.onnx', tokens: 'vits-piper-en_US-amy-low/tokens.txt', - dataDir: 'vits-piper-en_US-amy-low/espeak-ng-data', + lexicon: usePhonemize ? '' : 'lexicon-en-us.txt', ), numThreads: 2, debug: true, ), - maxNumSenetences: 1, + maxNumSentences: 1, ), // ── VITS Piper (Chinese) ───────────────────────────────────────────── @@ -61,7 +84,7 @@ final OfflineTtsConfig selectedTtsConfig = switch (selectedModelIndex) { debug: true, ), ruleFsts: 'vits-piper-zh_CN-xiao_ya-medium/phone.fst,vits-piper-zh_CN-xiao_ya-medium/date.fst,vits-piper-zh_CN-xiao_ya-medium/number.fst', - maxNumSenetences: 1, + maxNumSentences: 1, ), // ── VITS Piper (English, libritts) ──────────────────────────────────── @@ -70,12 +93,12 @@ final OfflineTtsConfig selectedTtsConfig = switch (selectedModelIndex) { vits: OfflineTtsVitsModelConfig( model: 'vits-piper-en_US-libritts_r-medium/en_US-libritts_r-medium.onnx', tokens: 'vits-piper-en_US-libritts_r-medium/tokens.txt', - dataDir: 'vits-piper-en_US-libritts_r-medium/espeak-ng-data', + lexicon: usePhonemize ? '' : 'lexicon-en-us.txt', ), numThreads: 2, debug: true, ), - maxNumSenetences: 1, + maxNumSentences: 1, ), // ── VITS (English, inflect-nano-v2) ────────────────────────────────── @@ -85,12 +108,12 @@ final OfflineTtsConfig selectedTtsConfig = switch (selectedModelIndex) { vits: OfflineTtsVitsModelConfig( model: 'vits-inflect-en-nano-v2/model.onnx', tokens: 'vits-inflect-en-nano-v2/tokens.txt', - dataDir: 'vits-inflect-en-nano-v2/espeak-ng-data', + lexicon: usePhonemize ? '' : 'lexicon-en-us.txt', ), numThreads: 2, debug: true, ), - maxNumSenetences: 1, + maxNumSentences: 1, ), // ── Kokoro (English) ────────────────────────────────────────────────── @@ -101,12 +124,12 @@ final OfflineTtsConfig selectedTtsConfig = switch (selectedModelIndex) { model: 'kokoro-int8-en-v0_19/model.int8.onnx', voices: 'kokoro-int8-en-v0_19/voices.bin', tokens: 'kokoro-int8-en-v0_19/tokens.txt', - dataDir: 'kokoro-int8-en-v0_19/espeak-ng-data', + lexicon: usePhonemize ? '' : 'lexicon-en-us.txt', ), numThreads: 2, debug: true, ), - maxNumSenetences: 1, + maxNumSentences: 1, ), // ── Kokoro (Chinese + English) ──────────────────────────────────────── @@ -117,13 +140,12 @@ final OfflineTtsConfig selectedTtsConfig = switch (selectedModelIndex) { model: 'kokoro-multi-lang-v1_0/model.onnx', voices: 'kokoro-multi-lang-v1_0/voices.bin', tokens: 'kokoro-multi-lang-v1_0/tokens.txt', - dataDir: 'kokoro-multi-lang-v1_0/espeak-ng-data', lexicon: 'kokoro-multi-lang-v1_0/lexicon-us-en.txt,kokoro-multi-lang-v1_0/lexicon-zh.txt', ), numThreads: 2, debug: true, ), - maxNumSenetences: 1, + maxNumSentences: 1, ), // ── MatchaTTS (English) ─────────────────────────────────────────────── @@ -133,12 +155,12 @@ final OfflineTtsConfig selectedTtsConfig = switch (selectedModelIndex) { acousticModel: 'matcha-icefall-en_US-ljspeech/model-steps-3.onnx', vocoder: 'vocos-22khz-univ.onnx', tokens: 'matcha-icefall-en_US-ljspeech/tokens.txt', - dataDir: 'matcha-icefall-en_US-ljspeech/espeak-ng-data', + lexicon: usePhonemize ? '' : 'lexicon-en-us.txt', ), numThreads: 2, debug: true, ), - maxNumSenetences: 1, + maxNumSentences: 1, ), // ── MatchaTTS (Chinese + English) ──────────────────────────────────── @@ -150,13 +172,12 @@ final OfflineTtsConfig selectedTtsConfig = switch (selectedModelIndex) { vocoder: 'vocos-16khz-univ.onnx', lexicon: 'matcha-icefall-zh-en/lexicon.txt', tokens: 'matcha-icefall-zh-en/tokens.txt', - dataDir: 'matcha-icefall-zh-en/espeak-ng-data', ), numThreads: 2, debug: true, ), ruleFsts: 'matcha-icefall-zh-en/phone-zh.fst,matcha-icefall-zh-en/date-zh.fst,matcha-icefall-zh-en/number-zh.fst', - maxNumSenetences: 1, + maxNumSentences: 1, ), // ── KittenTTS (English) ─────────────────────────────────────────────── @@ -166,12 +187,12 @@ final OfflineTtsConfig selectedTtsConfig = switch (selectedModelIndex) { model: 'kitten-nano-en-v0_1-fp16/model.fp16.onnx', voices: 'kitten-nano-en-v0_1-fp16/voices.bin', tokens: 'kitten-nano-en-v0_1-fp16/tokens.txt', - dataDir: 'kitten-nano-en-v0_1-fp16/espeak-ng-data', + lexicon: usePhonemize ? '' : 'lexicon-en-us.txt', ), numThreads: 2, debug: true, ), - maxNumSenetences: 1, + maxNumSentences: 1, ), // ── Pocket TTS (English) ────────────────────────────────────────────── diff --git a/flutter-examples/tts/lib/model_web.dart b/flutter-examples/tts/lib/model_web.dart index 729200240..bc81b7ef8 100644 --- a/flutter-examples/tts/lib/model_web.dart +++ b/flutter-examples/tts/lib/model_web.dart @@ -59,17 +59,38 @@ Future> loadModelFileBytes() async { // Collect all model directory names from the config. final modelDirs = {}; + // lexicon is the text frontend; it may hold several files separated by + // commas (e.g. "lexicon-us-en.txt,lexicon-zh.txt"), so each entry is + // handled on its own. + // Note: dataDir is deliberately not consulted - it is deprecated and + // ignored since v2.0.0 (espeak-ng is no longer used). for (final path in [ - cfg.model.vits.model, cfg.model.vits.tokens, cfg.model.vits.dataDir, - cfg.model.kokoro.model, cfg.model.kokoro.voices, cfg.model.kokoro.tokens, - cfg.model.kitten.model, cfg.model.kitten.voices, cfg.model.kitten.tokens, - cfg.model.matcha.acousticModel, cfg.model.matcha.vocoder, - cfg.model.pocket.lmFlow, cfg.model.pocket.lmMain, + cfg.model.vits.model, + cfg.model.vits.tokens, + cfg.model.vits.lexicon, + cfg.model.kokoro.model, + cfg.model.kokoro.voices, + cfg.model.kokoro.tokens, + cfg.model.kokoro.lexicon, + cfg.model.kitten.model, + cfg.model.kitten.voices, + cfg.model.kitten.tokens, + cfg.model.kitten.lexicon, + cfg.model.matcha.acousticModel, + cfg.model.matcha.vocoder, + cfg.model.matcha.lexicon, + cfg.model.pocket.lmFlow, + cfg.model.pocket.lmMain, cfg.model.supertonic.durationPredictor, - cfg.model.zipvoice.encoder, cfg.model.zipvoice.tokens, + cfg.model.zipvoice.encoder, + cfg.model.zipvoice.tokens, + cfg.model.zipvoice.lexicon, ]) { - if (path.isNotEmpty) { - modelDirs.add(path.split('/').first); + for (final entry in path.split(',')) { + final p = entry.trim(); + if (p.isNotEmpty) { + modelDirs.add(p.split('/').first); + } } } diff --git a/flutter-examples/tts/lib/tts_manager.dart b/flutter-examples/tts/lib/tts_manager.dart index 45abed08f..d3e929dc8 100644 --- a/flutter-examples/tts/lib/tts_manager.dart +++ b/flutter-examples/tts/lib/tts_manager.dart @@ -67,10 +67,12 @@ import 'dart:isolate'; import 'dart:typed_data'; import 'package:flutter/foundation.dart' show kDebugMode, kIsWeb; +import 'package:piper_phonemize/piper_phonemize.dart' as piper; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; import './generated_audio.dart'; import './model.dart' if (dart.library.js_interop) './model_web.dart' as m; +import './model_config.dart' as cfg; import './utils.dart' if (dart.library.js_interop) './utils_web.dart' as u; import './web_audio.dart' if (dart.library.io) './web_audio_stub.dart' as web_audio; @@ -92,8 +94,12 @@ class _GenerateRequest extends _ToWorker { final Float32List? referenceAudio; final int referenceSampleRate; final int numSteps; + final List>? phonemeCodepoints; _GenerateRequest(this.text, this.sid, this.speed, this.generationId, - {this.referenceAudio, this.referenceSampleRate = 0, this.numSteps = 5}); + {this.referenceAudio, + this.referenceSampleRate = 0, + this.numSteps = 5, + this.phonemeCodepoints}); } class _DisposeRequest extends _ToWorker {} @@ -186,6 +192,18 @@ class TtsManager { if (_state != TtsState.uninitialized) return; _state = TtsState.initializing; + if (cfg.selectedModelUsesPhonemize) { + // The phonemizer runs on this isolate; on web its espeak-ng-data is + // embedded in the WASM bundle, on native it comes from the package + // assets (extracted once, cached). + await piper.initBindingsAsync(); + final dataDir = kIsWeb ? '' : await piper.extractEspeakNgData(); + final ret = piper.PiperPhonemize.initialize(dataDir); + if (ret < 0) { + _logController.add('Error: failed to initialize the piper phonemizer'); + } + } + if (kIsWeb) { await _initWeb(); } else { @@ -214,19 +232,32 @@ class TtsManager { final id = _nextId++; + // Phonemize on this isolate (that is where the piper package is + // initialized); the background isolate/worker only receives the resulting + // codepoints. piper may emit empty sentences (e.g. a trailing segment + // after punctuation) — the C API requires every sentence to be non-empty. + List>? codepoints; + var outText = text; + if (cfg.selectedModelUsesPhonemize) { + codepoints = piper.PiperPhonemize.phonemize(text, voice: 'en-us') + .where((sentence) => sentence.isNotEmpty) + .toList(); + outText = ''; + } + if (kIsWeb) { _worker?.generate( - text: text, sid: sid, speed: speed, generationId: generationId, + text: outText, sid: sid, speed: speed, generationId: generationId, referenceAudio: referenceAudio, referenceSampleRate: referenceSampleRate, - numSteps: numSteps, + numSteps: numSteps, phonemeCodepoints: codepoints, ); } else { _pending[id] = _PendingGenerate( text: text, sid: sid, speed: speed, generationId: generationId, ); - _sendPort!.send(_GenerateRequest(text, sid, speed, generationId, + _sendPort!.send(_GenerateRequest(outText, sid, speed, generationId, referenceAudio: referenceAudio, referenceSampleRate: referenceSampleRate, - numSteps: numSteps, + numSteps: numSteps, phonemeCodepoints: codepoints, )); } @@ -420,6 +451,7 @@ class TtsManager { referenceAudio: req.referenceAudio, referenceSampleRate: req.referenceSampleRate, numSteps: req.numSteps, + phonemeCodepoints: req.phonemeCodepoints, ); final sampleRate = tts.sampleRate; diff --git a/flutter-examples/tts/lib/worker_stub.dart b/flutter-examples/tts/lib/worker_stub.dart index d0ad38771..ea6b50118 100644 --- a/flutter-examples/tts/lib/worker_stub.dart +++ b/flutter-examples/tts/lib/worker_stub.dart @@ -24,6 +24,7 @@ class TtsWorker { Float32List? referenceAudio, int referenceSampleRate = 0, int numSteps = 5, + List>? phonemeCodepoints, }) {} void cancel() {} void dispose() {} diff --git a/flutter-examples/tts/lib/worker_web.dart b/flutter-examples/tts/lib/worker_web.dart index 287f5ed16..ef01fe1a9 100644 --- a/flutter-examples/tts/lib/worker_web.dart +++ b/flutter-examples/tts/lib/worker_web.dart @@ -95,6 +95,7 @@ class TtsWorker { Float32List? referenceAudio, int referenceSampleRate = 0, int numSteps = 5, + List>? phonemeCodepoints, }) { _pendingLabel = GeneratedAudioItem.makeLabel(text); final msg = JSObject(); @@ -109,6 +110,15 @@ class TtsWorker { msg['referenceAudio'] = referenceAudio.buffer.toJS; msg['referenceSampleRate'] = referenceSampleRate.toJS; } + + if (phonemeCodepoints != null && phonemeCodepoints.isNotEmpty) { + // number[][] via the host JSON.parse (structured clone keeps it as a + // plain JS array for the worker). + final json = globalContext.getProperty('JSON'.toJS) as JSObject; + final parse = json.getProperty('parse'.toJS) as JSFunction; + msg['phonemeCodepoints'] = + parse.callAsFunction(null, jsonEncode(phonemeCodepoints).toJS); + } _worker?.postMessage(msg); } diff --git a/flutter-examples/tts/pubspec.yaml b/flutter-examples/tts/pubspec.yaml index 2ffb19c47..45d33c809 100644 --- a/flutter-examples/tts/pubspec.yaml +++ b/flutter-examples/tts/pubspec.yaml @@ -22,6 +22,7 @@ dependencies: sherpa_onnx: 1.13.8 # sherpa_onnx: # path: ../../flutter/sherpa_onnx + piper_phonemize: ^1.5.8 url_launcher: 6.2.6 url_launcher_linux: 3.1.0 audioplayers: ^5.0.0 diff --git a/flutter-examples/tts/web/tts-worker.js b/flutter-examples/tts/web/tts-worker.js index bc2e4eb53..d9f31e573 100644 --- a/flutter-examples/tts/web/tts-worker.js +++ b/flutter-examples/tts/web/tts-worker.js @@ -162,6 +162,11 @@ self.onmessage = async function(e) { genCfg.numSteps = msg.numSteps || 5; } + // Pre-phonemized input (number[][]); used in place of msg.text. + if (msg.phonemeCodepoints) { + genCfg.phonemeCodepoints = msg.phonemeCodepoints; + } + const cfgWasm = initSherpaOnnxGenerationConfig(genCfg, Module); // Set up callback for streaming chunks. diff --git a/flutter/sherpa_onnx/example/example.md b/flutter/sherpa_onnx/example/example.md index 8d4384ca7..d016da362 100644 --- a/flutter/sherpa_onnx/example/example.md +++ b/flutter/sherpa_onnx/example/example.md @@ -1,5 +1,57 @@ # sherpa-onnx app example +## Contents + +- [Pre-built Flutter apps](#pre-built-flutter-apps) + - [VAD from file](#vad-from-file) + - [VAD from microphone](#vad-from-microphone) + - [VAD + ASR from file](#vad--asr-from-file) + - [Zipformer CTC (Chinese)](#zipformer-ctc-chinese) + - [SenseVoice (Chinese/English/Japanese/Korean/Cantonese)](#sensevoice-chineseenglishjapanesekoreancantonese) + - [Whisper tiny.en (English)](#whisper-tinyen-english) + - [NeMo Parakeet TDT v2 (English)](#nemo-parakeet-tdt-v2-english) + - [NeMo Parakeet TDT v3 (25 European languages)](#nemo-parakeet-tdt-v3-25-european-languages) + - [Moonshine tiny (English)](#moonshine-tiny-english) + - [Qwen3 ASR 0.6B (multilingual)](#qwen3-asr-06b-multilingual) + - [FunASR Nano (Chinese/English/Japanese)](#funasr-nano-chineseenglishjapanese) + - [FireRed ASR CTC v2 (Chinese/English)](#firered-asr-ctc-v2-chineseenglish) + - [VAD + ASR from microphone](#vad--asr-from-microphone) + - [Zipformer CTC (Chinese)](#zipformer-ctc-chinese-1) + - [SenseVoice (Chinese/English/Japanese/Korean/Cantonese)](#sensevoice-chineseenglishjapanesekoreancantonese-1) + - [Whisper tiny.en (English)](#whisper-tinyen-english-1) + - [NeMo Parakeet TDT v2 (English)](#nemo-parakeet-tdt-v2-english-1) + - [NeMo Parakeet TDT v3 (25 European languages)](#nemo-parakeet-tdt-v3-25-european-languages-1) + - [Moonshine tiny (English)](#moonshine-tiny-english-1) + - [Qwen3 ASR 0.6B (multilingual)](#qwen3-asr-06b-multilingual-1) + - [FunASR Nano (Chinese/English/Japanese)](#funasr-nano-chineseenglishjapanese-1) + - [FireRed ASR CTC v2 (Chinese/English)](#firered-asr-ctc-v2-chineseenglish-1) + - [Streaming speech recognition](#streaming-speech-recognition) + - [Streaming Zipformer Bilingual (Chinese/English)](#streaming-zipformer-bilingual-chineseenglish) + - [Streaming Zipformer (English)](#streaming-zipformer-english) + - [Streaming Zipformer WenetSpeech (Chinese)](#streaming-zipformer-wenetspeech-chinese) + - [Streaming Zipformer (French)](#streaming-zipformer-french) + - [X-ASR 960ms (Chinese/English, with punctuation)](#x-asr-960ms-chineseenglish-with-punctuation) + - [Nemotron 3.5 ASR Streaming 560ms (multilingual)](#nemotron-35-asr-streaming-560ms-multilingual) + - [X-ASR 480ms (Chinese/English, with punctuation)](#x-asr-480ms-chineseenglish-with-punctuation) + - [Nemotron 3.5 ASR Streaming 1120ms (multilingual)](#nemotron-35-asr-streaming-1120ms-multilingual) + - [Text to speech](#text-to-speech) + - [VITS Piper en_US-amy-low (English)](#vits-piper-en_us-amy-low-english) + - [VITS Piper zh_CN-xiao_ya-medium (Chinese)](#vits-piper-zh_cn-xiao_ya-medium-chinese) + - [VITS Piper en_US-libritts_r-medium (English)](#vits-piper-en_us-libritts_r-medium-english) + - [VITS Inflect en-nano-v2 (English)](#vits-inflect-en-nano-v2-english) + - [Kokoro v0.19 (English)](#kokoro-v019-english) + - [Kokoro v1.0 (Chinese/English)](#kokoro-v10-chineseenglish) + - [Matcha en_US-ljspeech (English)](#matcha-en_us-ljspeech-english) + - [Matcha zh-en (Chinese/English)](#matcha-zh-en-chineseenglish) + - [Kitten Nano v0.1 (English)](#kitten-nano-v01-english) + - [PocketTTS (multilingual)](#pockettts-multilingual) + - [Supertonic 3 (multilingual)](#supertonic-3-multilingual) + - [Offline punctuation restoration](#offline-punctuation-restoration) + - [Online punctuation restoration](#online-punctuation-restoration) +- [Web demos](#web-demos) +- [Flutter source code](#flutter-source-code) +- [Pure dart-examples](#pure-dart-examples) + ## Pre-built Flutter apps Download pre-built Flutter apps for different platforms at @@ -293,6 +345,268 @@ ASR model: [sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25](https://k2-fsa. | Windows (x64) | [flutter-vad-non-streaming-asr-from-microphone-firered-ctc-zh-en-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-vad-non-streaming-asr-from-microphone-firered-ctc-zh-en-win-x64.zip) | | Web | [flutter-vad-non-streaming-asr-from-microphone-firered-ctc-zh-en-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-vad-non-streaming-asr-from-microphone-firered-ctc-zh-en-web.zip) | +### Streaming speech recognition + +#### Streaming Zipformer Bilingual (Chinese/English) + +ASR model: [sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-x86_64.apk) | +| Linux (x64) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-linux-x64.zip) | +| macOS | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-macos.zip) | +| Windows (x64) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-win-x64.zip) | + +#### Streaming Zipformer (English) + +ASR model: [sherpa-onnx-streaming-zipformer-en-2023-06-26](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#csukuangfj-sherpa-onnx-streaming-zipformer-en-2023-06-26-english) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-2023-06-26.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-x86_64.apk) | +| Linux (x64) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-linux-x64.zip) | +| macOS | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-macos.zip) | +| Windows (x64) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-en-2023-06-26-win-x64.zip) | + +#### Streaming Zipformer WenetSpeech (Chinese) + +ASR model: [icefall-asr-zipformer-streaming-wenetspeech-20230615](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#pkufool-icefall-asr-zipformer-streaming-wenetspeech-20230615-chinese) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/icefall-asr-zipformer-streaming-wenetspeech-20230615.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-x86_64.apk) | +| Linux (x64) | [flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-linux-x64.zip) | +| macOS | [flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-macos.zip) | +| Windows (x64) | [flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-icefall-asr-zipformer-streaming-wenetspeech-20230615-win-x64.zip) | + +#### Streaming Zipformer (French) + +ASR model: [sherpa-onnx-streaming-zipformer-fr-2023-04-14](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/online-transducer/zipformer-transducer-models.html#shaojieli-sherpa-onnx-streaming-zipformer-fr-2023-04-14-french) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-fr-2023-04-14.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-x86_64.apk) | +| Linux (x64) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-linux-x64.zip) | +| macOS | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-macos.zip) | +| Windows (x64) | [flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-streaming-zipformer-fr-2023-04-14-win-x64.zip) | + +#### X-ASR 960ms (Chinese/English, with punctuation) + +ASR model: sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05 — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-x86_64.apk) | +| Linux (x64) | [flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-linux-x64.zip) | +| macOS | [flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-macos.zip) | +| Windows (x64) | [flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-960ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-win-x64.zip) | + +#### Nemotron 3.5 ASR Streaming 560ms (multilingual) + +ASR model: sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11 — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-x86_64.apk) | +| Linux (x64) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-linux-x64.zip) | +| macOS | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-macos.zip) | +| Windows (x64) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11-win-x64.zip) | + +#### X-ASR 480ms (Chinese/English, with punctuation) + +ASR model: sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05 — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-x86_64.apk) | +| Linux (x64) | [flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-linux-x64.zip) | +| macOS | [flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-macos.zip) | +| Windows (x64) | [flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-x-asr-480ms-streaming-zipformer-transducer-zh-en-punct-int8-2026-06-05-win-x64.zip) | + +#### Nemotron 3.5 ASR Streaming 1120ms (multilingual) + +ASR model: sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11 — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-x86_64.apk) | +| Linux (x64) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-linux-x64.zip) | +| macOS | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-macos.zip) | +| Windows (x64) | [flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-streaming_asr-sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11-win-x64.zip) | + +### Text to speech + +#### VITS Piper en_US-amy-low (English) + +TTS model: [vits-piper-en_US-amy-low](https://k2-fsa.github.io/sherpa/onnx/tts/all/English/vits-piper-en_US-amy-low.html) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-vits-piper-en_US-amy-low-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-amy-low-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-vits-piper-en_US-amy-low-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-amy-low-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-vits-piper-en_US-amy-low-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-amy-low-x86_64.apk) | +| Linux (x64) | [flutter-tts-vits-piper-en_US-amy-low-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-amy-low-linux-x64.zip) | +| macOS | [flutter-tts-vits-piper-en_US-amy-low-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-amy-low-macos.zip) | +| Windows (x64) | [flutter-tts-vits-piper-en_US-amy-low-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-amy-low-win-x64.zip) | +| Web | [flutter-tts-vits-piper-en_US-amy-low-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-amy-low-web.zip) | + +#### VITS Piper zh_CN-xiao_ya-medium (Chinese) + +TTS model: [vits-piper-zh_CN-xiao_ya-medium](https://k2-fsa.github.io/sherpa/onnx/tts/all/Chinese/vits-piper-zh_CN-xiao_ya-medium.html) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-zh_CN-xiao_ya-medium.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-vits-piper-zh_CN-xiao_ya-medium-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-zh_CN-xiao_ya-medium-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-vits-piper-zh_CN-xiao_ya-medium-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-zh_CN-xiao_ya-medium-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-vits-piper-zh_CN-xiao_ya-medium-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-zh_CN-xiao_ya-medium-x86_64.apk) | +| Linux (x64) | [flutter-tts-vits-piper-zh_CN-xiao_ya-medium-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-zh_CN-xiao_ya-medium-linux-x64.zip) | +| macOS | [flutter-tts-vits-piper-zh_CN-xiao_ya-medium-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-zh_CN-xiao_ya-medium-macos.zip) | +| Windows (x64) | [flutter-tts-vits-piper-zh_CN-xiao_ya-medium-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-zh_CN-xiao_ya-medium-win-x64.zip) | +| Web | [flutter-tts-vits-piper-zh_CN-xiao_ya-medium-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-zh_CN-xiao_ya-medium-web.zip) | + +#### VITS Piper en_US-libritts_r-medium (English) + +TTS model: [vits-piper-en_US-libritts_r-medium](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#vits-piper-en-us-libritts-r-medium-english-904-speakers) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-libritts_r-medium.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-vits-piper-en_US-libritts_r-medium-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-libritts_r-medium-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-vits-piper-en_US-libritts_r-medium-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-libritts_r-medium-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-vits-piper-en_US-libritts_r-medium-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-libritts_r-medium-x86_64.apk) | +| Linux (x64) | [flutter-tts-vits-piper-en_US-libritts_r-medium-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-libritts_r-medium-linux-x64.zip) | +| macOS | [flutter-tts-vits-piper-en_US-libritts_r-medium-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-libritts_r-medium-macos.zip) | +| Windows (x64) | [flutter-tts-vits-piper-en_US-libritts_r-medium-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-libritts_r-medium-win-x64.zip) | +| Web | [flutter-tts-vits-piper-en_US-libritts_r-medium-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-piper-en_US-libritts_r-medium-web.zip) | + +#### VITS Inflect en-nano-v2 (English) + +TTS model: [vits-inflect-en-nano-v2](https://k2-fsa.github.io/sherpa/onnx/tts/all/English/vits-inflect-en-nano-v2.html) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-inflect-en-nano-v2.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-vits-inflect-en-nano-v2-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-inflect-en-nano-v2-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-vits-inflect-en-nano-v2-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-inflect-en-nano-v2-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-vits-inflect-en-nano-v2-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-inflect-en-nano-v2-x86_64.apk) | +| Linux (x64) | [flutter-tts-vits-inflect-en-nano-v2-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-inflect-en-nano-v2-linux-x64.zip) | +| macOS | [flutter-tts-vits-inflect-en-nano-v2-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-inflect-en-nano-v2-macos.zip) | +| Windows (x64) | [flutter-tts-vits-inflect-en-nano-v2-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-inflect-en-nano-v2-win-x64.zip) | +| Web | [flutter-tts-vits-inflect-en-nano-v2-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-vits-inflect-en-nano-v2-web.zip) | + +#### Kokoro v0.19 (English) + +TTS model: [kokoro-int8-en-v0_19](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html#kokoro-en-v0-19-english-11-speakers) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-int8-en-v0_19.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-kokoro-int8-en-v0_19-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-int8-en-v0_19-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-kokoro-int8-en-v0_19-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-int8-en-v0_19-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-kokoro-int8-en-v0_19-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-int8-en-v0_19-x86_64.apk) | +| Linux (x64) | [flutter-tts-kokoro-int8-en-v0_19-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-int8-en-v0_19-linux-x64.zip) | +| macOS | [flutter-tts-kokoro-int8-en-v0_19-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-int8-en-v0_19-macos.zip) | +| Windows (x64) | [flutter-tts-kokoro-int8-en-v0_19-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-int8-en-v0_19-win-x64.zip) | +| Web | [flutter-tts-kokoro-int8-en-v0_19-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-int8-en-v0_19-web.zip) | + +#### Kokoro v1.0 (Chinese/English) + +TTS model: [kokoro-multi-lang-v1_0](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html#kokoro-multi-lang-v1-0-chinese-english-53-speakers) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-kokoro-multi-lang-v1_0-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-multi-lang-v1_0-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-kokoro-multi-lang-v1_0-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-multi-lang-v1_0-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-kokoro-multi-lang-v1_0-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-multi-lang-v1_0-x86_64.apk) | +| Linux (x64) | [flutter-tts-kokoro-multi-lang-v1_0-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-multi-lang-v1_0-linux-x64.zip) | +| macOS | [flutter-tts-kokoro-multi-lang-v1_0-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-multi-lang-v1_0-macos.zip) | +| Windows (x64) | [flutter-tts-kokoro-multi-lang-v1_0-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-multi-lang-v1_0-win-x64.zip) | +| Web | [flutter-tts-kokoro-multi-lang-v1_0-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kokoro-multi-lang-v1_0-web.zip) | + +#### Matcha en_US-ljspeech (English) + +TTS model: [matcha-icefall-en_US-ljspeech](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-matcha-icefall-en_US-ljspeech-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-en_US-ljspeech-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-matcha-icefall-en_US-ljspeech-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-en_US-ljspeech-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-matcha-icefall-en_US-ljspeech-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-en_US-ljspeech-x86_64.apk) | +| Linux (x64) | [flutter-tts-matcha-icefall-en_US-ljspeech-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-en_US-ljspeech-linux-x64.zip) | +| macOS | [flutter-tts-matcha-icefall-en_US-ljspeech-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-en_US-ljspeech-macos.zip) | +| Windows (x64) | [flutter-tts-matcha-icefall-en_US-ljspeech-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-en_US-ljspeech-win-x64.zip) | +| Web | [flutter-tts-matcha-icefall-en_US-ljspeech-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-en_US-ljspeech-web.zip) | + +#### Matcha zh-en (Chinese/English) + +TTS model: [matcha-icefall-zh-en](https://k2-fsa.github.io/sherpa/onnx/tts/all/Chinese-English/matcha-icefall-zh-en.html) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-en.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-matcha-icefall-zh-en-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-zh-en-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-matcha-icefall-zh-en-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-zh-en-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-matcha-icefall-zh-en-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-zh-en-x86_64.apk) | +| Linux (x64) | [flutter-tts-matcha-icefall-zh-en-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-zh-en-linux-x64.zip) | +| macOS | [flutter-tts-matcha-icefall-zh-en-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-zh-en-macos.zip) | +| Windows (x64) | [flutter-tts-matcha-icefall-zh-en-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-zh-en-win-x64.zip) | +| Web | [flutter-tts-matcha-icefall-zh-en-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-matcha-icefall-zh-en-web.zip) | + +#### Kitten Nano v0.1 (English) + +TTS model: [kitten-nano-en-v0_1-fp16](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kitten.html#kitten-nano-en-v0-1-fp16) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-kitten-nano-en-v0_1-fp16-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kitten-nano-en-v0_1-fp16-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-kitten-nano-en-v0_1-fp16-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kitten-nano-en-v0_1-fp16-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-kitten-nano-en-v0_1-fp16-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kitten-nano-en-v0_1-fp16-x86_64.apk) | +| Linux (x64) | [flutter-tts-kitten-nano-en-v0_1-fp16-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kitten-nano-en-v0_1-fp16-linux-x64.zip) | +| macOS | [flutter-tts-kitten-nano-en-v0_1-fp16-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kitten-nano-en-v0_1-fp16-macos.zip) | +| Windows (x64) | [flutter-tts-kitten-nano-en-v0_1-fp16-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kitten-nano-en-v0_1-fp16-win-x64.zip) | +| Web | [flutter-tts-kitten-nano-en-v0_1-fp16-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-kitten-nano-en-v0_1-fp16-web.zip) | + +#### PocketTTS (multilingual) + +TTS model: [sherpa-onnx-pocket-tts-int8-2026-01-26](https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html#download-a-pre-trained-model) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-x86_64.apk) | +| Linux (x64) | [flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-linux-x64.zip) | +| macOS | [flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-macos.zip) | +| Windows (x64) | [flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-win-x64.zip) | +| Web | [flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-pocket-tts-int8-2026-01-26-web.zip) | + +#### Supertonic 3 (multilingual) + +TTS model: [sherpa-onnx-supertonic-3-tts-int8-2026-05-11](https://k2-fsa.github.io/sherpa/onnx/tts/supertonic.html#download-a-pre-trained-model) — [download](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2) + +| Platform | File | +|---|---| +| Android (arm64-v8a) | [flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-arm64-v8a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-arm64-v8a.apk) | +| Android (armeabi-v7a) | [flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-armeabi-v7a.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-armeabi-v7a.apk) | +| Android (x86_64) | [flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-x86_64.apk](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-x86_64.apk) | +| Linux (x64) | [flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-linux-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-linux-x64.zip) | +| macOS | [flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-macos.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-macos.zip) | +| Windows (x64) | [flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-win-x64.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-win-x64.zip) | +| Web | [flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-web.zip](https://github.com/k2-fsa/sherpa-onnx/releases/download/flutter/flutter-tts-sherpa-onnx-supertonic-3-tts-int8-2026-05-11-web.zip) | + ### Offline punctuation restoration | Platform | File | diff --git a/flutter/sherpa_onnx/lib/src/offline_recognizer.dart b/flutter/sherpa_onnx/lib/src/offline_recognizer.dart index ed018fd61..1e692eae6 100644 --- a/flutter/sherpa_onnx/lib/src/offline_recognizer.dart +++ b/flutter/sherpa_onnx/lib/src/offline_recognizer.dart @@ -74,13 +74,15 @@ class OfflineRecognizer { /// Create a recognizer from [config]. factory OfflineRecognizer(OfflineRecognizerConfig config) { - final c = convertConfig(config); - if (SherpaOnnxBindings.createOfflineRecognizer == null) { throw Exception("Please initialize sherpa-onnx first"); } + final c = convertConfig(config); final ptr = SherpaOnnxBindings.createOfflineRecognizer?.call(c) ?? nullptr; + // Free before any throw so a failed create does not leak the marshalled + // strings. The online factory does the same. + freeConfig(c); if (ptr == nullptr) { throw Exception( @@ -88,8 +90,6 @@ class OfflineRecognizer { ); } - freeConfig(c); - return OfflineRecognizer._(ptr: ptr, config: config); } @@ -283,10 +283,15 @@ class OfflineRecognizer { c.ref.hr.lexicon = config.hr.lexicon.toNativeUtf8(); c.ref.hr.ruleFsts = config.hr.ruleFsts.toNativeUtf8(); + c.ref.ctcFstDecoderConfig.graph = + config.ctcFstDecoderConfig.graph.toNativeUtf8(); + c.ref.ctcFstDecoderConfig.maxActive = config.ctcFstDecoderConfig.maxActive; + return c; } static void freeConfig(Pointer c) { + calloc.free(c.ref.ctcFstDecoderConfig.graph); calloc.free(c.ref.hr.lexicon); calloc.free(c.ref.hr.ruleFsts); calloc.free(c.ref.ruleFars); diff --git a/flutter/sherpa_onnx/lib/src/offline_recognizer_config.dart b/flutter/sherpa_onnx/lib/src/offline_recognizer_config.dart index e8d37aa5f..aca242284 100644 --- a/flutter/sherpa_onnx/lib/src/offline_recognizer_config.dart +++ b/flutter/sherpa_onnx/lib/src/offline_recognizer_config.dart @@ -805,6 +805,30 @@ class OfflineModelConfig { /// This combines feature extraction, the selected model family, optional /// language model settings, hotwords, grammar resources, and optional /// homophone replacement resources. +class OfflineCtcFstDecoderConfig { + const OfflineCtcFstDecoderConfig({this.graph = '', this.maxActive = 3000}); + + factory OfflineCtcFstDecoderConfig.fromJson(Map json) { + return OfflineCtcFstDecoderConfig( + graph: json['graph'] as String? ?? '', + maxActive: json['maxActive'] as int? ?? 3000, + ); + } + + @override + String toString() { + return 'OfflineCtcFstDecoderConfig(graph: $graph, maxActive: $maxActive)'; + } + + Map toJson() => { + 'graph': graph, + 'maxActive': maxActive, + }; + + final String graph; + final int maxActive; +} + class OfflineRecognizerConfig { const OfflineRecognizerConfig({ this.feat = const FeatureConfig(), @@ -818,6 +842,7 @@ class OfflineRecognizerConfig { this.ruleFars = '', this.blankPenalty = 0.0, this.hr = const HomophoneReplacerConfig(), + this.ctcFstDecoderConfig = const OfflineCtcFstDecoderConfig(), }); factory OfflineRecognizerConfig.fromJson(Map json) { @@ -839,12 +864,16 @@ class OfflineRecognizerConfig { hr: json['hr'] != null ? HomophoneReplacerConfig.fromJson(json['hr'] as Map) : const HomophoneReplacerConfig(), + ctcFstDecoderConfig: json['ctcFstDecoderConfig'] != null + ? OfflineCtcFstDecoderConfig.fromJson( + json['ctcFstDecoderConfig'] as Map) + : const OfflineCtcFstDecoderConfig(), ); } @override String toString() { - return 'OfflineRecognizerConfig(feat: $feat, model: $model, lm: $lm, decodingMethod: $decodingMethod, maxActivePaths: $maxActivePaths, hotwordsFile: $hotwordsFile, hotwordsScore: $hotwordsScore, ruleFsts: $ruleFsts, ruleFars: $ruleFars, blankPenalty: $blankPenalty, hr: $hr)'; + return 'OfflineRecognizerConfig(feat: $feat, model: $model, lm: $lm, decodingMethod: $decodingMethod, maxActivePaths: $maxActivePaths, hotwordsFile: $hotwordsFile, hotwordsScore: $hotwordsScore, ruleFsts: $ruleFsts, ruleFars: $ruleFars, blankPenalty: $blankPenalty, hr: $hr, ctcFstDecoderConfig: $ctcFstDecoderConfig)'; } Map toJson() => { @@ -859,6 +888,7 @@ class OfflineRecognizerConfig { 'ruleFars': ruleFars, 'blankPenalty': blankPenalty, 'hr': hr.toJson(), + 'ctcFstDecoderConfig': ctcFstDecoderConfig.toJson(), }; final FeatureConfig feat; @@ -877,6 +907,7 @@ class OfflineRecognizerConfig { final double blankPenalty; final HomophoneReplacerConfig hr; + final OfflineCtcFstDecoderConfig ctcFstDecoderConfig; } /// Recognition result returned by [OfflineRecognizer.getResult]. diff --git a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart index 120c54884..1d5313425 100644 --- a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart +++ b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart @@ -234,6 +234,8 @@ final class SherpaOnnxOfflineTtsKittenModelConfig extends Struct { @Float() external double lengthScale; + + external Pointer lexicon; } final class SherpaOnnxOfflineTtsZipVoiceModelConfig extends Struct { @@ -302,7 +304,7 @@ final class SherpaOnnxOfflineTtsConfig extends Struct { external Pointer ruleFsts; @Int32() - external int maxNumSenetences; + external int maxNumSentences; external Pointer ruleFars; @@ -334,6 +336,26 @@ final class SherpaOnnxGenerationConfig extends Struct { external int numSteps; external Pointer extra; + + // Pre-phonemized input as Unicode codepoints (flattened; see + // phonemeCodepointsLens for the per-sentence boundaries). When set, it is + // used in place of the text argument. Mutually exclusive with tokens. + external Pointer phonemeCodepoints; + + external Pointer phonemeCodepointsLens; + + @Int32() + external int phonemeCodepointsNumSentences; + + // Pre-tokenized input, e.g. Chinese pinyin like "zhong1" (flattened; see + // tokensLens for the per-sentence boundaries). Mutually exclusive with + // phonemeCodepoints. + external Pointer> tokens; + + external Pointer tokensLens; + + @Int32() + external int tokensNumSentences; } final class SherpaOnnxGeneratedAudio extends Struct { @@ -571,6 +593,14 @@ final class SherpaOnnxOfflineRecognizerConfig extends Struct { @Float() external double blankPenalty; external SherpaOnnxHomophoneReplacerConfig hr; + external SherpaOnnxOfflineCtcFstDecoderConfig ctcFstDecoderConfig; +} + +final class SherpaOnnxOfflineCtcFstDecoderConfig extends Struct { + external Pointer graph; + + @Int32() + external int maxActive; } final class SherpaOnnxOnlineTransducerModelConfig extends Struct { @@ -1270,6 +1300,16 @@ typedef SherpaOnnxOfflineTtsNumSpeakersNative = typedef SherpaOnnxOfflineTtsNumSpeakers = int Function(Pointer); +typedef SherpaOnnxOfflineTtsLangNative = + Pointer Function(Pointer); + +typedef SherpaOnnxOfflineTtsLang = + Pointer Function(Pointer); + +typedef SherpaOnnxDestroyOfflineTtsLangNative = Void Function(Pointer); + +typedef SherpaOnnxDestroyOfflineTtsLang = void Function(Pointer); + typedef SherpaOnnxOfflineTtsGenerateNative = Pointer Function( Pointer, @@ -1968,6 +2008,8 @@ class SherpaOnnxBindings { static SherpaOnnxDestroyOfflineTts? destroyOfflineTts; static SherpaOnnxOfflineTtsSampleRate? offlineTtsSampleRate; static SherpaOnnxOfflineTtsNumSpeakers? offlineTtsNumSpeakers; + static SherpaOnnxOfflineTtsLang? offlineTtsLang; + static SherpaOnnxDestroyOfflineTtsLang? destroyOfflineTtsLang; static SherpaOnnxOfflineTtsGenerate? offlineTtsGenerate; static SherpaOnnxDestroyOfflineTtsGeneratedAudio? destroyOfflineTtsGeneratedAudio; @@ -2461,6 +2503,18 @@ class SherpaOnnxBindings { ) .asFunction(); + offlineTtsLang ??= dynamicLibrary + .lookup>( + 'SherpaOnnxOfflineTtsLang', + ) + .asFunction(); + + destroyOfflineTtsLang ??= dynamicLibrary + .lookup>( + 'SherpaOnnxDestroyOfflineTtsLang', + ) + .asFunction(); + offlineTtsGenerate ??= dynamicLibrary .lookup>( 'SherpaOnnxOfflineTtsGenerate', diff --git a/flutter/sherpa_onnx/lib/src/tts.dart b/flutter/sherpa_onnx/lib/src/tts.dart index cd82a8ae2..d5d4f9a0b 100644 --- a/flutter/sherpa_onnx/lib/src/tts.dart +++ b/flutter/sherpa_onnx/lib/src/tts.dart @@ -14,17 +14,24 @@ export './tts_config.dart'; /// /// This module supports VITS, Matcha, Kokoro, Kitten, ZipVoice, Pocket TTS, /// and Supertonic model families. See `dart-api-examples/tts/bin/` for working -/// examples such as `pocket-en.dart`, `kokoro-en.dart`, `kokoro-zh-en.dart`, -/// `matcha-en.dart`, and `zipvoice-zh-en.dart`. +/// examples such as `pocket-en.dart`, `piper-lexicon.dart`, +/// `piper-phonemize.dart`, `kokoro-en-lexicon.dart`, `matcha-en-lexicon.dart`, +/// and `zipvoice-zh-en-lexicon.dart`. /// /// Example: /// /// ```dart /// final model = OfflineTtsModelConfig( -/// pocketTts: const OfflineTtsPocketSphinxModelConfig( -/// model: './sherpa-onnx-pocket-tts/model.int8.onnx', -/// tokens: './sherpa-onnx-pocket-tts/tokens.txt', -/// dataDir: './sherpa-onnx-pocket-tts/espeak-ng-data', +/// pocket: const OfflineTtsPocketModelConfig( +/// lmFlow: './sherpa-onnx-pocket-tts-int8-2026-01-26/lm_flow.int8.onnx', +/// lmMain: './sherpa-onnx-pocket-tts-int8-2026-01-26/lm_main.int8.onnx', +/// encoder: './sherpa-onnx-pocket-tts-int8-2026-01-26/encoder.onnx', +/// decoder: './sherpa-onnx-pocket-tts-int8-2026-01-26/decoder.int8.onnx', +/// textConditioner: +/// './sherpa-onnx-pocket-tts-int8-2026-01-26/text_conditioner.onnx', +/// vocabJson: './sherpa-onnx-pocket-tts-int8-2026-01-26/vocab.json', +/// tokenScoresJson: +/// './sherpa-onnx-pocket-tts-int8-2026-01-26/token_scores.json', /// ), /// numThreads: 1, /// ); @@ -67,6 +74,28 @@ extension OfflineTtsGenerationConfigFfi on OfflineTtsGenerationConfig { } Pointer toNative() { + // Validate before anything is allocated: sub-arrays must be non-empty. + // The mutual exclusion of phonemeCodepoints and tokens is NOT checked here; + // it is enforced by the C API, which logs a warning and returns no audio. + // generateWithConfig then returns an empty GeneratedAudio, so all bindings + // report that failure the same way. + final codepoints = phonemeCodepoints; + final tokenSentences = tokens; + if (codepoints != null) { + for (var i = 0; i < codepoints.length; i++) { + if (codepoints[i].isEmpty) { + throw ArgumentError('phonemeCodepoints[$i] must be a non-empty array'); + } + } + } + if (tokenSentences != null) { + for (var i = 0; i < tokenSentences.length; i++) { + if (tokenSentences[i].isEmpty) { + throw ArgumentError('tokens[$i] must be a non-empty array'); + } + } + } + final p = calloc(); p.ref.silenceScale = silenceScale; @@ -92,6 +121,50 @@ extension OfflineTtsGenerationConfigFfi on OfflineTtsGenerationConfig { p.ref.extra = extraToNativeUtf8(); + // Pre-phonemized input: all codepoints of all sentences in one flat + // Int32 array, with per-sentence lengths next to it. + if (codepoints != null && codepoints.isNotEmpty) { + var total = 0; + for (final sentence in codepoints) { + total += sentence.length; + } + final flat = calloc(total); + final lens = calloc(codepoints.length); + final flatList = flat.asTypedList(total); + var offset = 0; + for (var i = 0; i < codepoints.length; i++) { + final sentence = codepoints[i]; + flatList.setRange(offset, offset + sentence.length, sentence); + lens[i] = sentence.length; + offset += sentence.length; + } + p.ref.phonemeCodepoints = flat; + p.ref.phonemeCodepointsLens = lens; + p.ref.phonemeCodepointsNumSentences = codepoints.length; + } + + // Pre-tokenized input: all token strings of all sentences in one flat + // char* array, with per-sentence lengths next to it. + if (tokenSentences != null && tokenSentences.isNotEmpty) { + var total = 0; + for (final sentence in tokenSentences) { + total += sentence.length; + } + final ptrs = calloc>(total); + final lens = calloc(tokenSentences.length); + var offset = 0; + for (var i = 0; i < tokenSentences.length; i++) { + final sentence = tokenSentences[i]; + for (final token in sentence) { + ptrs[offset++] = token.toNativeUtf8(); + } + lens[i] = sentence.length; + } + p.ref.tokens = ptrs; + p.ref.tokensLens = lens; + p.ref.tokensNumSentences = tokenSentences.length; + } + return p; } @@ -105,6 +178,25 @@ extension OfflineTtsGenerationConfigFfi on OfflineTtsGenerationConfig { if (p.ref.extra != nullptr) { calloc.free(p.ref.extra); } + if (p.ref.phonemeCodepoints != nullptr) { + calloc.free(p.ref.phonemeCodepoints); + } + if (p.ref.phonemeCodepointsLens != nullptr) { + calloc.free(p.ref.phonemeCodepointsLens); + } + if (p.ref.tokens != nullptr) { + var total = 0; + for (var i = 0; i < p.ref.tokensNumSentences; i++) { + total += p.ref.tokensLens[i]; + } + for (var i = 0; i < total; i++) { + calloc.free(p.ref.tokens[i]); + } + calloc.free(p.ref.tokens); + } + if (p.ref.tokensLens != nullptr) { + calloc.free(p.ref.tokensLens); + } calloc.free(p); } } @@ -157,6 +249,7 @@ class OfflineTts { c.ref.model.kitten.tokens = config.model.kitten.tokens.toNativeUtf8(); c.ref.model.kitten.dataDir = config.model.kitten.dataDir.toNativeUtf8(); c.ref.model.kitten.lengthScale = config.model.kitten.lengthScale; + c.ref.model.kitten.lexicon = config.model.kitten.lexicon.toNativeUtf8(); c.ref.model.zipvoice.tokens = config.model.zipvoice.tokens.toNativeUtf8(); c.ref.model.zipvoice.encoder = config.model.zipvoice.encoder.toNativeUtf8(); @@ -201,7 +294,7 @@ class OfflineTts { c.ref.model.provider = config.model.provider.toNativeUtf8(); c.ref.ruleFsts = config.ruleFsts.toNativeUtf8(); - c.ref.maxNumSenetences = config.maxNumSenetences; + c.ref.maxNumSentences = config.maxNumSentences; c.ref.ruleFars = config.ruleFars.toNativeUtf8(); c.ref.silenceScale = config.silenceScale; @@ -234,6 +327,7 @@ class OfflineTts { calloc.free(c.ref.model.zipvoice.encoder); calloc.free(c.ref.model.zipvoice.tokens); + calloc.free(c.ref.model.kitten.lexicon); calloc.free(c.ref.model.kitten.dataDir); calloc.free(c.ref.model.kitten.tokens); calloc.free(c.ref.model.kitten.voices); @@ -395,6 +489,26 @@ class OfflineTts { return SherpaOnnxBindings.offlineTtsNumSpeakers?.call(ptr) ?? 0; } + /// Return the language/voice of the model, e.g., "en-us". + /// Returns an empty string if such information is not available. + String get lang { + if (SherpaOnnxBindings.offlineTtsLang == null) { + throw Exception("Please initialize sherpa-onnx first"); + } + + if (ptr == nullptr) { + return ""; + } + + final p = SherpaOnnxBindings.offlineTtsLang?.call(ptr); + if (p == null) { + return ""; + } + final ans = p.toDartString(); + SherpaOnnxBindings.destroyOfflineTtsLang?.call(p); + return ans; + } + Pointer ptr; OfflineTtsConfig config; } diff --git a/flutter/sherpa_onnx/lib/src/tts_config.dart b/flutter/sherpa_onnx/lib/src/tts_config.dart index 48a35df69..69d12bc55 100644 --- a/flutter/sherpa_onnx/lib/src/tts_config.dart +++ b/flutter/sherpa_onnx/lib/src/tts_config.dart @@ -17,6 +17,8 @@ class OfflineTtsGenerationConfig { this.referenceText = '', this.numSteps = 5, this.extra = const {}, + this.phonemeCodepoints, + this.tokens, }); final double silenceScale; @@ -33,6 +35,22 @@ class OfflineTtsGenerationConfig { /// key: string /// value: string | int | double final Map extra; + + /// Pre-phonemized input as Unicode codepoints, e.g. produced by the + /// `piper_phonemize` package. One sub-array per sentence. + /// + /// When set, it is used in place of the `text` argument of + /// `generateWithConfig` (pass `''` as the text). Every sub-array must be + /// non-empty. Mutually exclusive with [tokens]. + final List>? phonemeCodepoints; + + /// Pre-tokenized input, e.g. Chinese pinyin like `zhong1`. One sub-array + /// per sentence. + /// + /// When set, it is used in place of the `text` argument of + /// `generateWithConfig` (pass `''` as the text). Every sub-array must be + /// non-empty. Mutually exclusive with [phonemeCodepoints]. + final List>? tokens; } /// VITS model configuration. @@ -78,6 +96,12 @@ class OfflineTtsVitsModelConfig { final String model; final String lexicon; final String tokens; + /// Path to espeak-ng-data. + /// + /// Deprecated. Ignored since v2.0.0. Use a lexicon file or pass + /// pre-phonemized input via [OfflineTtsGenerationConfig.phonemeCodepoints] + /// (or [OfflineTtsGenerationConfig.tokens]) instead. + @Deprecated('Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead.') final String dataDir; final double noiseScale; final double noiseScaleW; @@ -129,6 +153,12 @@ class OfflineTtsMatchaModelConfig { final String vocoder; final String lexicon; final String tokens; + /// Path to espeak-ng-data. + /// + /// Deprecated. Ignored since v2.0.0. Use a lexicon file or pass + /// pre-phonemized input via [OfflineTtsGenerationConfig.phonemeCodepoints] + /// (or [OfflineTtsGenerationConfig.tokens]) instead. + @Deprecated('Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead.') final String dataDir; final double noiseScale; final double lengthScale; @@ -178,6 +208,12 @@ class OfflineTtsKokoroModelConfig { final String model; final String voices; final String tokens; + /// Path to espeak-ng-data. + /// + /// Deprecated. Ignored since v2.0.0. Use a lexicon file or pass + /// pre-phonemized input via [OfflineTtsGenerationConfig.phonemeCodepoints] + /// (or [OfflineTtsGenerationConfig.tokens]) instead. + @Deprecated('Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead.') final String dataDir; final double lengthScale; final String dictDir; // unused @@ -193,6 +229,7 @@ class OfflineTtsKittenModelConfig { this.tokens = '', this.dataDir = '', this.lengthScale = 1.0, + this.lexicon = '', }); factory OfflineTtsKittenModelConfig.fromJson(Map json) { @@ -202,12 +239,13 @@ class OfflineTtsKittenModelConfig { tokens: json['tokens'] as String? ?? '', dataDir: json['dataDir'] as String? ?? '', lengthScale: (json['lengthScale'] as num?)?.toDouble() ?? 1.0, + lexicon: json['lexicon'] as String? ?? '', ); } @override String toString() { - return 'OfflineTtsKittenModelConfig(model: $model, voices: $voices, tokens: $tokens, dataDir: $dataDir, lengthScale: $lengthScale)'; + return 'OfflineTtsKittenModelConfig(model: $model, voices: $voices, tokens: $tokens, dataDir: $dataDir, lengthScale: $lengthScale, lexicon: $lexicon)'; } Map toJson() => { @@ -216,13 +254,23 @@ class OfflineTtsKittenModelConfig { 'tokens': tokens, 'dataDir': dataDir, 'lengthScale': lengthScale, + 'lexicon': lexicon, }; final String model; final String voices; final String tokens; + /// Path to espeak-ng-data. + /// + /// Deprecated. Ignored since v2.0.0. Use a lexicon file or pass + /// pre-phonemized input via [OfflineTtsGenerationConfig.phonemeCodepoints] + /// (or [OfflineTtsGenerationConfig.tokens]) instead. + @Deprecated('Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead.') final String dataDir; final double lengthScale; + + /// Path to the lexicon file. + final String lexicon; } /// ZipVoice model configuration. @@ -277,6 +325,12 @@ class OfflineTtsZipVoiceModelConfig { final String encoder; final String decoder; final String vocoder; + /// Path to espeak-ng-data. + /// + /// Deprecated. Ignored since v2.0.0. Use a lexicon file or pass + /// pre-phonemized input via [OfflineTtsGenerationConfig.phonemeCodepoints] + /// (or [OfflineTtsGenerationConfig.tokens]) instead. + @Deprecated('Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead.') final String dataDir; final String lexicon; final double featScale; @@ -402,7 +456,7 @@ class OfflineTtsModelConfig { this.pocket = const OfflineTtsPocketModelConfig(), this.supertonic = const OfflineTtsSupertonicModelConfig(), this.numThreads = 1, - this.debug = true, + this.debug = false, this.provider = 'cpu', }); @@ -470,7 +524,7 @@ class OfflineTtsConfig { const OfflineTtsConfig({ required this.model, this.ruleFsts = '', - this.maxNumSenetences = 1, + this.maxNumSentences = 1, this.ruleFars = '', this.silenceScale = 0.2, }); @@ -481,7 +535,7 @@ class OfflineTtsConfig { json['model'] as Map, ), ruleFsts: json['ruleFsts'] as String? ?? '', - maxNumSenetences: json['maxNumSenetences'] as int? ?? 1, + maxNumSentences: json['maxNumSentences'] as int? ?? 1, ruleFars: json['ruleFars'] as String? ?? '', silenceScale: (json['silenceScale'] as num?)?.toDouble() ?? 0.2, ); @@ -489,20 +543,20 @@ class OfflineTtsConfig { @override String toString() { - return 'OfflineTtsConfig(model: $model, ruleFsts: $ruleFsts, maxNumSenetences: $maxNumSenetences, ruleFars: $ruleFars, silenceScale: $silenceScale)'; + return 'OfflineTtsConfig(model: $model, ruleFsts: $ruleFsts, maxNumSentences: $maxNumSentences, ruleFars: $ruleFars, silenceScale: $silenceScale)'; } Map toJson() => { 'model': model.toJson(), 'ruleFsts': ruleFsts, - 'maxNumSenetences': maxNumSenetences, + 'maxNumSentences': maxNumSentences, 'ruleFars': ruleFars, 'silenceScale': silenceScale, }; final OfflineTtsModelConfig model; final String ruleFsts; - final int maxNumSenetences; + final int maxNumSentences; final String ruleFars; final double silenceScale; } diff --git a/flutter/sherpa_onnx/lib/src/web/offline_recognizer.dart b/flutter/sherpa_onnx/lib/src/web/offline_recognizer.dart index d7561842b..4243c3393 100644 --- a/flutter/sherpa_onnx/lib/src/web/offline_recognizer.dart +++ b/flutter/sherpa_onnx/lib/src/web/offline_recognizer.dart @@ -121,6 +121,12 @@ JSObject _buildConfig(OfflineRecognizerConfig config) { hr['ruleFsts'] = config.hr.ruleFsts.toJS; jsConfig['hr'] = hr; + // CTC FST decoder config. + final ctcFstDecoderConfig = JSObject(); + ctcFstDecoderConfig['graph'] = config.ctcFstDecoderConfig.graph.toJS; + ctcFstDecoderConfig['maxActive'] = config.ctcFstDecoderConfig.maxActive.toJS; + jsConfig['ctcFstDecoderConfig'] = ctcFstDecoderConfig; + // Scalar fields. jsConfig['decodingMethod'] = config.decodingMethod.toJS; jsConfig['maxActivePaths'] = config.maxActivePaths.toJS; diff --git a/flutter/sherpa_onnx/lib/src/web/tts.dart b/flutter/sherpa_onnx/lib/src/web/tts.dart index b7b6443b5..a46821ba5 100644 --- a/flutter/sherpa_onnx/lib/src/web/tts.dart +++ b/flutter/sherpa_onnx/lib/src/web/tts.dart @@ -1,5 +1,6 @@ // Copyright (c) 2026 Xiaomi Corporation // Web implementation of OfflineTTS using dart:js_interop. +import 'dart:convert'; import 'dart:js_interop'; import 'dart:js_interop_unsafe'; import 'dart:typed_data'; @@ -9,6 +10,13 @@ import 'init.dart'; export '../tts_config.dart'; +/// Parse [source] into a JS value with the host's JSON.parse. +JSAny _jsonParse(String source) { + final json = globalContext.getProperty('JSON'.toJS) as JSObject; + final parse = json.getProperty('parse'.toJS) as JSFunction; + return parse.callAsFunction(null, source.toJS)!; +} + /// Offline text-to-speech engine (web implementation). class OfflineTts { OfflineTts.fromPtr({required this.ptr, required this.config}); @@ -102,6 +110,19 @@ class OfflineTts { genConfig['extra'] = extraObj; } + // Pre-phonemized / pre-tokenized input (number[][] and string[][]). + // The JS glue validates the shapes (same strict contract as the native + // bindings), so just forward them. Built with JSON.parse to stay within + // the oldest supported dart:js_interop surface. + final codepoints = config.phonemeCodepoints; + if (codepoints != null && codepoints.isNotEmpty) { + genConfig['phonemeCodepoints'] = _jsonParse(jsonEncode(codepoints)); + } + final tokenSentences = config.tokens; + if (tokenSentences != null && tokenSentences.isNotEmpty) { + genConfig['tokens'] = _jsonParse(jsonEncode(tokenSentences)); + } + if (onProgress != null) { // Create a JS callback that calls the Dart callback. genConfig['callback'] = (JSAny samplesPtr, JSAny n, JSAny progress, JSAny arg) { @@ -136,6 +157,14 @@ class OfflineTts { return val is JSNumber ? val.toDartInt : 0; } + /// Return the language/voice of the model, e.g., "en-us". + /// Returns an empty string if such information is not available. + String get lang { + final handle = ptr as JSObject; + final val = handle.getProperty('lang'.toJS); + return val is JSString ? val.toDart : ""; + } + dynamic ptr; OfflineTtsConfig config; bool _freed = false; @@ -183,6 +212,7 @@ JSObject _configToJs(OfflineTtsConfig config) { kitten['tokens'] = config.model.kitten.tokens.toJS; kitten['dataDir'] = config.model.kitten.dataDir.toJS; kitten['lengthScale'] = config.model.kitten.lengthScale.toJS; + kitten['lexicon'] = config.model.kitten.lexicon.toJS; model['kitten'] = kitten; final zipvoice = JSObject(); @@ -227,7 +257,7 @@ JSObject _configToJs(OfflineTtsConfig config) { jsConfig['offlineTtsModelConfig'] = model; jsConfig['ruleFsts'] = config.ruleFsts.toJS; jsConfig['ruleFars'] = config.ruleFars.toJS; - jsConfig['maxNumSentences'] = config.maxNumSenetences.toJS; + jsConfig['maxNumSentences'] = config.maxNumSentences.toJS; jsConfig['silenceScale'] = config.silenceScale.toJS; return jsConfig; diff --git a/go-api-examples/README.md b/go-api-examples/README.md index 7e1277371..edc2b06e8 100644 --- a/go-api-examples/README.md +++ b/go-api-examples/README.md @@ -31,7 +31,12 @@ for details. the online speech denoiser API with DPDFNet models. - [./non-streaming-tts](./non-streaming-tts) It shows how to use a non-streaming TTS - model to convert text to speech + model to convert text to speech. It uses a lexicon file for text normalization. + +- [./non-streaming-tts-phonemize](./non-streaming-tts-phonemize) It shows how to use a + non-streaming TTS model to convert text to speech. It phonemizes the text outside of + sherpa-onnx with [piper-phonemize-go](https://github.com/csukuangfj/piper-phonemize-go) + and passes the phoneme codepoints to sherpa-onnx. - [./offline-tts-play](./offline-tts-play) It shows how to use a non-streaming TTS model to convert text to speech. It plays the audio back as it is being generated. diff --git a/go-api-examples/non-streaming-tts-phonemize/go.mod b/go-api-examples/non-streaming-tts-phonemize/go.mod new file mode 100644 index 000000000..16bef31b3 --- /dev/null +++ b/go-api-examples/non-streaming-tts-phonemize/go.mod @@ -0,0 +1,3 @@ +module non-streaming-tts-phonemize + +go 1.17 diff --git a/go-api-examples/non-streaming-tts-phonemize/main.go b/go-api-examples/non-streaming-tts-phonemize/main.go new file mode 100644 index 000000000..bd403e6f4 --- /dev/null +++ b/go-api-examples/non-streaming-tts-phonemize/main.go @@ -0,0 +1,135 @@ +package main + +import ( + "log" + "math" + + pp "github.com/csukuangfj/piper-phonemize-go/piper_phonemize" + sherpa "github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx" + flag "github.com/spf13/pflag" +) + +// This example shows the piper-phonemize text frontend for TTS. +// +// The text is phonemized outside of sherpa-onnx with +// https://github.com/csukuangfj/piper-phonemize-go and the resulting phoneme +// codepoints are passed to sherpa-onnx via GenerationConfig.PhonemeCodepoints. +// No lexicon file is needed. Please see +// https://github.com/k2-fsa/sherpa-onnx/tree/master/python-api-examples +// for the corresponding Python examples, e.g., +// test-offline-tts-piper-phonemize.py +func main() { + log.SetFlags(log.LstdFlags | log.Lmicroseconds) + + config := sherpa.OfflineTtsConfig{} + lang := "" + sid := 0 + filename := "./generated.wav" + + var speed float32 + + flag.StringVar(&config.Model.Vits.Model, "vits-model", "", "Path to the vits ONNX model") + flag.StringVar(&config.Model.Vits.Tokens, "vits-tokens", "", "Path to tokens.txt") + flag.Float32Var(&config.Model.Vits.NoiseScale, "vits-noise-scale", 0.667, "noise_scale for VITS") + flag.Float32Var(&config.Model.Vits.NoiseScaleW, "vits-noise-scale-w", 0.8, "noise_scale_w for VITS") + flag.Float32Var(&config.Model.Vits.LengthScale, "vits-length-scale", 1.0, "length_scale for VITS. small -> faster; large -> slower") + + flag.StringVar(&config.Model.Matcha.AcousticModel, "matcha-acoustic-model", "", "Path to the matcha acoustic model") + flag.StringVar(&config.Model.Matcha.Vocoder, "matcha-vocoder", "", "Path to the matcha vocoder model") + flag.StringVar(&config.Model.Matcha.Tokens, "matcha-tokens", "", "Path to tokens.txt") + flag.Float32Var(&config.Model.Matcha.NoiseScale, "matcha-noise-scale", 0.667, "noise_scale for Matcha") + flag.Float32Var(&config.Model.Matcha.LengthScale, "matcha-length-scale", 1.0, "length_scale for Matcha. small -> faster; large -> slower") + + flag.StringVar(&config.Model.Kokoro.Model, "kokoro-model", "", "Path to the Kokoro ONNX model") + flag.StringVar(&config.Model.Kokoro.Voices, "kokoro-voices", "", "Path to voices.bin for Kokoro") + flag.StringVar(&config.Model.Kokoro.Tokens, "kokoro-tokens", "", "Path to tokens.txt for Kokoro") + flag.Float32Var(&config.Model.Kokoro.LengthScale, "kokoro-length-scale", 1.0, "length_scale for Kokoro. small -> faster; large -> slower") + + flag.StringVar(&config.Model.Kitten.Model, "kitten-model", "", "Path to the kitten ONNX model") + flag.StringVar(&config.Model.Kitten.Voices, "kitten-voices", "", "Path to voices.bin for kitten") + flag.StringVar(&config.Model.Kitten.Tokens, "kitten-tokens", "", "Path to tokens.txt for kitten") + flag.Float32Var(&config.Model.Kitten.LengthScale, "kitten-length-scale", 1.0, "length_scale for kitten. small -> faster; large -> slower") + + flag.StringVar(&lang, "lang", lang, "espeak-ng voice used by the piper phonemizer, e.g., en-us. Empty means use the language of the model") + flag.Float32Var(&speed, "speed", 1.0, "Speech speed. larger->faster; smaller->slower") + + flag.IntVar(&config.Model.NumThreads, "num-threads", 1, "Number of threads for computing") + flag.IntVar(&config.Model.Debug, "debug", 0, "Whether to show debug message") + flag.StringVar(&config.Model.Provider, "provider", "cpu", "Provider to use: cpu/cuda/coreml") + flag.StringVar(&config.RuleFsts, "tts-rule-fsts", "", "Path to rule.fst") + flag.StringVar(&config.RuleFars, "tts-rule-fars", "", "Path to rule.far") + flag.IntVar(&config.MaxNumSentences, "tts-max-num-sentences", 1, "Batch size (split long text to avoid OOM)") + + flag.IntVar(&sid, "sid", sid, "Speaker ID (multi-speaker models only)") + flag.StringVar(&filename, "output-filename", filename, "Output wav filename") + + flag.Parse() + + if len(flag.Args()) != 1 { + log.Fatalf("Please provide the text to generate audios") + } + text := flag.Arg(0) + + log.Println("Input text:", text) + log.Println("Speaker ID:", sid) + log.Println("Output filename:", filename) + + log.Println("Initializing model (may take several seconds)") + tts := sherpa.NewOfflineTts(&config) + defer sherpa.DeleteOfflineTts(tts) + log.Println("Model created!") + + // Determine the language for the phonemizer. Use the one from the model + // metadata unless --lang is given explicitly. + if lang == "" { + lang = tts.Lang() + if lang == "" { + log.Println("Warning: the model has no language information. " + + "Falling back to en-us. Please use --lang to set it explicitly.") + lang = "en-us" + } + } + log.Println("Phonemizer language:", lang) + + // Phonemize the text outside of sherpa-onnx. + // + // Initialize("") uses the espeak-ng-data embedded in piper-phonemize-go. + // It has nothing to do with sherpa-onnx; sherpa-onnx itself does not use + // espeak-ng anymore. + log.Println("piper-phonemize version:", pp.GetVersionStr()) + if ret := pp.Initialize(""); ret < 0 { + log.Fatalf("Failed to initialize piper-phonemize") + } + + result := pp.Phonemize(text, lang) + if result == nil { + log.Fatalf("Failed to phonemize the text") + } + defer pp.DeletePhonemizeResult(result) + + var sentences [][]int32 + for i := 0; i < result.GetNumSentences(); i++ { + var s []int32 + for _, cp := range result.GetPhonemes(i) { + s = append(s, int32(cp)) + } + sentences = append(sentences, s) + } + log.Println("Number of phonemized sentences:", len(sentences)) + + log.Println("Start generating!") + cfg := sherpa.GenerationConfig{ + SilenceScale: 0.2, + Speed: float32(math.Max(float64(speed), 1e-6)), + Sid: sid, + PhonemeCodepoints: sentences, + } + // The text has already been phonemized; pass an empty string here. + audio := tts.GenerateWithConfig("", &cfg, nil) + + log.Println("Done!") + if ok := audio.Save(filename); !ok { + log.Fatalf("Failed to write %s", filename) + } + log.Println("Saved to", filename) +} diff --git a/go-api-examples/non-streaming-tts-phonemize/run-kitten-en-phonemize.sh b/go-api-examples/non-streaming-tts-phonemize/run-kitten-en-phonemize.sh new file mode 100755 index 000000000..01044c59a --- /dev/null +++ b/go-api-examples/non-streaming-tts-phonemize/run-kitten-en-phonemize.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -ex + +export CGO_ENABLED=1 + +if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 + tar xf kitten-nano-en-v0_1-fp16.tar.bz2 + rm kitten-nano-en-v0_1-fp16.tar.bz2 +fi + +go mod tidy +go build + +./non-streaming-tts-phonemize \ + --kitten-model=./kitten-nano-en-v0_1-fp16/model.fp16.onnx \ + --kitten-voices=./kitten-nano-en-v0_1-fp16/voices.bin \ + --kitten-tokens=./kitten-nano-en-v0_1-fp16/tokens.txt \ + --debug=1 \ + --output-filename=./test-kitten-en-phonemize.wav \ + "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/go-api-examples/non-streaming-tts-phonemize/run-kokoro-en-phonemize.sh b/go-api-examples/non-streaming-tts-phonemize/run-kokoro-en-phonemize.sh new file mode 100755 index 000000000..65c0d13d3 --- /dev/null +++ b/go-api-examples/non-streaming-tts-phonemize/run-kokoro-en-phonemize.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -ex + +export CGO_ENABLED=1 + +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +go mod tidy +go build + +./non-streaming-tts-phonemize \ + --kokoro-model=./kokoro-en-v0_19/model.onnx \ + --kokoro-voices=./kokoro-en-v0_19/voices.bin \ + --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ + --debug=1 \ + --output-filename=./test-kokoro-en-phonemize.wav \ + "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/go-api-examples/non-streaming-tts-phonemize/run-matcha-en-phonemize.sh b/go-api-examples/non-streaming-tts-phonemize/run-matcha-en-phonemize.sh new file mode 100755 index 000000000..6f18cc3e2 --- /dev/null +++ b/go-api-examples/non-streaming-tts-phonemize/run-matcha-en-phonemize.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -ex + +export CGO_ENABLED=1 + +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./vocos-22khz-univ.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +fi + +go mod tidy +go build + +./non-streaming-tts-phonemize \ + --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ + --matcha-vocoder=./vocos-22khz-univ.onnx \ + --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ + --debug=1 \ + --output-filename=./test-matcha-en-phonemize.wav \ + "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/go-api-examples/non-streaming-tts-phonemize/run-vits-piper-phonemize.sh b/go-api-examples/non-streaming-tts-phonemize/run-vits-piper-phonemize.sh new file mode 100755 index 000000000..7698b26cc --- /dev/null +++ b/go-api-examples/non-streaming-tts-phonemize/run-vits-piper-phonemize.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -ex + +export CGO_ENABLED=1 + +if [ ! -d vits-piper-en_US-lessac-medium ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-lessac-medium.tar.bz2 + tar xf vits-piper-en_US-lessac-medium.tar.bz2 + rm vits-piper-en_US-lessac-medium.tar.bz2 +fi + +go mod tidy +go build + +./non-streaming-tts-phonemize \ + --vits-model=./vits-piper-en_US-lessac-medium/en_US-lessac-medium.onnx \ + --vits-tokens=./vits-piper-en_US-lessac-medium/tokens.txt \ + --output-filename=./liliana-piper-en_US-lessac-medium-phonemize.wav \ + 'liliana, the most beautiful and lovely assistant of our team!' diff --git a/go-api-examples/non-streaming-tts/main.go b/go-api-examples/non-streaming-tts/main.go index 5f6b8e7d3..27cb06b8a 100644 --- a/go-api-examples/non-streaming-tts/main.go +++ b/go-api-examples/non-streaming-tts/main.go @@ -20,7 +20,7 @@ func main() { flag.StringVar(&config.Model.Vits.Model, "vits-model", "", "Path to the vits ONNX model") flag.StringVar(&config.Model.Vits.Lexicon, "vits-lexicon", "", "Path to lexicon.txt") flag.StringVar(&config.Model.Vits.Tokens, "vits-tokens", "", "Path to tokens.txt") - flag.StringVar(&config.Model.Vits.DataDir, "vits-data-dir", "", "Path to espeak-ng-data") + flag.StringVar(&config.Model.Vits.DataDir, "vits-data-dir", "", "Deprecated. Ignored.") flag.Float32Var(&config.Model.Vits.NoiseScale, "vits-noise-scale", 0.667, "noise_scale for VITS") flag.Float32Var(&config.Model.Vits.NoiseScaleW, "vits-noise-scale-w", 0.8, "noise_scale_w for VITS") flag.Float32Var(&config.Model.Vits.LengthScale, "vits-length-scale", 1.0, "length_scale for VITS. small -> faster; large -> slower") @@ -29,21 +29,22 @@ func main() { flag.StringVar(&config.Model.Matcha.Vocoder, "matcha-vocoder", "", "Path to the matcha vocoder model") flag.StringVar(&config.Model.Matcha.Lexicon, "matcha-lexicon", "", "Path to lexicon.txt") flag.StringVar(&config.Model.Matcha.Tokens, "matcha-tokens", "", "Path to tokens.txt") - flag.StringVar(&config.Model.Matcha.DataDir, "matcha-data-dir", "", "Path to espeak-ng-data") + flag.StringVar(&config.Model.Matcha.DataDir, "matcha-data-dir", "", "Deprecated. Ignored.") flag.Float32Var(&config.Model.Matcha.NoiseScale, "matcha-noise-scale", 0.667, "noise_scale for Matcha") flag.Float32Var(&config.Model.Matcha.LengthScale, "matcha-length-scale", 1.0, "length_scale for Matcha. small -> faster; large -> slower") flag.StringVar(&config.Model.Kokoro.Model, "kokoro-model", "", "Path to the Kokoro ONNX model") flag.StringVar(&config.Model.Kokoro.Voices, "kokoro-voices", "", "Path to voices.bin for Kokoro") flag.StringVar(&config.Model.Kokoro.Tokens, "kokoro-tokens", "", "Path to tokens.txt for Kokoro") - flag.StringVar(&config.Model.Kokoro.DataDir, "kokoro-data-dir", "", "Path to espeak-ng-data for Kokoro") + flag.StringVar(&config.Model.Kokoro.DataDir, "kokoro-data-dir", "", "Deprecated. Ignored.") flag.StringVar(&config.Model.Kokoro.Lexicon, "kokoro-lexicon", "", "Path to lexicon files for Kokoro") flag.Float32Var(&config.Model.Kokoro.LengthScale, "kokoro-length-scale", 1.0, "length_scale for Kokoro. small -> faster; large -> slower") flag.StringVar(&config.Model.Kitten.Model, "kitten-model", "", "Path to the kitten ONNX model") flag.StringVar(&config.Model.Kitten.Voices, "kitten-voices", "", "Path to voices.bin for kitten") flag.StringVar(&config.Model.Kitten.Tokens, "kitten-tokens", "", "Path to tokens.txt for kitten") - flag.StringVar(&config.Model.Kitten.DataDir, "kitten-data-dir", "", "Path to espeak-ng-data for kitten") + flag.StringVar(&config.Model.Kitten.DataDir, "kitten-data-dir", "", "Deprecated. Ignored.") + flag.StringVar(&config.Model.Kitten.Lexicon, "kitten-lexicon", "", "Path to lexicon.txt for kitten") flag.Float32Var(&config.Model.Kitten.LengthScale, "kitten-length-scale", 1.0, "length_scale for kitten. small -> faster; large -> slower") flag.Float32Var(&speed, "speed", 1.0, "Speech speed. larger->faster; smaller->slower") diff --git a/go-api-examples/non-streaming-tts/run-kitten-en.sh b/go-api-examples/non-streaming-tts/run-kitten-en.sh index 4d2a4705a..e621c709a 100755 --- a/go-api-examples/non-streaming-tts/run-kitten-en.sh +++ b/go-api-examples/non-streaming-tts/run-kitten-en.sh @@ -10,6 +10,10 @@ if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then rm kitten-nano-en-v0_1-fp16.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build @@ -17,7 +21,7 @@ go build --kitten-model=./kitten-nano-en-v0_1-fp16/model.fp16.onnx \ --kitten-voices=./kitten-nano-en-v0_1-fp16/voices.bin \ --kitten-tokens=./kitten-nano-en-v0_1-fp16/tokens.txt \ - --kitten-data-dir=./kitten-nano-en-v0_1-fp16/espeak-ng-data \ + --kitten-lexicon=./lexicon-en-us.txt \ --debug=1 \ --output-filename=./test-kitten-en.wav \ "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/go-api-examples/non-streaming-tts/run-kokoro-en.sh b/go-api-examples/non-streaming-tts/run-kokoro-en.sh index a13facf41..0d2bda079 100755 --- a/go-api-examples/non-streaming-tts/run-kokoro-en.sh +++ b/go-api-examples/non-streaming-tts/run-kokoro-en.sh @@ -10,6 +10,10 @@ if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then rm kokoro-en-v0_19.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build @@ -17,7 +21,7 @@ go build --kokoro-model=./kokoro-en-v0_19/model.onnx \ --kokoro-voices=./kokoro-en-v0_19/voices.bin \ --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ - --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --kokoro-lexicon=./lexicon-en-us.txt \ --debug=1 \ --output-filename=./test-kokoro-en.wav \ "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/go-api-examples/non-streaming-tts/run-kokoro-zh-en.sh b/go-api-examples/non-streaming-tts/run-kokoro-zh-en.sh index 3fc6e8159..b09603bdc 100755 --- a/go-api-examples/non-streaming-tts/run-kokoro-zh-en.sh +++ b/go-api-examples/non-streaming-tts/run-kokoro-zh-en.sh @@ -17,7 +17,6 @@ go build --kokoro-model=./kokoro-multi-lang-v1_0/model.onnx \ --kokoro-voices=./kokoro-multi-lang-v1_0/voices.bin \ --kokoro-tokens=./kokoro-multi-lang-v1_0/tokens.txt \ - --kokoro-data-dir=./kokoro-multi-lang-v1_0/espeak-ng-data \ --kokoro-lexicon=./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt \ --debug=1 \ --output-filename=./test-kokoro-zh-en.wav \ diff --git a/go-api-examples/non-streaming-tts/run-matcha-en.sh b/go-api-examples/non-streaming-tts/run-matcha-en.sh index f8208dedb..7aa92a4d0 100755 --- a/go-api-examples/non-streaming-tts/run-matcha-en.sh +++ b/go-api-examples/non-streaming-tts/run-matcha-en.sh @@ -18,6 +18,10 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build @@ -25,7 +29,7 @@ go build --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ --matcha-vocoder=./vocos-22khz-univ.onnx \ --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ - --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --matcha-lexicon=./lexicon-en-us.txt \ --debug=1 \ --output-filename=./test-matcha-en.wav \ "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/go-api-examples/non-streaming-tts/run-vits-piper-en_US-lessac-medium.sh b/go-api-examples/non-streaming-tts/run-vits-piper-en_US-lessac-medium.sh index 4b820598f..957beccb3 100755 --- a/go-api-examples/non-streaming-tts/run-vits-piper-en_US-lessac-medium.sh +++ b/go-api-examples/non-streaming-tts/run-vits-piper-en_US-lessac-medium.sh @@ -10,12 +10,16 @@ if [ ! -d vits-piper-en_US-lessac-medium ]; then rm vits-piper-en_US-lessac-medium.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build ./non-streaming-tts \ --vits-model=./vits-piper-en_US-lessac-medium/en_US-lessac-medium.onnx \ - --vits-data-dir=./vits-piper-en_US-lessac-medium/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --vits-tokens=./vits-piper-en_US-lessac-medium/tokens.txt \ --output-filename=./liliana-piper-en_US-lessac-medium.wav \ 'liliana, the most beautiful and lovely assistant of our team!' diff --git a/go-api-examples/offline-tts-play/main.go b/go-api-examples/offline-tts-play/main.go index 9be920b9d..35f5489de 100644 --- a/go-api-examples/offline-tts-play/main.go +++ b/go-api-examples/offline-tts-play/main.go @@ -98,7 +98,7 @@ func main() { flag.StringVar(&config.Model.Vits.Model, "vits-model", "", "Path to the vits ONNX model") flag.StringVar(&config.Model.Vits.Lexicon, "vits-lexicon", "", "Path to lexicon.txt") flag.StringVar(&config.Model.Vits.Tokens, "vits-tokens", "", "Path to tokens.txt") - flag.StringVar(&config.Model.Vits.DataDir, "vits-data-dir", "", "Path to espeak-ng-data") + flag.StringVar(&config.Model.Vits.DataDir, "vits-data-dir", "", "Deprecated. Ignored.") flag.Float32Var(&config.Model.Vits.NoiseScale, "vits-noise-scale", 0.667, "noise_scale for VITS") flag.Float32Var(&config.Model.Vits.NoiseScaleW, "vits-noise-scale-w", 0.8, "noise_scale_w for VITS") @@ -108,7 +108,7 @@ func main() { flag.StringVar(&config.Model.Matcha.Vocoder, "matcha-vocoder", "", "Path to the matcha vocoder model") flag.StringVar(&config.Model.Matcha.Lexicon, "matcha-lexicon", "", "Path to lexicon.txt") flag.StringVar(&config.Model.Matcha.Tokens, "matcha-tokens", "", "Path to tokens.txt") - flag.StringVar(&config.Model.Matcha.DataDir, "matcha-data-dir", "", "Path to espeak-ng-data") + flag.StringVar(&config.Model.Matcha.DataDir, "matcha-data-dir", "", "Deprecated. Ignored.") flag.Float32Var(&config.Model.Matcha.NoiseScale, "matcha-noise-scale", 0.667, "noise_scale for Matcha") flag.Float32Var(&config.Model.Matcha.LengthScale, "matcha-length-scale", 1.0, "length_scale for Matcha. small -> faster in speech speed; large -> slower") @@ -116,14 +116,15 @@ func main() { flag.StringVar(&config.Model.Kokoro.Model, "kokoro-model", "", "Path to the Kokoro ONNX model") flag.StringVar(&config.Model.Kokoro.Voices, "kokoro-voices", "", "Path to voices.bin for Kokoro") flag.StringVar(&config.Model.Kokoro.Tokens, "kokoro-tokens", "", "Path to tokens.txt for Kokoro") - flag.StringVar(&config.Model.Kokoro.DataDir, "kokoro-data-dir", "", "Path to espeak-ng-data for Kokoro") + flag.StringVar(&config.Model.Kokoro.DataDir, "kokoro-data-dir", "", "Deprecated. Ignored.") flag.StringVar(&config.Model.Kokoro.Lexicon, "kokoro-lexicon", "", "Path to lexicon files for Kokoro") flag.Float32Var(&config.Model.Kokoro.LengthScale, "kokoro-length-scale", 1.0, "length_scale for Kokoro. small -> faster in speech speed; large -> slower") flag.StringVar(&config.Model.Kitten.Model, "kitten-model", "", "Path to the kitten ONNX model") flag.StringVar(&config.Model.Kitten.Voices, "kitten-voices", "", "Path to voices.bin for kitten") flag.StringVar(&config.Model.Kitten.Tokens, "kitten-tokens", "", "Path to tokens.txt for kitten") - flag.StringVar(&config.Model.Kitten.DataDir, "kitten-data-dir", "", "Path to espeak-ng-data for kitten") + flag.StringVar(&config.Model.Kitten.DataDir, "kitten-data-dir", "", "Deprecated. Ignored.") + flag.StringVar(&config.Model.Kitten.Lexicon, "kitten-lexicon", "", "Path to lexicon.txt for kitten") flag.Float32Var(&config.Model.Kitten.LengthScale, "kitten-length-scale", 1.0, "length_scale for kitten. small -> faster in speech speed; large -> slower") flag.IntVar(&config.Model.NumThreads, "num-threads", 1, "Number of threads for computing") diff --git a/go-api-examples/offline-tts-play/run-kitten-en.sh b/go-api-examples/offline-tts-play/run-kitten-en.sh index 437bbe127..cf648c82b 100755 --- a/go-api-examples/offline-tts-play/run-kitten-en.sh +++ b/go-api-examples/offline-tts-play/run-kitten-en.sh @@ -10,6 +10,10 @@ if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then rm kitten-nano-en-v0_1-fp16.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build @@ -17,6 +21,6 @@ go build --kitten-model=./kitten-nano-en-v0_1-fp16/model.fp16.onnx \ --kitten-voices=./kitten-nano-en-v0_1-fp16/voices.bin \ --kitten-tokens=./kitten-nano-en-v0_1-fp16/tokens.txt \ - --kitten-data-dir=./kitten-nano-en-v0_1-fp16/espeak-ng-data \ + --kitten-lexicon=./lexicon-en-us.txt \ --debug=1 \ "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/go-api-examples/offline-tts-play/run-kokoro-en.sh b/go-api-examples/offline-tts-play/run-kokoro-en.sh index 72ae4a086..935c57dd4 100755 --- a/go-api-examples/offline-tts-play/run-kokoro-en.sh +++ b/go-api-examples/offline-tts-play/run-kokoro-en.sh @@ -10,6 +10,10 @@ if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then rm kokoro-en-v0_19.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build @@ -17,6 +21,6 @@ go build --kokoro-model=./kokoro-en-v0_19/model.onnx \ --kokoro-voices=./kokoro-en-v0_19/voices.bin \ --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ - --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --kokoro-lexicon=./lexicon-en-us.txt \ --debug=1 \ "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/go-api-examples/offline-tts-play/run-kokoro-zh-en.sh b/go-api-examples/offline-tts-play/run-kokoro-zh-en.sh index 33c07d625..e854f0777 100755 --- a/go-api-examples/offline-tts-play/run-kokoro-zh-en.sh +++ b/go-api-examples/offline-tts-play/run-kokoro-zh-en.sh @@ -17,7 +17,6 @@ go build --kokoro-model=./kokoro-multi-lang-v1_0/model.onnx \ --kokoro-voices=./kokoro-multi-lang-v1_0/voices.bin \ --kokoro-tokens=./kokoro-multi-lang-v1_0/tokens.txt \ - --kokoro-data-dir=./kokoro-multi-lang-v1_0/espeak-ng-data \ --kokoro-lexicon=./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt \ --debug=1 \ "中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？" diff --git a/go-api-examples/offline-tts-play/run-matcha-en.sh b/go-api-examples/offline-tts-play/run-matcha-en.sh index a80cb70af..8dc628e50 100755 --- a/go-api-examples/offline-tts-play/run-matcha-en.sh +++ b/go-api-examples/offline-tts-play/run-matcha-en.sh @@ -18,6 +18,10 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build @@ -25,7 +29,7 @@ go build --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ --matcha-vocoder=./vocos-22khz-univ.onnx \ --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ - --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --matcha-lexicon=./lexicon-en-us.txt \ --debug=1 \ "Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." diff --git a/go-api-examples/offline-tts-play/run-vits-piper-en_US-lessac-medium.sh b/go-api-examples/offline-tts-play/run-vits-piper-en_US-lessac-medium.sh index a9925f6cf..a59299d58 100755 --- a/go-api-examples/offline-tts-play/run-vits-piper-en_US-lessac-medium.sh +++ b/go-api-examples/offline-tts-play/run-vits-piper-en_US-lessac-medium.sh @@ -10,11 +10,15 @@ if [ ! -d vits-piper-en_US-lessac-medium ]; then rm vits-piper-en_US-lessac-medium.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build ./offline-tts-play \ --vits-model=./vits-piper-en_US-lessac-medium/en_US-lessac-medium.onnx \ - --vits-data-dir=./vits-piper-en_US-lessac-medium/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --vits-tokens=./vits-piper-en_US-lessac-medium/tokens.txt \ 'liliana, the most beautiful and lovely assistant of our team!' diff --git a/go-api-examples/offline-zipformer-ctc-hlg/go.mod b/go-api-examples/offline-zipformer-ctc-hlg/go.mod new file mode 100644 index 000000000..3a2716252 --- /dev/null +++ b/go-api-examples/offline-zipformer-ctc-hlg/go.mod @@ -0,0 +1,3 @@ +module offline-zipformer-ctc-hlg + +go 1.17 diff --git a/go-api-examples/offline-zipformer-ctc-hlg/main.go b/go-api-examples/offline-zipformer-ctc-hlg/main.go new file mode 100644 index 000000000..9f71e4813 --- /dev/null +++ b/go-api-examples/offline-zipformer-ctc-hlg/main.go @@ -0,0 +1,52 @@ +// go-api-examples/offline-zipformer-ctc-hlg/main.go +// +// Copyright (c) 2026 Xiaomi Corporation +// +// Offline CTC decoding with an HLG graph via CtcFstDecoderConfig. +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +package main + +import ( + "flag" + "fmt" + + sherpa "github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx" +) + +func main() { + var config sherpa.OfflineRecognizerConfig + + flag.StringVar(&config.ModelConfig.ZipformerCtc.Model, "zipformer-ctc", "", "Path to the Zipformer CTC model") + flag.StringVar(&config.ModelConfig.Tokens, "tokens", "", "Path to tokens.txt") + + // This is what the example exercises: the graph and max-active reach the + // C API through CtcFstDecoderConfig. + flag.StringVar(&config.CtcFstDecoderConfig.Graph, "graph", "", "Path to H.fst, HL.fst or HLG.fst") + flag.IntVar(&config.CtcFstDecoderConfig.MaxActive, "max-active", 3000, "Decoder max-active setting") + + flag.Parse() + + if len(flag.Args()) != 1 { + panic("Usage: ./offline-zipformer-ctc-hlg --zipformer-ctc --tokens --graph ") + } + waveFilename := flag.Arg(0) + + config.ModelConfig.Debug = 0 + config.ModelConfig.Provider = "cpu" + config.ModelConfig.NumThreads = 1 + + wave := sherpa.ReadWave(waveFilename) + + recognizer := sherpa.NewOfflineRecognizer(&config) + defer sherpa.DeleteOfflineRecognizer(recognizer) + stream := sherpa.NewOfflineStream(recognizer) + defer sherpa.DeleteOfflineStream(stream) + stream.AcceptWaveform(wave.SampleRate, wave.Samples) + recognizer.Decode(stream) + + result := stream.GetResult() + fmt.Printf("text: %s\n", result.Text) +} diff --git a/go-api-examples/offline-zipformer-ctc-hlg/run.sh b/go-api-examples/offline-zipformer-ctc-hlg/run.sh new file mode 100755 index 000000000..6f98c5c47 --- /dev/null +++ b/go-api-examples/offline-zipformer-ctc-hlg/run.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -ex + +export CGO_ENABLED=1 + +if [ ! -f ./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + ls -lh sherpa-onnx-zipformer-ctc-en-2023-10-02 +fi + +go mod tidy +go build + +./offline-zipformer-ctc-hlg \ + --zipformer-ctc=./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx \ + --tokens=./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt \ + --graph=./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst \ + --max-active=3000 \ + ./sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav diff --git a/go-api-examples/zero-shot-zipvoice-tts-play/main.go b/go-api-examples/zero-shot-zipvoice-tts-play/main.go index 4cab34956..50cc5ba07 100644 --- a/go-api-examples/zero-shot-zipvoice-tts-play/main.go +++ b/go-api-examples/zero-shot-zipvoice-tts-play/main.go @@ -110,10 +110,8 @@ func main() { "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx" config.Model.Zipvoice.Decoder = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx" - config.Model.Zipvoice.DataDir = - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data" config.Model.Zipvoice.Lexicon = - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt" + "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt" config.Model.Zipvoice.Tokens = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt" config.Model.Zipvoice.Vocoder = "./vocos_24khz.onnx" diff --git a/go-api-examples/zero-shot-zipvoice-tts-play/run.sh b/go-api-examples/zero-shot-zipvoice-tts-play/run.sh index b375674cf..11331534c 100755 --- a/go-api-examples/zero-shot-zipvoice-tts-play/run.sh +++ b/go-api-examples/zero-shot-zipvoice-tts-play/run.sh @@ -14,6 +14,10 @@ if [ ! -f vocos_24khz.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build diff --git a/go-api-examples/zero-shot-zipvoice-tts/main.go b/go-api-examples/zero-shot-zipvoice-tts/main.go index 8ce464e0f..fb88d77d8 100644 --- a/go-api-examples/zero-shot-zipvoice-tts/main.go +++ b/go-api-examples/zero-shot-zipvoice-tts/main.go @@ -32,10 +32,8 @@ func main() { "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx" config.Model.Zipvoice.Decoder = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx" - config.Model.Zipvoice.DataDir = - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data" config.Model.Zipvoice.Lexicon = - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt" + "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt" config.Model.Zipvoice.Tokens = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt" config.Model.Zipvoice.Vocoder = "./vocos_24khz.onnx" diff --git a/go-api-examples/zero-shot-zipvoice-tts/run.sh b/go-api-examples/zero-shot-zipvoice-tts/run.sh index 282f1525f..b951eeb42 100755 --- a/go-api-examples/zero-shot-zipvoice-tts/run.sh +++ b/go-api-examples/zero-shot-zipvoice-tts/run.sh @@ -14,6 +14,10 @@ if [ ! -f vocos_24khz.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + go mod tidy go build diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets index 8ff60a4c0..48be3afed 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -26,6 +26,7 @@ export { Samples, OfflineZipformerCtcModelConfig, OfflineModelConfig, OfflineLMConfig, + OfflineCtcFstDecoderConfig, OfflineRecognizerConfig, OfflineRecognizerResult, OfflineRecognizer, diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc index d9a5790a2..12eb1e726 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-asr.cc @@ -414,6 +414,24 @@ static SherpaOnnxOfflineLMConfig GetOfflineLMConfig(Napi::Object obj) { return c; } +static SherpaOnnxOfflineCtcFstDecoderConfig GetOfflineCtcFstDecoderConfig( + Napi::Object obj) { + SherpaOnnxOfflineCtcFstDecoderConfig c; + memset(&c, 0, sizeof(c)); + + if (!obj.Has("ctcFstDecoderConfig") || + !obj.Get("ctcFstDecoderConfig").IsObject()) { + return c; + } + + Napi::Object o = obj.Get("ctcFstDecoderConfig").As(); + + SHERPA_ONNX_ASSIGN_ATTR_STR(graph, graph); + SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active, maxActive); + + return c; +} + static SherpaOnnxOfflineRecognizerConfig ParseConfig(Napi::Object o) { SherpaOnnxOfflineRecognizerConfig c; memset(&c, 0, sizeof(c)); @@ -421,6 +439,7 @@ static SherpaOnnxOfflineRecognizerConfig ParseConfig(Napi::Object o) { c.model_config = GetOfflineModelConfig(o); c.lm_config = GetOfflineLMConfig(o); c.hr = GetHomophoneReplacerConfig(o); + c.ctc_fst_decoder_config = GetOfflineCtcFstDecoderConfig(o); SHERPA_ONNX_ASSIGN_ATTR_STR(decoding_method, decodingMethod); SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active_paths, maxActivePaths); @@ -509,6 +528,7 @@ static void FreeConfig(const SherpaOnnxOfflineRecognizerConfig &c) { SHERPA_ONNX_DELETE_C_STR(c.rule_fars); SHERPA_ONNX_DELETE_C_STR(c.hr.lexicon); SHERPA_ONNX_DELETE_C_STR(c.hr.rule_fsts); + SHERPA_ONNX_DELETE_C_STR(c.ctc_fst_decoder_config.graph); } static Napi::External diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc index ab6b0d81b..7a5746f3f 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc @@ -38,6 +38,7 @@ SHERPA_ONNX_DELETE_C_STR(c.model.kitten.voices); \ SHERPA_ONNX_DELETE_C_STR(c.model.kitten.tokens); \ SHERPA_ONNX_DELETE_C_STR(c.model.kitten.data_dir); \ + SHERPA_ONNX_DELETE_C_STR(c.model.kitten.lexicon); \ \ SHERPA_ONNX_DELETE_C_STR(c.model.zipvoice.tokens); \ SHERPA_ONNX_DELETE_C_STR(c.model.zipvoice.encoder); \ @@ -75,13 +76,36 @@ SHERPA_ONNX_DELETE_C_STR(c.rule_fars); \ } while (0) -#define SHERPA_ONNX_DELETE_GENERATION_C_STR(c) \ - do { \ - SHERPA_ONNX_DELETE_C_STR(c.reference_text); \ - SHERPA_ONNX_DELETE_C_STR(c.extra); \ - if (c.reference_audio) { \ - delete[] c.reference_audio; \ - } \ +#define SHERPA_ONNX_DELETE_GENERATION_C_STR(c) \ + do { \ + SHERPA_ONNX_DELETE_C_STR(c.reference_text); \ + SHERPA_ONNX_DELETE_C_STR(c.extra); \ + if (c.reference_audio) { \ + delete[] c.reference_audio; \ + } \ + if (c.phoneme_codepoints) { \ + delete[] c.phoneme_codepoints; \ + } \ + if (c.phoneme_codepoints_lens) { \ + delete[] c.phoneme_codepoints_lens; \ + } \ + if (c.tokens) { \ + /* c.tokens is a FLAT array of sum(tokens_lens) strings, \ + not one per sentence, so free every token. */ \ + int32_t num_tokens = 0; \ + if (c.tokens_lens) { \ + for (int32_t i = 0; i != c.tokens_num_sentences; ++i) { \ + num_tokens += c.tokens_lens[i]; \ + } \ + } \ + for (int32_t i = 0; i != num_tokens; ++i) { \ + SHERPA_ONNX_DELETE_C_STR(c.tokens[i]); \ + } \ + delete[] c.tokens; \ + } \ + if (c.tokens_lens) { \ + delete[] c.tokens_lens; \ + } \ } while (0) static std::string JsObjectToJson(Napi::Env env, const Napi::Object &obj) { @@ -94,6 +118,70 @@ static SherpaOnnxGenerationConfig GetGenerationConfig(Napi::Object o) { SherpaOnnxGenerationConfig c; memset(&c, 0, sizeof(c)); + Napi::Env env = o.Env(); + + // Validate before marshalling anything: phonemeCodepoints must be an array + // of arrays of numbers and tokens an array of arrays of strings. Anything + // else is a caller bug, so throw a JS exception and bail out. + struct { + const char *name; + const char *element_type; + bool numeric; + } nested_arrays[] = { + {"phonemeCodepoints", "number", true}, + {"tokens", "string", false}, + }; + for (const auto &f : nested_arrays) { + if (!o.Has(f.name)) { + continue; + } + Napi::Value v = o.Get(f.name); + // An optional field that was declared but left unset arrives here as + // undefined/null. Treat that as "not provided" instead of a type error, + // matching how referenceAudio is handled below. + if (v.IsUndefined() || v.IsNull()) { + continue; + } + if (!v.IsArray()) { + Napi::TypeError::New(env, std::string(f.name) + " must be an array of arrays") + .ThrowAsJavaScriptException(); + return c; + } + Napi::Array outer = v.As(); + for (uint32_t i = 0; i < outer.Length(); ++i) { + Napi::Value e = outer.Get(i); + if (!e.IsArray()) { + Napi::TypeError::New( + env, + std::string(f.name) + "[" + std::to_string(i) + "] must be an array") + .ThrowAsJavaScriptException(); + return c; + } + Napi::Array inner = e.As(); + if (inner.Length() == 0) { + // The C API requires every sentence to have at least one entry. + Napi::TypeError::New( + env, + std::string(f.name) + "[" + std::to_string(i) + + "] must be a non-empty array") + .ThrowAsJavaScriptException(); + return c; + } + for (uint32_t j = 0; j < inner.Length(); ++j) { + Napi::Value item = inner.Get(j); + bool ok = f.numeric ? item.IsNumber() : item.IsString(); + if (!ok) { + Napi::TypeError::New( + env, + std::string(f.name) + "[" + std::to_string(i) + "][" + + std::to_string(j) + "] must be a " + f.element_type) + .ThrowAsJavaScriptException(); + return c; + } + } + } + } + SHERPA_ONNX_ASSIGN_ATTR_FLOAT(silence_scale, silenceScale); SHERPA_ONNX_ASSIGN_ATTR_FLOAT(speed, speed); SHERPA_ONNX_ASSIGN_ATTR_INT32(sid, sid); @@ -124,6 +212,66 @@ static SherpaOnnxGenerationConfig GetGenerationConfig(Napi::Object o) { c.extra = p; } + // phonemeCodepoints: JS number[][] -> one flat int32 array plus the + // per-sentence lengths, as expected by the C API. + if (o.Has("phonemeCodepoints") && o.Get("phonemeCodepoints").IsArray()) { + Napi::Array outer = o.Get("phonemeCodepoints").As(); + uint32_t n = outer.Length(); + + if (n > 0) { + std::vector flat; + int32_t *lens = new int32_t[n]; + + for (uint32_t i = 0; i < n; ++i) { + Napi::Array inner = outer.Get(i).As(); + uint32_t m = inner.Length(); + lens[i] = static_cast(m); + for (uint32_t j = 0; j < m; ++j) { + flat.push_back(inner.Get(j).ToNumber().Int32Value()); + } + } + + int32_t *buf = new int32_t[flat.empty() ? 1 : flat.size()]; + std::copy(flat.begin(), flat.end(), buf); + + c.phoneme_codepoints = buf; + c.phoneme_codepoints_lens = lens; + c.phoneme_codepoints_num_sentences = static_cast(n); + } + } + + // tokens: JS string[][] -> one flat char** array plus the per-sentence + // lengths, as expected by the C API. + if (o.Has("tokens") && o.Get("tokens").IsArray()) { + Napi::Array outer = o.Get("tokens").As(); + uint32_t n = outer.Length(); + + if (n > 0) { + std::vector flat; + int32_t *lens = new int32_t[n]; + + for (uint32_t i = 0; i < n; ++i) { + Napi::Array inner = outer.Get(i).As(); + uint32_t m = inner.Length(); + lens[i] = static_cast(m); + for (uint32_t j = 0; j < m; ++j) { + std::string s = inner.Get(j).ToString().Utf8Value(); + char *q = new char[s.size() + 1]; + std::copy(s.begin(), s.end(), q); + q[s.size()] = 0; + flat.push_back(q); + } + } + + const char **buf = new const char *[flat.empty() ? 1 : flat.size()]; + std::copy(flat.begin(), flat.end(), buf); + + c.tokens = buf; + c.tokens_lens = lens; + c.tokens_num_sentences = static_cast(n); + } + } + return c; } @@ -140,6 +288,9 @@ static SherpaOnnxOfflineTtsVitsModelConfig GetOfflineTtsVitsModelConfig( SHERPA_ONNX_ASSIGN_ATTR_STR(model, model); SHERPA_ONNX_ASSIGN_ATTR_STR(lexicon, lexicon); SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); + // data_dir is deprecated and ignored since v2.0.0 (espeak-ng is gone). Kept + // only so the C struct slot stays valid; the same applies to the matcha, + // kokoro, kitten and zipvoice model configs below. SHERPA_ONNX_ASSIGN_ATTR_STR(data_dir, dataDir); SHERPA_ONNX_ASSIGN_ATTR_FLOAT(noise_scale, noiseScale); SHERPA_ONNX_ASSIGN_ATTR_FLOAT(noise_scale_w, noiseScaleW); @@ -205,6 +356,7 @@ static SherpaOnnxOfflineTtsKittenModelConfig GetOfflineTtsKittenModelConfig( SHERPA_ONNX_ASSIGN_ATTR_STR(tokens, tokens); SHERPA_ONNX_ASSIGN_ATTR_STR(data_dir, dataDir); SHERPA_ONNX_ASSIGN_ATTR_FLOAT(length_scale, lengthScale); + SHERPA_ONNX_ASSIGN_ATTR_STR(lexicon, lexicon); return c; } @@ -526,6 +678,35 @@ static Napi::Number OfflineTtsNumSpeakersWrapper( return Napi::Number::New(env, num_speakers); } +static Napi::String OfflineTtsLangWrapper(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + std::ostringstream os; + os << "Expect only 1 argument. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsExternal()) { + Napi::TypeError::New(env, "Argument 0 should be an offline tts pointer.") + .ThrowAsJavaScriptException(); + + return {}; + } + + const SherpaOnnxOfflineTts *tts = + info[0].As>().Data(); + + const char *s = SherpaOnnxOfflineTtsLang(tts); + std::string lang = s; + SherpaOnnxDestroyOfflineTtsLang(s); + + return Napi::String::New(env, lang); +} + // synchronous version static Napi::Object OfflineTtsGenerateWithConfigWrapper( const Napi::CallbackInfo &info) { @@ -575,6 +756,11 @@ static Napi::Object OfflineTtsGenerateWithConfigWrapper( : Napi::Object::New(env); SherpaOnnxGenerationConfig gen_config = GetGenerationConfig(genObj); + if (env.IsExceptionPending()) { + // GetGenerationConfig throws before allocating anything, so there is + // nothing to free here. + return {}; + } const SherpaOnnxGeneratedAudio *audio = SherpaOnnxOfflineTtsGenerateWithConfig(tts, text.c_str(), &gen_config, @@ -583,8 +769,13 @@ static Napi::Object OfflineTtsGenerateWithConfigWrapper( SHERPA_ONNX_DELETE_GENERATION_C_STR(gen_config); if (!audio) { - Napi::Error::New(env, "TTS generation failed").ThrowAsJavaScriptException(); - return {}; + // Generation failed. The C API already logged the reason (for instance + // phonemeCodepoints and tokens are mutually exclusive). Return empty + // audio instead of throwing, matching C++/C/Python/Rust/Go/Java. + Napi::Object result = Napi::Object::New(env); + result.Set("samples", Napi::Float32Array::New(env, 0)); + result.Set("sampleRate", 0); + return result; } Napi::Object result = Napi::Object::New(env); @@ -1304,18 +1495,22 @@ static Napi::Object OfflineTtsGenerateAsyncWithConfigWrapper( cb = obj.Get("callback").As(); } - auto context = new TtsTsfnContext(env, info.This()); - TSFN tsfn = TSFN::New( - env, cb, "TtsGenerateWithConfig", kMaxPendingTtsChunks, 1, context, - [](Napi::Env, void *, TtsTsfnContext *ctx) { delete ctx; }); - + // Parse and validate the generation config before creating any resources. SherpaOnnxGenerationConfig gen_config; memset(&gen_config, 0, sizeof(gen_config)); if (obj.Has("generationConfig") && obj.Get("generationConfig").IsObject()) { gen_config = GetGenerationConfig(obj.Get("generationConfig").As()); + if (env.IsExceptionPending()) { + return {}; + } } + auto context = new TtsTsfnContext(env, info.This()); + TSFN tsfn = TSFN::New( + env, cb, "TtsGenerateWithConfig", kMaxPendingTtsChunks, 1, context, + [](Napi::Env, void *, TtsTsfnContext *ctx) { delete ctx; }); + TtsGenerateWithConfigWorker *worker = new TtsGenerateWithConfigWorker( env, tsfn, context->state, tts, text, gen_config, enable_external_buffer); @@ -1336,6 +1531,9 @@ void InitNonStreamingTts(Napi::Env env, Napi::Object exports) { exports.Set(Napi::String::New(env, "getOfflineTtsNumSpeakers"), Napi::Function::New(env, OfflineTtsNumSpeakersWrapper)); + exports.Set(Napi::String::New(env, "getOfflineTtsLang"), + Napi::Function::New(env, OfflineTtsLangWrapper)); + exports.Set(Napi::String::New(env, "offlineTtsGenerate"), Napi::Function::New(env, OfflineTtsGenerateWrapper)); diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts index 8f3971c34..f8eaa5eed 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts @@ -40,6 +40,7 @@ export const getOnlineStreamResultAsJson: (handle: object, streamHandle: object) export const createOfflineTts: (config: object, mgr?: object) => object; export const getOfflineTtsNumSpeakers: (handle: object) => number; export const getOfflineTtsSampleRate: (handle: object) => number; +export const getOfflineTtsLang: (handle: object) => string; export type TtsOutput = { samples: Float32Array; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets index 084ec4393..2782486bb 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingAsr.ets @@ -179,18 +179,24 @@ export class OfflineLMConfig { public scale: number = 1.0; } +export class OfflineCtcFstDecoderConfig { + public graph: string = ''; + public maxActive: number = 3000; +} + export class OfflineRecognizerConfig { public featConfig: FeatureConfig = new FeatureConfig(); public modelConfig: OfflineModelConfig = new OfflineModelConfig(); public lmConfig: OfflineLMConfig = new OfflineLMConfig(); public decodingMethod: string = "greedy_search"; public maxActivePaths: number = 4; - public hotwordsFfile: string = ''; + public hotwordsFile: string = ''; public hotwordsScore: number = 1.5; public ruleFsts: string = ''; public ruleFars: string = ''; public blankPenalty: number = 0; public hr: HomophoneReplacerConfig = new HomophoneReplacerConfig(); + public ctcFstDecoderConfig: OfflineCtcFstDecoderConfig = new OfflineCtcFstDecoderConfig(); } export class OfflineRecognizerResult { diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets index 47ced7940..7c98cd18b 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets @@ -1,5 +1,6 @@ import { createOfflineTts, + getOfflineTtsLang, getOfflineTtsNumSpeakers, getOfflineTtsSampleRate, offlineTtsGenerate, @@ -12,6 +13,9 @@ export class OfflineTtsVitsModelConfig { public model: string = ''; public lexicon: string = ''; public tokens: string = ''; + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon, or GenerationConfig.phonemeCodepoints / tokens with an + // external phonemizer instead. The field is kept only for ABI stability. public dataDir: string = ''; public dictDir: String = ''; // unused public noiseScale: number = 0.667; @@ -24,6 +28,9 @@ export class OfflineTtsMatchaModelConfig { public vocoder: string = ''; public lexicon: string = ''; public tokens: string = ''; + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon, or GenerationConfig.phonemeCodepoints / tokens with an + // external phonemizer instead. The field is kept only for ABI stability. public dataDir: string = ''; public dictDir: String = ''; // unused public noiseScale: number = 0.667; @@ -34,6 +41,9 @@ export class OfflineTtsKokoroModelConfig { public model: string = ''; public voices: string = ''; public tokens: string = ''; + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon, or GenerationConfig.phonemeCodepoints / tokens with an + // external phonemizer instead. The field is kept only for ABI stability. public dataDir: string = ''; public lengthScale: number = 1.0; public dictDir: string = ''; // unused @@ -45,6 +55,9 @@ export class OfflineTtsKittenModelConfig { public model: string = ''; public voices: string = ''; public tokens: string = ''; + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon, or GenerationConfig.phonemeCodepoints / tokens with an + // external phonemizer instead. The field is kept only for ABI stability. public dataDir: string = ''; public lengthScale: number = 1.0; } @@ -54,6 +67,9 @@ export class OfflineTtsZipvoiceModelConfig { public encoder: string = ''; public decoder: string = ''; public vocoder: string = ''; + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon, or GenerationConfig.phonemeCodepoints / tokens with an + // external phonemizer instead. The field is kept only for ABI stability. public dataDir: string = ''; public lexicon: string = ''; public featScale: number = 0.1; @@ -123,6 +139,20 @@ export class TtsGenerationConfig { public referenceText: string = ''; public numSteps: number = 5; public extra: object = new Object(); + + /** + * Pre-phonemized input as Unicode codepoints. Each inner array is one + * sentence. When set, it is used in place of the text argument; leave + * unset to use text with a lexicon. Mutually exclusive with `tokens`: + * if both are set, a warning is printed and generation fails. + */ + public phonemeCodepoints?: Array>; + + /** + * Pre-tokenized input, e.g. Chinese pinyin like "zhong1". Each inner + * array is one sentence. Mutually exclusive with `phonemeCodepoints`. + */ + public tokens?: Array>; } export class TtsInput { @@ -144,6 +174,8 @@ export class OfflineTts { public config: OfflineTtsConfig; public numSpeakers: number; public sampleRate: number; + // Language/voice of the model, e.g., "en-us". Empty if not available. + public lang: string; private handle: object; constructor(config: OfflineTtsConfig, mgr?: object) { @@ -152,6 +184,7 @@ export class OfflineTts { this.numSpeakers = getOfflineTtsNumSpeakers(this.handle); this.sampleRate = getOfflineTtsSampleRate(this.handle); + this.lang = getOfflineTtsLang(this.handle); } /* diff --git a/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets b/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets index dafac71ec..4733deab1 100644 --- a/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets +++ b/harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets @@ -95,7 +95,6 @@ function initTts(context: Context): OfflineTts { let ruleFsts = ''; let ruleFars = ''; let lexicon = ''; - let dataDir = ''; /* You can select an example below and change it according to match your selected tts model @@ -115,7 +114,10 @@ function initTts(context: Context): OfflineTts { // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 // modelDir = 'vits-piper-en_US-amy-low'; // modelName = 'en_US-amy-low.onnx'; - // dataDir = 'espeak-ng-data'; + // lexicon = 'lexicon-en-us.txt'; + // The lexicon is not inside the model package; download it from + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + // and put it next to the model. espeak-ng is no longer used. // Example 3: // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 @@ -141,7 +143,6 @@ function initTts(context: Context): OfflineTts { // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-libritts_r-medium.tar.bz2 // modelDir = 'vits-piper-en_US-libritts_r-medium'; // modelName = 'en_US-libritts_r-medium.onnx'; - // dataDir = 'espeak-ng-data'; // Example 7 // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models @@ -166,21 +167,18 @@ function initTts(context: Context): OfflineTts { // modelDir = 'matcha-icefall-en_US-ljspeech'; // acousticModelName = 'model-steps-3.onnx'; // vocoder = 'vocos-22khz-univ.onnx'; - // dataDir = 'espeak-ng-data'; // Example 10 // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html#kokoro-en-v0-19-english-11-speakers // modelDir = 'kokoro-en-v0_19'; // modelName = 'model.onnx'; // voices = 'voices.bin' - // dataDir = 'espeak-ng-data'; // Example 11 // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html // modelDir = 'kokoro-multi-lang-v1_0'; // modelName = 'model.onnx'; // voices = 'voices.bin' - // dataDir = 'espeak-ng-data'; // lexicon = 'lexicon-us-en.txt,lexicon-zh.txt'; // ruleFsts = `date-zh.fst,phone-zh.fst,number-zh.fst`; @@ -241,12 +239,6 @@ function initTts(context: Context): OfflineTts { lexicon = modelDir + '/' + lexicon; } - if (dataDir != '') { - copyRawFileDirToSandbox(context, modelDir + '/' + dataDir) - let sandboxPath: string = context.getApplicationContext().filesDir; - dataDir = sandboxPath + '/' + modelDir + '/' + dataDir; - } - const tokens = modelDir + '/tokens.txt'; const config: OfflineTtsConfig = new OfflineTtsConfig(); @@ -259,13 +251,11 @@ function initTts(context: Context): OfflineTts { if (voices == '') { config.model.vits.lexicon = lexicon; config.model.vits.tokens = tokens; - config.model.vits.dataDir = dataDir; config.model.matcha.acousticModel = acousticModelName; config.model.matcha.vocoder = vocoder; config.model.matcha.lexicon = lexicon; config.model.matcha.tokens = tokens; - config.model.matcha.dataDir = dataDir; } if (voices != '') { @@ -277,7 +267,6 @@ function initTts(context: Context): OfflineTts { if (voices != '') { config.model.kokoro.voices = voices; config.model.kokoro.tokens = tokens; - config.model.kokoro.dataDir = dataDir; config.model.kokoro.lexicon = lexicon; } diff --git a/ios-swiftui/README.md b/ios-swiftui/README.md index 695081b5c..2eff37948 100644 --- a/ios-swiftui/README.md +++ b/ios-swiftui/README.md @@ -160,7 +160,8 @@ This is a known Xcode issue with SPM dynamic (shared) frameworks — Xcode somet 1. Close Xcode 2. Delete DerivedData: ```bash - rm -rf ~/Library/Developer/Xcode/DerivedData/SherpaOnnxTts-* + rm -rf ~/Library/Developer/Xcode/DerivedData/SherpaOnnxTtsLexicon-* + rm -rf ~/Library/Developer/Xcode/DerivedData/SherpaOnnxTtsPhonemize-* ``` 3. Open Xcode 4. `File → Packages → Reset Package Caches` diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts.xcodeproj/project.pbxproj b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon.xcodeproj/project.pbxproj similarity index 88% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts.xcodeproj/project.pbxproj rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon.xcodeproj/project.pbxproj index bc8c1134e..1c75f473e 100644 --- a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts.xcodeproj/project.pbxproj +++ b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon.xcodeproj/project.pbxproj @@ -7,7 +7,7 @@ objects = { /* Begin PBXBuildFile section */ - C917B4E52B0EEF3B005245AC /* SherpaOnnxTtsApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = C917B4E42B0EEF3B005245AC /* SherpaOnnxTtsApp.swift */; }; + C917B4E52B0EEF3B005245AC /* SherpaOnnxTtsLexiconApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = C917B4E42B0EEF3B005245AC /* SherpaOnnxTtsLexiconApp.swift */; }; C917B4E72B0EEF3B005245AC /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C917B4E62B0EEF3B005245AC /* ContentView.swift */; }; C917B4E92B0EEF3C005245AC /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C917B4E82B0EEF3C005245AC /* Assets.xcassets */; }; C917B4EC2B0EEF3C005245AC /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C917B4EB2B0EEF3C005245AC /* Preview Assets.xcassets */; }; @@ -16,8 +16,8 @@ /* End PBXBuildFile section */ /* Begin PBXFileReference section */ - C917B4E12B0EEF3B005245AC /* SherpaOnnxTts.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SherpaOnnxTts.app; sourceTree = BUILT_PRODUCTS_DIR; }; - C917B4E42B0EEF3B005245AC /* SherpaOnnxTtsApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SherpaOnnxTtsApp.swift; sourceTree = ""; }; + C917B4E12B0EEF3B005245AC /* SherpaOnnxTtsLexicon.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SherpaOnnxTtsLexicon.app; sourceTree = BUILT_PRODUCTS_DIR; }; + C917B4E42B0EEF3B005245AC /* SherpaOnnxTtsLexiconApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SherpaOnnxTtsLexiconApp.swift; sourceTree = ""; }; C917B4E62B0EEF3B005245AC /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; C917B4E82B0EEF3C005245AC /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; C917B4EB2B0EEF3C005245AC /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; @@ -39,7 +39,7 @@ C917B4D82B0EEF3B005245AC = { isa = PBXGroup; children = ( - C917B4E32B0EEF3B005245AC /* SherpaOnnxTts */, + C917B4E32B0EEF3B005245AC /* SherpaOnnxTtsLexicon */, C917B4E22B0EEF3B005245AC /* Products */, ); sourceTree = ""; @@ -47,21 +47,21 @@ C917B4E22B0EEF3B005245AC /* Products */ = { isa = PBXGroup; children = ( - C917B4E12B0EEF3B005245AC /* SherpaOnnxTts.app */, + C917B4E12B0EEF3B005245AC /* SherpaOnnxTtsLexicon.app */, ); name = Products; sourceTree = ""; }; - C917B4E32B0EEF3B005245AC /* SherpaOnnxTts */ = { + C917B4E32B0EEF3B005245AC /* SherpaOnnxTtsLexicon */ = { isa = PBXGroup; children = ( C9FE9FE42B0F33CD009F1003 /* ViewModel.swift */, - C917B4E42B0EEF3B005245AC /* SherpaOnnxTtsApp.swift */, + C917B4E42B0EEF3B005245AC /* SherpaOnnxTtsLexiconApp.swift */, C917B4E62B0EEF3B005245AC /* ContentView.swift */, C917B4E82B0EEF3C005245AC /* Assets.xcassets */, C917B4EA2B0EEF3C005245AC /* Preview Content */, ); - path = SherpaOnnxTts; + path = SherpaOnnxTtsLexicon; sourceTree = ""; }; C917B4EA2B0EEF3C005245AC /* Preview Content */ = { @@ -75,9 +75,9 @@ /* End PBXGroup section */ /* Begin PBXNativeTarget section */ - C917B4E02B0EEF3B005245AC /* SherpaOnnxTts */ = { + C917B4E02B0EEF3B005245AC /* SherpaOnnxTtsLexicon */ = { isa = PBXNativeTarget; - buildConfigurationList = C917B4EF2B0EEF3C005245AC /* Build configuration list for PBXNativeTarget "SherpaOnnxTts" */; + buildConfigurationList = C917B4EF2B0EEF3C005245AC /* Build configuration list for PBXNativeTarget "SherpaOnnxTtsLexicon" */; buildPhases = ( C917B4DD2B0EEF3B005245AC /* Sources */, C917B4DE2B0EEF3B005245AC /* Frameworks */, @@ -87,12 +87,12 @@ ); dependencies = ( ); - name = SherpaOnnxTts; + name = SherpaOnnxTtsLexicon; packageProductDependencies = ( C9A4C698302B3BD60004DECB /* sherpa-onnx */, ); - productName = SherpaOnnxTts; - productReference = C917B4E12B0EEF3B005245AC /* SherpaOnnxTts.app */; + productName = SherpaOnnxTtsLexicon; + productReference = C917B4E12B0EEF3B005245AC /* SherpaOnnxTtsLexicon.app */; productType = "com.apple.product-type.application"; }; /* End PBXNativeTarget section */ @@ -110,7 +110,7 @@ }; }; }; - buildConfigurationList = C917B4DC2B0EEF3B005245AC /* Build configuration list for PBXProject "SherpaOnnxTts" */; + buildConfigurationList = C917B4DC2B0EEF3B005245AC /* Build configuration list for PBXProject "SherpaOnnxTtsLexicon" */; compatibilityVersion = "Xcode 14.0"; developmentRegion = en; hasScannedForEncodings = 0; @@ -126,7 +126,7 @@ projectDirPath = ""; projectRoot = ""; targets = ( - C917B4E02B0EEF3B005245AC /* SherpaOnnxTts */, + C917B4E02B0EEF3B005245AC /* SherpaOnnxTtsLexicon */, ); }; /* End PBXProject section */ @@ -150,7 +150,7 @@ files = ( C917B4E72B0EEF3B005245AC /* ContentView.swift in Sources */, C9FE9FE52B0F33CD009F1003 /* ViewModel.swift in Sources */, - C917B4E52B0EEF3B005245AC /* SherpaOnnxTtsApp.swift in Sources */, + C917B4E52B0EEF3B005245AC /* SherpaOnnxTtsLexiconApp.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -278,7 +278,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_ASSET_PATHS = "\"SherpaOnnxTts/Preview Content\""; + DEVELOPMENT_ASSET_PATHS = "\"SherpaOnnxTtsLexicon/Preview Content\""; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; @@ -291,7 +291,7 @@ "@executable_path/Frameworks", ); MARKETING_VERSION = 1.0; - PRODUCT_BUNDLE_IDENTIFIER = "com.k2-fsa.org.SherpaOnnxTts"; + PRODUCT_BUNDLE_IDENTIFIER = "com.k2-fsa.org.SherpaOnnxTtsLexicon"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; @@ -306,7 +306,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_ASSET_PATHS = "\"SherpaOnnxTts/Preview Content\""; + DEVELOPMENT_ASSET_PATHS = "\"SherpaOnnxTtsLexicon/Preview Content\""; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; @@ -319,7 +319,7 @@ "@executable_path/Frameworks", ); MARKETING_VERSION = 1.0; - PRODUCT_BUNDLE_IDENTIFIER = "com.k2-fsa.org.SherpaOnnxTts"; + PRODUCT_BUNDLE_IDENTIFIER = "com.k2-fsa.org.SherpaOnnxTtsLexicon"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_VERSION = 5.0; @@ -330,7 +330,7 @@ /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ - C917B4DC2B0EEF3B005245AC /* Build configuration list for PBXProject "SherpaOnnxTts" */ = { + C917B4DC2B0EEF3B005245AC /* Build configuration list for PBXProject "SherpaOnnxTtsLexicon" */ = { isa = XCConfigurationList; buildConfigurations = ( C917B4ED2B0EEF3C005245AC /* Debug */, @@ -339,7 +339,7 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; - C917B4EF2B0EEF3C005245AC /* Build configuration list for PBXNativeTarget "SherpaOnnxTts" */ = { + C917B4EF2B0EEF3C005245AC /* Build configuration list for PBXNativeTarget "SherpaOnnxTtsLexicon" */ = { isa = XCConfigurationList; buildConfigurations = ( C917B4F02B0EEF3C005245AC /* Debug */, diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon.xcodeproj/project.xcworkspace/contents.xcworkspacedata similarity index 100% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts.xcodeproj/project.xcworkspace/contents.xcworkspacedata rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon.xcodeproj/project.xcworkspace/contents.xcworkspacedata diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist similarity index 100% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Assets.xcassets/AccentColor.colorset/Contents.json b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Assets.xcassets/AccentColor.colorset/Contents.json similarity index 100% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Assets.xcassets/AccentColor.colorset/Contents.json rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Assets.xcassets/AccentColor.colorset/Contents.json diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Assets.xcassets/AppIcon.appiconset/Contents.json b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Assets.xcassets/AppIcon.appiconset/Contents.json similarity index 100% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Assets.xcassets/AppIcon.appiconset/Contents.json rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Assets.xcassets/AppIcon.appiconset/Contents.json diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Assets.xcassets/Contents.json b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Assets.xcassets/Contents.json similarity index 100% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Assets.xcassets/Contents.json rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Assets.xcassets/Contents.json diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ContentView.swift b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/ContentView.swift similarity index 99% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ContentView.swift rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/ContentView.swift index 601ced307..849db7a9c 100644 --- a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ContentView.swift +++ b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/ContentView.swift @@ -1,6 +1,6 @@ // // ContentView.swift -// SherpaOnnxTts +// SherpaOnnxTtsLexicon // // Created by fangjun on 2023/11/23. // diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Info.plist b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Info.plist similarity index 100% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Info.plist rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Info.plist diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Preview Content/Preview Assets.xcassets/Contents.json b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Preview Content/Preview Assets.xcassets/Contents.json similarity index 100% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/Preview Content/Preview Assets.xcassets/Contents.json rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/Preview Content/Preview Assets.xcassets/Contents.json diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/SherpaOnnxTtsApp.swift b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexiconApp.swift similarity index 62% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/SherpaOnnxTtsApp.swift rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexiconApp.swift index bd0be8dab..7d227e472 100644 --- a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/SherpaOnnxTtsApp.swift +++ b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexiconApp.swift @@ -1,6 +1,6 @@ // -// SherpaOnnxTtsApp.swift -// SherpaOnnxTts +// SherpaOnnxTtsLexiconApp.swift +// SherpaOnnxTtsLexicon // // Created by fangjun on 2023/11/23. // @@ -8,7 +8,7 @@ import SwiftUI @main -struct SherpaOnnxTtsApp: App { +struct SherpaOnnxTtsLexiconApp: App { var body: some Scene { WindowGroup { ContentView() diff --git a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/ViewModel.swift similarity index 91% rename from ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift rename to ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/ViewModel.swift index f0bacdc1e..30fcd0756 100644 --- a/ios-swiftui/SherpaOnnxTts/SherpaOnnxTts/ViewModel.swift +++ b/ios-swiftui/SherpaOnnxTtsLexicon/SherpaOnnxTtsLexicon/ViewModel.swift @@ -1,6 +1,6 @@ // // ViewModel.swift -// SherpaOnnxTts +// SherpaOnnxTtsLexicon // // Created by fangjun on 2023/11/23. // @@ -14,11 +14,6 @@ import SherpaOnnxShared #endif import Foundation -// used to get the path to espeak-ng-data -func resourceURL(to path: String) -> String { - return URL(string: path, relativeTo: Bundle.main.resourceURL)!.path -} - func getResource(_ forResource: String, _ ofType: String) -> String { let path = Bundle.main.path(forResource: forResource, ofType: ofType) precondition( @@ -89,11 +84,13 @@ func getTtsFor_en_US_amy_low() -> SherpaOnnxOfflineTtsWrapper { // tokens.txt let tokens = getResource("tokens", "txt") - // in this case, we don't need lexicon.txt - let dataDir = resourceURL(to: "espeak-ng-data") + // lexicon-en-us.txt; the model package has no lexicon, please download + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + // and add it to Build Phases -> Copy Bundle Resources + let lexicon = getResource("lexicon-en-us", "txt") let vits = sherpaOnnxOfflineTtsVitsModelConfig( - model: model, lexicon: "", tokens: tokens, dataDir: dataDir) + model: model, lexicon: lexicon, tokens: tokens) let modelConfig = sherpaOnnxOfflineTtsModelConfig(vits: vits) var config = sherpaOnnxOfflineTtsConfig(model: modelConfig) @@ -116,7 +113,6 @@ func getTtsFor_zh_en_melo_tts() -> SherpaOnnxOfflineTtsWrapper { let vits = sherpaOnnxOfflineTtsVitsModelConfig( model: model, lexicon: lexicon, tokens: tokens, - dataDir: "", noiseScale: 0.667, noiseScaleW: 0.8, lengthScale: 1.0 @@ -170,11 +166,13 @@ func getTtsFor_kokoro_en_v0_19() -> SherpaOnnxOfflineTtsWrapper { // tokens.txt let tokens = getResource("tokens", "txt") - // in this case, we don't need lexicon.txt - let dataDir = resourceURL(to: "espeak-ng-data") + // lexicon-en-us.txt; the model package has no lexicon, please download + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + // and add it to Build Phases -> Copy Bundle Resources + let lexicon = getResource("lexicon-en-us", "txt") let kokoro = sherpaOnnxOfflineTtsKokoroModelConfig( - model: model, voices: voices, tokens: tokens, dataDir: dataDir) + model: model, voices: voices, tokens: tokens, lexicon: lexicon) let modelConfig = sherpaOnnxOfflineTtsModelConfig(kokoro: kokoro) var config = sherpaOnnxOfflineTtsConfig(model: modelConfig) @@ -194,8 +192,6 @@ func getTtsFor_kokoro_multi_lang_v1_0() -> SherpaOnnxOfflineTtsWrapper { let lexicon_zh = getResource("lexicon-zh", "txt") let lexicon = "\(lexicon_en),\(lexicon_zh)" - // in this case, we don't need lexicon.txt - let dataDir = resourceURL(to: "espeak-ng-data") let numFst = getResource("number-zh", "fst") let dateFst = getResource("date-zh", "fst") @@ -203,8 +199,7 @@ func getTtsFor_kokoro_multi_lang_v1_0() -> SherpaOnnxOfflineTtsWrapper { let ruleFsts = "\(dateFst),\(phoneFst),\(numFst)" let kokoro = sherpaOnnxOfflineTtsKokoroModelConfig( - model: model, voices: voices, tokens: tokens, dataDir: dataDir, - lexicon: lexicon) + model: model, voices: voices, tokens: tokens, lexicon: lexicon) let modelConfig = sherpaOnnxOfflineTtsModelConfig(kokoro: kokoro) var config = sherpaOnnxOfflineTtsConfig(model: modelConfig, ruleFsts: ruleFsts) diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.pbxproj b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.pbxproj new file mode 100644 index 000000000..f51e526c9 --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.pbxproj @@ -0,0 +1,390 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 56; + objects = { + +/* Begin PBXBuildFile section */ + C917B4E52B0EEF3B005245AC /* SherpaOnnxTtsPhonemizeApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = C917B4E42B0EEF3B005245AC /* SherpaOnnxTtsPhonemizeApp.swift */; }; + C917B4E72B0EEF3B005245AC /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C917B4E62B0EEF3B005245AC /* ContentView.swift */; }; + C917B4E92B0EEF3C005245AC /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C917B4E82B0EEF3C005245AC /* Assets.xcassets */; }; + C917B4EC2B0EEF3C005245AC /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = C917B4EB2B0EEF3C005245AC /* Preview Assets.xcassets */; }; + C9A4C699302B3BD60004DECB /* sherpa-onnx in Frameworks */ = {isa = PBXBuildFile; productRef = C9A4C698302B3BD60004DECB /* sherpa-onnx */; }; + C9A4C6A2302B3BD60004DECB /* piper-phonemize in Frameworks */ = {isa = PBXBuildFile; productRef = C9A4C6A1302B3BD60004DECB /* piper-phonemize */; }; + C9FE9FE52B0F33CD009F1003 /* ViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = C9FE9FE42B0F33CD009F1003 /* ViewModel.swift */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + C917B4E12B0EEF3B005245AC /* SherpaOnnxTtsPhonemize.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SherpaOnnxTtsPhonemize.app; sourceTree = BUILT_PRODUCTS_DIR; }; + C917B4E42B0EEF3B005245AC /* SherpaOnnxTtsPhonemizeApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SherpaOnnxTtsPhonemizeApp.swift; sourceTree = ""; }; + C917B4E62B0EEF3B005245AC /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; + C917B4E82B0EEF3C005245AC /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + C917B4EB2B0EEF3C005245AC /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = ""; }; + C9FE9FE42B0F33CD009F1003 /* ViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewModel.swift; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + C917B4DE2B0EEF3B005245AC /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + C9A4C699302B3BD60004DECB /* sherpa-onnx in Frameworks */, + C9A4C6A2302B3BD60004DECB /* piper-phonemize in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + C917B4D82B0EEF3B005245AC = { + isa = PBXGroup; + children = ( + C917B4E32B0EEF3B005245AC /* SherpaOnnxTtsPhonemize */, + C917B4E22B0EEF3B005245AC /* Products */, + ); + sourceTree = ""; + }; + C917B4E22B0EEF3B005245AC /* Products */ = { + isa = PBXGroup; + children = ( + C917B4E12B0EEF3B005245AC /* SherpaOnnxTtsPhonemize.app */, + ); + name = Products; + sourceTree = ""; + }; + C917B4E32B0EEF3B005245AC /* SherpaOnnxTtsPhonemize */ = { + isa = PBXGroup; + children = ( + C9FE9FE42B0F33CD009F1003 /* ViewModel.swift */, + C917B4E42B0EEF3B005245AC /* SherpaOnnxTtsPhonemizeApp.swift */, + C917B4E62B0EEF3B005245AC /* ContentView.swift */, + C917B4E82B0EEF3C005245AC /* Assets.xcassets */, + C917B4EA2B0EEF3C005245AC /* Preview Content */, + ); + path = SherpaOnnxTtsPhonemize; + sourceTree = ""; + }; + C917B4EA2B0EEF3C005245AC /* Preview Content */ = { + isa = PBXGroup; + children = ( + C917B4EB2B0EEF3C005245AC /* Preview Assets.xcassets */, + ); + path = "Preview Content"; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + C917B4E02B0EEF3B005245AC /* SherpaOnnxTtsPhonemize */ = { + isa = PBXNativeTarget; + buildConfigurationList = C917B4EF2B0EEF3C005245AC /* Build configuration list for PBXNativeTarget "SherpaOnnxTtsPhonemize" */; + buildPhases = ( + C917B4DD2B0EEF3B005245AC /* Sources */, + C917B4DE2B0EEF3B005245AC /* Frameworks */, + C917B4DF2B0EEF3B005245AC /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = SherpaOnnxTtsPhonemize; + packageProductDependencies = ( + C9A4C698302B3BD60004DECB /* sherpa-onnx */, + C9A4C6A1302B3BD60004DECB /* piper-phonemize */, + ); + productName = SherpaOnnxTtsPhonemize; + productReference = C917B4E12B0EEF3B005245AC /* SherpaOnnxTtsPhonemize.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + C917B4D92B0EEF3B005245AC /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastSwiftUpdateCheck = 1420; + LastUpgradeCheck = 1420; + TargetAttributes = { + C917B4E02B0EEF3B005245AC = { + CreatedOnToolsVersion = 14.2; + }; + }; + }; + buildConfigurationList = C917B4DC2B0EEF3B005245AC /* Build configuration list for PBXProject "SherpaOnnxTtsPhonemize" */; + compatibilityVersion = "Xcode 14.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = C917B4D82B0EEF3B005245AC; + packageReferences = ( + C9A4C697302B3BD60004DECB /* XCRemoteSwiftPackageReference "sherpa-onnx" */, + C9A4C6A0302B3BD60004DECB /* XCRemoteSwiftPackageReference "piper-phonemize" */, + ); + productRefGroup = C917B4E22B0EEF3B005245AC /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + C917B4E02B0EEF3B005245AC /* SherpaOnnxTtsPhonemize */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + C917B4DF2B0EEF3B005245AC /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + C917B4EC2B0EEF3C005245AC /* Preview Assets.xcassets in Resources */, + C917B4E92B0EEF3C005245AC /* Assets.xcassets in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + C917B4DD2B0EEF3B005245AC /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + C917B4E72B0EEF3B005245AC /* ContentView.swift in Sources */, + C9FE9FE52B0F33CD009F1003 /* ViewModel.swift in Sources */, + C917B4E52B0EEF3B005245AC /* SherpaOnnxTtsPhonemizeApp.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + C917B4ED2B0EEF3C005245AC /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 16.2; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + C917B4EE2B0EEF3C005245AC /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 16.2; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + C917B4F02B0EEF3C005245AC /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_ASSET_PATHS = "\"SherpaOnnxTtsPhonemize/Preview Content\""; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "com.k2-fsa.org.SherpaOnnxTtsPhonemize"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + C917B4F12B0EEF3C005245AC /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_ASSET_PATHS = "\"SherpaOnnxTtsPhonemize/Preview Content\""; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "com.k2-fsa.org.SherpaOnnxTtsPhonemize"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + C917B4DC2B0EEF3B005245AC /* Build configuration list for PBXProject "SherpaOnnxTtsPhonemize" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + C917B4ED2B0EEF3C005245AC /* Debug */, + C917B4EE2B0EEF3C005245AC /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + C917B4EF2B0EEF3C005245AC /* Build configuration list for PBXNativeTarget "SherpaOnnxTtsPhonemize" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + C917B4F02B0EEF3C005245AC /* Debug */, + C917B4F12B0EEF3C005245AC /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + +/* Begin XCRemoteSwiftPackageReference section */ + C9A4C697302B3BD60004DECB /* XCRemoteSwiftPackageReference "sherpa-onnx" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/k2-fsa/sherpa-onnx"; + requirement = { + kind = exactVersion; + version = 1.13.8; + }; + }; + C9A4C6A0302B3BD60004DECB /* XCRemoteSwiftPackageReference "piper-phonemize" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/csukuangfj/piper-phonemize"; + requirement = { + kind = exactVersion; + version = 1.5.8; + }; + }; +/* End XCRemoteSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + C9A4C698302B3BD60004DECB /* sherpa-onnx */ = { + isa = XCSwiftPackageProductDependency; + package = C9A4C697302B3BD60004DECB /* XCRemoteSwiftPackageReference "sherpa-onnx" */; + productName = "sherpa-onnx"; + }; + C9A4C6A1302B3BD60004DECB /* piper-phonemize */ = { + isa = XCSwiftPackageProductDependency; + package = C9A4C6A0302B3BD60004DECB /* XCRemoteSwiftPackageReference "piper-phonemize" */; + productName = "piper-phonemize"; + }; +/* End XCSwiftPackageProductDependency section */ + }; + rootObject = C917B4D92B0EEF3B005245AC /* Project object */; +} diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 000000000..919434a62 --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 000000000..18d981003 --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Assets.xcassets/AccentColor.colorset/Contents.json b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Assets.xcassets/AccentColor.colorset/Contents.json new file mode 100644 index 000000000..eb8789700 --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Assets.xcassets/AccentColor.colorset/Contents.json @@ -0,0 +1,11 @@ +{ + "colors" : [ + { + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Assets.xcassets/AppIcon.appiconset/Contents.json b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 000000000..13613e3ee --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,13 @@ +{ + "images" : [ + { + "idiom" : "universal", + "platform" : "ios", + "size" : "1024x1024" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Assets.xcassets/Contents.json b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Assets.xcassets/Contents.json new file mode 100644 index 000000000..73c00596a --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/ContentView.swift b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/ContentView.swift new file mode 100644 index 000000000..783db3362 --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/ContentView.swift @@ -0,0 +1,454 @@ +// +// ContentView.swift +// SherpaOnnxTtsPhonemize +// +// Created by fangjun on 2023/11/23. +// +// Text-to-speech with Next-gen Kaldi on iOS without Internet connection + +#if canImport(SherpaOnnx) +import SherpaOnnx +#elseif canImport(SherpaOnnxShared) +import SherpaOnnxShared +#else +#error("SherpaOnnx module not found. Please check your SPM dependency configuration.") +#endif +#if canImport(piper_phonemize) +import piper_phonemize +#else +#error("piper_phonemize module not found. Please check your SPM dependency configuration.") +#endif +import SwiftUI +import AVFoundation +import UniformTypeIdentifiers + +class TtsProgressHandler: ObservableObject { + @Published var progress: Float = 0.0 + @Published var isGenerating = false + private var audioEngine: AVAudioEngine? + private var playerNode: AVAudioPlayerNode? + private var audioFormat: AVAudioFormat? + private var sampleRate: Float = 22050 + private var pendingBuffers = 0 + private var shouldStop = false + private let lock = NSLock() + + func startPlayback(sampleRate: Float) { + self.sampleRate = sampleRate + lock.lock() + self.pendingBuffers = 0 + self.shouldStop = false + lock.unlock() + + do { + let session = AVAudioSession.sharedInstance() + try session.setCategory(.playback, mode: .default) + try session.setActive(true) + } catch { + print("AVAudioSession error: \(error)") + } + + let engine = AVAudioEngine() + let player = AVAudioPlayerNode() + let format = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: Double(sampleRate), + channels: 1, + interleaved: false) + + engine.attach(player) + engine.connect(player, to: engine.mainMixerNode, format: format) + + do { + try engine.start() + } catch { + print("AVAudioEngine start error: \(error)") + return + } + + self.audioEngine = engine + self.playerNode = player + self.audioFormat = format + + player.play() + + DispatchQueue.main.async { + self.isGenerating = true + self.progress = 0.0 + } + } + + func appendSamples(_ samples: UnsafePointer?, count: Int32, progress: Float) { + lock.lock() + let stop = shouldStop + lock.unlock() + guard !stop, + let playerNode = playerNode, + let audioFormat = audioFormat, + let samples = samples, + count > 0 + else { return } + + let frameCount = AVAudioFrameCount(count) + guard let buffer = AVAudioPCMBuffer( + pcmFormat: audioFormat, frameCapacity: frameCount) + else { return } + + buffer.frameLength = frameCount + let channelData = buffer.floatChannelData![0] + memcpy(channelData, samples, Int(count) * MemoryLayout.size) + + lock.lock() + pendingBuffers += 1 + lock.unlock() + + playerNode.scheduleBuffer(buffer) { [weak self] in + guard let self = self else { return } + self.lock.lock() + self.pendingBuffers -= 1 + self.lock.unlock() + } + + DispatchQueue.main.async { + self.progress = progress + } + } + + /// Returns 0 to stop generation, 1 to continue + var stopFlag: Int32 { + lock.lock() + defer { lock.unlock() } + return shouldStop ? 0 : 1 + } + + func requestStop() { + lock.lock() + shouldStop = true + lock.unlock() + } + + func finishGeneration() { + // Wait for all scheduled buffers to finish playing, then clean up + DispatchQueue.global(qos: .background).async { [weak self] in + guard let self = self else { return } + while true { + self.lock.lock() + let pending = self.pendingBuffers + let stop = self.shouldStop + self.lock.unlock() + if pending <= 0 || stop { break } + Thread.sleep(forTimeInterval: 0.05) + } + + self.lock.lock() + let stopped = self.shouldStop + self.lock.unlock() + + if stopped { + self.playerNode?.stop() + } else { + Thread.sleep(forTimeInterval: 0.3) + } + + DispatchQueue.main.async { + self.playerNode?.stop() + self.audioEngine?.stop() + self.audioEngine = nil + self.playerNode = nil + self.audioFormat = nil + self.isGenerating = false + if !stopped { + self.progress = 1.0 + } + } + } + } +} + +struct ContentView: View { + @State private var sid = "0" + @State private var speed = 1.0 + @State private var text = "" + @State private var showAlert = false + @State var filename: URL = NSURL() as URL + @State var audioPlayer: AVAudioPlayer! + + @State private var lang = "en" + @State private var numSteps = 5 + + @StateObject private var progressHandler = TtsProgressHandler() + + @State private var showSavePicker = false + + private let languages = [ + "en", "ko", "ja", "ar", "bg", "cs", "da", "de", "el", "es", + "et", "fi", "fr", "hi", "hr", "hu", "id", "it", "lt", "lv", + "nl", "pl", "pt", "ro", "ru", "sk", "sl", "sv", "tr", "uk", "vi" + ] + + private let ttsSetup = createOfflineTts() + private var tts: SherpaOnnxOfflineTtsWrapper { ttsSetup.tts } + + var body: some View { + + VStack(alignment: .leading) { + HStack { + Spacer() + Text("Next-gen Kaldi: TTS").font(.title) + Spacer() + } + if tts.numSpeakers > 1 { + HStack { + Text("Speaker (1-\(tts.numSpeakers))") + Stepper("\((Int(sid) ?? 0) + 1)", value: Binding( + get: { (Int(sid) ?? 0) + 1 }, + set: { sid = "\($0 - 1)" } + ), in: 1...Int(tts.numSpeakers)) + } + } + HStack{ + Text("Speed \(String(format: "%.1f", speed))") + .padding(.trailing) + Slider(value: $speed, in: 0.5...2.0, step: 0.1) { + Text("Speech speed") + } + } + + if tts.isSupertonic { + HStack { + Text("Language") + Picker("Language", selection: $lang) { + ForEach(languages, id: \.self) { l in + Text(l).tag(l) + } + } + .pickerStyle(.menu) + + Spacer() + + Text("Steps") + Stepper("\(numSteps)", value: $numSteps, in: 1...20) + } + } + + if progressHandler.isGenerating { + VStack(spacing: 4) { + ProgressView(value: Double(progressHandler.progress)) + .progressViewStyle(.linear) + Text(String(format: "%.0f%%", progressHandler.progress * 100)) + .font(.caption) + .foregroundColor(.secondary) + } + .padding(.vertical, 4) + } + + Text("Please input your text below").padding([.trailing, .top, .bottom]) + + TextEditor(text: $text) + .font(.body) + .opacity(self.text.isEmpty ? 0.25 : 1) + .disableAutocorrection(true) + .border(Color.black) + .frame(minHeight: 100) + + Spacer() + HStack { + Spacer() + if progressHandler.isGenerating { + Button(action: { + progressHandler.requestStop() + }) { + Text("Stop") + } + } else { + Button(action: { + generate() + }) { + Text("Generate") + } + .alert(isPresented: $showAlert) { + Alert( + title: Text("Empty text"), + message: Text( + "Please input your text before clicking the Generate button")) + } + } + Spacer() + Button(action: { + self.audioPlayer.play() + }) { + Text("Play") + }.disabled(filename.absoluteString.isEmpty || progressHandler.isGenerating) + Spacer() + Button(action: { + showSavePicker = true + }) { + Text("Save") + } + .disabled(filename.absoluteString.isEmpty || progressHandler.isGenerating) + .fileExporter( + isPresented: $showSavePicker, + document: WavDocument(url: filename), + contentType: .wav, + defaultFilename: "sherpa-onnx-tts-output" + ) { result in + // fileExporter handles the save + } + Spacer() + Button(action: { + shareWav() + }) { + Text("Share") + }.disabled(filename.absoluteString.isEmpty || progressHandler.isGenerating) + Spacer() + } + Spacer() + } + .padding() + } + + private func generate() { + let speakerId = Int(self.sid) ?? 0 + let t = self.text.trimmingCharacters(in: .whitespacesAndNewlines) + if t.isEmpty { + self.showAlert = true + return + } + + if self.filename.absoluteString.isEmpty { + let tempDirectoryURL = NSURL.fileURL( + withPath: NSTemporaryDirectory(), isDirectory: true) + self.filename = tempDirectoryURL.appendingPathComponent("test.wav") + } + + let handler = progressHandler + let sampleRate = Float(tts.sampleRate) + let currentSpeed = Float(self.speed) + let currentNumSteps = self.numSteps + let currentLang = self.lang + let currentFilename = self.filename + let isSupertonic = tts.isSupertonic + // piper-phonemize is only used for models with a .phonemize frontend. + let usePhonemize = ttsSetup.frontend == .phonemize + let ttsLang = tts.lang + + DispatchQueue.global(qos: .userInitiated).async { + handler.startPlayback(sampleRate: sampleRate) + + let arg = Unmanaged.passUnretained(handler).toOpaque() + + let audio: SherpaOnnxGeneratedAudioWrapper + + let progressCallback: TtsProgressCallbackWithArg = { + samples, n, progress, arg in + let h = Unmanaged.fromOpaque(arg!) + .takeUnretainedValue() + h.appendSamples(samples, count: n, progress: progress) + return h.stopFlag + } + + if isSupertonic { + var genConfig = SherpaOnnxGenerationConfigSwift() + genConfig.sid = speakerId + genConfig.speed = currentSpeed + genConfig.numSteps = currentNumSteps + genConfig.extra = ["lang": currentLang] + audio = tts.generateWithConfig( + text: t, config: genConfig, + callback: progressCallback, arg: arg) + } else if usePhonemize { + // The text is phonemized outside of sherpa-onnx with + // piper-phonemize; the phoneme codepoints are passed via the + // generation config and the text itself is left empty. + // The espeak-ng voice is taken from the model metadata. + // Fall back to en-us if the model has no language information. + let voice: String + if ttsLang.isEmpty { + print("Warning: the model has no language information. Falling back to en-us.") + voice = "en-us" + } else { + voice = ttsLang + } + + guard let phonemes = piperPhonemizeText(text: t, voice: voice) else { + print("Error: piper-phonemize failed to phonemize the text") + handler.finishGeneration() + return + } + + var codepoints: [[Int32]] = [] + for i in 0...fromOpaque(arg!) + .takeUnretainedValue() + h.appendSamples(samples, count: n, progress: 1.0) + return h.stopFlag + } + + audio = tts.generateWithCallbackWithArg( + text: t, callback: simpleCallback, arg: arg, + sid: speakerId, speed: currentSpeed) + } + + let _ = audio.save(filename: currentFilename.path) + + handler.finishGeneration() + + DispatchQueue.main.async { + self.audioPlayer = try? AVAudioPlayer(contentsOf: self.filename) + } + } + } + + private func shareWav() { + guard !filename.absoluteString.isEmpty else { return } + let activityVC = UIActivityViewController( + activityItems: [filename], applicationActivities: nil) + guard let scene = UIApplication.shared.connectedScenes.first as? UIWindowScene, + let rootVC = scene.windows.first?.rootViewController + else { return } + rootVC.present(activityVC, animated: true) + } +} + +struct WavDocument: FileDocument { + static var readableContentTypes: [UTType] { [.wav] } + + let url: URL + + init(url: URL) { + self.url = url + } + + init(configuration: ReadConfiguration) throws { + fatalError("init(configuration:) not implemented") + } + + func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { + let data = try Data(contentsOf: url) + return FileWrapper(regularFileWithContents: data) + } +} + +struct ContentView_Previews: PreviewProvider { + static var previews: some View { + ContentView() + } +} diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Info.plist b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Info.plist new file mode 100644 index 000000000..a52458039 --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Info.plist @@ -0,0 +1,8 @@ + + + + + NSMicrophoneUsageDescription + Need microphone access for Next-gen Kaldi to work + + diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Preview Content/Preview Assets.xcassets/Contents.json b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Preview Content/Preview Assets.xcassets/Contents.json new file mode 100644 index 000000000..73c00596a --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/Preview Content/Preview Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemizeApp.swift b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemizeApp.swift new file mode 100644 index 000000000..bfb468352 --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemizeApp.swift @@ -0,0 +1,22 @@ +// +// SherpaOnnxTtsPhonemizeApp.swift +// SherpaOnnxTtsPhonemize +// +// Created by fangjun on 2023/11/23. +// + +import SwiftUI + +@main +struct SherpaOnnxTtsPhonemizeApp: App { + init() { + // The phonemizer is only used by models with a .phonemize frontend. + initPiperPhonemize() + } + + var body: some Scene { + WindowGroup { + ContentView() + } + } +} diff --git a/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/ViewModel.swift b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/ViewModel.swift new file mode 100644 index 000000000..8423c0052 --- /dev/null +++ b/ios-swiftui/SherpaOnnxTtsPhonemize/SherpaOnnxTtsPhonemize/ViewModel.swift @@ -0,0 +1,294 @@ +// +// ViewModel.swift +// SherpaOnnxTtsPhonemize +// +// Created by fangjun on 2023/11/23. +// + +#if canImport(SherpaOnnx) +import SherpaOnnx +#elseif canImport(SherpaOnnxShared) +import SherpaOnnxShared +#else +#error("SherpaOnnx module not found. Please check your SPM dependency configuration.") +#endif +#if canImport(piper_phonemize) +import piper_phonemize +#else +#error("piper_phonemize module not found. Please check your SPM dependency configuration.") +#endif +import Foundation + +func getResource(_ forResource: String, _ ofType: String) -> String { + let path = Bundle.main.path(forResource: forResource, ofType: ofType) + precondition( + path != nil, + "\(forResource).\(ofType) does not exist!\n" + "Remember to change \n" + + " Build Phases -> Copy Bundle Resources\n" + "to add it!" + ) + return path! +} + +/// How the input text is converted to phonemes before synthesis. +enum TextFrontend { + /// Phonemize the text with piper-phonemize and pass the resulting phoneme + /// codepoints to sherpa-onnx. No lexicon file is needed. + case phonemize + + /// Let sherpa-onnx convert the text with the model's own lexicon (or its + /// built-in frontend). piper-phonemize is not used. + case lexicon +} + +/// A TTS model together with the text frontend it expects. +struct TtsSetup { + let tts: SherpaOnnxOfflineTtsWrapper + let frontend: TextFrontend +} + +/// Initialize the piper-phonemize phonemizer. +/// +/// The espeak-ng-data used here is bundled inside the piper-phonemize package +/// and only feeds the piper phonemizer. sherpa-onnx itself does not use +/// espeak-ng anymore. +func initPiperPhonemize() { + print("piper-phonemize version: \(piperPhonemizeGetVersionStr())") + + guard let dataDir = piperPhonemizeBundledDataDir() else { + print("Error: bundled espeak-ng-data not found") + return + } + + let sampleRate = piperPhonemizeInitialize(dataDir: dataDir) + if sampleRate < 0 { + print("Error: failed to initialize piper-phonemize") + } +} + +/// Please refer to +/// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html +/// to download pre-trained models + +func getTtsForVCTK() -> SherpaOnnxOfflineTtsWrapper { + // See the following link + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#vctk-english-multi-speaker-109-speakers + + // vits-vctk.onnx + let model = getResource("vits-vctk", "onnx") + + // lexicon.txt + let lexicon = getResource("lexicon", "txt") + + // tokens.txt + let tokens = getResource("tokens", "txt") + + let vits = sherpaOnnxOfflineTtsVitsModelConfig(model: model, lexicon: lexicon, tokens: tokens) + let modelConfig = sherpaOnnxOfflineTtsModelConfig(vits: vits) + var config = sherpaOnnxOfflineTtsConfig(model: modelConfig) + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + +func getTtsForAishell3() -> SherpaOnnxOfflineTtsWrapper { + // See the following link + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#vits-model-aishell3 + + let model = getResource("model", "onnx") + + // lexicon.txt + let lexicon = getResource("lexicon", "txt") + + // tokens.txt + let tokens = getResource("tokens", "txt") + + // rule.fst + let ruleFsts = getResource("rule", "fst") + + // rule.far + let ruleFars = getResource("rule", "far") + + let vits = sherpaOnnxOfflineTtsVitsModelConfig(model: model, lexicon: lexicon, tokens: tokens) + let modelConfig = sherpaOnnxOfflineTtsModelConfig(vits: vits) + var config = sherpaOnnxOfflineTtsConfig( + model: modelConfig, + ruleFsts: ruleFsts, + ruleFars: ruleFars + ) + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +func getTtsFor_en_US_amy_low() -> SherpaOnnxOfflineTtsWrapper { + // please see https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + + let model = getResource("en_US-amy-low", "onnx") + + // tokens.txt + let tokens = getResource("tokens", "txt") + + // This is a piper English model: the text is phonemized with piper-phonemize + // and passed as phoneme codepoints, so no lexicon is used. + let vits = sherpaOnnxOfflineTtsVitsModelConfig(model: model, tokens: tokens) + let modelConfig = sherpaOnnxOfflineTtsModelConfig(vits: vits) + var config = sherpaOnnxOfflineTtsConfig(model: modelConfig) + + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#vits-melo-tts-zh-en-chinese-english-1-speaker +func getTtsFor_zh_en_melo_tts() -> SherpaOnnxOfflineTtsWrapper { + // please see https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-melo-tts-zh_en.tar.bz2 + + let model = getResource("model", "onnx") + + let tokens = getResource("tokens", "txt") + let lexicon = getResource("lexicon", "txt") + + let numFst = getResource("number", "fst") + let dateFst = getResource("date", "fst") + let phoneFst = getResource("phone", "fst") + let ruleFsts = "\(dateFst),\(phoneFst),\(numFst)" + + let vits = sherpaOnnxOfflineTtsVitsModelConfig( + model: model, lexicon: lexicon, tokens: tokens, + noiseScale: 0.667, + noiseScaleW: 0.8, + lengthScale: 1.0 + ) + + let modelConfig = sherpaOnnxOfflineTtsModelConfig(vits: vits) + var config = sherpaOnnxOfflineTtsConfig( + model: modelConfig, + ruleFsts: ruleFsts + ) + + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + +func getTtsFor_matcha_icefall_zh_baker() -> SherpaOnnxOfflineTtsWrapper { + // please see https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker + + let acousticModel = getResource("model-steps-3", "onnx") + let vocoder = getResource("vocos-22khz-univ", "onnx") + + let tokens = getResource("tokens", "txt") + let lexicon = getResource("lexicon", "txt") + + let numFst = getResource("number", "fst") + let dateFst = getResource("date", "fst") + let phoneFst = getResource("phone", "fst") + let ruleFsts = "\(dateFst),\(phoneFst),\(numFst)" + + let matcha = sherpaOnnxOfflineTtsMatchaModelConfig( + acousticModel: acousticModel, + vocoder: vocoder, + lexicon: lexicon, + tokens: tokens + ) + + let modelConfig = sherpaOnnxOfflineTtsModelConfig(matcha: matcha) + var config = sherpaOnnxOfflineTtsConfig( + model: modelConfig, + ruleFsts: ruleFsts + ) + + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + +func getTtsFor_kokoro_en_v0_19() -> SherpaOnnxOfflineTtsWrapper { + // please see https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html#kokoro-en-v0-19-english-11-speakers + + let model = getResource("model", "onnx") + let voices = getResource("voices", "bin") + + // tokens.txt + let tokens = getResource("tokens", "txt") + + // This is a piper-family English model: the text is phonemized with + // piper-phonemize and passed as phoneme codepoints, so no lexicon is used. + let kokoro = sherpaOnnxOfflineTtsKokoroModelConfig( + model: model, voices: voices, tokens: tokens) + let modelConfig = sherpaOnnxOfflineTtsModelConfig(kokoro: kokoro) + var config = sherpaOnnxOfflineTtsConfig(model: modelConfig) + + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + +func getTtsFor_kokoro_multi_lang_v1_0() -> SherpaOnnxOfflineTtsWrapper { + // please see https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html + + let model = getResource("model", "onnx") + let voices = getResource("voices", "bin") + + // tokens.txt + let tokens = getResource("tokens", "txt") + + let lexicon_en = getResource("lexicon-us-en", "txt") + let lexicon_zh = getResource("lexicon-zh", "txt") + let lexicon = "\(lexicon_en),\(lexicon_zh)" + + + let numFst = getResource("number-zh", "fst") + let dateFst = getResource("date-zh", "fst") + let phoneFst = getResource("phone-zh", "fst") + let ruleFsts = "\(dateFst),\(phoneFst),\(numFst)" + + let kokoro = sherpaOnnxOfflineTtsKokoroModelConfig( + model: model, voices: voices, tokens: tokens, lexicon: lexicon) + let modelConfig = sherpaOnnxOfflineTtsModelConfig(kokoro: kokoro) + var config = sherpaOnnxOfflineTtsConfig(model: modelConfig, ruleFsts: ruleFsts) + + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + +func getTtsForSupertonic() -> SherpaOnnxOfflineTtsWrapper { + // please see https://k2-fsa.github.io/sherpa/onnx/tts/supertonic.html + // Download: https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 + + let durationPredictor = getResource("duration_predictor", "int8.onnx") + let textEncoder = getResource("text_encoder", "int8.onnx") + let vectorEstimator = getResource("vector_estimator", "int8.onnx") + let vocoder = getResource("vocoder", "int8.onnx") + let ttsJson = getResource("tts", "json") + let unicodeIndexer = getResource("unicode_indexer", "bin") + let voiceStyle = getResource("voice", "bin") + + let supertonic = sherpaOnnxOfflineTtsSupertonicModelConfig( + durationPredictor: durationPredictor, + textEncoder: textEncoder, + vectorEstimator: vectorEstimator, + vocoder: vocoder, + ttsJson: ttsJson, + unicodeIndexer: unicodeIndexer, + voiceStyle: voiceStyle + ) + let modelConfig = sherpaOnnxOfflineTtsModelConfig(numThreads: 2, supertonic: supertonic) + var config = sherpaOnnxOfflineTtsConfig(model: modelConfig) + return SherpaOnnxOfflineTtsWrapper(config: &config) +} + +func createOfflineTts() -> TtsSetup { + // Please enable only one of them. + // + // The text frontend is chosen per model: piper-phonemize emits espeak IPA, + // which only the piper-family English models accept. Chinese/multi-lingual + // models need their own lexicon; VCTK uses a coqui phone inventory; and + // supertonic has its own text frontend. + + // return TtsSetup(tts: getTtsForSupertonic(), frontend: .lexicon) + + // return TtsSetup(tts: getTtsFor_kokoro_multi_lang_v1_0(), frontend: .lexicon) + + // return TtsSetup(tts: getTtsFor_kokoro_en_v0_19(), frontend: .phonemize) + + // return TtsSetup(tts: getTtsFor_matcha_icefall_zh_baker(), frontend: .lexicon) + + return TtsSetup(tts: getTtsFor_en_US_amy_low(), frontend: .phonemize) + + // return TtsSetup(tts: getTtsForVCTK(), frontend: .lexicon) + + // return TtsSetup(tts: getTtsForAishell3(), frontend: .lexicon) + + // return TtsSetup(tts: getTtsFor_zh_en_melo_tts(), frontend: .lexicon) + + // please add more models on need by following the above two examples +} diff --git a/java-api-examples/NonStreamingTtsKittenEn.java b/java-api-examples/NonStreamingTtsKittenEnLexicon.java similarity index 92% rename from java-api-examples/NonStreamingTtsKittenEn.java rename to java-api-examples/NonStreamingTtsKittenEnLexicon.java index 91026fcb4..1cfb5a698 100644 --- a/java-api-examples/NonStreamingTtsKittenEn.java +++ b/java-api-examples/NonStreamingTtsKittenEnLexicon.java @@ -4,7 +4,7 @@ // to convert text to speech import com.k2fsa.sherpa.onnx.*; -public class NonStreamingTtsKittenEn { +public class NonStreamingTtsKittenEnLexicon { public static void main(String[] args) { LibraryUtils.enableDebug(); // please visit @@ -13,7 +13,7 @@ public class NonStreamingTtsKittenEn { String model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; String voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; String tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; - String dataDir = "./kitten-nano-en-v0_1-fp16/espeak-ng-data"; + String lexicon = "./lexicon-en-us.txt"; String text = "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" @@ -24,7 +24,7 @@ public class NonStreamingTtsKittenEn { .setModel(model) .setVoices(voices) .setTokens(tokens) - .setDataDir(dataDir) + .setLexicon(lexicon) .build(); OfflineTtsModelConfig modelConfig = @@ -52,7 +52,7 @@ public class NonStreamingTtsKittenEn { float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); float realTimeFactor = timeElapsedSeconds / audioDuration; - String waveFilename = "tts-kitten-en.wav"; + String waveFilename = "tts-kitten-en-lexicon.wav"; audio.save(waveFilename); System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); diff --git a/java-api-examples/NonStreamingTtsKittenEnPhonemize.java b/java-api-examples/NonStreamingTtsKittenEnPhonemize.java new file mode 100644 index 000000000..d4e3c3bc1 --- /dev/null +++ b/java-api-examples/NonStreamingTtsKittenEnPhonemize.java @@ -0,0 +1,98 @@ +// Copyright 2026 Xiaomi Corporation + +// This file shows how to use aKittenTTS English KittenTTS English TTS model to convert text to +// speech. +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://github.com/csukuangfj/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. +// If you want to use a lexicon instead, please see +// NonStreamingTtsKittenEnLexicon.java. +import com.github.csukuangfj.piper.phonemize.PiperPhonemize; +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingTtsKittenEnPhonemize { + public static void main(String[] args) { + // please visit + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kitten.html + // to download model files + String model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; + String voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; + String tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; + String text = + "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + + "two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar."; + + OfflineTtsKittenModelConfig kittenModelConfig = + OfflineTtsKittenModelConfig.builder() + .setModel(model) + .setVoices(voices) + .setTokens(tokens) + .build(); + + OfflineTtsModelConfig modelConfig = + OfflineTtsModelConfig.builder() + .setKitten(kittenModelConfig) + .setNumThreads(2) + .setDebug(true) + .build(); + OfflineTtsConfig config = OfflineTtsConfig.builder().setModel(modelConfig).build(); + OfflineTts tts = new OfflineTts(config); + + System.out.printf("piper-phonemize version: %s\n", PiperPhonemize.getVersionStr()); + // initialize() extracts the espeak-ng-data embedded in piper-phonemize. + // That data only feeds the piper phonemizer; sherpa-onnx itself does not + // use espeak-ng anymore. + PiperPhonemize.initialize(); + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + String voice = tts.getLang(); + if (voice == null || voice.isEmpty()) { + System.out.println("Warning: the model has no language information. Falling back to en-us."); + voice = "en-us"; + } + System.out.printf("Phonemizer language: %s\n", voice); + + int[][] phonemeCodepoints; + try (PiperPhonemize.Result result = PiperPhonemize.text(text, voice)) { + if (result == null) { + throw new RuntimeException("Failed to phonemize the text"); + } + phonemeCodepoints = new int[result.getNumSentences()][]; + for (int i = 0; i < result.getNumSentences(); i++) { + phonemeCodepoints[i] = result.getPhonemes(i); + } + } + System.out.printf("Number of phonemized sentences: %d\n", phonemeCodepoints.length); + + int sid = 0; + float speed = 1.0f; + GenerationConfig genConfig = new GenerationConfig(); + genConfig.setSid(sid); + genConfig.setSpeed(speed); + genConfig.setSilenceScale(config.getSilenceScale()); + genConfig.setPhonemeCodepoints(phonemeCodepoints); + + long start = System.currentTimeMillis(); + // The text has already been phonemized; pass an empty string here. + GeneratedAudio audio = tts.generateWithConfig("", genConfig); + long stop = System.currentTimeMillis(); + + float timeElapsedSeconds = (stop - start) / 1000.0f; + + float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); + float realTimeFactor = timeElapsedSeconds / audioDuration; + + String waveFilename = "tts-kitten-en-phonemize.wav"; + audio.save(waveFilename); + System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); + System.out.printf("-- real-time factor (RTF): %.3f\n", realTimeFactor); + System.out.printf("-- text: %s\n", text); + System.out.printf("-- Saved to %s\n", waveFilename); + + tts.release(); + } +} diff --git a/java-api-examples/NonStreamingTtsKokoroEn.java b/java-api-examples/NonStreamingTtsKokoroEnLexicon.java similarity index 92% rename from java-api-examples/NonStreamingTtsKokoroEn.java rename to java-api-examples/NonStreamingTtsKokoroEnLexicon.java index 527047724..aa4cfa126 100644 --- a/java-api-examples/NonStreamingTtsKokoroEn.java +++ b/java-api-examples/NonStreamingTtsKokoroEnLexicon.java @@ -4,7 +4,7 @@ // to convert text to speech import com.k2fsa.sherpa.onnx.*; -public class NonStreamingTtsKokoroEn { +public class NonStreamingTtsKokoroEnLexicon { public static void main(String[] args) { // please visit // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html @@ -12,7 +12,7 @@ public class NonStreamingTtsKokoroEn { String model = "./kokoro-en-v0_19/model.onnx"; String voices = "./kokoro-en-v0_19/voices.bin"; String tokens = "./kokoro-en-v0_19/tokens.txt"; - String dataDir = "./kokoro-en-v0_19/espeak-ng-data"; + String lexicon = "./lexicon-en-us.txt"; String text = "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" @@ -23,7 +23,7 @@ public class NonStreamingTtsKokoroEn { .setModel(model) .setVoices(voices) .setTokens(tokens) - .setDataDir(dataDir) + .setLexicon(lexicon) .build(); OfflineTtsModelConfig modelConfig = @@ -51,7 +51,7 @@ public class NonStreamingTtsKokoroEn { float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); float realTimeFactor = timeElapsedSeconds / audioDuration; - String waveFilename = "tts-kokoro-en.wav"; + String waveFilename = "tts-kokoro-en-lexicon.wav"; audio.save(waveFilename); System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); diff --git a/java-api-examples/NonStreamingTtsKokoroEnPhonemize.java b/java-api-examples/NonStreamingTtsKokoroEnPhonemize.java new file mode 100644 index 000000000..7849955ac --- /dev/null +++ b/java-api-examples/NonStreamingTtsKokoroEnPhonemize.java @@ -0,0 +1,98 @@ +// Copyright 2026 Xiaomi Corporation + +// This file shows how to use aKokoro English Kokoro English TTS model to convert text to +// speech. +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://github.com/csukuangfj/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. +// If you want to use a lexicon instead, please see +// NonStreamingTtsKokoroEnLexicon.java. +import com.github.csukuangfj.piper.phonemize.PiperPhonemize; +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingTtsKokoroEnPhonemize { + public static void main(String[] args) { + // please visit + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html + // to download model files + String model = "./kokoro-en-v0_19/model.onnx"; + String voices = "./kokoro-en-v0_19/voices.bin"; + String tokens = "./kokoro-en-v0_19/tokens.txt"; + String text = + "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + + "two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar."; + + OfflineTtsKokoroModelConfig kokoroModelConfig = + OfflineTtsKokoroModelConfig.builder() + .setModel(model) + .setVoices(voices) + .setTokens(tokens) + .build(); + + OfflineTtsModelConfig modelConfig = + OfflineTtsModelConfig.builder() + .setKokoro(kokoroModelConfig) + .setNumThreads(2) + .setDebug(true) + .build(); + OfflineTtsConfig config = OfflineTtsConfig.builder().setModel(modelConfig).build(); + OfflineTts tts = new OfflineTts(config); + + System.out.printf("piper-phonemize version: %s\n", PiperPhonemize.getVersionStr()); + // initialize() extracts the espeak-ng-data embedded in piper-phonemize. + // That data only feeds the piper phonemizer; sherpa-onnx itself does not + // use espeak-ng anymore. + PiperPhonemize.initialize(); + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + String voice = tts.getLang(); + if (voice == null || voice.isEmpty()) { + System.out.println("Warning: the model has no language information. Falling back to en-us."); + voice = "en-us"; + } + System.out.printf("Phonemizer language: %s\n", voice); + + int[][] phonemeCodepoints; + try (PiperPhonemize.Result result = PiperPhonemize.text(text, voice)) { + if (result == null) { + throw new RuntimeException("Failed to phonemize the text"); + } + phonemeCodepoints = new int[result.getNumSentences()][]; + for (int i = 0; i < result.getNumSentences(); i++) { + phonemeCodepoints[i] = result.getPhonemes(i); + } + } + System.out.printf("Number of phonemized sentences: %d\n", phonemeCodepoints.length); + + int sid = 0; + float speed = 1.0f; + GenerationConfig genConfig = new GenerationConfig(); + genConfig.setSid(sid); + genConfig.setSpeed(speed); + genConfig.setSilenceScale(config.getSilenceScale()); + genConfig.setPhonemeCodepoints(phonemeCodepoints); + + long start = System.currentTimeMillis(); + // The text has already been phonemized; pass an empty string here. + GeneratedAudio audio = tts.generateWithConfig("", genConfig); + long stop = System.currentTimeMillis(); + + float timeElapsedSeconds = (stop - start) / 1000.0f; + + float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); + float realTimeFactor = timeElapsedSeconds / audioDuration; + + String waveFilename = "tts-kokoro-en-phonemize.wav"; + audio.save(waveFilename); + System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); + System.out.printf("-- real-time factor (RTF): %.3f\n", realTimeFactor); + System.out.printf("-- text: %s\n", text); + System.out.printf("-- Saved to %s\n", waveFilename); + + tts.release(); + } +} diff --git a/java-api-examples/NonStreamingTtsKokoroZhEn.java b/java-api-examples/NonStreamingTtsKokoroZhEnLexicon.java similarity index 92% rename from java-api-examples/NonStreamingTtsKokoroZhEn.java rename to java-api-examples/NonStreamingTtsKokoroZhEnLexicon.java index 24781b743..25e6118ab 100644 --- a/java-api-examples/NonStreamingTtsKokoroZhEn.java +++ b/java-api-examples/NonStreamingTtsKokoroZhEnLexicon.java @@ -4,7 +4,7 @@ // to convert Chinese and English text to speech import com.k2fsa.sherpa.onnx.*; -public class NonStreamingTtsKokoroZhEn { +public class NonStreamingTtsKokoroZhEnLexicon { public static void main(String[] args) { // please visit // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html @@ -12,7 +12,6 @@ public class NonStreamingTtsKokoroZhEn { String model = "./kokoro-multi-lang-v1_0/model.onnx"; String voices = "./kokoro-multi-lang-v1_0/voices.bin"; String tokens = "./kokoro-multi-lang-v1_0/tokens.txt"; - String dataDir = "./kokoro-multi-lang-v1_0/espeak-ng-data"; String lexicon = "./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt"; String text = @@ -24,7 +23,6 @@ public class NonStreamingTtsKokoroZhEn { .setModel(model) .setVoices(voices) .setTokens(tokens) - .setDataDir(dataDir) .setLexicon(lexicon) .build(); @@ -53,7 +51,7 @@ public class NonStreamingTtsKokoroZhEn { float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); float realTimeFactor = timeElapsedSeconds / audioDuration; - String waveFilename = "tts-kokoro-zh-en.wav"; + String waveFilename = "tts-kokoro-zh-en-lexicon.wav"; audio.save(waveFilename); System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); diff --git a/java-api-examples/NonStreamingTtsMatchaEn.java b/java-api-examples/NonStreamingTtsMatchaEnLexicon.java similarity index 92% rename from java-api-examples/NonStreamingTtsMatchaEn.java rename to java-api-examples/NonStreamingTtsMatchaEnLexicon.java index f4088f3a1..8838064bc 100644 --- a/java-api-examples/NonStreamingTtsMatchaEn.java +++ b/java-api-examples/NonStreamingTtsMatchaEnLexicon.java @@ -4,7 +4,7 @@ // to convert text to speech import com.k2fsa.sherpa.onnx.*; -public class NonStreamingTtsMatchaEn { +public class NonStreamingTtsMatchaEnLexicon { public static void main(String[] args) { // please visit // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker @@ -12,7 +12,7 @@ public class NonStreamingTtsMatchaEn { String acousticModel = "./matcha-icefall-en_US-ljspeech/model-steps-3.onnx"; String vocoder = "./vocos-22khz-univ.onnx"; String tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt"; - String dataDir = "./matcha-icefall-en_US-ljspeech/espeak-ng-data"; + String lexicon = "./lexicon-en-us.txt"; String text = "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" @@ -23,7 +23,7 @@ public class NonStreamingTtsMatchaEn { .setAcousticModel(acousticModel) .setVocoder(vocoder) .setTokens(tokens) - .setDataDir(dataDir) + .setLexicon(lexicon) .build(); OfflineTtsModelConfig modelConfig = @@ -50,7 +50,7 @@ public class NonStreamingTtsMatchaEn { float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); float realTimeFactor = timeElapsedSeconds / audioDuration; - String waveFilename = "tts-matcha-en.wav"; + String waveFilename = "tts-matcha-en-lexicon.wav"; audio.save(waveFilename); System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); diff --git a/java-api-examples/NonStreamingTtsMatchaEnPhonemize.java b/java-api-examples/NonStreamingTtsMatchaEnPhonemize.java new file mode 100644 index 000000000..ddfca1515 --- /dev/null +++ b/java-api-examples/NonStreamingTtsMatchaEnPhonemize.java @@ -0,0 +1,98 @@ +// Copyright 2026 Xiaomi Corporation + +// This file shows how to use amatcha English matcha English TTS model to convert text to +// speech. +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://github.com/csukuangfj/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. +// If you want to use a lexicon instead, please see +// NonStreamingTtsMatchaEnLexicon.java. +import com.github.csukuangfj.piper.phonemize.PiperPhonemize; +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingTtsMatchaEnPhonemize { + public static void main(String[] args) { + // please visit + // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker + // to download model files + String acousticModel = "./matcha-icefall-en_US-ljspeech/model-steps-3.onnx"; + String vocoder = "./vocos-22khz-univ.onnx"; + String tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt"; + String text = + "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + + "two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar."; + + OfflineTtsMatchaModelConfig matchaModelConfig = + OfflineTtsMatchaModelConfig.builder() + .setAcousticModel(acousticModel) + .setVocoder(vocoder) + .setTokens(tokens) + .build(); + + OfflineTtsModelConfig modelConfig = + OfflineTtsModelConfig.builder() + .setMatcha(matchaModelConfig) + .setNumThreads(1) + .setDebug(true) + .build(); + OfflineTtsConfig config = OfflineTtsConfig.builder().setModel(modelConfig).build(); + OfflineTts tts = new OfflineTts(config); + + System.out.printf("piper-phonemize version: %s\n", PiperPhonemize.getVersionStr()); + // initialize() extracts the espeak-ng-data embedded in piper-phonemize. + // That data only feeds the piper phonemizer; sherpa-onnx itself does not + // use espeak-ng anymore. + PiperPhonemize.initialize(); + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + String voice = tts.getLang(); + if (voice == null || voice.isEmpty()) { + System.out.println("Warning: the model has no language information. Falling back to en-us."); + voice = "en-us"; + } + System.out.printf("Phonemizer language: %s\n", voice); + + int[][] phonemeCodepoints; + try (PiperPhonemize.Result result = PiperPhonemize.text(text, voice)) { + if (result == null) { + throw new RuntimeException("Failed to phonemize the text"); + } + phonemeCodepoints = new int[result.getNumSentences()][]; + for (int i = 0; i < result.getNumSentences(); i++) { + phonemeCodepoints[i] = result.getPhonemes(i); + } + } + System.out.printf("Number of phonemized sentences: %d\n", phonemeCodepoints.length); + + int sid = 0; + float speed = 1.0f; + GenerationConfig genConfig = new GenerationConfig(); + genConfig.setSid(sid); + genConfig.setSpeed(speed); + genConfig.setSilenceScale(config.getSilenceScale()); + genConfig.setPhonemeCodepoints(phonemeCodepoints); + + long start = System.currentTimeMillis(); + // The text has already been phonemized; pass an empty string here. + GeneratedAudio audio = tts.generateWithConfig("", genConfig); + long stop = System.currentTimeMillis(); + + float timeElapsedSeconds = (stop - start) / 1000.0f; + + float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); + float realTimeFactor = timeElapsedSeconds / audioDuration; + + String waveFilename = "tts-matcha-en-phonemize.wav"; + audio.save(waveFilename); + System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); + System.out.printf("-- real-time factor (RTF): %.3f\n", realTimeFactor); + System.out.printf("-- text: %s\n", text); + System.out.printf("-- Saved to %s\n", waveFilename); + + tts.release(); + } +} diff --git a/java-api-examples/NonStreamingTtsMatchaZh.java b/java-api-examples/NonStreamingTtsMatchaZhLexicon.java similarity index 96% rename from java-api-examples/NonStreamingTtsMatchaZh.java rename to java-api-examples/NonStreamingTtsMatchaZhLexicon.java index 818f294be..c271e4e9d 100644 --- a/java-api-examples/NonStreamingTtsMatchaZh.java +++ b/java-api-examples/NonStreamingTtsMatchaZhLexicon.java @@ -4,7 +4,7 @@ // to convert text to speech import com.k2fsa.sherpa.onnx.*; -public class NonStreamingTtsMatchaZh { +public class NonStreamingTtsMatchaZhLexicon { public static void main(String[] args) { // please visit // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker @@ -54,7 +54,7 @@ public class NonStreamingTtsMatchaZh { float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); float realTimeFactor = timeElapsedSeconds / audioDuration; - String waveFilename = "tts-matcha-zh.wav"; + String waveFilename = "tts-matcha-zh-lexicon.wav"; audio.save(waveFilename); System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); diff --git a/java-api-examples/NonStreamingTtsPiperEn.java b/java-api-examples/NonStreamingTtsPiperEnLexicon.java similarity index 92% rename from java-api-examples/NonStreamingTtsPiperEn.java rename to java-api-examples/NonStreamingTtsPiperEnLexicon.java index 56f6bd241..541e4e077 100644 --- a/java-api-examples/NonStreamingTtsPiperEn.java +++ b/java-api-examples/NonStreamingTtsPiperEnLexicon.java @@ -4,14 +4,14 @@ // to convert text to speech import com.k2fsa.sherpa.onnx.*; -public class NonStreamingTtsPiperEn { +public class NonStreamingTtsPiperEnLexicon { public static void main(String[] args) { // please visit // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models // to download model files String model = "./vits-piper-en_GB-cori-medium/en_GB-cori-medium.onnx"; String tokens = "./vits-piper-en_GB-cori-medium/tokens.txt"; - String dataDir = "./vits-piper-en_GB-cori-medium/espeak-ng-data"; + String lexicon = "./lexicon-en-us.txt"; String text = "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" @@ -21,7 +21,7 @@ public class NonStreamingTtsPiperEn { OfflineTtsVitsModelConfig.builder() .setModel(model) .setTokens(tokens) - .setDataDir(dataDir) + .setLexicon(lexicon) .build(); OfflineTtsModelConfig modelConfig = @@ -50,7 +50,7 @@ public class NonStreamingTtsPiperEn { float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); float realTimeFactor = timeElapsedSeconds / audioDuration; - String waveFilename = "tts-piper-en.wav"; + String waveFilename = "tts-piper-en-lexicon.wav"; audio.save(waveFilename); System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); diff --git a/java-api-examples/NonStreamingTtsPiperEnPhonemize.java b/java-api-examples/NonStreamingTtsPiperEnPhonemize.java new file mode 100644 index 000000000..0399769b1 --- /dev/null +++ b/java-api-examples/NonStreamingTtsPiperEnPhonemize.java @@ -0,0 +1,99 @@ +// Copyright 2024 Xiaomi Corporation + +// This file shows how to use a piper VITS English TTS model +// to convert text to speech. +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://github.com/csukuangfj/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. +// +// If you want to use a lexicon instead, please see +// NonStreamingTtsPiperEnLexicon.java. +import com.github.csukuangfj.piper.phonemize.PiperPhonemize; +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingTtsPiperEnPhonemize { + public static void main(String[] args) { + // please visit + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // to download model files + String model = "./vits-piper-en_GB-cori-medium/en_GB-cori-medium.onnx"; + String tokens = "./vits-piper-en_GB-cori-medium/tokens.txt"; + String text = + "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" + + " businessman, an official, or a scholar."; + + OfflineTtsVitsModelConfig vitsModelConfig = + OfflineTtsVitsModelConfig.builder() + .setModel(model) + .setTokens(tokens) + .build(); + + OfflineTtsModelConfig modelConfig = + OfflineTtsModelConfig.builder() + .setVits(vitsModelConfig) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineTtsConfig config = OfflineTtsConfig.builder().setModel(modelConfig).build(); + OfflineTts tts = new OfflineTts(config); + + System.out.printf("piper-phonemize version: %s\n", PiperPhonemize.getVersionStr()); + // initialize() extracts the espeak-ng-data embedded in piper-phonemize. + // That data only feeds the piper phonemizer; sherpa-onnx itself does not + // use espeak-ng anymore. + PiperPhonemize.initialize(); + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + String voice = tts.getLang(); + if (voice == null || voice.isEmpty()) { + System.out.println("Warning: the model has no language information. Falling back to en-us."); + voice = "en-us"; + } + System.out.printf("Phonemizer language: %s\n", voice); + + int[][] phonemeCodepoints; + try (PiperPhonemize.Result result = PiperPhonemize.text(text, voice)) { + if (result == null) { + throw new RuntimeException("Failed to phonemize the text"); + } + phonemeCodepoints = new int[result.getNumSentences()][]; + for (int i = 0; i < result.getNumSentences(); i++) { + phonemeCodepoints[i] = result.getPhonemes(i); + } + } + System.out.printf("Number of phonemized sentences: %d\n", phonemeCodepoints.length); + + int sid = 0; + float speed = 1.0f; + GenerationConfig genConfig = new GenerationConfig(); + genConfig.setSid(sid); + genConfig.setSpeed(speed); + genConfig.setSilenceScale(config.getSilenceScale()); + genConfig.setPhonemeCodepoints(phonemeCodepoints); + + long start = System.currentTimeMillis(); + // The text has already been phonemized; pass an empty string here. + GeneratedAudio audio = tts.generateWithConfig("", genConfig); + long stop = System.currentTimeMillis(); + + float timeElapsedSeconds = (stop - start) / 1000.0f; + + float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); + float realTimeFactor = timeElapsedSeconds / audioDuration; + + String waveFilename = "tts-piper-en-phonemize.wav"; + audio.save(waveFilename); + System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); + System.out.printf("-- real-time factor (RTF): %.3f\n", realTimeFactor); + System.out.printf("-- text: %s\n", text); + System.out.printf("-- Saved to %s\n", waveFilename); + + tts.release(); + } +} diff --git a/java-api-examples/NonStreamingTtsPiperEnWithCallback.java b/java-api-examples/NonStreamingTtsPiperEnWithCallbackLexicon.java similarity index 96% rename from java-api-examples/NonStreamingTtsPiperEnWithCallback.java rename to java-api-examples/NonStreamingTtsPiperEnWithCallbackLexicon.java index fe0da9901..eff0552ff 100644 --- a/java-api-examples/NonStreamingTtsPiperEnWithCallback.java +++ b/java-api-examples/NonStreamingTtsPiperEnWithCallbackLexicon.java @@ -19,14 +19,14 @@ import java.util.concurrent.*; import java.util.concurrent.ConcurrentLinkedQueue; import javax.sound.sampled.*; -public class NonStreamingTtsPiperEn { +public class NonStreamingTtsPiperEnWithCallbackLexicon { public static void main(String[] args) { // please visit // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models // to download model files String model = "./vits-piper-en_GB-cori-medium/en_GB-cori-medium.onnx"; String tokens = "./vits-piper-en_GB-cori-medium/tokens.txt"; - String dataDir = "./vits-piper-en_GB-cori-medium/espeak-ng-data"; + String lexicon = "./lexicon-en-us.txt"; String text = "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" @@ -36,7 +36,7 @@ public class NonStreamingTtsPiperEn { OfflineTtsVitsModelConfig.builder() .setModel(model) .setTokens(tokens) - .setDataDir(dataDir) + .setLexicon(lexicon) .build(); OfflineTtsModelConfig modelConfig = @@ -176,7 +176,7 @@ public class NonStreamingTtsPiperEn { return; } - String waveFilename = "tts-piper-en.wav"; + String waveFilename = "tts-piper-en-with-callback-lexicon.wav"; audio.save(waveFilename); System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); diff --git a/java-api-examples/NonStreamingTtsPiperEnWithCallbackPhonemize.java b/java-api-examples/NonStreamingTtsPiperEnWithCallbackPhonemize.java new file mode 100644 index 000000000..ec7920d6d --- /dev/null +++ b/java-api-examples/NonStreamingTtsPiperEnWithCallbackPhonemize.java @@ -0,0 +1,224 @@ +// Copyright 2024 Xiaomi Corporation +// +// References +// https://www.baeldung.com/java-passing-method-parameter +// https://www.geeksforgeeks.org/how-to-create-a-thread-safe-queue-in-java/ +// https://stackoverflow.com/questions/74077394/java-audio-how-to-continuously-write-bytes-to-an-audio-file-as-they-are-being-g + +// This file shows how to use a piper VITS English TTS model +// to convert text to speech. The text is phonemized outside of sherpa-onnx +// with piper-phonemize (https://github.com/csukuangfj/piper-phonemize) and +// the resulting phoneme codepoints are passed via +// GenerationConfig.phonemeCodepoints. No lexicon file and no espeak-ng-data +// are needed. If you want to use a lexicon instead, please see +// NonStreamingTtsPiperEnWithCallbackLexicon.java. +// +// You can pass a callback to the generation call, +// which is invoked whenever max_num_sentences sentences have been +// finished generation. +// +// The callback saves the generated samples into a queue, which are played +// by a separate thread. + +import com.github.csukuangfj.piper.phonemize.PiperPhonemize; +import com.k2fsa.sherpa.onnx.*; +import java.util.Queue; +import java.util.concurrent.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import javax.sound.sampled.*; + +public class NonStreamingTtsPiperEnWithCallbackPhonemize { + public static void main(String[] args) { + // please visit + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // to download model files + String model = "./vits-piper-en_GB-cori-medium/en_GB-cori-medium.onnx"; + String tokens = "./vits-piper-en_GB-cori-medium/tokens.txt"; + String text = + "Today as always, men fall into two groups: slaves and free men. Whoever does not have" + + " two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a" + + " businessman, an official, or a scholar."; + + OfflineTtsVitsModelConfig vitsModelConfig = + OfflineTtsVitsModelConfig.builder() + .setModel(model) + .setTokens(tokens) + .build(); + + OfflineTtsModelConfig modelConfig = + OfflineTtsModelConfig.builder() + .setVits(vitsModelConfig) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineTtsConfig config = OfflineTtsConfig.builder().setModel(modelConfig).build(); + OfflineTts tts = new OfflineTts(config); + + System.out.printf("piper-phonemize version: %s\n", PiperPhonemize.getVersionStr()); + // initialize() extracts the espeak-ng-data embedded in piper-phonemize. + // That data only feeds the piper phonemizer; sherpa-onnx itself does not + // use espeak-ng anymore. + PiperPhonemize.initialize(); + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + String voice = tts.getLang(); + if (voice == null || voice.isEmpty()) { + System.out.println("Warning: the model has no language information. Falling back to en-us."); + voice = "en-us"; + } + System.out.printf("Phonemizer language: %s\n", voice); + + int[][] phonemeCodepoints; + try (PiperPhonemize.Result result = PiperPhonemize.text(text, voice)) { + if (result == null) { + throw new RuntimeException("Failed to phonemize the text"); + } + phonemeCodepoints = new int[result.getNumSentences()][]; + for (int i = 0; i < result.getNumSentences(); i++) { + phonemeCodepoints[i] = result.getPhonemes(i); + } + } + System.out.printf("Number of phonemized sentences: %d\n", phonemeCodepoints.length); + + Queue samplesQueue = new ConcurrentLinkedQueue<>(); + + Semaphore canPlaySem = new Semaphore(1); + try { + canPlaySem.acquire(); + } catch (InterruptedException ex) { + System.out.println("Failed to acquire the play semaphore in the main thread"); + return; + } + + Runnable playRuannable = + () -> { + try { + canPlaySem.acquire(); + } catch (InterruptedException e) { + System.out.println("Failed to get canPlay semaphore in the play thread"); + return; + } + + // https://docs.oracle.com/javase/8/docs/api/javax/sound/sampled/AudioFormat.html + AudioFormat format = + new AudioFormat( + tts.getSampleRate(), // sampleRate + 16, // sampleSizeInBits + 1, // channels + true, // signed + false // bigEndian + ); + DataLine.Info info = new DataLine.Info(SourceDataLine.class, format); + SourceDataLine line; + try { + line = (SourceDataLine) AudioSystem.getLine(info); + + int bufferSizeInBytes = tts.getSampleRate(); // 0.5 seconds + line.open(format, bufferSizeInBytes); + } catch (LineUnavailableException ex) { + System.out.println("Failed to open a device for playing"); + return; + } + line.start(); + + while (true) { + if (samplesQueue.isEmpty()) { + // Do nothing. + // + // If the generating speed is very slow, we can sleep + // for some time here to save some CPU. + } else { + byte[] samples = samplesQueue.poll(); + if (samples.length == 1) { + // end of the generating + break; + } + line.write(samples, 0, samples.length); + } + } + + line.drain(); + line.close(); + }; + + Thread playThread = new Thread(playRuannable); + playThread.start(); + + int sid = 0; + float speed = 1.0f; + GenerationConfig genConfig = new GenerationConfig(); + genConfig.setSid(sid); + genConfig.setSpeed(speed); + genConfig.setSilenceScale(config.getSilenceScale()); + genConfig.setPhonemeCodepoints(phonemeCodepoints); + long start = System.currentTimeMillis(); + // The text has already been phonemized; pass an empty string here. + GeneratedAudio audio = + tts.generateWithConfigAndCallback( + "", + genConfig, + (float[] samples) -> { + + // we use a byte array to save int16 samples + byte[] samplesInt16 = new byte[samples.length * 2]; + for (int i = 0; i < samples.length; ++i) { + float s = samples[i]; + if (s > 1) { + s = 1; + } + + if (s < -1) { + s = -1; + } + + short t = (short) (s * 32767); + + // we use little endian + samplesInt16[2 * i] = (byte) (t & 0xff); + samplesInt16[2 * i + 1] = (byte) ((t & 0xff00) >> 8); + } + + samplesQueue.add(samplesInt16); + + canPlaySem.release(); + + // Note: You can play the samples. + // warning: You need to save a copy of samples since it is freed + // when this function returns + + // return 1 to continue generation + // return 0 to stop generation + return 1; + }); + + // Since a sample always has two bytes. We put a single byte + // into the queue to indicate that we have finished processing. + samplesQueue.add(new byte[1]); + + long stop = System.currentTimeMillis(); + + float timeElapsedSeconds = (stop - start) / 1000.0f; + + float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); + float realTimeFactor = timeElapsedSeconds / audioDuration; + + try { + playThread.join(); + } catch (InterruptedException ex) { + System.out.println("Failed to join the play thread"); + return; + } + + String waveFilename = "tts-piper-en-with-callback-phonemize.wav"; + audio.save(waveFilename); + System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); + System.out.printf("-- real-time factor (RTF): %.3f\n", realTimeFactor); + System.out.printf("-- text: %s\n", text); + System.out.printf("-- Saved to %s\n", waveFilename); + + tts.release(); + } +} diff --git a/java-api-examples/NonStreamingTtsVitsDePhonemize.java b/java-api-examples/NonStreamingTtsVitsDePhonemize.java new file mode 100644 index 000000000..41afa817c --- /dev/null +++ b/java-api-examples/NonStreamingTtsVitsDePhonemize.java @@ -0,0 +1,95 @@ +// Copyright 2026 Xiaomi Corporation + +// This file shows how to use apiper VITS German piper VITS German TTS model to convert text to +// speech. +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://github.com/csukuangfj/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. +// There is no German lexicon, so this model is phonemize-only. +import com.github.csukuangfj.piper.phonemize.PiperPhonemize; +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingTtsVitsDePhonemize { + public static void main(String[] args) { + // please visit + // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // to download model files + String model = "./vits-piper-de_DE-glados-low/de_DE-glados-low.onnx"; + String tokens = "./vits-piper-de_DE-glados-low/tokens.txt"; + String text = + "Alles hat ein Ende, nur die Wurst hat zwei." + + ""; + + OfflineTtsVitsModelConfig vitsModelConfig = + OfflineTtsVitsModelConfig.builder() + .setModel(model) + .setTokens(tokens) + .build(); + + OfflineTtsModelConfig modelConfig = + OfflineTtsModelConfig.builder() + .setVits(vitsModelConfig) + .setNumThreads(1) + .setDebug(true) + .build(); + OfflineTtsConfig config = OfflineTtsConfig.builder().setModel(modelConfig).build(); + OfflineTts tts = new OfflineTts(config); + + System.out.printf("piper-phonemize version: %s\n", PiperPhonemize.getVersionStr()); + // initialize() extracts the espeak-ng-data embedded in piper-phonemize. + // That data only feeds the piper phonemizer; sherpa-onnx itself does not + // use espeak-ng anymore. + PiperPhonemize.initialize(); + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to de if the model has no language information. + String voice = tts.getLang(); + if (voice == null || voice.isEmpty()) { + System.out.println("Warning: the model has no language information. Falling back to de."); + voice = "de"; + } + System.out.printf("Phonemizer language: %s\n", voice); + + int[][] phonemeCodepoints; + try (PiperPhonemize.Result result = PiperPhonemize.text(text, voice)) { + if (result == null) { + throw new RuntimeException("Failed to phonemize the text"); + } + phonemeCodepoints = new int[result.getNumSentences()][]; + for (int i = 0; i < result.getNumSentences(); i++) { + phonemeCodepoints[i] = result.getPhonemes(i); + } + } + System.out.printf("Number of phonemized sentences: %d\n", phonemeCodepoints.length); + + int sid = 0; + float speed = 1.0f; + GenerationConfig genConfig = new GenerationConfig(); + genConfig.setSid(sid); + genConfig.setSpeed(speed); + genConfig.setSilenceScale(config.getSilenceScale()); + genConfig.setPhonemeCodepoints(phonemeCodepoints); + + long start = System.currentTimeMillis(); + // The text has already been phonemized; pass an empty string here. + GeneratedAudio audio = tts.generateWithConfig("", genConfig); + long stop = System.currentTimeMillis(); + + float timeElapsedSeconds = (stop - start) / 1000.0f; + + float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); + float realTimeFactor = timeElapsedSeconds / audioDuration; + + String waveFilename = "tts-vits-de-phonemize.wav"; + audio.save(waveFilename); + System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); + System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); + System.out.printf("-- real-time factor (RTF): %.3f\n", realTimeFactor); + System.out.printf("-- text: %s\n", text); + System.out.printf("-- Saved to %s\n", waveFilename); + + tts.release(); + } +} diff --git a/java-api-examples/NonStreamingTtsVitsZh.java b/java-api-examples/NonStreamingTtsVitsZhLexicon.java similarity index 96% rename from java-api-examples/NonStreamingTtsVitsZh.java rename to java-api-examples/NonStreamingTtsVitsZhLexicon.java index 7e12b23c9..ba5ba0717 100644 --- a/java-api-examples/NonStreamingTtsVitsZh.java +++ b/java-api-examples/NonStreamingTtsVitsZhLexicon.java @@ -7,7 +7,7 @@ // to train your model import com.k2fsa.sherpa.onnx.*; -public class NonStreamingTtsPiperEn { +public class NonStreamingTtsVitsZhLexicon { public static void main(String[] args) { // please visit // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models @@ -54,7 +54,7 @@ public class NonStreamingTtsPiperEn { float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); float realTimeFactor = timeElapsedSeconds / audioDuration; - String waveFilename = "tts-vits-zh.wav"; + String waveFilename = "tts-vits-zh-lexicon.wav"; audio.save(waveFilename); System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); diff --git a/java-api-examples/NonStreamingZipformerCtcHlg.java b/java-api-examples/NonStreamingZipformerCtcHlg.java new file mode 100644 index 000000000..269806b0e --- /dev/null +++ b/java-api-examples/NonStreamingZipformerCtcHlg.java @@ -0,0 +1,61 @@ +// Copyright 2026 Xiaomi Corporation + +// This file shows how to use a non-streaming Zipformer CTC model with an HLG +// decoding graph via sherpa-onnx's Java API. +// +// The HLG graph is passed to the C API through CtcFstDecoderConfig. +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingZipformerCtcHlg { + public static void main(String[] args) { + // please refer to + // https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + // to download model files + String dir = "./sherpa-onnx-zipformer-ctc-en-2023-10-02"; + String model = dir + "/model.int8.onnx"; + String tokens = dir + "/tokens.txt"; + String graph = dir + "/HLG.fst"; + String waveFilename = dir + "/test_wavs/0.wav"; + + WaveReader reader = new WaveReader(waveFilename); + + OfflineZipformerCtcModelConfig zipformerCtc = + OfflineZipformerCtcModelConfig.builder().setModel(model).build(); + + OfflineModelConfig modelConfig = + OfflineModelConfig.builder() + .setZipformerCtc(zipformerCtc) + .setTokens(tokens) + .setNumThreads(1) + .setDebug(true) + .build(); + + // This is what the example exercises: the graph and max_active reach the C + // API through OfflineCtcFstDecoderConfig. + OfflineCtcFstDecoderConfig ctcFstDecoderConfig = + OfflineCtcFstDecoderConfig.builder() + .setGraph(graph) + .setMaxActive(3000) + .build(); + + OfflineRecognizerConfig config = + OfflineRecognizerConfig.builder() + .setOfflineModelConfig(modelConfig) + .setDecodingMethod("greedy_search") + .setCtcFstDecoderConfig(ctcFstDecoderConfig) + .build(); + + OfflineRecognizer recognizer = new OfflineRecognizer(config); + OfflineStream stream = recognizer.createStream(); + stream.acceptWaveform(reader.getSamples(), reader.getSampleRate()); + + recognizer.decode(stream); + + String text = recognizer.getResult(stream).getText(); + + System.out.printf("filename:%s\nresult:%s\n", waveFilename, text); + + stream.release(); + recognizer.release(); + } +} diff --git a/java-api-examples/README.md b/java-api-examples/README.md index 90944748e..7164e5750 100755 --- a/java-api-examples/README.md +++ b/java-api-examples/README.md @@ -78,22 +78,22 @@ downstream ASR and `dpdfnet2_48khz_hr.onnx` for 48 kHz enhancement output. ```bash ./run-non-streaming-tts-coqui-de.sh -./run-non-streaming-tts-kitten-en.sh -./run-non-streaming-tts-kokoro-en.sh -./run-non-streaming-tts-kokoro-zh-en.sh -./run-non-streaming-tts-matcha-en.sh -./run-non-streaming-tts-matcha-zh.sh -./run-non-streaming-tts-piper-en-with-callback.sh -./run-non-streaming-tts-piper-en.sh -./run-non-streaming-tts-vits-zh.sh +./run-non-streaming-tts-kitten-en-lexicon.sh +./run-non-streaming-tts-kokoro-en-lexicon.sh +./run-non-streaming-tts-kokoro-zh-en-lexicon.sh +./run-non-streaming-tts-matcha-en-lexicon.sh +./run-non-streaming-tts-matcha-zh-lexicon.sh +./run-non-streaming-tts-piper-en-with-callback-lexicon.sh +./run-non-streaming-tts-piper-en-lexicon.sh +./run-non-streaming-tts-vits-zh-lexicon.sh ./run-pocket-tts.sh -./run-zipvoice-tts.sh +./run-zipvoice-tts-lexicon.sh ``` ## Non-Streaming text-to-speech (Playback the audio as it is being generated) ```bash -./run-non-streaming-tts-piper-en-with-callback.sh +./run-non-streaming-tts-piper-en-with-callback-lexicon.sh ``` ## Spoken language identification @@ -151,8 +151,8 @@ The punctuation model supports both English and Chinese. ## VAD (Remove silence) ```bash -./run-vad-remove-slience.sh -./run-ten-vad-remove-slience.sh +./run-vad-remove-silence.sh +./run-ten-vad-remove-silence.sh ``` ## VAD + Non-streaming Dolphin CTC for speech recognition diff --git a/java-api-examples/ZipVoiceTts.java b/java-api-examples/ZipVoiceTtsLexicon.java similarity index 95% rename from java-api-examples/ZipVoiceTts.java rename to java-api-examples/ZipVoiceTtsLexicon.java index 1810cf0e1..a66646c31 100644 --- a/java-api-examples/ZipVoiceTts.java +++ b/java-api-examples/ZipVoiceTtsLexicon.java @@ -6,7 +6,7 @@ import com.k2fsa.sherpa.onnx.*; import java.util.HashMap; import java.util.Map; -public class ZipVoiceTts { +public class ZipVoiceTtsLexicon { public static void main(String[] args) { LibraryUtils.enableDebug(); // please visit @@ -23,7 +23,6 @@ public class ZipVoiceTts { .setEncoder(modelDir + "/encoder.int8.onnx") .setDecoder(modelDir + "/decoder.int8.onnx") .setVocoder("./vocos_24khz.onnx") - .setDataDir(modelDir + "/espeak-ng-data") .setLexicon(modelDir + "/lexicon.txt") .build(); @@ -64,7 +63,7 @@ public class ZipVoiceTts { float audioDuration = audio.getSamples().length / (float) audio.getSampleRate(); float realTimeFactor = timeElapsedSeconds / audioDuration; - String waveFilename = "generated-zipvoice-zh-en-java.wav"; + String waveFilename = "generated-zipvoice-zh-en-lexicon-java.wav"; audio.save(waveFilename); System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds); System.out.printf("-- audio duration: %.3f seconds\n", audioDuration); diff --git a/java-api-examples/run-non-streaming-tts-kitten-en-lexicon.sh b/java-api-examples/run-non-streaming-tts-kitten-en-lexicon.sh new file mode 100755 index 000000000..f0ec165da --- /dev/null +++ b/java-api-examples/run-non-streaming-tts-kitten-en-lexicon.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kitten.html +# to download more models + +if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 + tar xf kitten-nano-en-v0_1-fp16.tar.bz2 + rm kitten-nano-en-v0_1-fp16.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + NonStreamingTtsKittenEnLexicon.java diff --git a/java-api-examples/run-non-streaming-tts-kitten-en.sh b/java-api-examples/run-non-streaming-tts-kitten-en-phonemize.sh similarity index 82% rename from java-api-examples/run-non-streaming-tts-kitten-en.sh rename to java-api-examples/run-non-streaming-tts-kitten-en-phonemize.sh index 8cc709c44..f1419e42b 100755 --- a/java-api-examples/run-non-streaming-tts-kitten-en.sh +++ b/java-api-examples/run-non-streaming-tts-kitten-en-phonemize.sh @@ -14,7 +14,10 @@ if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then rm kitten-nano-en-v0_1-fp16.tar.bz2 fi + +source ./setup-phonemize.sh + java \ -Dsherpa_onnx.native.path=$PWD/../build/lib \ - -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ - NonStreamingTtsKittenEn.java + -cp "$PHONEMIZE_CLASSPATH" \ + NonStreamingTtsKittenEnPhonemize.java diff --git a/java-api-examples/run-non-streaming-tts-kokoro-en-lexicon.sh b/java-api-examples/run-non-streaming-tts-kokoro-en-lexicon.sh new file mode 100755 index 000000000..e9d2d4d5d --- /dev/null +++ b/java-api-examples/run-non-streaming-tts-kokoro-en-lexicon.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html +# to download more models +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + NonStreamingTtsKokoroEnLexicon.java diff --git a/java-api-examples/run-non-streaming-tts-kokoro-en.sh b/java-api-examples/run-non-streaming-tts-kokoro-en-phonemize.sh similarity index 81% rename from java-api-examples/run-non-streaming-tts-kokoro-en.sh rename to java-api-examples/run-non-streaming-tts-kokoro-en-phonemize.sh index a09ca0cfa..4737a73fb 100755 --- a/java-api-examples/run-non-streaming-tts-kokoro-en.sh +++ b/java-api-examples/run-non-streaming-tts-kokoro-en-phonemize.sh @@ -13,7 +13,10 @@ if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then rm kokoro-en-v0_19.tar.bz2 fi + +source ./setup-phonemize.sh + java \ -Dsherpa_onnx.native.path=$PWD/../build/lib \ - -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ - NonStreamingTtsKokoroEn.java + -cp "$PHONEMIZE_CLASSPATH" \ + NonStreamingTtsKokoroEnPhonemize.java diff --git a/java-api-examples/run-non-streaming-tts-kokoro-zh-en.sh b/java-api-examples/run-non-streaming-tts-kokoro-zh-en-lexicon.sh similarity index 92% rename from java-api-examples/run-non-streaming-tts-kokoro-zh-en.sh rename to java-api-examples/run-non-streaming-tts-kokoro-zh-en-lexicon.sh index 07b8e786b..708df4af9 100755 --- a/java-api-examples/run-non-streaming-tts-kokoro-zh-en.sh +++ b/java-api-examples/run-non-streaming-tts-kokoro-zh-en-lexicon.sh @@ -16,4 +16,4 @@ fi java \ -Dsherpa_onnx.native.path=$PWD/../build/lib \ -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ - NonStreamingTtsKokoroZhEn.java + NonStreamingTtsKokoroZhEnLexicon.java diff --git a/java-api-examples/run-non-streaming-tts-matcha-en-lexicon.sh b/java-api-examples/run-non-streaming-tts-matcha-en-lexicon.sh new file mode 100755 index 000000000..93fb90fa0 --- /dev/null +++ b/java-api-examples/run-non-streaming-tts-matcha-en-lexicon.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +# please visit +# https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +# to download more models +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./vocos-22khz-univ.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + NonStreamingTtsMatchaEnLexicon.java diff --git a/java-api-examples/run-non-streaming-tts-matcha-en.sh b/java-api-examples/run-non-streaming-tts-matcha-en-phonemize.sh similarity index 88% rename from java-api-examples/run-non-streaming-tts-matcha-en.sh rename to java-api-examples/run-non-streaming-tts-matcha-en-phonemize.sh index acbfc44e8..afed9e867 100755 --- a/java-api-examples/run-non-streaming-tts-matcha-en.sh +++ b/java-api-examples/run-non-streaming-tts-matcha-en-phonemize.sh @@ -18,7 +18,10 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi + +source ./setup-phonemize.sh + java \ -Dsherpa_onnx.native.path=$PWD/../build/lib \ - -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ - NonStreamingTtsMatchaEn.java + -cp "$PHONEMIZE_CLASSPATH" \ + NonStreamingTtsMatchaEnPhonemize.java diff --git a/java-api-examples/run-non-streaming-tts-matcha-zh.sh b/java-api-examples/run-non-streaming-tts-matcha-zh-lexicon.sh similarity index 95% rename from java-api-examples/run-non-streaming-tts-matcha-zh.sh rename to java-api-examples/run-non-streaming-tts-matcha-zh-lexicon.sh index 2e3f0fccb..1754f1ea2 100755 --- a/java-api-examples/run-non-streaming-tts-matcha-zh.sh +++ b/java-api-examples/run-non-streaming-tts-matcha-zh-lexicon.sh @@ -20,4 +20,4 @@ fi java \ -Dsherpa_onnx.native.path=$PWD/../build/lib \ -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ - NonStreamingTtsMatchaZh.java + NonStreamingTtsMatchaZhLexicon.java diff --git a/java-api-examples/run-non-streaming-tts-piper-en-lexicon.sh b/java-api-examples/run-non-streaming-tts-piper-en-lexicon.sh new file mode 100755 index 000000000..1e25e9f0c --- /dev/null +++ b/java-api-examples/run-non-streaming-tts-piper-en-lexicon.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +# please visit +# https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +# to download more models +if [ ! -f ./vits-piper-en_GB-cori-medium/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_GB-cori-medium.tar.bz2 + tar xf vits-piper-en_GB-cori-medium.tar.bz2 + rm vits-piper-en_GB-cori-medium.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + NonStreamingTtsPiperEnLexicon.java diff --git a/java-api-examples/run-non-streaming-tts-piper-en.sh b/java-api-examples/run-non-streaming-tts-piper-en-phonemize.sh similarity index 82% rename from java-api-examples/run-non-streaming-tts-piper-en.sh rename to java-api-examples/run-non-streaming-tts-piper-en-phonemize.sh index 31f0465b0..a896873b4 100755 --- a/java-api-examples/run-non-streaming-tts-piper-en.sh +++ b/java-api-examples/run-non-streaming-tts-piper-en-phonemize.sh @@ -13,7 +13,10 @@ if [ ! -f ./vits-piper-en_GB-cori-medium/tokens.txt ]; then rm vits-piper-en_GB-cori-medium.tar.bz2 fi + +source ./setup-phonemize.sh + java \ -Dsherpa_onnx.native.path=$PWD/../build/lib \ - -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ - NonStreamingTtsPiperEn.java + -cp "$PHONEMIZE_CLASSPATH" \ + NonStreamingTtsPiperEnPhonemize.java diff --git a/java-api-examples/run-non-streaming-tts-piper-en-with-callback-lexicon.sh b/java-api-examples/run-non-streaming-tts-piper-en-with-callback-lexicon.sh new file mode 100755 index 000000000..04b70a21d --- /dev/null +++ b/java-api-examples/run-non-streaming-tts-piper-en-with-callback-lexicon.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +# please visit +# https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +# to download more models +if [ ! -f ./vits-piper-en_GB-cori-medium/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_GB-cori-medium.tar.bz2 + tar xf vits-piper-en_GB-cori-medium.tar.bz2 + rm vits-piper-en_GB-cori-medium.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + NonStreamingTtsPiperEnWithCallbackLexicon.java diff --git a/java-api-examples/run-non-streaming-tts-piper-en-with-callback.sh b/java-api-examples/run-non-streaming-tts-piper-en-with-callback-phonemize.sh similarity index 81% rename from java-api-examples/run-non-streaming-tts-piper-en-with-callback.sh rename to java-api-examples/run-non-streaming-tts-piper-en-with-callback-phonemize.sh index f0b3e72aa..c56e6fe33 100755 --- a/java-api-examples/run-non-streaming-tts-piper-en-with-callback.sh +++ b/java-api-examples/run-non-streaming-tts-piper-en-with-callback-phonemize.sh @@ -13,7 +13,10 @@ if [ ! -f ./vits-piper-en_GB-cori-medium/tokens.txt ]; then rm vits-piper-en_GB-cori-medium.tar.bz2 fi + +source ./setup-phonemize.sh + java \ -Dsherpa_onnx.native.path=$PWD/../build/lib \ - -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ - NonStreamingTtsPiperEnWithCallback.java + -cp "$PHONEMIZE_CLASSPATH" \ + NonStreamingTtsPiperEnWithCallbackPhonemize.java diff --git a/java-api-examples/run-non-streaming-tts-vits-de-phonemize.sh b/java-api-examples/run-non-streaming-tts-vits-de-phonemize.sh new file mode 100755 index 000000000..1903e19f4 --- /dev/null +++ b/java-api-examples/run-non-streaming-tts-vits-de-phonemize.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +# please visit +# https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +# to download more models +# +# There is no German lexicon, so this model is phonemize-only. +if [ ! -f ./vits-piper-de_DE-glados-low/de_DE-glados-low.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-glados-low.tar.bz2 + tar xf vits-piper-de_DE-glados-low.tar.bz2 + rm vits-piper-de_DE-glados-low.tar.bz2 +fi + +source ./setup-phonemize.sh + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp "$PHONEMIZE_CLASSPATH" \ + NonStreamingTtsVitsDePhonemize.java diff --git a/java-api-examples/run-non-streaming-tts-vits-zh.sh b/java-api-examples/run-non-streaming-tts-vits-zh-lexicon.sh similarity index 93% rename from java-api-examples/run-non-streaming-tts-vits-zh.sh rename to java-api-examples/run-non-streaming-tts-vits-zh-lexicon.sh index 6ecc61900..17d37ee80 100755 --- a/java-api-examples/run-non-streaming-tts-vits-zh.sh +++ b/java-api-examples/run-non-streaming-tts-vits-zh-lexicon.sh @@ -16,4 +16,4 @@ fi java \ -Dsherpa_onnx.native.path=$PWD/../build/lib \ -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ - NonStreamingTtsVitsZh.java + NonStreamingTtsVitsZhLexicon.java diff --git a/java-api-examples/run-non-streaming-zipformer-ctc-hlg.sh b/java-api-examples/run-non-streaming-zipformer-ctc-hlg.sh new file mode 100755 index 000000000..48515de57 --- /dev/null +++ b/java-api-examples/run-non-streaming-zipformer-ctc-hlg.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -ex + +source ./setup.sh + +if [ ! -f ./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + + tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +fi + +java \ + -Dsherpa_onnx.native.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ + NonStreamingZipformerCtcHlg.java diff --git a/java-api-examples/run-zipvoice-tts.sh b/java-api-examples/run-zipvoice-tts-lexicon.sh similarity index 96% rename from java-api-examples/run-zipvoice-tts.sh rename to java-api-examples/run-zipvoice-tts-lexicon.sh index 49647ecf5..8d3696d30 100755 --- a/java-api-examples/run-zipvoice-tts.sh +++ b/java-api-examples/run-zipvoice-tts-lexicon.sh @@ -20,4 +20,4 @@ fi java \ -Dsherpa_onnx.native.path=$PWD/../build/lib \ -cp ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar \ - ZipVoiceTts.java + ZipVoiceTtsLexicon.java diff --git a/java-api-examples/setup-phonemize.sh b/java-api-examples/setup-phonemize.sh new file mode 100755 index 000000000..6f3224caa --- /dev/null +++ b/java-api-examples/setup-phonemize.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Common setup for the piper-phonemize based TTS examples. +# Source this file after ./setup.sh in each run-*-phonemize.sh script: +# source ./setup.sh +# source ./setup-phonemize.sh +# +# It downloads the piper-phonemize JVM jar and the native jar for the current +# platform from JitPack and sets PIPER_CLASSPATH for you. The espeak-ng-data +# used by the phonemizer is embedded in those jars; it only feeds piper +# phonemize, not sherpa-onnx. + +PIPER_PHONEMIZE_VERSION=v1.5.8 +PIPER_BASE_URL=https://jitpack.io/com/github/csukuangfj/piper-phonemize + +# The native jar must match the JVM's architecture, not the shell's: Git +# Bash on Windows ARM reports x86_64 while the JVM is aarch64. This mirrors +# LibraryUtils.getOsArch() so the jar's internal resource path is found. +jvm_props=$(java -XshowSettings:properties -version 2>&1) +jvm_os=$(printf '%s\n' "$jvm_props" | sed -n 's/.*os\.name = //p' | head -1 | + tr '[:upper:]' '[:lower:]') +jvm_arch=$(printf '%s\n' "$jvm_props" | sed -n 's/.*os\.arch = //p' | head -1 | + tr '[:upper:]' '[:lower:]') + +case "$jvm_os" in + *mac* | *darwin*) piper_os=osx ;; + *win*) piper_os=win ;; + *nux*) piper_os=linux ;; + *) + echo "Unsupported OS: $jvm_os" >&2 + exit 1 + ;; +esac + +case "$jvm_arch" in + amd64* | x86_64*) piper_arch=x64 ;; + aarch64* | arm64*) + if [ "$piper_os" = "win" ]; then piper_arch=arm64; else piper_arch=aarch64; fi + ;; + x86*) piper_arch=x86 ;; + arm*) piper_arch=arm ;; + *) + echo "Unsupported architecture: $jvm_arch" >&2 + exit 1 + ;; +esac + +piper_plat="$piper_os-$piper_arch" + +piper_jvm_jar=piper-phonemize-jvm-$PIPER_PHONEMIZE_VERSION.jar +piper_native_jar=piper-phonemize-native-lib-$piper_plat-$PIPER_PHONEMIZE_VERSION.jar + +if [ ! -f ./$piper_jvm_jar ]; then + curl -fSL -O $PIPER_BASE_URL/piper-phonemize-jvm/$PIPER_PHONEMIZE_VERSION/$piper_jvm_jar +fi + +if [ ! -f ./$piper_native_jar ]; then + curl -fSL -O \ + $PIPER_BASE_URL/piper-phonemize-native-lib-$piper_plat/$PIPER_PHONEMIZE_VERSION/$piper_native_jar +fi + +case "$jvm_os" in + *win*) piper_sep=';' ;; + *) piper_sep=':' ;; +esac + +# The sherpa-onnx jar name contains a version glob; expand it here. +sherpa_jar=$(ls ../sherpa-onnx/java-api/target/sherpa-onnx-jvm-*.jar) + +# Complete classpath for the phonemize examples: sherpa-onnx + piper-phonemize +export PHONEMIZE_CLASSPATH="$sherpa_jar$piper_sep$piper_jvm_jar$piper_sep$piper_native_jar" diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh index 0a85158d5..0edecbf1b 100755 --- a/kotlin-api-examples/run.sh +++ b/kotlin-api-examples/run.sh @@ -39,7 +39,7 @@ function testVersion() { function testPocketTts() { if [ ! -f ./sherpa-onnx-pocket-tts-int8-2026-01-26/encoder.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 tar xvf sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 rm sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 fi @@ -57,20 +57,20 @@ function testPocketTts() { java -Djava.library.path=../build/lib -jar $out_filename } -function testZipVoiceTts() { +function testZipVoiceTtsLexicon() { if [ ! -f ./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 tar xvf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 fi if [ ! -f ./vocos_24khz.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx fi - out_filename=test_zipvoice_tts.jar + out_filename=test_zipvoice_tts_lexicon.jar kotlinc-jvm -include-runtime -d $out_filename \ - test_zipvoice_tts.kt \ + test_zipvoice_tts_lexicon.kt \ Tts.kt \ WaveReader.kt \ faked-asset-manager.kt \ @@ -83,7 +83,7 @@ function testZipVoiceTts() { function testSupertonicTts() { if [ ! -f ./sherpa-onnx-supertonic-3-tts-int8-2026-05-11/duration_predictor.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 tar xvf sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 rm sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 fi @@ -103,19 +103,19 @@ function testSupertonicTts() { function testSpeakerEmbeddingExtractor() { if [ ! -f ./3dspeaker_speech_eres2net_large_sv_zh-cn_3dspeaker_16k.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_large_sv_zh-cn_3dspeaker_16k.onnx + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_large_sv_zh-cn_3dspeaker_16k.onnx fi if [ ! -f ./speaker1_a_cn_16k.wav ]; then - curl -SL -O https://github.com/csukuangfj/sr-data/raw/main/test/3d-speaker/speaker1_a_cn_16k.wav + curl -fSL -O https://github.com/csukuangfj/sr-data/raw/main/test/3d-speaker/speaker1_a_cn_16k.wav fi if [ ! -f ./speaker1_b_cn_16k.wav ]; then - curl -SL -O https://github.com/csukuangfj/sr-data/raw/main/test/3d-speaker/speaker1_b_cn_16k.wav + curl -fSL -O https://github.com/csukuangfj/sr-data/raw/main/test/3d-speaker/speaker1_b_cn_16k.wav fi if [ ! -f ./speaker2_a_cn_16k.wav ]; then - curl -SL -O https://github.com/csukuangfj/sr-data/raw/main/test/3d-speaker/speaker2_a_cn_16k.wav + curl -fSL -O https://github.com/csukuangfj/sr-data/raw/main/test/3d-speaker/speaker2_a_cn_16k.wav fi out_filename=test_speaker_id.jar @@ -136,7 +136,7 @@ function testSpeakerEmbeddingExtractor() { function testOnlineAsr() { if [ ! -f ./sherpa-onnx-streaming-t-one-russian-2025-09-08/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-t-one-russian-2025-09-08.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-t-one-russian-2025-09-08.tar.bz2 tar xvf sherpa-onnx-streaming-t-one-russian-2025-09-08.tar.bz2 rm sherpa-onnx-streaming-t-one-russian-2025-09-08.tar.bz2 fi @@ -147,19 +147,19 @@ function testOnlineAsr() { fi if [ ! -f ./sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2 tar xvf sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2 rm sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-80ms.tar.bz2 fi if [ ! -d ./sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13 ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2 tar xvf sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2 rm sherpa-onnx-streaming-zipformer-ctc-multi-zh-hans-2023-12-13.tar.bz2 fi if [ ! -d ./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18 ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 tar xvf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 rm sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 fi @@ -181,44 +181,109 @@ function testOnlineAsr() { java -Djava.library.path=../build/lib -jar $out_filename } -function testTts() { +function setupPhonemize() { + # Download the piper-phonemize JVM jar and the native jar for this platform. + # The espeak-ng-data used by the phonemizer is embedded in those jars; it + # only feeds piper-phonemize, not sherpa-onnx. + PIPER_PHONEMIZE_VERSION=v1.5.7 + PIPER_BASE_URL=https://jitpack.io/com/github/csukuangfj/piper-phonemize + + # The native jar must match the JVM's architecture, not the shell's: Git + # Bash on Windows ARM reports x86_64 while the JVM is aarch64. This mirrors + # LibraryUtils.getOsArch() so the jar's internal resource path is found. + jvm_props=$(java -XshowSettings:properties -version 2>&1) + jvm_os=$(printf '%s\n' "$jvm_props" | sed -n 's/.*os\.name = //p' | head -1 | + tr '[:upper:]' '[:lower:]') + jvm_arch=$(printf '%s\n' "$jvm_props" | sed -n 's/.*os\.arch = //p' | head -1 | + tr '[:upper:]' '[:lower:]') + + case "$jvm_os" in + *mac* | *darwin*) piper_os=osx ;; + *win*) piper_os=win ;; + *nux*) piper_os=linux ;; + *) + echo "Unsupported OS: $jvm_os" >&2 + exit 1 + ;; + esac + + case "$jvm_arch" in + amd64* | x86_64*) piper_arch=x64 ;; + aarch64* | arm64*) + if [ "$piper_os" = "win" ]; then piper_arch=arm64; else piper_arch=aarch64; fi + ;; + x86*) piper_arch=x86 ;; + arm*) piper_arch=arm ;; + *) + echo "Unsupported architecture: $jvm_arch" >&2 + exit 1 + ;; + esac + + piper_plat="$piper_os-$piper_arch" + + PIPER_JVM_JAR=piper-phonemize-jvm-$PIPER_PHONEMIZE_VERSION.jar + piper_native_jar=piper-phonemize-native-lib-$piper_plat-$PIPER_PHONEMIZE_VERSION.jar + + if [ ! -f ./$PIPER_JVM_JAR ]; then + curl -fSL -O $PIPER_BASE_URL/piper-phonemize-jvm/$PIPER_PHONEMIZE_VERSION/$PIPER_JVM_JAR + fi + + if [ ! -f ./$piper_native_jar ]; then + curl -fSL -O \ + $PIPER_BASE_URL/piper-phonemize-native-lib-$piper_plat/$PIPER_PHONEMIZE_VERSION/$piper_native_jar + fi + + case "$jvm_os" in + *win*) piper_sep=';' ;; + *) piper_sep=':' ;; + esac + + export PIPER_CLASSPATH="$PIPER_JVM_JAR$piper_sep$piper_native_jar" +} + +function testTtsLexicon() { if [ ! -f ./vits-piper-en_US-amy-low/en_US-amy-low.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 rm vits-piper-en_US-amy-low.tar.bz2 fi if [ ! -f ./matcha-icefall-zh-baker/model-steps-3.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 tar xvf matcha-icefall-zh-baker.tar.bz2 rm matcha-icefall-zh-baker.tar.bz2 fi if [ ! -f ./vocos-22khz-univ.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi if [ ! -f ./kokoro-multi-lang-v1_0/model.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 tar xf kokoro-multi-lang-v1_0.tar.bz2 rm kokoro-multi-lang-v1_0.tar.bz2 fi if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 fi if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 fi - out_filename=test_tts.jar + if [ ! -f ./lexicon-en-us.txt ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + fi + + out_filename=test_tts_lexicon.jar kotlinc-jvm -include-runtime -d $out_filename \ - test_tts.kt \ + test_tts_lexicon.kt \ Tts.kt \ faked-asset-manager.kt \ faked-log.kt @@ -228,10 +293,50 @@ function testTts() { java -Djava.library.path=../build/lib -jar $out_filename } +function testTtsPhonemize() { + if [ ! -f ./vits-piper-en_US-amy-low/en_US-amy-low.onnx ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + tar xf vits-piper-en_US-amy-low.tar.bz2 + rm vits-piper-en_US-amy-low.tar.bz2 + fi + + if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 + fi + + if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 + tar xf kitten-nano-en-v0_1-fp16.tar.bz2 + rm kitten-nano-en-v0_1-fp16.tar.bz2 + fi + + if [ ! -f ./vits-piper-de_DE-glados-low/de_DE-glados-low.onnx ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-glados-low.tar.bz2 + tar xf vits-piper-de_DE-glados-low.tar.bz2 + rm vits-piper-de_DE-glados-low.tar.bz2 + fi + + setupPhonemize + + out_filename=test_tts_phonemize.jar + kotlinc-jvm -include-runtime -cp "$PIPER_JVM_JAR" -d $out_filename \ + test_tts_phonemize.kt \ + Tts.kt \ + faked-asset-manager.kt \ + faked-log.kt + + ls -lh $out_filename + + java -Djava.library.path=../build/lib \ + -cp "$out_filename:$PIPER_CLASSPATH" \ + com.k2fsa.sherpa.onnx.Test_tts_phonemizeKt +} function testAudioTagging() { if [ ! -d sherpa-onnx-zipformer-audio-tagging-2024-04-09 ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/audio-tagging-models/sherpa-onnx-zipformer-audio-tagging-2024-04-09.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/audio-tagging-models/sherpa-onnx-zipformer-audio-tagging-2024-04-09.tar.bz2 tar xvf sherpa-onnx-zipformer-audio-tagging-2024-04-09.tar.bz2 rm sherpa-onnx-zipformer-audio-tagging-2024-04-09.tar.bz2 fi @@ -253,13 +358,13 @@ function testAudioTagging() { function testSpokenLanguageIdentification() { if [ ! -f ./sherpa-onnx-whisper-tiny/tiny-encoder.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.tar.bz2 tar xvf sherpa-onnx-whisper-tiny.tar.bz2 rm sherpa-onnx-whisper-tiny.tar.bz2 fi if [ ! -f ./spoken-language-identification-test-wavs/ar-arabic.wav ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/spoken-language-identification-test-wavs.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/spoken-language-identification-test-wavs.tar.bz2 tar xvf spoken-language-identification-test-wavs.tar.bz2 rm spoken-language-identification-test-wavs.tar.bz2 fi @@ -280,57 +385,57 @@ function testSpokenLanguageIdentification() { function testOfflineAsr() { if [ ! -f ./sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02/model.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02.tar.bz2 tar xvf sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02.tar.bz2 rm sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02.tar.bz2 ls -lh sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02 fi if [ ! -f ./sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/encoder.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16.tar.bz2 tar xvf sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16.tar.bz2 rm sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16.tar.bz2 ls -lh sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16 fi if [ ! -f ./sherpa-onnx-moonshine-tiny-en-int8/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 tar xvf sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 rm sherpa-onnx-moonshine-tiny-en-int8.tar.bz2 fi if [ ! -f ./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 fi if [ ! -f ./sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2 tar xvf sherpa-onnx-whisper-tiny.en.tar.bz2 rm sherpa-onnx-whisper-tiny.en.tar.bz2 fi if [ ! -f ./sherpa-onnx-nemo-ctc-en-citrinet-512/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 tar xvf sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 rm sherpa-onnx-nemo-ctc-en-citrinet-512.tar.bz2 fi if [ ! -f ./sherpa-onnx-paraformer-zh-2023-09-14/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 tar xvf sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 rm sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 fi if [ ! -f ./sherpa-onnx-zipformer-multi-zh-hans-2023-9-2/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-multi-zh-hans-2023-9-2.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-multi-zh-hans-2023-9-2.tar.bz2 tar xvf sherpa-onnx-zipformer-multi-zh-hans-2023-9-2.tar.bz2 rm sherpa-onnx-zipformer-multi-zh-hans-2023-9-2.tar.bz2 fi if [ ! -f ./sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03/model.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03.tar.bz2 tar xvf sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03.tar.bz2 rm sherpa-onnx-zipformer-ctc-zh-int8-2025-07-03.tar.bz2 @@ -353,17 +458,17 @@ function testOfflineAsr() { function testInverseTextNormalizationOfflineAsr() { if [ ! -f ./sherpa-onnx-paraformer-zh-2023-09-14/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 tar xvf sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 rm sherpa-onnx-paraformer-zh-2023-09-14.tar.bz2 fi if [ ! -f ./itn-zh-number.wav ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn-zh-number.wav + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn-zh-number.wav fi if [ ! -f ./itn_zh_number.fst ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst fi out_filename=test_itn_offline_asr.jar @@ -383,17 +488,17 @@ function testInverseTextNormalizationOfflineAsr() { function testInverseTextNormalizationOnlineAsr() { if [ ! -f ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 rm sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 fi if [ ! -f ./itn-zh-number.wav ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn-zh-number.wav + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn-zh-number.wav fi if [ ! -f ./itn_zh_number.fst ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst fi out_filename=test_itn_online_asr.jar @@ -413,7 +518,7 @@ function testInverseTextNormalizationOnlineAsr() { function testOfflinePunctuation() { if [ ! -f ./sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12/model.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2 tar xvf sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2 rm sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2 fi @@ -452,7 +557,7 @@ function testOfflineDiacritization() { function testOnlinePunctuation() { if [ ! -f ./sherpa-onnx-online-punct-en-2024-08-06/model.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2 tar xvf sherpa-onnx-online-punct-en-2024-08-06.tar.bz2 rm sherpa-onnx-online-punct-en-2024-08-06.tar.bz2 fi @@ -471,17 +576,17 @@ function testOnlinePunctuation() { function testOfflineSpeakerDiarization() { if [ ! -f ./sherpa-onnx-pyannote-segmentation-3-0/model.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 tar xvf sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 rm sherpa-onnx-pyannote-segmentation-3-0.tar.bz2 fi if [ ! -f ./3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx fi if [ ! -f ./0-four-speakers-zh.wav ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/0-four-speakers-zh.wav fi out_filename=test_offline_speaker_diarization.jar @@ -502,15 +607,15 @@ function testOfflineSpeakerDiarization() { function testOfflineSpeechDenoiser() { if [ ! -f ./gtcrn_simple.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/gtcrn_simple.onnx + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/gtcrn_simple.onnx fi if [ ! -f ./dpdfnet_baseline.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/dpdfnet_baseline.onnx + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/dpdfnet_baseline.onnx fi if [ ! -f ./inp_16k.wav ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/inp_16k.wav + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/inp_16k.wav fi out_filename=test_offline_speech_denoiser_gtcrn.jar @@ -544,15 +649,15 @@ function testOfflineSpeechDenoiser() { function testOnlineSpeechDenoiser() { if [ ! -f ./gtcrn_simple.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/gtcrn_simple.onnx + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/gtcrn_simple.onnx fi if [ ! -f ./dpdfnet_baseline.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/dpdfnet_baseline.onnx + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/dpdfnet_baseline.onnx fi if [ ! -f ./inp_16k.wav ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/inp_16k.wav + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/inp_16k.wav fi out_filename=test_online_speech_denoiser.jar @@ -574,19 +679,19 @@ function testOnlineSpeechDenoiser() { function testOfflineSenseVoiceWithHr() { if [ ! -f ./sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 tar xvf sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 rm sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2 fi if [ ! -d dict ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/hr-files/dict.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/hr-files/dict.tar.bz2 tar xf dict.tar.bz2 rm dict.tar.bz2 - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/hr-files/replace.fst - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/hr-files/test-hr.wav - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/hr-files/lexicon.txt + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/hr-files/replace.fst + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/hr-files/test-hr.wav + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/hr-files/lexicon.txt fi out_filename=test_offline_sense_voice_with_hr.jar @@ -606,7 +711,7 @@ function testOfflineSenseVoiceWithHr() { function testOfflineNeMoCanary() { if [ ! -f sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 tar xvf sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 rm sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 fi @@ -629,7 +734,7 @@ function testOfflineNeMoCanary() { function testOfflineOmnilingualAsrCtc() { if [ ! -f sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2 tar xvf sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2 rm sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2 fi @@ -651,7 +756,7 @@ function testOfflineOmnilingualAsrCtc() { function testOfflineMedAsrCtc() { if [ ! -f ./sherpa-onnx-medasr-ctc-en-int8-2025-12-25/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2 tar xvf sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2 rm sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2 fi @@ -673,7 +778,7 @@ function testOfflineMedAsrCtc() { function testOfflineMoonshineAsrV2() { if [ ! -f ./sherpa-onnx-moonshine-tiny-en-quantized-2026-02-27/encoder_model.ort ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-quantized-2026-02-27.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-quantized-2026-02-27.tar.bz2 tar xvf sherpa-onnx-moonshine-tiny-en-quantized-2026-02-27.tar.bz2 rm sherpa-onnx-moonshine-tiny-en-quantized-2026-02-27.tar.bz2 fi @@ -695,7 +800,7 @@ function testOfflineMoonshineAsrV2() { function testOfflineFireRedAsrCtc() { if [ ! -f ./sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25/tokens.txt ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25.tar.bz2 tar xvf sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25.tar.bz2 rm sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25.tar.bz2 fi @@ -717,7 +822,7 @@ function testOfflineFireRedAsrCtc() { function testOfflineFunAsrNano() { if [ ! -f ./sherpa-onnx-funasr-nano-int8-2025-12-30/embedding.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-funasr-nano-int8-2025-12-30.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-funasr-nano-int8-2025-12-30.tar.bz2 tar xvf sherpa-onnx-funasr-nano-int8-2025-12-30.tar.bz2 rm sherpa-onnx-funasr-nano-int8-2025-12-30.tar.bz2 fi @@ -739,7 +844,7 @@ function testOfflineFunAsrNano() { function testOfflineQwen3Asr() { if [ ! -f ./sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25/encoder.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2 tar xvf sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2 rm sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2 fi @@ -761,7 +866,7 @@ function testOfflineQwen3Asr() { function testOfflineCohereTranscribe() { if [ ! -f ./sherpa-onnx-cohere-transcribe-14-lang-int8-2026-04-01/encoder.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-cohere-transcribe-14-lang-int8-2026-04-01.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-cohere-transcribe-14-lang-int8-2026-04-01.tar.bz2 tar xvf sherpa-onnx-cohere-transcribe-14-lang-int8-2026-04-01.tar.bz2 rm sherpa-onnx-cohere-transcribe-14-lang-int8-2026-04-01.tar.bz2 fi @@ -783,7 +888,7 @@ function testOfflineCohereTranscribe() { function testOfflineWenetCtc() { if [ ! -f sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10/model.int8.onnx ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10.tar.bz2 + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10.tar.bz2 tar xvf sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10.tar.bz2 rm sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10.tar.bz2 fi @@ -803,26 +908,50 @@ function testOfflineWenetCtc() { java -Djava.library.path=../build/lib -jar $out_filename } +function testOfflineZipformerCtcHlg() { + if [ ! -f sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx ]; then + curl -fSL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + fi + + out_filename=test_offline_zipformer_ctc_hlg.jar + kotlinc-jvm -include-runtime -d $out_filename \ + test_offline_zipformer_ctc_hlg.kt \ + FeatureConfig.kt \ + QnnConfig.kt \ + HomophoneReplacerConfig.kt \ + OfflineRecognizer.kt \ + OfflineStream.kt \ + WaveReader.kt \ + faked-asset-manager.kt + + ls -lh $out_filename + java -Djava.library.path=../build/lib -jar $out_filename +} + testVersion +testPocketTts +testZipVoiceTtsLexicon +testSupertonicTts +testTtsLexicon +testTtsPhonemize testOfflineSpeakerDiarization testOfflineCohereTranscribe testOfflineQwen3Asr testOfflineMoonshineAsrV2 testOfflineFireRedAsrCtc -testPocketTts -testZipVoiceTts -testSupertonicTts testOfflineFunAsrNano testOfflineMedAsrCtc testOfflineOmnilingualAsrCtc testOfflineWenetCtc +testOfflineZipformerCtcHlg testOfflineNeMoCanary testOfflineSenseVoiceWithHr testOfflineSpeechDenoiser testOnlineSpeechDenoiser testSpeakerEmbeddingExtractor testOnlineAsr -testTts testAudioTagging testSpokenLanguageIdentification testOfflineAsr diff --git a/kotlin-api-examples/test_offline_zipformer_ctc_hlg.kt b/kotlin-api-examples/test_offline_zipformer_ctc_hlg.kt new file mode 100644 index 000000000..f5dab7051 --- /dev/null +++ b/kotlin-api-examples/test_offline_zipformer_ctc_hlg.kt @@ -0,0 +1,46 @@ +package com.k2fsa.sherpa.onnx + +// This file shows how to use a non-streaming Zipformer CTC model with an HLG +// decoding graph via sherpa-onnx's Kotlin API. +// +// The HLG graph is passed to the C API through ctcFstDecoderConfig. + +fun main() { + val recognizer = createOfflineRecognizer() + val waveFilename = "./sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav" + + val waveData = WaveReader.readWaveFromFile( + filename = waveFilename, + ) + + var stream = recognizer.createStream() + stream.acceptWaveform(waveData.samples, sampleRate = waveData.sampleRate) + recognizer.decode(stream) + + var result = recognizer.getResult(stream) + println(result) + + stream.release() + recognizer.release() +} + +fun createOfflineRecognizer(): OfflineRecognizer { + val config = OfflineRecognizerConfig( + modelConfig = OfflineModelConfig( + zipformerCtc = OfflineZipformerCtcModelConfig( + model = "./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx", + ), + tokens = "./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt", + numThreads = 1, + debug = true, + ), + // This is what the example exercises: the graph and maxActive reach the + // C API through ctcFstDecoderConfig. + ctcFstDecoderConfig = OfflineCtcFstDecoderConfig( + graph = "./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst", + maxActive = 3000, + ), + ) + + return OfflineRecognizer(config = config) +} diff --git a/kotlin-api-examples/test_pocket_tts.kt b/kotlin-api-examples/test_pocket_tts.kt index dc1483c95..8e60599b5 100644 --- a/kotlin-api-examples/test_pocket_tts.kt +++ b/kotlin-api-examples/test_pocket_tts.kt @@ -4,26 +4,26 @@ fun main() { testPocketTts() } +// Builds the config through getOfflineTtsConfig so the Pocket branch of that +// helper is exercised end to end, not just the raw data class. fun testPocketTts() { // see https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models - val config = OfflineTtsConfig( - model=OfflineTtsModelConfig( - pocket=OfflineTtsPocketModelConfig( - lmFlow="./sherpa-onnx-pocket-tts-int8-2026-01-26/lm_flow.int8.onnx", - lmMain="./sherpa-onnx-pocket-tts-int8-2026-01-26/lm_main.int8.onnx", - encoder="./sherpa-onnx-pocket-tts-int8-2026-01-26/encoder.onnx", - decoder="./sherpa-onnx-pocket-tts-int8-2026-01-26/decoder.int8.onnx", - textConditioner="./sherpa-onnx-pocket-tts-int8-2026-01-26/text_conditioner.onnx", - vocabJson="./sherpa-onnx-pocket-tts-int8-2026-01-26/vocab.json", - tokenScoresJson="./sherpa-onnx-pocket-tts-int8-2026-01-26/token_scores.json", - ), - numThreads=2, - debug=true, - ), - ) - val tts = OfflineTts(config=config) + val modelDir = "./sherpa-onnx-pocket-tts-int8-2026-01-26" - val referenceAudioFilename = "./sherpa-onnx-pocket-tts-int8-2026-01-26/test_wavs/bria.wav" + val config = getOfflineTtsConfig( + modelDir = modelDir, + isPocket = true, + lmFlow = "lm_flow.int8.onnx", + lmMain = "lm_main.int8.onnx", + pocketEncoder = "encoder.onnx", + pocketDecoder = "decoder.int8.onnx", + textConditioner = "text_conditioner.onnx", + vocabJson = "vocab.json", + tokenScoresJson = "token_scores.json", + ) + val tts = OfflineTts(config = config) + + val referenceAudioFilename = "$modelDir/test_wavs/bria.wav" val wave = WaveReader.readWave( filename = referenceAudioFilename, ) @@ -41,6 +41,10 @@ fun testPocketTts() { val text = "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be, a statesman, a businessman, an official, or a scholar." val audio = tts.generateWithConfigAndCallback(text=text, config=genConfig, callback=::callback) + if (audio.samples.isEmpty()) { + println("FAIL - PocketTTS produced no audio") + System.exit(1) + } audio.save(filename="out-bria.wav") tts.release() println("Saved to out-bria.wav") diff --git a/kotlin-api-examples/test_tts.kt b/kotlin-api-examples/test_tts_lexicon.kt similarity index 86% rename from kotlin-api-examples/test_tts.kt rename to kotlin-api-examples/test_tts_lexicon.kt index 1bb05fe83..d76b6f1a3 100644 --- a/kotlin-api-examples/test_tts.kt +++ b/kotlin-api-examples/test_tts_lexicon.kt @@ -1,3 +1,9 @@ +// This file shows the lexicon-based text frontend for TTS. +// If you want to use piper-phonemize instead, please see +// test_tts_phonemize.kt. +// +// Download https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +// for the English models. package com.k2fsa.sherpa.onnx fun main() { @@ -16,7 +22,6 @@ fun testKokoroZhEn() { model="./kokoro-multi-lang-v1_0/model.onnx", voices="./kokoro-multi-lang-v1_0/voices.bin", tokens="./kokoro-multi-lang-v1_0/tokens.txt", - dataDir="./kokoro-multi-lang-v1_0/espeak-ng-data", lexicon="./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt", ), numThreads=2, @@ -28,9 +33,9 @@ fun testKokoroZhEn() { silenceScale = 0.2f, ) val audio = tts.generateWithConfigAndCallback(text="中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？", config=genConfig, callback=::callback) - audio.save(filename="test-kokoro-zh-en.wav") + audio.save(filename="test-kokoro-zh-en-lexicon.wav") tts.release() - println("Saved to test-kokoro-zh-en.wav") + println("Saved to test-kokoro-zh-en-lexicon.wav") } fun testKokoroEn() { @@ -41,7 +46,7 @@ fun testKokoroEn() { model="./kokoro-en-v0_19/model.onnx", voices="./kokoro-en-v0_19/voices.bin", tokens="./kokoro-en-v0_19/tokens.txt", - dataDir="./kokoro-en-v0_19/espeak-ng-data", + lexicon="./lexicon-en-us.txt", ), numThreads=2, debug=true, @@ -52,9 +57,9 @@ fun testKokoroEn() { silenceScale = 0.2f, ) val audio = tts.generateWithConfigAndCallback(text="How are you doing today?", config=genConfig, callback=::callback) - audio.save(filename="test-kokoro-en.wav") + audio.save(filename="test-kokoro-en-lexicon.wav") tts.release() - println("Saved to test-kokoro-en.wav") + println("Saved to test-kokoro-en-lexicon.wav") } fun testMatcha() { @@ -78,9 +83,9 @@ fun testMatcha() { silenceScale = 0.2f, ) val audio = tts.generateWithConfigAndCallback(text="某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。", config=genConfig, callback=::callback) - audio.save(filename="test-matcha-zh.wav") + audio.save(filename="test-matcha-zh-lexicon.wav") tts.release() - println("Saved to test-matcha-zh.wav") + println("Saved to test-matcha-zh-lexicon.wav") } fun testVits() { @@ -91,7 +96,7 @@ fun testVits() { vits=OfflineTtsVitsModelConfig( model="./vits-piper-en_US-amy-low/en_US-amy-low.onnx", tokens="./vits-piper-en_US-amy-low/tokens.txt", - dataDir="./vits-piper-en_US-amy-low/espeak-ng-data", + lexicon="./lexicon-en-us.txt", ), numThreads=1, debug=true, @@ -102,9 +107,9 @@ fun testVits() { silenceScale = 0.2f, ) val audio = tts.generateWithConfigAndCallback(text="“Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.”", config=genConfig, callback=::callback) - audio.save(filename="test-en.wav") + audio.save(filename="test-piper-en-lexicon.wav") tts.release() - println("Saved to test-en.wav") + println("Saved to test-piper-en-lexicon.wav") } fun testKittenEn() { @@ -115,7 +120,7 @@ fun testKittenEn() { model="./kitten-nano-en-v0_1-fp16/model.fp16.onnx", voices="./kitten-nano-en-v0_1-fp16/voices.bin", tokens="./kitten-nano-en-v0_1-fp16/tokens.txt", - dataDir="./kitten-nano-en-v0_1-fp16/espeak-ng-data", + lexicon="./lexicon-en-us.txt", ), numThreads=2, debug=true, @@ -127,9 +132,9 @@ fun testKittenEn() { silenceScale = 0.2f, ) val audio = tts.generateWithConfigAndCallback(text="How are you doing today?", config=genConfig, callback=::callback) - audio.save(filename="test-kitten-en.wav") + audio.save(filename="test-kitten-en-lexicon.wav") tts.release() - println("Saved to test-kitten-en.wav") + println("Saved to test-kitten-en-lexicon.wav") } /* diff --git a/kotlin-api-examples/test_tts_phonemize.kt b/kotlin-api-examples/test_tts_phonemize.kt new file mode 100644 index 000000000..41a644670 --- /dev/null +++ b/kotlin-api-examples/test_tts_phonemize.kt @@ -0,0 +1,152 @@ +// This file shows the piper-phonemize text frontend for TTS. +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://github.com/csukuangfj/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. +// +// If you want to use a lexicon instead, please see test_tts_lexicon.kt. +package com.k2fsa.sherpa.onnx + +import com.github.csukuangfj.piper.phonemize.PiperPhonemize + +fun main() { + testPiperEnPhonemize() + testKittenEnPhonemize() + testKokoroEnPhonemize() + testVitsDePhonemize() +} + +// Phonemize the text and return the phoneme codepoints, one inner array per +// sentence. The espeak-ng voice is taken from the model metadata; fall back +// to [fallbackVoice] if the model has no language information. +fun phonemize(tts: OfflineTts, text: String, fallbackVoice: String): Array { + var voice = tts.lang() + if (voice.isEmpty()) { + println("Warning: the model has no language information. Falling back to $fallbackVoice.") + voice = fallbackVoice + } + println("Phonemizer language: $voice") + + val result = PiperPhonemize.text(text, voice) + ?: throw RuntimeException("Failed to phonemize the text") + result.use { + return Array(it.numSentences) { i -> it.getPhonemes(i) } + } +} + +fun testPiperEnPhonemize() { + // see https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + PiperPhonemize.initialize() + var config = OfflineTtsConfig( + model=OfflineTtsModelConfig( + vits=OfflineTtsVitsModelConfig( + model="./vits-piper-en_US-amy-low/en_US-amy-low.onnx", + tokens="./vits-piper-en_US-amy-low/tokens.txt", + ), + numThreads=1, + debug=true, + ) + ) + val tts = OfflineTts(config=config) + val text = "“Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.”" + val genConfig = GenerationConfig( + silenceScale = 0.2f, + phonemeCodepoints = phonemize(tts, text, "en-us"), + ) + // The text has already been phonemized; pass an empty string here. + val audio = tts.generateWithConfigAndCallback(text="", config=genConfig, callback=::callback) + audio.save(filename="test-piper-en-phonemize.wav") + tts.release() + println("Saved to test-piper-en-phonemize.wav") +} + +fun testKittenEnPhonemize() { + // see https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + PiperPhonemize.initialize() + var config = OfflineTtsConfig( + model=OfflineTtsModelConfig( + kitten=OfflineTtsKittenModelConfig( + model="./kitten-nano-en-v0_1-fp16/model.fp16.onnx", + voices="./kitten-nano-en-v0_1-fp16/voices.bin", + tokens="./kitten-nano-en-v0_1-fp16/tokens.txt", + ), + numThreads=2, + debug=true, + ), + ) + val tts = OfflineTts(config=config) + val text = "How are you doing today?" + val genConfig = GenerationConfig( + sid = 7, + silenceScale = 0.2f, + phonemeCodepoints = phonemize(tts, text, "en-us"), + ) + val audio = tts.generateWithConfigAndCallback(text="", config=genConfig, callback=::callback) + audio.save(filename="test-kitten-en-phonemize.wav") + tts.release() + println("Saved to test-kitten-en-phonemize.wav") +} + +fun testKokoroEnPhonemize() { + // see https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + PiperPhonemize.initialize() + var config = OfflineTtsConfig( + model=OfflineTtsModelConfig( + kokoro=OfflineTtsKokoroModelConfig( + model="./kokoro-en-v0_19/model.onnx", + voices="./kokoro-en-v0_19/voices.bin", + tokens="./kokoro-en-v0_19/tokens.txt", + ), + numThreads=2, + debug=true, + ), + ) + val tts = OfflineTts(config=config) + val text = "How are you doing today?" + val genConfig = GenerationConfig( + silenceScale = 0.2f, + phonemeCodepoints = phonemize(tts, text, "en-us"), + ) + val audio = tts.generateWithConfigAndCallback(text="", config=genConfig, callback=::callback) + audio.save(filename="test-kokoro-en-phonemize.wav") + tts.release() + println("Saved to test-kokoro-en-phonemize.wav") +} + +fun testVitsDePhonemize() { + // see https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models + // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-glados-low.tar.bz2 + // + // There is no German lexicon, so this model is phonemize-only. + PiperPhonemize.initialize() + var config = OfflineTtsConfig( + model=OfflineTtsModelConfig( + vits=OfflineTtsVitsModelConfig( + model="./vits-piper-de_DE-glados-low/de_DE-glados-low.onnx", + tokens="./vits-piper-de_DE-glados-low/tokens.txt", + ), + numThreads=1, + debug=true, + ) + ) + val tts = OfflineTts(config=config) + val text = "Alles hat ein Ende, nur die Wurst hat zwei." + val genConfig = GenerationConfig( + silenceScale = 0.2f, + phonemeCodepoints = phonemize(tts, text, "de"), + ) + val audio = tts.generateWithConfigAndCallback(text="", config=genConfig, callback=::callback) + audio.save(filename="test-vits-de-phonemize.wav") + tts.release() + println("Saved to test-vits-de-phonemize.wav") +} + +fun callback(samples: FloatArray): Int { + println("callback got called with ${samples.size} samples"); + + // 1 means to continue + // 0 means to stop + return 1 +} diff --git a/kotlin-api-examples/test_zipvoice_tts.kt b/kotlin-api-examples/test_zipvoice_tts_lexicon.kt similarity index 61% rename from kotlin-api-examples/test_zipvoice_tts.kt rename to kotlin-api-examples/test_zipvoice_tts_lexicon.kt index 4033b17c1..a8571a3f4 100644 --- a/kotlin-api-examples/test_zipvoice_tts.kt +++ b/kotlin-api-examples/test_zipvoice_tts_lexicon.kt @@ -4,24 +4,21 @@ fun main() { testZipVoiceTts() } +// Builds the config through getOfflineTtsConfig so the ZipVoice branch of that +// helper is exercised end to end, not just the raw data class. fun testZipVoiceTts() { val modelDir = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia" val referenceAudioFilename = "$modelDir/test_wavs/leijun-1.wav" val wave = WaveReader.readWave(filename = referenceAudioFilename) - val config = OfflineTtsConfig( - model = OfflineTtsModelConfig( - zipvoice = OfflineTtsZipVoiceModelConfig( - tokens = "$modelDir/tokens.txt", - encoder = "$modelDir/encoder.int8.onnx", - decoder = "$modelDir/decoder.int8.onnx", - vocoder = "./vocos_24khz.onnx", - dataDir = "$modelDir/espeak-ng-data", - lexicon = "$modelDir/lexicon.txt", - ), - numThreads = 2, - debug = false, - ), + val config = getOfflineTtsConfig( + modelDir = modelDir, + lexicon = "lexicon.txt", + isZipvoice = true, + zipvoiceEncoder = "encoder.int8.onnx", + zipvoiceDecoder = "decoder.int8.onnx", + // The vocoder is downloaded next to the model dir, not inside it. + zipvoiceVocoder = "./vocos_24khz.onnx", ) val tts = OfflineTts(config = config) @@ -36,9 +33,13 @@ fun testZipVoiceTts() { ) val audio = tts.generateWithConfigAndCallback(text = text, config = genConfig, callback = ::callback) - audio.save(filename = "test-zipvoice-zh-en.wav") + if (audio.samples.isEmpty()) { + println("FAIL - ZipVoice produced no audio") + System.exit(1) + } + audio.save(filename = "test-zipvoice-zh-en-lexicon.wav") tts.release() - println("Saved to test-zipvoice-zh-en.wav") + println("Saved to test-zipvoice-zh-en-lexicon.wav") } fun callback(samples: FloatArray): Int { diff --git a/mfc-examples/NonStreamingSpeechRecognition/sherpa-onnx-deps.props b/mfc-examples/NonStreamingSpeechRecognition/sherpa-onnx-deps.props index 9b35d475d..88bfabf11 100644 --- a/mfc-examples/NonStreamingSpeechRecognition/sherpa-onnx-deps.props +++ b/mfc-examples/NonStreamingSpeechRecognition/sherpa-onnx-deps.props @@ -16,9 +16,6 @@ kaldi-native-fbank-core.lib; kissfft-float.lib; onnxruntime.lib; - piper_phonemize.lib; - espeak-ng.lib; - ucd.lib; ssentencepiece_core.lib; diff --git a/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.cpp b/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.cpp index 626327293..cf415e89b 100644 --- a/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.cpp +++ b/mfc-examples/NonStreamingTextToSpeech/NonStreamingTextToSpeechDlg.cpp @@ -446,8 +446,10 @@ void CNonStreamingTextToSpeechDlg::Init() { error_message += "Cannot find ./tokens.txt\r\n"; ok = false; } - // it is OK to leave lexicon.txt and espeak-ng-data empty - // since models using characters don't need them + // it is OK to leave the lexicon files empty + // since models using characters don't need them. + // Note: espeak-ng is no longer used; text goes through a lexicon (or + // GenerationConfig.phoneme_codepoints / .tokens). if (!ok) { generate_btn_.EnableWindow(FALSE); @@ -541,10 +543,22 @@ void CNonStreamingTextToSpeechDlg::Init() { config.model.kokoro.model = "./model.onnx"; config.model.kokoro.voices = "./voices.bin"; config.model.kokoro.tokens = "./tokens.txt"; - config.model.kokoro.data_dir = "./espeak-ng-data"; - if (Exists("./dict/jieba.dict.utf8") && Exists("./lexicon-zh.txt")) { + // Text frontend is the lexicon; espeak-ng is no longer used. + std::string lex; + if (Exists("./lexicon-us-en.txt")) { + lex = "./lexicon-us-en.txt"; + } + if (Exists("./lexicon-zh.txt")) { + if (!lex.empty()) { + lex += ","; + } + lex += "./lexicon-zh.txt"; + } + if (!lex.empty()) { + config.model.kokoro.lexicon = lex.c_str(); + } + if (Exists("./dict/jieba.dict.utf8")) { config.model.kokoro.dict_dir = "./dict"; - config.model.kokoro.lexicon = "./lexicon-us-en.txt,./lexicon-zh.txt"; } } else if (Exists("./hifigan.onnx") || Exists("./vocos.onnx")) { // it is a matcha tts model @@ -558,10 +572,7 @@ void CNonStreamingTextToSpeechDlg::Init() { config.model.matcha.tokens = "./tokens.txt"; - if (Exists("./espeak-ng-data/phontab")) { - config.model.matcha.data_dir = "./espeak-ng-data"; - } - + // Text frontend is the lexicon; espeak-ng is no longer used. if(Exists("./lexicon.txt")) { config.model.matcha.lexicon = "./lexicon.txt"; } @@ -573,10 +584,8 @@ void CNonStreamingTextToSpeechDlg::Init() { // it is a vits tts model config.model.vits.model = "./model.onnx"; config.model.vits.tokens = "./tokens.txt"; - if (Exists("./espeak-ng-data/phontab")) { - config.model.vits.data_dir = "./espeak-ng-data"; - } + // Text frontend is the lexicon; espeak-ng is no longer used. if (Exists("./lexicon.txt")) { config.model.vits.lexicon = "./lexicon.txt"; } diff --git a/mfc-examples/NonStreamingTextToSpeech/sherpa-onnx-deps.props b/mfc-examples/NonStreamingTextToSpeech/sherpa-onnx-deps.props index 9b35d475d..88bfabf11 100644 --- a/mfc-examples/NonStreamingTextToSpeech/sherpa-onnx-deps.props +++ b/mfc-examples/NonStreamingTextToSpeech/sherpa-onnx-deps.props @@ -16,9 +16,6 @@ kaldi-native-fbank-core.lib; kissfft-float.lib; onnxruntime.lib; - piper_phonemize.lib; - espeak-ng.lib; - ucd.lib; ssentencepiece_core.lib; diff --git a/mfc-examples/StreamingSpeechRecognition/sherpa-onnx-deps.props b/mfc-examples/StreamingSpeechRecognition/sherpa-onnx-deps.props index 9b35d475d..88bfabf11 100644 --- a/mfc-examples/StreamingSpeechRecognition/sherpa-onnx-deps.props +++ b/mfc-examples/StreamingSpeechRecognition/sherpa-onnx-deps.props @@ -16,9 +16,6 @@ kaldi-native-fbank-core.lib; kissfft-float.lib; onnxruntime.lib; - piper_phonemize.lib; - espeak-ng.lib; - ucd.lib; ssentencepiece_core.lib; diff --git a/nodejs-addon-examples/README.md b/nodejs-addon-examples/README.md index 5f004cf40..adc3d73f7 100644 --- a/nodejs-addon-examples/README.md +++ b/nodejs-addon-examples/README.md @@ -173,30 +173,30 @@ The following tables list the examples in this folder. |[./test_tts_non_streaming_pocket_en.js](./test_tts_non_streaming_pocket_en.js)| Zero-shot text-to-speech with a PocketTTS English Model| |[./test_tts_non_streaming_pocket_en_async.js](./test_tts_non_streaming_pocket_en_async.js)| Zero-shot text-to-speech with a PocketTTS English Model using async JS API| |[./test_tts_non_streaming_pocket_en_play_async.js](./test_tts_non_streaming_pocket_en_play_async.js)| Zero-shot text-to-speech with a PocketTTS English Model using async JS API and live audio playback| -|[./test_tts_non_streaming_zipvoice_zh_en.js](./test_tts_non_streaming_zipvoice_zh_en.js)| Zero-shot text-to-speech with a ZipVoice Chinese/English Model| -|[./test_tts_non_streaming_zipvoice_zh_en_async.js](./test_tts_non_streaming_zipvoice_zh_en_async.js)| Zero-shot text-to-speech with a ZipVoice Chinese/English Model using async JS API| -|[./test_tts_non_streaming_zipvoice_zh_en_play_async.js](./test_tts_non_streaming_zipvoice_zh_en_play_async.js)| Zero-shot text-to-speech with a ZipVoice Chinese/English Model using async JS API and live audio playback| -|[./test_tts_non_streaming_kitten_en.js](./test_tts_non_streaming_kitten_en.js)| Text-to-speech with a KittenTTS English Model (async API)| -|[./test_tts_non_streaming_kitten_en_sync.js](./test_tts_non_streaming_kitten_en_sync.js)| Text-to-speech with a KittenTTS English Model (sync API)| -|[./test_tts_non_streaming_kokoro_en.js](./test_tts_non_streaming_kokoro_en.js)| Text-to-speech with a Kokoro English Model (sync API)| -|[./test_tts_non_streaming_kokoro_en_async.js](./test_tts_non_streaming_kokoro_en_async.js)| Text-to-speech with a Kokoro English Model (async API)| -|[./test_tts_non_streaming_kokoro_zh_en.js](./test_tts_non_streaming_kokoro_zh_en.js)| Text-to-speech with a Kokoro Model supporting Chinese and English (sync API)| -|[./test_tts_non_streaming_kokoro_zh_en_async.js](./test_tts_non_streaming_kokoro_zh_en_async.js)| Text-to-speech with a Kokoro Model supporting Chinese and English (async API)| -|[./test_tts_non_streaming_matcha_icefall_en.js](./test_tts_non_streaming_matcha_icefall_en.js)| Text-to-speech with a [MatchaTTS English Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker) (sync API)| -|[./test_tts_non_streaming_matcha_icefall_en_async.js](./test_tts_non_streaming_matcha_icefall_en_async.js)| Text-to-speech with a [MatchaTTS English Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker) (async API)| -|[./test_tts_non_streaming_matcha_icefall_zh.js](./test_tts_non_streaming_matcha_icefall_zh.js)| Text-to-speech with a [MatchaTTS Chinese Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker) (sync API)| -|[./test_tts_non_streaming_matcha_icefall_zh_async.js](./test_tts_non_streaming_matcha_icefall_zh_async.js)| Text-to-speech with a [MatchaTTS Chinese Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker) (async API)| +|[./test_tts_non_streaming_zipvoice_zh_en_lexicon.js](./test_tts_non_streaming_zipvoice_zh_en_lexicon.js)| Zero-shot text-to-speech with a ZipVoice Chinese/English Model| +|[./test_tts_non_streaming_zipvoice_zh_en_lexicon_async.js](./test_tts_non_streaming_zipvoice_zh_en_lexicon_async.js)| Zero-shot text-to-speech with a ZipVoice Chinese/English Model using async JS API| +|[./test_tts_non_streaming_zipvoice_zh_en_lexicon_play_async.js](./test_tts_non_streaming_zipvoice_zh_en_lexicon_play_async.js)| Zero-shot text-to-speech with a ZipVoice Chinese/English Model using async JS API and live audio playback| +|[./test_tts_non_streaming_kitten_en_lexicon.js](./test_tts_non_streaming_kitten_en_lexicon.js)| Text-to-speech with a KittenTTS English Model (async API)| +|[./test_tts_non_streaming_kitten_en_lexicon_sync.js](./test_tts_non_streaming_kitten_en_lexicon_sync.js)| Text-to-speech with a KittenTTS English Model (sync API)| +|[./test_tts_non_streaming_kokoro_en_lexicon.js](./test_tts_non_streaming_kokoro_en_lexicon.js)| Text-to-speech with a Kokoro English Model (sync API)| +|[./test_tts_non_streaming_kokoro_en_lexicon_async.js](./test_tts_non_streaming_kokoro_en_lexicon_async.js)| Text-to-speech with a Kokoro English Model (async API)| +|[./test_tts_non_streaming_kokoro_zh_en_lexicon.js](./test_tts_non_streaming_kokoro_zh_en_lexicon.js)| Text-to-speech with a Kokoro Model supporting Chinese and English (sync API)| +|[./test_tts_non_streaming_kokoro_zh_en_lexicon_async.js](./test_tts_non_streaming_kokoro_zh_en_lexicon_async.js)| Text-to-speech with a Kokoro Model supporting Chinese and English (async API)| +|[./test_tts_non_streaming_matcha_icefall_en_lexicon.js](./test_tts_non_streaming_matcha_icefall_en_lexicon.js)| Text-to-speech with a [MatchaTTS English Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker) (sync API)| +|[./test_tts_non_streaming_matcha_icefall_en_lexicon_async.js](./test_tts_non_streaming_matcha_icefall_en_lexicon_async.js)| Text-to-speech with a [MatchaTTS English Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker) (async API)| +|[./test_tts_non_streaming_matcha_icefall_zh_lexicon.js](./test_tts_non_streaming_matcha_icefall_zh_lexicon.js)| Text-to-speech with a [MatchaTTS Chinese Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker) (sync API)| +|[./test_tts_non_streaming_matcha_icefall_zh_lexicon_async.js](./test_tts_non_streaming_matcha_icefall_zh_lexicon_async.js)| Text-to-speech with a [MatchaTTS Chinese Model](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker) (async API)| |[./test_tts_non_streaming_supertonic_en.js](./test_tts_non_streaming_supertonic_en.js)| Text-to-speech with a Supertonic English Model (sync API)| |[./test_tts_non_streaming_supertonic_en_async.js](./test_tts_non_streaming_supertonic_en_async.js)| Text-to-speech with a Supertonic English Model (async API)| |[./test_tts_non_streaming_supertonic_en_play_async.js](./test_tts_non_streaming_supertonic_en_play_async.js)| Text-to-speech with a Supertonic English Model (async API + live playback)| -|[./test_tts_non_streaming_vits_piper_en.js](./test_tts_non_streaming_vits_piper_en.js)| Text-to-speech with a [piper](https://github.com/rhasspy/piper) English model (sync API)| -|[./test_tts_non_streaming_vits_piper_en_async.js](./test_tts_non_streaming_vits_piper_en_async.js)| Text-to-speech with a [piper](https://github.com/rhasspy/piper) English model (async API)| +|[./test_tts_non_streaming_vits_piper_en_lexicon.js](./test_tts_non_streaming_vits_piper_en_lexicon.js)| Text-to-speech with a [piper](https://github.com/rhasspy/piper) English model (sync API)| +|[./test_tts_non_streaming_vits_piper_en_lexicon_async.js](./test_tts_non_streaming_vits_piper_en_lexicon_async.js)| Text-to-speech with a [piper](https://github.com/rhasspy/piper) English model (async API)| |[./test_tts_non_streaming_vits_coqui_de.js](./test_tts_non_streaming_vits_coqui_de.js)| Text-to-speech with a [coqui](https://github.com/coqui-ai/TTS) German model (sync API)| |[./test_tts_non_streaming_vits_coqui_de_async.js](./test_tts_non_streaming_vits_coqui_de_async.js)| Text-to-speech with a [coqui](https://github.com/coqui-ai/TTS) German model (async API)| -|[./test_tts_non_streaming_vits_zh_ll.js](./test_tts_non_streaming_vits_zh_ll.js)| Text-to-speech with a Chinese model using [cppjieba](https://github.com/yanyiwu/cppjieba) (sync API)| -|[./test_tts_non_streaming_vits_zh_ll_async.js](./test_tts_non_streaming_vits_zh_ll_async.js)| Text-to-speech with a Chinese model using [cppjieba](https://github.com/yanyiwu/cppjieba) (async API)| -|[./test_tts_non_streaming_vits_zh_aishell3.js](./test_tts_non_streaming_vits_zh_aishell3.js)| Text-to-speech with a Chinese TTS model (sync API)| -|[./test_tts_non_streaming_vits_zh_aishell3_async.js](./test_tts_non_streaming_vits_zh_aishell3_async.js)| Text-to-speech with a Chinese TTS model (async API)| +|[./test_tts_non_streaming_vits_zh_ll_lexicon.js](./test_tts_non_streaming_vits_zh_ll_lexicon.js)| Text-to-speech with a Chinese model using [cppjieba](https://github.com/yanyiwu/cppjieba) (sync API)| +|[./test_tts_non_streaming_vits_zh_ll_lexicon_async.js](./test_tts_non_streaming_vits_zh_ll_lexicon_async.js)| Text-to-speech with a Chinese model using [cppjieba](https://github.com/yanyiwu/cppjieba) (async API)| +|[./test_tts_non_streaming_vits_zh_aishell3_lexicon.js](./test_tts_non_streaming_vits_zh_aishell3_lexicon.js)| Text-to-speech with a Chinese TTS model (sync API)| +|[./test_tts_non_streaming_vits_zh_aishell3_lexicon_async.js](./test_tts_non_streaming_vits_zh_aishell3_lexicon_async.js)| Text-to-speech with a Chinese TTS model (async API)| ### Speaker diarization @@ -675,7 +675,7 @@ rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx # The reference text must match the reference audio transcript. -node ./test_tts_non_streaming_zipvoice_zh_en_async.js +node ./test_tts_non_streaming_zipvoice_zh_en_lexicon_async.js ``` ### Zero-shot text-to-speech with ZipVoice models (Chinese/English TTS, async API + playback) @@ -691,7 +691,7 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-mode npm install speaker # The reference text must match the reference audio transcript. -node ./test_tts_non_streaming_zipvoice_zh_en_play_async.js +node ./test_tts_non_streaming_zipvoice_zh_en_lexicon_play_async.js ``` ### Zero-shot text-to-speech with ZipVoice models (Chinese/English TTS, sync API) @@ -704,7 +704,7 @@ rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx # The reference text must match the reference audio transcript. -node ./test_tts_non_streaming_zipvoice_zh_en.js +node ./test_tts_non_streaming_zipvoice_zh_en_lexicon.js ``` ### Text-to-speech with KittenTTS models (English TTS, async API) @@ -714,7 +714,7 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 -node ./test_tts_non_streaming_kitten_en.js +node ./test_tts_non_streaming_kitten_en_lexicon.js ``` ### Text-to-speech with KittenTTS models (English TTS, sync API) @@ -724,7 +724,7 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 -node ./test_tts_non_streaming_kitten_en_sync.js +node ./test_tts_non_streaming_kitten_en_lexicon_sync.js ``` ### Text-to-speech with Supertonic TTS models (English TTS, sync API) @@ -766,7 +766,7 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kokoro-multi-lang-v1_0.tar.bz2 rm kokoro-multi-lang-v1_0.tar.bz2 -node ./test_tts_non_streaming_kokoro_zh_en.js +node ./test_tts_non_streaming_kokoro_zh_en_lexicon.js ``` ### Text-to-speech with Kokoro TTS models (Chinese + English TTS, async API) @@ -776,7 +776,7 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kokoro-multi-lang-v1_0.tar.bz2 rm kokoro-multi-lang-v1_0.tar.bz2 -node ./test_tts_non_streaming_kokoro_zh_en_async.js +node ./test_tts_non_streaming_kokoro_zh_en_lexicon_async.js ``` ### Text-to-speech with Kokoro TTS models (English TTS, sync API) @@ -786,7 +786,7 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 -node ./test_tts_non_streaming_kokoro_en.js +node ./test_tts_non_streaming_kokoro_en_lexicon.js ``` ### Text-to-speech with Kokoro TTS models (English TTS, async API) @@ -796,7 +796,7 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 -node ./test_tts_non_streaming_kokoro_en_async.js +node ./test_tts_non_streaming_kokoro_en_lexicon_async.js ``` ### Text-to-speech with MatchaTTS models (English TTS, sync API) @@ -807,7 +807,7 @@ rm matcha-icefall-en_US-ljspeech.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx -node ./test_tts_non_streaming_matcha_icefall_en.js +node ./test_tts_non_streaming_matcha_icefall_en_lexicon.js ``` ### Text-to-speech with MatchaTTS models (English TTS, async API) @@ -818,7 +818,7 @@ rm matcha-icefall-en_US-ljspeech.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx -node ./test_tts_non_streaming_matcha_icefall_en_async.js +node ./test_tts_non_streaming_matcha_icefall_en_lexicon_async.js ``` ### Text-to-speech with MatchaTTS models (Chinese TTS, sync API) @@ -829,7 +829,7 @@ rm matcha-icefall-zh-baker.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx -node ./test_tts_non_streaming_matcha_icefall_zh.js +node ./test_tts_non_streaming_matcha_icefall_zh_lexicon.js ``` ### Text-to-speech with MatchaTTS models (Chinese TTS, async API) @@ -840,7 +840,7 @@ rm matcha-icefall-zh-baker.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx -node ./test_tts_non_streaming_matcha_icefall_zh_async.js +node ./test_tts_non_streaming_matcha_icefall_zh_lexicon_async.js ``` ### Text-to-speech with piper VITS models (sync API) @@ -850,7 +850,7 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-pip tar xvf vits-piper-en_GB-cori-medium.tar.bz2 rm vits-piper-en_GB-cori-medium.tar.bz2 -node ./test_tts_non_streaming_vits_piper_en.js +node ./test_tts_non_streaming_vits_piper_en_lexicon.js ``` ### Text-to-speech with piper VITS models (async API) @@ -860,7 +860,7 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-pip tar xvf vits-piper-en_GB-cori-medium.tar.bz2 rm vits-piper-en_GB-cori-medium.tar.bz2 -node ./test_tts_non_streaming_vits_piper_en_async.js +node ./test_tts_non_streaming_vits_piper_en_lexicon_async.js ``` ### Text-to-speech with Coqui-ai/TTS VITS models (German, sync API) @@ -890,7 +890,7 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-o tar xvf sherpa-onnx-vits-zh-ll.tar.bz2 rm sherpa-onnx-vits-zh-ll.tar.bz2 -node ./test_tts_non_streaming_vits_zh_ll.js +node ./test_tts_non_streaming_vits_zh_ll_lexicon.js ``` ### Text-to-speech with vits Chinese zh-ll models (async API) @@ -900,7 +900,7 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-o tar xvf sherpa-onnx-vits-zh-ll.tar.bz2 rm sherpa-onnx-vits-zh-ll.tar.bz2 -node ./test_tts_non_streaming_vits_zh_ll_async.js +node ./test_tts_non_streaming_vits_zh_ll_lexicon_async.js ``` ### Text-to-speech with vits Chinese AiShell3 models (sync API) @@ -910,7 +910,7 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-ice tar xvf vits-icefall-zh-aishell3.tar.bz2 rm vits-icefall-zh-aishell3.tar.bz2 -node ./test_tts_non_streaming_vits_zh_aishell3.js +node ./test_tts_non_streaming_vits_zh_aishell3_lexicon.js ``` ### Text-to-speech with vits Chinese AiShell3 models (async API) @@ -920,7 +920,7 @@ wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-ice tar xvf vits-icefall-zh-aishell3.tar.bz2 rm vits-icefall-zh-aishell3.tar.bz2 -node ./test_tts_non_streaming_vits_zh_aishell3_async.js +node ./test_tts_non_streaming_vits_zh_aishell3_lexicon_async.js ``` ### Spoken language identification with Whisper multi-lingual models diff --git a/nodejs-addon-examples/package.json b/nodejs-addon-examples/package.json index 8b1fd9183..7dd121eae 100644 --- a/nodejs-addon-examples/package.json +++ b/nodejs-addon-examples/package.json @@ -1,6 +1,7 @@ { "dependencies": { "node-cpal": "^0.1.1", + "piper-phonemize-node": "^1.5.8", "sherpa-onnx-node": "^1.13.8" } } diff --git a/nodejs-addon-examples/test_asr_non_streaming_zipformer_ctc_hlg.js b/nodejs-addon-examples/test_asr_non_streaming_zipformer_ctc_hlg.js new file mode 100644 index 000000000..1c8ea5356 --- /dev/null +++ b/nodejs-addon-examples/test_asr_non_streaming_zipformer_ctc_hlg.js @@ -0,0 +1,56 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file shows how to use a non-streaming Zipformer CTC model with an HLG +// decoding graph via sherpa-onnx's Node-addon API. +// +// The HLG graph is passed to the C API through ctcFstDecoderConfig. +const sherpa_onnx = require('sherpa-onnx-node'); + +// Please download test files from +// https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +const config = { + 'featConfig': { + 'sampleRate': 16000, + 'featureDim': 80, + }, + 'modelConfig': { + 'zipformerCtc': { + 'model': './sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx', + }, + 'tokens': './sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt', + 'numThreads': 2, + 'provider': 'cpu', + 'debug': 1, + }, + // This is what the example exercises: the graph reaches the C API through + // ctcFstDecoderConfig. + 'ctcFstDecoderConfig': { + 'graph': './sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst', + 'maxActive': 3000, + }, +}; + +const waveFilename = './sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav'; + +const recognizer = new sherpa_onnx.OfflineRecognizer(config); +console.log('Started'); +let start = Date.now(); +const stream = recognizer.createStream(); +const wave = sherpa_onnx.readWave(waveFilename); +stream.acceptWaveform({sampleRate: wave.sampleRate, samples: wave.samples}); + +recognizer.decode(stream); +const result = recognizer.getResult(stream); +let stop = Date.now(); +console.log('Done'); + +const elapsed_seconds = (stop - start) / 1000; +const duration = wave.samples.length / wave.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'seconds'); +console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); +console.log(waveFilename); +console.log('result\n', result); diff --git a/nodejs-addon-examples/test_tts_async_callback_stress.js b/nodejs-addon-examples/test_tts_async_callback_stress_lexicon.js similarity index 97% rename from nodejs-addon-examples/test_tts_async_callback_stress.js rename to nodejs-addon-examples/test_tts_async_callback_stress_lexicon.js index 7f54ff8c3..a67e256a1 100644 --- a/nodejs-addon-examples/test_tts_async_callback_stress.js +++ b/nodejs-addon-examples/test_tts_async_callback_stress_lexicon.js @@ -1,12 +1,14 @@ // Copyright (c) 2026 Kevin Castillo // // Stress test for the async TTS progress-callback path (generateAsync + onProgress). +// Text frontend: lexicon (lexicon-en-us.txt); see +// test_tts_async_callback_stress_phonemize.js for the piper-phonemize variant. // Covers: repeated generation, cancellation via callback return value, a throwing // callback (cancels the generation and rejects the promise with the thrown // error), concurrent generations, and RSS. // // Usage: -// node test_tts_async_callback_stress.js [model-dir] [iterations] +// node test_tts_async_callback_stress_lexicon.js [model-dir] [iterations] // // please download model files from // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models @@ -28,7 +30,7 @@ const makeTts = () => new sherpa_onnx.OfflineTts({ vits: { model: `${modelDir}/${base}.onnx`, tokens: `${modelDir}/tokens.txt`, - dataDir: `${modelDir}/espeak-ng-data`, + lexicon: 'lexicon-en-us.txt', }, numThreads: 2, provider: 'cpu', diff --git a/nodejs-addon-examples/test_tts_async_callback_stress_phonemize.js b/nodejs-addon-examples/test_tts_async_callback_stress_phonemize.js new file mode 100644 index 000000000..84f99119d --- /dev/null +++ b/nodejs-addon-examples/test_tts_async_callback_stress_phonemize.js @@ -0,0 +1,255 @@ +// Copyright (c) 2026 Kevin Castillo +// +// Stress test for the async TTS progress-callback path (generateAsync + onProgress). +// Text frontend: piper-phonemize; see +// test_tts_async_callback_stress_lexicon.js for the lexicon variant. +// Covers: repeated generation, cancellation via callback return value, a throwing +// callback (cancels the generation and rejects the promise with the thrown +// error), concurrent generations, and RSS. +// +// Usage: +// node test_tts_async_callback_stress_phonemize.js [model-dir] [iterations] +// +// please download model files from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// +// The model onnx filename is derived from the directory name (vits-piper-X -> X.onnx), +// so any vits-piper voice works. Short runs (e.g. 30) suit CI; RSS is informational +// unless STRICT_RSS=1 (steady growth with varying shapes is the known ONNX Runtime +// memory-pattern issue #1939, shared with the sync path — not the callback machinery). + +const path = require('path'); +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +const modelDir = process.argv[2] || './vits-piper-en_GB-cori-medium'; +const iterations = Number(process.argv[3] || 100); + +const base = path.basename(path.resolve(modelDir)).replace(/^vits-piper-/, ''); +const makeTts = () => new sherpa_onnx.OfflineTts({ + model: { + vits: { + model: `${modelDir}/${base}.onnx`, + tokens: `${modelDir}/tokens.txt`, + }, + numThreads: 2, + provider: 'cpu', + debug: false, + }, + maxNumSentences: 1, +}); + +const TEXT = + 'Hello there, how are you doing on this fine morning today my dear friend?'; + +// The stress asserts callback counts and isolation, not audio quality, so a +// fixed voice is fine; any piper voice works. +const voice = 'en-us'; +const phonemeCodepoints = piperPhonemize.phonemize(TEXT, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); +const rssMb = () => Math.round(process.memoryUsage().rss / (1024 * 1024)); + +// (A child_process.fork() harness for the throwing case was removed: in a +// forked child — and only there; spawn() is fine — a pending throwing +// callback coincides with a native crash in a later batch's phonemize, +// reproducible on stock v1.13.4 as well. Pre-existing and fork-specific; +// root cause tracked separately from this fix.) + +async function sequential(tts) { + let totalChunks = 0; + let steadyStateBase = 0; + const warmup = Math.min(10, Math.max(1, Math.floor(iterations / 3))); + for (let i = 0; i < iterations; i++) { + let chunks = 0; + // Both requests go through the generationConfig path here: the legacy + // request shape cannot carry phonemeCodepoints. The _lexicon twin + // exercises both native workers. + const req = i % 2 === 0 ? + {text: '', generationConfig: {phonemeCodepoints, sid: 0, speed: 1.0}} : + {text: '', generationConfig: {phonemeCodepoints, sid: 0, speed: 1.0}}; + await tts.generateAsync({ + ...req, + onProgress: () => { + chunks++; + return 1; + }, + }); + // drain-before-settle guarantees every delivered callback ran by now + if (chunks === 0) { + console.error(`FAIL: no progress callbacks fired in iteration ${i}`); + process.exit(1); + } + totalChunks += chunks; + if (i === warmup - 1) steadyStateBase = rssMb(); + } + const growth = rssMb() - steadyStateBase; + console.log(`sequential: ${iterations} calls, ${ + totalChunks} chunks, steady-state RSS growth ${growth} MB`); + if (process.env.STRICT_RSS === '1' && growth > 100) { + console.error('FAIL: steady-state RSS grew > 100 MB'); + process.exit(1); + } +} + +async function cancellation(tts) { + for (const [label, req] of [ + ['withConfig', {text: '', generationConfig: {phonemeCodepoints, sid: 0, speed: 1.0}}], + ['legacy', {text: '', generationConfig: {phonemeCodepoints, sid: 0, speed: 1.0}}], + ]) { + let calls = 0; + const r = await tts.generateAsync({ + ...req, + onProgress: () => { + calls++; + return 0; // cancel on first chunk + }, + }); + await new Promise((res) => setTimeout(res, 200)); + if (calls !== 1) { + console.error( + `FAIL(${label}): expected exactly 1 callback after cancel, got ${calls}`); + process.exit(1); + } + console.log( + `cancellation (${label}): exactly 1 callback, samples: ${r.samples.length}`); + } +} + +async function throwing(tts) { + // Contract: a throwing onProgress must not unwind through the N-API + // boundary; the addon cancels the generation and the promise REJECTS with + // the thrown error. Nothing may reach uncaughtException. + let uncaught = null; + const handler = (err) => { + uncaught = err; + }; + process.on('uncaughtException', handler); + let calls = 0; + let rejection = null; + try { + await tts.generateAsync({ + text: TEXT, + generationConfig: {phonemeCodepoints, sid: 0, speed: 1.0}, + onProgress: () => { + calls++; + throw new Error('callback boom'); + }, + }); + } catch (err) { + rejection = err; + } + await new Promise((res) => setTimeout(res, 200)); + process.removeListener('uncaughtException', handler); + if (uncaught !== null) { + console.error(`FAIL: throwing callback leaked to uncaughtException: ${ + uncaught.message}`); + process.exit(1); + } + if (rejection === null || !(rejection instanceof Error) || + !rejection.message.includes('callback boom')) { + console.error( + `FAIL: expected rejection containing the thrown message, got ${ + rejection === null ? 'resolution' : rejection.message}`); + process.exit(1); + } + if (calls !== 1) { + console.error( + `FAIL: expected exactly 1 callback before cancellation, got ${calls}`); + process.exit(1); + } + console.log( + `throwing callback: cancelled generation, promise rejected with "${ + rejection.message}"`); +} + +async function concurrent(tts) { + // Cancellation is BEST-EFFORT: the producer checks the flag before starting + // the next batch, but batches already synthesized/queued are not rolled + // back, so on fast hardware the cancelled result may still contain most or + // all of the audio. The hard guarantees tested here are callback + // suppression (exactly one invocation) and per-worker isolation (the other + // seven generations run all their batches). Sample counts are logged for + // information only. + const multi = 'First sentence to speak aloud. Second sentence follows here. ' + + 'And a third sentence closes it out.'; + const multiCodepoints = piperPhonemize.phonemize(multi, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + const fullChunkCounts = Array.from({length: 7}, () => 0); + let cancelledCalls = 0; + const jobs = fullChunkCounts.map( + (_, i) => tts.generateAsync({ + text: '', + generationConfig: {phonemeCodepoints: multiCodepoints, sid: 0, speed: 1.0}, + onProgress: () => { + fullChunkCounts[i]++; + return 1; + }, + })); + jobs.push(tts.generateAsync({ + text: '', + generationConfig: {phonemeCodepoints: multiCodepoints, sid: 0, speed: 1.0}, + onProgress: () => { + cancelledCalls++; + return 0; + }, + })); + const results = await Promise.all(jobs); + const full = results.slice(0, 7).map((r) => r.samples.length); + const cancelled = results[7].samples.length; + const minFull = Math.min(...full); + const minChunks = Math.min(...fullChunkCounts); + if (cancelledCalls !== 1) { + console.error(`FAIL: concurrent cancel got ${cancelledCalls} callbacks`); + process.exit(1); + } + if (minChunks < 2) { + console.error(`FAIL: a full generation saw only ${ + minChunks} callbacks — cancellation leaked across workers`); + process.exit(1); + } + console.log(`concurrent: 7 full (min ${minChunks} chunks, min ${ + minFull} samples) + 1 cancelled (${ + cancelled} samples, best-effort truncation), no leakage`); +} + +async function copyBuffer(tts) { + // Exercise the enableExternalBuffer:false marshalling branch + let chunks = 0; + const r = await tts.generateAsync({ + text: TEXT, + generationConfig: {phonemeCodepoints, sid: 0, speed: 1.0}, + enableExternalBuffer: false, + onProgress: () => { + chunks++; + return 1; + }, + }); + if (!r.samples || r.samples.length === 0) { + console.error('FAIL: enableExternalBuffer:false returned no samples'); + process.exit(1); + } + if (chunks === 0) { + console.error('FAIL: no progress callback fired in copy-buffer generation'); + process.exit(1); + } + console.log(`copy-buffer: ${chunks} chunks, ${r.samples.length} samples`); +} + +async function main() { + const tts = makeTts(); + await sequential(tts); + await cancellation(tts); + await throwing(tts); + await copyBuffer(tts); + await concurrent(tts); + console.log('PASS'); +} + +main().catch((err) => { + console.error('FAIL:', err); + process.exit(1); +}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_kitten_en.js b/nodejs-addon-examples/test_tts_non_streaming_kitten_en_lexicon.js similarity index 96% rename from nodejs-addon-examples/test_tts_non_streaming_kitten_en.js rename to nodejs-addon-examples/test_tts_non_streaming_kitten_en_lexicon.js index c862b1c6f..054fe89fd 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_kitten_en.js +++ b/nodejs-addon-examples/test_tts_non_streaming_kitten_en_lexicon.js @@ -14,8 +14,8 @@ async function createOfflineTtsAsync() { kitten: { model: './kitten-nano-en-v0_1-fp16/model.fp16.onnx', voices: './kitten-nano-en-v0_1-fp16/voices.bin', + lexicon: './lexicon-en-us.txt', tokens: './kitten-nano-en-v0_1-fp16/tokens.txt', - dataDir: './kitten-nano-en-v0_1-fp16/espeak-ng-data', }, debug: true, numThreads: 1, @@ -76,7 +76,7 @@ async function main() { `RTF = ${elapsedSeconds.toFixed(3)} / ${durationSeconds.toFixed(3)} =`, realTimeFactor.toFixed(3)); - const filename = 'test-kitten-en-6.wav'; + const filename = 'test-kitten-en-6-lexicon.wav'; sherpa_onnx.writeWave(filename, { samples: audio.samples, sampleRate: audio.sampleRate, diff --git a/nodejs-addon-examples/test_tts_non_streaming_kitten_en_sync.js b/nodejs-addon-examples/test_tts_non_streaming_kitten_en_lexicon_sync.js similarity index 93% rename from nodejs-addon-examples/test_tts_non_streaming_kitten_en_sync.js rename to nodejs-addon-examples/test_tts_non_streaming_kitten_en_lexicon_sync.js index 5675091c1..ea289af40 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_kitten_en_sync.js +++ b/nodejs-addon-examples/test_tts_non_streaming_kitten_en_lexicon_sync.js @@ -10,8 +10,8 @@ function createOfflineTts() { kitten: { model: './kitten-nano-en-v0_1-fp16/model.fp16.onnx', voices: './kitten-nano-en-v0_1-fp16/voices.bin', + lexicon: './lexicon-en-us.txt', tokens: './kitten-nano-en-v0_1-fp16/tokens.txt', - dataDir: './kitten-nano-en-v0_1-fp16/espeak-ng-data', }, debug: true, numThreads: 1, @@ -45,7 +45,7 @@ console.log( `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); -const filename = 'test-kitten-en-sync.wav'; +const filename = 'test-kitten-en-lexicon-sync.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_kitten_en_phonemize.js b/nodejs-addon-examples/test_tts_non_streaming_kitten_en_phonemize.js new file mode 100644 index 000000000..b3bfb8e3b --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_kitten_en_phonemize.js @@ -0,0 +1,108 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test_tts_non_streaming_kitten_en_lexicon.js. + +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +/** + * Create an offline TTS instance asynchronously using the Kitten model. + * + * Model files can be downloaded from: + * https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kitten.html + */ +async function createOfflineTtsAsync() { + const config = { + model: { + kitten: { + model: './kitten-nano-en-v0_1-fp16/model.fp16.onnx', + voices: './kitten-nano-en-v0_1-fp16/voices.bin', + tokens: './kitten-nano-en-v0_1-fp16/tokens.txt', + }, + debug: true, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + + // Use the async factory (non-blocking) + return await sherpa_onnx.OfflineTts.createAsync(config); +} + +async function main() { + // Asynchronously create the OfflineTts instance + const tts = await createOfflineTtsAsync(); + + const text = + 'Today as always, men fall into two groups: slaves and free men. ' + + 'Whoever does not have two-thirds of his day for himself, is a slave, ' + + 'whatever he may be: a statesman, a businessman, an official, or a scholar.'; + + console.log('Number of speakers:', tts.numSpeakers); + console.log('Sample rate:', tts.sampleRate); + + const start = Date.now(); + // The espeak-ng voice is taken from the model metadata. Fall back to + // en-us if the model has no language information. + const voice = tts.lang || 'en-us'; + const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + + const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 6, + speed: 1.0, + silenceScale: 0.2, + }); + + // Asynchronous generation with progress reporting + // The text has already been phonemized; pass an empty string here. + const audio = await tts.generateAsync({ + text: '', + generationConfig, + + // Progress callback receives audio chunks + onProgress({samples, progress}) { + // samples is Float32Array for this chunk + process.stdout.write(`\rGenerating... ${ + (progress * 100).toFixed(1)}% (chunk length: ${samples.length})`); + + // Return 0 or false to cancel, any other value to continue + return true; + }, + }); + + console.log('\nGeneration finished.'); + + const stop = Date.now(); + const elapsedSeconds = (stop - start) / 1000; + const durationSeconds = audio.samples.length / audio.sampleRate; + const realTimeFactor = elapsedSeconds / durationSeconds; + + console.log('Wave duration:', durationSeconds.toFixed(3), 'seconds'); + console.log('Elapsed time:', elapsedSeconds.toFixed(3), 'seconds'); + console.log( + `RTF = ${elapsedSeconds.toFixed(3)} / ${durationSeconds.toFixed(3)} =`, + realTimeFactor.toFixed(3)); + + const filename = 'test-kitten-en-6-phonemize.wav'; + sherpa_onnx.writeWave(filename, { + samples: audio.samples, + sampleRate: audio.sampleRate, + }); + + console.log(`Saved to ${filename}`); +} + +// Run the demo +main().catch((err) => { + console.error('TTS failed:', err); + process.exit(1); +}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_kitten_en_phonemize_sync.js b/nodejs-addon-examples/test_tts_non_streaming_kitten_en_phonemize_sync.js new file mode 100644 index 000000000..b9b2567f3 --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_kitten_en_phonemize_sync.js @@ -0,0 +1,67 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test_tts_non_streaming_kitten_en_lexicon_sync.js. +// +// Synchronous text-to-speech with the Kitten model. +// +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +function createOfflineTts() { + const config = { + model: { + kitten: { + model: './kitten-nano-en-v0_1-fp16/model.fp16.onnx', + voices: './kitten-nano-en-v0_1-fp16/voices.bin', + tokens: './kitten-nano-en-v0_1-fp16/tokens.txt', + }, + debug: true, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return new sherpa_onnx.OfflineTts(config); +} + +const tts = createOfflineTts(); + +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// en-us if the model has no language information. +const voice = tts.lang || 'en-us'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 6, + speed: 1.0, + silenceScale: 0.2, +}); + +let start = Date.now(); +const audio = tts.generate({text, generationConfig}); +let stop = Date.now(); +const elapsed_seconds = (stop - start) / 1000; +const duration = audio.samples.length / audio.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'seconds'); +console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); + +const filename = 'test-kitten-en-phonemize-sync.wav'; +sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + +console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_kokoro_en.js b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_lexicon.js similarity index 94% rename from nodejs-addon-examples/test_tts_non_streaming_kokoro_en.js rename to nodejs-addon-examples/test_tts_non_streaming_kokoro_en_lexicon.js index 76a3d39fe..d78fc3e46 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_kokoro_en.js +++ b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_lexicon.js @@ -10,8 +10,8 @@ function createOfflineTts() { kokoro: { model: './kokoro-en-v0_19/model.onnx', voices: './kokoro-en-v0_19/voices.bin', + lexicon: './lexicon-en-us.txt', tokens: './kokoro-en-v0_19/tokens.txt', - dataDir: './kokoro-en-v0_19/espeak-ng-data', }, debug: true, numThreads: 1, @@ -46,7 +46,7 @@ console.log( `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); -const filename = 'test-kokoro-en-6.wav'; +const filename = 'test-kokoro-en-6-lexicon.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_async.js b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_lexicon_async.js similarity index 95% rename from nodejs-addon-examples/test_tts_non_streaming_kokoro_en_async.js rename to nodejs-addon-examples/test_tts_non_streaming_kokoro_en_lexicon_async.js index 465f3c14e..783d6c7c1 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_async.js +++ b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_lexicon_async.js @@ -10,8 +10,8 @@ async function createOfflineTts() { kokoro: { model: './kokoro-en-v0_19/model.onnx', voices: './kokoro-en-v0_19/voices.bin', + lexicon: './lexicon-en-us.txt', tokens: './kokoro-en-v0_19/tokens.txt', - dataDir: './kokoro-en-v0_19/espeak-ng-data', }, debug: false, numThreads: 1, @@ -58,7 +58,7 @@ async function main() { `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); - const filename = 'test-kokoro-en-async.wav'; + const filename = 'test-kokoro-en-lexicon-async.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_phonemize.js b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_phonemize.js new file mode 100644 index 000000000..b3e1835d7 --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_phonemize.js @@ -0,0 +1,68 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test_tts_non_streaming_kokoro_en_lexicon.js. +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +// please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/kokoro.html +// to download model files +function createOfflineTts() { + const config = { + model: { + kokoro: { + model: './kokoro-en-v0_19/model.onnx', + voices: './kokoro-en-v0_19/voices.bin', + tokens: './kokoro-en-v0_19/tokens.txt', + }, + debug: true, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return new sherpa_onnx.OfflineTts(config); +} + +const tts = createOfflineTts(); + +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// en-us if the model has no language information. +const voice = tts.lang || 'en-us'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 6, + speed: 1.0, + silenceScale: 0.2, +}); + + +let start = Date.now(); +const audio = tts.generate({text, generationConfig}); +let stop = Date.now(); +const elapsed_seconds = (stop - start) / 1000; +const duration = audio.samples.length / audio.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'seconds'); +console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); + +const filename = 'test-kokoro-en-6-phonemize.wav'; +sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + +console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_phonemize_async.js b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_phonemize_async.js new file mode 100644 index 000000000..6eaa11389 --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_kokoro_en_phonemize_async.js @@ -0,0 +1,86 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test_tts_non_streaming_kokoro_en_lexicon_async.js. +// +// Asynchronous text-to-speech with the Kokoro English model. +// +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +async function createOfflineTts() { + const config = { + model: { + kokoro: { + model: './kokoro-en-v0_19/model.onnx', + voices: './kokoro-en-v0_19/voices.bin', + tokens: './kokoro-en-v0_19/tokens.txt', + }, + debug: false, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return await sherpa_onnx.OfflineTts.createAsync(config); +} + +async function main() { + const tts = await createOfflineTts(); + + const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + + // The espeak-ng voice is taken from the model metadata. Fall back to + // en-us if the model has no language information. + const voice = tts.lang || 'en-us'; + const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + + const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 6, + speed: 1.0, + silenceScale: 0.2, + }); + + const start = Date.now(); + // The text has already been phonemized; pass an empty string here. + const audio = await tts.generateAsync({ + text: '', + enableExternalBuffer: true, + generationConfig, + onProgress: ({samples, progress}) => { + process.stdout.write( + `Progress: ${(progress * 100).toFixed(1)}%, ` + + `Samples: ${samples.length}\r`); + return 1; + }, + }); + + console.log(''); + const stop = Date.now(); + const elapsed_seconds = (stop - start) / 1000; + const duration = audio.samples.length / audio.sampleRate; + const real_time_factor = elapsed_seconds / duration; + console.log('Wave duration', duration.toFixed(3), 'seconds'); + console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); + console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); + + const filename = 'test-kokoro-en-phonemize-async.wav'; + sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + console.log(`Saved to ${filename}`); +} + +main().catch((err) => { + console.error('Error:', err); + process.exitCode = 1; +}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en.js b/nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en_lexicon.js similarity index 94% rename from nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en.js rename to nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en_lexicon.js index 761cc67d9..8bdd2d7dd 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en.js +++ b/nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en_lexicon.js @@ -11,7 +11,6 @@ function createOfflineTts() { model: './kokoro-multi-lang-v1_0/model.onnx', voices: './kokoro-multi-lang-v1_0/voices.bin', tokens: './kokoro-multi-lang-v1_0/tokens.txt', - dataDir: './kokoro-multi-lang-v1_0/espeak-ng-data', lexicon: './kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt', }, @@ -47,7 +46,7 @@ console.log( `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); -const filename = 'test-kokoro-zh-en-48.wav'; +const filename = 'test-kokoro-zh-en-48-lexicon.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en_async.js b/nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en_lexicon_async.js similarity index 94% rename from nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en_async.js rename to nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en_lexicon_async.js index 4ad7a84c6..62be0f641 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en_async.js +++ b/nodejs-addon-examples/test_tts_non_streaming_kokoro_zh_en_lexicon_async.js @@ -11,7 +11,6 @@ async function createOfflineTts() { model: './kokoro-multi-lang-v1_0/model.onnx', voices: './kokoro-multi-lang-v1_0/voices.bin', tokens: './kokoro-multi-lang-v1_0/tokens.txt', - dataDir: './kokoro-multi-lang-v1_0/espeak-ng-data', lexicon: './kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt', }, @@ -60,7 +59,7 @@ async function main() { `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); - const filename = 'test-kokoro-zh-en-async.wav'; + const filename = 'test-kokoro-zh-en-lexicon-async.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en.js b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_lexicon.js similarity index 94% rename from nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en.js rename to nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_lexicon.js index 7aa576eaf..4e647784b 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en.js +++ b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_lexicon.js @@ -10,8 +10,8 @@ function createOfflineTts() { matcha: { acousticModel: './matcha-icefall-en_US-ljspeech/model-steps-3.onnx', vocoder: './vocos-22khz-univ.onnx', + lexicon: './lexicon-en-us.txt', tokens: './matcha-icefall-en_US-ljspeech/tokens.txt', - dataDir: './matcha-icefall-en_US-ljspeech/espeak-ng-data', }, debug: true, numThreads: 1, @@ -46,7 +46,7 @@ console.log( `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); -const filename = 'test-matcha-en.wav'; +const filename = 'test-matcha-en-lexicon.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_async.js b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_lexicon_async.js similarity index 94% rename from nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_async.js rename to nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_lexicon_async.js index 6cab837d8..a7d318ae1 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_async.js +++ b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_lexicon_async.js @@ -10,8 +10,8 @@ async function createOfflineTts() { matcha: { acousticModel: './matcha-icefall-en_US-ljspeech/model-steps-3.onnx', vocoder: './vocos-22khz-univ.onnx', + lexicon: './lexicon-en-us.txt', tokens: './matcha-icefall-en_US-ljspeech/tokens.txt', - dataDir: './matcha-icefall-en_US-ljspeech/espeak-ng-data', }, debug: false, numThreads: 1, @@ -58,7 +58,7 @@ async function main() { `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); - const filename = 'test-matcha-en-async.wav'; + const filename = 'test-matcha-en-lexicon-async.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_phonemize.js b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_phonemize.js new file mode 100644 index 000000000..33c3c405c --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_phonemize.js @@ -0,0 +1,68 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test_tts_non_streaming_matcha_icefall_en_lexicon.js. +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +// please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker +// to download model files +function createOfflineTts() { + const config = { + model: { + matcha: { + acousticModel: './matcha-icefall-en_US-ljspeech/model-steps-3.onnx', + vocoder: './vocos-22khz-univ.onnx', + tokens: './matcha-icefall-en_US-ljspeech/tokens.txt', + }, + debug: true, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return new sherpa_onnx.OfflineTts(config); +} + +const tts = createOfflineTts(); + +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// en-us if the model has no language information. +const voice = tts.lang || 'en-us'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 0, + speed: 1.0, + silenceScale: 0.2, +}); + + +let start = Date.now(); +const audio = tts.generate({text, generationConfig}); +let stop = Date.now(); +const elapsed_seconds = (stop - start) / 1000; +const duration = audio.samples.length / audio.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'seconds'); +console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); + +const filename = 'test-matcha-en-phonemize.wav'; +sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + +console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_phonemize_async.js b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_phonemize_async.js new file mode 100644 index 000000000..94290de60 --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_en_phonemize_async.js @@ -0,0 +1,86 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test_tts_non_streaming_matcha_icefall_en_lexicon_async.js. +// +// Asynchronous text-to-speech with the Matcha English model. +// +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +async function createOfflineTts() { + const config = { + model: { + matcha: { + acousticModel: './matcha-icefall-en_US-ljspeech/model-steps-3.onnx', + vocoder: './vocos-22khz-univ.onnx', + tokens: './matcha-icefall-en_US-ljspeech/tokens.txt', + }, + debug: false, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return await sherpa_onnx.OfflineTts.createAsync(config); +} + +async function main() { + const tts = await createOfflineTts(); + + const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + + // The espeak-ng voice is taken from the model metadata. Fall back to + // en-us if the model has no language information. + const voice = tts.lang || 'en-us'; + const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + + const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 0, + speed: 1.0, + silenceScale: 0.2, + }); + + const start = Date.now(); + // The text has already been phonemized; pass an empty string here. + const audio = await tts.generateAsync({ + text: '', + enableExternalBuffer: true, + generationConfig, + onProgress: ({samples, progress}) => { + process.stdout.write( + `Progress: ${(progress * 100).toFixed(1)}%, ` + + `Samples: ${samples.length}\r`); + return 1; + }, + }); + + console.log(''); + const stop = Date.now(); + const elapsed_seconds = (stop - start) / 1000; + const duration = audio.samples.length / audio.sampleRate; + const real_time_factor = elapsed_seconds / duration; + console.log('Wave duration', duration.toFixed(3), 'seconds'); + console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); + console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); + + const filename = 'test-matcha-en-phonemize-async.wav'; + sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + console.log(`Saved to ${filename}`); +} + +main().catch((err) => { + console.error('Error:', err); + process.exitCode = 1; +}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh.js b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh_lexicon.js similarity index 97% rename from nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh.js rename to nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh_lexicon.js index d006cf6cb..35012add0 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh.js +++ b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh_lexicon.js @@ -48,7 +48,7 @@ console.log( `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); -const filename = 'test-matcha-zh.wav'; +const filename = 'test-matcha-zh-lexicon.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh_async.js b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh_lexicon_async.js similarity index 97% rename from nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh_async.js rename to nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh_lexicon_async.js index 4439f0a6a..3536c1382 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh_async.js +++ b/nodejs-addon-examples/test_tts_non_streaming_matcha_icefall_zh_lexicon_async.js @@ -60,7 +60,7 @@ async function main() { `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); - const filename = 'test-matcha-zh-async.wav'; + const filename = 'test-matcha-zh-lexicon-async.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_de_phonemize.js b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_de_phonemize.js new file mode 100644 index 000000000..3680cc5cb --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_de_phonemize.js @@ -0,0 +1,68 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// +// +// There is no German lexicon, so this model is phonemize-only. +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +// please download model files from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +function createOfflineTts() { + const config = { + model: { + vits: { + model: './vits-piper-de_DE-glados-low/de_DE-glados-low.onnx', + tokens: './vits-piper-de_DE-glados-low/tokens.txt', + }, + debug: true, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return new sherpa_onnx.OfflineTts(config); +} + +const tts = createOfflineTts(); + +const text = 'Alles hat ein Ende, nur die Wurst hat zwei.'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// de if the model has no language information. +const voice = tts.lang || 'de'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 0, + speed: 1.0, + silenceScale: 0.2, +}); + + +let start = Date.now(); +// The text has already been phonemized; pass an empty string here. +const audio = tts.generate({text: '', generationConfig}); +let stop = Date.now(); +const elapsed_seconds = (stop - start) / 1000; +const duration = audio.samples.length / audio.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'seconds'); +console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); + +const filename = 'test-vits-de-phonemize.wav'; +sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + +console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_de_phonemize_async.js b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_de_phonemize_async.js new file mode 100644 index 000000000..4dbda6c99 --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_de_phonemize_async.js @@ -0,0 +1,84 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test_tts_non_streaming_vits_piper_en_lexicon_async.js. +// +// Asynchronous text-to-speech with the VITS Piper German model. +// +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +async function createOfflineTts() { + const config = { + model: { + vits: { + model: './vits-piper-de_DE-glados-low/de_DE-glados-low.onnx', + tokens: './vits-piper-de_DE-glados-low/tokens.txt', + }, + debug: false, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return await sherpa_onnx.OfflineTts.createAsync(config); +} + +async function main() { + const tts = await createOfflineTts(); + + const text = 'Alles hat ein Ende, nur die Wurst hat zwei.'; + + // The espeak-ng voice is taken from the model metadata. Fall back to + // de if the model has no language information. + const voice = tts.lang || 'de'; + const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + + const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 0, + speed: 1.0, + silenceScale: 0.2, + }); + + const start = Date.now(); + // The text has already been phonemized; pass an empty string here. + const audio = await tts.generateAsync({ + text: '', + enableExternalBuffer: true, + generationConfig, + onProgress: ({samples, progress}) => { + process.stdout.write( + `Progress: ${(progress * 100).toFixed(1)}%, ` + + `Samples: ${samples.length}\r`); + return 1; + }, + }); + + console.log(''); + const stop = Date.now(); + const elapsed_seconds = (stop - start) / 1000; + const duration = audio.samples.length / audio.sampleRate; + const real_time_factor = elapsed_seconds / duration; + console.log('Wave duration', duration.toFixed(3), 'seconds'); + console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); + console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); + + const filename = 'test-vits-de-phonemize-async.wav'; + sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + console.log(`Saved to ${filename}`); +} + +main().catch((err) => { + console.error('Error:', err); + process.exitCode = 1; +}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en.js b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_lexicon.js similarity index 93% rename from nodejs-addon-examples/test_tts_non_streaming_vits_piper_en.js rename to nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_lexicon.js index a1827706f..41b11d880 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_lexicon.js @@ -8,8 +8,8 @@ function createOfflineTts() { model: { vits: { model: './vits-piper-en_GB-cori-medium/en_GB-cori-medium.onnx', + lexicon: './lexicon-en-us.txt', tokens: './vits-piper-en_GB-cori-medium/tokens.txt', - dataDir: './vits-piper-en_GB-cori-medium/espeak-ng-data', }, debug: true, numThreads: 1, @@ -44,7 +44,7 @@ console.log( `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); -const filename = 'test-piper-en.wav'; +const filename = 'test-piper-en-lexicon.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_async.js b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_lexicon_async.js similarity index 94% rename from nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_async.js rename to nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_lexicon_async.js index 4ecef2625..155dc01eb 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_async.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_lexicon_async.js @@ -9,8 +9,8 @@ async function createOfflineTts() { model: { vits: { model: './vits-piper-en_GB-cori-medium/en_GB-cori-medium.onnx', + lexicon: './lexicon-en-us.txt', tokens: './vits-piper-en_GB-cori-medium/tokens.txt', - dataDir: './vits-piper-en_GB-cori-medium/espeak-ng-data', }, debug: false, numThreads: 1, @@ -57,7 +57,7 @@ async function main() { `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); - const filename = 'test-vits-piper-en-async.wav'; + const filename = 'test-vits-piper-en-lexicon-async.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_phonemize.js b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_phonemize.js new file mode 100644 index 000000000..5705d11cd --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_phonemize.js @@ -0,0 +1,67 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test_tts_non_streaming_vits_piper_en_lexicon.js. +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +// please download model files from +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +function createOfflineTts() { + const config = { + model: { + vits: { + model: './vits-piper-en_GB-cori-medium/en_GB-cori-medium.onnx', + tokens: './vits-piper-en_GB-cori-medium/tokens.txt', + }, + debug: true, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return new sherpa_onnx.OfflineTts(config); +} + +const tts = createOfflineTts(); + +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// en-us if the model has no language information. +const voice = tts.lang || 'en-us'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 0, + speed: 1.0, + silenceScale: 0.2, +}); + + +let start = Date.now(); +// The text has already been phonemized; pass an empty string here. +const audio = tts.generate({text: '', generationConfig}); +let stop = Date.now(); +const elapsed_seconds = (stop - start) / 1000; +const duration = audio.samples.length / audio.sampleRate; +const real_time_factor = elapsed_seconds / duration; +console.log('Wave duration', duration.toFixed(3), 'seconds'); +console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); +console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); + +const filename = 'test-piper-en-phonemize.wav'; +sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + +console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_phonemize_async.js b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_phonemize_async.js new file mode 100644 index 000000000..3bdb67a38 --- /dev/null +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_piper_en_phonemize_async.js @@ -0,0 +1,85 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize-node) and the resulting +// phoneme codepoints are passed via GenerationConfig.phonemeCodepoints. No +// lexicon file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test_tts_non_streaming_vits_piper_en_lexicon_async.js. +// +// Asynchronous text-to-speech with the VITS Piper English model. +// +const sherpa_onnx = require('sherpa-onnx-node'); +const piperPhonemize = require('piper-phonemize-node'); + +async function createOfflineTts() { + const config = { + model: { + vits: { + model: './vits-piper-en_GB-cori-medium/en_GB-cori-medium.onnx', + tokens: './vits-piper-en_GB-cori-medium/tokens.txt', + }, + debug: false, + numThreads: 1, + provider: 'cpu', + }, + maxNumSentences: 1, + }; + return await sherpa_onnx.OfflineTts.createAsync(config); +} + +async function main() { + const tts = await createOfflineTts(); + + const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + + // The espeak-ng voice is taken from the model metadata. Fall back to + // en-us if the model has no language information. + const voice = tts.lang || 'en-us'; + const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + + const generationConfig = new sherpa_onnx.GenerationConfig({ + phonemeCodepoints, + sid: 0, + speed: 1.0, + silenceScale: 0.2, + }); + + const start = Date.now(); + // The text has already been phonemized; pass an empty string here. + const audio = await tts.generateAsync({ + text: '', + enableExternalBuffer: true, + generationConfig, + onProgress: ({samples, progress}) => { + process.stdout.write( + `Progress: ${(progress * 100).toFixed(1)}%, ` + + `Samples: ${samples.length}\r`); + return 1; + }, + }); + + console.log(''); + const stop = Date.now(); + const elapsed_seconds = (stop - start) / 1000; + const duration = audio.samples.length / audio.sampleRate; + const real_time_factor = elapsed_seconds / duration; + console.log('Wave duration', duration.toFixed(3), 'seconds'); + console.log('Elapsed', elapsed_seconds.toFixed(3), 'seconds'); + console.log( + `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); + + const filename = 'test-vits-piper-en-phonemize-async.wav'; + sherpa_onnx.writeWave( + filename, {samples: audio.samples, sampleRate: audio.sampleRate}); + console.log(`Saved to ${filename}`); +} + +main().catch((err) => { + console.error('Error:', err); + process.exitCode = 1; +}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3.js b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3_lexicon.js similarity index 97% rename from nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3.js rename to nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3_lexicon.js index 2fe593ed1..cb63ea55a 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3_lexicon.js @@ -46,7 +46,7 @@ console.log( `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); -const filename = 'test-zh-aishell3.wav'; +const filename = 'test-zh-aishell3-lexicon.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3_async.js b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3_lexicon_async.js similarity index 97% rename from nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3_async.js rename to nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3_lexicon_async.js index e94c8c210..e985405f3 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3_async.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_aishell3_lexicon_async.js @@ -60,7 +60,7 @@ async function main() { `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); - const filename = 'test-vits-zh-aishell3-async.wav'; + const filename = 'test-vits-zh-aishell3-lexicon-async.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll.js b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll_lexicon.js similarity index 97% rename from nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll.js rename to nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll_lexicon.js index 9a2f95604..678d8eebd 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll_lexicon.js @@ -45,7 +45,7 @@ console.log( `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); -const filename = 'test-zh-ll.wav'; +const filename = 'test-zh-ll-lexicon.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll_async.js b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll_lexicon_async.js similarity index 97% rename from nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll_async.js rename to nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll_lexicon_async.js index 6e83e0011..883f5d387 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll_async.js +++ b/nodejs-addon-examples/test_tts_non_streaming_vits_zh_ll_lexicon_async.js @@ -59,7 +59,7 @@ async function main() { `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); - const filename = 'test-vits-zh-ll-async.wav'; + const filename = 'test-vits-zh-ll-lexicon-async.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); console.log(`Saved to ${filename}`); diff --git a/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en.js b/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_lexicon.js similarity index 93% rename from nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en.js rename to nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_lexicon.js index c88b76eb0..a5d1e02c3 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en.js +++ b/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_lexicon.js @@ -14,8 +14,6 @@ function createOfflineTts() { decoder: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx', vocoder: './vocos_24khz.onnx', - dataDir: - './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data', lexicon: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt', }, debug: true, @@ -58,7 +56,7 @@ console.log( `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); -const filename = 'test-zipvoice-zh-en.wav'; +const filename = 'test-zipvoice-zh-en-lexicon.wav'; sherpa_onnx.writeWave( filename, {samples: audio.samples, sampleRate: audio.sampleRate}); diff --git a/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_async.js b/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_lexicon_async.js similarity index 95% rename from nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_async.js rename to nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_lexicon_async.js index 2d72cc6ae..598336e7f 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_async.js +++ b/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_lexicon_async.js @@ -11,8 +11,6 @@ async function createOfflineTts() { decoder: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx', vocoder: './vocos_24khz.onnx', - dataDir: - './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data', lexicon: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt', }, debug: false, // set to true to see verbose logs @@ -85,7 +83,7 @@ async function main() { `RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); - const filename = 'test-zipvoice-zh-en-async.wav'; + const filename = 'test-zipvoice-zh-en-lexicon-async.wav'; sherpa_onnx.writeWave(filename, { samples: audio.samples, sampleRate: audio.sampleRate, diff --git a/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_play_async.js b/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_lexicon_play_async.js similarity index 96% rename from nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_play_async.js rename to nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_lexicon_play_async.js index 4b4a02dd6..94eaf9d7b 100644 --- a/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_play_async.js +++ b/nodejs-addon-examples/test_tts_non_streaming_zipvoice_zh_en_lexicon_play_async.js @@ -15,8 +15,6 @@ async function createOfflineTts() { decoder: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx', vocoder: './vocos_24khz.onnx', - dataDir: - './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data', lexicon: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt', }, debug: false, // set to true to see verbose logs @@ -130,7 +128,7 @@ async function main() { `RTF = ${generationElapsedSeconds.toFixed(3)}/${duration.toFixed(3)} =`, real_time_factor.toFixed(3)); - const filename = 'test-zipvoice-zh-en-play-async.wav'; + const filename = 'test-zipvoice-zh-en-lexicon-play-async.wav'; sherpa_onnx.writeWave(filename, { samples: audio.samples, sampleRate: audio.sampleRate, diff --git a/nodejs-addon-examples/test_version.js b/nodejs-addon-examples/test_version.js new file mode 100644 index 000000000..bc6580891 --- /dev/null +++ b/nodejs-addon-examples/test_version.js @@ -0,0 +1,21 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// Print the sherpa-onnx and onnxruntime versions. +// +// Usage: +// node test_version.js + +const sherpa_onnx = require('sherpa-onnx-node'); + +console.log(`sherpa-onnx version: ${sherpa_onnx.version}`); +console.log(`sherpa-onnx gitSha1: ${sherpa_onnx.gitSha1}`); +console.log(`sherpa-onnx gitDate: ${sherpa_onnx.gitDate}`); +console.log(`onnxruntime version: ${sherpa_onnx.onnxruntimeVersion}`); + +if (typeof sherpa_onnx.version !== 'string' || sherpa_onnx.version.length === 0) { + throw new Error('sherpa_onnx.version is missing or empty'); +} +if (typeof sherpa_onnx.onnxruntimeVersion !== 'string' || + sherpa_onnx.onnxruntimeVersion.length === 0) { + throw new Error('sherpa_onnx.onnxruntimeVersion is missing or empty'); +} diff --git a/nodejs-examples/README.md b/nodejs-examples/README.md index 55c40ac64..bdedf4970 100644 --- a/nodejs-examples/README.md +++ b/nodejs-examples/README.md @@ -106,9 +106,9 @@ node ./test-offline-speaker-diarization.js In the following, we demonstrate how to run text-to-speech. -## ./test-offline-tts-zipvoice-zh-en.js +## ./test-offline-tts-zipvoice-zh-en-lexicon.js -[./test-offline-tts-zipvoice-zh-en.js](./test-offline-tts-zipvoice-zh-en.js) +[./test-offline-tts-zipvoice-zh-en-lexicon.js](./test-offline-tts-zipvoice-zh-en-lexicon.js) shows how to use ZipVoice for Zero-shot TTS in Chinese and English. Please make sure that the reference transcript matches the reference audio. @@ -121,7 +121,7 @@ rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx -node ./test-offline-tts-zipvoice-zh-en.js +node ./test-offline-tts-zipvoice-zh-en-lexicon.js ``` ## ./test-offline-tts-supertonic.js @@ -153,9 +153,9 @@ rm sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 node ./test-offline-tts-pocket-en.js ``` -## ./test-offline-tts-kitten-en.js +## ./test-offline-tts-kitten-en-lexicon.js -[./test-offline-tts-kitten-en.js](./test-offline-tts-kitten-en.js) shows how to use +[./test-offline-tts-kitten-en-lexicon.js](./test-offline-tts-kitten-en-lexicon.js) shows how to use [kitten-nano-en-v0_1-fp16](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2) for text-to-speech. @@ -166,12 +166,12 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 -node ./test-offline-tts-kitten-en.js +node ./test-offline-tts-kitten-en-lexicon.js ``` -## ./test-offline-tts-kokoro-en.js +## ./test-offline-tts-kokoro-en-lexicon.js -[./test-offline-tts-kokoro-en.js](./test-offline-tts-kokoro-en.js) shows how to use +[./test-offline-tts-kokoro-en-lexicon.js](./test-offline-tts-kokoro-en-lexicon.js) shows how to use [kokoro-en-v0_19](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2) for text-to-speech. @@ -182,12 +182,12 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 -node ./test-offline-tts-kokoro-en.js +node ./test-offline-tts-kokoro-en-lexicon.js ``` -## ./test-offline-tts-kokoro-zh-en.js +## ./test-offline-tts-kokoro-zh-en-lexicon.js -[./test-offline-tts-kokoro-zh-en.js](./test-offline-tts-kokoro-zh-en.js) shows how to use +[./test-offline-tts-kokoro-zh-en-lexicon.js](./test-offline-tts-kokoro-zh-en-lexicon.js) shows how to use [kokoro-multi-lang-v1_0](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2) for text-to-speech in Chinese and English. @@ -198,12 +198,12 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kokoro-multi-lang-v1_0.tar.bz2 rm kokoro-multi-lang-v1_0.tar.bz2 -node ./test-offline-tts-kokoro-zh-en.js +node ./test-offline-tts-kokoro-zh-en-lexicon.js ``` -## ./test-offline-tts-matcha-zh.js +## ./test-offline-tts-matcha-zh-lexicon.js -[./test-offline-tts-matcha-zh.js](./test-offline-tts-matcha-zh.js) shows how to use +[./test-offline-tts-matcha-zh-lexicon.js](./test-offline-tts-matcha-zh-lexicon.js) shows how to use [matcha-icefall-zh-baker](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-zh-baker-chinese-1-female-speaker) for text-to-speech. @@ -216,12 +216,12 @@ rm matcha-icefall-zh-baker.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx -node ./test-offline-tts-matcha-zh.js +node ./test-offline-tts-matcha-zh-lexicon.js ``` -## ./test-offline-tts-matcha-en.js +## ./test-offline-tts-matcha-en-lexicon.js -[./test-offline-tts-matcha-en.js](./test-offline-tts-matcha-en.js) shows how to use +[./test-offline-tts-matcha-en-lexicon.js](./test-offline-tts-matcha-en-lexicon.js) shows how to use [matcha-icefall-en_US-ljspeech](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/matcha.html#matcha-icefall-en-us-ljspeech-american-english-1-female-speaker) for text-to-speech. @@ -234,12 +234,12 @@ rm matcha-icefall-en_US-ljspeech.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx -node ./test-offline-tts-matcha-en.js +node ./test-offline-tts-matcha-en-lexicon.js ``` -## ./test-offline-tts-vits-en.js +## ./test-offline-tts-vits-en-lexicon.js -[./test-offline-tts-vits-en.js](./test-offline-tts-vits-en.js) shows how to use +[./test-offline-tts-vits-en-lexicon.js](./test-offline-tts-vits-en-lexicon.js) shows how to use [vits-piper-en_US-amy-low.tar.bz2](https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2) for text-to-speech. @@ -248,12 +248,12 @@ You can use the following command to run it: ```bash wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xvf vits-piper-en_US-amy-low.tar.bz2 -node ./test-offline-tts-vits-en.js +node ./test-offline-tts-vits-en-lexicon.js ``` -## ./test-offline-tts-vits-zh.js +## ./test-offline-tts-vits-zh-lexicon.js -[./test-offline-tts-vits-zh.js](./test-offline-tts-vits-zh.js) shows how to use +[./test-offline-tts-vits-zh-lexicon.js](./test-offline-tts-vits-zh-lexicon.js) shows how to use a VITS pretrained model [aishell3](https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#vits-model-aishell3) for text-to-speech. @@ -263,7 +263,7 @@ You can use the following command to run it: ```bash wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2 tar xvf vits-icefall-zh-aishell3.tar.bz2 -node ./test-offline-tts-vits-zh.js +node ./test-offline-tts-vits-zh-lexicon.js ``` # Speech-to-text diff --git a/nodejs-examples/package.json b/nodejs-examples/package.json index 166a5852b..e795cb449 100644 --- a/nodejs-examples/package.json +++ b/nodejs-examples/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "piper-phonemize": "^1.5.8", "sherpa-onnx": "^1.13.8", "wav": "^1.0.2" } diff --git a/nodejs-examples/test-offline-tts-kitten-en.js b/nodejs-examples/test-offline-tts-kitten-en-lexicon.js similarity index 88% rename from nodejs-examples/test-offline-tts-kitten-en.js rename to nodejs-examples/test-offline-tts-kitten-en-lexicon.js index 06e9b6ca0..c65d7847f 100644 --- a/nodejs-examples/test-offline-tts-kitten-en.js +++ b/nodejs-examples/test-offline-tts-kitten-en-lexicon.js @@ -6,8 +6,8 @@ function createOfflineTts() { let offlineTtsKittenModelConfig = { model: './kitten-nano-en-v0_1-fp16/model.fp16.onnx', voices: './kitten-nano-en-v0_1-fp16/voices.bin', + lexicon: './lexicon-en-us.txt', tokens: './kitten-nano-en-v0_1-fp16/tokens.txt', - dataDir: './kitten-nano-en-v0_1-fp16/espeak-ng-data', lengthScale: 1.0, }; let offlineTtsModelConfig = { @@ -37,6 +37,6 @@ const text = 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; const audio = tts.generateWithConfig(text, generationConfig); -tts.save('./test-kitten-en.wav', audio); -console.log('Saved to test-kitten-en.wav successfully.'); +tts.save('./test-kitten-en-lexicon.wav', audio); +console.log('Saved to test-kitten-en-lexicon.wav successfully.'); tts.free(); diff --git a/nodejs-examples/test-offline-tts-kitten-en-phonemize.js b/nodejs-examples/test-offline-tts-kitten-en-phonemize.js new file mode 100644 index 000000000..1dd878e83 --- /dev/null +++ b/nodejs-examples/test-offline-tts-kitten-en-phonemize.js @@ -0,0 +1,58 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test-offline-tts-kitten-en-lexicon.js. + +const sherpa_onnx = require('sherpa-onnx'); +const piperPhonemize = require('piper-phonemize'); + +function createOfflineTts() { + let offlineTtsKittenModelConfig = { + model: './kitten-nano-en-v0_1-fp16/model.fp16.onnx', + voices: './kitten-nano-en-v0_1-fp16/voices.bin', + tokens: './kitten-nano-en-v0_1-fp16/tokens.txt', + lengthScale: 1.0, + }; + let offlineTtsModelConfig = { + offlineTtsKittenModelConfig: offlineTtsKittenModelConfig, + numThreads: 1, + debug: 1, + provider: 'cpu', + }; + + let offlineTtsConfig = { + offlineTtsModelConfig: offlineTtsModelConfig, + maxNumSentences: 1, + }; + + return sherpa_onnx.createOfflineTts(offlineTtsConfig); +} + +const tts = createOfflineTts(); +const speakerId = 0; +const speed = 1.0; +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// en-us if the model has no language information. +const voice = tts.lang || 'en-us'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = { + phonemeCodepoints: phonemeCodepoints, + sid: speakerId, + speed: speed, + silenceScale: 0.2, +}; +// The text has already been phonemized; pass an empty string here. +const audio = tts.generateWithConfig('', generationConfig); +tts.save('./test-kitten-en-phonemize.wav', audio); +console.log('Saved to test-kitten-en-phonemize.wav successfully.'); +tts.free(); diff --git a/nodejs-examples/test-offline-tts-kokoro-en.js b/nodejs-examples/test-offline-tts-kokoro-en-lexicon.js similarity index 88% rename from nodejs-examples/test-offline-tts-kokoro-en.js rename to nodejs-examples/test-offline-tts-kokoro-en-lexicon.js index 76614249f..f40474ff3 100644 --- a/nodejs-examples/test-offline-tts-kokoro-en.js +++ b/nodejs-examples/test-offline-tts-kokoro-en-lexicon.js @@ -6,8 +6,8 @@ function createOfflineTts() { let offlineTtsKokoroModelConfig = { model: './kokoro-en-v0_19/model.onnx', voices: './kokoro-en-v0_19/voices.bin', + lexicon: './lexicon-en-us.txt', tokens: './kokoro-en-v0_19/tokens.txt', - dataDir: './kokoro-en-v0_19/espeak-ng-data', lengthScale: 1.0, }; let offlineTtsModelConfig = { @@ -37,6 +37,6 @@ const text = 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; const audio = tts.generateWithConfig(text, generationConfig); -tts.save('./test-kokoro-en.wav', audio); -console.log('Saved to test-kokoro-en.wav successfully.'); +tts.save('./test-kokoro-en-lexicon.wav', audio); +console.log('Saved to test-kokoro-en-lexicon.wav successfully.'); tts.free(); diff --git a/nodejs-examples/test-offline-tts-kokoro-en-phonemize.js b/nodejs-examples/test-offline-tts-kokoro-en-phonemize.js new file mode 100644 index 000000000..295f1a464 --- /dev/null +++ b/nodejs-examples/test-offline-tts-kokoro-en-phonemize.js @@ -0,0 +1,58 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test-offline-tts-kokoro-en-lexicon.js. + +const sherpa_onnx = require('sherpa-onnx'); +const piperPhonemize = require('piper-phonemize'); + +function createOfflineTts() { + let offlineTtsKokoroModelConfig = { + model: './kokoro-en-v0_19/model.onnx', + voices: './kokoro-en-v0_19/voices.bin', + tokens: './kokoro-en-v0_19/tokens.txt', + lengthScale: 1.0, + }; + let offlineTtsModelConfig = { + offlineTtsKokoroModelConfig: offlineTtsKokoroModelConfig, + numThreads: 1, + debug: 1, + provider: 'cpu', + }; + + let offlineTtsConfig = { + offlineTtsModelConfig: offlineTtsModelConfig, + maxNumSentences: 1, + }; + + return sherpa_onnx.createOfflineTts(offlineTtsConfig); +} + +const tts = createOfflineTts(); +const speakerId = 0; +const speed = 1.0; +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// en-us if the model has no language information. +const voice = tts.lang || 'en-us'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = { + phonemeCodepoints: phonemeCodepoints, + sid: speakerId, + speed: speed, + silenceScale: 0.2, +}; +// The text has already been phonemized; pass an empty string here. +const audio = tts.generateWithConfig('', generationConfig); +tts.save('./test-kokoro-en-phonemize.wav', audio); +console.log('Saved to test-kokoro-en-phonemize.wav successfully.'); +tts.free(); diff --git a/nodejs-examples/test-offline-tts-kokoro-zh-en.js b/nodejs-examples/test-offline-tts-kokoro-zh-en-lexicon.js similarity index 87% rename from nodejs-examples/test-offline-tts-kokoro-zh-en.js rename to nodejs-examples/test-offline-tts-kokoro-zh-en-lexicon.js index 63bf4f943..7efaa36a9 100644 --- a/nodejs-examples/test-offline-tts-kokoro-zh-en.js +++ b/nodejs-examples/test-offline-tts-kokoro-zh-en-lexicon.js @@ -7,7 +7,6 @@ function createOfflineTts() { model: './kokoro-multi-lang-v1_0/model.onnx', voices: './kokoro-multi-lang-v1_0/voices.bin', tokens: './kokoro-multi-lang-v1_0/tokens.txt', - dataDir: './kokoro-multi-lang-v1_0/espeak-ng-data', lexicon: './kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt', lengthScale: 1.0, @@ -39,6 +38,6 @@ const text = '中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？'; const audio = tts.generateWithConfig(text, generationConfig); -tts.save('./test-kokoro-zh-en-49.wav', audio); -console.log('Saved to test-kokoro-zh-en-49.wav successfully.'); +tts.save('./test-kokoro-zh-en-49-lexicon.wav', audio); +console.log('Saved to test-kokoro-zh-en-49-lexicon.wav successfully.'); tts.free(); diff --git a/nodejs-examples/test-offline-tts-matcha-en.js b/nodejs-examples/test-offline-tts-matcha-en-lexicon.js similarity index 88% rename from nodejs-examples/test-offline-tts-matcha-en.js rename to nodejs-examples/test-offline-tts-matcha-en-lexicon.js index 1878ad1b8..8cdad372f 100644 --- a/nodejs-examples/test-offline-tts-matcha-en.js +++ b/nodejs-examples/test-offline-tts-matcha-en-lexicon.js @@ -8,8 +8,8 @@ function createOfflineTts() { let offlineTtsMatchaModelConfig = { acousticModel: './matcha-icefall-en_US-ljspeech/model-steps-3.onnx', vocoder: './vocos-22khz-univ.onnx', + lexicon: './lexicon-en-us.txt', tokens: './matcha-icefall-en_US-ljspeech/tokens.txt', - dataDir: './matcha-icefall-en_US-ljspeech/espeak-ng-data', noiseScale: 0.667, lengthScale: 1.0, @@ -41,6 +41,6 @@ const generationConfig = { }; const audio = tts.generateWithConfig(text, generationConfig); -tts.save('./test-matcha-en.wav', audio); -console.log('Saved to test-matcha-en.wav successfully.'); +tts.save('./test-matcha-en-lexicon.wav', audio); +console.log('Saved to test-matcha-en-lexicon.wav successfully.'); tts.free(); diff --git a/nodejs-examples/test-offline-tts-matcha-en-phonemize.js b/nodejs-examples/test-offline-tts-matcha-en-phonemize.js new file mode 100644 index 000000000..6add182d7 --- /dev/null +++ b/nodejs-examples/test-offline-tts-matcha-en-phonemize.js @@ -0,0 +1,62 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test-offline-tts-matcha-en-lexicon.js. + +const sherpa_onnx = require('sherpa-onnx'); +const piperPhonemize = require('piper-phonemize'); + +const silenceScale = 0.2; + +function createOfflineTts() { + let offlineTtsMatchaModelConfig = { + acousticModel: './matcha-icefall-en_US-ljspeech/model-steps-3.onnx', + vocoder: './vocos-22khz-univ.onnx', + tokens: './matcha-icefall-en_US-ljspeech/tokens.txt', + + noiseScale: 0.667, + lengthScale: 1.0, + }; + let offlineTtsModelConfig = { + offlineTtsMatchaModelConfig: offlineTtsMatchaModelConfig, + numThreads: 1, + debug: 1, + provider: 'cpu', + }; + + let offlineTtsConfig = { + offlineTtsModelConfig: offlineTtsModelConfig, + maxNumSentences: 1, + silenceScale: silenceScale, + }; + + return sherpa_onnx.createOfflineTts(offlineTtsConfig); +} + +const tts = createOfflineTts(); +const text = + 'Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// en-us if the model has no language information. +const voice = tts.lang || 'en-us'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = { + phonemeCodepoints: phonemeCodepoints, + sid: 0, + speed: 1.0, + silenceScale: silenceScale, +}; + +// The text has already been phonemized; pass an empty string here. +const audio = tts.generateWithConfig('', generationConfig); +tts.save('./test-matcha-en-phonemize.wav', audio); +console.log('Saved to test-matcha-en-phonemize.wav successfully.'); +tts.free(); diff --git a/nodejs-examples/test-offline-tts-matcha-zh.js b/nodejs-examples/test-offline-tts-matcha-zh-lexicon.js similarity index 93% rename from nodejs-examples/test-offline-tts-matcha-zh.js rename to nodejs-examples/test-offline-tts-matcha-zh-lexicon.js index d89c9c68d..42b4ec909 100644 --- a/nodejs-examples/test-offline-tts-matcha-zh.js +++ b/nodejs-examples/test-offline-tts-matcha-zh-lexicon.js @@ -42,6 +42,6 @@ const generationConfig = { }; const audio = tts.generateWithConfig(text, generationConfig); -tts.save('./test-matcha-zh.wav', audio); -console.log('Saved to test-matcha-zh.wav successfully.'); +tts.save('./test-matcha-zh-lexicon.wav', audio); +console.log('Saved to test-matcha-zh-lexicon.wav successfully.'); tts.free(); diff --git a/nodejs-examples/test-offline-tts-vits-de-phonemize.js b/nodejs-examples/test-offline-tts-vits-de-phonemize.js new file mode 100644 index 000000000..7ac57cf3d --- /dev/null +++ b/nodejs-examples/test-offline-tts-vits-de-phonemize.js @@ -0,0 +1,62 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. +// +// There is no German lexicon, so this model is phonemize-only. + +const sherpa_onnx = require('sherpa-onnx'); +const piperPhonemize = require('piper-phonemize'); + +function createOfflineTts() { + let offlineTtsVitsModelConfig = { + model: './vits-piper-de_DE-glados-low/de_DE-glados-low.onnx', + tokens: './vits-piper-de_DE-glados-low/tokens.txt', + noiseScale: 0.667, + noiseScaleW: 0.8, + lengthScale: 1.0, + }; + let offlineTtsModelConfig = { + offlineTtsVitsModelConfig: offlineTtsVitsModelConfig, + numThreads: 1, + debug: 1, + provider: 'cpu', + }; + + let offlineTtsConfig = { + offlineTtsModelConfig: offlineTtsModelConfig, + maxNumSentences: 1, + }; + + return sherpa_onnx.createOfflineTts(offlineTtsConfig); +} + + +const tts = createOfflineTts(); +const speakerId = 0; +const speed = 1.0; +const text = + 'Alles hat ein Ende, nur die Wurst hat zwei.'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// de if the model has no language information. +const voice = tts.lang || 'de'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = { + phonemeCodepoints: phonemeCodepoints, + sid: speakerId, + speed: speed, + silenceScale: 0.2, +}; +// The text has already been phonemized; pass an empty string here. +const audio = tts.generateWithConfig('', generationConfig); +tts.save('./test-vits-de-phonemize.wav', audio); +console.log('Saved to test-vits-de-phonemize.wav successfully.'); + +tts.free(); diff --git a/nodejs-examples/test-offline-tts-vits-en.js b/nodejs-examples/test-offline-tts-vits-en-lexicon.js similarity index 88% rename from nodejs-examples/test-offline-tts-vits-en.js rename to nodejs-examples/test-offline-tts-vits-en-lexicon.js index cf28189c9..0e031e3ad 100644 --- a/nodejs-examples/test-offline-tts-vits-en.js +++ b/nodejs-examples/test-offline-tts-vits-en-lexicon.js @@ -5,8 +5,8 @@ const sherpa_onnx = require('sherpa-onnx'); function createOfflineTts() { let offlineTtsVitsModelConfig = { model: './vits-piper-en_US-amy-low/en_US-amy-low.onnx', + lexicon: './lexicon-en-us.txt', tokens: './vits-piper-en_US-amy-low/tokens.txt', - dataDir: './vits-piper-en_US-amy-low/espeak-ng-data', noiseScale: 0.667, noiseScaleW: 0.8, lengthScale: 1.0, @@ -39,7 +39,7 @@ const audio = tts.generateWithConfig( '“Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.”', generationConfig); -tts.save('./test-vits-en.wav', audio); -console.log('Saved to test-vits-en.wav successfully.'); +tts.save('./test-vits-en-lexicon.wav', audio); +console.log('Saved to test-vits-en-lexicon.wav successfully.'); tts.free(); diff --git a/nodejs-examples/test-offline-tts-vits-en-phonemize.js b/nodejs-examples/test-offline-tts-vits-en-phonemize.js new file mode 100644 index 000000000..49a81af18 --- /dev/null +++ b/nodejs-examples/test-offline-tts-vits-en-phonemize.js @@ -0,0 +1,61 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// The text is phonemized outside of sherpa-onnx with piper-phonemize +// (https://www.npmjs.com/package/piper-phonemize) and the resulting phoneme +// codepoints are passed via GenerationConfig.phonemeCodepoints. No lexicon +// file and no espeak-ng-data are needed. If you want to use a lexicon +// instead, please see test-offline-tts-vits-en-lexicon.js. + +const sherpa_onnx = require('sherpa-onnx'); +const piperPhonemize = require('piper-phonemize'); + +function createOfflineTts() { + let offlineTtsVitsModelConfig = { + model: './vits-piper-en_US-amy-low/en_US-amy-low.onnx', + tokens: './vits-piper-en_US-amy-low/tokens.txt', + noiseScale: 0.667, + noiseScaleW: 0.8, + lengthScale: 1.0, + }; + let offlineTtsModelConfig = { + offlineTtsVitsModelConfig: offlineTtsVitsModelConfig, + numThreads: 1, + debug: 1, + provider: 'cpu', + }; + + let offlineTtsConfig = { + offlineTtsModelConfig: offlineTtsModelConfig, + maxNumSentences: 1, + }; + + return sherpa_onnx.createOfflineTts(offlineTtsConfig); +} + + +const tts = createOfflineTts(); +const speakerId = 0; +const speed = 1.0; +const text = + '“Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar.”'; + +// The espeak-ng voice is taken from the model metadata. Fall back to +// en-us if the model has no language information. +const voice = tts.lang || 'en-us'; +const phonemeCodepoints = piperPhonemize.phonemize(text, voice) + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + .filter((sentence) => sentence.length > 0); + +const generationConfig = { + phonemeCodepoints: phonemeCodepoints, + sid: speakerId, + speed: speed, + silenceScale: 0.2, +}; +// The text has already been phonemized; pass an empty string here. +const audio = tts.generateWithConfig('', generationConfig); +tts.save('./test-vits-en-phonemize.wav', audio); +console.log('Saved to test-vits-en-phonemize.wav successfully.'); + +tts.free(); diff --git a/nodejs-examples/test-offline-tts-vits-zh.js b/nodejs-examples/test-offline-tts-vits-zh-lexicon.js similarity index 91% rename from nodejs-examples/test-offline-tts-vits-zh.js rename to nodejs-examples/test-offline-tts-vits-zh-lexicon.js index ebeee64e4..440b2ff8a 100644 --- a/nodejs-examples/test-offline-tts-vits-zh.js +++ b/nodejs-examples/test-offline-tts-vits-zh-lexicon.js @@ -38,6 +38,6 @@ const generationConfig = { silenceScale: 0.2, }; const audio = tts.generateWithConfig('3年前中国总人口是1411778724人', generationConfig); -tts.save('./test-vits-zh.wav', audio); -console.log('Saved to test-vits-zh.wav successfully.'); +tts.save('./test-vits-zh-lexicon.wav', audio); +console.log('Saved to test-vits-zh-lexicon.wav successfully.'); tts.free(); diff --git a/nodejs-examples/test-offline-tts-zipvoice-zh-en.js b/nodejs-examples/test-offline-tts-zipvoice-zh-en-lexicon.js similarity index 89% rename from nodejs-examples/test-offline-tts-zipvoice-zh-en.js rename to nodejs-examples/test-offline-tts-zipvoice-zh-en-lexicon.js index 7fec98ca8..ee7fc2661 100644 --- a/nodejs-examples/test-offline-tts-zipvoice-zh-en.js +++ b/nodejs-examples/test-offline-tts-zipvoice-zh-en-lexicon.js @@ -8,7 +8,6 @@ function createOfflineTts() { decoder: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx', tokens: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt', lexicon: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt', - dataDir: './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data', vocoder: './vocos_24khz.onnx', }; @@ -47,6 +46,6 @@ const generationConfig = { const tts = createOfflineTts(); const audio = tts.generateWithConfig(text, generationConfig); -tts.save('./test-zipvoice-zh-en.wav', audio); -console.log('Saved to test-zipvoice-zh-en.wav successfully.'); +tts.save('./test-zipvoice-zh-en-lexicon.wav', audio); +console.log('Saved to test-zipvoice-zh-en-lexicon.wav successfully.'); tts.free(); diff --git a/nodejs-examples/test-offline-zipformer-ctc-hlg.js b/nodejs-examples/test-offline-zipformer-ctc-hlg.js new file mode 100644 index 000000000..3fe7ed4f3 --- /dev/null +++ b/nodejs-examples/test-offline-zipformer-ctc-hlg.js @@ -0,0 +1,44 @@ +// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang) +// +// This file shows how to use a non-streaming Zipformer CTC model with an HLG +// decoding graph via sherpa-onnx's Node API. +// +// The HLG graph is passed to the C API through ctcFstDecoderConfig. +const fs = require('fs'); +const {Readable} = require('stream'); +const wav = require('wav'); + +const sherpa_onnx = require('sherpa-onnx'); + +function createOfflineRecognizer() { + let config = { + modelConfig: { + zipformerCtc: { + model: './sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx', + }, + tokens: './sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt', + }, + // This is what the example exercises: the graph reaches the C API through + // ctcFstDecoderConfig. + ctcFstDecoderConfig: { + graph: './sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst', + maxActive: 3000, + }, + }; + + return sherpa_onnx.createOfflineRecognizer(config); +} + +const recognizer = createOfflineRecognizer(); +const stream = recognizer.createStream(); + +const waveFilename = './sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav'; +const wave = sherpa_onnx.readWave(waveFilename); +stream.acceptWaveform(wave.sampleRate, wave.samples); + +recognizer.decode(stream); +const text = recognizer.getResult(stream).text; +console.log(text); + +stream.free(); +recognizer.free(); diff --git a/pascal-api-examples/non-streaming-asr/run-zipformer-ctc-hlg.sh b/pascal-api-examples/non-streaming-asr/run-zipformer-ctc-hlg.sh new file mode 100755 index 000000000..1e8ffe3a1 --- /dev/null +++ b/pascal-api-examples/non-streaming-asr/run-zipformer-ctc-hlg.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -ex + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) + +echo "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" + +if [[ ! -f ../../build/install/lib/libsherpa-onnx-c-api.dylib && ! -f ../../build/install/lib/libsherpa-onnx-c-api.so && ! -f ../../build/install/lib/sherpa-onnx-c-api.dll ]]; then + mkdir -p ../../build + pushd ../../build + cmake \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + .. + + cmake --build . --target install --config Release + ls -lh lib + popd +fi + +if [ ! -f ./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + + tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +fi + +fpc \ + -dSHERPA_ONNX_USE_SHARED_LIBS \ + -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ + -Fl$SHERPA_ONNX_DIR/build/install/lib \ + ./zipformer_ctc_hlg.pas + +export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$DYLD_LIBRARY_PATH + +./zipformer_ctc_hlg diff --git a/pascal-api-examples/non-streaming-asr/zipformer_ctc_hlg.pas b/pascal-api-examples/non-streaming-asr/zipformer_ctc_hlg.pas new file mode 100644 index 000000000..a4e782f45 --- /dev/null +++ b/pascal-api-examples/non-streaming-asr/zipformer_ctc_hlg.pas @@ -0,0 +1,83 @@ +{ Copyright (c) 2026 Xiaomi Corporation } + +{ +This file shows how to use a non-streaming Zipformer CTC model with an HLG +decoding graph via sherpa-onnx's Pascal API. + +The HLG graph is passed to the C API through CtcFstDecoderConfig. + +You can download the model files from +https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +} + +program zipformer_ctc_hlg; + +{$mode objfpc} + +uses + sherpa_onnx, + DateUtils, + SysUtils; + +var + Wave: TSherpaOnnxWave; + WaveFilename: AnsiString; + + Config: TSherpaOnnxOfflineRecognizerConfig; + Recognizer: TSherpaOnnxOfflineRecognizer; + Stream: TSherpaOnnxOfflineStream; + RecognitionResult: TSherpaOnnxOfflineRecognizerResult; + + Start: TDateTime; + Stop: TDateTime; + + Elapsed: Single; + Duration: Single; + RealTimeFactor: Single; +begin + Initialize(Config); + + Config.ModelConfig.ZipformerCtc.Model := './sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx'; + Config.ModelConfig.Tokens := './sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt'; + Config.ModelConfig.Provider := 'cpu'; + Config.ModelConfig.NumThreads := 1; + Config.ModelConfig.Debug := False; + + { This is what the example exercises: the graph and max-active reach the C + API through CtcFstDecoderConfig. } + Config.CtcFstDecoderConfig.Graph := './sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst'; + Config.CtcFstDecoderConfig.MaxActive := 3000; + + WaveFilename := './sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav'; + + Wave := SherpaOnnxReadWave(WaveFilename); + + Recognizer := TSherpaOnnxOfflineRecognizer.Create(Config); + Stream := Recognizer.CreateStream(); + Start := Now; + + Stream.AcceptWaveform(Wave.Samples, Wave.SampleRate); + Recognizer.Decode(Stream); + + RecognitionResult := Recognizer.GetResult(Stream); + + Stop := Now; + + Elapsed := MilliSecondsBetween(Stop, Start) / 1000; + Duration := Length(Wave.Samples) / Wave.SampleRate; + RealTimeFactor := Elapsed / Duration; + + WriteLn(RecognitionResult.ToString); + WriteLn(Format('NumThreads %d', [Config.ModelConfig.NumThreads])); + WriteLn(Format('Elapsed %.3f s', [Elapsed])); + WriteLn(Format('Wave duration %.3f s', [Duration])); + WriteLn(Format('RTF = %.3f/%.3f = %.3f', [Elapsed, Duration, RealTimeFactor])); + + {Free resources to avoid memory leak. + + Note: You don't need to invoke them for this simple script. + However, you have to invoke them in your own large/complex project. + } + FreeAndNil(Stream); + FreeAndNil(Recognizer); +end. diff --git a/pascal-api-examples/portaudio-test/README.md b/pascal-api-examples/portaudio-test/README.md index 78e2c98c0..59990b6fc 100644 --- a/pascal-api-examples/portaudio-test/README.md +++ b/pascal-api-examples/portaudio-test/README.md @@ -1,7 +1,7 @@ # Introduction -[portaudio.pas](./portaudio.pas) -requires that the portaudio library is installed on your system. +[test-play.pas](./test-play.pas) and [test-record.pas](./test-record.pas) +require that the portaudio library is installed on your system. On macOS, you can use @@ -10,4 +10,13 @@ On macOS, you can use brew install portaudio ``` -and it will install `portaudio` into `/usr/local/Cellar/portaudio/19.7.0`. +On macOS with an Intel CPU, it installs `portaudio` into +`/usr/local/Cellar/portaudio/19.7.0`. + +On macOS with Apple silicon (arm64), e.g., M1/M2/M3/M4, it installs +`portaudio` into `/opt/homebrew/Cellar/portaudio/19.7.0`. + +The run scripts in this directory and in [../tts](../tts) pass both +locations to the Free Pascal compiler via `-Fl`, so they work on Intel and +Apple silicon Macs. If your Homebrew installs a different portaudio +version, please adjust the paths in the run scripts accordingly. diff --git a/pascal-api-examples/portaudio-test/run-test-play.sh b/pascal-api-examples/portaudio-test/run-test-play.sh index 741cd358e..a58764c64 100755 --- a/pascal-api-examples/portaudio-test/run-test-play.sh +++ b/pascal-api-examples/portaudio-test/run-test-play.sh @@ -29,6 +29,7 @@ fpc \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ -Fl$SHERPA_ONNX_DIR/build/install/lib \ -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + -Fl/opt/homebrew/Cellar/portaudio/19.7.0/lib \ ./test-play.pas export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH diff --git a/pascal-api-examples/portaudio-test/run-test-record.sh b/pascal-api-examples/portaudio-test/run-test-record.sh index 151711b3a..5a31f5cd2 100755 --- a/pascal-api-examples/portaudio-test/run-test-record.sh +++ b/pascal-api-examples/portaudio-test/run-test-record.sh @@ -29,6 +29,7 @@ fpc \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ -Fl$SHERPA_ONNX_DIR/build/install/lib \ -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + -Fl/opt/homebrew/Cellar/portaudio/19.7.0/lib \ ./test-record.pas export LD_LIBRARY_PATH=$SHERPA_ONNX_DIR/build/install/lib:$LD_LIBRARY_PATH diff --git a/pascal-api-examples/tts/kitten-en-playback.pas b/pascal-api-examples/tts/kitten-en-playback.pas index 96b415684..b9897481d 100644 --- a/pascal-api-examples/tts/kitten-en-playback.pas +++ b/pascal-api-examples/tts/kitten-en-playback.pas @@ -118,7 +118,7 @@ begin Config.Model.Kitten.Model := './kitten-nano-en-v0_1-fp16/model.fp16.onnx'; Config.Model.Kitten.Voices := './kitten-nano-en-v0_1-fp16/voices.bin'; Config.Model.Kitten.Tokens := './kitten-nano-en-v0_1-fp16/tokens.txt'; - Config.Model.Kitten.DataDir := './kitten-nano-en-v0_1-fp16/espeak-ng-data'; + Config.Model.Kitten.Lexicon := './lexicon-en-us.txt'; Config.Model.NumThreads := 2; Config.Model.Debug := False; Config.MaxNumSentences := 1; diff --git a/pascal-api-examples/tts/kitten-en.pas b/pascal-api-examples/tts/kitten-en.pas index 7eb2108af..178efcae8 100644 --- a/pascal-api-examples/tts/kitten-en.pas +++ b/pascal-api-examples/tts/kitten-en.pas @@ -23,7 +23,7 @@ begin Config.Model.Kitten.Model := './kitten-nano-en-v0_1-fp16/model.fp16.onnx'; Config.Model.Kitten.Voices := './kitten-nano-en-v0_1-fp16/voices.bin'; Config.Model.Kitten.Tokens := './kitten-nano-en-v0_1-fp16/tokens.txt'; - Config.Model.Kitten.DataDir := './kitten-nano-en-v0_1-fp16/espeak-ng-data'; + Config.Model.Kitten.Lexicon := './lexicon-en-us.txt'; Config.Model.NumThreads := 2; Config.Model.Debug := False; Config.MaxNumSentences := 1; diff --git a/pascal-api-examples/tts/kokoro-en-playback.pas b/pascal-api-examples/tts/kokoro-en-playback.pas index b9395c2d6..5dd2094e3 100644 --- a/pascal-api-examples/tts/kokoro-en-playback.pas +++ b/pascal-api-examples/tts/kokoro-en-playback.pas @@ -118,7 +118,7 @@ begin Config.Model.Kokoro.Model := './kokoro-en-v0_19/model.onnx'; Config.Model.Kokoro.Voices := './kokoro-en-v0_19/voices.bin'; Config.Model.Kokoro.Tokens := './kokoro-en-v0_19/tokens.txt'; - Config.Model.Kokoro.DataDir := './kokoro-en-v0_19/espeak-ng-data'; + Config.Model.Kokoro.Lexicon := './lexicon-en-us.txt'; Config.Model.NumThreads := 2; Config.Model.Debug := False; Config.MaxNumSentences := 1; diff --git a/pascal-api-examples/tts/kokoro-en.pas b/pascal-api-examples/tts/kokoro-en.pas index ee623d0f7..ff2921cd9 100644 --- a/pascal-api-examples/tts/kokoro-en.pas +++ b/pascal-api-examples/tts/kokoro-en.pas @@ -23,7 +23,7 @@ begin Config.Model.Kokoro.Model := './kokoro-en-v0_19/model.onnx'; Config.Model.Kokoro.Voices := './kokoro-en-v0_19/voices.bin'; Config.Model.Kokoro.Tokens := './kokoro-en-v0_19/tokens.txt'; - Config.Model.Kokoro.DataDir := './kokoro-en-v0_19/espeak-ng-data'; + Config.Model.Kokoro.Lexicon := './lexicon-en-us.txt'; Config.Model.NumThreads := 2; Config.Model.Debug := False; Config.MaxNumSentences := 1; diff --git a/pascal-api-examples/tts/kokoro-zh-en-playback.pas b/pascal-api-examples/tts/kokoro-zh-en-playback.pas index 8435c466d..e3702b67f 100644 --- a/pascal-api-examples/tts/kokoro-zh-en-playback.pas +++ b/pascal-api-examples/tts/kokoro-zh-en-playback.pas @@ -118,8 +118,6 @@ begin Config.Model.Kokoro.Model := './kokoro-multi-lang-v1_0/model.onnx'; Config.Model.Kokoro.Voices := './kokoro-multi-lang-v1_0/voices.bin'; Config.Model.Kokoro.Tokens := './kokoro-multi-lang-v1_0/tokens.txt'; - Config.Model.Kokoro.DataDir := './kokoro-multi-lang-v1_0/espeak-ng-data'; - Config.Model.Kokoro.DictDir := './kokoro-multi-lang-v1_0/dict'; Config.Model.Kokoro.Lexicon := './kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt'; Config.Model.NumThreads := 2; Config.Model.Debug := False; diff --git a/pascal-api-examples/tts/kokoro-zh-en.pas b/pascal-api-examples/tts/kokoro-zh-en.pas index 48e472128..9d8b505bb 100644 --- a/pascal-api-examples/tts/kokoro-zh-en.pas +++ b/pascal-api-examples/tts/kokoro-zh-en.pas @@ -23,8 +23,6 @@ begin Config.Model.Kokoro.Model := './kokoro-multi-lang-v1_0/model.onnx'; Config.Model.Kokoro.Voices := './kokoro-multi-lang-v1_0/voices.bin'; Config.Model.Kokoro.Tokens := './kokoro-multi-lang-v1_0/tokens.txt'; - Config.Model.Kokoro.DataDir := './kokoro-multi-lang-v1_0/espeak-ng-data'; - Config.Model.Kokoro.DictDir := './kokoro-multi-lang-v1_0/dict'; Config.Model.Kokoro.Lexicon := './kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt'; Config.Model.NumThreads := 2; Config.Model.Debug := False; diff --git a/pascal-api-examples/tts/matcha-en-playback.pas b/pascal-api-examples/tts/matcha-en-playback.pas index f98f90c27..d17822f22 100644 --- a/pascal-api-examples/tts/matcha-en-playback.pas +++ b/pascal-api-examples/tts/matcha-en-playback.pas @@ -118,7 +118,7 @@ begin Config.Model.Matcha.AcousticModel := './matcha-icefall-en_US-ljspeech/model-steps-3.onnx'; Config.Model.Matcha.Vocoder := './vocos-22khz-univ.onnx'; Config.Model.Matcha.Tokens := './matcha-icefall-en_US-ljspeech/tokens.txt'; - Config.Model.Matcha.DataDir := './matcha-icefall-en_US-ljspeech/espeak-ng-data'; + Config.Model.Matcha.Lexicon := './lexicon-en-us.txt'; Config.Model.NumThreads := 1; Config.Model.Debug := False; Config.MaxNumSentences := 1; diff --git a/pascal-api-examples/tts/matcha-en.pas b/pascal-api-examples/tts/matcha-en.pas index 56a93f392..0ef4360e9 100644 --- a/pascal-api-examples/tts/matcha-en.pas +++ b/pascal-api-examples/tts/matcha-en.pas @@ -23,7 +23,7 @@ begin Config.Model.Matcha.AcousticModel := './matcha-icefall-en_US-ljspeech/model-steps-3.onnx'; Config.Model.Matcha.Vocoder := './vocos-22khz-univ.onnx'; Config.Model.Matcha.Tokens := './matcha-icefall-en_US-ljspeech/tokens.txt'; - Config.Model.Matcha.DataDir := './matcha-icefall-en_US-ljspeech/espeak-ng-data'; + Config.Model.Matcha.Lexicon := './lexicon-en-us.txt'; Config.Model.NumThreads := 1; Config.Model.Debug := False; Config.MaxNumSentences := 1; diff --git a/pascal-api-examples/tts/matcha-zh-playback.pas b/pascal-api-examples/tts/matcha-zh-playback.pas index e4515fce7..af3fcb347 100644 --- a/pascal-api-examples/tts/matcha-zh-playback.pas +++ b/pascal-api-examples/tts/matcha-zh-playback.pas @@ -119,7 +119,6 @@ begin Config.Model.Matcha.Vocoder := './vocos-22khz-univ.onnx'; Config.Model.Matcha.Lexicon := './matcha-icefall-zh-baker/lexicon.txt'; Config.Model.Matcha.Tokens := './matcha-icefall-zh-baker/tokens.txt'; - Config.Model.Matcha.DictDir := './matcha-icefall-zh-baker/dict'; Config.Model.NumThreads := 1; Config.Model.Debug := False; Config.RuleFsts := './matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst'; diff --git a/pascal-api-examples/tts/matcha-zh.pas b/pascal-api-examples/tts/matcha-zh.pas index 0e7a9ed89..41e07722e 100644 --- a/pascal-api-examples/tts/matcha-zh.pas +++ b/pascal-api-examples/tts/matcha-zh.pas @@ -24,7 +24,6 @@ begin Config.Model.Matcha.Vocoder := './vocos-22khz-univ.onnx'; Config.Model.Matcha.Lexicon := './matcha-icefall-zh-baker/lexicon.txt'; Config.Model.Matcha.Tokens := './matcha-icefall-zh-baker/tokens.txt'; - Config.Model.Matcha.DictDir := './matcha-icefall-zh-baker/dict'; Config.Model.NumThreads := 1; Config.Model.Debug := False; Config.RuleFsts := './matcha-icefall-zh-baker/phone.fst,./matcha-icefall-zh-baker/date.fst,./matcha-icefall-zh-baker/number.fst'; diff --git a/pascal-api-examples/tts/piper-playback.pas b/pascal-api-examples/tts/piper-playback.pas index a378b4dc6..cac69c1f3 100644 --- a/pascal-api-examples/tts/piper-playback.pas +++ b/pascal-api-examples/tts/piper-playback.pas @@ -117,7 +117,7 @@ var begin Config.Model.Vits.Model := './vits-piper-en_US-libritts_r-medium/en_US-libritts_r-medium.onnx'; Config.Model.Vits.Tokens := './vits-piper-en_US-libritts_r-medium/tokens.txt'; - Config.Model.Vits.DataDir := './vits-piper-en_US-libritts_r-medium/espeak-ng-data'; + Config.Model.Vits.Lexicon := './lexicon-en-us.txt'; Config.Model.NumThreads := 1; Config.Model.Debug := False; Config.MaxNumSentences := 1; diff --git a/pascal-api-examples/tts/piper.pas b/pascal-api-examples/tts/piper.pas index e18fd4a44..99675cdb4 100644 --- a/pascal-api-examples/tts/piper.pas +++ b/pascal-api-examples/tts/piper.pas @@ -22,7 +22,7 @@ var begin Config.Model.Vits.Model := './vits-piper-en_US-libritts_r-medium/en_US-libritts_r-medium.onnx'; Config.Model.Vits.Tokens := './vits-piper-en_US-libritts_r-medium/tokens.txt'; - Config.Model.Vits.DataDir := './vits-piper-en_US-libritts_r-medium/espeak-ng-data'; + Config.Model.Vits.Lexicon := './lexicon-en-us.txt'; Config.Model.NumThreads := 1; Config.Model.Debug := False; Config.MaxNumSentences := 1; diff --git a/pascal-api-examples/tts/run-kitten-en-playback.sh b/pascal-api-examples/tts/run-kitten-en-playback.sh index f7571f4fd..b01e70366 100755 --- a/pascal-api-examples/tts/run-kitten-en-playback.sh +++ b/pascal-api-examples/tts/run-kitten-en-playback.sh @@ -31,11 +31,16 @@ if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then rm kitten-nano-en-v0_1-fp16.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + fpc \ -dSHERPA_ONNX_USE_SHARED_LIBS \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ -Fl$SHERPA_ONNX_DIR/build/install/lib \ -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + -Fl/opt/homebrew/Cellar/portaudio/19.7.0/lib \ ./kitten-en-playback.pas # Please see ../portaudio-test/README.md diff --git a/pascal-api-examples/tts/run-kitten-en.sh b/pascal-api-examples/tts/run-kitten-en.sh index 181b5d112..2c693a965 100755 --- a/pascal-api-examples/tts/run-kitten-en.sh +++ b/pascal-api-examples/tts/run-kitten-en.sh @@ -31,6 +31,10 @@ if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then rm kitten-nano-en-v0_1-fp16.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + fpc \ -dSHERPA_ONNX_USE_SHARED_LIBS \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ diff --git a/pascal-api-examples/tts/run-kokoro-en-playback.sh b/pascal-api-examples/tts/run-kokoro-en-playback.sh index 49e280d7a..935400c1d 100755 --- a/pascal-api-examples/tts/run-kokoro-en-playback.sh +++ b/pascal-api-examples/tts/run-kokoro-en-playback.sh @@ -31,11 +31,16 @@ if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then rm kokoro-en-v0_19.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + fpc \ -dSHERPA_ONNX_USE_SHARED_LIBS \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ -Fl$SHERPA_ONNX_DIR/build/install/lib \ -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + -Fl/opt/homebrew/Cellar/portaudio/19.7.0/lib \ ./kokoro-en-playback.pas # Please see ../portaudio-test/README.md diff --git a/pascal-api-examples/tts/run-kokoro-en.sh b/pascal-api-examples/tts/run-kokoro-en.sh index f26fb335d..277a3166c 100755 --- a/pascal-api-examples/tts/run-kokoro-en.sh +++ b/pascal-api-examples/tts/run-kokoro-en.sh @@ -31,6 +31,10 @@ if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then rm kokoro-en-v0_19.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + fpc \ -dSHERPA_ONNX_USE_SHARED_LIBS \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ diff --git a/pascal-api-examples/tts/run-kokoro-zh-en-playback.sh b/pascal-api-examples/tts/run-kokoro-zh-en-playback.sh index cc6e2526d..6746de846 100755 --- a/pascal-api-examples/tts/run-kokoro-zh-en-playback.sh +++ b/pascal-api-examples/tts/run-kokoro-zh-en-playback.sh @@ -36,6 +36,7 @@ fpc \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ -Fl$SHERPA_ONNX_DIR/build/install/lib \ -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + -Fl/opt/homebrew/Cellar/portaudio/19.7.0/lib \ ./kokoro-zh-en-playback.pas # Please see ../portaudio-test/README.md diff --git a/pascal-api-examples/tts/run-matcha-en-playback.sh b/pascal-api-examples/tts/run-matcha-en-playback.sh index d50ddc51d..e1152a197 100755 --- a/pascal-api-examples/tts/run-matcha-en-playback.sh +++ b/pascal-api-examples/tts/run-matcha-en-playback.sh @@ -37,11 +37,16 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + fpc \ -dSHERPA_ONNX_USE_SHARED_LIBS \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ -Fl$SHERPA_ONNX_DIR/build/install/lib \ -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + -Fl/opt/homebrew/Cellar/portaudio/19.7.0/lib \ ./matcha-en-playback.pas # Please see ../portaudio-test/README.md diff --git a/pascal-api-examples/tts/run-matcha-en.sh b/pascal-api-examples/tts/run-matcha-en.sh index 85dbba3a0..b7ce3a060 100755 --- a/pascal-api-examples/tts/run-matcha-en.sh +++ b/pascal-api-examples/tts/run-matcha-en.sh @@ -37,6 +37,10 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + fpc \ -dSHERPA_ONNX_USE_SHARED_LIBS \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ diff --git a/pascal-api-examples/tts/run-matcha-zh-playback.sh b/pascal-api-examples/tts/run-matcha-zh-playback.sh index 8cf3675fe..d1fc17ea3 100755 --- a/pascal-api-examples/tts/run-matcha-zh-playback.sh +++ b/pascal-api-examples/tts/run-matcha-zh-playback.sh @@ -41,6 +41,7 @@ fpc \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ -Fl$SHERPA_ONNX_DIR/build/install/lib \ -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + -Fl/opt/homebrew/Cellar/portaudio/19.7.0/lib \ ./matcha-zh-playback.pas # Please see ../portaudio-test/README.md diff --git a/pascal-api-examples/tts/run-piper-playback.sh b/pascal-api-examples/tts/run-piper-playback.sh index 7076dee05..c37239406 100755 --- a/pascal-api-examples/tts/run-piper-playback.sh +++ b/pascal-api-examples/tts/run-piper-playback.sh @@ -29,11 +29,16 @@ if [[ ! -f ./vits-piper-en_US-libritts_r-medium/tokens.txt ]]; then rm vits-piper-en_US-libritts_r-medium.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + fpc \ -dSHERPA_ONNX_USE_SHARED_LIBS \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ -Fl$SHERPA_ONNX_DIR/build/install/lib \ -Fl/usr/local/Cellar/portaudio/19.7.0/lib \ + -Fl/opt/homebrew/Cellar/portaudio/19.7.0/lib \ ./piper-playback.pas # Please see ../portaudio-test/README.md diff --git a/pascal-api-examples/tts/run-piper.sh b/pascal-api-examples/tts/run-piper.sh index 7abed25ee..747979472 100755 --- a/pascal-api-examples/tts/run-piper.sh +++ b/pascal-api-examples/tts/run-piper.sh @@ -29,6 +29,10 @@ if [[ ! -f ./vits-piper-en_US-libritts_r-medium/tokens.txt ]]; then rm vits-piper-en_US-libritts_r-medium.tar.bz2 fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + fpc \ -dSHERPA_ONNX_USE_SHARED_LIBS \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ diff --git a/pascal-api-examples/tts/run-zipvoice-zh-en.sh b/pascal-api-examples/tts/run-zipvoice-zh-en.sh index 39ac20d50..826ae3d39 100755 --- a/pascal-api-examples/tts/run-zipvoice-zh-en.sh +++ b/pascal-api-examples/tts/run-zipvoice-zh-en.sh @@ -35,6 +35,10 @@ if [ ! -f ./vocos_24khz.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx fi +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + fpc \ -dSHERPA_ONNX_USE_SHARED_LIBS \ -Fu$SHERPA_ONNX_DIR/sherpa-onnx/pascal-api \ diff --git a/pascal-api-examples/tts/zipvoice-zh-en.pas b/pascal-api-examples/tts/zipvoice-zh-en.pas index 3a01c5366..3d1a866ca 100644 --- a/pascal-api-examples/tts/zipvoice-zh-en.pas +++ b/pascal-api-examples/tts/zipvoice-zh-en.pas @@ -34,8 +34,7 @@ begin Config.Model.ZipVoice.Encoder := './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx'; Config.Model.ZipVoice.Decoder := './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx'; Config.Model.ZipVoice.Vocoder := './vocos_24khz.onnx'; - Config.Model.ZipVoice.DataDir := './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data'; - Config.Model.ZipVoice.Lexicon := './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt'; + Config.Model.ZipVoice.Lexicon := './sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt'; Config.Model.ZipVoice.FeatScale := 0.1; Config.Model.ZipVoice.Tshift := 0.5; Config.Model.ZipVoice.TargetRms := 0.1; diff --git a/python-api-examples/offline-tts-play.py b/python-api-examples/offline-tts-play.py index 974ee097f..e5d524305 100755 --- a/python-api-examples/offline-tts-play.py +++ b/python-api-examples/offline-tts-play.py @@ -16,10 +16,12 @@ Example (1/8) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/offline-tts-play.py \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." @@ -77,11 +79,13 @@ rm matcha-icefall-en_US-ljspeech.tar.bz2 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/offline-tts-play.py \ --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ --matcha-vocoder=./vocos-22khz-univ.onnx \ --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ - --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --matcha-lexicon=./lexicon-en-us.txt \ --output-filename=./test-matcha-ljspeech-en.wav \ --num-threads=2 \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." @@ -94,12 +98,14 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/offline-tts.py \ --debug=1 \ --kokoro-model=./kokoro-en-v0_19/model.onnx \ --kokoro-voices=./kokoro-en-v0_19/voices.bin \ --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ - --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --kokoro-lexicon=./lexicon-en-us.txt \ --num-threads=2 \ --sid=10 \ --output-filename="./kokoro-10.wav" \ @@ -118,7 +124,6 @@ python3 ./python-api-examples/offline-tts-play.py \ --kokoro-model=./kokoro-multi-lang-v1_0/model.onnx \ --kokoro-voices=./kokoro-multi-lang-v1_0/voices.bin \ --kokoro-tokens=./kokoro-multi-lang-v1_0/tokens.txt \ - --kokoro-data-dir=./kokoro-multi-lang-v1_0/espeak-ng-data \ --kokoro-lexicon=./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt \ --num-threads=2 \ --sid=18 \ @@ -131,12 +136,14 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/offline-tts-play.py \ --debug=1 \ --kitten-model=./kitten-nano-en-v0_1-fp16/model.fp16.onnx \ --kitten-voices=./kitten-nano-en-v0_1-fp16/voices.bin \ --kitten-tokens=./kitten-nano-en-v0_1-fp16/tokens.txt \ - --kitten-data-dir=./kitten-nano-en-v0_1-fp16/espeak-ng-data \ + --kitten-lexicon=./lexicon-en-us.txt \ --num-threads=2 \ --sid=0 \ --output-filename="./kitten-0.wav" \ @@ -194,13 +201,6 @@ def add_vits_args(parser): help="Path to tokens.txt", ) - parser.add_argument( - "--vits-data-dir", - type=str, - default="", - help="""Path to the dict directory of espeak-ng. If it is specified, - --vits-lexicon and --vits-tokens are ignored""", - ) def add_matcha_args(parser): @@ -232,13 +232,6 @@ def add_matcha_args(parser): help="Path to tokens.txt for matcha", ) - parser.add_argument( - "--matcha-data-dir", - type=str, - default="", - help="""Path to the dict directory of espeak-ng. If it is specified, - --matcha-lexicon and --matcha-tokens are ignored""", - ) def add_kokoro_args(parser): @@ -263,12 +256,6 @@ def add_kokoro_args(parser): help="Path to tokens.txt for kokoro", ) - parser.add_argument( - "--kokoro-data-dir", - type=str, - default="", - help="Path to the dict directory of espeak-ng.", - ) parser.add_argument( "--kokoro-lexicon", @@ -301,13 +288,14 @@ def add_kitten_args(parser): ) parser.add_argument( - "--kitten-data-dir", + "--kitten-lexicon", type=str, default="", - help="Path to the dict directory of espeak-ng.", + help="Path to lexicon.txt for kitten", ) + def get_args(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -503,7 +491,6 @@ def main(): vits=sherpa_onnx.OfflineTtsVitsModelConfig( model=args.vits_model, lexicon=args.vits_lexicon, - data_dir=args.vits_data_dir, tokens=args.vits_tokens, ), matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( @@ -511,20 +498,18 @@ def main(): vocoder=args.matcha_vocoder, lexicon=args.matcha_lexicon, tokens=args.matcha_tokens, - data_dir=args.matcha_data_dir, ), kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig( model=args.kokoro_model, voices=args.kokoro_voices, tokens=args.kokoro_tokens, - data_dir=args.kokoro_data_dir, lexicon=args.kokoro_lexicon, ), kitten=sherpa_onnx.OfflineTtsKittenModelConfig( model=args.kitten_model, voices=args.kitten_voices, tokens=args.kitten_tokens, - data_dir=args.kitten_data_dir, + lexicon=args.kitten_lexicon, ), provider=args.provider, debug=args.debug, diff --git a/python-api-examples/offline-tts.py b/python-api-examples/offline-tts.py index 20157c44f..023337b19 100755 --- a/python-api-examples/offline-tts.py +++ b/python-api-examples/offline-tts.py @@ -17,10 +17,12 @@ Example (1/8) wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/offline-tts.py \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." @@ -78,11 +80,13 @@ rm matcha-icefall-en_US-ljspeech.tar.bz2 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/offline-tts.py \ --matcha-acoustic-model=./matcha-icefall-en_US-ljspeech/model-steps-3.onnx \ --matcha-vocoder=./vocos-22khz-univ.onnx \ --matcha-tokens=./matcha-icefall-en_US-ljspeech/tokens.txt \ - --matcha-data-dir=./matcha-icefall-en_US-ljspeech/espeak-ng-data \ + --matcha-lexicon=./lexicon-en-us.txt \ --output-filename=./test-matcha-ljspeech-en.wav \ --num-threads=2 \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." @@ -95,12 +99,14 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kokoro-en-v0_19.tar.bz2 rm kokoro-en-v0_19.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/offline-tts.py \ --debug=1 \ --kokoro-model=./kokoro-en-v0_19/model.onnx \ --kokoro-voices=./kokoro-en-v0_19/voices.bin \ --kokoro-tokens=./kokoro-en-v0_19/tokens.txt \ - --kokoro-data-dir=./kokoro-en-v0_19/espeak-ng-data \ + --kokoro-lexicon=./lexicon-en-us.txt \ --num-threads=2 \ --sid=10 \ --output-filename="./kokoro-10.wav" \ @@ -119,7 +125,6 @@ python3 ./python-api-examples/offline-tts.py \ --kokoro-model=./kokoro-multi-lang-v1_0/model.onnx \ --kokoro-voices=./kokoro-multi-lang-v1_0/voices.bin \ --kokoro-tokens=./kokoro-multi-lang-v1_0/tokens.txt \ - --kokoro-data-dir=./kokoro-multi-lang-v1_0/espeak-ng-data \ --kokoro-lexicon=./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt \ --num-threads=2 \ --sid=18 \ @@ -132,12 +137,14 @@ curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/k tar xf kitten-nano-en-v0_1-fp16.tar.bz2 rm kitten-nano-en-v0_1-fp16.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/offline-tts.py \ --debug=1 \ --kitten-model=./kitten-nano-en-v0_1-fp16/model.fp16.onnx \ --kitten-voices=./kitten-nano-en-v0_1-fp16/voices.bin \ --kitten-tokens=./kitten-nano-en-v0_1-fp16/tokens.txt \ - --kitten-data-dir=./kitten-nano-en-v0_1-fp16/espeak-ng-data \ + --kitten-lexicon=./lexicon-en-us.txt \ --num-threads=2 \ --sid=0 \ --output-filename="./kitten-0.wav" \ @@ -181,13 +188,6 @@ def add_vits_args(parser): help="Path to tokens.txt", ) - parser.add_argument( - "--vits-data-dir", - type=str, - default="", - help="""Path to the dict directory of espeak-ng. If it is specified, - --vits-lexicon and --vits-tokens are ignored""", - ) def add_matcha_args(parser): @@ -219,13 +219,6 @@ def add_matcha_args(parser): help="Path to tokens.txt for matcha", ) - parser.add_argument( - "--matcha-data-dir", - type=str, - default="", - help="""Path to the dict directory of espeak-ng. If it is specified, - --matcha-lexicon and --matcha-tokens are ignored""", - ) def add_kokoro_args(parser): @@ -250,12 +243,6 @@ def add_kokoro_args(parser): help="Path to tokens.txt for kokoro", ) - parser.add_argument( - "--kokoro-data-dir", - type=str, - default="", - help="Path to the dict directory of espeak-ng.", - ) parser.add_argument( "--kokoro-lexicon", @@ -288,13 +275,14 @@ def add_kitten_args(parser): ) parser.add_argument( - "--kitten-data-dir", + "--kitten-lexicon", type=str, default="", - help="Path to the dict directory of espeak-ng.", + help="Path to lexicon.txt for kitten", ) + def get_args(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -386,7 +374,6 @@ def main(): vits=sherpa_onnx.OfflineTtsVitsModelConfig( model=args.vits_model, lexicon=args.vits_lexicon, - data_dir=args.vits_data_dir, tokens=args.vits_tokens, ), matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( @@ -394,20 +381,18 @@ def main(): vocoder=args.matcha_vocoder, lexicon=args.matcha_lexicon, tokens=args.matcha_tokens, - data_dir=args.matcha_data_dir, ), kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig( model=args.kokoro_model, voices=args.kokoro_voices, tokens=args.kokoro_tokens, - data_dir=args.kokoro_data_dir, lexicon=args.kokoro_lexicon, ), kitten=sherpa_onnx.OfflineTtsKittenModelConfig( model=args.kitten_model, voices=args.kitten_voices, tokens=args.kitten_tokens, - data_dir=args.kitten_data_dir, + lexicon=args.kitten_lexicon, ), provider=args.provider, debug=args.debug, diff --git a/python-api-examples/offline-zipformer-ctc-hlg-decode-files.py b/python-api-examples/offline-zipformer-ctc-hlg-decode-files.py new file mode 100644 index 000000000..512ea7152 --- /dev/null +++ b/python-api-examples/offline-zipformer-ctc-hlg-decode-files.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +# This file shows how to use a non-streaming zipformer CTC model with an HLG +# decoding graph via sherpa-onnx's Python API. +# +# The HLG graph is passed to the C API through ctc_fst_decoder_config. +# +# clang-format off +# +# wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +# tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +# rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +# +# python3 ./python-api-examples/offline-zipformer-ctc-hlg-decode-files.py \ +# --tokens ./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt \ +# --model ./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx \ +# --graph ./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst \ +# ./sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav +# +# clang-format on + +import argparse +from pathlib import Path +from typing import Tuple + +import numpy as np +import sherpa_onnx + + +def get_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--tokens", + type=str, + required=True, + help="Path to tokens.txt", + ) + + parser.add_argument( + "--model", + type=str, + required=True, + help="Path to the ONNX model", + ) + + parser.add_argument( + "--graph", + type=str, + required=True, + help="Path to H.fst, HL.fst, or HLG.fst", + ) + + parser.add_argument( + "--num-threads", + type=int, + default=1, + help="Number of threads for neural network computation", + ) + + parser.add_argument( + "--provider", + type=str, + default="cpu", + help="Valid values: cpu, cuda, coreml", + ) + + parser.add_argument( + "--debug", + type=int, + default=0, + help="Valid values: 1, 0", + ) + + parser.add_argument( + "sound_file", + type=str, + help="The input sound file to decode. It must be of WAVE" + "format with a single channel, and each sample has 16-bit, " + "i.e., int16_t. " + "The sample rate of the file can be arbitrary and does not need to " + "be 16 kHz", + ) + + return parser.parse_args() + + +def assert_file_exists(filename: str): + assert Path(filename).is_file(), ( + f"{filename} does not exist!\n" + "Please refer to " + "https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html to download it" + ) + + +def read_wave(wave_filename: str) -> Tuple[np.ndarray, int]: + """ + Args: + wave_filename: + Path to a wave file. It should be single channel and each sample should + be 16-bit. Its sample rate does not need to be 16kHz. + Returns: + Return a tuple containing: + - A 1-D array of dtype np.float32 containing the samples, which are + normalized to the range [-1, 1]. + - sample rate of the wave file + """ + + import wave + + with wave.open(wave_filename) as f: + assert f.getnchannels() == 1, f.getnchannels() + assert f.getsampwidth() == 2, f.getsampwidth() # it is in bytes + num_samples = f.getnframes() + samples = f.readframes(num_samples) + samples_int16 = np.frombuffer(samples, dtype=np.int16) + samples_float32 = samples_int16.astype(np.float32) + + samples_float32 = samples_float32 / 32768 + return samples_float32, f.getframerate() + + +def main(): + args = get_args() + print(vars(args)) + + assert_file_exists(args.tokens) + assert_file_exists(args.graph) + assert_file_exists(args.model) + + # This is what the example exercises: the graph reaches the C API through + # ctc_graph, which is passed to ctc_fst_decoder_config. + recognizer = sherpa_onnx.OfflineRecognizer.from_zipformer_ctc( + model=args.model, + tokens=args.tokens, + num_threads=args.num_threads, + provider=args.provider, + debug=bool(args.debug), + ctc_graph=args.graph, + ) + + audio, sample_rate = read_wave(args.sound_file) + + stream = recognizer.create_stream() + stream.accept_waveform(sample_rate, audio) + recognizer.decode_stream(stream) + + result = stream.result + print(args.sound_file) + print(result.text) + # When an HLG graph is used, the graph's output labels are in result.words. + print(result.words) + # str(result) is the JSON serialization of the result. + print(result) + + +if __name__ == "__main__": + main() diff --git a/python-api-examples/test-offline-tts-coqui-phonemize.py b/python-api-examples/test-offline-tts-coqui-phonemize.py new file mode 100755 index 000000000..944d3a5aa --- /dev/null +++ b/python-api-examples/test-offline-tts-coqui-phonemize.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +""" +Test OfflineTtsVitsExtImpl with a coqui model (uses character frontend). + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-coqui-de-css10.tar.bz2 + tar xf vits-coqui-de-css10.tar.bz2 +""" + +import time + +import sherpa_onnx +import soundfile as sf + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model="./vits-coqui-de-css10/model.onnx", + tokens="./vits-coqui-de-css10/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Alles hat ein Ende, nur die Wurst hat zwei." + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-coqui-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-inflect-lexicon.py b/python-api-examples/test-offline-tts-inflect-lexicon.py new file mode 100755 index 000000000..4227e2915 --- /dev/null +++ b/python-api-examples/test-offline-tts-inflect-lexicon.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +""" +Test OfflineTtsVitsExtImpl with an inflect model using lexicon-based phonemization. + +Download model and lexicon: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-inflect-en-nano-v2.tar.bz2 + tar xf vits-inflect-en-nano-v2.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +""" + +import time + +import sherpa_onnx +import soundfile as sf + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model="./vits-inflect-en-nano-v2/model.onnx", + tokens="./vits-inflect-en-nano-v2/tokens.txt", + lexicon="./lexicon-en-us.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Friends fell out often because life was changing so fast." + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-inflect-lexicon.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-inflect-phonemize.py b/python-api-examples/test-offline-tts-inflect-phonemize.py new file mode 100755 index 000000000..cae6b2330 --- /dev/null +++ b/python-api-examples/test-offline-tts-inflect-phonemize.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +Test OfflineTtsVitsExtImpl with an inflect model using piper_phonemize. + +Install piper_phonemize: + pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-inflect-en-nano-v2.tar.bz2 + tar xf vits-inflect-en-nano-v2.tar.bz2 +""" + +import time + +import sherpa_onnx +import soundfile as sf +from piper_phonemize import phonemize_espeak + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model="./vits-inflect-en-nano-v2/model.onnx", + tokens="./vits-inflect-en-nano-v2/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Friends fell out often because life was changing so fast." +# The espeak-ng voice for the phonemizer is taken from the model metadata. +# Fall back to en-us if the model has no language information. +lang = tts.lang +if not lang: + print("Warning: the model has no language information. Falling back to en-us.") + lang = "en-us" +print("Phonemizer language:", lang) + +sentences = phonemize_espeak(text, lang) +phoneme_codepoints = [[ord(c) for c in sentence] for sentence in sentences] + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.phoneme_codepoints = phoneme_codepoints + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-inflect-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-kitten-lexicon.py b/python-api-examples/test-offline-tts-kitten-lexicon.py new file mode 100755 index 000000000..d29f57d27 --- /dev/null +++ b/python-api-examples/test-offline-tts-kitten-lexicon.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +""" +Test a Kitten English TTS model using lexicon-based phonemization +(no espeak-ng data, no external phonemizer needed). + +If you don't want to use a lexicon, please see +test-offline-tts-kitten-phonemize.py which uses piper_phonemize instead. + +Download model and lexicon: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-micro-en-v0_8.tar.bz2 + tar xf kitten-micro-en-v0_8.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +""" + +import time + +import sherpa_onnx +import soundfile as sf + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + kitten=sherpa_onnx.OfflineTtsKittenModelConfig( + model="./kitten-micro-en-v0_8/model.onnx", + voices="./kitten-micro-en-v0_8/voices.bin", + tokens="./kitten-micro-en-v0_8/tokens.txt", + lexicon="./lexicon-en-us.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Today as always, men fall into two groups: slaves and free men." + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-kitten-lexicon.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-kitten-phonemize.py b/python-api-examples/test-offline-tts-kitten-phonemize.py new file mode 100755 index 000000000..2f45ee9fe --- /dev/null +++ b/python-api-examples/test-offline-tts-kitten-phonemize.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +""" +Test a Kitten English TTS model using piper_phonemize (no espeak-ng data). + +If you want to use a lexicon instead, please see +test-offline-tts-kitten-lexicon.py. + +Install piper_phonemize: + pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-micro-en-v0_8.tar.bz2 + tar xf kitten-micro-en-v0_8.tar.bz2 +""" + +import time + +import sherpa_onnx +import soundfile as sf + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + kitten=sherpa_onnx.OfflineTtsKittenModelConfig( + model="./kitten-micro-en-v0_8/model.onnx", + voices="./kitten-micro-en-v0_8/voices.bin", + tokens="./kitten-micro-en-v0_8/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Today as always, men fall into two groups: slaves and free men." +# The espeak-ng voice for the phonemizer is taken from the model metadata. +# Fall back to en-us if the model has no language information. +lang = tts.lang +if not lang: + print("Warning: the model has no language information. Falling back to en-us.") + lang = "en-us" +print("Phonemizer language:", lang) + +sentences = phonemize_espeak(text, lang) +phoneme_codepoints = [[ord(c) for c in sentence] for sentence in sentences] + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.phoneme_codepoints = phoneme_codepoints + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-kitten-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-kokoro-en-lexicon.py b/python-api-examples/test-offline-tts-kokoro-en-lexicon.py new file mode 100644 index 000000000..fd9fde4fd --- /dev/null +++ b/python-api-examples/test-offline-tts-kokoro-en-lexicon.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +""" +Test Kokoro English TTS model using lexicon +(no espeak-ng data needed). + +Supported extra parameters in gen_config.extra (all optional): + - debug, int, default from model config. + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +""" + +import time + +import sherpa_onnx +import soundfile as sf + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig( + model="./kokoro-en-v0_19/model.onnx", + voices="./kokoro-en-v0_19/voices.bin", + tokens="./kokoro-en-v0_19/tokens.txt", + lexicon="./lexicon-en-us.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Today as always, men fall into two groups: slaves and free men. Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-kokoro-en-lexicon.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-kokoro-en-phonemize.py b/python-api-examples/test-offline-tts-kokoro-en-phonemize.py new file mode 100644 index 000000000..8705af418 --- /dev/null +++ b/python-api-examples/test-offline-tts-kokoro-en-phonemize.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +Test Kokoro English TTS model using piper_phonemize +(no espeak-ng data needed). + +If you want to use a lexicon instead, please see +test-offline-tts-kokoro-en-lexicon.py. + +Install dependencies: + pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 +""" + +import time + +import sherpa_onnx +import soundfile as sf + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig( + model="./kokoro-en-v0_19/model.onnx", + voices="./kokoro-en-v0_19/voices.bin", + tokens="./kokoro-en-v0_19/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Today as always, men fall into two groups: slaves and free men. Friends fell out often because life was changing so fast. The easiest thing in the world was to lose touch with someone." +# The espeak-ng voice for the phonemizer is taken from the model metadata. +# Fall back to en-us if the model has no language information. +lang = tts.lang +if not lang: + print("Warning: the model has no language information. Falling back to en-us.") + lang = "en-us" +print("Phonemizer language:", lang) + +sentences = phonemize_espeak(text, lang) +phoneme_codepoints = [[ord(c) for c in sentence] for sentence in sentences] + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.phoneme_codepoints = phoneme_codepoints + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-kokoro-en-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") + diff --git a/python-api-examples/test-offline-tts-kokoro-multilang-lexicon.py b/python-api-examples/test-offline-tts-kokoro-multilang-lexicon.py new file mode 100644 index 000000000..f5c87f487 --- /dev/null +++ b/python-api-examples/test-offline-tts-kokoro-multilang-lexicon.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +""" +Test Kokoro Chinese+English TTS model using lexicon +(no espeak-ng data needed). + +Supported extra parameters in gen_config.extra (all optional): + - lang, str, Language override (e.g., "en-us", "zh"). Defaults to kokoro.lang. + - debug, int, default from model config. + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 + tar xf kokoro-multi-lang-v1_0.tar.bz2 +""" + +import time + +import sherpa_onnx +import soundfile as sf + +model_dir = "./kokoro-multi-lang-v1_0" + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig( + model=f"{model_dir}/model.onnx", + voices=f"{model_dir}/voices.bin", + tokens=f"{model_dir}/tokens.txt", + lexicon=f"{model_dir}/lexicon-us-en.txt,{model_dir}/lexicon-zh.txt", + ), + debug=True, + ), + rule_fsts=f"{model_dir}/phone-zh.fst,{model_dir}/date-zh.fst,{model_dir}/number-zh.fst", +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "在这个快速发展的时代，人工智能artificial intelligence技术正在改变我们的生活方式。 有困难，请拨打110。" + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-kokoro-multilang-lexicon.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-kokoro-multilang-phonemize.py b/python-api-examples/test-offline-tts-kokoro-multilang-phonemize.py new file mode 100644 index 000000000..d66ace7d0 --- /dev/null +++ b/python-api-examples/test-offline-tts-kokoro-multilang-phonemize.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +""" +Test Kokoro Chinese+English TTS model using misaki + piper_phonemize +(no espeak-ng data needed). + +Chinese text uses misaki.zh.ZHG2P for IPA phonemization. +English text uses piper_phonemize (espeak-ng) for IPA phonemization. + +Install dependencies: + pip install 'misaki[zh]' kaldifst + pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2 + tar xf kokoro-multi-lang-v1_0.tar.bz2 +""" + +import re +import time + +import kaldifst +import sherpa_onnx +import soundfile as sf + +try: + from misaki import zh +except Exception as ex: + raise RuntimeError(f"{ex}\nPlease run\npip install 'misaki[zh]' kaldifst") + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + +# Initialize Chinese G2P +_zh_g2p = zh.ZHG2P() + + +def tokenize_mixed(text, lang): + """Tokenize mixed Chinese+English text for Kokoro. + + Chinese: misaki zh.ZHG2P -> IPA phonemes (individual characters) + English: piper_phonemize -> IPA phonemes + """ + # Replace Chinese punctuations with ASCII equivalents + for zh_p, en_p in [ + ("，", ","), + ("。", "."), + ("！", "!"), + ("？", "?"), + ("；", ";"), + ("：", ":"), + ("、", ","), + ]: + text = text.replace(zh_p, en_p) + + # Split into: Chinese chars | punctuation | English/ASCII text + segments = re.findall( + r"[一-鿿]+" r"|[,.\!?;:\-…]" r"|[a-zA-Z0-9 ]+", + text, + ) + + all_tokens = [] + for seg in segments: + if re.match(r"[一-鿿]+", seg): + # Chinese segment: convert to IPA using misaki + ipa = _zh_g2p.word2ipa(seg) + # Strip U+032F (COMBINING INVERTED BREVE BELOW), same as + # generate_lexicon_zh.py does + ipa = ipa.replace(chr(815), "") + for c in ipa: + all_tokens.append(c) + elif re.match(r"[,.\!?;:\-…]", seg): + # Punctuation: add directly as ASCII token + all_tokens.append(seg) + else: + # English/ASCII text: use piper_phonemize + seg = seg.strip() + if not seg: + continue + sentences = phonemize_espeak(seg, lang) + for sentence in sentences: + for c in sentence: + all_tokens.append(c) + + return [all_tokens] + + +model_dir = "./kokoro-multi-lang-v1_0" + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig( + model=f"{model_dir}/model.onnx", + voices=f"{model_dir}/voices.bin", + tokens=f"{model_dir}/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "在这个快速发展的时代，人工智能artificial intelligence技术正在改变我们的生活方式。语音合成speech synthesis作为重要应用之一。有困难，请拨打110。" + +# Normalize text using kaldifst rule FSTs (same as C++ rule_fsts) +# Converts numbers, dates, phone numbers to Chinese characters +rule_fsts_files = [ + f"{model_dir}/phone-zh.fst", + f"{model_dir}/date-zh.fst", + f"{model_dir}/number-zh.fst", +] +rule_fsts = [kaldifst.TextNormalizer(fst_path) for fst_path in rule_fsts_files] +for tn in rule_fsts: + text = tn.normalize(text) + +print(f"Normalized text: {text}") + +# The espeak-ng voice for English segments is taken from the model metadata. +# Fall back to en-us if the model has no language information. +lang = tts.lang +if not lang: + print("Warning: the model has no language information. Falling back to en-us.") + lang = "en-us" +print("Phonemizer language:", lang) + +tokens = tokenize_mixed(text, lang) +print(f"Total tokens: {len(tokens[0])}") + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.tokens = tokens + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-kokoro-multilang-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") + diff --git a/python-api-examples/test-offline-tts-matcha-en-lexicon.py b/python-api-examples/test-offline-tts-matcha-en-lexicon.py new file mode 100755 index 000000000..aa3e0d616 --- /dev/null +++ b/python-api-examples/test-offline-tts-matcha-en-lexicon.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +Test a Matcha English TTS model using lexicon-based phonemization +(no espeak-ng data, no external phonemizer needed). + +If you don't want to use a lexicon, please see +test-offline-tts-matcha-en-phonemize.py which uses piper_phonemize instead. + +Supported extra parameters in gen_config.extra (all optional): + - min_words_in_sentence, int, default 5. + Merge adjacent sentences if the number of words is less than this. + Each CJK character counts as one word; English words are space-separated. + - max_words_in_sentence, int, default 20. + Split a sentence into chunks if the number of words exceeds this. + Splits at punctuation or space boundaries. + - max_codepoints_in_sentence, int, default 200. + For the phoneme_codepoints path: split a phoneme sequence if it exceeds + this many codepoints. Splits at space boundaries. + - debug, int, default from model config. + +Download model, vocoder, and lexicon: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +""" + +import time + +import sherpa_onnx +import soundfile as sf + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( + acoustic_model="./matcha-icefall-en_US-ljspeech/model-steps-3.onnx", + vocoder="./vocos-22khz-univ.onnx", + tokens="./matcha-icefall-en_US-ljspeech/tokens.txt", + lexicon="./lexicon-en-us.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Today as always, men fall into two groups: slaves and free men." + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-matcha-en-lexicon.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-matcha-en-phonemize.py b/python-api-examples/test-offline-tts-matcha-en-phonemize.py new file mode 100755 index 000000000..81b8f1e35 --- /dev/null +++ b/python-api-examples/test-offline-tts-matcha-en-phonemize.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +""" +Test a Matcha English TTS model using piper_phonemize (no espeak-ng data). + +If you want to use a lexicon instead, please see +test-offline-tts-matcha-en-lexicon.py. + +Supported extra parameters in gen_config.extra (all optional): + - min_words_in_sentence, int, default 5. + Merge adjacent sentences if the number of words is less than this. + - max_words_in_sentence, int, default 20. + Split a sentence into chunks if the number of words exceeds this. + - max_codepoints_in_sentence, int, default 200. + Split a phoneme sequence if it exceeds this many codepoints. + - debug, int, default from model config. + +Install piper_phonemize: + pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +Download model and vocoder: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +""" + +import time + +import sherpa_onnx +import soundfile as sf + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( + acoustic_model="./matcha-icefall-en_US-ljspeech/model-steps-3.onnx", + vocoder="./vocos-22khz-univ.onnx", + tokens="./matcha-icefall-en_US-ljspeech/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Today as always, men fall into two groups: slaves and free men." +# The espeak-ng voice for the phonemizer is taken from the model metadata. +# Fall back to en-us if the model has no language information. +lang = tts.lang +if not lang: + print("Warning: the model has no language information. Falling back to en-us.") + lang = "en-us" +print("Phonemizer language:", lang) + +sentences = phonemize_espeak(text, lang) +phoneme_codepoints = [[ord(c) for c in sentence] for sentence in sentences] + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.phoneme_codepoints = phoneme_codepoints + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-matcha-en-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-matcha-zh-en-lexicon.py b/python-api-examples/test-offline-tts-matcha-zh-en-lexicon.py new file mode 100755 index 000000000..b48a2d699 --- /dev/null +++ b/python-api-examples/test-offline-tts-matcha-zh-en-lexicon.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +Test a Matcha Chinese+English TTS model using lexicon-based phonemization +(no espeak-ng data, no external phonemizer needed). + +If you don't want to use a lexicon, please see +test-offline-tts-matcha-zh-en-phonemize.py which uses pypinyin + piper_phonemize instead. + +Passes two lexicon files (comma-separated): the zh lexicon from the model +directory and the en lexicon (lexicon-en-us.txt). + +Supported extra parameters in gen_config.extra (all optional): + - min_words_in_sentence, int, default 5. + Merge adjacent sentences if the number of words is less than this. + Each CJK character counts as one word; English words are space-separated. + - max_words_in_sentence, int, default 20. + Split a sentence into chunks if the number of words exceeds this. + Splits at punctuation or space boundaries. + - debug, int, default from model config. + +Download model, vocoder, and lexicon: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-en.tar.bz2 + tar xf matcha-icefall-zh-en.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-16khz-univ.onnx + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +""" + +import time + +import sherpa_onnx +import soundfile as sf + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( + acoustic_model="./matcha-icefall-zh-en/model-steps-3.onnx", + vocoder="./vocos-16khz-univ.onnx", + tokens="./matcha-icefall-zh-en/tokens.txt", + lexicon="./matcha-icefall-zh-en/lexicon.txt,./lexicon-en-us.txt", + ), + debug=True, + ), + rule_fsts="./matcha-icefall-zh-en/phone-zh.fst,./matcha-icefall-zh-en/date-zh.fst,./matcha-icefall-zh-en/number-zh.fst", +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = ( + "我最近在学习machine learning，希望能够在未来的artificial intelligence领域有所建树。" + "在这次vocation中，我们计划去Paris欣赏埃菲尔铁塔和卢浮宫的美景。" + "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。" + "开始数字测试。2025年12月4号，拨打110或者189202512043。123456块钱。" + "在这个快速发展的时代，人工智能技术正在改变我们的生活方式。" + "语音合成作为人工智能的重要应用之一，让机器能够用自然流畅的语音与人类进行交流。" +) + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-matcha-zh-en-lexicon.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-matcha-zh-en-phonemize.py b/python-api-examples/test-offline-tts-matcha-zh-en-phonemize.py new file mode 100755 index 000000000..0d7576434 --- /dev/null +++ b/python-api-examples/test-offline-tts-matcha-zh-en-phonemize.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +""" +Test a Matcha Chinese+English TTS model using pinyin + espeak phonemization +(no espeak-ng data needed at runtime). + +If you want to use a lexicon instead, please see +test-offline-tts-matcha-zh-en-lexicon.py. + +Uses pypinyin for Chinese text and piper_phonemize for English text, +then passes combined tokens via GenerationConfig.tokens. + +Supported extra parameters in gen_config.extra (all optional): + - min_words_in_sentence, int, default 5. + Merge adjacent sentences if the number of words is less than this. + - max_words_in_sentence, int, default 20. + Split a sentence into chunks if the number of words exceeds this. + - max_codepoints_in_sentence, int, default 200. + Split a phoneme sequence if it exceeds this many codepoints. + - debug, int, default from model config. + +Install dependencies: + pip install pypinyin kaldifst + pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +Download model and vocoder: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-en.tar.bz2 + tar xf matcha-icefall-zh-en.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-16khz-univ.onnx +""" + +import re +import time + +import sherpa_onnx +import soundfile as sf + +import kaldifst + +try: + from pypinyin import lazy_pinyin, Style +except Exception as ex: + raise RuntimeError(f"{ex}\nPlease run\npip install pypinyin kaldifst") + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( + acoustic_model="./matcha-icefall-zh-en/model-steps-3.onnx", + vocoder="./vocos-16khz-univ.onnx", + tokens="./matcha-icefall-zh-en/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = ( + "我最近在学习machine learning，希望能够在未来的artificial intelligence领域有所建树。" + "在这次vocation中，我们计划去Paris欣赏埃菲尔铁塔和卢浮宫的美景。" + "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。" + "开始数字测试。2025年12月4号，拨打110或者189202512043。123456块钱。" + "在这个快速发展的时代，人工智能技术正在改变我们的生活方式。" + "语音合成作为人工智能的重要应用之一，让机器能够用自然流畅的语音与人类进行交流。" +) + +# Normalize text using kaldifst rule FSTs (same as C++ rule_fsts) +# Converts numbers, dates, phone numbers to Chinese characters +rule_fsts = [ + "./matcha-icefall-zh-en/phone-zh.fst", + "./matcha-icefall-zh-en/date-zh.fst", + "./matcha-icefall-zh-en/number-zh.fst", +] +for fst_path in rule_fsts: + tn = kaldifst.TextNormalizer(fst_path) + text = tn.normalize(text) + +print(f"Normalized text: {text}") + + +def tokenize_mixed(text, lang): + """Split text into Chinese and non-Chinese segments, tokenize each. + + Returns a list of sentences, each sentence is a list of token strings. + Chinese pinyin tokens are multi-char (e.g., 'zhong1', 'guo2'). + English IPA tokens are single-char (e.g., 'h', 'ə', 'l'). + Punctuation is preserved as tokens (e.g., ',', '.', '?'). + """ + # Split into: Chinese chars | ASCII+IPA chars | punctuation chars + segments = re.findall( + r"[一-鿿]+" r"|[a-zA-Z0-9 ,;:!?\-']+" r"|[，。！？；：、…—]", + text, + ) + + all_tokens = [] + for seg in segments: + if re.match(r"[一-鿿]+", seg): + # Chinese segment: convert to pinyin token strings + pinyin_list = lazy_pinyin(seg, style=Style.TONE3) + for py in pinyin_list: + # Neutral tone characters (的, 着, 了, etc.) get no tone number + # from pypinyin. Append '5' to match the lexicon convention. + if not py[-1].isdigit(): + py += "5" + all_tokens.append(py) + elif re.match(r"[，。！？；：、…—]", seg): + # Chinese punctuation: convert to ASCII equivalent + punct_map = { + "，": ",", + "。": ".", + "！": "!", + "？": "?", + "；": ";", + "：": ":", + "、": ",", + "…": "...", + "—": "-", + } + all_tokens.append(punct_map.get(seg, seg)) + else: + # English/ASCII segment: use piper_phonemize + seg = seg.strip() + if not seg: + continue + sentences = phonemize_espeak(seg, lang) + for sentence in sentences: + for c in sentence: + all_tokens.append(c) + + return [all_tokens] + + +# The espeak-ng voice for English segments is taken from the model metadata. +# Fall back to en-us if the model has no language information. +lang = tts.lang +if not lang: + print("Warning: the model has no language information. Falling back to en-us.") + lang = "en-us" +print("Phonemizer language:", lang) + +tokens = tokenize_mixed(text, lang) +print(f"Total tokens: {len(tokens[0])}") + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.tokens = tokens + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-matcha-zh-en-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-matcha-zh-lexicon.py b/python-api-examples/test-offline-tts-matcha-zh-lexicon.py new file mode 100755 index 000000000..27440fe8e --- /dev/null +++ b/python-api-examples/test-offline-tts-matcha-zh-lexicon.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Test a Matcha Chinese TTS model using lexicon-based phonemization +(no espeak-ng data, no external phonemizer needed). + +If you don't want to use a lexicon, please see +test-offline-tts-matcha-zh-phonemize.py which uses pypinyin instead. + +Supported extra parameters in gen_config.extra (all optional): + - min_words_in_sentence, int, default 5. + Merge adjacent sentences if the number of words is less than this. + Each CJK character counts as one word. + - max_words_in_sentence, int, default 20. + Split a sentence into chunks if the number of words exceeds this. + Splits at punctuation or space boundaries. + - debug, int, default from model config. + +Download model and vocoder: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xf matcha-icefall-zh-baker.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +""" + +import time + +import sherpa_onnx +import soundfile as sf + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( + acoustic_model="./matcha-icefall-zh-baker/model-steps-3.onnx", + vocoder="./vocos-22khz-univ.onnx", + tokens="./matcha-icefall-zh-baker/tokens.txt", + lexicon="./matcha-icefall-zh-baker/lexicon.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转。" + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-matcha-zh-lexicon.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-matcha-zh-phonemize.py b/python-api-examples/test-offline-tts-matcha-zh-phonemize.py new file mode 100755 index 000000000..6d4220a12 --- /dev/null +++ b/python-api-examples/test-offline-tts-matcha-zh-phonemize.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +""" +Test a Matcha Chinese TTS model using pinyin phonemization +(no espeak-ng data needed). + +If you want to use a lexicon instead, please see +test-offline-tts-matcha-zh-lexicon.py. + +Uses pypinyin to convert Chinese text to pinyin tokens, then passes them +via GenerationConfig.tokens. + +Install dependencies: + pip install pypinyin kaldifst + +Download model and vocoder: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2 + tar xf matcha-icefall-zh-baker.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +""" + +import time + +import kaldifst +import sherpa_onnx +import soundfile as sf + +try: + from pypinyin import lazy_pinyin, Style +except Exception as ex: + raise RuntimeError(f"{ex}\nPlease run\npip install pypinyin kaldifst") + +text = ( + "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。" + "开始数字测试。2025年12月4号，拨打110或者189202512043。123456块钱。" + "在这个快速发展的时代，人工智能技术正在改变我们的生活方式。" + "语音合成作为人工智能的重要应用之一，让机器能够用自然流畅的语音与人类进行交流。" +) + +# Normalize text using kaldifst rule FSTs (same as C++ rule_fsts) +rule_fsts_files = [ + "./matcha-icefall-zh-baker/phone.fst", + "./matcha-icefall-zh-baker/date.fst", + "./matcha-icefall-zh-baker/number.fst", +] +rule_fsts = [kaldifst.TextNormalizer(fst_path) for fst_path in rule_fsts_files] +for tn in rule_fsts: + text = tn.normalize(text) + +print(f"Normalized text: {text}") + +# Convert Chinese text to pinyin token strings +# lazy_pinyin with TONE3 returns: ['dang1', 'ye4', '，', 'xing1', ...] +# Punctuation is included as separate tokens. +pinyin_list = lazy_pinyin(text, style=Style.TONE3) +tokens = [] +for py in pinyin_list: + py = py.strip() + if not py: + continue + # Punctuation tokens: convert Chinese to ASCII + punct_map = { + "，": ",", + "。": ".", + "！": "!", + "？": "?", + "；": ",", + "：": ",", + "、": ",", + ";": ",", + ":": ",", + } + if py in punct_map: + tokens.append(punct_map[py]) + else: + # Use pinyin as-is (zh model expects no tone suffix for neutral tone) + tokens.append(py) + +print(f"Total tokens: {len(tokens)}") + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + matcha=sherpa_onnx.OfflineTtsMatchaModelConfig( + acoustic_model="./matcha-icefall-zh-baker/model-steps-3.onnx", + vocoder="./vocos-22khz-univ.onnx", + tokens="./matcha-icefall-zh-baker/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.tokens = [tokens] + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-matcha-zh-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-piper-lexicon.py b/python-api-examples/test-offline-tts-piper-lexicon.py new file mode 100755 index 000000000..9c487961e --- /dev/null +++ b/python-api-examples/test-offline-tts-piper-lexicon.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +""" +Test a piper tts model using lexicon-based phonemization +(no espeak-ng data, no external phonemizer needed). + +Download model and lexicon: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + tar xf vits-piper-en_US-amy-low.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +""" + +import time + +import sherpa_onnx +import soundfile as sf + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model="./vits-piper-en_US-amy-low/en_US-amy-low.onnx", + tokens="./vits-piper-en_US-amy-low/tokens.txt", + lexicon="./lexicon-en-us.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Today as always, men fall into two groups: slaves and free men." + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-piper-lexicon.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-piper-phonemize.py b/python-api-examples/test-offline-tts-piper-phonemize.py new file mode 100755 index 000000000..2d90c6805 --- /dev/null +++ b/python-api-examples/test-offline-tts-piper-phonemize.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +""" +Test a piper tts model using piper_phonemize. + +Install piper_phonemize: + pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + tar xf vits-piper-en_US-amy-low.tar.bz2 +""" + +import time + +import sherpa_onnx +import soundfile as sf + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model="./vits-piper-en_US-amy-low/en_US-amy-low.onnx", + tokens="./vits-piper-en_US-amy-low/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Today as always, men fall into two groups: slaves and free men." + +# The espeak-ng voice for the phonemizer is taken from the model metadata. +# Fall back to en-us if the model has no language information. +lang = tts.lang +if not lang: + print("Warning: the model has no language information. Falling back to en-us.") + lang = "en-us" +print("Phonemizer language:", lang) + +sentences = phonemize_espeak(text, lang) +phoneme_codepoints = [[ord(c) for c in sentence] for sentence in sentences] + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.phoneme_codepoints = phoneme_codepoints + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-piper-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-piper-tokens.py b/python-api-examples/test-offline-tts-piper-tokens.py new file mode 100644 index 000000000..e3efa0c2b --- /dev/null +++ b/python-api-examples/test-offline-tts-piper-tokens.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +""" +Test a piper tts model with pre-tokenized input (GenerationConfig.tokens). + +This is the GenerationConfig.tokens counterpart of +./test-offline-tts-piper-phonemize.py: instead of passing +GenerationConfig.phoneme_codepoints (Unicode codepoints) we pass +GenerationConfig.tokens (token strings) and let sherpa-onnx map them to IDs +with the model's tokens.txt. + +Install piper_phonemize: + pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + tar xf vits-piper-en_US-amy-low.tar.bz2 +""" + +import time + +import sherpa_onnx +import soundfile as sf + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model="./vits-piper-en_US-amy-low/en_US-amy-low.onnx", + tokens="./vits-piper-en_US-amy-low/tokens.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +text = "Today as always, men fall into two groups: slaves and free men." + +# The espeak-ng voice for the phonemizer is taken from the model metadata. +# Fall back to en-us if the model has no language information. +lang = tts.lang +if not lang: + print("Warning: the model has no language information. Falling back to en-us.") + lang = "en-us" +print("Phonemizer language:", lang) + +sentences = phonemize_espeak(text, lang) + +# Each sentence is a string of phonemes; each phoneme is one token string. +# sherpa-onnx adds the bos/pad/eos markers itself, exactly as it does for +# phoneme_codepoints, so we pass the bare tokens here. +token_sentences = [[c for c in sentence] for sentence in sentences] +print("tokens", token_sentences) + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.tokens = token_sentences + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-piper-tokens.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/test-offline-tts-zipvoice-lexicon.py b/python-api-examples/test-offline-tts-zipvoice-lexicon.py new file mode 100644 index 000000000..f0d1ffc7e --- /dev/null +++ b/python-api-examples/test-offline-tts-zipvoice-lexicon.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +Test ZipVoice Chinese+English TTS model using lexicon +(no espeak-ng data needed). + +If you don't want to use a lexicon, please see +test-offline-tts-zipvoice-phonemize.py which uses piper_phonemize instead. + +Passes two lexicon files (comma-separated): the zh lexicon from the model +directory and the en lexicon (lexicon-en-us.txt). + +Supported extra parameters in gen_config.extra (all optional): + - min_words_in_sentence, int, default 5. + Merge adjacent sentences if the number of words is less than this. + Each CJK character counts as one word; English words are space-separated. + - max_words_in_sentence, int, default 20. + Split a sentence into chunks if the number of words exceeds this. + Splits at punctuation or space boundaries. + - num_steps, int, default 4. + Number of flow-matching denoising steps. + - debug, int, default from model config. + +Download model, vocoder, and lexicon: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 + tar xf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +""" + +import time + +import numpy as np +import sherpa_onnx +import soundfile as sf + +model_dir = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia" + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + zipvoice=sherpa_onnx.OfflineTtsZipvoiceModelConfig( + tokens=f"{model_dir}/tokens.txt", + encoder=f"{model_dir}/encoder.int8.onnx", + decoder=f"{model_dir}/decoder.int8.onnx", + vocoder="./vocos_24khz.onnx", + lexicon=f"{model_dir}/lexicon.txt,./lexicon-en-us.txt", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +# Load reference audio +ref_wav = f"{model_dir}/test_wavs/news-female.wav" +ref_audio, ref_sr = sf.read(ref_wav) +ref_audio = ref_audio.astype(np.float32) + +text = "在这个快速发展的时代，人工智能artificial intelligence技术正在改变我们的生活方式。语音合成speech synthesis作为重要应用之一，让机器能够用自然流畅的语音与人类进行交流。" + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +gen_config.reference_audio = ref_audio.tolist() +gen_config.reference_sample_rate = ref_sr +gen_config.reference_text = "各位村民, 大家新年好! 近期, 湖北省武汉市等多个地区" + +start = time.time() +audio = tts.generate(text, gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-zipvoice-lexicon.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") \ No newline at end of file diff --git a/python-api-examples/test-offline-tts-zipvoice-phonemize.py b/python-api-examples/test-offline-tts-zipvoice-phonemize.py new file mode 100644 index 000000000..14c084da8 --- /dev/null +++ b/python-api-examples/test-offline-tts-zipvoice-phonemize.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +""" +Test ZipVoice Chinese+English TTS model. + +If you want to use a lexicon instead, please see +test-offline-tts-zipvoice-lexicon.py. + +Chinese text uses pypinyin for phonemization (initial0 final_tone format, +e.g., "好" -> "h0 ao3"). +English text uses piper_phonemize (espeak-ng) for IPA phonemization. + +Supported extra parameters in gen_config.extra (all optional): + - min_words_in_sentence, int, default 5. + Merge adjacent sentences if the number of words is less than this. + - max_words_in_sentence, int, default 20. + Split a sentence into chunks if the number of words exceeds this. + - num_steps, int, default 4. + Number of flow-matching denoising steps. + - debug, int, default from model config. + +Install dependencies: + pip install pypinyin + pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + +Download model: + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 + tar xf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx +""" + +import re +import time + +import numpy as np +import sherpa_onnx +import soundfile as sf + +try: + from pypinyin import Style, lazy_pinyin + from pypinyin.contrib.tone_convert import to_finals_tone3, to_initials +except Exception as ex: + raise RuntimeError(f"{ex}\nPlease run\npip install pypinyin") + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + + +def get_initial_final(token): + """Convert a pinyin token to ZipVoice format: initial0 final_tone. + + Example: "hao3" -> "h0 ao3", "de" -> "d0 e5" + """ + initial = to_initials(token, strict=False) + final = to_finals_tone3(token, strict=False, neutral_tone_with_five=True) + + ans = "" + if initial: + ans = initial + "0" + if final: + ans += f" {final}" + return ans + + +def tokenize_mixed(text, lang): + """Tokenize mixed Chinese+English text for ZipVoice. + + Chinese: pypinyin -> initial0 final_tone format + English: piper_phonemize -> IPA phonemes + """ + # Replace Chinese punctuations with ASCII equivalents + for zh, en in [("，", ","), ("。", "."), ("！", "!"), ("？", "?"), + ("；", ";"), ("：", ":"), ("、", ",")]: + text = text.replace(zh, en) + + # Split into: Chinese chars | punctuation | English/ASCII text + segments = re.findall( + r"[一-鿿]+" + r"|[,.\!?;:]" + r"|[a-zA-Z0-9 ]+", + text, + ) + + all_tokens = [] + for seg in segments: + if re.match(r"[一-鿿]+", seg): + # Chinese segment: convert each character to initial0 final + for ch in seg: + py = lazy_pinyin(ch, style=Style.TONE3, + tone_sandhi=True, + neutral_tone_with_five=True) + if py: + initial_final = get_initial_final(py[0]) + for token in initial_final.split(): + all_tokens.append(token) + elif re.match(r"[,.\!?;:]", seg): + # Punctuation: add directly as ASCII token + all_tokens.append(seg) + else: + # English/ASCII text: use piper_phonemize + seg = seg.strip() + if not seg: + continue + sentences = phonemize_espeak(seg, lang) + for sentence in sentences: + for c in sentence: + all_tokens.append(c) + + return [all_tokens] + + +model_dir = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia" + +tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + zipvoice=sherpa_onnx.OfflineTtsZipvoiceModelConfig( + tokens=f"{model_dir}/tokens.txt", + encoder=f"{model_dir}/encoder.int8.onnx", + decoder=f"{model_dir}/decoder.int8.onnx", + vocoder="./vocos_24khz.onnx", + ), + debug=True, + ), +) +tts = sherpa_onnx.OfflineTts(tts_config) + +# Load reference audio +ref_wav = f"{model_dir}/test_wavs/news-female.wav" +ref_audio, ref_sr = sf.read(ref_wav) +ref_audio = ref_audio.astype(np.float32) + +ref_text = "各位村民, 大家新年好! 近期, 湖北省武汉市等多个地区" +text = "在这个快速发展的时代，人工智能artificial intelligence技术正在改变我们的生活方式。语音合成speech synthesis作为重要应用之一，让机器能够用自然流畅的语音与人类进行交流。" + +# Tokenize reference text and generated text separately +# The espeak-ng voice for English segments is taken from the model metadata. +# Fall back to en-us if the model has no language information. +lang = tts.lang +if not lang: + print("Warning: the model has no language information. Falling back to en-us.") + lang = "en-us" +print("Phonemizer language:", lang) + +ref_tokens = tokenize_mixed(ref_text, lang) +gen_tokens = tokenize_mixed(text, lang) + +gen_config = sherpa_onnx.GenerationConfig() +gen_config.sid = 0 +gen_config.speed = 1.0 +# tokens[0] = reference text tokens, tokens[1..] = generated text tokens +gen_config.tokens = ref_tokens + gen_tokens +gen_config.reference_audio = ref_audio.tolist() +gen_config.reference_sample_rate = ref_sr +gen_config.reference_text = ref_text + +print(f"Reference text: {ref_text}") +print(f"Synthesis text: {text}") + +start = time.time() +audio = tts.generate("", gen_config) +end = time.time() + +assert len(audio.samples) > 0, "No audio generated!" +assert audio.sample_rate > 0, "Invalid sample rate!" + +filename = "test-zipvoice-phonemize.wav" +sf.write(filename, audio.samples, samplerate=audio.sample_rate) + +elapsed_seconds = end - start +audio_duration = len(audio.samples) / audio.sample_rate +rtf = elapsed_seconds / audio_duration + +print(f"Saved to {filename}") +print(f"Elapsed seconds: {elapsed_seconds:.3f}") +print(f"Audio duration in seconds: {audio_duration:.3f}") +print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {rtf:.3f}") diff --git a/python-api-examples/zipvoice-tts-play.py b/python-api-examples/zipvoice-tts-play.py index 15a1fe58f..3a966ff13 100644 --- a/python-api-examples/zipvoice-tts-play.py +++ b/python-api-examples/zipvoice-tts-play.py @@ -17,6 +17,8 @@ rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/zipvoice-tts-play.py You can find more models at @@ -56,8 +58,7 @@ def create_tts(): tokens="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt", encoder="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx", decoder="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx", - data_dir="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data", - lexicon="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt", + lexicon="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt", vocoder="./vocos_24khz.onnx", ), debug=False, diff --git a/python-api-examples/zipvoice-tts.py b/python-api-examples/zipvoice-tts.py index 35d256af3..9ee6b4bd1 100644 --- a/python-api-examples/zipvoice-tts.py +++ b/python-api-examples/zipvoice-tts.py @@ -15,6 +15,8 @@ rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + python3 ./python-api-examples/zipvoice-tts.py You can find more models at @@ -40,8 +42,7 @@ def create_tts(): tokens="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt", encoder="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx", decoder="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx", - data_dir="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data", - lexicon="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt", + lexicon="./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt", vocoder="./vocos_24khz.onnx", ), debug=False, diff --git a/rust-api-examples/.cargo/config.toml b/rust-api-examples/.cargo/config.toml new file mode 100644 index 000000000..f18fca75a --- /dev/null +++ b/rust-api-examples/.cargo/config.toml @@ -0,0 +1,3 @@ +[patch.crates-io] +sherpa-onnx = { path = "../sherpa-onnx/rust/sherpa-onnx" } +sherpa-onnx-sys = { path = "../sherpa-onnx/rust/sherpa-onnx-sys" } diff --git a/rust-api-examples/Cargo.lock b/rust-api-examples/Cargo.lock index 8566c0399..e8724fbbb 100644 --- a/rust-api-examples/Cargo.lock +++ b/rust-api-examples/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "alsa" version = "0.9.1" @@ -86,6 +97,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -110,6 +130,15 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -125,6 +154,12 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" @@ -141,6 +176,15 @@ dependencies = [ "libc", ] +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + [[package]] name = "bzip2-sys" version = "0.1.13+1.0.8" @@ -158,6 +202,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -179,6 +225,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.60" @@ -210,7 +266,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -235,6 +291,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "coreaudio-rs" version = "0.13.0" @@ -275,6 +337,30 @@ dependencies = [ "windows", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -284,6 +370,22 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctrlc" version = "3.5.2" @@ -301,6 +403,40 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "deflate64" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_arbitrary" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "dispatch2" version = "0.3.0" @@ -321,7 +457,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -400,6 +536,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -411,6 +557,31 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -423,6 +594,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -535,6 +715,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -558,7 +747,7 @@ dependencies = [ "combine", "jni-sys", "log", - "thiserror", + "thiserror 1.0.69", "walkdir", "windows-sys 0.45.0", ] @@ -569,6 +758,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.89" @@ -615,6 +814,27 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "mach2" version = "0.4.3" @@ -651,7 +871,7 @@ dependencies = [ "log", "ndk-sys", "num_enum", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -681,6 +901,12 @@ dependencies = [ "libc", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-derive" version = "0.4.2" @@ -689,7 +915,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -720,7 +946,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -807,6 +1033,16 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -819,6 +1055,29 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "piper-phonemize" +version = "1.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c8504db04647f8929286fcffeb503a75c0e20bc9092d396b1208ae908251d4" +dependencies = [ + "bzip2 0.4.4", + "piper-phonemize-sys", + "tar", + "walkdir", +] + +[[package]] +name = "piper-phonemize-sys" +version = "1.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1ee1bf4669c378c95cb3848d11cb1dd29d0d0dcc98690b20845401b80ca5a1" +dependencies = [ + "bzip2 0.4.4", + "tar", + "ureq", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -840,6 +1099,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -867,6 +1132,18 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "redox_syscall" version = "0.7.3" @@ -884,7 +1161,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -892,12 +1169,13 @@ dependencies = [ [[package]] name = "rust-api-examples" -version = "1.13.6" +version = "1.13.8" dependencies = [ "anyhow", "clap", "cpal", "ctrlc", + "piper-phonemize", "serde_json", "sherpa-onnx", ] @@ -992,7 +1270,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1009,10 +1287,19 @@ dependencies = [ ] [[package]] -name = "sherpa-onnx" -version = "1.13.6" +name = "sha1" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93440301c4240bd5d83496ab9adeee889be48d9428f9c01c37e5ae4ef33ba9e2" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sherpa-onnx" +version = "1.13.8" dependencies = [ "serde", "serde_json", @@ -1021,13 +1308,12 @@ dependencies = [ [[package]] name = "sherpa-onnx-sys" -version = "1.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d627abc4624c845e628ad87cca4ff9ac265fc643021d155f2721ac589eeb324" +version = "1.13.8" dependencies = [ - "bzip2", + "bzip2 0.4.4", "tar", "ureq", + "zip", ] [[package]] @@ -1083,6 +1369,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d62a2e0561533f2ca2561d0cf27fd9fedb640a1bf2616ff5d5c80d99017faadc" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1091,7 +1388,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1111,7 +1408,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", ] [[package]] @@ -1122,9 +1428,39 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + [[package]] name = "tinystr" version = "0.8.2" @@ -1165,6 +1501,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1217,6 +1559,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -1233,6 +1581,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.112" @@ -1279,7 +1636,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1521,6 +1878,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.2" @@ -1537,6 +1900,15 @@ dependencies = [ "rustix", ] +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.8.1" @@ -1556,7 +1928,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -1577,7 +1949,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -1586,6 +1958,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" @@ -1617,7 +2003,37 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.5.2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.4", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.21", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", ] [[package]] @@ -1625,3 +2041,43 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.1+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/rust-api-examples/Cargo.toml b/rust-api-examples/Cargo.toml index a00b3e8e1..67f4488b3 100644 --- a/rust-api-examples/Cargo.toml +++ b/rust-api-examples/Cargo.toml @@ -13,15 +13,44 @@ serde_json = "1.0" cpal = { version = "0.16", optional = true } # cross-platform audio I/O ctrlc = "3.4" +# External phonemizer used only by the *-phonemize examples. Optional so the +# lexicon-based examples never build it. Linked statically by default (its +# own default feature) so the examples run without extra library-path setup; +# on Android it links shared instead (piper-phonemize-sys forces this, same +# as sherpa-onnx-sys). +piper-phonemize = { version = "1.5.8", optional = true } + [features] # Default to static linking so users can run examples without setting library paths default = ["static"] # Feature for using microphone mic = ["cpal"] +# Feature for the *-phonemize examples +phonemize = ["dep:piper-phonemize"] static = ["sherpa-onnx/static"] shared = ["sherpa-onnx/shared"] +[[example]] +name = "vits_tts_phonemize" +required-features = ["phonemize"] + +[[example]] +name = "vits_tts_de_phonemize" +required-features = ["phonemize"] + +[[example]] +name = "kitten_tts_en_phonemize" +required-features = ["phonemize"] + +[[example]] +name = "kokoro_tts_en_phonemize" +required-features = ["phonemize"] + +[[example]] +name = "matcha_tts_en_phonemize" +required-features = ["phonemize"] + [[example]] name = "streaming_zipformer_microphone" required-features = ["mic"] diff --git a/rust-api-examples/README.md b/rust-api-examples/README.md index 5917ad5c5..29d87176e 100644 --- a/rust-api-examples/README.md +++ b/rust-api-examples/README.md @@ -46,52 +46,56 @@ in your own Cargo project, see | 1 | [version](#example-1-show-sherpa-onnx-version) | Show the sherpa-onnx version | | 2 | [pocket_tts](#example-2-tts-with-pocket-tts-zero-shot-voice-cloning) | Text-to-speech with zero-shot voice cloning using a reference audio | | 3 | [supertonic_tts](#example-3-tts-with-supertonic-tts) | Text-to-speech with Supertonic TTS (multi-speaker, multi-language) | -| 4 | [zipvoice_tts](#example-4-tts-with-zipvoice-zero-shot-voice-cloning) | Text-to-speech with ZipVoice zero-shot voice cloning | -| 5 | [vits_tts](#example-5-tts-with-vits-english-piper) | Text-to-speech with a standalone VITS Piper model (English) | -| 6 | [vits_tts](#example-6-tts-with-vits-german-piper) | Text-to-speech with a standalone VITS Piper model (German) | -| 7 | [matcha_tts_en](#example-7-tts-with-matcha-english) | Text-to-speech with Matcha TTS (English) | -| 8 | [matcha_tts_zh](#example-8-tts-with-matcha-chinese) | Text-to-speech with Matcha TTS (Chinese) | -| 9 | [kokoro_tts_en](#example-9-tts-with-kokoro-english) | Text-to-speech with Kokoro TTS (English) | -| 10 | [kokoro_tts_zh_en](#example-10-tts-with-kokoro-chinese--english) | Text-to-speech with Kokoro TTS (Chinese + English) | -| 11 | [kitten_tts_en](#example-11-tts-with-kitten-english) | Text-to-speech with Kitten TTS (English) | -| 12 | [streaming_zipformer_en](#example-12-asr-with-streaming-zipformer-english) | Streaming ASR with zipformer transducer (English) | -| 13 | [streaming_zipformer_zh_en](#example-13-asr-with-streaming-zipformer-chinese--english) | Streaming ASR with zipformer transducer (Chinese + English) | -| 14 | [streaming_zipformer_microphone](#example-14-asr-with-streaming-zipformer-with-a-microphone-real-time-asr) | Real-time streaming ASR from microphone input | -| 15 | [zipformer_en](#example-15-asr-with-non-streaming-zipformer-english) | Non-streaming ASR with zipformer transducer (English) | -| 16 | [zipformer_zh_en](#example-16-asr-with-non-streaming-zipformer-chinese--english) | Non-streaming ASR with zipformer transducer (Chinese + English) | -| 17 | [zipformer_vi](#example-17-asr-with-non-streaming-zipformer-vietnamese) | Non-streaming ASR with zipformer transducer (Vietnamese) | -| 18 | [nemo_parakeet](#example-18-asr-with-non-streaming-nemo-parakeet-english) | Non-streaming ASR with Nemo Parakeet TDT transducer (English) | -| 19 | [fire_red_asr_ctc](#example-19-asr-with-non-streaming-fireredasr-ctc-chinese--english) | Non-streaming ASR with FireRedASR CTC model (Chinese + English) | -| 20 | [moonshine_v2](#example-20-asr-with-non-streaming-moonshine-v2-english) | Non-streaming ASR with Moonshine v2 (English) | -| 21 | [sense_voice](#example-21-asr-with-non-streaming-sensevoice) | Non-streaming ASR with SenseVoice (Chinese, English, Japanese, Korean, Cantonese) | -| 22 | [qwen3_asr](#example-22-asr-with-non-streaming-qwen3-asr) | Non-streaming ASR with Qwen3 ASR (multilingual) | -| 23 | [cohere_transcribe](#example-23-asr-with-non-streaming-cohere-transcribe) | Non-streaming ASR with Cohere Transcribe (multilingual) | -| 24 | [silero_vad_remove_silence](#example-24-remove-silences-from-a-file-using-silerovad) | Remove silences from an audio file using Silero VAD | -| 25 | [offline_speech_enhancement_gtcrn](#example-25-offline-speech-enhancement-with-gtcrn) | Offline speech enhancement with GTCRN | -| 26 | [offline_speech_enhancement_dpdfnet](#example-26-offline-speech-enhancement-with-dpdfnet) | Offline speech enhancement with DPDFNet | -| 27 | [streaming_speech_enhancement_gtcrn](#example-27-streaming-speech-enhancement-with-gtcrn) | Streaming speech enhancement with GTCRN | -| 28 | [streaming_speech_enhancement_dpdfnet](#example-28-streaming-speech-enhancement-with-dpdfnet) | Streaming speech enhancement with DPDFNet | -| 29 | [online_punctuation](#example-29-online-punctuation) | Add punctuation to text using online punctuation model | -| 30 | [keyword_spotter](#example-30-keyword-spotter) | Detect keywords from audio using a Zipformer KWS model | -| 31 | [spoken_language_identification](#example-31-spoken-language-identification) | Detect the spoken language in a wave file using Whisper | -| 32 | [offline_punctuation](#example-32-offline-punctuation) | Add punctuation to text using an offline punctuation model | -| 33 | [audio_tagging_zipformer](#example-33-audio-tagging-with-a-zipformer-model) | Audio tagging with a Zipformer model | -| 34 | [audio_tagging_ced](#example-34-audio-tagging-with-a-ced-model) | Audio tagging with a CED model | -| 35 | [speaker_embedding_extractor](#example-35-speaker-embedding-extractor) | Compute a speaker embedding from a wave file | -| 36 | [speaker_embedding_manager](#example-36-speaker-embedding-manager) | Register, search, verify, and remove speakers using embeddings | -| 37 | [speaker_embedding_cosine_similarity](#example-37-speaker-embedding-cosine-similarity) | Compute cosine similarity from three speaker embeddings | -| 38 | [offline_speaker_diarization](#example-38-offline-speaker-diarization) | Offline speaker diarization with pyannote segmentation and 3D-Speaker embeddings | -| 39 | [sense_voice_simulate_streaming_microphone](#example-39-simulated-streaming-asr-with-sensevoice-and-vad-from-microphone) | Simulated streaming ASR with SenseVoice and VAD from microphone | -| 40 | [fire_red_asr_ctc_simulate_streaming_microphone](#example-40-simulated-streaming-asr-with-fireredasr-ctc-and-vad-from-microphone) | Simulated streaming ASR with FireRedASR CTC and VAD from microphone | -| 41 | [parakeet_tdt_ctc_simulate_streaming_microphone](#example-41-simulated-streaming-asr-with-parakeet-tdt-ctc-and-vad-from-microphone) | Simulated streaming ASR with Parakeet TDT CTC and VAD from microphone | -| 42 | [parakeet_tdt_simulate_streaming_microphone](#example-42-simulated-streaming-asr-with-parakeet-tdt-transducer-and-vad-from-microphone) | Simulated streaming ASR with Parakeet TDT transducer and VAD from microphone | -| 43 | [wenet_ctc_simulate_streaming_microphone](#example-43-simulated-streaming-asr-with-wenet-ctc-and-vad-from-microphone) | Simulated streaming ASR with WeNet CTC and VAD from microphone | -| 44 | [zipformer_ctc_simulate_streaming_microphone](#example-44-simulated-streaming-asr-with-zipformer-ctc-and-vad-from-microphone) | Simulated streaming ASR with Zipformer CTC and VAD from microphone | -| 45 | [zipformer_transducer_simulate_streaming_microphone](#example-45-simulated-streaming-asr-with-zipformer-transducer-and-vad-from-microphone) | Simulated streaming ASR with Zipformer transducer and VAD from microphone | -| 46 | [zipformer_transducer_simulate_streaming_microphone](#example-46-simulated-streaming-asr-with-zipformer-transducer-japanese-and-vad-from-microphone) | Simulated streaming ASR with Zipformer transducer (Japanese) and VAD from microphone | -| 47 | [qwen3_asr_simulate_streaming_microphone](#example-47-simulated-streaming-asr-with-qwen3-asr-and-vad-from-microphone) | Simulated streaming ASR with Qwen3 ASR and VAD from microphone | -| 48 | [whisper](#example-48-asr-with-non-streaming-whisper) | Non-streaming ASR with Whisper (multilingual) | -| 51 | [paraformer](#example-51-asr-with-non-streaming-paraformer) | Non-streaming ASR with Paraformer | +| 4 | [zipvoice_tts_lexicon](#example-4-tts-with-zipvoice-zero-shot-voice-cloning) | Text-to-speech with ZipVoice zero-shot voice cloning (lexicon) | +| 5 | [vits_tts_lexicon](#example-5-tts-with-vits-english-piper-lexicon) | Text-to-speech with a standalone VITS Piper model (English, lexicon) | +| 6 | [vits_tts_phonemize](#example-6-tts-with-vits-english-piper-piper-phonemize) | Text-to-speech with a standalone VITS Piper model (English, piper-phonemize) | +| 7 | [vits_tts_de_phonemize](#example-7-tts-with-vits-german-piper-piper-phonemize) | Text-to-speech with a standalone VITS Piper model (German, piper-phonemize — no German lexicon is published yet) | +| 8 | [matcha_tts_en_lexicon](#example-8-tts-with-matcha-english-lexicon) | Text-to-speech with Matcha TTS (English, lexicon) | +| 9 | [matcha_tts_en_phonemize](#example-9-tts-with-matcha-english-piper-phonemize) | Text-to-speech with Matcha TTS (English, piper-phonemize) | +| 10 | [matcha_tts_zh_lexicon](#example-10-tts-with-matcha-chinese) | Text-to-speech with Matcha TTS (Chinese, lexicon) | +| 11 | [kokoro_tts_en_lexicon](#example-11-tts-with-kokoro-english-lexicon) | Text-to-speech with Kokoro TTS (English, lexicon) | +| 12 | [kokoro_tts_en_phonemize](#example-12-tts-with-kokoro-english-piper-phonemize) | Text-to-speech with Kokoro TTS (English, piper-phonemize) | +| 13 | [kokoro_tts_zh_en_lexicon](#example-13-tts-with-kokoro-chinese--english) | Text-to-speech with Kokoro TTS (Chinese + English, lexicon) | +| 14 | [kitten_tts_en_lexicon](#example-14-tts-with-kitten-english-lexicon) | Text-to-speech with Kitten TTS (English, lexicon) | +| 15 | [kitten_tts_en_phonemize](#example-15-tts-with-kitten-english-piper-phonemize) | Text-to-speech with Kitten TTS (English, piper-phonemize) | +| 16 | [streaming_zipformer_en](#example-16-asr-with-streaming-zipformer-english) | Streaming ASR with zipformer transducer (English) | +| 17 | [streaming_zipformer_zh_en](#example-17-asr-with-streaming-zipformer-chinese--english) | Streaming ASR with zipformer transducer (Chinese + English) | +| 18 | [streaming_zipformer_microphone](#example-18-asr-with-streaming-zipformer-with-a-microphone-real-time-asr) | Real-time streaming ASR from microphone input | +| 19 | [zipformer_en](#example-19-asr-with-non-streaming-zipformer-english) | Non-streaming ASR with zipformer transducer (English) | +| 20 | [zipformer_zh_en](#example-20-asr-with-non-streaming-zipformer-chinese--english) | Non-streaming ASR with zipformer transducer (Chinese + English) | +| 21 | [zipformer_vi](#example-21-asr-with-non-streaming-zipformer-vietnamese) | Non-streaming ASR with zipformer transducer (Vietnamese) | +| 22 | [nemo_parakeet](#example-22-asr-with-non-streaming-nemo-parakeet-english) | Non-streaming ASR with Nemo Parakeet TDT transducer (English) | +| 23 | [fire_red_asr_ctc](#example-23-asr-with-non-streaming-fireredasr-ctc-chinese--english) | Non-streaming ASR with FireRedASR CTC model (Chinese + English) | +| 24 | [moonshine_v2](#example-24-asr-with-non-streaming-moonshine-v2-english) | Non-streaming ASR with Moonshine v2 (English) | +| 25 | [sense_voice](#example-25-asr-with-non-streaming-sensevoice) | Non-streaming ASR with SenseVoice (Chinese, English, Japanese, Korean, Cantonese) | +| 26 | [qwen3_asr](#example-26-asr-with-non-streaming-qwen3-asr) | Non-streaming ASR with Qwen3 ASR (multilingual) | +| 27 | [cohere_transcribe](#example-27-asr-with-non-streaming-cohere-transcribe) | Non-streaming ASR with Cohere Transcribe (multilingual) | +| 28 | [silero_vad_remove_silence](#example-28-remove-silences-from-a-file-using-silerovad) | Remove silences from an audio file using Silero VAD | +| 29 | [offline_speech_enhancement_gtcrn](#example-29-offline-speech-enhancement-with-gtcrn) | Offline speech enhancement with GTCRN | +| 30 | [offline_speech_enhancement_dpdfnet](#example-30-offline-speech-enhancement-with-dpdfnet) | Offline speech enhancement with DPDFNet | +| 31 | [streaming_speech_enhancement_gtcrn](#example-31-streaming-speech-enhancement-with-gtcrn) | Streaming speech enhancement with GTCRN | +| 32 | [streaming_speech_enhancement_dpdfnet](#example-32-streaming-speech-enhancement-with-dpdfnet) | Streaming speech enhancement with DPDFNet | +| 33 | [online_punctuation](#example-33-online-punctuation) | Add punctuation to text using online punctuation model | +| 34 | [keyword_spotter](#example-34-keyword-spotter) | Detect keywords from audio using a Zipformer KWS model | +| 35 | [spoken_language_identification](#example-35-spoken-language-identification) | Detect the spoken language in a wave file using Whisper | +| 36 | [offline_punctuation](#example-36-offline-punctuation) | Add punctuation to text using an offline punctuation model | +| 37 | [audio_tagging_zipformer](#example-37-audio-tagging-with-a-zipformer-model) | Audio tagging with a Zipformer model | +| 38 | [audio_tagging_ced](#example-38-audio-tagging-with-a-ced-model) | Audio tagging with a CED model | +| 39 | [speaker_embedding_extractor](#example-39-speaker-embedding-extractor) | Compute a speaker embedding from a wave file | +| 40 | [speaker_embedding_manager](#example-40-speaker-embedding-manager) | Register, search, verify, and remove speakers using embeddings | +| 41 | [speaker_embedding_cosine_similarity](#example-41-speaker-embedding-cosine-similarity) | Compute cosine similarity from three speaker embeddings | +| 42 | [offline_speaker_diarization](#example-42-offline-speaker-diarization) | Offline speaker diarization with pyannote segmentation and 3D-Speaker embeddings | +| 43 | [sense_voice_simulate_streaming_microphone](#example-43-simulated-streaming-asr-with-sensevoice-and-vad-from-microphone) | Simulated streaming ASR with SenseVoice and VAD from microphone | +| 44 | [fire_red_asr_ctc_simulate_streaming_microphone](#example-44-simulated-streaming-asr-with-fireredasr-ctc-and-vad-from-microphone) | Simulated streaming ASR with FireRedASR CTC and VAD from microphone | +| 45 | [parakeet_tdt_ctc_simulate_streaming_microphone](#example-45-simulated-streaming-asr-with-parakeet-tdt-ctc-and-vad-from-microphone) | Simulated streaming ASR with Parakeet TDT CTC and VAD from microphone | +| 46 | [parakeet_tdt_simulate_streaming_microphone](#example-46-simulated-streaming-asr-with-parakeet-tdt-transducer-and-vad-from-microphone) | Simulated streaming ASR with Parakeet TDT transducer and VAD from microphone | +| 47 | [wenet_ctc_simulate_streaming_microphone](#example-47-simulated-streaming-asr-with-wenet-ctc-and-vad-from-microphone) | Simulated streaming ASR with WeNet CTC and VAD from microphone | +| 48 | [zipformer_ctc_simulate_streaming_microphone](#example-48-simulated-streaming-asr-with-zipformer-ctc-and-vad-from-microphone) | Simulated streaming ASR with Zipformer CTC and VAD from microphone | +| 49 | [zipformer_transducer_simulate_streaming_microphone](#example-49-simulated-streaming-asr-with-zipformer-transducer-and-vad-from-microphone) | Simulated streaming ASR with Zipformer transducer and VAD from microphone | +| 50 | [zipformer_transducer_simulate_streaming_microphone](#example-50-simulated-streaming-asr-with-zipformer-transducer-japanese-and-vad-from-microphone) | Simulated streaming ASR with Zipformer transducer (Japanese) and VAD from microphone | +| 51 | [qwen3_asr_simulate_streaming_microphone](#example-51-simulated-streaming-asr-with-qwen3-asr-and-vad-from-microphone) | Simulated streaming ASR with Qwen3 ASR and VAD from microphone | +| 52 | [whisper](#example-52-asr-with-non-streaming-whisper) | Non-streaming ASR with Whisper (multilingual) | +| 55 | [paraformer](#example-55-asr-with-non-streaming-paraformer) | Non-streaming ASR with Paraformer | ## Run it @@ -124,218 +128,247 @@ to check the RPATH for shared builds. ### Example 4: TTS with ZipVoice zero-shot voice cloning ```bash -./run-zipvoice-tts.sh +./run-zipvoice-tts-lexicon.sh ``` -### Example 5: TTS with VITS (English Piper) +### Example 5: TTS with VITS (English Piper, lexicon) ```bash -./run-vits-en.sh +./run-vits-en-lexicon.sh ``` -### Example 6: TTS with VITS (German Piper) +### Example 6: TTS with VITS (English Piper, piper-phonemize) + +The text is phonemized outside of sherpa-onnx with the piper-phonemize crate. ```bash -./run-vits-de.sh +./run-vits-en-phonemize.sh ``` -### Example 7: TTS with Matcha (English) +### Example 7: TTS with VITS (German Piper, piper-phonemize) + +There is no published German lexicon yet, so this example uses the +piper-phonemize frontend. ```bash -./run-matcha-tts-en.sh +./run-vits-de-phonemize.sh ``` -### Example 8: TTS with Matcha (Chinese) +### Example 8: TTS with Matcha (English, lexicon) ```bash -./run-matcha-tts-zh.sh +./run-matcha-tts-en-lexicon.sh ``` -### Example 9: TTS with Kokoro (English) +### Example 9: TTS with Matcha (English, piper-phonemize) ```bash -./run-kokoro-tts-en.sh +./run-matcha-tts-en-phonemize.sh ``` -### Example 10: TTS with Kokoro (Chinese + English) +### Example 10: TTS with Matcha (Chinese) ```bash -./run-kokoro-tts-zh-en.sh +./run-matcha-tts-zh-lexicon.sh ``` -### Example 11: TTS with Kitten (English) +### Example 11: TTS with Kokoro (English, lexicon) ```bash -./run-kitten-tts-en.sh +./run-kokoro-tts-en-lexicon.sh ``` -### Example 12: ASR with streaming zipformer (English) +### Example 12: TTS with Kokoro (English, piper-phonemize) + +```bash +./run-kokoro-tts-en-phonemize.sh +``` + +### Example 13: TTS with Kokoro (Chinese + English) + +```bash +./run-kokoro-tts-zh-en-lexicon.sh +``` + +### Example 14: TTS with Kitten (English, lexicon) + +```bash +./run-kitten-tts-en-lexicon.sh +``` + +### Example 15: TTS with Kitten (English, piper-phonemize) + +```bash +./run-kitten-tts-en-phonemize.sh +``` + +### Example 16: ASR with streaming zipformer (English) ```bash ./run-streaming-zipformer-en.sh ``` -### Example 13: ASR with streaming zipformer (Chinese + English) +### Example 17: ASR with streaming zipformer (Chinese + English) ```bash ./run-streaming-zipformer-zh-en.sh ``` -### Example 14: ASR with streaming zipformer (with a microphone, real-time ASR) +### Example 18: ASR with streaming zipformer (with a microphone, real-time ASR) ```bash ./run-streaming-zipformer-microphone-zh-en.sh ``` -### Example 15: ASR with non-streaming zipformer (English) +### Example 19: ASR with non-streaming zipformer (English) ```bash ./run-zipformer-en.sh ``` -### Example 16: ASR with non-streaming zipformer (Chinese + English) +### Example 20: ASR with non-streaming zipformer (Chinese + English) ```bash ./run-zipformer-zh-en.sh ``` -### Example 17: ASR with non-streaming zipformer (Vietnamese) +### Example 21: ASR with non-streaming zipformer (Vietnamese) ```bash ./run-zipformer-vi.sh ``` -### Example 18: ASR with non-streaming Nemo Parakeet (English) +### Example 22: ASR with non-streaming Nemo Parakeet (English) ```bash ./run-nemo-parakeet-en.sh ``` -### Example 19: ASR with non-streaming FireRedASR CTC (Chinese + English) +### Example 23: ASR with non-streaming FireRedASR CTC (Chinese + English) ```bash ./run-fire-red-asr-ctc.sh ``` -### Example 20: ASR with non-streaming Moonshine v2 (English) +### Example 24: ASR with non-streaming Moonshine v2 (English) ```bash ./run-moonshine-v2.sh ``` -### Example 21: ASR with non-streaming SenseVoice +### Example 25: ASR with non-streaming SenseVoice ```bash ./run-sense-voice.sh ``` -### Example 22: ASR with non-streaming Qwen3 ASR +### Example 26: ASR with non-streaming Qwen3 ASR ```bash ./run-qwen3-asr.sh ``` -### Example 23: ASR with non-streaming Cohere Transcribe +### Example 27: ASR with non-streaming Cohere Transcribe ```bash ./run-cohere-transcribe.sh ``` -### Example 24: Remove silences from a file using SileroVAD +### Example 28: Remove silences from a file using SileroVAD ```bash ./run-silero-vad-remove-silence.sh ``` -### Example 25: Offline speech enhancement with GTCRN +### Example 29: Offline speech enhancement with GTCRN ```bash ./run-offline-speech-enhancement-gtcrn.sh ``` -### Example 26: Offline speech enhancement with DPDFNet +### Example 30: Offline speech enhancement with DPDFNet ```bash ./run-offline-speech-enhancement-dpdfnet.sh ``` -### Example 27: Streaming speech enhancement with GTCRN +### Example 31: Streaming speech enhancement with GTCRN ```bash ./run-streaming-speech-enhancement-gtcrn.sh ``` -### Example 28: Streaming speech enhancement with DPDFNet +### Example 32: Streaming speech enhancement with DPDFNet ```bash ./run-streaming-speech-enhancement-dpdfnet.sh ``` -### Example 29: Online punctuation +### Example 33: Online punctuation ```bash ./run-online-punctuation.sh ``` -### Example 30: Keyword spotter +### Example 34: Keyword spotter ```bash ./run-keyword-spotter.sh ``` -### Example 31: Spoken language identification +### Example 35: Spoken language identification ```bash ./run-spoken-language-identification.sh ``` -### Example 32: Offline punctuation +### Example 36: Offline punctuation ```bash ./run-offline-punctuation.sh ``` -### Example 33: Audio tagging with a Zipformer model +### Example 37: Audio tagging with a Zipformer model ```bash ./run-audio-tagging-zipformer.sh ``` -### Example 34: Audio tagging with a CED model +### Example 38: Audio tagging with a CED model ```bash ./run-audio-tagging-ced.sh ``` -### Example 35: Speaker embedding extractor +### Example 39: Speaker embedding extractor ```bash ./run-speaker-embedding-extractor.sh ``` -### Example 36: Speaker embedding manager +### Example 40: Speaker embedding manager ```bash ./run-speaker-embedding-manager.sh ``` -### Example 37: Speaker embedding cosine similarity +### Example 41: Speaker embedding cosine similarity ```bash ./run-speaker-embedding-cosine-similarity.sh ``` -### Example 38: Offline speaker diarization +### Example 42: Offline speaker diarization ```bash ./run-offline-speaker-diarization.sh ``` -### Example 39: Simulated streaming ASR with SenseVoice and VAD from microphone +### Example 43: Simulated streaming ASR with SenseVoice and VAD from microphone This example uses Silero VAD to detect speech segments and runs the offline SenseVoice recognizer on each detected segment, providing an experience @@ -345,7 +378,7 @@ similar to streaming ASR. ./run-sense-voice-simulate-streaming-microphone.sh ``` -### Example 40: Simulated streaming ASR with FireRedASR CTC and VAD from microphone +### Example 44: Simulated streaming ASR with FireRedASR CTC and VAD from microphone This example uses Silero VAD to detect speech segments and runs the offline FireRedASR CTC recognizer on each detected segment. @@ -354,7 +387,7 @@ FireRedASR CTC recognizer on each detected segment. ./run-fire-red-asr-ctc-simulate-streaming-microphone.sh ``` -### Example 41: Simulated streaming ASR with Parakeet TDT CTC and VAD from microphone +### Example 45: Simulated streaming ASR with Parakeet TDT CTC and VAD from microphone This example uses Silero VAD to detect speech segments and runs the offline Parakeet TDT CTC recognizer on each detected segment (Japanese). @@ -363,7 +396,7 @@ Parakeet TDT CTC recognizer on each detected segment (Japanese). ./run-parakeet-tdt-ctc-simulate-streaming-microphone.sh ``` -### Example 42: Simulated streaming ASR with Parakeet TDT transducer and VAD from microphone +### Example 46: Simulated streaming ASR with Parakeet TDT transducer and VAD from microphone This example uses Silero VAD to detect speech segments and runs the offline Parakeet TDT transducer recognizer on each detected segment (English). @@ -372,7 +405,7 @@ Parakeet TDT transducer recognizer on each detected segment (English). ./run-parakeet-tdt-simulate-streaming-microphone.sh ``` -### Example 43: Simulated streaming ASR with WeNet CTC and VAD from microphone +### Example 47: Simulated streaming ASR with WeNet CTC and VAD from microphone This example uses Silero VAD to detect speech segments and runs the offline WeNet CTC recognizer on each detected segment (Cantonese). @@ -381,7 +414,7 @@ WeNet CTC recognizer on each detected segment (Cantonese). ./run-wenet-ctc-simulate-streaming-microphone.sh ``` -### Example 44: Simulated streaming ASR with Zipformer CTC and VAD from microphone +### Example 48: Simulated streaming ASR with Zipformer CTC and VAD from microphone This example uses Silero VAD to detect speech segments and runs the offline Zipformer CTC recognizer on each detected segment (Chinese). @@ -390,7 +423,7 @@ Zipformer CTC recognizer on each detected segment (Chinese). ./run-zipformer-ctc-simulate-streaming-microphone.sh ``` -### Example 45: Simulated streaming ASR with Zipformer transducer and VAD from microphone +### Example 49: Simulated streaming ASR with Zipformer transducer and VAD from microphone This example uses Silero VAD to detect speech segments and runs the offline Zipformer transducer recognizer on each detected segment (Chinese). @@ -399,7 +432,7 @@ Zipformer transducer recognizer on each detected segment (Chinese). ./run-zipformer-transducer-simulate-streaming-microphone.sh ``` -### Example 46: Simulated streaming ASR with Zipformer transducer (Japanese) and VAD from microphone +### Example 50: Simulated streaming ASR with Zipformer transducer (Japanese) and VAD from microphone This example uses Silero VAD to detect speech segments and runs the offline Zipformer transducer recognizer on each detected segment (Japanese, @@ -409,7 +442,7 @@ reazonspeech model). ./run-zipformer-ja-reazonspeech-simulate-streaming-microphone.sh ``` -### Example 47: Simulated streaming ASR with Qwen3 ASR and VAD from microphone +### Example 51: Simulated streaming ASR with Qwen3 ASR and VAD from microphone This example uses Silero VAD to detect speech segments and runs the offline Qwen3 ASR recognizer on each detected segment. @@ -418,25 +451,25 @@ Qwen3 ASR recognizer on each detected segment. ./run-qwen3-asr-simulate-streaming-microphone.sh ``` -### Example 48: ASR with non-streaming Whisper +### Example 52: ASR with non-streaming Whisper ```bash ./run-whisper.sh ``` -### Example 49: ASR with non-streaming FunASR Nano +### Example 53: ASR with non-streaming FunASR Nano ```bash ./run-funasr-nano.sh ``` -### Example 50: Remove silences from a file using ten-vad +### Example 54: Remove silences from a file using ten-vad ```bash ./run-ten-vad-remove-silence.sh ``` -### Example 51: ASR with non-streaming Paraformer +### Example 55: ASR with non-streaming Paraformer ```bash ./run-paraformer.sh \ No newline at end of file diff --git a/rust-api-examples/examples/kitten_tts_en.rs b/rust-api-examples/examples/kitten_tts_en_lexicon.rs similarity index 94% rename from rust-api-examples/examples/kitten_tts_en.rs rename to rust-api-examples/examples/kitten_tts_en_lexicon.rs index 0c7cd8445..c56023610 100644 --- a/rust-api-examples/examples/kitten_tts_en.rs +++ b/rust-api-examples/examples/kitten_tts_en_lexicon.rs @@ -15,8 +15,9 @@ fn main() { model: Some("./kitten-nano-en-v0_1-fp16/model.fp16.onnx".into()), voices: Some("./kitten-nano-en-v0_1-fp16/voices.bin".into()), tokens: Some("./kitten-nano-en-v0_1-fp16/tokens.txt".into()), - data_dir: Some("./kitten-nano-en-v0_1-fp16/espeak-ng-data".into()), + lexicon: Some("./lexicon-en-us.txt".into()), length_scale: 1.0, + ..Default::default() }, num_threads: 2, debug: false, @@ -67,7 +68,7 @@ fn main() { elapsed_seconds, duration, rtf ); - let filename = "./generated-kitten-en-rust.wav"; + let filename = "./generated-kitten-en-lexicon-rust.wav"; if audio.save(filename) { println!("Saved to: {}", filename); } else { diff --git a/rust-api-examples/examples/kitten_tts_en_phonemize.rs b/rust-api-examples/examples/kitten_tts_en_phonemize.rs new file mode 100644 index 000000000..f2284b357 --- /dev/null +++ b/rust-api-examples/examples/kitten_tts_en_phonemize.rs @@ -0,0 +1,116 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file demonstrates how to use Kitten TTS with sherpa-onnx's Rust API +// for offline English text-to-speech. +// +// The text is phonemized outside of sherpa-onnx with the piper-phonemize crate +// and the resulting phoneme codepoints are passed via +// GenerationConfig::phoneme_codepoints. No lexicon file is needed. +// +// The Rust counterpart of +// https://github.com/k2-fsa/sherpa-onnx/blob/master/python-api-examples/test-offline-tts-kitten-phonemize.py + +use sherpa_onnx::{ + GenerationConfig, OfflineTts, OfflineTtsConfig, OfflineTtsKittenModelConfig, +}; +use std::time::Instant; + +fn main() { + println!("piper-phonemize version: {}", piper_phonemize::get_version()); + + // Note: the espeak-ng-data used by piper-phonemize is embedded in the + // crate and initialized automatically on first use. It has nothing to do + // with sherpa-onnx; sherpa-onnx itself does not use espeak-ng anymore. + + let config = OfflineTtsConfig { + model: sherpa_onnx::OfflineTtsModelConfig { + kitten: OfflineTtsKittenModelConfig { + model: Some("./kitten-nano-en-v0_1-fp16/model.fp16.onnx".into()), + voices: Some("./kitten-nano-en-v0_1-fp16/voices.bin".into()), + tokens: Some("./kitten-nano-en-v0_1-fp16/tokens.txt".into()), + length_scale: 1.0, + ..Default::default() + }, + num_threads: 2, + debug: false, + ..Default::default() + }, + ..Default::default() + }; + + let tts = OfflineTts::create(&config).expect("Failed to create OfflineTts"); + + println!("Sample rate: {}", tts.sample_rate()); + println!("Num speakers: {}", tts.num_speakers()); + + let text = "Today as always, men fall into two groups: slaves and free men. Whoever \ + does not have two-thirds of his day for himself, is a slave, whatever \ + he may be: a statesman, a businessman, an official, or a scholar. \ + Friends fell out often because life was changing so fast. The easiest \ + thing in the world was to lose touch with someone."; + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + let lang = { + let l = tts.lang(); + if l.is_empty() { + eprintln!( + "Warning: the model has no language information. Falling back to en-us." + ); + "en-us".to_string() + } else { + l + } + }; + println!("Phonemizer language: {}", lang); + + let result = + piper_phonemize::phonemize(text, &lang).expect("Failed to phonemize the text"); + + let mut phoneme_codepoints = Vec::new(); + for i in 0..result.num_sentences() { + let phonemes = result.get_phonemes(i).unwrap_or_default(); + phoneme_codepoints.push(phonemes.into_iter().map(|c| c as i32).collect::>()); + } + println!("Number of phonemized sentences: {}", phoneme_codepoints.len()); + + let gen_config = GenerationConfig { + sid: 0, + speed: 1.0, + phoneme_codepoints: Some(phoneme_codepoints), + ..Default::default() + }; + + let start = Instant::now(); + + // The text has already been phonemized; pass an empty string here. + let audio = tts + .generate_with_config( + "", + &gen_config, + Some(|_samples: &[f32], progress: f32| -> bool { + println!("Progress: {:.1}%", progress * 100.0); + true + }), + ) + .expect("Generation failed"); + + let elapsed_seconds = start.elapsed().as_secs_f32(); + let duration = audio.samples().len() as f32 / audio.sample_rate() as f32; + let rtf = elapsed_seconds / duration; + + println!("Number of threads: {}", config.model.num_threads); + println!("Elapsed seconds: {:.3} s", elapsed_seconds); + println!("Audio duration: {:.3} s", duration); + println!( + "Real-time factor (RTF): {:.3}/{:.3} = {:.3}", + elapsed_seconds, duration, rtf + ); + + let filename = "./generated-kitten-en-phonemize-rust.wav"; + if audio.save(filename) { + println!("Saved to: {}", filename); + } else { + eprintln!("Failed to save {}", filename); + } +} diff --git a/rust-api-examples/examples/kokoro_tts_en.rs b/rust-api-examples/examples/kokoro_tts_en_lexicon.rs similarity index 95% rename from rust-api-examples/examples/kokoro_tts_en.rs rename to rust-api-examples/examples/kokoro_tts_en_lexicon.rs index 734f39410..1d01fe1cc 100644 --- a/rust-api-examples/examples/kokoro_tts_en.rs +++ b/rust-api-examples/examples/kokoro_tts_en_lexicon.rs @@ -15,7 +15,7 @@ fn main() { model: Some("./kokoro-en-v0_19/model.onnx".into()), voices: Some("./kokoro-en-v0_19/voices.bin".into()), tokens: Some("./kokoro-en-v0_19/tokens.txt".into()), - data_dir: Some("./kokoro-en-v0_19/espeak-ng-data".into()), + lexicon: Some("./lexicon-en-us.txt".into()), length_scale: 1.0, ..Default::default() }, @@ -68,7 +68,7 @@ fn main() { elapsed_seconds, duration, rtf ); - let filename = "./generated-kokoro-en-rust.wav"; + let filename = "./generated-kokoro-en-lexicon-rust.wav"; if audio.save(filename) { println!("Saved to: {}", filename); } else { diff --git a/rust-api-examples/examples/kokoro_tts_en_phonemize.rs b/rust-api-examples/examples/kokoro_tts_en_phonemize.rs new file mode 100644 index 000000000..22e52ae91 --- /dev/null +++ b/rust-api-examples/examples/kokoro_tts_en_phonemize.rs @@ -0,0 +1,116 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file demonstrates how to use Kokoro TTS with sherpa-onnx's Rust API +// for offline English text-to-speech. +// +// The text is phonemized outside of sherpa-onnx with the piper-phonemize crate +// and the resulting phoneme codepoints are passed via +// GenerationConfig::phoneme_codepoints. No lexicon file is needed. +// +// The Rust counterpart of +// https://github.com/k2-fsa/sherpa-onnx/blob/master/python-api-examples/test-offline-tts-kokoro-en-phonemize.py + +use sherpa_onnx::{ + GenerationConfig, OfflineTts, OfflineTtsConfig, OfflineTtsKokoroModelConfig, +}; +use std::time::Instant; + +fn main() { + println!("piper-phonemize version: {}", piper_phonemize::get_version()); + + // Note: the espeak-ng-data used by piper-phonemize is embedded in the + // crate and initialized automatically on first use. It has nothing to do + // with sherpa-onnx; sherpa-onnx itself does not use espeak-ng anymore. + + let config = OfflineTtsConfig { + model: sherpa_onnx::OfflineTtsModelConfig { + kokoro: OfflineTtsKokoroModelConfig { + model: Some("./kokoro-en-v0_19/model.onnx".into()), + voices: Some("./kokoro-en-v0_19/voices.bin".into()), + tokens: Some("./kokoro-en-v0_19/tokens.txt".into()), + length_scale: 1.0, + ..Default::default() + }, + num_threads: 2, + debug: false, + ..Default::default() + }, + ..Default::default() + }; + + let tts = OfflineTts::create(&config).expect("Failed to create OfflineTts"); + + println!("Sample rate: {}", tts.sample_rate()); + println!("Num speakers: {}", tts.num_speakers()); + + let text = "Today as always, men fall into two groups: slaves and free men. Whoever \ + does not have two-thirds of his day for himself, is a slave, whatever \ + he may be: a statesman, a businessman, an official, or a scholar. \ + Friends fell out often because life was changing so fast. The easiest \ + thing in the world was to lose touch with someone."; + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + let lang = { + let l = tts.lang(); + if l.is_empty() { + eprintln!( + "Warning: the model has no language information. Falling back to en-us." + ); + "en-us".to_string() + } else { + l + } + }; + println!("Phonemizer language: {}", lang); + + let result = + piper_phonemize::phonemize(text, &lang).expect("Failed to phonemize the text"); + + let mut phoneme_codepoints = Vec::new(); + for i in 0..result.num_sentences() { + let phonemes = result.get_phonemes(i).unwrap_or_default(); + phoneme_codepoints.push(phonemes.into_iter().map(|c| c as i32).collect::>()); + } + println!("Number of phonemized sentences: {}", phoneme_codepoints.len()); + + let gen_config = GenerationConfig { + sid: 0, + speed: 1.0, + phoneme_codepoints: Some(phoneme_codepoints), + ..Default::default() + }; + + let start = Instant::now(); + + // The text has already been phonemized; pass an empty string here. + let audio = tts + .generate_with_config( + "", + &gen_config, + Some(|_samples: &[f32], progress: f32| -> bool { + println!("Progress: {:.1}%", progress * 100.0); + true + }), + ) + .expect("Generation failed"); + + let elapsed_seconds = start.elapsed().as_secs_f32(); + let duration = audio.samples().len() as f32 / audio.sample_rate() as f32; + let rtf = elapsed_seconds / duration; + + println!("Number of threads: {}", config.model.num_threads); + println!("Elapsed seconds: {:.3} s", elapsed_seconds); + println!("Audio duration: {:.3} s", duration); + println!( + "Real-time factor (RTF): {:.3}/{:.3} = {:.3}", + elapsed_seconds, duration, rtf + ); + + let filename = "./generated-kokoro-en-phonemize-rust.wav"; + if audio.save(filename) { + println!("Saved to: {}", filename); + } else { + eprintln!("Failed to save {}", filename); + } +} diff --git a/rust-api-examples/examples/kokoro_tts_zh_en.rs b/rust-api-examples/examples/kokoro_tts_zh_en_lexicon.rs similarity index 92% rename from rust-api-examples/examples/kokoro_tts_zh_en.rs rename to rust-api-examples/examples/kokoro_tts_zh_en_lexicon.rs index bc1d1cdec..dba10a199 100644 --- a/rust-api-examples/examples/kokoro_tts_zh_en.rs +++ b/rust-api-examples/examples/kokoro_tts_zh_en_lexicon.rs @@ -15,8 +15,6 @@ fn main() { model: Some("./kokoro-multi-lang-v1_0/model.onnx".into()), voices: Some("./kokoro-multi-lang-v1_0/voices.bin".into()), tokens: Some("./kokoro-multi-lang-v1_0/tokens.txt".into()), - data_dir: Some("./kokoro-multi-lang-v1_0/espeak-ng-data".into()), - dict_dir: Some("./kokoro-multi-lang-v1_0/dict".into()), lexicon: Some( "./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt".into(), ), @@ -69,7 +67,7 @@ fn main() { elapsed_seconds, duration, rtf ); - let filename = "./generated-kokoro-zh-en-rust.wav"; + let filename = "./generated-kokoro-zh-en-lexicon-rust.wav"; if audio.save(filename) { println!("Saved to: {}", filename); } else { diff --git a/rust-api-examples/examples/matcha_tts_en.rs b/rust-api-examples/examples/matcha_tts_en_lexicon.rs similarity index 94% rename from rust-api-examples/examples/matcha_tts_en.rs rename to rust-api-examples/examples/matcha_tts_en_lexicon.rs index 0d659ddce..9cea5b974 100644 --- a/rust-api-examples/examples/matcha_tts_en.rs +++ b/rust-api-examples/examples/matcha_tts_en_lexicon.rs @@ -15,7 +15,7 @@ fn main() { acoustic_model: Some("./matcha-icefall-en_US-ljspeech/model-steps-3.onnx".into()), vocoder: Some("./vocos-22khz-univ.onnx".into()), tokens: Some("./matcha-icefall-en_US-ljspeech/tokens.txt".into()), - data_dir: Some("./matcha-icefall-en_US-ljspeech/espeak-ng-data".into()), + lexicon: Some("./lexicon-en-us.txt".into()), noise_scale: 0.667, length_scale: 1.0, ..Default::default() @@ -69,7 +69,7 @@ fn main() { elapsed_seconds, duration, rtf ); - let filename = "./generated-matcha-en-rust.wav"; + let filename = "./generated-matcha-en-lexicon-rust.wav"; if audio.save(filename) { println!("Saved to: {}", filename); } else { diff --git a/rust-api-examples/examples/matcha_tts_en_phonemize.rs b/rust-api-examples/examples/matcha_tts_en_phonemize.rs new file mode 100644 index 000000000..254b98342 --- /dev/null +++ b/rust-api-examples/examples/matcha_tts_en_phonemize.rs @@ -0,0 +1,117 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file demonstrates how to use Matcha TTS with sherpa-onnx's Rust API +// for offline English text-to-speech. +// +// The text is phonemized outside of sherpa-onnx with the piper-phonemize crate +// and the resulting phoneme codepoints are passed via +// GenerationConfig::phoneme_codepoints. No lexicon file is needed. +// +// The Rust counterpart of +// https://github.com/k2-fsa/sherpa-onnx/blob/master/python-api-examples/test-offline-tts-matcha-en-phonemize.py + +use sherpa_onnx::{ + GenerationConfig, OfflineTts, OfflineTtsConfig, OfflineTtsMatchaModelConfig, +}; +use std::time::Instant; + +fn main() { + println!("piper-phonemize version: {}", piper_phonemize::get_version()); + + // Note: the espeak-ng-data used by piper-phonemize is embedded in the + // crate and initialized automatically on first use. It has nothing to do + // with sherpa-onnx; sherpa-onnx itself does not use espeak-ng anymore. + + let config = OfflineTtsConfig { + model: sherpa_onnx::OfflineTtsModelConfig { + matcha: OfflineTtsMatchaModelConfig { + acoustic_model: Some("./matcha-icefall-en_US-ljspeech/model-steps-3.onnx".into()), + vocoder: Some("./vocos-22khz-univ.onnx".into()), + tokens: Some("./matcha-icefall-en_US-ljspeech/tokens.txt".into()), + noise_scale: 0.667, + length_scale: 1.0, + ..Default::default() + }, + num_threads: 2, + debug: false, + ..Default::default() + }, + ..Default::default() + }; + + let tts = OfflineTts::create(&config).expect("Failed to create OfflineTts"); + + println!("Sample rate: {}", tts.sample_rate()); + println!("Num speakers: {}", tts.num_speakers()); + + let text = "Today as always, men fall into two groups: slaves and free men. Whoever \ + does not have two-thirds of his day for himself, is a slave, whatever \ + he may be: a statesman, a businessman, an official, or a scholar. \ + Friends fell out often because life was changing so fast. The easiest \ + thing in the world was to lose touch with someone."; + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + let lang = { + let l = tts.lang(); + if l.is_empty() { + eprintln!( + "Warning: the model has no language information. Falling back to en-us." + ); + "en-us".to_string() + } else { + l + } + }; + println!("Phonemizer language: {}", lang); + + let result = + piper_phonemize::phonemize(text, &lang).expect("Failed to phonemize the text"); + + let mut phoneme_codepoints = Vec::new(); + for i in 0..result.num_sentences() { + let phonemes = result.get_phonemes(i).unwrap_or_default(); + phoneme_codepoints.push(phonemes.into_iter().map(|c| c as i32).collect::>()); + } + println!("Number of phonemized sentences: {}", phoneme_codepoints.len()); + + let gen_config = GenerationConfig { + sid: 0, + speed: 1.0, + phoneme_codepoints: Some(phoneme_codepoints), + ..Default::default() + }; + + let start = Instant::now(); + + // The text has already been phonemized; pass an empty string here. + let audio = tts + .generate_with_config( + "", + &gen_config, + Some(|_samples: &[f32], progress: f32| -> bool { + println!("Progress: {:.1}%", progress * 100.0); + true + }), + ) + .expect("Generation failed"); + + let elapsed_seconds = start.elapsed().as_secs_f32(); + let duration = audio.samples().len() as f32 / audio.sample_rate() as f32; + let rtf = elapsed_seconds / duration; + + println!("Number of threads: {}", config.model.num_threads); + println!("Elapsed seconds: {:.3} s", elapsed_seconds); + println!("Audio duration: {:.3} s", duration); + println!( + "Real-time factor (RTF): {:.3}/{:.3} = {:.3}", + elapsed_seconds, duration, rtf + ); + + let filename = "./generated-matcha-en-phonemize-rust.wav"; + if audio.save(filename) { + println!("Saved to: {}", filename); + } else { + eprintln!("Failed to save {}", filename); + } +} diff --git a/rust-api-examples/examples/matcha_tts_zh.rs b/rust-api-examples/examples/matcha_tts_zh_lexicon.rs similarity index 97% rename from rust-api-examples/examples/matcha_tts_zh.rs rename to rust-api-examples/examples/matcha_tts_zh_lexicon.rs index 7e3dce107..50c19fc52 100644 --- a/rust-api-examples/examples/matcha_tts_zh.rs +++ b/rust-api-examples/examples/matcha_tts_zh_lexicon.rs @@ -73,7 +73,7 @@ fn main() { elapsed_seconds, duration, rtf ); - let filename = "./generated-matcha-zh-rust.wav"; + let filename = "./generated-matcha-zh-lexicon-rust.wav"; if audio.save(filename) { println!("Saved to: {}", filename); } else { diff --git a/rust-api-examples/examples/streaming_zipformer_ctc_hlg.rs b/rust-api-examples/examples/streaming_zipformer_ctc_hlg.rs new file mode 100644 index 000000000..b462a04e4 --- /dev/null +++ b/rust-api-examples/examples/streaming_zipformer_ctc_hlg.rs @@ -0,0 +1,131 @@ +// rust-api-examples/examples/streaming_zipformer_ctc_hlg.rs +// +// Copyright (c) 2026 Xiaomi Corporation +// +// This file demonstrates how to decode a streaming CTC model with an HLG +// decoding graph using sherpa-onnx's Rust API. +// +// The HLG graph is passed to the C API through ctc_fst_decoder_config. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 +// tar xvf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 +// rm sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 +// +// cargo run --release --example streaming_zipformer_ctc_hlg -- \ +// --model ./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/ctc-epoch-30-avg-3-chunk-16-left-128.int8.onnx \ +// --tokens ./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/tokens.txt \ +// --graph ./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/HLG.fst \ +// --wav ./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/test_wavs/8k.wav +// +// clang-format on + +use clap::Parser; +use sherpa_onnx::{OnlineCtcFstDecoderConfig, OnlineRecognizer, OnlineRecognizerConfig, Wave}; + +/// Streaming CTC decoding with an HLG graph via ctc_fst_decoder_config. +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Path to the streaming zipformer CTC model + #[arg(long)] + model: String, + + /// Path to tokens.txt + #[arg(long)] + tokens: String, + + /// Path to the decoding graph (H.fst, HL.fst or HLG.fst) + #[arg(long)] + graph: String, + + /// Path to a wav file + #[arg(long)] + wav: String, + + /// Number of threads + #[arg(long, default_value_t = 1)] + num_threads: i32, + + /// Enable debug logs + #[arg(long, default_value_t = false)] + debug: bool, +} + +fn main() { + let args = Args::parse(); + + let wave = Wave::read(&args.wav).expect("Failed to read WAV file"); + + let mut config = OnlineRecognizerConfig::default(); + config.model_config.zipformer2_ctc.model = Some(args.model); + config.model_config.tokens = Some(args.tokens); + config.model_config.num_threads = args.num_threads; + config.model_config.debug = args.debug; + + // This is what the example exercises: the graph and max_active reach the C + // API through OnlineCtcFstDecoderConfig. + config.ctc_fst_decoder_config = OnlineCtcFstDecoderConfig { + graph: Some(args.graph), + max_active: 3000, + }; + + let recognizer = + OnlineRecognizer::create(&config).expect("Failed to create OnlineRecognizer"); + + let stream = recognizer.create_stream(); + let mut segment_id = 0; + + println!( + "Sample rate: {}, num samples: {}, duration: {:.2}s", + wave.sample_rate(), + wave.num_samples(), + wave.num_samples() as f32 / wave.sample_rate() as f32 + ); + + // simulate streaming. You can choose an arbitrary N + const CHUNK_SIZE: usize = 3200; + + for chunk in wave.samples().chunks(CHUNK_SIZE) { + stream.accept_waveform(wave.sample_rate(), chunk); + + while recognizer.is_ready(&stream) { + recognizer.decode(&stream); + } + + if let Some(result) = recognizer.get_result(&stream) { + if !result.text.is_empty() { + println!("Segment {}: {}", segment_id, result.text); + } + } + + if recognizer.is_endpoint(&stream) { + if let Some(result) = recognizer.get_result(&stream) { + if !result.text.is_empty() { + segment_id += 1; + } + } + recognizer.reset(&stream); + } + } + + // Tail padding (~0.3s) + let tail_padding_len = (wave.sample_rate() as f32 * 0.3).round() as usize; + let tail_padding = vec![0.0f32; tail_padding_len]; + stream.accept_waveform(wave.sample_rate(), &tail_padding); + + stream.input_finished(); + + while recognizer.is_ready(&stream) { + recognizer.decode(&stream); + } + + if let Some(result) = recognizer.get_result(&stream) { + if !result.text.is_empty() { + println!("Segment {}: {}", segment_id, result.text); + } + } + + println!("Transcription finished."); +} diff --git a/rust-api-examples/examples/vits_tts_de_phonemize.rs b/rust-api-examples/examples/vits_tts_de_phonemize.rs new file mode 100644 index 000000000..89824da49 --- /dev/null +++ b/rust-api-examples/examples/vits_tts_de_phonemize.rs @@ -0,0 +1,110 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file demonstrates how to use a German Piper VITS TTS model with +// sherpa-onnx's Rust API for offline text-to-speech. +// +// There is no published German lexicon, so the text is phonemized outside of +// sherpa-onnx with the piper-phonemize crate and the resulting phoneme +// codepoints are passed via GenerationConfig::phoneme_codepoints. + +use sherpa_onnx::{ + GenerationConfig, OfflineTts, OfflineTtsConfig, OfflineTtsVitsModelConfig, +}; +use std::time::Instant; + +fn main() { + println!("piper-phonemize version: {}", piper_phonemize::get_version()); + + // Note: the espeak-ng-data used by piper-phonemize is embedded in the + // crate and initialized automatically on first use. It has nothing to do + // with sherpa-onnx; sherpa-onnx itself does not use espeak-ng anymore. + + let config = OfflineTtsConfig { + model: sherpa_onnx::OfflineTtsModelConfig { + vits: OfflineTtsVitsModelConfig { + model: Some("./vits-piper-de_DE-glados-high/de_DE-glados-high.onnx".into()), + tokens: Some("./vits-piper-de_DE-glados-high/tokens.txt".into()), + noise_scale: 0.667, + noise_scale_w: 0.8, + length_scale: 1.0, + ..Default::default() + }, + num_threads: 2, + debug: false, + ..Default::default() + }, + ..Default::default() + }; + + let tts = OfflineTts::create(&config).expect("Failed to create OfflineTts"); + + println!("Sample rate: {}", tts.sample_rate()); + println!("Num speakers: {}", tts.num_speakers()); + + let text = "Alles hat ein Ende, nur die Wurst hat zwei."; + + // The espeak-ng voice for the phonemizer is taken from the model metadata + // (for this model, "de"). Fall back to en-us if unavailable. + let lang = { + let l = tts.lang(); + if l.is_empty() { + eprintln!( + "Warning: the model has no language information. Falling back to en-us." + ); + "en-us".to_string() + } else { + l + } + }; + println!("Phonemizer language: {}", lang); + + let result = + piper_phonemize::phonemize(text, &lang).expect("Failed to phonemize the text"); + + let mut phoneme_codepoints = Vec::new(); + for i in 0..result.num_sentences() { + let phonemes = result.get_phonemes(i).unwrap_or_default(); + phoneme_codepoints.push(phonemes.into_iter().map(|c| c as i32).collect::>()); + } + println!("Number of phonemized sentences: {}", phoneme_codepoints.len()); + + let gen_config = GenerationConfig { + sid: 0, + speed: 1.0, + phoneme_codepoints: Some(phoneme_codepoints), + ..Default::default() + }; + + let start = Instant::now(); + + // The text has already been phonemized; pass an empty string here. + let audio = tts + .generate_with_config( + "", + &gen_config, + Some(|_samples: &[f32], progress: f32| -> bool { + println!("Progress: {:.1}%", progress * 100.0); + true + }), + ) + .expect("Generation failed"); + + let elapsed_seconds = start.elapsed().as_secs_f32(); + let duration = audio.samples().len() as f32 / audio.sample_rate() as f32; + let rtf = elapsed_seconds / duration; + + println!("Number of threads: {}", config.model.num_threads); + println!("Elapsed seconds: {:.3} s", elapsed_seconds); + println!("Audio duration: {:.3} s", duration); + println!( + "Real-time factor (RTF): {:.3}/{:.3} = {:.3}", + elapsed_seconds, duration, rtf + ); + + let filename = "./generated-vits-de-phonemize-rust.wav"; + if audio.save(filename) { + println!("Saved to: {}", filename); + } else { + eprintln!("Failed to save {}", filename); + } +} diff --git a/rust-api-examples/examples/vits_tts.rs b/rust-api-examples/examples/vits_tts_lexicon.rs similarity index 94% rename from rust-api-examples/examples/vits_tts.rs rename to rust-api-examples/examples/vits_tts_lexicon.rs index ecb8588b5..3cc47b95e 100644 --- a/rust-api-examples/examples/vits_tts.rs +++ b/rust-api-examples/examples/vits_tts_lexicon.rs @@ -20,16 +20,16 @@ struct Args { #[arg(long)] tokens: String, - /// Path to espeak-ng-data + /// Path to lexicon.txt #[arg(long)] - data_dir: String, + lexicon: String, /// Input text to synthesize #[arg(long)] text: String, /// Output wave filename - #[arg(long, default_value = "./generated-vits-rust.wav")] + #[arg(long, default_value = "./generated-vits-lexicon-rust.wav")] output: String, /// Speaker ID for multi-speaker models @@ -60,7 +60,7 @@ fn main() { noise_scale: 0.667, noise_scale_w: 0.8, length_scale: 1.0, - data_dir: Some(args.data_dir.clone()), + lexicon: Some(args.lexicon.clone()), ..Default::default() }, num_threads: args.num_threads, @@ -69,7 +69,6 @@ fn main() { }, ..Default::default() }; - let tts = OfflineTts::create(&config).expect("Failed to create OfflineTts"); println!("Sample rate: {}", tts.sample_rate()); @@ -82,7 +81,6 @@ fn main() { }; let start = Instant::now(); - let audio = tts .generate_with_config( &args.text, @@ -105,7 +103,6 @@ fn main() { "Real-time factor (RTF): {:.3}/{:.3} = {:.3}", elapsed_seconds, duration, rtf ); - if audio.save(&args.output) { println!("Saved to: {}", args.output); } else { diff --git a/rust-api-examples/examples/vits_tts_phonemize.rs b/rust-api-examples/examples/vits_tts_phonemize.rs new file mode 100644 index 000000000..d82993776 --- /dev/null +++ b/rust-api-examples/examples/vits_tts_phonemize.rs @@ -0,0 +1,113 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file demonstrates how to use a Piper VITS TTS model with sherpa-onnx's +// Rust API for offline text-to-speech. +// +// The text is phonemized outside of sherpa-onnx with the piper-phonemize crate +// and the resulting phoneme codepoints are passed via +// GenerationConfig::phoneme_codepoints. No lexicon file is needed. +// +// The Rust counterpart of +// https://github.com/k2-fsa/sherpa-onnx/blob/master/python-api-examples/test-offline-tts-piper-phonemize.py + +use sherpa_onnx::{ + GenerationConfig, OfflineTts, OfflineTtsConfig, OfflineTtsVitsModelConfig, +}; +use std::time::Instant; + +fn main() { + println!("piper-phonemize version: {}", piper_phonemize::get_version()); + + // Note: the espeak-ng-data used by piper-phonemize is embedded in the + // crate and initialized automatically on first use. It has nothing to do + // with sherpa-onnx; sherpa-onnx itself does not use espeak-ng anymore. + + let config = OfflineTtsConfig { + model: sherpa_onnx::OfflineTtsModelConfig { + vits: OfflineTtsVitsModelConfig { + model: Some("./vits-piper-en_US-amy-low/en_US-amy-low.onnx".into()), + tokens: Some("./vits-piper-en_US-amy-low/tokens.txt".into()), + noise_scale: 0.667, + noise_scale_w: 0.8, + length_scale: 1.0, + ..Default::default() + }, + num_threads: 2, + debug: false, + ..Default::default() + }, + ..Default::default() + }; + + let tts = OfflineTts::create(&config).expect("Failed to create OfflineTts"); + + println!("Sample rate: {}", tts.sample_rate()); + println!("Num speakers: {}", tts.num_speakers()); + + let text = "Liliana, the most beautiful and lovely assistant of our team!"; + + // The espeak-ng voice for the phonemizer is taken from the model metadata. + // Fall back to en-us if the model has no language information. + let lang = { + let l = tts.lang(); + if l.is_empty() { + eprintln!( + "Warning: the model has no language information. Falling back to en-us." + ); + "en-us".to_string() + } else { + l + } + }; + println!("Phonemizer language: {}", lang); + + let result = + piper_phonemize::phonemize(text, &lang).expect("Failed to phonemize the text"); + + let mut phoneme_codepoints = Vec::new(); + for i in 0..result.num_sentences() { + let phonemes = result.get_phonemes(i).unwrap_or_default(); + phoneme_codepoints.push(phonemes.into_iter().map(|c| c as i32).collect::>()); + } + println!("Number of phonemized sentences: {}", phoneme_codepoints.len()); + + let gen_config = GenerationConfig { + sid: 0, + speed: 1.0, + phoneme_codepoints: Some(phoneme_codepoints), + ..Default::default() + }; + + let start = Instant::now(); + + // The text has already been phonemized; pass an empty string here. + let audio = tts + .generate_with_config( + "", + &gen_config, + Some(|_samples: &[f32], progress: f32| -> bool { + println!("Progress: {:.1}%", progress * 100.0); + true + }), + ) + .expect("Generation failed"); + + let elapsed_seconds = start.elapsed().as_secs_f32(); + let duration = audio.samples().len() as f32 / audio.sample_rate() as f32; + let rtf = elapsed_seconds / duration; + + println!("Number of threads: {}", config.model.num_threads); + println!("Elapsed seconds: {:.3} s", elapsed_seconds); + println!("Audio duration: {:.3} s", duration); + println!( + "Real-time factor (RTF): {:.3}/{:.3} = {:.3}", + elapsed_seconds, duration, rtf + ); + + let filename = "./generated-vits-en-phonemize-rust.wav"; + if audio.save(filename) { + println!("Saved to: {}", filename); + } else { + eprintln!("Failed to save {}", filename); + } +} diff --git a/rust-api-examples/examples/zipformer_ctc_hlg.rs b/rust-api-examples/examples/zipformer_ctc_hlg.rs new file mode 100644 index 000000000..fe736c97e --- /dev/null +++ b/rust-api-examples/examples/zipformer_ctc_hlg.rs @@ -0,0 +1,85 @@ +// rust-api-examples/examples/zipformer_ctc_hlg.rs +// +// Copyright (c) 2026 Xiaomi Corporation +// +// Offline CTC decoding with an HLG graph via ctc_fst_decoder_config. +// +// clang-format off +// +// wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +// +// cargo run --release --example zipformer_ctc_hlg -- \ +// --model ./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.onnx \ +// --tokens ./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt \ +// --graph ./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst \ +// --wav ./sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav +// +// clang-format on + +use clap::Parser; +use sherpa_onnx::{ + OfflineCtcFstDecoderConfig, OfflineRecognizer, OfflineRecognizerConfig, OfflineZipformerCtcModelConfig, + Wave, +}; + +#[derive(Parser, Debug)] +struct Args { + /// Path to the zipformer CTC model + #[arg(long)] + model: String, + + /// Path to tokens.txt + #[arg(long)] + tokens: String, + + /// Path to the decoding graph (H.fst, HL.fst or HLG.fst) + #[arg(long)] + graph: String, + + /// Path to a wav file + #[arg(long)] + wav: String, + + /// Number of threads + #[arg(long, default_value_t = 1)] + num_threads: i32, + + /// Enable debug logs + #[arg(long, default_value_t = false)] + debug: bool, +} + +fn main() { + let args = Args::parse(); + + let mut config = OfflineRecognizerConfig::default(); + config.model_config.zipformer_ctc = OfflineZipformerCtcModelConfig { + model: Some(args.model), + }; + config.model_config.tokens = Some(args.tokens); + config.model_config.num_threads = args.num_threads; + config.model_config.debug = args.debug; + + // This is what the example exercises: the graph and max_active reach the C + // API through OfflineCtcFstDecoderConfig. + config.ctc_fst_decoder_config = OfflineCtcFstDecoderConfig { + graph: Some(args.graph), + max_active: 3000, + }; + + let recognizer = + OfflineRecognizer::create(&config).expect("Failed to create OfflineRecognizer"); + + let wave = Wave::read(&args.wav).expect("Failed to read WAV file"); + let stream = recognizer.create_stream(); + stream.accept_waveform(wave.sample_rate(), wave.samples()); + recognizer.decode(&stream); + + if let Some(result) = stream.get_result() { + println!("Decoded text: {}", result.text); + } else { + eprintln!("No result produced"); + } +} diff --git a/rust-api-examples/examples/zipvoice_tts.rs b/rust-api-examples/examples/zipvoice_tts_lexicon.rs similarity index 93% rename from rust-api-examples/examples/zipvoice_tts.rs rename to rust-api-examples/examples/zipvoice_tts_lexicon.rs index e3e6b6463..5ff39c3a3 100644 --- a/rust-api-examples/examples/zipvoice_tts.rs +++ b/rust-api-examples/examples/zipvoice_tts_lexicon.rs @@ -21,16 +21,15 @@ fn main() { "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx".into(), ), vocoder: Some("./vocos_24khz.onnx".into()), - data_dir: Some( - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data".into(), - ), lexicon: Some( - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt".into(), + "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt" + .into(), ), feat_scale: 0.1, t_shift: 0.5, target_rms: 0.1, guidance_scale: 1.0, + ..Default::default() }, num_threads: 2, debug: false, @@ -89,7 +88,7 @@ fn main() { elapsed_seconds, duration, rtf ); - let filename = "./generated-zipvoice-zh-en-rust.wav"; + let filename = "./generated-zipvoice-zh-en-lexicon-rust.wav"; if audio.save(filename) { println!("Saved to: {}", filename); } else { diff --git a/rust-api-examples/for-advanced-users.md b/rust-api-examples/for-advanced-users.md index 0818a862d..8d25f8060 100644 --- a/rust-api-examples/for-advanced-users.md +++ b/rust-api-examples/for-advanced-users.md @@ -384,7 +384,6 @@ cmake \ -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. @@ -624,7 +623,6 @@ cmake \ -DBUILD_SHARED_LIBS=ON \ -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \ - -DBUILD_ESPEAK_NG_EXE=OFF \ -DSHERPA_ONNX_ENABLE_BINARY=OFF \ -DCMAKE_INSTALL_PREFIX=./install \ .. diff --git a/rust-api-examples/run-kitten-tts-en-lexicon.sh b/rust-api-examples/run-kitten-tts-en-lexicon.sh new file mode 100755 index 000000000..56f432abe --- /dev/null +++ b/rust-api-examples/run-kitten-tts-en-lexicon.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kitten-nano-en-v0_1-fp16.tar.bz2 + tar xf kitten-nano-en-v0_1-fp16.tar.bz2 + rm kitten-nano-en-v0_1-fp16.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +cargo run --example kitten_tts_en_lexicon diff --git a/dotnet-examples/kitten-tts/run-kitten.sh b/rust-api-examples/run-kitten-tts-en-phonemize.sh old mode 100755 new mode 100644 similarity index 81% rename from dotnet-examples/kitten-tts/run-kitten.sh rename to rust-api-examples/run-kitten-tts-en-phonemize.sh index 72a912fff..e910df90e --- a/dotnet-examples/kitten-tts/run-kitten.sh +++ b/rust-api-examples/run-kitten-tts-en-phonemize.sh @@ -7,5 +7,4 @@ if [ ! -f ./kitten-nano-en-v0_1-fp16/model.fp16.onnx ]; then rm kitten-nano-en-v0_1-fp16.tar.bz2 fi - -dotnet run +cargo run --features phonemize --example kitten_tts_en_phonemize diff --git a/rust-api-examples/run-kokoro-tts-en-lexicon.sh b/rust-api-examples/run-kokoro-tts-en-lexicon.sh new file mode 100755 index 000000000..13117d46f --- /dev/null +++ b/rust-api-examples/run-kokoro-tts-en-lexicon.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2 + tar xf kokoro-en-v0_19.tar.bz2 + rm kokoro-en-v0_19.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +cargo run --example kokoro_tts_en_lexicon diff --git a/rust-api-examples/run-kokoro-tts-en.sh b/rust-api-examples/run-kokoro-tts-en-phonemize.sh old mode 100755 new mode 100644 similarity index 79% rename from rust-api-examples/run-kokoro-tts-en.sh rename to rust-api-examples/run-kokoro-tts-en-phonemize.sh index 5e51aecb5..73ab98f20 --- a/rust-api-examples/run-kokoro-tts-en.sh +++ b/rust-api-examples/run-kokoro-tts-en-phonemize.sh @@ -7,4 +7,4 @@ if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then rm kokoro-en-v0_19.tar.bz2 fi -cargo run --example kokoro_tts_en +cargo run --features phonemize --example kokoro_tts_en_phonemize diff --git a/rust-api-examples/run-kokoro-tts-zh-en.sh b/rust-api-examples/run-kokoro-tts-zh-en-lexicon.sh similarity index 85% rename from rust-api-examples/run-kokoro-tts-zh-en.sh rename to rust-api-examples/run-kokoro-tts-zh-en-lexicon.sh index c6f58ba7a..18c3b1095 100755 --- a/rust-api-examples/run-kokoro-tts-zh-en.sh +++ b/rust-api-examples/run-kokoro-tts-zh-en-lexicon.sh @@ -7,4 +7,4 @@ if [ ! -f ./kokoro-multi-lang-v1_0/model.onnx ]; then rm kokoro-multi-lang-v1_0.tar.bz2 fi -cargo run --example kokoro_tts_zh_en +cargo run --example kokoro_tts_zh_en_lexicon diff --git a/rust-api-examples/run-matcha-tts-en-lexicon.sh b/rust-api-examples/run-matcha-tts-en-lexicon.sh new file mode 100755 index 000000000..bc7a97c46 --- /dev/null +++ b/rust-api-examples/run-matcha-tts-en-lexicon.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -f ./matcha-icefall-en_US-ljspeech/model-steps-3.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2 + tar xf matcha-icefall-en_US-ljspeech.tar.bz2 + rm matcha-icefall-en_US-ljspeech.tar.bz2 +fi + +if [ ! -f ./vocos-22khz-univ.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +cargo run --example matcha_tts_en_lexicon diff --git a/rust-api-examples/run-matcha-tts-en.sh b/rust-api-examples/run-matcha-tts-en-phonemize.sh old mode 100755 new mode 100644 similarity index 87% rename from rust-api-examples/run-matcha-tts-en.sh rename to rust-api-examples/run-matcha-tts-en-phonemize.sh index 817931989..3508b878c --- a/rust-api-examples/run-matcha-tts-en.sh +++ b/rust-api-examples/run-matcha-tts-en-phonemize.sh @@ -11,4 +11,4 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi -cargo run --example matcha_tts_en +cargo run --features phonemize --example matcha_tts_en_phonemize diff --git a/rust-api-examples/run-matcha-tts-zh.sh b/rust-api-examples/run-matcha-tts-zh-lexicon.sh similarity index 91% rename from rust-api-examples/run-matcha-tts-zh.sh rename to rust-api-examples/run-matcha-tts-zh-lexicon.sh index 2bbf77a29..90b0fd93c 100755 --- a/rust-api-examples/run-matcha-tts-zh.sh +++ b/rust-api-examples/run-matcha-tts-zh-lexicon.sh @@ -11,4 +11,4 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi -cargo run --example matcha_tts_zh +cargo run --example matcha_tts_zh_lexicon diff --git a/rust-api-examples/run-paraformer.sh b/rust-api-examples/run-paraformer.sh index 40155c1a5..f37a3e0aa 100755 --- a/rust-api-examples/run-paraformer.sh +++ b/rust-api-examples/run-paraformer.sh @@ -13,12 +13,12 @@ model_dir="$script_dir/$model" archive_path="$script_dir/$archive" if [ ! -d "$model_dir" ]; then - if command -v wget >/dev/null 2>&1; then - wget -O "$archive_path" "$repo_url/$archive" - elif command -v curl >/dev/null 2>&1; then + if command -v curl >/dev/null 2>&1; then curl -L -o "$archive_path" "$repo_url/$archive" + elif command -v wget >/dev/null 2>&1; then + wget -O "$archive_path" "$repo_url/$archive" else - echo "Error: Neither wget nor curl found. Please install one of them." + echo "Error: Neither curl nor wget found. Please install one of them." exit 1 fi diff --git a/rust-api-examples/run-streaming-zipformer-ctc-hlg.sh b/rust-api-examples/run-streaming-zipformer-ctc-hlg.sh new file mode 100755 index 000000000..ad30b91f2 --- /dev/null +++ b/rust-api-examples/run-streaming-zipformer-ctc-hlg.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +# see also +# https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 +if [ ! -f "./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/ctc-epoch-30-avg-3-chunk-16-left-128.int8.onnx" ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 + + tar xvf sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 + rm sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18.tar.bz2 + ls -lh sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18 +fi + +# Streaming CTC decoding with an HLG graph via ctc_fst_decoder_config. +cargo run --example streaming_zipformer_ctc_hlg -- \ + --model ./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/ctc-epoch-30-avg-3-chunk-16-left-128.int8.onnx \ + --tokens ./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/tokens.txt \ + --graph ./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/HLG.fst \ + --wav ./sherpa-onnx-streaming-zipformer-ctc-small-2024-03-18/test_wavs/8k.wav diff --git a/rust-api-examples/run-vits-de-phonemize.sh b/rust-api-examples/run-vits-de-phonemize.sh new file mode 100755 index 000000000..4aeedf1c6 --- /dev/null +++ b/rust-api-examples/run-vits-de-phonemize.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -d ./vits-piper-de_DE-glados-high ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-glados-high.tar.bz2 + tar xf vits-piper-de_DE-glados-high.tar.bz2 + rm vits-piper-de_DE-glados-high.tar.bz2 +fi + +cargo run --features phonemize --example vits_tts_de_phonemize diff --git a/rust-api-examples/run-vits-de.sh b/rust-api-examples/run-vits-de.sh deleted file mode 100755 index 3ca7b0eb6..000000000 --- a/rust-api-examples/run-vits-de.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -ex - -if [ ! -d ./vits-piper-de_DE-glados-high ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-de_DE-glados-high.tar.bz2 - tar xf vits-piper-de_DE-glados-high.tar.bz2 - rm vits-piper-de_DE-glados-high.tar.bz2 -fi - -cargo run --example vits_tts -- --model ./vits-piper-de_DE-glados-high/de_DE-glados-high.onnx --tokens ./vits-piper-de_DE-glados-high/tokens.txt --data-dir ./vits-piper-de_DE-glados-high/espeak-ng-data --output ./generated-vits-de-rust.wav --text "Alles hat ein Ende, nur die Wurst hat zwei." diff --git a/rust-api-examples/run-vits-en-lexicon.sh b/rust-api-examples/run-vits-en-lexicon.sh new file mode 100755 index 000000000..5cec1f2ea --- /dev/null +++ b/rust-api-examples/run-vits-en-lexicon.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -d ./vits-piper-en_US-amy-low ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + tar xf vits-piper-en_US-amy-low.tar.bz2 + rm vits-piper-en_US-amy-low.tar.bz2 +fi + +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +cargo run --example vits_tts_lexicon -- --model ./vits-piper-en_US-amy-low/en_US-amy-low.onnx --tokens ./vits-piper-en_US-amy-low/tokens.txt --lexicon ./lexicon-en-us.txt --debug --output ./generated-vits-en-lexicon-rust.wav --text "Liliana, the most beautiful and lovely assistant of our team!" diff --git a/rust-api-examples/run-vits-en-phonemize.sh b/rust-api-examples/run-vits-en-phonemize.sh new file mode 100644 index 000000000..82cfb732e --- /dev/null +++ b/rust-api-examples/run-vits-en-phonemize.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -d ./vits-piper-en_US-amy-low ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 + tar xf vits-piper-en_US-amy-low.tar.bz2 + rm vits-piper-en_US-amy-low.tar.bz2 +fi + +cargo run --features phonemize --example vits_tts_phonemize diff --git a/rust-api-examples/run-vits-en.sh b/rust-api-examples/run-vits-en.sh deleted file mode 100755 index ae4101556..000000000 --- a/rust-api-examples/run-vits-en.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -ex - -if [ ! -d ./vits-piper-en_US-amy-low ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 - tar xf vits-piper-en_US-amy-low.tar.bz2 - rm vits-piper-en_US-amy-low.tar.bz2 -fi - -cargo run --example vits_tts -- --model ./vits-piper-en_US-amy-low/en_US-amy-low.onnx --tokens ./vits-piper-en_US-amy-low/tokens.txt --data-dir ./vits-piper-en_US-amy-low/espeak-ng-data --output ./generated-vits-en-rust.wav --text "Liliana, the most beautiful and lovely assistant of our team!" diff --git a/rust-api-examples/run-zipformer-ctc-hlg.sh b/rust-api-examples/run-zipformer-ctc-hlg.sh new file mode 100755 index 000000000..504d8ed28 --- /dev/null +++ b/rust-api-examples/run-zipformer-ctc-hlg.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -ex + +# see also +# https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 +if [ ! -f "./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx" ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + + tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + ls -lh sherpa-onnx-zipformer-ctc-en-2023-10-02 +fi + +# Offline CTC decoding with an HLG graph via ctc_fst_decoder_config. +cargo run --example zipformer_ctc_hlg -- \ + --model ./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx \ + --tokens ./sherpa-onnx-zipformer-ctc-en-2023-10-02/tokens.txt \ + --graph ./sherpa-onnx-zipformer-ctc-en-2023-10-02/HLG.fst \ + --wav ./sherpa-onnx-zipformer-ctc-en-2023-10-02/test_wavs/0.wav diff --git a/rust-api-examples/run-zipvoice-tts.sh b/rust-api-examples/run-zipvoice-tts-lexicon.sh similarity index 74% rename from rust-api-examples/run-zipvoice-tts.sh rename to rust-api-examples/run-zipvoice-tts-lexicon.sh index 4ec898a72..73f3ba124 100755 --- a/rust-api-examples/run-zipvoice-tts.sh +++ b/rust-api-examples/run-zipvoice-tts-lexicon.sh @@ -11,4 +11,8 @@ if [ ! -f ./vocos_24khz.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx fi -cargo run --example zipvoice_tts +if [ ! -f ./lexicon-en-us.txt ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +cargo run --example zipvoice_tts_lexicon diff --git a/scripts/apk/build-apk-tts-engine.sh.in b/scripts/apk/build-apk-tts-engine.sh.in index b405fb41b..c526f2341 100644 --- a/scripts/apk/build-apk-tts-engine.sh.in +++ b/scripts/apk/build-apk-tts-engine.sh.in @@ -56,6 +56,10 @@ wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/$mod tar xf $model_dir.tar.bz2 rm $model_dir.tar.bz2 +# espeak-ng-data is unused: sherpa-onnx ignores data_dir since v2.0.0 and the +# phonemizer (piper-phonemize) bundles its own copy. Drop it to shrink the APK. +rm -rf $model_dir/espeak-ng-data + {% if tts_model.vocoder %} curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/$vocoder {% endif %} @@ -114,16 +118,13 @@ sed -i.bak s/"lang = null"/"lang = \"$lang_iso_639_3\""/ ./TtsEngine.kt sed -i.bak s%"ruleFsts = null"%"ruleFars = \"$rule_fars\""% ./TtsEngine.kt {% endif %} -{% if tts_model.data_dir %} - data_dir={{ tts_model.data_dir }} - sed -i.bak s%"dataDir = null"%"dataDir = \"$data_dir\""% ./TtsEngine.kt -{% elif not tts_model.is_char %} - sed -i.bak s/"lexicon = null"/"lexicon = \"lexicon.txt\""/ ./TtsEngine.kt -{% endif %} - -{% if tts_model.lexicon %} +{% if tts_model.phonemize %} + sed -i.bak s/"usePhonemize = false"/"usePhonemize = true"/ ./TtsEngine.kt +{% elif tts_model.lexicon %} lexicon={{ tts_model.lexicon }} sed -i.bak s%"lexicon = null"%"lexicon = \"$lexicon\""% ./TtsEngine.kt +{% elif not tts_model.is_char %} + sed -i.bak s/"lexicon = null"/"lexicon = \"lexicon.txt\""/ ./TtsEngine.kt {% endif %} git diff diff --git a/scripts/apk/build-apk-tts.sh.in b/scripts/apk/build-apk-tts.sh.in index fa1fe9a3e..6f04eed84 100644 --- a/scripts/apk/build-apk-tts.sh.in +++ b/scripts/apk/build-apk-tts.sh.in @@ -55,6 +55,10 @@ wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/$mod tar xf $model_dir.tar.bz2 rm $model_dir.tar.bz2 +# espeak-ng-data is unused: sherpa-onnx ignores data_dir since v2.0.0 and the +# phonemizer (piper-phonemize) bundles its own copy. Drop it to shrink the APK. +rm -rf $model_dir/espeak-ng-data + {% if tts_model.vocoder %} curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/$vocoder {% endif %} @@ -93,16 +97,13 @@ sed -i.bak s/"modelDir = null"/"modelDir = \"$model_dir\""/ ./MainActivity.kt sed -i.bak s%"ruleFsts = null"%"ruleFars = \"$rule_fars\""% ./MainActivity.kt {% endif %} -{% if tts_model.data_dir %} - data_dir={{ tts_model.data_dir }} - sed -i.bak s%"dataDir = null"%"dataDir = \"$data_dir\""% ./MainActivity.kt -{% elif not tts_model.is_char %} - sed -i.bak s/"lexicon = null"/"lexicon = \"lexicon.txt\""/ ./MainActivity.kt -{% endif %} - -{% if tts_model.lexicon %} +{% if tts_model.phonemize %} + sed -i.bak s/"usePhonemize = false"/"usePhonemize = true"/ ./MainActivity.kt +{% elif tts_model.lexicon %} lexicon={{ tts_model.lexicon }} sed -i.bak s%"lexicon = null"%"lexicon = \"$lexicon\""% ./MainActivity.kt +{% elif not tts_model.is_char %} + sed -i.bak s/"lexicon = null"/"lexicon = \"lexicon.txt\""/ ./MainActivity.kt {% endif %} {% if tts_model.is_kitten %} diff --git a/scripts/apk/generate-tts-apk-script.py b/scripts/apk/generate-tts-apk-script.py index 947e7ada0..7fa0ca7b5 100755 --- a/scripts/apk/generate-tts-apk-script.py +++ b/scripts/apk/generate-tts-apk-script.py @@ -43,7 +43,7 @@ class TtsModel: lang2: str = "" # en, zh, fr, de, etc. rule_fsts: Optional[List[str]] = None rule_fars: Optional[List[str]] = None - data_dir: Optional[str] = None + phonemize: bool = False dict_dir: Optional[str] = None is_char: bool = False lang_iso_639_3: str = "" @@ -78,7 +78,7 @@ def get_coqui_models() -> List[TtsModel]: ] for m in models: - m.data_dir = m.model_dir + "/" + "espeak-ng-data" + m.phonemize = True m.model_name = "model.onnx" m.lang = "en" @@ -134,6 +134,7 @@ def get_piper_models() -> List[TtsModel]: TtsModel(model_dir="vits-piper-de_DE-eva_k-x_low"), TtsModel(model_dir="vits-piper-de_DE-karlsson-low"), TtsModel(model_dir="vits-piper-de_DE-kerstin-low"), + TtsModel(model_dir="vits-piper-de_DE-glados-low"), TtsModel(model_dir="vits-piper-de_DE-dii-high"), TtsModel(model_dir="vits-piper-de_DE-miro-high"), # TtsModel(model_dir="vits-piper-de_DE-mls-medium"), @@ -305,10 +306,10 @@ def get_piper_models() -> List[TtsModel]: if "zh_CN" in m.model_dir and ( "xiao_ya" in m.model_dir or "chaowen" in m.model_dir ): - m.lexicon = f"{m.model_dir}/lexicon.txt" + m.lexicon = "lexicon.txt" m.rule_fsts = f"{m.model_dir}/phone.fst,{m.model_dir}/date.fst,{m.model_dir}/number.fst" else: - m.data_dir = m.model_dir + "/" + "espeak-ng-data" + m.phonemize = True m.model_name = m.model_dir[len("vits-piper-") :] + ".onnx" m.lang = m.model_dir.split("-")[2][:2] @@ -332,7 +333,7 @@ def get_mimic3_models() -> List[TtsModel]: TtsModel(model_dir="vits-mimic3-vi_VN-vais1000_low"), ] for m in models: - m.data_dir = m.model_dir + "/" + "espeak-ng-data" + m.phonemize = True m.model_name = m.model_dir[len("vits-mimic3-") :] + ".onnx" m.lang = m.model_dir.split("-")[2][:2] @@ -478,7 +479,7 @@ def get_inflect_models() -> List[TtsModel]: ] for m in english_models: - m.data_dir = f"{m.model_dir}/espeak-ng-data" + m.phonemize = True return english_models @@ -511,7 +512,6 @@ def get_matcha_models() -> List[TtsModel]: s = [f"{m.model_dir}/{r}" for r in rule_fsts_zh] m.rule_fsts = ",".join(s) m.vocoder = "vocos-16khz-univ.onnx" - m.data_dir = f"{m.model_dir}/espeak-ng-data" english_persian_models = [ TtsModel( @@ -531,7 +531,7 @@ def get_matcha_models() -> List[TtsModel]: ), ] for m in english_persian_models: - m.data_dir = f"{m.model_dir}/espeak-ng-data" + m.phonemize = True m.vocoder = "vocos-22khz-univ.onnx" return chinese_models + english_persian_models + chinese_english_models @@ -546,7 +546,7 @@ def get_kokoro_models() -> List[TtsModel]: ) ] for m in english_models: - m.data_dir = f"{m.model_dir}/espeak-ng-data" + m.phonemize = True m.voices = "voices.bin" multi_lingual_models = [ @@ -579,13 +579,12 @@ def get_kokoro_models() -> List[TtsModel]: ), ] for m in multi_lingual_models: - m.data_dir = f"{m.model_dir}/espeak-ng-data" m.voices = "voices.bin" m.lexicon = f"{m.model_dir}/lexicon-us-en.txt,{m.model_dir}/lexicon-zh.txt" m.rule_fsts = f"{m.model_dir}/phone-zh.fst,{m.model_dir}/date-zh.fst,{m.model_dir}/number-zh.fst" for m in arabic_models: - m.data_dir = f"{m.model_dir}/espeak-ng-data" + m.phonemize = True m.voices = "voices.bin" return english_models + multi_lingual_models + arabic_models @@ -630,7 +629,7 @@ def get_kitten_models() -> List[TtsModel]: ), ] for m in kitten_models: - m.data_dir = f"{m.model_dir}/espeak-ng-data" + m.phonemize = True m.voices = "voices.bin" m.is_kitten = True diff --git a/scripts/dart/tts-pubspec.yaml b/scripts/dart/tts-pubspec.yaml index f857597e4..d1058f161 100644 --- a/scripts/dart/tts-pubspec.yaml +++ b/scripts/dart/tts-pubspec.yaml @@ -1,6 +1,7 @@ name: tts description: A sample command-line application. version: 1.0.0 +publish_to: 'none' # repository: https://github.com/my_org/my_repo environment: @@ -11,6 +12,8 @@ dependencies: sherpa_onnx: path: ../../flutter/sherpa_onnx + piper_phonemize: ^1.5.8 + path: ^1.9.0 args: ^2.5.0 diff --git a/scripts/dotnet/NativeResources.cs b/scripts/dotnet/NativeResources.cs new file mode 100644 index 000000000..042884f32 --- /dev/null +++ b/scripts/dotnet/NativeResources.cs @@ -0,0 +1,52 @@ +/// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang) + +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + /// + /// Owns the native/pinned memory created while marshalling + /// OfflineTtsGenerationConfig to its native form (reference audio, the + /// flattened phoneme_codepoints/tokens arrays and the per-token C + /// strings). Dispose after the native call returns. + /// + internal sealed class NativeResources : IDisposable + { + internal GCHandle? AudioHandle; + internal GCHandle CodepointsHandle; + internal GCHandle CodepointsLensHandle; + internal GCHandle TokensHandle; + internal GCHandle TokensLensHandle; + internal readonly List TokenStrings = new List(); + + public void Dispose() + { + if (AudioHandle.HasValue) + { + AudioHandle.Value.Free(); + AudioHandle = null; + } + + if (CodepointsHandle.IsAllocated) + CodepointsHandle.Free(); + + if (CodepointsLensHandle.IsAllocated) + CodepointsLensHandle.Free(); + + if (TokensHandle.IsAllocated) + TokensHandle.Free(); + + if (TokensLensHandle.IsAllocated) + TokensLensHandle.Free(); + + foreach (IntPtr p in TokenStrings) + { + if (p != IntPtr.Zero) + Marshal.FreeHGlobal(p); + } + TokenStrings.Clear(); + } + } +} diff --git a/scripts/dotnet/OfflineCtcFstDecoderConfig.cs b/scripts/dotnet/OfflineCtcFstDecoderConfig.cs new file mode 100644 index 000000000..ba075fdb7 --- /dev/null +++ b/scripts/dotnet/OfflineCtcFstDecoderConfig.cs @@ -0,0 +1,24 @@ +/// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang) + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + [StructLayout(LayoutKind.Sequential)] + public struct OfflineCtcFstDecoderConfig + { + public OfflineCtcFstDecoderConfig() + { + Graph = ""; + MaxActive = 3000; + } + + [MarshalAs(UnmanagedType.LPStr)] + public string Graph; + + /// + /// Decoder max-active setting. 0 means 3000. + /// + public int MaxActive; + } +} diff --git a/scripts/dotnet/OfflineRecognizerConfig.cs b/scripts/dotnet/OfflineRecognizerConfig.cs index fd1fd773c..711c96f51 100644 --- a/scripts/dotnet/OfflineRecognizerConfig.cs +++ b/scripts/dotnet/OfflineRecognizerConfig.cs @@ -21,6 +21,7 @@ namespace SherpaOnnx RuleFars = ""; BlankPenalty = 0.0F; Hr = new HomophoneReplacerConfig(); + CtcFstDecoderConfig = new OfflineCtcFstDecoderConfig(); } public FeatureConfig FeatConfig; public OfflineModelConfig ModelConfig; @@ -45,5 +46,7 @@ namespace SherpaOnnx public float BlankPenalty; public HomophoneReplacerConfig Hr; + + public OfflineCtcFstDecoderConfig CtcFstDecoderConfig; } } diff --git a/scripts/dotnet/OfflineRecognizerResult.cs b/scripts/dotnet/OfflineRecognizerResult.cs index 5a9d59f9b..bb3998bda 100644 --- a/scripts/dotnet/OfflineRecognizerResult.cs +++ b/scripts/dotnet/OfflineRecognizerResult.cs @@ -94,6 +94,12 @@ namespace SherpaOnnx } } + // This struct mirrors SherpaOnnxOfflineRecognizerResult in c-api.h. + // Field order and types must match the C layout exactly; a + // Marshal.PtrToStructure on this struct reads raw memory from the C + // side, so any deviation silently reads the wrong offsets. Note that + // Durations sits near the end (after tokens_arr/json/lang/emotion/event), + // not right after Tokens. [StructLayout(LayoutKind.Sequential)] struct Impl { @@ -101,7 +107,18 @@ namespace SherpaOnnx public IntPtr Timestamps; public int Count; public IntPtr Tokens; + public IntPtr TokensArr; + public IntPtr Json; + public IntPtr Lang; + public IntPtr Emotion; + public IntPtr Event; public IntPtr Durations; + public IntPtr YsLogProbs; + public IntPtr SegmentTimestamps; + public IntPtr SegmentDurations; + public IntPtr SegmentTexts; + public IntPtr SegmentTextsArr; + public int SegmentCount; } private String _text; diff --git a/scripts/dotnet/OfflineTts.cs b/scripts/dotnet/OfflineTts.cs index d5d3f47c0..2ec31ef01 100644 --- a/scripts/dotnet/OfflineTts.cs +++ b/scripts/dotnet/OfflineTts.cs @@ -34,7 +34,7 @@ namespace SherpaOnnx var config = new OfflineTtsGenerationConfig(); config.Speed = speed; config.Sid = speakerId; - var nativeConfig = config.ToNative(out GCHandle? audioHandle); + var nativeConfig = config.ToNative(out NativeResources nativeResources); try { @@ -48,8 +48,7 @@ namespace SherpaOnnx } finally { - if (audioHandle.HasValue) - audioHandle.Value.Free(); + nativeResources.Dispose(); } } @@ -78,7 +77,7 @@ namespace SherpaOnnx }; } - var nativeConfig = config.ToNative(out GCHandle? audioHandle); + var nativeConfig = config.ToNative(out NativeResources nativeResources); GCHandle callbackHandle = default(GCHandle); try @@ -100,8 +99,7 @@ namespace SherpaOnnx if (callbackHandle.IsAllocated) callbackHandle.Free(); - if (audioHandle.HasValue) - audioHandle.Value.Free(); + nativeResources.Dispose(); } } @@ -130,7 +128,7 @@ namespace SherpaOnnx }; } - var nativeConfig = config.ToNative(out GCHandle? audioHandle); + var nativeConfig = config.ToNative(out NativeResources nativeResources); GCHandle callbackHandle = default(GCHandle); try @@ -152,8 +150,7 @@ namespace SherpaOnnx if (callbackHandle.IsAllocated) callbackHandle.Free(); - if (audioHandle.HasValue) - audioHandle.Value.Free(); + nativeResources.Dispose(); } } @@ -169,9 +166,7 @@ namespace SherpaOnnx utf8BytesWithNull[utf8Bytes.Length] = 0; GCHandle callbackHandle = default(GCHandle); - GCHandle? audioHandle = null; - - var nativeConfig = config.ToNative(out audioHandle); + var nativeConfig = config.ToNative(out NativeResources nativeResources); try { @@ -192,8 +187,7 @@ namespace SherpaOnnx if (callbackHandle.IsAllocated) callbackHandle.Free(); - if (audioHandle.HasValue) - audioHandle.Value.Free(); + nativeResources.Dispose(); } } @@ -236,6 +230,20 @@ namespace SherpaOnnx } } + /// + /// Language/voice of the model, e.g., "en-us". Empty if not available. + /// + public string Lang + { + get + { + IntPtr p = SherpaOnnxOfflineTtsLang(_handle.Handle); + string ans = Marshal.PtrToStringAnsi(p) ?? ""; + SherpaOnnxDestroyOfflineTtsLang(p); + return ans; + } + } + [DllImport(Dll.Filename)] private static extern IntPtr SherpaOnnxCreateOfflineTts(ref OfflineTtsConfig config); @@ -248,6 +256,12 @@ namespace SherpaOnnx [DllImport(Dll.Filename)] private static extern int SherpaOnnxOfflineTtsNumSpeakers(IntPtr handle); + [DllImport(Dll.Filename)] + private static extern IntPtr SherpaOnnxOfflineTtsLang(IntPtr handle); + + [DllImport(Dll.Filename)] + private static extern void SherpaOnnxDestroyOfflineTtsLang(IntPtr s); + [DllImport(Dll.Filename)] private static extern IntPtr SherpaOnnxOfflineTtsGenerate(IntPtr handle, [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.I1)] byte[] utf8Text, int sid, float speed); diff --git a/scripts/dotnet/OfflineTtsGeneratedAudio.cs b/scripts/dotnet/OfflineTtsGeneratedAudio.cs index 282820790..cdfc61af5 100644 --- a/scripts/dotnet/OfflineTtsGeneratedAudio.cs +++ b/scripts/dotnet/OfflineTtsGeneratedAudio.cs @@ -14,6 +14,10 @@ namespace SherpaOnnx public bool SaveToWaveFile(String filename) { + // A NULL handle means generation failed (the C API reports errors by + // returning no audio). Report empty audio instead of dereferencing + // IntPtr.Zero below. + if (Handle == IntPtr.Zero) return false; Impl impl = (Impl)Marshal.PtrToStructure(Handle, typeof(Impl)); byte[] utf8Filename = Encoding.UTF8.GetBytes(filename); byte[] utf8FilenameWithNull = new byte[utf8Filename.Length + 1]; // +1 for null terminator @@ -38,7 +42,10 @@ namespace SherpaOnnx private void Cleanup() { - SherpaOnnxDestroyOfflineTtsGeneratedAudio(Handle); + if (Handle != IntPtr.Zero) + { + SherpaOnnxDestroyOfflineTtsGeneratedAudio(Handle); + } // Don't permit the handle to be used again. _handle = new HandleRef(this, IntPtr.Zero); @@ -59,6 +66,7 @@ namespace SherpaOnnx { get { + if (Handle == IntPtr.Zero) return 0; Impl impl = (Impl)Marshal.PtrToStructure(Handle, typeof(Impl)); return impl.NumSamples; } @@ -68,6 +76,7 @@ namespace SherpaOnnx { get { + if (Handle == IntPtr.Zero) return 0; Impl impl = (Impl)Marshal.PtrToStructure(Handle, typeof(Impl)); return impl.SampleRate; } @@ -77,6 +86,7 @@ namespace SherpaOnnx { get { + if (Handle == IntPtr.Zero) return new float[0]; Impl impl = (Impl)Marshal.PtrToStructure(Handle, typeof(Impl)); float[] samples = new float[impl.NumSamples]; diff --git a/scripts/dotnet/OfflineTtsGenerationConfig.cs b/scripts/dotnet/OfflineTtsGenerationConfig.cs index c14901df9..d1b5beea2 100644 --- a/scripts/dotnet/OfflineTtsGenerationConfig.cs +++ b/scripts/dotnet/OfflineTtsGenerationConfig.cs @@ -1,153 +1,249 @@ -/// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang) - -using System; -using System.Collections; -using System.Runtime.InteropServices; -using System.Text; - -namespace SherpaOnnx -{ - public class OfflineTtsGenerationConfig - { - public OfflineTtsGenerationConfig() - { - SilenceScale = 0.2f; - Speed = 1.0f; - Sid = 0; - ReferenceAudio = null; - ReferenceSampleRate = 0; - ReferenceText = ""; - NumSteps = 5; - Extra = new Hashtable(); - } - - public float SilenceScale; - public float Speed; - public int Sid; - - public float[] ReferenceAudio; - public int ReferenceSampleRate; - public string ReferenceText; - public int NumSteps; - - /// - /// Extra attributes stored as key/value pairs - /// - public Hashtable Extra; - - /// - /// Convert to native struct for P/Invoke - /// - internal NativeStruct ToNative(out GCHandle? audioHandle) - { - NativeStruct native = new NativeStruct(); - native.SilenceScale = SilenceScale; - native.Speed = Speed; - native.Sid = Sid; - - // Handle ReferenceAudio - audioHandle = null; - if (ReferenceAudio != null && ReferenceAudio.Length > 0) - { - audioHandle = GCHandle.Alloc(ReferenceAudio, GCHandleType.Pinned); - native.ReferenceAudio = audioHandle.Value.AddrOfPinnedObject(); - native.ReferenceAudioLen = ReferenceAudio.Length; - } - else - { - native.ReferenceAudio = IntPtr.Zero; - native.ReferenceAudioLen = 0; - } - - native.ReferenceSampleRate = ReferenceSampleRate; - native.ReferenceText = ReferenceText ?? ""; - native.NumSteps = NumSteps; - - // Handle Extra JSON - native.Extra = "{}"; - if (Extra != null && Extra.Count > 0) - { - StringBuilder json = new StringBuilder(); - json.Append("{"); - bool first = true; - - foreach (DictionaryEntry kv in Extra) - { - if (!first) json.Append(","); - first = false; - - string key = JsonEscape(kv.Key.ToString()); - string val; - - if (kv.Value is string) - val = JsonEscape((string)kv.Value); - else if (kv.Value is float || kv.Value is double) - val = ((IFormattable)kv.Value).ToString(null, System.Globalization.CultureInfo.InvariantCulture); - else if (kv.Value is bool) - val = (bool)kv.Value ? "true" : "false"; - else - val = kv.Value.ToString(); - - json.AppendFormat("{0}:{1}", key, val); - } - - json.Append("}"); - native.Extra = json.ToString(); - } - return native; - } - - /// - /// Escapes a string for JSON (for .NET 2.0) - /// - private static string JsonEscape(string s) - { - if (s == null) return "\"\""; - - StringBuilder sb = new StringBuilder(); - sb.Append('"'); - foreach (char c in s) - { - switch (c) - { - case '"': sb.Append("\\\""); break; - case '\\': sb.Append("\\\\"); break; - case '\b': sb.Append("\\b"); break; - case '\f': sb.Append("\\f"); break; - case '\n': sb.Append("\\n"); break; - case '\r': sb.Append("\\r"); break; - case '\t': sb.Append("\\t"); break; - default: - if (c < 32 || c > 126) - sb.AppendFormat("\\u{0:X4}", (int)c); - else - sb.Append(c); - break; - } - } - sb.Append('"'); - return sb.ToString(); - } - - [StructLayout(LayoutKind.Sequential)] - internal struct NativeStruct - { - public float SilenceScale; - public float Speed; - public int Sid; - - public IntPtr ReferenceAudio; - public int ReferenceAudioLen; - public int ReferenceSampleRate; - - [MarshalAs(UnmanagedType.LPStr)] - public string ReferenceText; - - public int NumSteps; - - [MarshalAs(UnmanagedType.LPStr)] - public string Extra; - } - } -} - +/// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang) + +using System; +using System.Collections; +using System.Runtime.InteropServices; +using System.Text; + +namespace SherpaOnnx +{ + public class OfflineTtsGenerationConfig + { + public OfflineTtsGenerationConfig() + { + SilenceScale = 0.2f; + Speed = 1.0f; + Sid = 0; + ReferenceAudio = null; + ReferenceSampleRate = 0; + ReferenceText = ""; + NumSteps = 5; + Extra = new Hashtable(); + PhonemeCodepoints = null; + Tokens = null; + } + + public float SilenceScale; + public float Speed; + public int Sid; + + public float[] ReferenceAudio; + public int ReferenceSampleRate; + public string ReferenceText; + public int NumSteps; + + /// + /// Extra attributes stored as key/value pairs + /// + public Hashtable Extra; + + /// + /// Pre-phonemized input as Unicode codepoints. Each inner array is one + /// sentence. When set, it is used in place of the text argument; + /// leave empty to use text with a lexicon. Mutually exclusive with + /// Tokens. + /// + public int[][] PhonemeCodepoints; + + /// + /// Pre-tokenized input, e.g. Chinese pinyin like "zhong1". Each inner + /// array is one sentence. Mutually exclusive with PhonemeCodepoints. + /// + public string[][] Tokens; + + /// + /// Convert to native struct for P/Invoke + /// + internal NativeStruct ToNative(out NativeResources resources) + { + resources = new NativeResources(); + NativeStruct native = new NativeStruct(); + native.SilenceScale = SilenceScale; + native.Speed = Speed; + native.Sid = Sid; + + // Handle ReferenceAudio + if (ReferenceAudio != null && ReferenceAudio.Length > 0) + { + resources.AudioHandle = GCHandle.Alloc(ReferenceAudio, GCHandleType.Pinned); + native.ReferenceAudio = resources.AudioHandle.Value.AddrOfPinnedObject(); + native.ReferenceAudioLen = ReferenceAudio.Length; + } + else + { + native.ReferenceAudio = IntPtr.Zero; + native.ReferenceAudioLen = 0; + } + + native.ReferenceSampleRate = ReferenceSampleRate; + native.ReferenceText = ReferenceText ?? ""; + native.NumSteps = NumSteps; + + // Handle Extra JSON + native.Extra = "{}"; + if (Extra != null && Extra.Count > 0) + { + StringBuilder json = new StringBuilder(); + json.Append("{"); + bool first = true; + + foreach (DictionaryEntry kv in Extra) + { + if (!first) json.Append(","); + first = false; + + string key = JsonEscape(kv.Key.ToString()); + string val; + + if (kv.Value is string) + val = JsonEscape((string)kv.Value); + else if (kv.Value is float || kv.Value is double) + val = ((IFormattable)kv.Value).ToString(null, System.Globalization.CultureInfo.InvariantCulture); + else if (kv.Value is bool) + val = (bool)kv.Value ? "true" : "false"; + else + val = kv.Value.ToString(); + + json.AppendFormat("{0}:{1}", key, val); + } + + json.Append("}"); + native.Extra = json.ToString(); + } + // Flatten PhonemeCodepoints; the C API takes a flat array plus + // the length of each sentence. + if (PhonemeCodepoints != null && PhonemeCodepoints.Length > 0) + { + int total = 0; + foreach (int[] sentence in PhonemeCodepoints) + { + if (sentence != null) + total += sentence.Length; + } + + if (total > 0) + { + int[] flat = new int[total]; + int[] lens = new int[PhonemeCodepoints.Length]; + int k = 0; + for (int i = 0; i < PhonemeCodepoints.Length; i++) + { + int[] sentence = PhonemeCodepoints[i]; + if (sentence == null) + continue; + lens[i] = sentence.Length; + Array.Copy(sentence, 0, flat, k, sentence.Length); + k += sentence.Length; + } + + resources.CodepointsHandle = GCHandle.Alloc(flat, GCHandleType.Pinned); + resources.CodepointsLensHandle = GCHandle.Alloc(lens, GCHandleType.Pinned); + native.PhonemeCodepoints = resources.CodepointsHandle.AddrOfPinnedObject(); + native.PhonemeCodepointsLens = resources.CodepointsLensHandle.AddrOfPinnedObject(); + native.PhonemeCodepointsNumSentences = PhonemeCodepoints.Length; + } + } + + // Flatten Tokens the same way. Each token string becomes a C + // string; the char** array is pinned. + if (Tokens != null && Tokens.Length > 0) + { + int total = 0; + foreach (string[] sentence in Tokens) + { + if (sentence != null) + total += sentence.Length; + } + + if (total > 0) + { + IntPtr[] flat = new IntPtr[total]; + int[] lens = new int[Tokens.Length]; + int k = 0; + for (int i = 0; i < Tokens.Length; i++) + { + string[] sentence = Tokens[i]; + if (sentence == null) + continue; + lens[i] = sentence.Length; + for (int j = 0; j < sentence.Length; j++) + { + IntPtr p = Marshal.StringToHGlobalAnsi(sentence[j] ?? ""); + resources.TokenStrings.Add(p); + flat[k++] = p; + } + } + + resources.TokensHandle = GCHandle.Alloc(flat, GCHandleType.Pinned); + resources.TokensLensHandle = GCHandle.Alloc(lens, GCHandleType.Pinned); + native.Tokens = resources.TokensHandle.AddrOfPinnedObject(); + native.TokensLens = resources.TokensLensHandle.AddrOfPinnedObject(); + native.TokensNumSentences = Tokens.Length; + } + } + + return native; + } + + /// + /// Escapes a string for JSON (for .NET 2.0) + /// + private static string JsonEscape(string s) + { + if (s == null) return "\"\""; + + StringBuilder sb = new StringBuilder(); + sb.Append('"'); + foreach (char c in s) + { + switch (c) + { + case '"': sb.Append("\\\""); break; + case '\\': sb.Append("\\\\"); break; + case '\b': sb.Append("\\b"); break; + case '\f': sb.Append("\\f"); break; + case '\n': sb.Append("\\n"); break; + case '\r': sb.Append("\\r"); break; + case '\t': sb.Append("\\t"); break; + default: + if (c < 32 || c > 126) + sb.AppendFormat("\\u{0:X4}", (int)c); + else + sb.Append(c); + break; + } + } + sb.Append('"'); + return sb.ToString(); + } + + [StructLayout(LayoutKind.Sequential)] + internal struct NativeStruct + { + public float SilenceScale; + public float Speed; + public int Sid; + + public IntPtr ReferenceAudio; + public int ReferenceAudioLen; + public int ReferenceSampleRate; + + [MarshalAs(UnmanagedType.LPStr)] + public string ReferenceText; + + public int NumSteps; + + [MarshalAs(UnmanagedType.LPStr)] + public string Extra; + + public IntPtr PhonemeCodepoints; + public IntPtr PhonemeCodepointsLens; + public int PhonemeCodepointsNumSentences; + + public IntPtr Tokens; + public IntPtr TokensLens; + public int TokensNumSentences; + } + } +} + diff --git a/scripts/dotnet/OfflineTtsKittenModelConfig.cs b/scripts/dotnet/OfflineTtsKittenModelConfig.cs index 366bc9f3a..8c3baece9 100644 --- a/scripts/dotnet/OfflineTtsKittenModelConfig.cs +++ b/scripts/dotnet/OfflineTtsKittenModelConfig.cs @@ -13,6 +13,7 @@ namespace SherpaOnnx Voices = ""; Tokens = ""; DataDir = ""; + Lexicon = ""; LengthScale = 1.0F; } @@ -25,9 +26,16 @@ namespace SherpaOnnx [MarshalAs(UnmanagedType.LPStr)] public string Tokens; + /// + /// Deprecated. Ignored since v2.0.0. Use Lexicon or + /// OfflineTtsGenerationConfig.PhonemeCodepoints instead. + /// [MarshalAs(UnmanagedType.LPStr)] public string DataDir; public float LengthScale; + + [MarshalAs(UnmanagedType.LPStr)] + public string Lexicon; } } diff --git a/scripts/dotnet/OfflineTtsKokoroModelConfig.cs b/scripts/dotnet/OfflineTtsKokoroModelConfig.cs index 1a76288a4..e81acd444 100644 --- a/scripts/dotnet/OfflineTtsKokoroModelConfig.cs +++ b/scripts/dotnet/OfflineTtsKokoroModelConfig.cs @@ -1,4 +1,4 @@ -/// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) +/// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) using System.Runtime.InteropServices; @@ -29,6 +29,10 @@ namespace SherpaOnnx [MarshalAs(UnmanagedType.LPStr)] public string Tokens; + /// + /// Deprecated. Ignored since v2.0.0. Use Lexicon or + /// OfflineTtsGenerationConfig.PhonemeCodepoints instead. + /// [MarshalAs(UnmanagedType.LPStr)] public string DataDir; diff --git a/scripts/dotnet/OfflineTtsMatchaModelConfig.cs b/scripts/dotnet/OfflineTtsMatchaModelConfig.cs index 8743e1223..60cc0843e 100644 --- a/scripts/dotnet/OfflineTtsMatchaModelConfig.cs +++ b/scripts/dotnet/OfflineTtsMatchaModelConfig.cs @@ -1,4 +1,4 @@ -/// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) +/// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) using System.Runtime.InteropServices; @@ -32,6 +32,10 @@ namespace SherpaOnnx [MarshalAs(UnmanagedType.LPStr)] public string Tokens; + /// + /// Deprecated. Ignored since v2.0.0. Use Lexicon or + /// OfflineTtsGenerationConfig.PhonemeCodepoints instead. + /// [MarshalAs(UnmanagedType.LPStr)] public string DataDir; diff --git a/scripts/dotnet/OfflineTtsVitsModelConfig.cs b/scripts/dotnet/OfflineTtsVitsModelConfig.cs index f72430b2d..023e36e91 100644 --- a/scripts/dotnet/OfflineTtsVitsModelConfig.cs +++ b/scripts/dotnet/OfflineTtsVitsModelConfig.cs @@ -1,4 +1,4 @@ -/// Copyright (c) 2024.5 by 东风破 +/// Copyright (c) 2024.5 by 东风破 using System.Runtime.InteropServices; @@ -29,6 +29,10 @@ namespace SherpaOnnx [MarshalAs(UnmanagedType.LPStr)] public string Tokens; + /// + /// Deprecated. Ignored since v2.0.0. Use Lexicon or + /// OfflineTtsGenerationConfig.PhonemeCodepoints instead. + /// [MarshalAs(UnmanagedType.LPStr)] public string DataDir; diff --git a/scripts/dotnet/OfflineTtsZipVoiceModelConfig.cs b/scripts/dotnet/OfflineTtsZipVoiceModelConfig.cs index 82ddeb5b5..d0852e035 100644 --- a/scripts/dotnet/OfflineTtsZipVoiceModelConfig.cs +++ b/scripts/dotnet/OfflineTtsZipVoiceModelConfig.cs @@ -1,4 +1,4 @@ -/// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) +/// Copyright (c) 2025 Xiaomi Corporation (authors: Fangjun Kuang) using System.Runtime.InteropServices; @@ -33,6 +33,10 @@ namespace SherpaOnnx [MarshalAs(UnmanagedType.LPStr)] public string Vocoder; + /// + /// Deprecated. Ignored since v2.0.0. Use Lexicon or + /// OfflineTtsGenerationConfig.PhonemeCodepoints instead. + /// [MarshalAs(UnmanagedType.LPStr)] public string DataDir; diff --git a/scripts/flutter/.gitignore b/scripts/flutter/.gitignore deleted file mode 100644 index 1ff7096c7..000000000 --- a/scripts/flutter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -!*.sh.in diff --git a/scripts/flutter/build-android-streaming-asr.sh.in b/scripts/flutter/build-android-streaming-asr.sh.in deleted file mode 100755 index b0bc944c5..000000000 --- a/scripts/flutter/build-android-streaming-asr.sh.in +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash -set -ex - -log() { - # This function is from espnet - local fname=${BASH_SOURCE[1]##*/} - echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" -} - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) -log "SCRIPT_DIR: $SCRIPT_DIR" -log "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" - -SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" $SHERPA_ONNX_DIR/CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) -log "SHERPA_ONNX_VERSION: $SHERPA_ONNX_VERSION" - -if [ -z $arch ]; then - arch=x86_64 -fi - -log "arch: $arch" - -{% for model in model_list %} -pushd $SHERPA_ONNX_DIR/flutter-examples/streaming_asr/ - -model_name={{ model.model_name }} -lang={{ model.lang }} -type={{ model.idx }} -short_name={{ model.short_name }} - -rm -rf assets -mkdir assets - -pushd assets - -curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${model_name}.tar.bz2 -tar xvf ${model_name}.tar.bz2 - -{{ model.cmd }} - -rm -rf *.tar.bz2 -ls -lh $model_name - -ls -lh -ls -lh * - -popd - -git checkout ./ -sed -i.bak "s| - assets/$| - assets/\n - assets/$model_name/|g" ./pubspec.yaml - -sed -i.bak "s/final type = .*;$/final type = $type;/g" ./lib/streaming_asr.dart - -{% if model.rule_fsts %} - rule_fsts={{ model.rule_fsts }} - sed -i.bak "s|ruleFsts: ''|ruleFsts: await copyAssetFile(\'assets/$rule_fsts\')|g" ./lib/streaming_asr.dart -{% endif %} - -git diff . - -flutter pub get - -flutter build apk --split-per-abi --release - -pushd build/app/outputs/flutter-apk -ls -lh - - - -for arch in armeabi-v7a arm64-v8a x86_64; do - src=app-$arch-release.apk - dst=$SHERPA_ONNX_DIR/sherpa-onnx-$SHERPA_ONNX_VERSION-$arch-asr-$short_name.apk - mv $src $dst -done - -pushd $SHERPA_ONNX_DIR -ls -lh *.apk -popd - -popd - -popd - -{% endfor %} diff --git a/scripts/flutter/build-android-tts.sh.in b/scripts/flutter/build-android-tts.sh.in deleted file mode 100755 index 252036952..000000000 --- a/scripts/flutter/build-android-tts.sh.in +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env bash -set -ex - -log() { - # This function is from espnet - local fname=${BASH_SOURCE[1]##*/} - echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" -} - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) -log "SCRIPT_DIR: $SCRIPT_DIR" -log "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" - -SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" $SHERPA_ONNX_DIR/CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) -log "SHERPA_ONNX_VERSION: $SHERPA_ONNX_VERSION" - -{% for tts_model in tts_model_list %} -pushd $SHERPA_ONNX_DIR/flutter-examples/tts/ - -git checkout . - -rm -rf assets -mkdir assets -pushd assets - -model_dir={{ tts_model.model_dir }} -model_name={{ tts_model.model_name }} -lang={{ tts_model.lang }} - -curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/$model_dir.tar.bz2 -tar xf $model_dir.tar.bz2 -rm $model_dir.tar.bz2 - -ls -lh -ls -lh * - -popd # assets -# Now we are at the project root directory - -./generate-asset-list.py - -pushd lib - -sed -i.bak "s|modelDir = ''|modelDir = \"$model_dir\"|" ./model.dart -sed -i.bak s/"modelName = ''"/"modelName = \"$model_name\""/ ./model.dart - -{% if tts_model.rule_fsts %} - rule_fsts={{ tts_model.rule_fsts }} - sed -i.bak "s|ruleFsts = ''|ruleFsts = \"$rule_fsts\"|" ./model.dart -{% endif %} - -{% if tts_model.rule_fars %} - rule_fars={{ tts_model.rule_fars }} - sed -i.bak "s|ruleFars = ''|ruleFars = \"$rule_fars\"|" ./model.dart -{% endif %} - -{% if tts_model.data_dir %} - data_dir={{ tts_model.data_dir }} - sed -i.bak "s|dataDir = ''|dataDir = \"$data_dir\"|" ./model.dart -{% elif not tts_model.is_char %} - sed -i.bak "s|lexicon = ''|lexicon = \"lexicon.txt\"|" ./model.dart -{% endif %} - -{% if tts_model.is_kitten %} - sed -i.bak "s|voices = ''|voices = \"voices.bin\"|" ./model.dart - sed -i.bak "s|isKitten = false|isKitten = true|" ./model.dart -{% endif %} - -git status - -git diff . - -popd #lib - -flutter pub get - -flutter build apk --split-per-abi --release - -pushd build/app/outputs/flutter-apk -ls -lh - -for arch in armeabi-v7a arm64-v8a x86_64; do - src=app-$arch-release.apk - dst=$SHERPA_ONNX_DIR/sherpa-onnx-$SHERPA_ONNX_VERSION-$arch-$lang-tts-$model_dir.apk - mv $src $dst -done - -pushd $SHERPA_ONNX_DIR -ls -lh *.apk -popd - -popd - -popd - -{% endfor %} diff --git a/scripts/flutter/build-linux-streaming-asr.sh.in b/scripts/flutter/build-linux-streaming-asr.sh.in deleted file mode 100755 index 62a855626..000000000 --- a/scripts/flutter/build-linux-streaming-asr.sh.in +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -set -ex - -log() { - # This function is from espnet - local fname=${BASH_SOURCE[1]##*/} - echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" -} - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) -log "SCRIPT_DIR: $SCRIPT_DIR" -log "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" - -SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" $SHERPA_ONNX_DIR/CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) -log "SHERPA_ONNX_VERSION: $SHERPA_ONNX_VERSION" - -if [ -z $arch ]; then - arch=x86_64 -fi - -log "arch: $arch" - -{% for model in model_list %} -pushd $SHERPA_ONNX_DIR/flutter-examples/streaming_asr/ - -model_name={{ model.model_name }} -lang={{ model.lang }} -type={{ model.idx }} -short_name={{ model.short_name }} - -rm -rf assets -mkdir assets - -pushd assets - -curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${model_name}.tar.bz2 -tar xvf ${model_name}.tar.bz2 - -{{ model.cmd }} - -rm -rf *.tar.bz2 -ls -lh $model_name - -ls -lh -ls -lh * - -popd - -git checkout ./ -sed -i.bak "s| - assets/$| - assets/\n - assets/$model_name/|g" ./pubspec.yaml - -sed -i.bak "s/final type = .*;$/final type = $type;/g" ./lib/streaming_asr.dart - -{% if model.rule_fsts %} - rule_fsts={{ model.rule_fsts }} - sed -i.bak "s|ruleFsts: ''|ruleFsts: await copyAssetFile(\'assets/$rule_fsts\')|g" ./lib/streaming_asr.dart -{% endif %} - -git diff . - -flutter pub get -# flutter upgrade -# flutter pub upgrade - -export FLUTTER_XCODE_ARCHS=$arch -log "FLUTTER_XCODE_ARCHS: $FLUTTER_XCODE_ARCHS" - -flutter build linux - -pushd build/linux/x64/release/ - -ls -lh -echo "----" -ls -lh bundle - -echo "----" -ls -lh bundle/lib - -echo "----" -ls -lh bundle/data -echo "======" -ls -lh bundle/data/flutter_assets - - -echo "----" -file bundle/streaming_asr - -echo "----" -readelf -d bundle/streaming_asr - -echo "----" -ldd bundle/streaming_asr - -app=sherpa-onnx-$SHERPA_ONNX_VERSION-linux-$arch-$lang-streaming_asr -app=sherpa-onnx-$SHERPA_ONNX_VERSION-linux-$arch-asr-$lang-$short_name - -mv bundle $app -tar cjf $app.tar.bz2 $app -rm -rf $app -mv $app.tar.bz2 $SHERPA_ONNX_DIR - -pushd $SHERPA_ONNX_DIR -ls -lh *.tar.bz2 -popd - -popd - -popd - -{% endfor %} diff --git a/scripts/flutter/build-linux-tts.sh.in b/scripts/flutter/build-linux-tts.sh.in deleted file mode 100755 index cf73bd301..000000000 --- a/scripts/flutter/build-linux-tts.sh.in +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env bash -set -ex - -log() { - # This function is from espnet - local fname=${BASH_SOURCE[1]##*/} - echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" -} - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) -log "SCRIPT_DIR: $SCRIPT_DIR" -log "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" - -SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" $SHERPA_ONNX_DIR/CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) -log "SHERPA_ONNX_VERSION: $SHERPA_ONNX_VERSION" - -if [ -z $arch ]; then - arch=x86_64 -fi - -log "arch: $arch" - -{% for tts_model in tts_model_list %} -pushd $SHERPA_ONNX_DIR/flutter-examples/tts/ - -git checkout . - -rm -rf assets -mkdir assets - -pushd assets - -model_dir={{ tts_model.model_dir }} -model_name={{ tts_model.model_name }} -lang={{ tts_model.lang }} - -curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/$model_dir.tar.bz2 -tar xf $model_dir.tar.bz2 -rm $model_dir.tar.bz2 - -ls -lh -ls -lh * - -popd # assets -# Now we are at the project root directory - -./generate-asset-list.py - -pushd lib - -sed -i.bak "s|modelDir = ''|modelDir = \"$model_dir\"|" ./model.dart -sed -i.bak s/"modelName = ''"/"modelName = \"$model_name\""/ ./model.dart - -{% if tts_model.rule_fsts %} - rule_fsts={{ tts_model.rule_fsts }} - sed -i.bak "s|ruleFsts = ''|ruleFsts = \"$rule_fsts\"|" ./model.dart -{% endif %} - -{% if tts_model.rule_fars %} - rule_fars={{ tts_model.rule_fars }} - sed -i.bak "s|ruleFars = ''|ruleFars = \"$rule_fars\"|" ./model.dart -{% endif %} - -{% if tts_model.data_dir %} - data_dir={{ tts_model.data_dir }} - sed -i.bak "s|dataDir = ''|dataDir = \"$data_dir\"|" ./model.dart -{% elif not tts_model.is_char %} - sed -i.bak "s|lexicon = ''|lexicon = \"lexicon.txt\"|" ./model.dart -{% endif %} - -{% if tts_model.is_kitten %} - sed -i.bak "s|voices = ''|voices = \"voices.bin\"|" ./model.dart - sed -i.bak "s|isKitten = false|isKitten = true|" ./model.dart -{% endif %} - -git status - -git diff . - -popd #lib - -flutter pub get - -flutter build linux - -pushd build/linux/x64/release/ - -ls -lh -echo "----" -ls -lh bundle - -echo "----" -ls -lh bundle/lib - -echo "----" -ls -lh bundle/data -echo "======" -ls -lh bundle/data/flutter_assets - - -echo "----" -file bundle/tts - -echo "----" -readelf -d bundle/tts - -echo "----" -ldd bundle/tts - -app=sherpa-onnx-$SHERPA_ONNX_VERSION-linux-$arch-$lang-tts-$model_dir - -mv bundle $app -tar cjf $app.tar.bz2 $app -rm -rf $app -mv $app.tar.bz2 $SHERPA_ONNX_DIR - -pushd $SHERPA_ONNX_DIR -ls -lh *.tar.bz2 -popd - -popd - -popd -{% endfor %} diff --git a/scripts/flutter/build-macos-streaming-asr.sh.in b/scripts/flutter/build-macos-streaming-asr.sh.in deleted file mode 100755 index 84dd6f4c5..000000000 --- a/scripts/flutter/build-macos-streaming-asr.sh.in +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash -set -ex - -log() { - # This function is from espnet - local fname=${BASH_SOURCE[1]##*/} - echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" -} - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) -log "SCRIPT_DIR: $SCRIPT_DIR" -log "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" - -SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" $SHERPA_ONNX_DIR/CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) -log "SHERPA_ONNX_VERSION: $SHERPA_ONNX_VERSION" - -if [ -z $arch ]; then - arch=x86_64 -fi - -log "arch: $arch" - -{% for model in model_list %} -pushd $SHERPA_ONNX_DIR/flutter-examples/streaming_asr/ - -model_name={{ model.model_name }} -lang={{ model.lang }} -type={{ model.idx }} -short_name={{ model.short_name }} - -rm -rf assets -mkdir assets - -pushd assets - -curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${model_name}.tar.bz2 -tar xvf ${model_name}.tar.bz2 - -{{ model.cmd }} - -rm -rf *.tar.bz2 -ls -lh $model_name - -ls -lh -ls -lh * - -popd - -git checkout ./ -sed -i.bak "s| - assets/$| - assets/\n - assets/$model_name/|g" ./pubspec.yaml - -sed -i.bak "s/final type = .*;$/final type = $type;/g" ./lib/streaming_asr.dart - -{% if model.rule_fsts %} - rule_fsts={{ model.rule_fsts }} - sed -i.bak "s|ruleFsts: ''|ruleFsts: await copyAssetFile(\'assets/$rule_fsts\')|g" ./lib/streaming_asr.dart -{% endif %} - -git diff . - -flutter pub get - -export FLUTTER_XCODE_ARCHS=$arch -log "FLUTTER_XCODE_ARCHS: $FLUTTER_XCODE_ARCHS" - -flutter build macos - -pushd build/macos/Build/Products/Release/ -ls -lh - -app=sherpa-onnx-$SHERPA_ONNX_VERSION-osx-$arch-asr-$lang-$short_name.app -mv streaming_asr.app $app -tar cjf $app.tar.bz2 $app -rm -rf $app -ls -lh -mv $app.tar.bz2 $SHERPA_ONNX_DIR - -pushd $SHERPA_ONNX_DIR -ls -lh *.tar.bz2 -popd - -popd - -popd - -{% endfor %} diff --git a/scripts/flutter/build-macos-tts.sh.in b/scripts/flutter/build-macos-tts.sh.in deleted file mode 100755 index 324ecbe43..000000000 --- a/scripts/flutter/build-macos-tts.sh.in +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -set -ex - -log() { - # This function is from espnet - local fname=${BASH_SOURCE[1]##*/} - echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" -} - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) -log "SCRIPT_DIR: $SCRIPT_DIR" -log "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" - -SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" $SHERPA_ONNX_DIR/CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) -log "SHERPA_ONNX_VERSION: $SHERPA_ONNX_VERSION" - -if [ -z $arch ]; then - arch=x86_64 -fi - -log "arch: $arch" - -{% for tts_model in tts_model_list %} -pushd $SHERPA_ONNX_DIR/flutter-examples/tts/ - -git checkout . - -rm -rf assets -mkdir assets - -pushd assets - -model_dir={{ tts_model.model_dir }} -model_name={{ tts_model.model_name }} -lang={{ tts_model.lang }} - -curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/$model_dir.tar.bz2 -tar xf $model_dir.tar.bz2 -rm $model_dir.tar.bz2 - -ls -lh -ls -lh * - -popd # assets -# Now we are at the project root directory - -./generate-asset-list.py - -pushd lib - -sed -i.bak "s|modelDir = ''|modelDir = \"$model_dir\"|" ./model.dart -sed -i.bak s/"modelName = ''"/"modelName = \"$model_name\""/ ./model.dart - -{% if tts_model.rule_fsts %} - rule_fsts={{ tts_model.rule_fsts }} - sed -i.bak "s|ruleFsts = ''|ruleFsts = \"$rule_fsts\"|" ./model.dart -{% endif %} - -{% if tts_model.rule_fars %} - rule_fars={{ tts_model.rule_fars }} - sed -i.bak "s|ruleFars = ''|ruleFars = \"$rule_fars\"|" ./model.dart -{% endif %} - -{% if tts_model.data_dir %} - data_dir={{ tts_model.data_dir }} - sed -i.bak "s|dataDir = ''|dataDir = \"$data_dir\"|" ./model.dart -{% elif not tts_model.is_char %} - sed -i.bak "s|lexicon = ''|lexicon = \"lexicon.txt\"|" ./model.dart -{% endif %} - -{% if tts_model.is_kitten %} - sed -i.bak "s|voices = ''|voices = \"voices.bin\"|" ./model.dart - sed -i.bak "s|isKitten = false|isKitten = true|" ./model.dart -{% endif %} - -git status - -git diff . - -popd #lib - -flutter pub get - -export FLUTTER_XCODE_ARCHS=$arch -log "FLUTTER_XCODE_ARCHS: $FLUTTER_XCODE_ARCHS" - -flutter build macos - -pushd build/macos/Build/Products/Release/ -ls -lh - -app=sherpa-onnx-$SHERPA_ONNX_VERSION-osx-$arch-$lang-tts-$model_dir.app - -mv tts.app $app -tar cjf $app.tar.bz2 $app -ls -lh -mv $app.tar.bz2 $SHERPA_ONNX_DIR -rm -rf $app - -pushd $SHERPA_ONNX_DIR -ls -lh *.tar.bz2 -popd - -popd - -popd - -{% endfor %} diff --git a/scripts/flutter/build-windows-streaming-asr.sh.in b/scripts/flutter/build-windows-streaming-asr.sh.in deleted file mode 100755 index a6ad57b1b..000000000 --- a/scripts/flutter/build-windows-streaming-asr.sh.in +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -set -ex - -log() { - # This function is from espnet - local fname=${BASH_SOURCE[1]##*/} - echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" -} - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) -log "SCRIPT_DIR: $SCRIPT_DIR" -log "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" - -SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" $SHERPA_ONNX_DIR/CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) -log "SHERPA_ONNX_VERSION: $SHERPA_ONNX_VERSION" - -{% for model in model_list %} -pushd $SHERPA_ONNX_DIR/flutter-examples/streaming_asr/ - -model_name={{ model.model_name }} -lang={{ model.lang }} -type={{ model.idx }} -short_name={{ model.short_name }} - -rm -rf build -rm -rf assets -mkdir assets - -pushd assets - -curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${model_name}.tar.bz2 -tar xvf ${model_name}.tar.bz2 - -{{ model.cmd }} - -rm -rf *.tar.bz2 -ls -lh $model_name - -ls -lh -ls -lh * - -popd - -git checkout ./ -sed -i.bak "s| - assets/$| - assets/\n - assets/$model_name/|g" ./pubspec.yaml - -sed -i.bak "s/final type = .*;$/final type = $type;/g" ./lib/streaming_asr.dart - -{% if model.rule_fsts %} - rule_fsts={{ model.rule_fsts }} - sed -i.bak "s|ruleFsts: ''|ruleFsts: await copyAssetFile(\'assets/$rule_fsts\')|g" ./lib/streaming_asr.dart -{% endif %} - -git diff . - -flutter pub get - -flutter build windows - -pushd build/windows/x64/runner/ -ls -lh - -dst=sherpa-onnx-$SHERPA_ONNX_VERSION-win-x64-asr-$lang-$short_name -mv Release $dst -tar cjf $dst.tar.bz2 ./$dst -rm -rf $dst -mv $dst.tar.bz2 $SHERPA_ONNX_DIR - -pushd $SHERPA_ONNX_DIR -ls -lh *.tar.bz2 -popd - -popd - -popd - -{% endfor %} diff --git a/scripts/flutter/build-windows-tts.sh.in b/scripts/flutter/build-windows-tts.sh.in deleted file mode 100755 index beb33643b..000000000 --- a/scripts/flutter/build-windows-tts.sh.in +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -set -ex - -log() { - # This function is from espnet - local fname=${BASH_SOURCE[1]##*/} - echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" -} - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -SHERPA_ONNX_DIR=$(cd $SCRIPT_DIR/../.. && pwd) -log "SCRIPT_DIR: $SCRIPT_DIR" -log "SHERPA_ONNX_DIR: $SHERPA_ONNX_DIR" - -SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" $SHERPA_ONNX_DIR/CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) -log "SHERPA_ONNX_VERSION: $SHERPA_ONNX_VERSION" - -if [ -z $arch ]; then - arch=x64 -fi - -log "arch: $arch" - -{% for tts_model in tts_model_list %} -pushd $SHERPA_ONNX_DIR/flutter-examples/tts/ - -git checkout . - -rm -rf assets - -mkdir -p assets - -pushd assets - -model_dir={{ tts_model.model_dir }} -model_name={{ tts_model.model_name }} -lang={{ tts_model.lang }} - -curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/$model_dir.tar.bz2 -tar xf $model_dir.tar.bz2 -rm $model_dir.tar.bz2 - -ls -lh - -ls -lh * - -popd # assets -# Now we are at the project root directory - -./generate-asset-list.py - -pushd lib - -sed -i.bak "s|modelDir = ''|modelDir = \"$model_dir\"|" ./model.dart -sed -i.bak s/"modelName = ''"/"modelName = \"$model_name\""/ ./model.dart - -{% if tts_model.rule_fsts %} - rule_fsts={{ tts_model.rule_fsts }} - sed -i.bak "s|ruleFsts = ''|ruleFsts = \"$rule_fsts\"|" ./model.dart -{% endif %} - -{% if tts_model.rule_fars %} - rule_fars={{ tts_model.rule_fars }} - sed -i.bak "s|ruleFars = ''|ruleFars = \"$rule_fars\"|" ./model.dart -{% endif %} - -{% if tts_model.data_dir %} - data_dir={{ tts_model.data_dir }} - sed -i.bak "s|dataDir = ''|dataDir = \"$data_dir\"|" ./model.dart -{% elif not tts_model.is_char %} - sed -i.bak "s|lexicon = ''|lexicon = \"lexicon.txt\"|" ./model.dart -{% endif %} - -{% if tts_model.is_kitten %} - sed -i.bak "s|voices = ''|voices = \"voices.bin\"|" ./model.dart - sed -i.bak "s|isKitten = false|isKitten = true|" ./model.dart -{% endif %} - -git status - -git diff . - -popd #lib - -flutter pub get - -flutter build windows - -pushd build/windows/x64/runner/ -ls -lh -echo "---" - -ls -lh ./Release/ - -dst=sherpa-onnx-$SHERPA_ONNX_VERSION-win-$arch-$lang-tts-$model_dir -mv Release $dst -tar cjf $dst.tar.bz2 ./$dst -rm -rf $dst -mv $dst.tar.bz2 $SHERPA_ONNX_DIR - -pushd $SHERPA_ONNX_DIR -ls -lh *.tar.bz2 -popd - -popd - -popd - -{% endfor %} diff --git a/scripts/flutter/generate-streaming-asr.py b/scripts/flutter/generate-streaming-asr.py deleted file mode 100755 index 1bdb2d4ad..000000000 --- a/scripts/flutter/generate-streaming-asr.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -from dataclasses import dataclass -import jinja2 - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--total", - type=int, - default=1, - help="Number of runners", - ) - parser.add_argument( - "--index", - type=int, - default=0, - help="Index of the current runner", - ) - - return parser.parse_args() - - -@dataclass -class Model: - # We will download - # https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/{model_name}.tar.bz2 - model_name: str - - # The type of the model, e..g, 0, 1, 2. It is hardcoded in the flutter code - # See flutter-example/streaming_asr/lib/online_model.dart - idx: int - - # e.g., zh, en, zh_en - lang: str - - # e.g., whisper, paraformer, zipformer - short_name: str = "" - - # cmd is used to remove extra files from the model directory - cmd: str = "" - - rule_fsts: str = "" - - -def get_models(): - models = [ - Model( - model_name="sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", - idx=0, - lang="bilingual_zh_en", - short_name="zipformer_2023_02_20", - rule_fsts="itn_zh_number.fst", - cmd=""" - if [ ! -f itn_zh_number.fst ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst - fi - pushd $model_name - rm -fv encoder-epoch-99-avg-1.onnx - rm -fv decoder-epoch-99-avg-1.int8.onnx - rm -fv joiner-epoch-99-avg-1.int8.onnx - - rm -fv *.sh - rm -fv bpe.model - rm -fv README.md - rm -fv .gitattributes - rm -fv *state* - rm -rfv test_wavs - - ls -lh - - popd - """, - ), - Model( - model_name="sherpa-onnx-streaming-zipformer-en-2023-06-26", - idx=1, - lang="en", - short_name="zipformer2_2023_06_26", - cmd=""" - pushd $model_name - rm -fv encoder-epoch-99-avg-1-chunk-16-left-128.onnx - rm -fv decoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx - rm -fv joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx - - rm -fv README.md - rm -fv bpe.model - rm -rfv test_wavs - - ls -lh - - popd - """, - ), - Model( - model_name="icefall-asr-zipformer-streaming-wenetspeech-20230615", - idx=2, - lang="zh", - short_name="zipformer2_wenetspeech_2023_06_15", - rule_fsts="itn_zh_number.fst", - cmd=""" - if [ ! -f itn_zh_number.fst ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/itn_zh_number.fst - fi - pushd $model_name - rm -fv exp/encoder-epoch-12-avg-4-chunk-16-left-128.onnx - rm -fv exp/decoder-epoch-12-avg-4-chunk-16-left-128.int8.onnx - rm -fv exp/joiner-epoch-12-avg-4-chunk-16-left-128.int8.onnx - - rm -fv data/lang_char/lexicon.txt - rm -fv data/lang_char/words.txt - rm -rfv test_wavs - rm -fv README.md - - ls -lh exp/ - ls -lh data/lang_char - - popd - """, - ), - Model( - model_name="sherpa-onnx-streaming-zipformer-fr-2023-04-14", - idx=3, - lang="fr", - short_name="zipformer_2023_04_14", - cmd=""" - pushd $model_name - rm -fv encoder-epoch-29-avg-9-with-averaged-model.onnx - rm -fv decoder-epoch-29-avg-9-with-averaged-model.int8.onnx - rm -fv joiner-epoch-29-avg-9-with-averaged-model.int8.onnx - - rm -fv *.sh - rm -rfv test_wavs - rm README.md - - ls -lh - - popd - """, - ), - ] - - return models - - -def main(): - args = get_args() - - index = args.index - total = args.total - assert 0 <= index < total, (index, total) - - all_model_list = get_models() - - num_models = len(all_model_list) - - num_per_runner = num_models // total - if num_per_runner <= 0: - raise ValueError(f"num_models: {num_models}, num_runners: {total}") - - start = index * num_per_runner - end = start + num_per_runner - - remaining = num_models - args.total * num_per_runner - - print(f"{index}/{total}: {start}-{end}/{num_models}") - - d = dict() - d["model_list"] = all_model_list[start:end] - if index < remaining: - s = args.total * num_per_runner + index - d["model_list"].append(all_model_list[s]) - print(f"{s}/{num_models}") - - filename_list = [ - "./build-macos-streaming-asr.sh", - "./build-linux-streaming-asr.sh", - "./build-windows-streaming-asr.sh", - "./build-android-streaming-asr.sh", - ] - for filename in filename_list: - environment = jinja2.Environment() - with open(f"{filename}.in") as f: - s = f.read() - template = environment.from_string(s) - - s = template.render(**d) - with open(filename, "w") as f: - print(s, file=f) - - -if __name__ == "__main__": - main() diff --git a/scripts/flutter/generate-tts.py b/scripts/flutter/generate-tts.py deleted file mode 100755 index 7c57cef62..000000000 --- a/scripts/flutter/generate-tts.py +++ /dev/null @@ -1,457 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -from dataclasses import dataclass -from typing import List, Optional - -import jinja2 - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--total", - type=int, - default=1, - help="Number of runners", - ) - parser.add_argument( - "--index", - type=int, - default=0, - help="Index of the current runner", - ) - return parser.parse_args() - - -@dataclass -class TtsModel: - model_dir: str - model_name: str = "" - lang: str = "" # en, zh, fr, de, etc. - rule_fsts: Optional[List[str]] = None - rule_fars: Optional[List[str]] = None - data_dir: Optional[str] = None - dict_dir: Optional[str] = None - is_char: bool = False - is_kitten: bool = False - - -def get_coqui_models() -> List[TtsModel]: - # English (coqui-ai/TTS) - models = [ - TtsModel(model_dir="vits-coqui-en-ljspeech"), - TtsModel(model_dir="vits-coqui-en-ljspeech-neon"), - TtsModel(model_dir="vits-coqui-en-vctk"), - # TtsModel(model_dir="vits-coqui-en-jenny"), - ] - - for m in models: - m.data_dir = m.model_dir + "/" + "espeak-ng-data" - m.model_name = "model.onnx" - m.lang = "en" - - character_models = [ - TtsModel(model_dir="vits-coqui-bg-cv", lang="bg"), - TtsModel(model_dir="vits-coqui-bn-custom_female", lang="bn"), - TtsModel(model_dir="vits-coqui-cs-cv", lang="cs"), - TtsModel(model_dir="vits-coqui-da-cv", lang="da"), - TtsModel(model_dir="vits-coqui-de-css10", lang="de"), - TtsModel(model_dir="vits-coqui-es-css10", lang="es"), - TtsModel(model_dir="vits-coqui-et-cv", lang="et"), - TtsModel(model_dir="vits-coqui-fi-css10", lang="fi"), - TtsModel(model_dir="vits-coqui-fr-css10", lang="fr"), - TtsModel(model_dir="vits-coqui-ga-cv", lang="ga"), - TtsModel(model_dir="vits-coqui-hr-cv", lang="hr"), - TtsModel(model_dir="vits-coqui-lt-cv", lang="lt"), - TtsModel(model_dir="vits-coqui-lv-cv", lang="lv"), - TtsModel(model_dir="vits-coqui-mt-cv", lang="mt"), - TtsModel(model_dir="vits-coqui-nl-css10", lang="nl"), - TtsModel(model_dir="vits-coqui-pl-mai_female", lang="pl"), - TtsModel(model_dir="vits-coqui-pt-cv", lang="pt"), - TtsModel(model_dir="vits-coqui-ro-cv", lang="ro"), - TtsModel(model_dir="vits-coqui-sk-cv", lang="sk"), - TtsModel(model_dir="vits-coqui-sl-cv", lang="sl"), - TtsModel(model_dir="vits-coqui-sv-cv", lang="sv"), - TtsModel(model_dir="vits-coqui-uk-mai", lang="uk"), - ] - for m in character_models: - m.is_char = True - m.model_name = "model.onnx" - - return models + character_models - - -def get_piper_models() -> List[TtsModel]: - models = [ - # TtsModel(model_dir="vits-piper-es_ES-mls_10246-low"), - # TtsModel(model_dir="vits-piper-es_ES-mls_9972-low"), - # TtsModel(model_dir="vits-piper-pl_PL-mls_6892-low"), - TtsModel(model_dir="vits-piper-ar_JO-kareem-low"), - TtsModel(model_dir="vits-piper-ar_JO-kareem-medium"), - TtsModel(model_dir="vits-piper-ca_ES-upc_ona-medium"), - TtsModel(model_dir="vits-piper-ca_ES-upc_ona-x_low"), - TtsModel(model_dir="vits-piper-ca_ES-upc_pau-x_low"), - TtsModel(model_dir="vits-piper-ca_ES-upc_pau-x_low"), - TtsModel(model_dir="vits-piper-cs_CZ-jirka-medium"), - TtsModel(model_dir="vits-piper-cy_GB-gwryw_gogleddol-medium"), - TtsModel(model_dir="vits-piper-da_DK-talesyntese-medium"), - TtsModel(model_dir="vits-piper-de_DE-eva_k-x_low"), - TtsModel(model_dir="vits-piper-de_DE-karlsson-low"), - TtsModel(model_dir="vits-piper-de_DE-kerstin-low"), - # TtsModel(model_dir="vits-piper-de_DE-mls-medium"), - TtsModel(model_dir="vits-piper-de_DE-pavoque-low"), - TtsModel(model_dir="vits-piper-de_DE-ramona-low"), - TtsModel(model_dir="vits-piper-de_DE-thorsten-high"), - TtsModel(model_dir="vits-piper-de_DE-thorsten-low"), - TtsModel(model_dir="vits-piper-de_DE-thorsten-medium"), - TtsModel(model_dir="vits-piper-de_DE-thorsten_emotional-medium"), - TtsModel(model_dir="vits-piper-el_GR-rapunzelina-low"), - TtsModel(model_dir="vits-piper-en_GB-alan-low"), - TtsModel(model_dir="vits-piper-en_GB-alan-medium"), - TtsModel(model_dir="vits-piper-en_GB-alba-medium"), - TtsModel(model_dir="vits-piper-en_GB-aru-medium"), - TtsModel(model_dir="vits-piper-en_GB-cori-high"), - TtsModel(model_dir="vits-piper-en_GB-cori-medium"), - TtsModel(model_dir="vits-piper-en_GB-jenny_dioco-medium"), - TtsModel(model_dir="vits-piper-en_GB-northern_english_male-medium"), - TtsModel(model_dir="vits-piper-en_GB-semaine-medium"), - TtsModel(model_dir="vits-piper-en_GB-southern_english_female-low"), - TtsModel(model_dir="vits-piper-en_GB-southern_english_female-medium"), - TtsModel(model_dir="vits-piper-en_GB-southern_english_male-medium"), - TtsModel(model_dir="vits-piper-en_GB-sweetbbak-amy"), - TtsModel(model_dir="vits-piper-en_GB-vctk-medium"), - TtsModel(model_dir="vits-piper-en_US-amy-low"), - TtsModel(model_dir="vits-piper-en_US-amy-medium"), - TtsModel(model_dir="vits-piper-en_US-arctic-medium"), - TtsModel(model_dir="vits-piper-en_US-bryce-medium"), - TtsModel(model_dir="vits-piper-en_US-danny-low"), - TtsModel(model_dir="vits-piper-en_US-glados"), - TtsModel(model_dir="vits-piper-en_US-hfc_female-medium"), - TtsModel(model_dir="vits-piper-en_US-hfc_male-medium"), - TtsModel(model_dir="vits-piper-en_US-joe-medium"), - TtsModel(model_dir="vits-piper-en_US-john-medium"), - TtsModel(model_dir="vits-piper-en_US-kathleen-low"), - TtsModel(model_dir="vits-piper-en_US-kristin-medium"), - TtsModel(model_dir="vits-piper-en_US-kusal-medium"), - TtsModel(model_dir="vits-piper-en_US-l2arctic-medium"), - TtsModel(model_dir="vits-piper-en_US-lessac-high"), - TtsModel(model_dir="vits-piper-en_US-lessac-low"), - TtsModel(model_dir="vits-piper-en_US-lessac-medium"), - TtsModel(model_dir="vits-piper-en_US-libritts-high"), - TtsModel(model_dir="vits-piper-en_US-libritts_r-medium"), - TtsModel(model_dir="vits-piper-en_US-ljspeech-high"), - TtsModel(model_dir="vits-piper-en_US-ljspeech-medium"), - TtsModel(model_dir="vits-piper-en_US-norman-medium"), - TtsModel(model_dir="vits-piper-en_US-ryan-high"), - TtsModel(model_dir="vits-piper-en_US-ryan-low"), - TtsModel(model_dir="vits-piper-en_US-ryan-medium"), - TtsModel(model_dir="vits-piper-es-glados-medium"), - TtsModel(model_dir="vits-piper-es_ES-carlfm-x_low"), - TtsModel(model_dir="vits-piper-es_ES-davefx-medium"), - TtsModel(model_dir="vits-piper-es_ES-sharvard-medium"), - TtsModel(model_dir="vits-piper-es_MX-ald-medium"), - TtsModel(model_dir="vits-piper-es_MX-claude-high"), - TtsModel(model_dir="vits-piper-fa_IR-amir-medium"), - TtsModel(model_dir="vits-piper-fa_IR-gyro-medium"), - TtsModel(model_dir="vits-piper-fi_FI-harri-low"), - TtsModel(model_dir="vits-piper-fi_FI-harri-medium"), - # TtsModel(model_dir="vits-piper-fr_FR-mls-medium"), - TtsModel(model_dir="vits-piper-fr_FR-siwis-low"), - TtsModel(model_dir="vits-piper-fr_FR-siwis-medium"), - TtsModel(model_dir="vits-piper-fr_FR-tom-medium"), - TtsModel(model_dir="vits-piper-fr_FR-upmc-medium"), - TtsModel(model_dir="vits-piper-hu_HU-anna-medium"), - TtsModel(model_dir="vits-piper-hu_HU-berta-medium"), - TtsModel(model_dir="vits-piper-hu_HU-imre-medium"), - TtsModel(model_dir="vits-piper-is_IS-bui-medium"), - TtsModel(model_dir="vits-piper-is_IS-salka-medium"), - TtsModel(model_dir="vits-piper-is_IS-steinn-medium"), - TtsModel(model_dir="vits-piper-is_IS-ugla-medium"), - TtsModel(model_dir="vits-piper-it_IT-paola-medium"), - TtsModel(model_dir="vits-piper-it_IT-riccardo-x_low"), - TtsModel(model_dir="vits-piper-ka_GE-natia-medium"), - TtsModel(model_dir="vits-piper-kk_KZ-iseke-x_low"), - TtsModel(model_dir="vits-piper-kk_KZ-issai-high"), - TtsModel(model_dir="vits-piper-kk_KZ-raya-x_low"), - TtsModel(model_dir="vits-piper-lb_LU-marylux-medium"), - TtsModel(model_dir="vits-piper-ne_NP-google-medium"), - TtsModel(model_dir="vits-piper-ne_NP-google-x_low"), - TtsModel(model_dir="vits-piper-nl_BE-nathalie-medium"), - TtsModel(model_dir="vits-piper-nl_BE-nathalie-x_low"), - TtsModel(model_dir="vits-piper-nl_BE-rdh-medium"), - TtsModel(model_dir="vits-piper-nl_BE-rdh-x_low"), - # TtsModel(model_dir="vits-piper-nl_NL-mls-medium"), - # TtsModel(model_dir="vits-piper-nl_NL-mls_5809-low"), - # TtsModel(model_dir="vits-piper-nl_NL-mls_7432-low"), - TtsModel(model_dir="vits-piper-no_NO-talesyntese-medium"), - TtsModel(model_dir="vits-piper-pl_PL-darkman-medium"), - TtsModel(model_dir="vits-piper-pl_PL-gosia-medium"), - TtsModel(model_dir="vits-piper-pl_PL-mc_speech-medium"), - TtsModel(model_dir="vits-piper-pt_BR-edresson-low"), - TtsModel(model_dir="vits-piper-pt_BR-faber-medium"), - TtsModel(model_dir="vits-piper-pt_PT-tugao-medium"), - TtsModel(model_dir="vits-piper-ro_RO-mihai-medium"), - TtsModel(model_dir="vits-piper-ru_RU-denis-medium"), - TtsModel(model_dir="vits-piper-ru_RU-dmitri-medium"), - TtsModel(model_dir="vits-piper-ru_RU-irina-medium"), - TtsModel(model_dir="vits-piper-ru_RU-ruslan-medium"), - TtsModel(model_dir="vits-piper-sk_SK-lili-medium"), - TtsModel(model_dir="vits-piper-sl_SI-artur-medium"), - TtsModel(model_dir="vits-piper-sr_RS-serbski_institut-medium"), - TtsModel(model_dir="vits-piper-sv_SE-nst-medium"), - TtsModel(model_dir="vits-piper-sw_CD-lanfrica-medium"), - TtsModel(model_dir="vits-piper-tr_TR-dfki-medium"), - TtsModel(model_dir="vits-piper-tr_TR-fahrettin-medium"), - TtsModel(model_dir="vits-piper-tr_TR-fettah-medium"), - TtsModel(model_dir="vits-piper-uk_UA-lada-x_low"), - TtsModel(model_dir="vits-piper-uk_UA-ukrainian_tts-medium"), - TtsModel(model_dir="vits-piper-vi_VN-25hours_single-low"), - TtsModel(model_dir="vits-piper-vi_VN-vais1000-medium"), - TtsModel(model_dir="vits-piper-vi_VN-vivos-x_low"), - TtsModel(model_dir="vits-piper-zh_CN-huayan-medium"), - ] - - for m in models: - m.data_dir = m.model_dir + "/" + "espeak-ng-data" - m.model_name = m.model_dir[len("vits-piper-") :] + ".onnx" - m.lang = m.model_dir.split("-")[2][:2] - - return models - - -def get_mimic3_models() -> List[TtsModel]: - models = [ - TtsModel(model_dir="vits-mimic3-af_ZA-google-nwu_low"), - TtsModel(model_dir="vits-mimic3-bn-multi_low"), - TtsModel(model_dir="vits-mimic3-es_ES-m-ailabs_low"), - TtsModel(model_dir="vits-mimic3-fa-haaniye_low"), - TtsModel(model_dir="vits-mimic3-fi_FI-harri-tapani-ylilammi_low"), - TtsModel(model_dir="vits-mimic3-gu_IN-cmu-indic_low"), - TtsModel(model_dir="vits-mimic3-hu_HU-diana-majlinger_low"), - TtsModel(model_dir="vits-mimic3-ko_KO-kss_low"), - TtsModel(model_dir="vits-mimic3-ne_NP-ne-google_low"), - TtsModel(model_dir="vits-mimic3-pl_PL-m-ailabs_low"), - TtsModel(model_dir="vits-mimic3-tn_ZA-google-nwu_low"), - TtsModel(model_dir="vits-mimic3-vi_VN-vais1000_low"), - ] - for m in models: - m.data_dir = m.model_dir + "/" + "espeak-ng-data" - m.model_name = m.model_dir[len("vits-mimic3-") :] + ".onnx" - m.lang = m.model_dir.split("-")[2][:2] - - return models - - -def get_vits_models() -> List[TtsModel]: - chinese_models = [ - # Chinese - TtsModel( - model_dir="vits-icefall-zh-aishell3", - model_name="model.onnx", - lang="zh", - rule_fsts="vits-icefall-zh-aishell3/phone.fst,vits-icefall-zh-aishell3/date.fst,vits-icefall-zh-aishell3/number.fst,vits-icefall-zh-aishell3/new_heteronym.fst", - rule_fars="vits-icefall-zh-aishell3/rule.far", - ), - TtsModel( - model_dir="vits-zh-aishell3", - model_name="vits-aishell3.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-doom", - model_name="doom.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-echo", - model_name="echo.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-zenyatta", - model_name="zenyatta.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-abyssinvoker", - model_name="abyssinvoker.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-keqing", - model_name="keqing.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-eula", - model_name="eula.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-bronya", - model_name="bronya.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-theresa", - model_name="theresa.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-fanchen-wnj", - model_name="vits-zh-hf-fanchen-wnj.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-melo-tts-zh_en", - model_name="model.onnx", - lang="zh_en", - ), - TtsModel( - model_dir="vits-zh-hf-fanchen-C", - model_name="vits-zh-hf-fanchen-C.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-fanchen-ZhiHuiLaoZhe", - model_name="vits-zh-hf-fanchen-ZhiHuiLaoZhe.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-fanchen-ZhiHuiLaoZhe_new", - model_name="vits-zh-hf-fanchen-ZhiHuiLaoZhe_new.onnx", - lang="zh", - ), - TtsModel( - model_dir="vits-zh-hf-fanchen-unity", - model_name="vits-zh-hf-fanchen-unity.onnx", - lang="zh", - ), - TtsModel( - model_dir="sherpa-onnx-vits-zh-ll", - model_name="model.onnx", - lang="zh", - ), - ] - - rule_fsts = ["phone.fst", "date.fst", "number.fst"] - for m in chinese_models: - s = [f"{m.model_dir}/{r}" for r in rule_fsts] - if ( - "vits-zh-hf" in m.model_dir - or "sherpa-onnx-vits-zh-ll" == m.model_dir - or "melo-tts" in m.model_dir - ): - m.dict_dir = m.model_dir + "/dict" - else: - m.rule_fars = f"{m.model_dir}/rule.far" - - m.rule_fsts = ",".join(s) - - all_models = chinese_models + [ - TtsModel( - model_dir="vits-cantonese-hf-xiaomaiiwn", - model_name="vits-cantonese-hf-xiaomaiiwn.onnx", - lang="cantonese", - rule_fsts="vits-cantonese-hf-xiaomaiiwn/rule.fst", - ), - # English (US) - TtsModel(model_dir="vits-vctk", model_name="vits-vctk.onnx", lang="en"), - # TtsModel(model_dir="vits-ljs", model_name="vits-ljs.onnx", lang="en"), - # fmt: on - ] - - return all_models - - -def get_kitten_models() -> List[TtsModel]: - models = [ - TtsModel( - model_dir="kitten-nano-en-v0_8-fp32", - model_name="model.fp32.onnx", - lang="en", - ), - TtsModel( - model_dir="kitten-nano-en-v0_8-int8", - model_name="model.int8.onnx", - lang="en", - ), - TtsModel( - model_dir="kitten-micro-en-v0_8", - model_name="model.onnx", - lang="en", - ), - TtsModel( - model_dir="kitten-mini-en-v0_8", - model_name="model.onnx", - lang="en", - ), - ] - - for m in models: - m.data_dir = f"{m.model_dir}/espeak-ng-data" - m.is_kitten = True - - return models - - -def main(): - args = get_args() - index = args.index - total = args.total - assert 0 <= index < total, (index, total) - d = dict() - - all_model_list = get_vits_models() - all_model_list += get_piper_models() - all_model_list += get_mimic3_models() - all_model_list += get_coqui_models() - all_model_list += get_kitten_models() - - num_models = len(all_model_list) - - num_per_runner = num_models // total - if num_per_runner <= 0: - raise ValueError(f"num_models: {num_models}, num_runners: {total}") - - start = index * num_per_runner - end = start + num_per_runner - - remaining = num_models - args.total * num_per_runner - - print( - "{index}/{total}: {start}-{end}/{num_models}".format( - index=index, - total=total, - start=start, - end=end, - num_models=num_models, - ) - ) - - d["tts_model_list"] = all_model_list[start:end] - if index < remaining: - s = args.total * num_per_runner + index - d["tts_model_list"].append(all_model_list[s]) - print("{s}/{num_models}".format(s=s, num_models=num_models)) - - filename_list = [ - "./build-macos-tts.sh", - "./build-linux-tts.sh", - "./build-android-tts.sh", - "./build-windows-tts.sh", - ] - for filename in filename_list: - environment = jinja2.Environment() - with open("{filename}.in".format(filename=filename)) as f: - s = f.read() - template = environment.from_string(s) - - s = template.render(**d) - with open(filename, "w") as f: - print(s, file=f) - - -if __name__ == "__main__": - main() diff --git a/scripts/go/_internal/non-streaming-tts-phonemize/go.mod b/scripts/go/_internal/non-streaming-tts-phonemize/go.mod new file mode 100644 index 000000000..3038d1d2f --- /dev/null +++ b/scripts/go/_internal/non-streaming-tts-phonemize/go.mod @@ -0,0 +1,5 @@ +module non-streaming-tts-phonemize + +go 1.17 + +replace github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx => ../ diff --git a/scripts/go/_internal/non-streaming-tts-phonemize/main.go b/scripts/go/_internal/non-streaming-tts-phonemize/main.go new file mode 120000 index 000000000..9bd62c9a6 --- /dev/null +++ b/scripts/go/_internal/non-streaming-tts-phonemize/main.go @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-tts-phonemize/main.go \ No newline at end of file diff --git a/scripts/go/_internal/non-streaming-tts-phonemize/run-kitten-en-phonemize.sh b/scripts/go/_internal/non-streaming-tts-phonemize/run-kitten-en-phonemize.sh new file mode 120000 index 000000000..d053f8200 --- /dev/null +++ b/scripts/go/_internal/non-streaming-tts-phonemize/run-kitten-en-phonemize.sh @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-tts-phonemize/run-kitten-en-phonemize.sh \ No newline at end of file diff --git a/scripts/go/_internal/non-streaming-tts-phonemize/run-kokoro-en-phonemize.sh b/scripts/go/_internal/non-streaming-tts-phonemize/run-kokoro-en-phonemize.sh new file mode 120000 index 000000000..a17a9956e --- /dev/null +++ b/scripts/go/_internal/non-streaming-tts-phonemize/run-kokoro-en-phonemize.sh @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-tts-phonemize/run-kokoro-en-phonemize.sh \ No newline at end of file diff --git a/scripts/go/_internal/non-streaming-tts-phonemize/run-matcha-en-phonemize.sh b/scripts/go/_internal/non-streaming-tts-phonemize/run-matcha-en-phonemize.sh new file mode 120000 index 000000000..fbebade4a --- /dev/null +++ b/scripts/go/_internal/non-streaming-tts-phonemize/run-matcha-en-phonemize.sh @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-tts-phonemize/run-matcha-en-phonemize.sh \ No newline at end of file diff --git a/scripts/go/_internal/non-streaming-tts-phonemize/run-vits-piper-phonemize.sh b/scripts/go/_internal/non-streaming-tts-phonemize/run-vits-piper-phonemize.sh new file mode 120000 index 000000000..0709e9e72 --- /dev/null +++ b/scripts/go/_internal/non-streaming-tts-phonemize/run-vits-piper-phonemize.sh @@ -0,0 +1 @@ +../../../../go-api-examples/non-streaming-tts-phonemize/run-vits-piper-phonemize.sh \ No newline at end of file diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index e9bd14567..ed73a85e8 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -124,6 +124,11 @@ type OnlineCtcFstDecoderConfig struct { MaxActive int } +type OfflineCtcFstDecoderConfig struct { + Graph string + MaxActive int +} + type HomophoneReplacerConfig struct { DictDir string // unused Lexicon string @@ -501,10 +506,10 @@ type OfflineCanaryModelConfig struct { } type OfflineCohereTranscribeModelConfig struct { - Encoder string - Decoder string - Language string - UsePunct int + Encoder string + Decoder string + Language string + UsePunct int UseInverseTextNormalization int } @@ -572,25 +577,25 @@ type OfflineLMConfig struct { } type OfflineModelConfig struct { - Transducer OfflineTransducerModelConfig - Paraformer OfflineParaformerModelConfig - NemoCTC OfflineNemoEncDecCtcModelConfig - Whisper OfflineWhisperModelConfig - Tdnn OfflineTdnnModelConfig - SenseVoice OfflineSenseVoiceModelConfig - Moonshine OfflineMoonshineModelConfig - FireRedAsr OfflineFireRedAsrModelConfig - FunAsrNano OfflineFunASRNanoModelConfig - Dolphin OfflineDolphinModelConfig - ZipformerCtc OfflineZipformerCtcModelConfig - Canary OfflineCanaryModelConfig - WenetCtc OfflineWenetCtcModelConfig - Omnilingual OfflineOmnilingualAsrCtcModelConfig - MedAsr OfflineMedAsrCtcModelConfig - FireRedAsrCtc OfflineFireRedAsrCtcModelConfig - Qwen3ASR OfflineQwen3ASRModelConfig + Transducer OfflineTransducerModelConfig + Paraformer OfflineParaformerModelConfig + NemoCTC OfflineNemoEncDecCtcModelConfig + Whisper OfflineWhisperModelConfig + Tdnn OfflineTdnnModelConfig + SenseVoice OfflineSenseVoiceModelConfig + Moonshine OfflineMoonshineModelConfig + FireRedAsr OfflineFireRedAsrModelConfig + FunAsrNano OfflineFunASRNanoModelConfig + Dolphin OfflineDolphinModelConfig + ZipformerCtc OfflineZipformerCtcModelConfig + Canary OfflineCanaryModelConfig + WenetCtc OfflineWenetCtcModelConfig + Omnilingual OfflineOmnilingualAsrCtcModelConfig + MedAsr OfflineMedAsrCtcModelConfig + FireRedAsrCtc OfflineFireRedAsrCtcModelConfig + Qwen3ASR OfflineQwen3ASRModelConfig CohereTranscribe OfflineCohereTranscribeModelConfig - Tokens string // Path to tokens.txt + Tokens string // Path to tokens.txt // Number of threads to use for neural network computation NumThreads int @@ -619,13 +624,14 @@ type OfflineRecognizerConfig struct { DecodingMethod string // Used only when DecodingMethod is modified_beam_search. - MaxActivePaths int - HotwordsFile string - HotwordsScore float32 - BlankPenalty float32 - RuleFsts string - RuleFars string - Hr HomophoneReplacerConfig + MaxActivePaths int + HotwordsFile string + HotwordsScore float32 + BlankPenalty float32 + RuleFsts string + RuleFars string + Hr HomophoneReplacerConfig + CtcFstDecoderConfig OfflineCtcFstDecoderConfig } // It wraps a pointer from C @@ -765,6 +771,9 @@ func newCOfflineRecognizerConfig(config *OfflineRecognizerConfig) *C.struct_Sher c.hr.lexicon = C.CString(config.Hr.Lexicon) c.hr.rule_fsts = C.CString(config.Hr.RuleFsts) + + c.ctc_fst_decoder_config.graph = C.CString(config.CtcFstDecoderConfig.Graph) + c.ctc_fst_decoder_config.max_active = C.int(config.CtcFstDecoderConfig.MaxActive) return &c } func freeCOfflineRecognizerConfig(c *C.struct_SherpaOnnxOfflineRecognizerConfig) { @@ -827,6 +836,7 @@ func freeCOfflineRecognizerConfig(c *C.struct_SherpaOnnxOfflineRecognizerConfig) &c.rule_fars, &c.hr.lexicon, &c.hr.rule_fsts, + &c.ctc_fst_decoder_config.graph, } for _, field := range stringFields { @@ -985,7 +995,7 @@ type OfflineTtsVitsModelConfig struct { Model string // Path to the VITS onnx model Lexicon string // Path to lexicon.txt Tokens string // Path to tokens.txt - DataDir string // Path to espeak-ng-data directory + DataDir string // Deprecated. Ignored. Use Lexicon or phoneme_codepoints instead. NoiseScale float32 // noise scale for vits models. Please use 0.667 in general NoiseScaleW float32 // noise scale for vits models. Please use 0.8 in general LengthScale float32 // Please use 1.0 in general. Smaller -> Faster speech speed. Larger -> Slower speech speed @@ -997,8 +1007,8 @@ type OfflineTtsMatchaModelConfig struct { Vocoder string // Path to the vocoder model for MatchaTTS Lexicon string // Path to lexicon.txt Tokens string // Path to tokens.txt - DataDir string // Path to espeak-ng-data directory - NoiseScale float32 // noise scale for vits models. Please use 0.667 in general + DataDir string // Deprecated. Ignored. Use Lexicon or PhonemeCodepoints instead. + NoiseScale float32 // noise scale for matcha models. Please use 1.0 in general LengthScale float32 // Please use 1.0 in general. Smaller -> Faster speech speed. Larger -> Slower speech speed DictDir string // unused } @@ -1007,7 +1017,7 @@ type OfflineTtsKokoroModelConfig struct { Model string // Path to the model for kokoro Voices string // Path to the voices.bin for kokoro Tokens string // Path to tokens.txt - DataDir string // Path to espeak-ng-data directory + DataDir string // Deprecated. Ignored. Use Lexicon or phoneme_codepoints instead. DictDir string // unused Lexicon string // Path to lexicon files Lang string // Example: es for Spanish, fr-fr for French. Can be empty @@ -1018,8 +1028,9 @@ type OfflineTtsKittenModelConfig struct { Model string // Path to the model for kitten Voices string // Path to the voices.bin for kitten Tokens string // Path to tokens.txt - DataDir string // Path to espeak-ng-data directory + DataDir string // Deprecated. Ignored. Use Lexicon or PhonemeCodepoints instead. LengthScale float32 // Please use 1.0 in general. Smaller -> Faster speech speed. Larger -> Slower speech speed + Lexicon string // Path to lexicon.txt } type OfflineTtsPocketModelConfig struct { @@ -1037,7 +1048,7 @@ type OfflineTtsZipvoiceModelConfig struct { Tokens string // Path to tokens.txt for ZipVoice Encoder string // Path to text encoder (e.g. encoder.onnx) Decoder string // Path to flow-matching decoder (e.g. fm_decoder.onnx) - DataDir string // Path to espeak-ng-data + DataDir string // Deprecated. Ignored. Use Lexicon or Tokens instead. (ZipVoice does not support PhonemeCodepoints.) Lexicon string // Path to lexicon.txt (needed for zh) Vocoder string // Path to vocoder (e.g. vocos_24khz.onnx) @@ -1104,6 +1115,22 @@ type GenerationConfig struct { // Opaque JSON passed directly to C Extra json.RawMessage + + // Pre-phonemized input as Unicode codepoints. Each inner slice is one + // sentence. When set, it is used in place of the text argument; leave + // empty to use text with a lexicon. + // + // Phonemize the text yourself, e.g., with + // https://github.com/csukuangfj/piper-phonemize-go + // + // Mutually exclusive with Tokens. + PhonemeCodepoints [][]int32 + + // Pre-tokenized input, e.g., Chinese pinyin like "zhong1". Each inner + // slice is one sentence. + // + // Mutually exclusive with PhonemeCodepoints. + Tokens [][]string } // The offline tts class. It wraps a pointer from C. @@ -1280,6 +1307,9 @@ func NewOfflineTts(config *OfflineTtsConfig) *OfflineTts { c.model.kitten.length_scale = C.float(config.Model.Kitten.LengthScale) + c.model.kitten.lexicon = C.CString(config.Model.Kitten.Lexicon) + defer C.free(unsafe.Pointer(c.model.kitten.lexicon)) + // zipvoice c.model.zipvoice.tokens = C.CString(config.Model.Zipvoice.Tokens) defer C.free(unsafe.Pointer(c.model.zipvoice.tokens)) @@ -1373,6 +1403,17 @@ func (tts *OfflineTts) SampleRate() int { return int(C.SherpaOnnxOfflineTtsSampleRate(tts.impl)) } +// Lang returns the language/voice of the model, e.g., "en-us". +// It returns an empty string if such information is not available. +func (tts *OfflineTts) Lang() string { + p := C.SherpaOnnxOfflineTtsLang(tts.impl) + if p == nil { + return "" + } + defer C.SherpaOnnxDestroyOfflineTtsLang(p) + return C.GoString(p) +} + func (tts *OfflineTts) Generate(text string, sid int, speed float32) *GeneratedAudio { s := C.CString(text) defer C.free(unsafe.Pointer(s)) @@ -1549,6 +1590,70 @@ func (tts *OfflineTts) GenerateWithConfig( cCfg.extra = cExtra + // Flatten PhonemeCodepoints; the C API takes a flat array plus the + // length of each sentence. The arrays are allocated in C memory to + // follow the cgo pointer rules and are freed after use. + if len(cfg.PhonemeCodepoints) > 0 { + numCodepoints := 0 + for _, sentence := range cfg.PhonemeCodepoints { + numCodepoints += len(sentence) + } + if numCodepoints > 0 { + cFlatCodepoints := (*C.int32_t)(C.malloc(C.size_t(numCodepoints) * C.size_t(unsafe.Sizeof(C.int32_t(0))))) + defer C.free(unsafe.Pointer(cFlatCodepoints)) + cCodepointLens := (*C.int32_t)(C.malloc(C.size_t(len(cfg.PhonemeCodepoints)) * C.size_t(unsafe.Sizeof(C.int32_t(0))))) + defer C.free(unsafe.Pointer(cCodepointLens)) + + flat := unsafe.Slice(cFlatCodepoints, numCodepoints) + lens := unsafe.Slice(cCodepointLens, len(cfg.PhonemeCodepoints)) + k := 0 + for i, sentence := range cfg.PhonemeCodepoints { + for _, cp := range sentence { + flat[k] = C.int32_t(cp) + k++ + } + lens[i] = C.int32_t(len(sentence)) + } + + cCfg.phoneme_codepoints = cFlatCodepoints + cCfg.phoneme_codepoints_lens = cCodepointLens + cCfg.phoneme_codepoints_num_sentences = C.int32_t(len(cfg.PhonemeCodepoints)) + } + } + + // Flatten Tokens; the C API takes a flat array plus the length of each + // sentence. Same C-memory allocation as above; the token strings are + // copied to C memory and freed after use. + if len(cfg.Tokens) > 0 { + numTokens := 0 + for _, sentence := range cfg.Tokens { + numTokens += len(sentence) + } + if numTokens > 0 { + cFlatTokens := (**C.char)(C.malloc(C.size_t(numTokens) * C.size_t(unsafe.Sizeof(uintptr(0))))) + defer C.free(unsafe.Pointer(cFlatTokens)) + cTokenLens := (*C.int32_t)(C.malloc(C.size_t(len(cfg.Tokens)) * C.size_t(unsafe.Sizeof(C.int32_t(0))))) + defer C.free(unsafe.Pointer(cTokenLens)) + + flat := unsafe.Slice(cFlatTokens, numTokens) + lens := unsafe.Slice(cTokenLens, len(cfg.Tokens)) + k := 0 + for i, sentence := range cfg.Tokens { + for _, tok := range sentence { + s := C.CString(tok) + defer C.free(unsafe.Pointer(s)) + flat[k] = s + k++ + } + lens[i] = C.int32_t(len(sentence)) + } + + cCfg.tokens = cFlatTokens + cCfg.tokens_lens = cTokenLens + cCfg.tokens_num_sentences = C.int32_t(len(cfg.Tokens)) + } + } + var audio *C.struct_SherpaOnnxGeneratedAudio if cb != nil { h := cgo.NewHandle(cb) diff --git a/scripts/node-addon-api/lib/non-streaming-tts.js b/scripts/node-addon-api/lib/non-streaming-tts.js index ce17eb9c3..2332499e2 100644 --- a/scripts/node-addon-api/lib/non-streaming-tts.js +++ b/scripts/node-addon-api/lib/non-streaming-tts.js @@ -44,6 +44,7 @@ class OfflineTts { // Common initialization this.numSpeakers = addon.getOfflineTtsNumSpeakers(this.handle); this.sampleRate = addon.getOfflineTtsSampleRate(this.handle); + this.lang = addon.getOfflineTtsLang(this.handle); } /** diff --git a/scripts/node-addon-api/lib/types.js b/scripts/node-addon-api/lib/types.js index 357ac92e1..a407f9a1f 100644 --- a/scripts/node-addon-api/lib/types.js +++ b/scripts/node-addon-api/lib/types.js @@ -396,11 +396,19 @@ * @property {number} [blankPenalty] */ +/** + * CTC FST decoder options for offline CTC models. + * @typedef {Object} OfflineCtcFstDecoderConfig + * @property {string} [graph] - Path to the decoding graph. + * @property {number} [maxActive] - Decoder max-active setting. 0 means 3000. + */ + /** * Offline recognizer config passed to createOfflineRecognizer. * @typedef {Object} OfflineRecognizerConfig * @property {FeatureConfig} [featConfig] * @property {OfflineModelConfig} [modelConfig] + * @property {OfflineCtcFstDecoderConfig} [ctcFstDecoderConfig] */ /** @@ -440,6 +448,14 @@ * @property {string=} referenceText * * @property {{[key: string]: number | string}} [extra] + * + * @property {number[][]} [phonemeCodepoints] - Pre-phonemized input as + * Unicode codepoints, one inner array per sentence. When set, it is + * used in place of the text argument. Mutually exclusive with + * `tokens`. + * @property {string[][]} [tokens] - Pre-tokenized input, e.g. Chinese pinyin + * like "zhong1", one inner array per sentence. Mutually exclusive + * with `phonemeCodepoints`. */ @@ -525,7 +541,8 @@ * @property {string} [model] * @property {string} [lexicon] * @property {string} [tokens] - * @property {string} [dataDir] + * @property {string} [dataDir] - Deprecated. Ignored. Use lexicon, or + * phonemeCodepoints or tokens instead. * @property {number} [noiseScale] * @property {number} [noiseScaleW] * @property {number} [lengthScale] @@ -537,7 +554,8 @@ * @property {string} [vocoder] * @property {string} [lexicon] * @property {string} [tokens] - * @property {string} [dataDir] + * @property {string} [dataDir] - Deprecated. Ignored. Use lexicon, or + * phonemeCodepoints or tokens instead. * @property {number} [noiseScale] * @property {number} [lengthScale] */ @@ -547,7 +565,8 @@ * @property {string} [model] * @property {string} [voices] * @property {string} [tokens] - * @property {string} [dataDir] + * @property {string} [dataDir] - Deprecated. Ignored. Use lexicon, or + * phonemeCodepoints or tokens instead. * @property {number} [lengthScale] * @property {string} [lexicon] * @property {string} [lang] @@ -558,7 +577,9 @@ * @property {string} [model] * @property {string} [voices] * @property {string} [tokens] - * @property {string} [dataDir] + * @property {string} [lexicon] + * @property {string} [dataDir] - Deprecated. Ignored. Use lexicon, or + * phonemeCodepoints or tokens instead. * @property {number} [lengthScale] */ @@ -568,7 +589,8 @@ * @property {string} [encoder] * @property {string} [decoder] * @property {string} [vocoder] - * @property {string} [dataDir] + * @property {string} [dataDir] - Deprecated. Ignored. Use lexicon, or + * phonemeCodepoints or tokens instead. * @property {string} [lexicon] * @property {number} [featScale] * @property {number} [tShift] @@ -616,10 +638,10 @@ */ /** - * Offline Speech Denoiser model config - * @typedef {Object} OfflineSpeechDenoiserDpdfNetModelConfig - * @property {string} [model] - * @property {number} [attenuationLimitDb] Offline attenuation limit in dB. + * Offline Speech Denoiser model config + * @typedef {Object} OfflineSpeechDenoiserDpdfNetModelConfig + * @property {string} [model] + * @property {number} [attenuationLimitDb] Offline attenuation limit in dB. */ /** @@ -633,10 +655,10 @@ */ /** - * Offline Speech Denoiser configuration (partial). - * @typedef {Object} OfflineSpeechDenoiserConfig - * @property {OfflineSpeechDenoiserModelConfig} [model] - */ + * Offline Speech Denoiser configuration (partial). + * @typedef {Object} OfflineSpeechDenoiserConfig + * @property {OfflineSpeechDenoiserModelConfig} [model] + */ /** * Online Speech Denoiser configuration (partial). @@ -645,11 +667,11 @@ */ /** - * Offline speaker segmentation (pyannote) model config - * @typedef {Object} OfflineSpeakerSegmentationPyannoteModelConfig - * @property {string} [model] - * @property {number} [windowShiftRatio=0.1] - */ + * Offline speaker segmentation (pyannote) model config + * @typedef {Object} OfflineSpeakerSegmentationPyannoteModelConfig + * @property {string} [model] + * @property {number} [windowShiftRatio=0.1] + */ /** * Offline speaker segmentation model config diff --git a/scripts/wasm/generate-tts.py b/scripts/wasm/generate-tts.py index ac9d98c64..f6ae6106a 100755 --- a/scripts/wasm/generate-tts.py +++ b/scripts/wasm/generate-tts.py @@ -42,7 +42,14 @@ def get_models(): mv -v *.onnx ../ mv -v tokens.txt ../ - mv -v espeak-ng-data ../ + popd + + pushd .. + rm -rf piper-phonemize + curl -SL -O https://github.com/csukuangfj/piper-phonemize/releases/download/wasm/piper-phonemize-v1.5.8-wasm-browser.zip + unzip -q piper-phonemize-v1.5.8-wasm-browser.zip + mv browser piper-phonemize + rm piper-phonemize-v1.5.8-wasm-browser.zip popd @@ -61,7 +68,14 @@ def get_models(): mv -v *.onnx ../ mv -v tokens.txt ../ - mv -v espeak-ng-data ../ + popd + + pushd .. + rm -rf piper-phonemize + curl -SL -O https://github.com/csukuangfj/piper-phonemize/releases/download/wasm/piper-phonemize-v1.5.8-wasm-browser.zip + unzip -q piper-phonemize-v1.5.8-wasm-browser.zip + mv browser piper-phonemize + rm piper-phonemize-v1.5.8-wasm-browser.zip popd @@ -82,7 +96,6 @@ def get_models(): mv -v *.onnx ../ mv -v tokens.txt ../ mv -v lexicon.txt ../ - mv -v espeak-ng-data ../ popd curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-16khz-univ.onnx @@ -126,7 +139,14 @@ def get_models(): mv -v *.onnx ../ mv -v tokens.txt ../ - mv -v espeak-ng-data ../ + popd + + pushd .. + rm -rf piper-phonemize + curl -SL -O https://github.com/csukuangfj/piper-phonemize/releases/download/wasm/piper-phonemize-v1.5.8-wasm-browser.zip + unzip -q piper-phonemize-v1.5.8-wasm-browser.zip + mv browser piper-phonemize + rm piper-phonemize-v1.5.8-wasm-browser.zip popd curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx @@ -150,7 +170,6 @@ def get_models(): mv -v decoder.int8.onnx ../ mv -v tokens.txt ../ mv -v lexicon.txt ../ - mv -v espeak-ng-data ../ popd curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx @@ -173,13 +192,14 @@ def get_models(): mv -v tokens.txt ../ mv -v lexicon-us-en.txt ../ mv -v lexicon-zh.txt ../ - mv -v espeak-ng-data ../ mv -v *.fst ../ popd git checkout . - sed -i.bak 's/let modelType = 0/let modelType = 7/g' ../sherpa-onnx-tts.js + # modelType 8: kokoro-multi-lang. Do not use 7 here, that is + # Supertonic's id in sherpa-onnx-tts.js. + sed -i.bak 's/let modelType = 0/let modelType = 8/g' ../sherpa-onnx-tts.js rm -rf $model_name git diff @@ -207,6 +227,28 @@ def get_models(): git diff """, ), + Model( + model_name="sherpa-onnx-supertonic-3-tts-int8-2026-05-11", + hf="k2-fsa/web-assembly-multilingual-tts-supertonic", + ms="csukuangfj/web-assembly-multilingual-tts-supertonic", + cmd=""" + pushd $model_name + + mv -v duration_predictor.int8.onnx ../ + mv -v text_encoder.int8.onnx ../ + mv -v vector_estimator.int8.onnx ../ + mv -v vocoder.int8.onnx ../ + mv -v tts.json ../ + mv -v unicode_indexer.bin ../ + mv -v voice.bin ../ + popd + + git checkout . + sed -i.bak 's/let modelType = 0/let modelType = 7/g' ../sherpa-onnx-tts.js + rm -rf $model_name + git diff + """, + ), ] return models diff --git a/scripts/wasm/run-tts.sh.in b/scripts/wasm/run-tts.sh.in index 8df14aaef..af0687098 100644 --- a/scripts/wasm/run-tts.sh.in +++ b/scripts/wasm/run-tts.sh.in @@ -21,6 +21,7 @@ ms_name={{ model.ms }} pushd wasm/tts git checkout . rm -rf assets +rm -rf piper-phonemize mkdir assets cd assets curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/${model_name}.tar.bz2 @@ -83,8 +84,8 @@ ls -lh *.tar.bz2 # git push https://csukuangfj:$HF_TOKEN@huggingface.co/spaces/$hf_name main || true # cd .. # rm -rf huggingface -# rm -rf $dst +rm -rf $dst -ls -lh *.tar.bz2 +# ls -lh *.tar.bz2 {% endfor %} diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 68eae1656..90c635f15 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1562,6 +1562,8 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( SHERPA_ONNX_OR(config->model.kitten.data_dir, ""); tts_config.model.kitten.length_scale = SHERPA_ONNX_OR(config->model.kitten.length_scale, 1.0); + tts_config.model.kitten.lexicon = + SHERPA_ONNX_OR(config->model.kitten.lexicon, ""); // zipvoice tts_config.model.zipvoice.tokens = @@ -1681,6 +1683,19 @@ int32_t SherpaOnnxOfflineTtsNumSpeakers(const SherpaOnnxOfflineTts *tts) { return tts->impl->NumSpeakers(); } +const char *SherpaOnnxOfflineTtsLang(const SherpaOnnxOfflineTts *tts) { + std::string lang = tts->impl->Lang(); + char *p = new char[lang.size() + 1]; + std::copy(lang.begin(), lang.end(), p); + p[lang.size()] = 0; + return p; +} + +void SherpaOnnxDestroyOfflineTtsLang(const char *s) { + if (!s) return; + delete[] s; +} + static const SherpaOnnxGeneratedAudio *SherpaOnnxOfflineTtsGenerateInternal( const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed, std::function callback) { @@ -1707,10 +1722,39 @@ static const SherpaOnnxGeneratedAudio *SherpaOnnxOfflineTtsGenerateInternal( return ans; } +// Sentence lengths must all be > 0: a non-positive n makes p + n a reversed +// range, which is undefined behavior in the vector range constructor below. +static bool ValidateSentenceLens(const int32_t *lens, int32_t num_sentences, + const char *name) { + for (int32_t i = 0; i != num_sentences; ++i) { + if (lens[i] <= 0) { + SHERPA_ONNX_LOGE("%s_lens[%d] = %d. It must be > 0", name, i, lens[i]); + return false; + } + } + return true; +} + static const SherpaOnnxGeneratedAudio *SherpaOnnxOfflineTtsGenerateInternal( const SherpaOnnxOfflineTts *tts, const char *text, const SherpaOnnxGenerationConfig *config, std::function callback) { + // phoneme_codepoints and tokens are mutually exclusive. Validate them + // before doing any work. Neither being set is fine: the caller may rely on + // text plus a lexicon. + bool has_phoneme_codepoints = config->phoneme_codepoints && + config->phoneme_codepoints_lens && + config->phoneme_codepoints_num_sentences > 0; + bool has_tokens = + config->tokens && config->tokens_lens && config->tokens_num_sentences > 0; + + if (has_phoneme_codepoints && has_tokens) { + SHERPA_ONNX_LOGE( + "Both phoneme_codepoints and tokens are given. Please set only one of " + "them."); + return nullptr; + } + sherpa_onnx::GenerationConfig cfg; if (config->reference_audio) { if (config->reference_audio_len <= 0) { @@ -1746,6 +1790,45 @@ static const SherpaOnnxGeneratedAudio *SherpaOnnxOfflineTtsGenerateInternal( } } + // phoneme_codepoints and tokens are flattened; unflatten them here. + if (has_phoneme_codepoints) { + if (!ValidateSentenceLens(config->phoneme_codepoints_lens, + config->phoneme_codepoints_num_sentences, + "phoneme_codepoints")) { + return nullptr; + } + + cfg.phoneme_codepoints.reserve(config->phoneme_codepoints_num_sentences); + const int32_t *p = config->phoneme_codepoints; + for (int32_t i = 0; i != config->phoneme_codepoints_num_sentences; ++i) { + int32_t n = config->phoneme_codepoints_lens[i]; + cfg.phoneme_codepoints.emplace_back(p, p + n); + p += n; + } + } + + if (has_tokens) { + if (!ValidateSentenceLens(config->tokens_lens, + config->tokens_num_sentences, "tokens")) { + return nullptr; + } + + cfg.tokens.reserve(config->tokens_num_sentences); + const char *const *p = config->tokens; + for (int32_t i = 0; i != config->tokens_num_sentences; ++i) { + int32_t n = config->tokens_lens[i]; + std::vector sentence; + sentence.reserve(n); + for (int32_t k = 0; k != n; ++k) { + // A C caller may legitimately pass a NULL entry here; constructing + // std::string from nullptr is undefined behaviour. + sentence.emplace_back(p[k] != nullptr ? p[k] : ""); + } + cfg.tokens.push_back(std::move(sentence)); + p += n; + } + } + sherpa_onnx::GeneratedAudio audio = tts->impl->Generate(text, cfg, callback); if (audio.samples.empty()) { @@ -2000,6 +2083,18 @@ int32_t SherpaOnnxOfflineTtsNumSpeakers(const SherpaOnnxOfflineTts *tts) { return 0; } +const char *SherpaOnnxOfflineTtsLang(const SherpaOnnxOfflineTts *tts) { + SHERPA_ONNX_LOGE("TTS is not enabled. Please rebuild sherpa-onnx"); + char *p = new char[1]; + p[0] = 0; + return p; +} + +void SherpaOnnxDestroyOfflineTtsLang(const char *s) { + if (!s) return; + delete[] s; +} + const SherpaOnnxGeneratedAudio *SherpaOnnxOfflineTtsGenerate( const SherpaOnnxOfflineTts *tts, const char *text, int32_t sid, float speed) { diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 2aa4b5a93..0cc26e47b 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -2268,11 +2268,16 @@ SHERPA_ONNX_API void SherpaOnnxVoiceActivityDetectorFlush( typedef struct SherpaOnnxOfflineTtsVitsModelConfig { /** Path to the VITS ONNX model, for example `./vits-ljs.onnx`. */ const char *model; - /** Path to the lexicon file. Ignored if @c data_dir is provided. */ + /** Path to the lexicon file. */ const char *lexicon; /** Path to the tokens file. */ const char *tokens; - /** Optional path to espeak-ng-data. */ + /** + * Path to espeak-ng-data. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * SherpaOnnxGenerationConfig.phoneme_codepoints (or .tokens) instead. + */ const char *data_dir; /** VITS noise scale. */ float noise_scale; @@ -2294,7 +2299,12 @@ typedef struct SherpaOnnxOfflineTtsMatchaModelConfig { const char *lexicon; /** Path to the tokens file. */ const char *tokens; - /** Optional path to espeak-ng-data. */ + /** + * Path to espeak-ng-data. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * SherpaOnnxGenerationConfig.phoneme_codepoints (or .tokens) instead. + */ const char *data_dir; /** Matcha noise scale. */ float noise_scale; @@ -2312,7 +2322,12 @@ typedef struct SherpaOnnxOfflineTtsKokoroModelConfig { const char *voices; /** Path to the tokens file. */ const char *tokens; - /** Optional path to espeak-ng-data. */ + /** + * Path to espeak-ng-data. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * SherpaOnnxGenerationConfig.phoneme_codepoints (or .tokens) instead. + */ const char *data_dir; /** Speech rate scale. Values < 1 are slower; values > 1 are faster. */ float length_scale; @@ -2332,10 +2347,17 @@ typedef struct SherpaOnnxOfflineTtsKittenModelConfig { const char *voices; /** Path to the tokens file. */ const char *tokens; - /** Optional path to espeak-ng-data. */ + /** + * Path to espeak-ng-data. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * SherpaOnnxGenerationConfig.phoneme_codepoints (or .tokens) instead. + */ const char *data_dir; /** Speech rate scale. Values < 1 are slower; values > 1 are faster. */ float length_scale; + /** Path to the lexicon file. */ + const char *lexicon; } SherpaOnnxOfflineTtsKittenModelConfig; /** @brief Configuration for a ZipVoice TTS model. */ @@ -2348,7 +2370,12 @@ typedef struct SherpaOnnxOfflineTtsZipvoiceModelConfig { const char *decoder; /** Path to the vocoder model. */ const char *vocoder; - /** Optional path to espeak-ng-data. */ + /** + * Path to espeak-ng-data. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * SherpaOnnxGenerationConfig.phoneme_codepoints (or .tokens) instead. + */ const char *data_dir; /** Path to the lexicon file. */ const char *lexicon; @@ -2450,7 +2477,9 @@ typedef struct SherpaOnnxOfflineTtsModelConfig { * config.model.kokoro.model = "./kokoro-en-v0_19/model.onnx"; * config.model.kokoro.voices = "./kokoro-en-v0_19/voices.bin"; * config.model.kokoro.tokens = "./kokoro-en-v0_19/tokens.txt"; - * config.model.kokoro.data_dir = "./kokoro-en-v0_19/espeak-ng-data"; + * // Download it from + * // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + * config.model.kokoro.lexicon = "./lexicon-en-us.txt"; * config.model.num_threads = 2; * config.model.provider = "cpu"; * config.model.debug = 0; @@ -2540,7 +2569,9 @@ typedef struct SherpaOnnxOfflineTts SherpaOnnxOfflineTts; * config.model.kokoro.model = "./kokoro-en-v0_19/model.onnx"; * config.model.kokoro.voices = "./kokoro-en-v0_19/voices.bin"; * config.model.kokoro.tokens = "./kokoro-en-v0_19/tokens.txt"; - * config.model.kokoro.data_dir = "./kokoro-en-v0_19/espeak-ng-data"; + * // Download it from + * // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + * config.model.kokoro.lexicon = "./lexicon-en-us.txt"; * config.model.num_threads = 2; * * const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); @@ -2579,6 +2610,26 @@ SherpaOnnxOfflineTtsSampleRate(const SherpaOnnxOfflineTts *tts); SHERPA_ONNX_API int32_t SherpaOnnxOfflineTtsNumSpeakers(const SherpaOnnxOfflineTts *tts); +/** + * @brief Return the language/voice of the model, e.g., "en-us". + * + * The returned string is empty if such information is not available. It is + * never NULL; free it with SherpaOnnxDestroyOfflineTtsLang(). + * + * @param tts A pointer returned by SherpaOnnxCreateOfflineTts(). + * @return Language/voice of the model. Free it with + * SherpaOnnxDestroyOfflineTtsLang(). + */ +SHERPA_ONNX_API const char * +SherpaOnnxOfflineTtsLang(const SherpaOnnxOfflineTts *tts); + +/** + * @brief Free a string returned by SherpaOnnxOfflineTtsLang(). + * + * @param s A pointer returned by SherpaOnnxOfflineTtsLang(). + */ +SHERPA_ONNX_API void SherpaOnnxDestroyOfflineTtsLang(const char *s); + /** * @brief Generate speech from text using the simple sid/speed interface. * @@ -2754,6 +2805,44 @@ typedef struct SherpaOnnxGenerationConfig { int32_t num_steps; /** Optional model-specific JSON string with extra key/value pairs. */ const char *extra; + + /** + * Pre-phonemized input as Unicode codepoints. + * + * Phonemize the text yourself (for instance with piper_phonemize) and pass + * the resulting codepoints here. When set, it is used in place of @c text. + * Leave as NULL to use @c text with a lexicon. + * + * Each sentence is one sub-array. All codepoints of all sentences are stored + * flattened in this array; see @c phoneme_codepoints_lens for the boundaries. + */ + const int32_t *phoneme_codepoints; + + /** Number of codepoints in each sentence. It has + * @c phoneme_codepoints_num_sentences entries. If not NULL, every entry + * must be > 0. */ + const int32_t *phoneme_codepoints_lens; + + /** Number of sentences in @c phoneme_codepoints_lens. */ + int32_t phoneme_codepoints_num_sentences; + + /** + * Pre-tokenized input, e.g. Chinese pinyin like "zhong1". + * + * Each sentence is one sub-array. All token strings of all sentences are + * stored flattened in this array; see @c tokens_lens for the boundaries. + * + * Mutually exclusive with @c phoneme_codepoints: if both are set, a warning + * is printed and generation fails. + */ + const char *const *tokens; + + /** Number of tokens in each sentence. It has @c tokens_num_sentences + * entries. If not NULL, every entry must be > 0. */ + const int32_t *tokens_lens; + + /** Number of sentences in @c tokens_lens. */ + int32_t tokens_num_sentences; } SherpaOnnxGenerationConfig; /** diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc index 65ef7096e..73e3b3ba2 100644 --- a/sherpa-onnx/c-api/cxx-api.cc +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -522,6 +522,7 @@ OfflineTts OfflineTts::Create(const OfflineTtsConfig &config) { c.model.kitten.tokens = config.model.kitten.tokens.c_str(); c.model.kitten.data_dir = config.model.kitten.data_dir.c_str(); c.model.kitten.length_scale = config.model.kitten.length_scale; + c.model.kitten.lexicon = config.model.kitten.lexicon.c_str(); c.model.zipvoice.tokens = config.model.zipvoice.tokens.c_str(); c.model.zipvoice.encoder = config.model.zipvoice.encoder.c_str(); @@ -589,6 +590,13 @@ int32_t OfflineTts::NumSpeakers() const { return SherpaOnnxOfflineTtsNumSpeakers(p_); } +std::string OfflineTts::Lang() const { + const char *s = SherpaOnnxOfflineTtsLang(p_); + std::string ans = s; + SherpaOnnxDestroyOfflineTtsLang(s); + return ans; +} + GeneratedAudio OfflineTts::Generate(const std::string &text, int32_t sid /*= 0*/, float speed /*= 1.0*/, OfflineTtsCallback callback /*= nullptr*/, @@ -635,6 +643,41 @@ GeneratedAudio OfflineTts::Generate(const std::string &text, std::string s = j.dump(); c.extra = s.c_str(); + // Flatten phoneme_codepoints; the C API takes a flat array + per-sentence + // lengths. + std::vector flat_codepoints; + std::vector codepoint_lens; + codepoint_lens.reserve(config.phoneme_codepoints.size()); + for (const auto &sentence : config.phoneme_codepoints) { + flat_codepoints.insert(flat_codepoints.end(), sentence.begin(), + sentence.end()); + codepoint_lens.push_back(static_cast(sentence.size())); + } + + if (!codepoint_lens.empty()) { + c.phoneme_codepoints = flat_codepoints.data(); + c.phoneme_codepoints_lens = codepoint_lens.data(); + c.phoneme_codepoints_num_sentences = + static_cast(codepoint_lens.size()); + } + + // Flatten tokens; the C API takes a flat array + per-sentence lengths. + std::vector flat_tokens; + std::vector token_lens; + token_lens.reserve(config.tokens.size()); + for (const auto &sentence : config.tokens) { + token_lens.push_back(static_cast(sentence.size())); + for (const auto &tok : sentence) { + flat_tokens.push_back(tok.c_str()); + } + } + + if (!token_lens.empty()) { + c.tokens = flat_tokens.data(); + c.tokens_lens = token_lens.data(); + c.tokens_num_sentences = static_cast(token_lens.size()); + } + const SherpaOnnxGeneratedAudio *audio = SherpaOnnxOfflineTtsGenerateWithConfig(p_, text.c_str(), &c, callback, arg); diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index 09acdab4c..0e1cada13 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -859,7 +859,12 @@ struct OfflineTtsVitsModelConfig { std::string lexicon; /** Token file. */ std::string tokens; - /** Data directory such as `espeak-ng-data`. */ + /** + * Data directory such as `espeak-ng-data`. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * GenerationConfig::phoneme_codepoints (or ::tokens) instead. + */ std::string data_dir; /** Reserved field. Currently unused by the wrapper. */ std::string dict_dir; @@ -882,7 +887,12 @@ struct OfflineTtsMatchaModelConfig { std::string lexicon; /** Token file. */ std::string tokens; - /** Data directory such as `espeak-ng-data`. */ + /** + * Data directory such as `espeak-ng-data`. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * GenerationConfig::phoneme_codepoints (or ::tokens) instead. + */ std::string data_dir; /** Reserved field. Currently unused by the wrapper. */ std::string dict_dir; @@ -901,7 +911,12 @@ struct OfflineTtsKokoroModelConfig { std::string voices; /** Token file. */ std::string tokens; - /** Data directory such as `espeak-ng-data`. */ + /** + * Data directory such as `espeak-ng-data`. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * GenerationConfig::phoneme_codepoints (or ::tokens) instead. + */ std::string data_dir; /** Reserved field. Currently unused by the wrapper. */ std::string dict_dir; @@ -922,11 +937,19 @@ struct OfflineTtsKittenModelConfig { std::string voices; /** Token file. */ std::string tokens; - /** Data directory. */ + /** + * Data directory. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * GenerationConfig::phoneme_codepoints (or ::tokens) instead. + */ std::string data_dir; /** Length scale. Values < 1 are faster; values > 1 are slower. */ float length_scale = 1.0; + + /** Lexicon file. */ + std::string lexicon; }; /** @brief ZipVoice model configuration. */ @@ -939,7 +962,12 @@ struct OfflineTtsZipvoiceModelConfig { std::string decoder; /** Vocoder model file. */ std::string vocoder; - /** Data directory. */ + /** + * Data directory. + * + * @deprecated Ignored since v2.0.0. Pass pre-phonemized input via + * GenerationConfig::phoneme_codepoints (or ::tokens) instead. + */ std::string data_dir; /** Lexicon file. */ std::string lexicon; @@ -1042,6 +1070,27 @@ struct GenerationConfig { /** Model-specific extra attributes serialized to JSON internally. */ std::unordered_map extra; + + /** + * Pre-phonemized input as Unicode codepoints. + * + * Phonemize the text yourself (for instance with piper_phonemize) and pass + * the resulting codepoints here. When set, it is used in place of the `text` + * argument. Leave empty to use `text` with a lexicon. + * + * Each inner vector is one sentence. + */ + std::vector> phoneme_codepoints; + + /** + * Pre-tokenized input, e.g. Chinese pinyin like "zhong1". + * + * Each inner vector is one sentence. + * + * Mutually exclusive with `phoneme_codepoints`: if both are set, a warning + * is printed and generation fails. + */ + std::vector> tokens; }; /** @brief Configuration for offline TTS. */ @@ -1113,6 +1162,11 @@ class SHERPA_ONNX_API OfflineTts /** @brief Return the number of supported speakers. */ int32_t NumSpeakers() const; + /** @brief Return the language/voice of the model, e.g., "en-us". + * + * Returns an empty string if such information is not available. */ + std::string Lang() const; + /** * @brief Generate speech using the simple speaker-id and speed interface. * diff --git a/sherpa-onnx/c-api/docs/tts.dox b/sherpa-onnx/c-api/docs/tts.dox index 9e63465ae..f3a53c1ec 100644 --- a/sherpa-onnx/c-api/docs/tts.dox +++ b/sherpa-onnx/c-api/docs/tts.dox @@ -9,6 +9,22 @@ family by filling in the corresponding sub-struct of @see SherpaOnnxCreateOfflineTts, SherpaOnnxOfflineTtsConfig, SherpaOnnxOfflineTtsModelConfig +@note Most families below need a lexicon file as their text frontend. For the +English models, download the shared lexicon once: + +@code +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +@endcode + +and pass it to the family you use, e.g. +`config.model.vits.lexicon = "./lexicon-en-us.txt";`. Models with their own +lexicon (e.g. Chinese or multi-language packages) ship `lexicon.txt` inside the +model tarball — use that path instead. PocketTTS and Supertonic have their own +built-in text frontend and need no lexicon at all. As an alternative to a +lexicon you can pass pre-phonemized input via +@ref SherpaOnnxGenerationConfig::phoneme_codepoints (or `::tokens`) and leave +the lexicon empty. + @tableofcontents @section tts_kokoro Kokoro @@ -22,7 +38,7 @@ memset(&config, 0, sizeof(config)); config.model.kokoro.model = "./kokoro-en-v0_19/model.onnx"; config.model.kokoro.voices = "./kokoro-en-v0_19/voices.bin"; config.model.kokoro.tokens = "./kokoro-en-v0_19/tokens.txt"; -config.model.kokoro.data_dir = "./kokoro-en-v0_19/espeak-ng-data"; +config.model.kokoro.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 2; const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); @@ -34,7 +50,10 @@ Example source: [kokoro-tts-en-c-api.c](https://github.com/k2-fsa/sherpa-onnx/bl @section tts_vits VITS (Piper) -VITS models from the Piper project use espeak-ng for phonemization. +VITS models from the Piper project. Note: `data_dir` is deprecated and +ignored since v2.0.0 (espeak-ng is no longer used). Pass a `lexicon`, or +pre-phonemized input via `SherpaOnnxGenerationConfig.phoneme_codepoints` +(or `.tokens`). @code SherpaOnnxOfflineTtsConfig config; @@ -43,8 +62,7 @@ config.model.vits.model = "./vits-piper-en_US-lessac-medium/en_US-lessac-medium.onnx"; config.model.vits.tokens = "./vits-piper-en_US-lessac-medium/tokens.txt"; -config.model.vits.data_dir = - "./vits-piper-en_US-lessac-medium/espeak-ng-data"; +config.model.vits.lexicon = "./lexicon-en-us.txt"; config.model.vits.noise_scale = 0.667f; config.model.vits.noise_scale_w = 0.8f; config.model.vits.length_scale = 1.0f; @@ -70,8 +88,7 @@ config.model.matcha.acoustic_model = config.model.matcha.vocoder = "./vocos-22khz-univ.onnx"; config.model.matcha.tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt"; -config.model.matcha.data_dir = - "./matcha-icefall-en_US-ljspeech/espeak-ng-data"; +config.model.matcha.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 2; const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); @@ -92,8 +109,7 @@ memset(&config, 0, sizeof(config)); config.model.kitten.model = "./kitten-nano-en-v0_1-fp16/model.fp16.onnx"; config.model.kitten.voices = "./kitten-nano-en-v0_1-fp16/voices.bin"; config.model.kitten.tokens = "./kitten-nano-en-v0_1-fp16/tokens.txt"; -config.model.kitten.data_dir = - "./kitten-nano-en-v0_1-fp16/espeak-ng-data"; +config.model.kitten.lexicon = "./lexicon-en-us.txt"; config.model.num_threads = 2; const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); @@ -120,8 +136,6 @@ config.model.zipvoice.tokens = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt"; config.model.zipvoice.lexicon = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt"; -config.model.zipvoice.data_dir = - "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data"; config.model.num_threads = 2; const SherpaOnnxOfflineTts *tts = SherpaOnnxCreateOfflineTts(&config); diff --git a/sherpa-onnx/c-api/sherpa-onnx-symbols-c.exp b/sherpa-onnx/c-api/sherpa-onnx-symbols-c.exp index f6d530958..fccef044d 100644 --- a/sherpa-onnx/c-api/sherpa-onnx-symbols-c.exp +++ b/sherpa-onnx/c-api/sherpa-onnx-symbols-c.exp @@ -62,6 +62,7 @@ _SherpaOnnxDestroyOfflineStream _SherpaOnnxDestroyOfflineStreamResultJson _SherpaOnnxDestroyOfflineTts _SherpaOnnxDestroyOfflineTtsGeneratedAudio +_SherpaOnnxDestroyOfflineTtsLang _SherpaOnnxDestroyOnlinePunctuation _SherpaOnnxDestroyOnlineRecognizer _SherpaOnnxDestroyOnlineRecognizerResult @@ -122,6 +123,7 @@ _SherpaOnnxOfflineTtsGenerateWithConfig _SherpaOnnxOfflineTtsGenerateWithProgressCallback _SherpaOnnxOfflineTtsGenerateWithProgressCallbackWithArg _SherpaOnnxOfflineTtsGenerateWithZipvoice +_SherpaOnnxOfflineTtsLang _SherpaOnnxOfflineTtsNumSpeakers _SherpaOnnxOfflineTtsSampleRate _SherpaOnnxOnlinePunctuationAddPunct diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index 2197e7d44..8278e41e4 100755 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -289,9 +289,7 @@ if(SHERPA_ONNX_ENABLE_TTS) list(APPEND sources character-lexicon.cc hifigan-vocoder.cc - kokoro-multi-lang-lexicon.cc lexicon.cc - matcha-tts-lexicon.cc melo-tts-lexicon.cc offline-tts-character-frontend.cc offline-tts-frontend.cc @@ -314,8 +312,8 @@ if(SHERPA_ONNX_ENABLE_TTS) offline-tts-zipvoice-model-config.cc offline-tts-zipvoice-model.cc offline-tts.cc - piper-phonemize-lexicon.cc sentence-piece-tokenizer.cc + tts-text-normalizer.cc vocoder.cc vocos-vocoder.cc ) @@ -477,11 +475,6 @@ endif() target_link_libraries(sherpa-onnx-core fstfar fst) -if(SHERPA_ONNX_ENABLE_TTS) - target_link_libraries(sherpa-onnx-core - piper_phonemize) -endif() - if(SHERPA_ONNX_ENABLE_CHECK) target_compile_definitions(sherpa-onnx-core PUBLIC SHERPA_ONNX_ENABLE_CHECK=1) @@ -854,7 +847,6 @@ if(SHERPA_ONNX_ENABLE_TESTS) if(SHERPA_ONNX_ENABLE_TTS) list(APPEND sherpa_onnx_test_srcs sentence-piece-tokenizer-test.cc - piper-phonemize-test.cc offline-tts-supertonic-unicode-processor-test.cc ) endif() diff --git a/sherpa-onnx/csrc/hifigan-vocoder.cc b/sherpa-onnx/csrc/hifigan-vocoder.cc index a990bddb6..b43a0ee3c 100644 --- a/sherpa-onnx/csrc/hifigan-vocoder.cc +++ b/sherpa-onnx/csrc/hifigan-vocoder.cc @@ -65,6 +65,8 @@ class HifiganVocoder::Impl { return {p, p + total}; } + int32_t SampleRate() const { return sample_rate_; } + private: void Init(void *model_data, size_t model_data_length) { if (model_data) { @@ -80,9 +82,13 @@ class HifiganVocoder::Impl { GetInputNames(sess_.get(), &input_names_, &input_names_ptr_); GetOutputNames(sess_.get(), &output_names_, &output_names_ptr_); + + Ort::ModelMetadata meta_data = sess_->GetModelMetadata(); + Ort::AllocatorWithDefaultOptions allocator; + SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(sample_rate_, "sample_rate", 22050); } - private: + int32_t sample_rate_ = 22050; Ort::Env env_; Ort::SessionOptions sess_opts_; Ort::AllocatorWithDefaultOptions allocator_; @@ -112,6 +118,8 @@ std::vector HifiganVocoder::Run(Ort::Value mel) const { return impl_->Run(std::move(mel)); } +int32_t HifiganVocoder::SampleRate() const { return impl_->SampleRate(); } + #if __ANDROID_API__ >= 9 template HifiganVocoder::HifiganVocoder(AAssetManager *mgr, int32_t num_threads, const std::string &provider, diff --git a/sherpa-onnx/csrc/hifigan-vocoder.h b/sherpa-onnx/csrc/hifigan-vocoder.h index f491c3f5e..b51c7eb94 100644 --- a/sherpa-onnx/csrc/hifigan-vocoder.h +++ b/sherpa-onnx/csrc/hifigan-vocoder.h @@ -30,6 +30,8 @@ class HifiganVocoder : public Vocoder { */ std::vector Run(Ort::Value mel) const override; + int32_t SampleRate() const override; + private: class Impl; std::unique_ptr impl_; diff --git a/sherpa-onnx/csrc/kokoro-multi-lang-lexicon.cc b/sherpa-onnx/csrc/kokoro-multi-lang-lexicon.cc deleted file mode 100644 index 9df90ca59..000000000 --- a/sherpa-onnx/csrc/kokoro-multi-lang-lexicon.cc +++ /dev/null @@ -1,593 +0,0 @@ -// sherpa-onnx/csrc/kokoro-multi-lang-lexicon.cc -// -// Copyright (c) 2025 Xiaomi Corporation - -#include "sherpa-onnx/csrc/kokoro-multi-lang-lexicon.h" -#include "sherpa-onnx/csrc/macros.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - -#if __OHOS__ -#include "rawfile/raw_file_manager.h" -#endif - -#include "espeak-ng/speak_lib.h" -#include "phoneme_ids.hpp" // NOLINT -#include "phonemize.hpp" // NOLINT -#include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/onnx-utils.h" -#include "sherpa-onnx/csrc/phrase-matcher.h" -#include "sherpa-onnx/csrc/symbol-table.h" -#include "sherpa-onnx/csrc/text-utils.h" - -namespace sherpa_onnx { - -void CallPhonemizeEspeak(const std::string &text, - piper::eSpeakPhonemeConfig &config, // NOLINT - std::vector> *phonemes); - -class KokoroMultiLangLexicon::Impl { - public: - Impl(const std::string &tokens, const std::string &lexicon, - const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &meta_data, bool debug) - : meta_data_(meta_data), debug_(debug) { - InitTokens(tokens); - - InitLexicon(lexicon); - - InitEspeak(data_dir); // See ./piper-phonemize-lexicon.cc - } - - template - Impl(Manager *mgr, const std::string &tokens, const std::string &lexicon, - const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &meta_data, bool debug) - : meta_data_(meta_data), debug_(debug) { - InitTokens(mgr, tokens); - - InitLexicon(mgr, lexicon); - - // we assume you have copied data_dir from assets to some path - - InitEspeak(data_dir); // See ./piper-phonemize-lexicon.cc - } - - std::vector ConvertTextToTokenIds(const std::string &_text, - const std::string &voice) const { - // we cannot convert text to lowercase here since it will affect - // how piper_phonemize handles punctuations inside the text - std::string text = _text; - - std::vector> replace_str_pairs = { - {"，", ","}, {":", ","}, {"、", ","}, {"；", ";"}, {"：", ":"}, - {"。", "."}, {"？", "?"}, {"！", "!"}, {"\\s+", " "}, - }; - for (const auto &p : replace_str_pairs) { - std::regex re(p.first); - text = std::regex_replace(text, re, p.second); - } - - if (debug_) { - SHERPA_ONNX_LOGE("After replacing punctuations and merging spaces:\n%s", - text.c_str()); - } - - // https://en.cppreference.com/w/cpp/regex - // https://stackoverflow.com/questions/37989081/how-to-use-unicode-range-in-c-regex - std::string expr_chinese = "([\\u4e00-\\u9fff]+)"; - std::string expr_not_chinese = "([^\\u4e00-\\u9fff]+)"; - - std::string expr_both = expr_chinese + "|" + expr_not_chinese; - - auto ws = ToWideString(text); - std::wstring wexpr_both = ToWideString(expr_both); - std::wregex we_both(wexpr_both); - - std::wstring wexpr_zh = ToWideString(expr_chinese); - std::wregex we_zh(wexpr_zh); - - auto begin = std::wsregex_iterator(ws.begin(), ws.end(), we_both); - auto end = std::wsregex_iterator(); - - std::vector ans; - - for (std::wsregex_iterator i = begin; i != end; ++i) { - std::wsmatch match = *i; - std::wstring match_str = match.str(); - - auto ms = ToString(match_str); - uint8_t c = reinterpret_cast(ms.data())[0]; - - std::vector> ids_vec; - if (std::regex_match(match_str, we_zh)) { - if (debug_) { - SHERPA_ONNX_LOGE("Chinese: %s", ms.c_str()); - } - ids_vec = ConvertChineseToTokenIDs(ms); - } else { - if (debug_) { - SHERPA_ONNX_LOGE("Non-Chinese: %s", ms.c_str()); - } - - ids_vec = ConvertNonChineseToTokenIDs(ms, voice); - } - - for (const auto &ids : ids_vec) { - if (ids.size() > 10 + 2) { - ans.emplace_back(ids); - } else { - if (ans.empty()) { - ans.emplace_back(ids); - } else { - if ((ans.back().tokens.size() + ids.size() < 50) || - (ids.size() < 5)) { - ans.back().tokens.back() = ids[1]; - ans.back().tokens.insert(ans.back().tokens.end(), ids.begin() + 2, - ids.end()); - } else { - ans.emplace_back(ids); - } - } - } - } - } - - if (debug_) { - for (const auto &v : ans) { - std::ostringstream os; - os << "\n"; - std::string sep; - for (auto i : v.tokens) { - os << sep << i; - sep = " "; - } - os << "\n"; - SHERPA_ONNX_LOGE("%s", os.str().c_str()); - } - } - - return ans; - } - - private: - bool IsPunctuation(const std::string &text) const { - if (text == ";" || text == ":" || text == "," || text == "." || - text == "!" || text == "?" || text == "—" || text == "…" || - text == "\"" || text == "(" || text == ")" || text == "“" || - text == "”") { - return true; - } - - return false; - } - - std::vector ConvertWordToIds(const std::string &w) const { - std::vector ans; - if (word2ids_.count(w)) { - ans = word2ids_.at(w); - } else { - std::vector words = SplitUtf8(w); - for (const auto &word : words) { - if (word2ids_.count(word)) { - auto ids = ConvertWordToIds(word); - ans.insert(ans.end(), ids.begin(), ids.end()); - } else { - if (debug_) { - SHERPA_ONNX_LOGE("Skip OOV: '%s'", word.c_str()); - } - } - } - } - - if (debug_ && !ans.empty()) { - std::ostringstream os; - os << w << ": "; - for (auto i : ans) { - os << id2token_.at(i) << " "; - } - os << "\n"; -#if __OHOS__ - SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); -#else - SHERPA_ONNX_LOGE("%s", os.str().c_str()); -#endif - } - - return ans; - } - - std::vector> ConvertChineseToTokenIDs( - const std::string &text) const { - std::vector words = SplitUtf8(text); - - if (debug_) { - std::ostringstream os; - std::string sep = ""; - for (const auto &w : words) { - os << sep << w; - sep = "_"; - } - -#if __OHOS__ - SHERPA_ONNX_LOGE("after splitting into UTF8:\n%{public}s", - os.str().c_str()); -#else - SHERPA_ONNX_LOGE("after splitting into UTF8:\n%s", os.str().c_str()); -#endif - } - - std::vector> ans; - std::vector this_sentence; - int32_t max_len = meta_data_.max_token_len; - - this_sentence.push_back(0); - - PhraseMatcher matcher(&all_words_, words, debug_); - - for (const std::string &w : matcher) { - auto ids = ConvertWordToIds(w); - if (ids.empty()) { -#if __OHOS__ - SHERPA_ONNX_LOGE("Ignore OOV '%{public}s'", w.c_str()); -#else - SHERPA_ONNX_LOGE("Ignore OOV '%s'", w.c_str()); -#endif - continue; - } - - if (this_sentence.size() + ids.size() > max_len - 2) { - this_sentence.push_back(0); - ans.push_back(std::move(this_sentence)); - - this_sentence.push_back(0); - } - - this_sentence.insert(this_sentence.end(), ids.begin(), ids.end()); - } // for (const std::string &w : matcher) - - if (this_sentence.size() > 1) { - this_sentence.push_back(0); - ans.push_back(std::move(this_sentence)); - } - - if (debug_) { - for (const auto &v : ans) { - std::ostringstream os; - os << "\n"; - std::string sep; - for (auto i : v) { - os << sep << i; - sep = " "; - } - os << "\n"; - SHERPA_ONNX_LOGE("%s", os.str().c_str()); - } - } - - return ans; - } - - std::vector> ConvertTextToTokenIDsWithEspeak( - const std::string &text, const std::string &voice) const { - auto temp = ConvertTextToTokenIdsKokoroOrKitten( - phoneme2id_, meta_data_.max_token_len, text, voice); - std::vector> ans; - ans.reserve(temp.size()); - - for (const auto &i : temp) { - ans.emplace_back(i.tokens.begin(), i.tokens.end()); - } - - return ans; - } - - std::vector> ConvertNonChineseToTokenIDs( - const std::string &text, const std::string &voice) const { - if (IsPunctuation(text)) { - return {std::vector{0, token2id_.at(text), 0}}; - } - - if (!voice.empty()) { - return ConvertTextToTokenIDsWithEspeak(text, voice); - } - - // If voice is empty, we split the text into words and use the lexicon - // to lookup the pronunciation of each word, fallback to espeak if - // a word is not in the lexicon. - - std::vector words = SplitUtf8(text); - if (debug_) { - std::ostringstream os; - os << "After splitting to words: "; - std::string sep; - for (const auto &w : words) { - os << sep << w; - sep = "_"; - } - SHERPA_ONNX_LOGE("%s", os.str().c_str()); - } - - std::vector> ans; - int32_t max_len = meta_data_.max_token_len; - std::vector this_sentence; - - int32_t space_id = token2id_.at(" "); - - this_sentence.push_back(0); - - for (const auto &_word : words) { - auto word = ToLowerCase(_word); - if (IsPunctuation(word)) { - this_sentence.push_back(token2id_.at(word)); - - if (this_sentence.size() > max_len - 2) { - // this sentence is too long, split it - this_sentence.push_back(0); - ans.push_back(std::move(this_sentence)); - - this_sentence.push_back(0); - continue; - } - - if (word == "." || word == "!" || word == "?" || word == ";") { - // Note: You can add more punctuations here to split the text - // into sentences. We just use four here: .!?; - this_sentence.push_back(0); - ans.push_back(std::move(this_sentence)); - - this_sentence.push_back(0); - } - } else if (word2ids_.count(word)) { - const auto &ids = word2ids_.at(word); - if (this_sentence.size() + ids.size() + 3 > max_len - 2) { - this_sentence.push_back(0); - ans.push_back(std::move(this_sentence)); - - this_sentence.push_back(0); - } - - this_sentence.insert(this_sentence.end(), ids.begin(), ids.end()); - this_sentence.push_back(space_id); - } else { - if (debug_) { - SHERPA_ONNX_LOGE("Use espeak-ng to handle the OOV: '%s'", - word.c_str()); - } - - piper::eSpeakPhonemeConfig config; - - config.voice = meta_data_.voice; - - std::vector> phonemes; - - CallPhonemizeEspeak(word, config, &phonemes); - // Note phonemes[i] contains a vector of unicode codepoints; - // we need to convert them to utf8 - - std::vector ids; - for (const auto &v : phonemes) { - for (const auto p : v) { - auto token = Utf32ToUtf8(p); - if (token2id_.count(token)) { - ids.push_back(token2id_.at(token)); - } else { - if (debug_) { - SHERPA_ONNX_LOGE("Skip OOV token '%s' from '%s'", token.c_str(), - word.c_str()); - } - } - } - } - - if (this_sentence.size() + ids.size() + 3 > max_len - 2) { - this_sentence.push_back(0); - ans.push_back(std::move(this_sentence)); - - this_sentence.push_back(0); - } - - this_sentence.insert(this_sentence.end(), ids.begin(), ids.end()); - this_sentence.push_back(space_id); - } - } - - if (this_sentence.size() > 1) { - this_sentence.push_back(0); - ans.push_back(std::move(this_sentence)); - } - - if (debug_) { - for (const auto &v : ans) { - std::ostringstream os; - os << "\n"; - std::string sep; - for (auto i : v) { - os << sep << i; - sep = " "; - } - os << "\n"; - SHERPA_ONNX_LOGE("%s", os.str().c_str()); - } - } - - return ans; - } - - void InitTokens(const std::string &tokens) { - auto is = OpenInputFile(tokens); - InitTokens(is); - } - - template - void InitTokens(Manager *mgr, const std::string &tokens) { - auto buf = ReadFile(mgr, tokens); - - std::istringstream is(std::string(buf.data(), buf.size())); - InitTokens(is); - } - - void InitTokens(std::istream &is) { - token2id_ = ReadTokens(is); // defined in ./symbol-table.cc - - if (debug_) { - for (const auto &p : token2id_) { - id2token_[p.second] = p.first; - } - } - - std::u32string s; - for (const auto &p : token2id_) { - s = Utf8ToUtf32(p.first); - - if (s.size() != 1) { - SHERPA_ONNX_LOGE("Error for token %s with id %d", p.first.c_str(), - p.second); - SHERPA_ONNX_EXIT(-1); - } - - char32_t c = s[0]; - phoneme2id_.insert({c, p.second}); - } - } - - void InitLexicon(const std::string &lexicon) { - if (lexicon.empty()) { - return; - } - - std::vector files; - SplitStringToVector(lexicon, ",", false, &files); - for (const auto &f : files) { - auto is = OpenInputFile(f); - InitLexicon(is); - } - } - - template - void InitLexicon(Manager *mgr, const std::string &lexicon) { - if (lexicon.empty()) { - return; - } - - std::vector files; - SplitStringToVector(lexicon, ",", false, &files); - for (const auto &f : files) { - auto buf = ReadFile(mgr, f); - - std::istringstream is(std::string(buf.data(), buf.size())); - InitLexicon(is); - } - } - - void InitLexicon(std::istream &is) { - std::string word; - std::vector token_list; - std::string token; - - std::string line; - int32_t line_num = 0; - int32_t num_warn = 0; - while (std::getline(is, line)) { - ++line_num; - std::istringstream iss(line); - - token_list.clear(); - iss >> word; - ToLowerCase(&word); - - if (word2ids_.count(word)) { - num_warn += 1; - if (num_warn < 10) { - SHERPA_ONNX_LOGE("Duplicated word: %s at line %d:%s. Ignore it.", - word.c_str(), line_num, line.c_str()); - } - continue; - } - - while (iss >> token) { - token_list.push_back(std::move(token)); - } - - std::vector ids = ConvertTokensToIds(token2id_, token_list); - - if (ids.empty() && word != "呣") { - SHERPA_ONNX_LOGE( - "Invalid pronunciation for word '%s' at line %d:%s. Ignore it", - word.c_str(), line_num, line.c_str()); - continue; - } - - word2ids_.insert({std::move(word), std::move(ids)}); - } - - for (const auto &[key, _] : word2ids_) { - all_words_.insert(key); - } - } - - private: - OfflineTtsKokoroModelMetaData meta_data_; - - // word to token IDs - std::unordered_map> word2ids_; - std::unordered_set all_words_; - - // tokens.txt is saved in token2id_ - std::unordered_map token2id_; - std::unordered_map id2token_; - - std::unordered_map phoneme2id_; - - bool debug_ = false; -}; - -KokoroMultiLangLexicon::~KokoroMultiLangLexicon() = default; - -KokoroMultiLangLexicon::KokoroMultiLangLexicon( - const std::string &tokens, const std::string &lexicon, - const std::string &data_dir, const OfflineTtsKokoroModelMetaData &meta_data, - bool debug) - : impl_(std::make_unique(tokens, lexicon, data_dir, meta_data, - debug)) {} // NOLINT - -template -KokoroMultiLangLexicon::KokoroMultiLangLexicon( - Manager *mgr, const std::string &tokens, const std::string &lexicon, - const std::string &data_dir, const OfflineTtsKokoroModelMetaData &meta_data, - bool debug) - : impl_(std::make_unique(mgr, tokens, lexicon, data_dir, meta_data, - debug)) {} // NOLINT - -std::vector KokoroMultiLangLexicon::ConvertTextToTokenIds( - const std::string &text, const std::string &voice /*= ""*/) const { - return impl_->ConvertTextToTokenIds(text, voice); -} - -#if __ANDROID_API__ >= 9 -template KokoroMultiLangLexicon::KokoroMultiLangLexicon( - AAssetManager *mgr, const std::string &tokens, const std::string &lexicon, - const std::string &data_dir, const OfflineTtsKokoroModelMetaData &meta_data, - bool debug); -#endif - -#if __OHOS__ -template KokoroMultiLangLexicon::KokoroMultiLangLexicon( - NativeResourceManager *mgr, const std::string &tokens, - const std::string &lexicon, const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &meta_data, bool debug); -#endif - -} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/kokoro-multi-lang-lexicon.h b/sherpa-onnx/csrc/kokoro-multi-lang-lexicon.h deleted file mode 100644 index a6d50963c..000000000 --- a/sherpa-onnx/csrc/kokoro-multi-lang-lexicon.h +++ /dev/null @@ -1,43 +0,0 @@ -// sherpa-onnx/csrc/kokoro-multi-lang-lexicon.h -// -// Copyright (c) 2025 Xiaomi Corporation - -#ifndef SHERPA_ONNX_CSRC_KOKORO_MULTI_LANG_LEXICON_H_ -#define SHERPA_ONNX_CSRC_KOKORO_MULTI_LANG_LEXICON_H_ - -#include -#include -#include - -#include "sherpa-onnx/csrc/offline-tts-frontend.h" -#include "sherpa-onnx/csrc/offline-tts-kokoro-model-meta-data.h" - -namespace sherpa_onnx { - -class KokoroMultiLangLexicon : public OfflineTtsFrontend { - public: - ~KokoroMultiLangLexicon() override; - - KokoroMultiLangLexicon(const std::string &tokens, const std::string &lexicon, - const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &meta_data, - bool debug); - - template - KokoroMultiLangLexicon(Manager *mgr, const std::string &tokens, - const std::string &lexicon, - const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &meta_data, - bool debug); - - std::vector ConvertTextToTokenIds( - const std::string &text, const std::string &voice = "") const override; - - private: - class Impl; - std::unique_ptr impl_; -}; - -} // namespace sherpa_onnx - -#endif // SHERPA_ONNX_CSRC_KOKORO_MULTI_LANG_LEXICON_H_ diff --git a/sherpa-onnx/csrc/matcha-tts-lexicon.cc b/sherpa-onnx/csrc/matcha-tts-lexicon.cc deleted file mode 100644 index fda27ee3d..000000000 --- a/sherpa-onnx/csrc/matcha-tts-lexicon.cc +++ /dev/null @@ -1,500 +0,0 @@ -// sherpa-onnx/csrc/matcha-tts-lexicon.cc -// -// Copyright (c) 2025 Xiaomi Corporation - -#include "sherpa-onnx/csrc/matcha-tts-lexicon.h" - -#include - -#include -#include -#include -#include // NOLINT -#include -#include -#include -#include -#include -#include - -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - -#if __OHOS__ -#include "rawfile/raw_file_manager.h" -#endif - -#include "espeak-ng/speak_lib.h" -#include "phoneme_ids.hpp" // NOLINT -#include "phonemize.hpp" // NOLINT -#include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/macros.h" -#include "sherpa-onnx/csrc/onnx-utils.h" -#include "sherpa-onnx/csrc/phrase-matcher.h" -#include "sherpa-onnx/csrc/symbol-table.h" -#include "sherpa-onnx/csrc/text-utils.h" - -namespace sherpa_onnx { - -namespace { -// Please see https://github.com/k2-fsa/sherpa-onnx/pull/2853 -// for why we need to do the replacement -static const std::vector> kReplacements = { - {"ɝ", "ɜɹ"}, {"ɚ", "əɹ"}, - - {"eɪ", "A"}, {"aɪ", "I"}, {"ɔɪ", "Y"}, - {"oʊ", "O"}, {"əʊ", "O"}, {"aʊ", "W"}, - - {"tʃ", "ʧ"}, {"dʒ", "ʤ"}, - - {"ː", ""}, - - {"g", "ɡ"}, {"r", "ɹ"}, - - {"e", "ɛ"}, -}; - -std::vector ConvertPhonemesToUTF8( - const std::vector> &phonemes) { - std::vector out; - - for (const auto &word : phonemes) { - for (char32_t cp : word) { - out.push_back(Utf32ToUtf8(cp)); - } - } - - return out; -} - -std::string ApplyReplacements(std::string s) { - for (const auto &p : kReplacements) { - const std::string &from = p.first; - const std::string &to = p.second; - - size_t pos = 0; - while ((pos = s.find(from, pos)) != std::string::npos) { - s.replace(pos, from.size(), to); - pos += to.size(); - } - } - return s; -} - -std::vector SplitTokensUTF8(const std::string &s) { - std::vector out; - - for (size_t i = 0; i < s.size();) { - unsigned char c = s[i]; - size_t len = (c < 0x80) ? 1 : (c < 0xE0) ? 2 : (c < 0xF0) ? 3 : 4; - - out.push_back(s.substr(i, len)); - i += len; - } - - return out; -} - -std::vector ProcessPhonemes( - const std::vector> &phonemes, bool skip_replacement) { - auto tokens = ConvertPhonemesToUTF8(phonemes); - if (skip_replacement) { - return tokens; - } - - std::string joined = Join(tokens); - std::string replaced = ApplyReplacements(joined); - return SplitTokensUTF8(replaced); -} - -} // namespace - -void CallPhonemizeEspeak(const std::string &text, - piper::eSpeakPhonemeConfig &config, // NOLINT - std::vector> *phonemes); - -class MatchaTtsLexicon::Impl { - public: - Impl(const std::string &lexicon, const std::string &tokens, - const std::string &data_dir, bool debug, bool skip_replacement) - : debug_(debug), skip_replacement_(skip_replacement) { - if (lexicon.empty()) { - SHERPA_ONNX_LOGE("Please provide lexicon.txt for this model"); - SHERPA_ONNX_EXIT(-1); - } - - { - auto is = OpenInputFile(tokens); - InitTokens(is); - } - - InitLexicon(lexicon); - - if (data_dir.empty()) { - SHERPA_ONNX_LOGE("Please provide data dir for this model"); - SHERPA_ONNX_EXIT(-1); - } - - InitEspeak(data_dir); // See ./piper-phonemize-lexicon.cc - } - - template - Impl(Manager *mgr, const std::string &lexicon, const std::string &tokens, - const std::string &data_dir, bool debug, bool skip_replacement) - : debug_(debug), skip_replacement_(skip_replacement) { - if (lexicon.empty()) { - SHERPA_ONNX_LOGE("Please provide lexicon.txt for this model"); - SHERPA_ONNX_EXIT(-1); - } - - { - auto buf = ReadFile(mgr, tokens); - std::istringstream is(std::string(buf.data(), buf.size())); - - InitTokens(is); - } - - std::vector files; - SplitStringToVector(lexicon, ",", false, &files); - for (const auto &f : files) { - auto buf = ReadFile(mgr, f); - - std::istringstream is(std::string(buf.data(), buf.size())); - InitLexicon(is); - } - - if (data_dir.empty()) { - SHERPA_ONNX_LOGE("Please provide data dir for this model"); - SHERPA_ONNX_EXIT(-1); - } - - InitEspeak(data_dir); // See ./piper-phonemize-lexicon.cc - } - - std::vector ConvertTextToTokenIds(const std::string &_text) const { - std::string text = _text; - std::vector> replace_str_pairs = { - {"，", ","}, {"、", ","}, {"；", ";"}, {"：", ","}, {":", ","}, - {"。", "."}, {"？", "?"}, {"！", "!"}, {"\\s+", " "}, - }; - for (const auto &p : replace_str_pairs) { - std::regex re(p.first); - text = std::regex_replace(text, re, p.second); - } - - if (debug_) { - SHERPA_ONNX_LOGE("After replacing punctuations and merging spaces:\n%s", - text.c_str()); - } - - std::vector words = SplitUtf8(text); - - if (debug_) { -#if __OHOS__ - SHERPA_ONNX_LOGE("input text:\n%{public}s", _text.c_str()); - SHERPA_ONNX_LOGE("after replacing punctuations:\n%{public}s", - text.c_str()); -#else - SHERPA_ONNX_LOGE("input text:\n%s", _text.c_str()); - SHERPA_ONNX_LOGE("after replacing punctuations:\n%s", text.c_str()); -#endif - - std::ostringstream os; - std::string sep = ""; - for (const auto &w : words) { - os << sep << w; - sep = "_"; - } - -#if __OHOS__ - SHERPA_ONNX_LOGE("after splitting into UTF8:\n%{public}s", - os.str().c_str()); -#else - SHERPA_ONNX_LOGE("after splitting into UTF8:\n%s", os.str().c_str()); -#endif - } - - // remove spaces after punctuations - std::vector words2 = std::move(words); - words.reserve(words2.size()); - - for (int32_t i = 0; i < words2.size(); ++i) { - if (i == 0) { - words.push_back(std::move(words2[i])); - } else if (words2[i] == " ") { - if (words.back() == " " || IsPunct(words.back())) { - continue; - } else { - words.push_back(std::move(words2[i])); - } - } else if (IsPunct(words2[i])) { - if (words.back() == " " || IsPunct(words.back())) { - continue; - } else { - words.push_back(std::move(words2[i])); - } - } else { - words.push_back(std::move(words2[i])); - } - } - - if (debug_) { - std::ostringstream os; - std::string sep = ""; - for (const auto &w : words) { - os << sep << w; - sep = "_"; - } - -#if __OHOS__ - SHERPA_ONNX_LOGE("after removing spaces after punctuations:\n%{public}s", - os.str().c_str()); -#else - SHERPA_ONNX_LOGE("after removing spaces after punctuations:\n%s", - os.str().c_str()); -#endif - } - - std::vector ans; - std::vector this_sentence; - - PhraseMatcher matcher(&all_words_, words, debug_); - - int32_t blank = token2id_.at(" "); - - std::vector ids; - std::string last_word; - for (const std::string &w : matcher) { - ids = ConvertWordToIds(w); - - if (ids.empty()) { -#if __OHOS__ - SHERPA_ONNX_LOGE("Ignore OOV '%{public}s'", w.c_str()); -#else - SHERPA_ONNX_LOGE("Ignore OOV '%s'", w.c_str()); -#endif - - last_word = w; - continue; - } - - if (!last_word.empty() && isalpha(last_word[0])) { - this_sentence.push_back(blank); - } - - this_sentence.insert(this_sentence.end(), ids.begin(), ids.end()); - - if (IsPunct(w)) { - if (debug_) { - std::ostringstream os; - std::string sep; - os << "new sentence: ["; - for (auto i : this_sentence) { - os << sep << i; - sep = ", "; - } - os << "]"; - SHERPA_ONNX_LOGE("%s", os.str().c_str()); - } - - ans.emplace_back(std::move(this_sentence)); - this_sentence = {}; - } - - last_word = w; - } // for (const std::string &w : matcher) - - if (!this_sentence.empty()) { - ans.emplace_back(std::move(this_sentence)); - } - - return ans; - } - - private: - std::vector ConvertWordToIds(const std::string &w) const { - std::vector ans; - if (word2ids_.count(w)) { - ans = word2ids_.at(w); - } else if (token2id_.count(w)) { - ans = {token2id_.at(w)}; - } else { - if (ContainsCJK(w)) { - std::vector words = SplitUtf8(w); - for (const auto &word : words) { - if (word2ids_.count(word)) { - auto ids = ConvertWordToIds(word); - ans.insert(ans.end(), ids.begin(), ids.end()); - } - } - } else { - if (debug_) { - SHERPA_ONNX_LOGE("use espeak for %s", w.c_str()); - } - // use espeak - piper::eSpeakPhonemeConfig config; - config.voice = "en-us"; - std::vector> phonemes; - CallPhonemizeEspeak(w, config, &phonemes); - - auto pp = ProcessPhonemes(phonemes, skip_replacement_); - - for (const auto &p : pp) { - if (token2id_.count(p)) { - ans.push_back(token2id_.at(p)); - } else { - SHERPA_ONNX_LOGE("Skip token: %s", p.c_str()); - } - } - } - } - - if (debug_) { - std::ostringstream os; - os << w << ": "; - for (auto i : ans) { - os << "'" << id2token_.at(i) << "'(" << i << ")" << ","; - } -#if __OHOS__ - SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); -#else - SHERPA_ONNX_LOGE("%s", os.str().c_str()); -#endif - } - - return ans; - } - - void InitTokens(std::istream &is) { - token2id_ = ReadTokens(is); - - if (debug_) { - for (const auto &p : token2id_) { - id2token_[p.second] = p.first; - } - } - } - - void InitLexicon(const std::string &lexicon) { - if (lexicon.empty()) { - SHERPA_ONNX_LOGE("Empty lexicon!"); - return; - } - - std::vector files; - SplitStringToVector(lexicon, ",", false, &files); - for (const auto &f : files) { - auto is = OpenInputFile(f); - InitLexicon(is); - } - } - - void InitLexicon(std::istream &is) { - std::string word; - std::vector token_list; - std::string line; - std::string phone; - int32_t line_num = 0; - - while (std::getline(is, line)) { - ++line_num; - - std::istringstream iss(line); - - token_list.clear(); - - iss >> word; - ToLowerCase(&word); - - if (word2ids_.count(word)) { -#if __OHOS__ - SHERPA_ONNX_LOGE( - "Duplicated word: %{public}s at line %{public}d:%{public}s. Ignore " - "it.", - word.c_str(), line_num, line.c_str()); -#else - SHERPA_ONNX_LOGE("Duplicated word: %s at line %d:%s. Ignore it.", - word.c_str(), line_num, line.c_str()); -#endif - continue; - } - - while (iss >> phone) { - token_list.push_back(std::move(phone)); - } - - std::vector ids = ConvertTokensToIds(token2id_, token_list); - if (ids.empty()) { - if (debug_) { -#if __OHOS__ - SHERPA_ONNX_LOGE("Empty token ids for '%{public}s'", line.c_str()); -#else - SHERPA_ONNX_LOGE("Empty token ids for '%s'", line.c_str()); -#endif - } - continue; - } - - word2ids_.insert({std::move(word), std::move(ids)}); - } - - for (const auto &[key, _] : word2ids_) { - all_words_.insert(key); - } - } - - private: - // lexicon.txt is saved in word2ids_ - std::unordered_map> word2ids_; - std::unordered_set all_words_; - - // tokens.txt is saved in token2id_ - std::unordered_map token2id_; - - std::unordered_map id2token_; - - bool debug_ = false; - bool skip_replacement_ = false; -}; // namespace sherpa_onnx - -MatchaTtsLexicon::~MatchaTtsLexicon() = default; - -MatchaTtsLexicon::MatchaTtsLexicon(const std::string &lexicon, - const std::string &tokens, - const std::string &data_dir, bool debug, - bool skip_replacement) - : impl_(std::make_unique(lexicon, tokens, data_dir, debug, - skip_replacement)) {} // NOLINT - -template -MatchaTtsLexicon::MatchaTtsLexicon(Manager *mgr, const std::string &lexicon, - const std::string &tokens, - const std::string &data_dir, bool debug, - bool skip_replacement) - : impl_(std::make_unique(mgr, lexicon, tokens, data_dir, debug, - skip_replacement)) {} // NOLINT - -std::vector MatchaTtsLexicon::ConvertTextToTokenIds( - const std::string &text, const std::string & /*unused_voice = ""*/) const { - return impl_->ConvertTextToTokenIds(text); -} - -#if __ANDROID_API__ >= 9 -template MatchaTtsLexicon::MatchaTtsLexicon(AAssetManager *mgr, - const std::string &lexicon, - const std::string &tokens, - const std::string &data_dir, - bool debug, bool skip_replacement); -#endif - -#if __OHOS__ -template MatchaTtsLexicon::MatchaTtsLexicon(NativeResourceManager *mgr, - const std::string &lexicon, - const std::string &tokens, - const std::string &data_dir, - bool debug, bool skip_replacement); -#endif - -} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/matcha-tts-lexicon.h b/sherpa-onnx/csrc/matcha-tts-lexicon.h deleted file mode 100644 index f23df4392..000000000 --- a/sherpa-onnx/csrc/matcha-tts-lexicon.h +++ /dev/null @@ -1,42 +0,0 @@ -// sherpa-onnx/csrc/matcha-tts-lexicon.h -// -// Copyright (c) 2025 Xiaomi Corporation - -#ifndef SHERPA_ONNX_CSRC_MATCHA_TTS_LEXICON_H_ -#define SHERPA_ONNX_CSRC_MATCHA_TTS_LEXICON_H_ - -#include -#include -#include -#include - -#include "sherpa-onnx/csrc/offline-tts-frontend.h" - -namespace sherpa_onnx { - -// For Chinese+English matcha tts -class MatchaTtsLexicon : public OfflineTtsFrontend { - public: - ~MatchaTtsLexicon() override; - - MatchaTtsLexicon(const std::string &lexicon, const std::string &tokens, - const std::string &data_dir, bool debug, - bool skip_replacement); - - template - MatchaTtsLexicon(Manager *mgr, const std::string &lexicon, - const std::string &tokens, const std::string &data_dir, - bool debug, bool skip_replacement); - - std::vector ConvertTextToTokenIds( - const std::string &text, - const std::string &unused_voice = "") const override; - - private: - class Impl; - std::unique_ptr impl_; -}; - -} // namespace sherpa_onnx - -#endif // SHERPA_ONNX_CSRC_MATCHA_TTS_LEXICON_H_ diff --git a/sherpa-onnx/csrc/offline-tts-character-frontend.h b/sherpa-onnx/csrc/offline-tts-character-frontend.h index 55bf6a70f..4c4e0f87f 100644 --- a/sherpa-onnx/csrc/offline-tts-character-frontend.h +++ b/sherpa-onnx/csrc/offline-tts-character-frontend.h @@ -28,7 +28,8 @@ class OfflineTtsCharacterFrontend : public OfflineTtsFrontend { * @param text The input text. * Example 1: "This is the first sample sentence; this is the * second one." Example 2: "这是第一句。这是第二句。" - * @param voice Optional. It is for espeak-ng. + * @param voice Optional. Unused by this frontend. (It used to select an + * espeak-ng voice; espeak-ng is no longer used.) * * @return Return a vector-of-vector of token IDs. Each subvector contains * a sentence that can be processed independently. diff --git a/sherpa-onnx/csrc/offline-tts-frontend.cc b/sherpa-onnx/csrc/offline-tts-frontend.cc index f083c75e3..f53436290 100644 --- a/sherpa-onnx/csrc/offline-tts-frontend.cc +++ b/sherpa-onnx/csrc/offline-tts-frontend.cc @@ -4,9 +4,14 @@ #include "sherpa-onnx/csrc/offline-tts-frontend.h" +#include #include #include +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/offline-tts-kitten-model-meta-data.h" +#include "sherpa-onnx/csrc/text-utils.h" + namespace sherpa_onnx { std::string TokenIDs::ToString() const { @@ -31,4 +36,249 @@ std::string TokenIDs::ToString() const { return os.str(); } +std::unordered_map ReadPiperTokens(std::istream &is) { + std::unordered_map token2id; + std::string line; + std::string sym; + std::u32string s; + int32_t id = 0; + while (std::getline(is, line)) { + std::istringstream iss(line); + iss >> sym; + if (iss.eof()) { + id = atoi(sym.c_str()); + sym = " "; + } else { + iss >> id; + } + + iss >> std::ws; + if (!iss.eof()) { + SHERPA_ONNX_LOGE("Error when reading tokens: %s", line.c_str()); + SHERPA_ONNX_EXIT(-1); + } + + s = Utf8ToUtf32(sym); + if (s.size() != 1) { + // for tokens.txt from coqui-ai/TTS, the last token is + if (s.size() == 6 && s[0] == '<' && s[1] == 'B' && s[2] == 'L' && + s[3] == 'N' && s[4] == 'K' && s[5] == '>') { + continue; + } + SHERPA_ONNX_LOGE("Error when reading tokens at Line %s. size: %d", + line.c_str(), static_cast(s.size())); + SHERPA_ONNX_EXIT(-1); + } + + char32_t c = s[0]; + if (token2id.count(c)) { + SHERPA_ONNX_LOGE("Duplicated token %s. Line %s. Existing ID: %d", + sym.c_str(), line.c_str(), token2id.at(c)); + SHERPA_ONNX_EXIT(-1); + } + token2id.insert({c, id}); + } + return token2id; +} + +std::vector PiperPhonemesToIdsVits( + const std::unordered_map &token2id, + const std::vector &phonemes, bool is_inflect) { + // These markers must exist in tokens.txt; look them up defensively so a + // vocab without them fails cleanly instead of throwing std::out_of_range + // across the C ABI (which would call std::terminate). + auto pad_it = token2id.find(U'_'); + if (pad_it == token2id.end()) { + SHERPA_ONNX_LOGE("tokens.txt is missing the required '_' symbol."); + return {}; + } + int32_t pad = pad_it->second; + int32_t bos = -1; + int32_t eos = -1; + if (!is_inflect) { + auto bos_it = token2id.find(U'^'); + auto eos_it = token2id.find(U'$'); + if (bos_it == token2id.end() || eos_it == token2id.end()) { + SHERPA_ONNX_LOGE("tokens.txt is missing the required '^' or '$' symbol."); + return {}; + } + bos = bos_it->second; + eos = eos_it->second; + } + + std::vector ans; + ans.reserve(phonemes.size() * 2 + 3); + + if (is_inflect) { + ans.push_back(pad); + } else { + ans.push_back(bos); + ans.push_back(pad); + } + + for (auto p : phonemes) { + if (token2id.count(p)) { + ans.push_back(token2id.at(p)); + ans.push_back(pad); + } else { + SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", + static_cast(p)); + } + } + + if (!is_inflect) { + ans.push_back(eos); + } + + return ans; +} + +std::vector> PiperPhonemesToIdsMatcha( + const std::unordered_map &token2id, + const std::vector &phonemes, bool use_eos_bos, + int32_t max_token_len /*= 400*/) { + std::vector> ans; + std::vector current; + + int32_t bos = -1; + int32_t eos = -1; + if (use_eos_bos && token2id.count(U'^') && token2id.count(U'$')) { + bos = token2id.at(U'^'); + eos = token2id.at(U'$'); + current.push_back(bos); + } + + for (auto p : phonemes) { + if (token2id.count(p)) { + current.push_back(token2id.at(p)); + } else { + SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", + static_cast(p)); + } + + if (current.size() > max_token_len + 1) { + if (eos >= 0) { + current.push_back(eos); + } + ans.push_back(std::move(current)); + if (bos >= 0) { + current.push_back(bos); + } + } + } + + if (!current.empty()) { + if (eos >= 0 && current.size() > 1) { + current.push_back(eos); + ans.push_back(std::move(current)); + } else { + ans.push_back(std::move(current)); + } + } + + return ans; +} + +std::vector> PiperPhonemesToIdsKitten( + const std::unordered_map &token2id, + const std::vector &phonemes, + const OfflineTtsKittenModelMetaData &meta_data) { + std::vector> ans; + std::vector current; + current.reserve(phonemes.size()); + current.push_back(meta_data.start_id); + + int32_t suffix_size = meta_data.add_pad_after_end ? 2 : 1; + for (auto p : phonemes) { + if (token2id.count(p)) { + int32_t emitted_tokens = p == '.' ? 2 : 1; + if (static_cast(current.size()) + emitted_tokens + suffix_size > + meta_data.max_token_len) { + current.push_back(meta_data.end_id); + if (meta_data.add_pad_after_end) { + current.push_back(meta_data.pad_id); + } + ans.push_back(std::move(current)); + current.reserve(phonemes.size()); + current.push_back(meta_data.start_id); + } + current.push_back(token2id.at(p)); + if (p == '.') { + auto sp = token2id.find(' '); + if (sp != token2id.end()) { + current.push_back(sp->second); + } + } + } else { + SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", + static_cast(p)); + } + } + + current.push_back(meta_data.end_id); + if (meta_data.add_pad_after_end) { + current.push_back(meta_data.pad_id); + } + ans.push_back(std::move(current)); + return ans; +} + +std::vector CoquiPhonemesToIds( + const std::unordered_map &token2id, + const std::vector &phonemes, + const OfflineTtsVitsModelMetaData &vits_meta_data) { + int32_t use_eos_bos = vits_meta_data.use_eos_bos; + int32_t bos_id = vits_meta_data.bos_id; + int32_t eos_id = vits_meta_data.eos_id; + int32_t blank_id = vits_meta_data.blank_id; + int32_t add_blank = vits_meta_data.add_blank; + auto comma_it = token2id.find(','); + if (comma_it == token2id.end()) { + SHERPA_ONNX_LOGE("tokens.txt is missing the required ',' symbol."); + return {}; + } + int32_t comma_id = comma_it->second; + + std::vector ans; + if (add_blank) { + ans.reserve(phonemes.size() * 2 + 3); + } else { + ans.reserve(phonemes.size() + 2); + } + + if (use_eos_bos) { + ans.push_back(bos_id); + } + + if (add_blank) { + ans.push_back(blank_id); + for (auto p : phonemes) { + if (token2id.count(p)) { + ans.push_back(token2id.at(p)); + ans.push_back(blank_id); + } else { + SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", + static_cast(p)); + } + } + } else { + for (auto p : phonemes) { + if (token2id.count(p)) { + ans.push_back(token2id.at(p)); + } else { + SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", + static_cast(p)); + } + } + } + + ans.push_back(comma_id); + + if (use_eos_bos) { + ans.push_back(eos_id); + } + + return ans; +} + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-frontend.h b/sherpa-onnx/csrc/offline-tts-frontend.h index e58ffd5d9..9c9bb87d9 100644 --- a/sherpa-onnx/csrc/offline-tts-frontend.h +++ b/sherpa-onnx/csrc/offline-tts-frontend.h @@ -5,12 +5,14 @@ #ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_FRONTEND_H_ #define SHERPA_ONNX_CSRC_OFFLINE_TTS_FRONTEND_H_ #include +#include #include #include #include #include -#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/offline-tts-kitten-model-meta-data.h" +#include "sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h" namespace sherpa_onnx { @@ -44,7 +46,7 @@ class OfflineTtsFrontend { * @param text The input text. * Example 1: "This is the first sample sentence; this is the * second one." Example 2: "这是第一句。这是第二句。" - * @param voice Optional. It is for espeak-ng. + * @param voice Optional voice identifier. * * @return Return a vector-of-vector of token IDs. Each subvector contains * a sentence that can be processed independently. @@ -55,14 +57,28 @@ class OfflineTtsFrontend { const std::string &text, const std::string &voice = "") const = 0; }; -// implementation is in ./piper-phonemize-lexicon.cc -void InitEspeak(const std::string &data_dir); +// Tokenization utilities (no espeak-ng dependency) -// implementation in ./piper-phonemize-lexicon.cc -std::vector ConvertTextToTokenIdsKokoroOrKitten( +std::unordered_map ReadPiperTokens(std::istream &is); + +std::vector PiperPhonemesToIdsVits( const std::unordered_map &token2id, - int32_t max_token_len, const std::string &text, - const std::string &voice = ""); + const std::vector &phonemes, bool is_inflect); + +std::vector> PiperPhonemesToIdsMatcha( + const std::unordered_map &token2id, + const std::vector &phonemes, bool use_eos_bos, + int32_t max_token_len = 400); + +std::vector> PiperPhonemesToIdsKitten( + const std::unordered_map &token2id, + const std::vector &phonemes, + const OfflineTtsKittenModelMetaData &meta_data); + +std::vector CoquiPhonemesToIds( + const std::unordered_map &token2id, + const std::vector &phonemes, + const OfflineTtsVitsModelMetaData &vits_meta_data); } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-impl.h b/sherpa-onnx/csrc/offline-tts-impl.h index 60eefdc98..5671de6ce 100644 --- a/sherpa-onnx/csrc/offline-tts-impl.h +++ b/sherpa-onnx/csrc/offline-tts-impl.h @@ -57,8 +57,20 @@ class OfflineTtsImpl { // If it supports only a single speaker, then it return 0 or 1. virtual int32_t NumSpeakers() const { return 1; } + // Language/voice of the model, e.g., "en-us". Empty if not available. + virtual std::string Lang() const { return ""; } + std::vector AddBlank(const std::vector &x, int32_t blank_id = 0) const; + + protected: + // The model metadata `voice` may contain several space-separated languages, + // e.g. "zh en-us" for a bilingual model. The phonemizer needs a single voice, + // so return only the last one (e.g. "en-us"). + static std::string LastLang(const std::string &voice) { + auto pos = voice.rfind(' '); + return (pos == std::string::npos) ? voice : voice.substr(pos + 1); + } }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-kitten-impl.h b/sherpa-onnx/csrc/offline-tts-kitten-impl.h index 33d0d7cb6..0732739e1 100644 --- a/sherpa-onnx/csrc/offline-tts-kitten-impl.h +++ b/sherpa-onnx/csrc/offline-tts-kitten-impl.h @@ -4,26 +4,21 @@ #ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_KITTEN_IMPL_H_ #define SHERPA_ONNX_CSRC_OFFLINE_TTS_KITTEN_IMPL_H_ -#include -#include #include -#include #include +#include +#include #include #include -#include "fst/extensions/far/far.h" -#include "kaldifst/csrc/kaldi-fst-io.h" -#include "kaldifst/csrc/text-normalizer.h" #include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/fst-utils.h" -#include "sherpa-onnx/csrc/lexicon.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-tts-frontend.h" #include "sherpa-onnx/csrc/offline-tts-impl.h" #include "sherpa-onnx/csrc/offline-tts-kitten-model.h" -#include "sherpa-onnx/csrc/piper-phonemize-lexicon.h" +#include "sherpa-onnx/csrc/symbol-table.h" #include "sherpa-onnx/csrc/text-utils.h" +#include "sherpa-onnx/csrc/tts-text-normalizer.h" namespace sherpa_onnx { @@ -32,105 +27,89 @@ class OfflineTtsKittenImpl : public OfflineTtsImpl { explicit OfflineTtsKittenImpl(const OfflineTtsConfig &config) : config_(config), model_(std::make_unique(config.model)) { - InitFrontend(); + if (!config.model.kitten.tokens.empty()) { + auto is = OpenInputFile(config.model.kitten.tokens); + token_str2id_ = ReadTokens(is); - if (!config.rule_fsts.empty()) { - std::vector files; - SplitStringToVector(config.rule_fsts, ",", false, &files); - tn_list_.reserve(files.size()); - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); -#endif + // Also load char32_t → int32_t map for IPA single-codepoint tokens + auto is2 = OpenInputFile(config.model.kitten.tokens); + std::string line; + while (std::getline(is2, line)) { + std::istringstream iss(line); + std::string sym; + int32_t id = 0; + iss >> sym; + if (sym.empty()) continue; + if (iss.eof()) { + id = atoi(sym.c_str()); + sym = " "; + } else { + iss >> id; + } + std::u32string u32 = Utf8ToUtf32(sym); + if (u32.size() == 1) { + token2id_[u32[0]] = id; } - tn_list_.push_back(std::make_unique(f)); } } - if (!config.rule_fars.empty()) { - if (config.model.debug) { - SHERPA_ONNX_LOGE("Loading FST archives"); - } + if (!config.model.kitten.lexicon.empty()) { std::vector files; - SplitStringToVector(config.rule_fars, ",", false, &files); - - tn_list_.reserve(files.size() + tn_list_.size()); - + SplitStringToVector(config.model.kitten.lexicon, ",", false, &files); for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); -#endif - } - std::unique_ptr> reader( - fst::FarReader::Open(f)); - for (; !reader->Done(); reader->Next()) { - std::unique_ptr r( - fst::CastOrConvertToConstFst(reader->GetFst()->Copy())); - - tn_list_.push_back( - std::make_unique(std::move(r))); - } - } - - if (config.model.debug) { - SHERPA_ONNX_LOGE("FST archives loaded!"); + auto is = OpenInputFile(f); + LoadLexicon(is, config.model.debug); } } + + tn_list_ = LoadTextNormalizers(config.rule_fsts, config.rule_fars, + config.model.debug); } template OfflineTtsKittenImpl(Manager *mgr, const OfflineTtsConfig &config) : config_(config), model_(std::make_unique(mgr, config.model)) { - InitFrontend(mgr); + if (!config.model.kitten.tokens.empty()) { + auto buf = ReadFile(mgr, config.model.kitten.tokens); + std::istringstream is(std::string(buf.data(), buf.size())); + token_str2id_ = ReadTokens(is); - if (!config.rule_fsts.empty()) { - std::vector files; - SplitStringToVector(config.rule_fsts, ",", false, &files); - tn_list_.reserve(files.size()); - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); -#endif + // Also load char32_t → int32_t map for IPA single-codepoint tokens + auto buf2 = ReadFile(mgr, config.model.kitten.tokens); + std::istringstream is2(std::string(buf2.data(), buf2.size())); + std::string line; + while (std::getline(is2, line)) { + std::istringstream iss(line); + std::string sym; + int32_t id = 0; + iss >> sym; + if (sym.empty()) continue; + if (iss.eof()) { + id = atoi(sym.c_str()); + sym = " "; + } else { + iss >> id; + } + std::u32string u32 = Utf8ToUtf32(sym); + if (u32.size() == 1) { + token2id_[u32[0]] = id; } - auto buf = ReadFile(mgr, f); - std::istringstream is(std::string(buf.data(), buf.size())); - tn_list_.push_back(std::make_unique(is)); } } - if (!config.rule_fars.empty()) { + if (!config.model.kitten.lexicon.empty()) { std::vector files; - SplitStringToVector(config.rule_fars, ",", false, &files); - tn_list_.reserve(files.size() + tn_list_.size()); - + SplitStringToVector(config.model.kitten.lexicon, ",", false, &files); for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); -#endif - } - auto buf = ReadFile(mgr, f); + std::istringstream is(std::string(buf.data(), buf.size())); + LoadLexicon(is, config.model.debug); + } + } - auto fsts = ReadFstsFromFar(buf); - for (auto &r : fsts) { - tn_list_.push_back( - std::make_unique(std::move(r))); - } - } // for (const auto &f : files) - } // if (!config.rule_fars.empty()) + tn_list_ = LoadTextNormalizers(mgr, config.rule_fsts, config.rule_fars, + config.model.debug); } int32_t SampleRate() const override { @@ -141,17 +120,30 @@ class OfflineTtsKittenImpl : public OfflineTtsImpl { return model_->GetMetaData().num_speakers; } - // Supported options in GenerationConfig: - // - sid: Speaker ID for multi-speaker models - // - speed: Speech speed factor (default: 1.0) - // - silence_scale: Scale applied to pauses in the generated audio - // - // Supported extra options in config.extra: - // - None + std::string Lang() const override { return LastLang(model_->GetMetaData().voice); } + + /** + * + * Supported options in GenerationConfig: + * - sid: Speaker ID for multi-speaker models + * - speed: Speech speed factor (default: 1.0) + * - silence_scale: Scale applied to pauses in the generated audio + * + * Supported extra parameters: + * + * - debug, int, default from model config + * - min_words_in_sentence, int, default 5. + * Merge adjacent sentences if the number of words is less than this. + * - max_words_in_sentence, int, default 20. + * Split a sentence into chunks if the number of words exceeds this. + * Splits at punctuation or space boundaries. + */ GeneratedAudio Generate( const std::string &_text, const GenerationConfig &gen_config, GeneratedAudioCallback callback = nullptr) const override { - if (config_.model.debug) { + bool debug = gen_config.GetExtraInt("debug", config_.model.debug); + + if (debug) { SHERPA_ONNX_LOGE("%s", gen_config.ToString().c_str()); } @@ -195,43 +187,116 @@ class OfflineTtsKittenImpl : public OfflineTtsImpl { } std::string text = _text; - if (config_.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("Raw text: %{public}s", text.c_str()); -#else - SHERPA_ONNX_LOGE("Raw text: %s", text.c_str()); -#endif - std::ostringstream os; - os << "In bytes (hex):\n"; - const auto p = reinterpret_cast(text.c_str()); - for (int32_t i = 0; i != text.size(); ++i) { - os << std::setw(2) << std::setfill('0') << std::hex - << static_cast(p[i]) << " "; - } - os << "\n"; - -#if __OHOS__ - SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); -#else - SHERPA_ONNX_LOGE("%s", os.str().c_str()); -#endif + if (!text.empty()) { + text = NormalizeText(text, debug); } - if (!tn_list_.empty()) { - for (const auto &tn : tn_list_) { - text = tn->Normalize(text); - if (config_.model.debug) { + std::vector token_ids; + + if (!gen_config.tokens.empty()) { + // String tokens path — wrap with start_id/end_id like lexicon path + for (const auto &sentence : gen_config.tokens) { + std::vector ids; + ids.push_back(meta_data.start_id); + for (const auto &tok : sentence) { + auto it = token_str2id_.find(tok); + if (it != token_str2id_.end()) { + ids.push_back(it->second); + } else { + SHERPA_ONNX_LOGE("Skip unknown token: '%s'", tok.c_str()); + } + } + ids.push_back(meta_data.end_id); + if (meta_data.add_pad_after_end) { + ids.push_back(meta_data.pad_id); + } + if (ids.size() > 2) { // more than just start+end + token_ids.emplace_back(std::move(ids)); + } + } + } else if (!gen_config.phoneme_codepoints.empty()) { + // phoneme_codepoints path + for (const auto &sentence : gen_config.phoneme_codepoints) { + std::vector phonemes(sentence.begin(), sentence.end()); + auto ids_list = + PiperPhonemesToIdsKitten(token2id_, phonemes, meta_data); + for (auto &ids : ids_list) { + token_ids.emplace_back(std::move(ids)); + } + } + } else { + // Lexicon path + auto sentences = SplitByAllPunctuation(text); + if (sentences.empty()) { + SHERPA_ONNX_LOGE("No sentences after splitting"); + return {}; + } + + int32_t min_words = gen_config.GetExtraInt("min_words_in_sentence", 5); + int32_t max_words = gen_config.GetExtraInt("max_words_in_sentence", 20); + + sentences = MergeShortSentencesByWords(sentences, min_words); + + std::vector final_chunks; + for (const auto &s : sentences) { + auto pieces = SplitLongSentenceByWords(s, max_words); + final_chunks.insert(final_chunks.end(), pieces.begin(), pieces.end()); + } + + // Merge punctuation-only chunks into previous + std::vector merged_chunks; + for (const auto &s : final_chunks) { + std::u32string u32 = Utf8ToUtf32(Trim(s)); + bool all_punct = !u32.empty(); + for (char32_t c : u32) { + if (c != U',' && c != U'.' && c != U'!' && c != U'?' && c != U';' && + c != U':' && c != U' ') { + all_punct = false; + break; + } + } + if (all_punct && !merged_chunks.empty()) { + merged_chunks.back() += Trim(s); + } else { + merged_chunks.push_back(s); + } + } + sentences = std::move(merged_chunks); + + if (debug) { + SHERPA_ONNX_LOGE("After split/merge: %d sentences", + static_cast(sentences.size())); + for (int32_t i = 0; i < static_cast(sentences.size()); ++i) { #if __OHOS__ - SHERPA_ONNX_LOGE("After normalizing: %{public}s", text.c_str()); + SHERPA_ONNX_LOGE(" sentence %d: '%{public}s'", i, + sentences[i].c_str()); #else - SHERPA_ONNX_LOGE("After normalizing: %s", text.c_str()); + SHERPA_ONNX_LOGE(" sentence %d: '%s'", i, sentences[i].c_str()); #endif } } - } - std::vector token_ids = - frontend_->ConvertTextToTokenIds(text, meta_data.voice); + for (int32_t si = 0; si < static_cast(sentences.size()); ++si) { + auto ids = TokenizeSentence(sentences[si], meta_data, debug, si); + + // Add trailing punctuation if not already present + std::string trimmed = Trim(sentences[si]); + if (!trimmed.empty()) { + char last = trimmed.back(); + std::string punct(1, last); + if (IsPunctuation(punct) && !ids.empty() && + !ids.back().tokens.empty()) { + auto it = token_str2id_.find(punct); + if (it != token_str2id_.end() && + ids.back().tokens.back() != it->second) { + ids.back().tokens.push_back(it->second); + } + } + } + + token_ids.insert(token_ids.end(), ids.begin(), ids.end()); + } + } if (token_ids.empty() || (token_ids.size() == 1 && token_ids[0].tokens.empty())) { @@ -245,96 +310,39 @@ class OfflineTtsKittenImpl : public OfflineTtsImpl { } std::vector> x; - x.reserve(token_ids.size()); - for (auto &i : token_ids) { x.push_back(std::move(i.tokens)); } + if (debug) { + for (int32_t i = 0; i < static_cast(x.size()); ++i) { + std::ostringstream os; + os << "Sentence " << i << " final IDs (" << x[i].size() << "): ["; + for (size_t j = 0; j < x[i].size(); ++j) { + if (j > 0) os << ", "; + os << x[i][j]; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + } + int32_t x_size = static_cast(x.size()); - if (config_.max_num_sentences != 1) { -#if __OHOS__ - SHERPA_ONNX_LOGE( - "max_num_sentences (%{public}d) != 1 is ignored for Kitten TTS " - "models", - config_.max_num_sentences); -#else - SHERPA_ONNX_LOGE( - "max_num_sentences (%d) != 1 is ignored for Kitten TTS models", - config_.max_num_sentences); -#endif - } - - // the input text is too long, we process sentences within it in batches - // to avoid OOM. Batch size is config_.max_num_sentences + // Kitten processes all sub-sequences as a single batch std::vector> batch_x; - - int32_t batch_size = 1; - batch_x.reserve(batch_size); - int32_t num_batches = x_size / batch_size; - - if (config_.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE( - "Split it into %{public}d batches. batch size: " - "%{public}d. Number of sentences: %{public}d", - num_batches, batch_size, x_size); -#else - SHERPA_ONNX_LOGE( - "Split it into %d batches. batch size: %d. Number " - "of sentences: %d", - num_batches, batch_size, x_size); -#endif + batch_x.reserve(x_size); + for (auto &seq : x) { + batch_x.push_back(std::move(seq)); } - GeneratedAudio ans; - - int32_t should_continue = 1; - - int32_t k = 0; - - for (int32_t b = 0; b != num_batches && should_continue; ++b) { - batch_x.clear(); - for (int32_t i = 0; i != batch_size; ++i, ++k) { - batch_x.push_back(std::move(x[k])); - } - - auto audio = Process(batch_x, sid, speed, gen_config.silence_scale); - ans.sample_rate = audio.sample_rate; - ans.samples.insert(ans.samples.end(), audio.samples.begin(), - audio.samples.end()); - if (callback) { - should_continue = callback(audio.samples.data(), audio.samples.size(), - (b + 1) * 1.0 / num_batches); - // Caution(fangjun): audio is freed when the callback returns, so users - // should copy the data if they want to access the data after - // the callback returns to avoid segmentation fault. - } - } - - batch_x.clear(); - while (k < static_cast(x.size()) && should_continue) { - batch_x.push_back(std::move(x[k])); - - ++k; - } - - if (!batch_x.empty()) { - auto audio = Process(batch_x, sid, speed, gen_config.silence_scale); - ans.sample_rate = audio.sample_rate; - ans.samples.insert(ans.samples.end(), audio.samples.begin(), - audio.samples.end()); - if (callback) { - callback(audio.samples.data(), audio.samples.size(), 1.0); - // Caution(fangjun): audio is freed when the callback returns, so users - // should copy the data if they want to access the data after - // the callback returns to avoid segmentation fault. - } - } - - return ans; + auto audio = Process(batch_x, sid, speed, gen_config.silence_scale); + return audio; } [[deprecated("Use Generate(text, GenerationConfig, callback) instead")]] @@ -349,23 +357,328 @@ class OfflineTtsKittenImpl : public OfflineTtsImpl { } private: - template - void InitFrontend(Manager *mgr) { - const auto &meta_data = model_->GetMetaData(); - frontend_ = std::make_unique( - mgr, config_.model.kitten.tokens, config_.model.kitten.data_dir, - meta_data); + std::string NormalizeText(const std::string &text, bool debug) const { + std::string result = text; + + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Raw text: %{public}s", result.c_str()); +#else + SHERPA_ONNX_LOGE("Raw text: %s", result.c_str()); +#endif + } + + if (!tn_list_.empty()) { + for (const auto &tn : tn_list_) { + result = tn->Normalize(result); + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("After normalizing: %{public}s", result.c_str()); +#else + SHERPA_ONNX_LOGE("After normalizing: %s", result.c_str()); +#endif + } + } + } + + return result; } - void InitFrontend() { - const auto &meta_data = model_->GetMetaData(); - frontend_ = std::make_unique( - config_.model.kitten.tokens, config_.model.kitten.data_dir, meta_data); + std::vector TokenizeSentence( + const std::string &text, const OfflineTtsKittenModelMetaData &meta_data, + bool debug, int32_t sentence_index = 0) const { + if (!lexicon_str_.empty()) { + return TokenizeFromLexiconStr(text, meta_data, debug, sentence_index); + } + + SHERPA_ONNX_LOGE( + "No lexicon available. " + "Please provide --kitten-lexicon."); + return {}; + } + + std::vector TokenizeFromLexiconStr( + const std::string &text, const OfflineTtsKittenModelMetaData &meta_data, + bool debug, int32_t sentence_index = 0) const { + std::vector result; + + std::string normalized = ReplacePunctuations(text); + + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("TokenizeFromLexiconStr: text='%{public}s'", + normalized.c_str()); +#else + SHERPA_ONNX_LOGE("TokenizeFromLexiconStr: text='%s'", normalized.c_str()); +#endif + } + + { + // Split into words (UTF-8 aware) + std::vector words; + std::u32string u32 = Utf8ToUtf32(normalized); + size_t current_word_start = 0; + bool in_word = false; + + auto CjkPunctToAscii = [](char32_t c) -> const char * { + if (c == U'，') return ","; + if (c == U'、') return ","; + if (c == U'。') return "."; + if (c == U'！') return "!"; + if (c == U'？') return "?"; + if (c == U'；') return ";"; + if (c == U'：') return ","; + if (c == U'…') return "..."; + return nullptr; + }; + + for (size_t ci = 0; ci < u32.size(); ++ci) { + char32_t c = u32[ci]; + if (c == U' ' || c == U'\t' || c == U'\n' || c == U'\r') { + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(" "); + } else if (c == U',' || c == U'.' || c == U'!' || c == U'?' || + c == U';' || c == U':') { + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(Utf32ToUtf8(u32.substr(ci, 1))); + } else if (CjkPunctToAscii(c)) { + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(CjkPunctToAscii(c)); + } else if (IsCJK(c)) { + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(Utf32ToUtf8(u32.substr(ci, 1))); + } else { + if (!in_word) { + current_word_start = ci; + in_word = true; + } + } + } + if (in_word) { + words.push_back(Utf32ToUtf8(u32.substr(current_word_start))); + } + + if (words.empty()) return result; + + std::vector sentence_ids; + int32_t i = 0; + int32_t n = static_cast(words.size()); + + while (i < n) { + // Skip space tokens + if (words[i] == " ") { + auto it_space = token_str2id_.find(" "); + if (it_space != token_str2id_.end()) { + sentence_ids.push_back(it_space->second); + } + ++i; + continue; + } + + // Punctuation: look up in token_str2id_ + if (IsPunctuation(words[i])) { + auto it = token_str2id_.find(words[i]); + if (it != token_str2id_.end()) { + sentence_ids.push_back(it->second); + if (debug) { + SHERPA_ONNX_LOGE("Punctuation: '%s' -> %d", words[i].c_str(), + it->second); + } + } + ++i; + continue; + } + + // Lexicon lookup (longest match) + bool found = false; + int32_t max_try = std::min(max_lexicon_phrase_len_, n - i); + for (int32_t len = max_try; len >= 1; --len) { + std::string phrase; + for (int32_t j = i; j < i + len; ++j) { + bool prev_is_cjk = + !phrase.empty() && IsCJK(Utf8ToUtf32(phrase).back()); + bool curr_is_cjk = + !words[j].empty() && IsCJK(Utf8ToUtf32(words[j]).front()); + if (j > i && !(prev_is_cjk && curr_is_cjk)) { + phrase.push_back(' '); + } + phrase += ToLowerCase(words[j]); + } + + auto it = lexicon_str_.find(phrase); + if (it != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it->second) { + phones_str += p + " "; + } + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", phrase.c_str(), + phones_str.c_str()); + } + auto ids = ConvertPhonemeStringsToIds(it->second); + sentence_ids.insert(sentence_ids.end(), ids.begin(), ids.end()); + i += len; + found = true; + break; + } + } + + if (!found) { + if (ContainsCJK(words[i])) { + std::vector chars = SplitUtf8(words[i]); + for (const auto &ch : chars) { + std::string ch_lower = ToLowerCase(ch); + auto it = lexicon_str_.find(ch_lower); + if (it != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", + ch_lower.c_str(), phones_str.c_str()); + } + auto ids = ConvertPhonemeStringsToIds(it->second); + sentence_ids.insert(sentence_ids.end(), ids.begin(), ids.end()); + } else { + SHERPA_ONNX_LOGE("OOV character skipped: '%s'", ch.c_str()); + } + } + } else { + // The word is not in the lexicon. If it contains '-', split it + // and look up each part separately, e.g. "two-thirds" -> "two" + + // "thirds". + std::vector parts = SplitStringAndTrim(words[i], '-'); + if (words[i].find('-') != std::string::npos && !parts.empty()) { + for (const auto &part : parts) { + std::string part_lower = ToLowerCase(part); + auto it2 = lexicon_str_.find(part_lower); + if (it2 != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it2->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", + part_lower.c_str(), phones_str.c_str()); + } + auto ids = ConvertPhonemeStringsToIds(it2->second); + sentence_ids.insert(sentence_ids.end(), ids.begin(), + ids.end()); + } else { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", part.c_str()); + } + } + } else { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", words[i].c_str()); + } + } + ++i; + } + } + + if (!sentence_ids.empty()) { + if (debug) { + std::ostringstream os; + os << "Sentence " << sentence_index << " tokens: ["; + for (size_t j = 0; j < sentence_ids.size(); ++j) { + if (j > 0) os << ", "; + // Look up token string from id + std::string tok_str = "?"; + for (const auto &kv : token_str2id_) { + if (kv.second == sentence_ids[j]) { + tok_str = kv.first; + break; + } + } + os << tok_str << "(" << sentence_ids[j] << ")"; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + + // Wrap with start_id and end_id + std::vector wrapped; + wrapped.push_back(meta_data.start_id); + wrapped.insert(wrapped.end(), sentence_ids.begin(), sentence_ids.end()); + wrapped.push_back(meta_data.end_id); + if (meta_data.add_pad_after_end) { + wrapped.push_back(meta_data.pad_id); + } + result.emplace_back(std::move(wrapped)); + } + } + + return result; + } + + std::vector ConvertPhonemeStringsToIds( + const std::vector &phones) const { + std::vector ans; + for (const auto &p : phones) { + auto it = token_str2id_.find(p); + if (it != token_str2id_.end()) { + ans.push_back(it->second); + } else { + SHERPA_ONNX_LOGE("Skip unknown token: '%s'", p.c_str()); + } + } + return ans; + } + + void LoadLexicon(std::istream &is, bool debug) { + auto entries = ParseLexiconFile(is, &max_lexicon_phrase_len_); + for (auto &e : entries) { + lexicon_str_[std::move(e.key)] = std::move(e.phonemes); + } + if (debug) { + SHERPA_ONNX_LOGE("Loaded lexicon: %d entries, max phrase len %d", + static_cast(lexicon_str_.size()), + max_lexicon_phrase_len_); + } + } + + static std::string ReplacePunctuations(const std::string &s) { + static const std::vector> replacements = + { + {"，", ","}, {"、", ","}, {"；", ";"}, {"：", ","}, {":", ","}, + {"。", "."}, {"？", "?"}, {"！", "!"}, {"…", "..."}, + }; + std::string result = s; + for (const auto &p : replacements) { + size_t pos = 0; + while ((pos = result.find(p.first, pos)) != std::string::npos) { + result.replace(pos, p.first.size(), p.second); + pos += p.second.size(); + } + } + return result; + } + + static bool IsPunctuation(const std::string &s) { + if (s.empty()) return false; + static const std::string puncts = ",.;:!?"; + return s.size() == 1 && puncts.find(s[0]) != std::string::npos; } GeneratedAudio Process(const std::vector> &tokens, - int32_t sid, float speed, - float silence_scale) const { + int32_t sid, float speed, float silence_scale) const { int32_t num_tokens = 0; for (const auto &k : tokens) { num_tokens += k.size(); @@ -390,7 +703,6 @@ class OfflineTtsKittenImpl : public OfflineTtsImpl { audio.GetTensorTypeAndShapeInfo().GetShape(); int64_t total = 1; - // The output shape may be (1, 1, total) or (1, total) or (total,) for (auto i : audio_shape) { total *= i; } @@ -412,7 +724,10 @@ class OfflineTtsKittenImpl : public OfflineTtsImpl { OfflineTtsConfig config_; std::unique_ptr model_; std::vector> tn_list_; - std::unique_ptr frontend_; + std::unordered_map token2id_; // for IPA codepoints + std::unordered_map token_str2id_; // for string tokens + std::unordered_map> lexicon_str_; + int32_t max_lexicon_phrase_len_ = 1; }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-kitten-model-config.cc b/sherpa-onnx/csrc/offline-tts-kitten-model-config.cc index 2534b1be2..7c17830c2 100644 --- a/sherpa-onnx/csrc/offline-tts-kitten-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-kitten-model-config.cc @@ -19,8 +19,12 @@ void OfflineTtsKittenModelConfig::Register(ParseOptions *po) { "Path to voices.bin for kitten models"); po->Register("kitten-tokens", &tokens, "Path to tokens.txt for kitten models"); + po->Register("kitten-lexicon", &lexicon, + "Path to lexicon.txt for kitten models"); po->Register("kitten-data-dir", &data_dir, - "Path to the directory containing dict for espeak-ng."); + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. " + "Use --kitten-lexicon or " + "GenerationConfig.phoneme_codepoints instead."); po->Register("kitten-length-scale", &length_scale, "Inverse of speech speed. Larger->Slower; Smaller->faster."); } @@ -56,37 +60,25 @@ bool OfflineTtsKittenModelConfig::Validate() const { return false; } - if (data_dir.empty()) { - SHERPA_ONNX_LOGE("Please provide --kitten-data-dir"); - return false; + if (!data_dir.empty()) { + SHERPA_ONNX_LOGE( + "WARNING: --kitten-data-dir is deprecated and ignored in " + "sherpa-onnx >= v2.0.0. Please use GenerationConfig.phoneme_codepoints " + "with an external phonemizer (e.g. piper_phonemize) instead. " + "See python-api-examples/test-offline-tts-kitten-phonemize.py"); } - if (!FileExists(data_dir + "/phontab")) { - SHERPA_ONNX_LOGE( - "'%s/phontab' does not exist. Please check --kitten-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/phonindex")) { - SHERPA_ONNX_LOGE( - "'%s/phonindex' does not exist. Please check --kitten-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/phondata")) { - SHERPA_ONNX_LOGE( - "'%s/phondata' does not exist. Please check --kitten-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/intonations")) { - SHERPA_ONNX_LOGE( - "'%s/intonations' does not exist. Please check --kitten-data-dir", - data_dir.c_str()); - return false; + if (!lexicon.empty()) { + std::vector files; + SplitStringToVector(lexicon, ",", false, &files); + for (const auto &f : files) { + if (!FileExists(f)) { + SHERPA_ONNX_LOGE( + "lexicon '%s' does not exist. Please re-check --kitten-lexicon", + f.c_str()); + return false; + } + } } if (length_scale <= 0) { @@ -107,6 +99,7 @@ std::string OfflineTtsKittenModelConfig::ToString() const { os << "model=\"" << model << "\", "; os << "voices=\"" << voices << "\", "; os << "tokens=\"" << tokens << "\", "; + os << "lexicon=\"" << lexicon << "\", "; os << "data_dir=\"" << data_dir << "\", "; os << "length_scale=" << length_scale << ")"; diff --git a/sherpa-onnx/csrc/offline-tts-kitten-model-config.h b/sherpa-onnx/csrc/offline-tts-kitten-model-config.h index 15d816302..81b8b2281 100644 --- a/sherpa-onnx/csrc/offline-tts-kitten-model-config.h +++ b/sherpa-onnx/csrc/offline-tts-kitten-model-config.h @@ -16,21 +16,28 @@ struct OfflineTtsKittenModelConfig { std::string voices; std::string tokens; + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon for text input, or GenerationConfig.phoneme_codepoints / + // GenerationConfig.tokens with an external phonemizer instead. std::string data_dir; // speed = 1 / length_scale float length_scale = 1.0; + std::string lexicon; + OfflineTtsKittenModelConfig() = default; OfflineTtsKittenModelConfig(const std::string &model, const std::string &voices, const std::string &tokens, - const std::string &data_dir, float length_scale) + const std::string &data_dir, float length_scale, + const std::string &lexicon) : model(model), voices(voices), tokens(tokens), data_dir(data_dir), - length_scale(length_scale) {} + length_scale(length_scale), + lexicon(lexicon) {} void Register(ParseOptions *po); bool Validate() const; diff --git a/sherpa-onnx/csrc/offline-tts-kitten-model.cc b/sherpa-onnx/csrc/offline-tts-kitten-model.cc index bc1e3ad22..94e4604fe 100644 --- a/sherpa-onnx/csrc/offline-tts-kitten-model.cc +++ b/sherpa-onnx/csrc/offline-tts-kitten-model.cc @@ -176,7 +176,12 @@ class OfflineTtsKittenModel::Impl { SHERPA_ONNX_READ_META_DATA(meta_data_.sample_rate, "sample_rate"); SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.version, "version", 1); SHERPA_ONNX_READ_META_DATA(meta_data_.num_speakers, "n_speakers"); - SHERPA_ONNX_READ_META_DATA(meta_data_.has_espeak, "has_espeak"); + // has_espeak is legacy metadata stamped by older exports. It is not + // required any more: espeak-ng is no longer used, a lexicon (or phonemes + // from an external phonemizer) is enough. Kept optional so old models + // still load. + SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.has_espeak, "has_espeak", + 1); SHERPA_ONNX_READ_META_DATA_STR_WITH_DEFAULT(meta_data_.voice, "voice", "en-us"); SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.max_token_len, @@ -186,10 +191,6 @@ class OfflineTtsKittenModel::Impl { SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.pad_id, "pad_id", 0); SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.add_pad_after_end, "add_pad_after_end", 0); - if (meta_data_.has_espeak != 1) { - SHERPA_ONNX_LOGE("It should require espeak-ng"); - SHERPA_ONNX_EXIT(-1); - } if (config_.debug) { std::vector speaker_names; diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-impl.h b/sherpa-onnx/csrc/offline-tts-kokoro-impl.h index 1a58a6e99..a2ae036e8 100644 --- a/sherpa-onnx/csrc/offline-tts-kokoro-impl.h +++ b/sherpa-onnx/csrc/offline-tts-kokoro-impl.h @@ -4,27 +4,21 @@ #ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_IMPL_H_ #define SHERPA_ONNX_CSRC_OFFLINE_TTS_KOKORO_IMPL_H_ -#include -#include #include -#include #include +#include +#include #include #include -#include "fst/extensions/far/far.h" -#include "kaldifst/csrc/kaldi-fst-io.h" -#include "kaldifst/csrc/text-normalizer.h" #include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/fst-utils.h" -#include "sherpa-onnx/csrc/kokoro-multi-lang-lexicon.h" -#include "sherpa-onnx/csrc/lexicon.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/offline-tts-frontend.h" #include "sherpa-onnx/csrc/offline-tts-impl.h" #include "sherpa-onnx/csrc/offline-tts-kokoro-model.h" -#include "sherpa-onnx/csrc/piper-phonemize-lexicon.h" +#include "sherpa-onnx/csrc/symbol-table.h" #include "sherpa-onnx/csrc/text-utils.h" +#include "sherpa-onnx/csrc/tts-text-normalizer.h" namespace sherpa_onnx { @@ -33,105 +27,88 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { explicit OfflineTtsKokoroImpl(const OfflineTtsConfig &config) : config_(config), model_(std::make_unique(config.model)) { - InitFrontend(); + if (!config.model.kokoro.tokens.empty()) { + auto is = OpenInputFile(config.model.kokoro.tokens); + token_str2id_ = ReadTokens(is); - if (!config.rule_fsts.empty()) { - std::vector files; - SplitStringToVector(config.rule_fsts, ",", false, &files); - tn_list_.reserve(files.size()); - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); -#endif + // Also load char32_t → int32_t map for IPA single-codepoint tokens + auto is2 = OpenInputFile(config.model.kokoro.tokens); + std::string line; + while (std::getline(is2, line)) { + std::istringstream iss(line); + std::string sym; + int32_t id = 0; + iss >> sym; + if (sym.empty()) continue; + if (iss.eof()) { + id = atoi(sym.c_str()); + sym = " "; + } else { + iss >> id; + } + std::u32string u32 = Utf8ToUtf32(sym); + if (u32.size() == 1) { + token2id_[u32[0]] = id; } - tn_list_.push_back(std::make_unique(f)); } } - if (!config.rule_fars.empty()) { - if (config.model.debug) { - SHERPA_ONNX_LOGE("Loading FST archives"); - } + if (!config.model.kokoro.lexicon.empty()) { std::vector files; - SplitStringToVector(config.rule_fars, ",", false, &files); - - tn_list_.reserve(files.size() + tn_list_.size()); - + SplitStringToVector(config.model.kokoro.lexicon, ",", false, &files); for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); -#endif - } - std::unique_ptr> reader( - fst::FarReader::Open(f)); - for (; !reader->Done(); reader->Next()) { - std::unique_ptr r( - fst::CastOrConvertToConstFst(reader->GetFst()->Copy())); - - tn_list_.push_back( - std::make_unique(std::move(r))); - } - } - - if (config.model.debug) { - SHERPA_ONNX_LOGE("FST archives loaded!"); + auto is = OpenInputFile(f); + LoadLexicon(is, config.model.debug); } } + + tn_list_ = LoadTextNormalizers(config.rule_fsts, config.rule_fars, + config.model.debug); } template OfflineTtsKokoroImpl(Manager *mgr, const OfflineTtsConfig &config) : config_(config), model_(std::make_unique(mgr, config.model)) { - InitFrontend(mgr); + if (!config.model.kokoro.tokens.empty()) { + auto buf = ReadFile(mgr, config.model.kokoro.tokens); + std::istringstream is(std::string(buf.data(), buf.size())); + token_str2id_ = ReadTokens(is); - if (!config.rule_fsts.empty()) { - std::vector files; - SplitStringToVector(config.rule_fsts, ",", false, &files); - tn_list_.reserve(files.size()); - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); -#endif + auto buf2 = ReadFile(mgr, config.model.kokoro.tokens); + std::istringstream is2(std::string(buf2.data(), buf2.size())); + std::string line; + while (std::getline(is2, line)) { + std::istringstream iss(line); + std::string sym; + int32_t id = 0; + iss >> sym; + if (sym.empty()) continue; + if (iss.eof()) { + id = atoi(sym.c_str()); + sym = " "; + } else { + iss >> id; + } + std::u32string u32 = Utf8ToUtf32(sym); + if (u32.size() == 1) { + token2id_[u32[0]] = id; } - auto buf = ReadFile(mgr, f); - std::istringstream is(std::string(buf.data(), buf.size())); - tn_list_.push_back(std::make_unique(is)); } } - if (!config.rule_fars.empty()) { + if (!config.model.kokoro.lexicon.empty()) { std::vector files; - SplitStringToVector(config.rule_fars, ",", false, &files); - tn_list_.reserve(files.size() + tn_list_.size()); - + SplitStringToVector(config.model.kokoro.lexicon, ",", false, &files); for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); -#endif - } - auto buf = ReadFile(mgr, f); + std::istringstream is(std::string(buf.data(), buf.size())); + LoadLexicon(is, config.model.debug); + } + } - auto fsts = ReadFstsFromFar(buf); - for (auto &r : fsts) { - tn_list_.push_back( - std::make_unique(std::move(r))); - } - } // for (const auto &f : files) - } // if (!config.rule_fars.empty()) + tn_list_ = LoadTextNormalizers(mgr, config.rule_fsts, config.rule_fars, + config.model.debug); } int32_t SampleRate() const override { @@ -142,20 +119,36 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { return model_->GetMetaData().num_speakers; } - // Supported options in GenerationConfig: - // - sid: Speaker ID for multi-speaker models - // - speed: Speech speed factor. If left at 1.0, it falls back to the - // default implied by kokoro.length_scale. - // - silence_scale: Scale applied to pauses in the generated audio. If left - // at 0.2, it falls back to OfflineTtsConfig.silence_scale. - // - // Supported extra options in config.extra: - // - lang: Language override for Kokoro >= 1.0. Defaults to - // kokoro.lang if provided, otherwise meta_data.voice. + std::string Lang() const override { return LastLang(model_->GetMetaData().voice); } + + /** + * + * Supported options in GenerationConfig: + * - sid: Speaker ID for multi-speaker models + * - speed: Speech speed factor (default: 1.0) + * - silence_scale: Scale applied to pauses in the generated audio + * - tokens: Vector of sentences, each a vector of token strings. + * Takes precedence over text when provided. + * + * Supported extra parameters: + * + * - debug, int, default from model config + * - min_words_in_sentence, int, default 5. + * Merge adjacent sentences if the number of words is less than this. + * Each CJK character counts as one word; English words are + * space-separated. + * - max_words_in_sentence, int, default 20. + * Split a sentence into chunks if the number of words exceeds this. + * Splits at punctuation or space boundaries. + * - lang, str, default from model config. + * Language override for sentence splitting. + */ GeneratedAudio Generate( const std::string &_text, const GenerationConfig &gen_config, GeneratedAudioCallback callback = nullptr) const override { - if (config_.model.debug) { + bool debug = gen_config.GetExtraInt("debug", config_.model.debug); + + if (debug) { SHERPA_ONNX_LOGE("%s", gen_config.ToString().c_str()); } @@ -199,49 +192,236 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { } std::string text = _text; - if (config_.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("Raw text: %{public}s", text.c_str()); -#else - SHERPA_ONNX_LOGE("Raw text: %s", text.c_str()); -#endif - std::ostringstream os; - os << "In bytes (hex):\n"; - const auto p = reinterpret_cast(text.c_str()); - for (int32_t i = 0; i != text.size(); ++i) { - os << std::setw(2) << std::setfill('0') << std::hex - << static_cast(p[i]) << " "; - } - os << "\n"; - -#if __OHOS__ - SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); -#else - SHERPA_ONNX_LOGE("%s", os.str().c_str()); -#endif + if (!text.empty()) { + text = NormalizeText(text, debug); } - if (!tn_list_.empty()) { - for (const auto &tn : tn_list_) { - text = tn->Normalize(text); - if (config_.model.debug) { + std::vector token_ids; + + if (!gen_config.tokens.empty()) { + // tokens path: split at punctuation, merge short, split long + // Flatten all tokens + std::vector all_tokens; + for (const auto &sentence : gen_config.tokens) { + for (const auto &tok : sentence) { + all_tokens.push_back(tok); + } + } + + // Split at punctuation tokens into sub-sentences + auto IsPunctToken = [](const std::string &t) { + return t == "," || t == "." || t == "!" || t == "?" || t == ";" || + t == ":"; + }; + + std::vector> token_sentences; + std::vector current; + for (const auto &tok : all_tokens) { + current.push_back(tok); + if (IsPunctToken(tok)) { + token_sentences.push_back(std::move(current)); + current.clear(); + } + } + if (!current.empty()) { + token_sentences.push_back(std::move(current)); + } + + // Merge short token sentences + int32_t min_words = gen_config.GetExtraInt("min_words_in_sentence", 5); + std::vector> merged; + std::vector buffer; + for (auto &ts : token_sentences) { + buffer.insert(buffer.end(), ts.begin(), ts.end()); + if (static_cast(buffer.size()) >= min_words) { + merged.push_back(std::move(buffer)); + buffer.clear(); + } + } + if (!buffer.empty()) { + if (!merged.empty()) { + merged.back().insert(merged.back().end(), buffer.begin(), + buffer.end()); + } else { + merged.push_back(std::move(buffer)); + } + } + + // Merge single-token sentences into previous + std::vector> final_sentences; + for (auto &ts : merged) { + if (ts.size() == 1 && !final_sentences.empty()) { + final_sentences.back().insert(final_sentences.back().end(), + ts.begin(), ts.end()); + } else { + final_sentences.push_back(std::move(ts)); + } + } + + // Convert token strings to IDs, wrap with PAD + int32_t si = 0; + for (const auto &sentence : final_sentences) { + std::vector ids; + ids.push_back(0); // PAD at start + for (const auto &tok : sentence) { + auto it = token_str2id_.find(tok); + if (it != token_str2id_.end()) { + ids.push_back(it->second); + } else { + // Multi-character token not found as whole — split into + // individual characters and look up each one. + std::u32string u32 = Utf8ToUtf32(tok); + bool found_sub = false; + for (char32_t c : u32) { + std::string ch = Utf32ToUtf8(std::u32string(1, c)); + auto it2 = token_str2id_.find(ch); + if (it2 != token_str2id_.end()) { + ids.push_back(it2->second); + found_sub = true; + } + } + if (!found_sub && debug) { + SHERPA_ONNX_LOGE("Skip unknown token: '%s'", tok.c_str()); + } + } + } + ids.push_back(0); // PAD at end + + if (debug) { + std::ostringstream os; + os << "Sentence " << si << " tokens: ["; + for (size_t j = 0; j < ids.size(); ++j) { + if (j > 0) os << ", "; + // Look up token string from id + std::string tok_str = "?"; + for (const auto &kv : token_str2id_) { + if (kv.second == ids[j]) { + tok_str = kv.first; + break; + } + } + os << tok_str << "(" << ids[j] << ")"; + } + os << "]"; #if __OHOS__ - SHERPA_ONNX_LOGE("After normalizing: %{public}s", text.c_str()); + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); #else - SHERPA_ONNX_LOGE("After normalizing: %s", text.c_str()); + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + + if (ids.size() > 2) { // more than just PAD + PAD + token_ids.emplace_back(std::move(ids)); + } + ++si; + } + } else if (!gen_config.phoneme_codepoints.empty()) { + // phoneme_codepoints path + int32_t si = 0; + for (const auto &sentence : gen_config.phoneme_codepoints) { + std::vector phonemes(sentence.begin(), sentence.end()); + auto ids_list = PiperPhonemesToIdsMatcha(token2id_, phonemes, false); + for (auto &ids : ids_list) { + if (debug) { + std::ostringstream os; + os << "Sentence " << si << " IPA: '"; + for (char32_t cp : phonemes) { + os << Utf32ToUtf8(std::u32string(1, cp)); + } + os << "' -> IDs: ["; + for (size_t j = 0; j < ids.size(); ++j) { + if (j > 0) os << ", "; + os << ids[j]; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + // Wrap with PAD + ids.insert(ids.begin(), 0); + ids.push_back(0); + token_ids.emplace_back(std::move(ids)); + ++si; + } + } + } else { + // Lexicon path: split text, tokenize each sentence, wrap with PAD + std::string normalized = ReplacePunctuations(text); + + auto sentences = SplitByAllPunctuation(normalized); + if (sentences.empty()) { + SHERPA_ONNX_LOGE("No sentences after splitting"); + return {}; + } + + int32_t min_words = gen_config.GetExtraInt("min_words_in_sentence", 5); + int32_t max_words = gen_config.GetExtraInt("max_words_in_sentence", 20); + + sentences = MergeShortSentencesByWords(sentences, min_words); + + std::vector final_chunks; + for (const auto &s : sentences) { + auto pieces = SplitLongSentenceByWords(s, max_words); + final_chunks.insert(final_chunks.end(), pieces.begin(), pieces.end()); + } + + // Merge punctuation-only chunks into previous + std::vector merged_chunks; + for (const auto &s : final_chunks) { + std::u32string u32 = Utf8ToUtf32(Trim(s)); + bool all_punct = !u32.empty(); + for (char32_t c : u32) { + if (c != U',' && c != U'.' && c != U'!' && c != U'?' && c != U';' && + c != U':' && c != U' ') { + all_punct = false; + break; + } + } + if (all_punct && !merged_chunks.empty()) { + merged_chunks.back() += Trim(s); + } else { + merged_chunks.push_back(s); + } + } + sentences = std::move(merged_chunks); + + if (debug) { + SHERPA_ONNX_LOGE("After split/merge: %d sentences", + static_cast(sentences.size())); + for (int32_t i = 0; i < static_cast(sentences.size()); ++i) { +#if __OHOS__ + SHERPA_ONNX_LOGE(" sentence %d: '%{public}s'", i, + sentences[i].c_str()); +#else + SHERPA_ONNX_LOGE(" sentence %d: '%s'", i, sentences[i].c_str()); #endif } } - } - std::string lang = gen_config.GetExtraString("lang"); - if (lang.empty()) { - lang = config_.model.kokoro.lang.empty() ? meta_data.voice - : config_.model.kokoro.lang; - } + for (int32_t si = 0; si < static_cast(sentences.size()); ++si) { + auto ids = TokenizeSentence(sentences[si], meta_data, debug, si); - std::vector token_ids = frontend_->ConvertTextToTokenIds( - text, lang); + // Add trailing punctuation if not already present + std::string trimmed = Trim(sentences[si]); + if (!trimmed.empty()) { + char last = trimmed.back(); + std::string punct(1, last); + if (IsPunctuation(punct) && !ids.empty() && + !ids.back().tokens.empty()) { + auto it = token_str2id_.find(punct); + if (it != token_str2id_.end() && + ids.back().tokens.back() != it->second) { + ids.back().tokens.push_back(it->second); + } + } + } + + token_ids.insert(token_ids.end(), ids.begin(), ids.end()); + } + } if (token_ids.empty() || (token_ids.size() == 1 && token_ids[0].tokens.empty())) { @@ -255,9 +435,7 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { } std::vector> x; - x.reserve(token_ids.size()); - for (auto &i : token_ids) { x.push_back(std::move(i.tokens)); } @@ -277,10 +455,7 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { #endif } - // the input text is too long, we process sentences within it in batches - // to avoid OOM. Batch size is config_.max_num_sentences std::vector> batch_x; - int32_t batch_size = 1; batch_x.reserve(config_.max_num_sentences); int32_t num_batches = x_size / batch_size; @@ -300,9 +475,7 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { } GeneratedAudio ans; - int32_t should_continue = 1; - int32_t k = 0; for (int32_t b = 0; b != num_batches && should_continue; ++b) { @@ -311,38 +484,29 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { batch_x.push_back(std::move(x[k])); } - auto audio = - Process(batch_x, sid, speed, gen_config.silence_scale); + auto audio = Process(batch_x, sid, speed, gen_config.silence_scale); ans.sample_rate = audio.sample_rate; ans.samples.insert(ans.samples.end(), audio.samples.begin(), audio.samples.end()); if (callback) { should_continue = callback(audio.samples.data(), audio.samples.size(), (b + 1) * 1.0 / num_batches); - // Caution(fangjun): audio is freed when the callback returns, so users - // should copy the data if they want to access the data after - // the callback returns to avoid segmentation fault. } } batch_x.clear(); while (k < static_cast(x.size()) && should_continue) { batch_x.push_back(std::move(x[k])); - ++k; } if (!batch_x.empty()) { - auto audio = - Process(batch_x, sid, speed, gen_config.silence_scale); + auto audio = Process(batch_x, sid, speed, gen_config.silence_scale); ans.sample_rate = audio.sample_rate; ans.samples.insert(ans.samples.end(), audio.samples.begin(), audio.samples.end()); if (callback) { callback(audio.samples.data(), audio.samples.size(), 1.0); - // Caution(fangjun): audio is freed when the callback returns, so users - // should copy the data if they want to access the data after - // the callback returns to avoid segmentation fault. } } @@ -357,69 +521,322 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { gen_config.sid = sid; gen_config.speed = speed; gen_config.silence_scale = config_.silence_scale; - if (!config_.model.kokoro.lang.empty()) { - gen_config.extra["lang"] = config_.model.kokoro.lang; - } - return Generate(text, gen_config, std::move(callback)); } private: - template - void InitFrontend(Manager *mgr) { - const auto &meta_data = model_->GetMetaData(); - - if (meta_data.version >= 2) { - // this is a multi-lingual model, we require that you pass lexicon - if (config_.model.kokoro.lexicon.empty() && - config_.model.kokoro.lang.empty()) { - SHERPA_ONNX_LOGE("Current model version: '%d'", meta_data.version); - SHERPA_ONNX_LOGE( - "You are using a multi-lingual Kokoro model (e.g., Kokoro >= " - "v1.0). Please pass --kokoro-lexicon or provide --kokoro-lang"); - SHERPA_ONNX_EXIT(-1); - } - - frontend_ = std::make_unique( - mgr, config_.model.kokoro.tokens, config_.model.kokoro.lexicon, - config_.model.kokoro.data_dir, meta_data, config_.model.debug); - - return; + std::string NormalizeText(const std::string &text, bool debug) const { + std::string result = text; + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Raw text: %{public}s", result.c_str()); +#else + SHERPA_ONNX_LOGE("Raw text: %s", result.c_str()); +#endif } - - frontend_ = std::make_unique( - mgr, config_.model.kokoro.tokens, config_.model.kokoro.data_dir, - meta_data); + if (!tn_list_.empty()) { + for (const auto &tn : tn_list_) { + result = tn->Normalize(result); + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("After normalizing: %{public}s", result.c_str()); +#else + SHERPA_ONNX_LOGE("After normalizing: %s", result.c_str()); +#endif + } + } + } + return result; } - void InitFrontend() { - const auto &meta_data = model_->GetMetaData(); - if (meta_data.version >= 2) { - // this is a multi-lingual model, we require that you pass lexicon - if (config_.model.kokoro.lexicon.empty() && - config_.model.kokoro.lang.empty()) { - SHERPA_ONNX_LOGE("Current model version: '%d'", meta_data.version); - SHERPA_ONNX_LOGE( - "You are using a multi-lingual Kokoro model (e.g., Kokoro >= " - "v1.0). please pass --kokoro-lexicon or --kokoro-lang"); - SHERPA_ONNX_EXIT(-1); - } - - frontend_ = std::make_unique( - config_.model.kokoro.tokens, config_.model.kokoro.lexicon, - config_.model.kokoro.data_dir, meta_data, config_.model.debug); - - return; + std::vector TokenizeSentence( + const std::string &text, const OfflineTtsKokoroModelMetaData &meta_data, + bool debug, int32_t sentence_index = 0) const { + if (!lexicon_str_.empty()) { + return TokenizeFromLexiconStr(text, meta_data, debug, sentence_index); } - // this is for kokoro v0.19, which supports only English - frontend_ = std::make_unique( - config_.model.kokoro.tokens, config_.model.kokoro.data_dir, meta_data); + SHERPA_ONNX_LOGE( + "No lexicon available. " + "Please provide --kokoro-lexicon."); + return {}; + } + + std::vector TokenizeFromLexiconStr( + const std::string &text, const OfflineTtsKokoroModelMetaData &meta_data, + bool debug, int32_t sentence_index = 0) const { + std::vector result; + + std::string normalized = ReplacePunctuations(text); + + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("TokenizeFromLexiconStr: text='%{public}s'", + normalized.c_str()); +#else + SHERPA_ONNX_LOGE("TokenizeFromLexiconStr: text='%s'", normalized.c_str()); +#endif + } + + { + std::vector words; + std::u32string u32 = Utf8ToUtf32(normalized); + size_t current_word_start = 0; + bool in_word = false; + + auto CjkPunctToAscii = [](char32_t c) -> const char * { + if (c == U'，') return ","; + if (c == U'、') return ","; + if (c == U'。') return "."; + if (c == U'！') return "!"; + if (c == U'？') return "?"; + if (c == U'；') return ";"; + if (c == U'：') return ","; + if (c == U'…') return "..."; + return nullptr; + }; + + for (size_t ci = 0; ci < u32.size(); ++ci) { + char32_t c = u32[ci]; + if (c == U' ' || c == U'\t' || c == U'\n' || c == U'\r') { + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(" "); + } else if (c == U',' || c == U'.' || c == U'!' || c == U'?' || + c == U';' || c == U':') { + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(Utf32ToUtf8(u32.substr(ci, 1))); + } else if (CjkPunctToAscii(c)) { + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(CjkPunctToAscii(c)); + } else if (IsCJK(c)) { + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(Utf32ToUtf8(u32.substr(ci, 1))); + } else { + if (!in_word) { + current_word_start = ci; + in_word = true; + } + } + } + if (in_word) { + words.push_back(Utf32ToUtf8(u32.substr(current_word_start))); + } + + if (words.empty()) return result; + + std::vector sentence_ids; + int32_t i = 0; + int32_t n = static_cast(words.size()); + + while (i < n) { + // Skip space tokens + if (words[i] == " ") { + auto it_space = token_str2id_.find(" "); + if (it_space != token_str2id_.end()) { + sentence_ids.push_back(it_space->second); + } + ++i; + continue; + } + + // Punctuation: look up in token_str2id_ + if (IsPunctuation(words[i])) { + auto it = token_str2id_.find(words[i]); + if (it != token_str2id_.end()) { + sentence_ids.push_back(it->second); + if (debug) { + SHERPA_ONNX_LOGE("Punctuation: '%s' -> %d", words[i].c_str(), + it->second); + } + } + ++i; + continue; + } + + // Lexicon lookup (longest match) + bool found = false; + int32_t max_try = std::min(max_lexicon_phrase_len_, n - i); + for (int32_t len = max_try; len >= 1; --len) { + std::string phrase; + for (int32_t j = i; j < i + len; ++j) { + bool prev_is_cjk = + !phrase.empty() && IsCJK(Utf8ToUtf32(phrase).back()); + bool curr_is_cjk = + !words[j].empty() && IsCJK(Utf8ToUtf32(words[j]).front()); + if (j > i && !(prev_is_cjk && curr_is_cjk)) { + phrase.push_back(' '); + } + phrase += ToLowerCase(words[j]); + } + + auto it = lexicon_str_.find(phrase); + if (it != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", phrase.c_str(), + phones_str.c_str()); + } + auto ids = ConvertPhonemeStringsToIds(it->second); + sentence_ids.insert(sentence_ids.end(), ids.begin(), ids.end()); + i += len; + found = true; + break; + } + } + + if (!found) { + if (ContainsCJK(words[i])) { + std::vector chars = SplitUtf8(words[i]); + for (const auto &ch : chars) { + std::string ch_lower = ToLowerCase(ch); + auto it = lexicon_str_.find(ch_lower); + if (it != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", + ch_lower.c_str(), phones_str.c_str()); + } + auto ids = ConvertPhonemeStringsToIds(it->second); + sentence_ids.insert(sentence_ids.end(), ids.begin(), ids.end()); + } else { + SHERPA_ONNX_LOGE("OOV character skipped: '%s'", ch.c_str()); + } + } + } else { + // The word is not in the lexicon. If it contains '-', split it + // and look up each part separately, e.g. "two-thirds" -> "two" + + // "thirds". + std::vector parts = SplitStringAndTrim(words[i], '-'); + if (words[i].find('-') != std::string::npos && !parts.empty()) { + for (const auto &part : parts) { + std::string part_lower = ToLowerCase(part); + auto it2 = lexicon_str_.find(part_lower); + if (it2 != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it2->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", + part_lower.c_str(), phones_str.c_str()); + } + auto ids = ConvertPhonemeStringsToIds(it2->second); + sentence_ids.insert(sentence_ids.end(), ids.begin(), + ids.end()); + } else { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", part.c_str()); + } + } + } else { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", words[i].c_str()); + } + } + ++i; + } + } + + if (!sentence_ids.empty()) { + if (debug) { + std::ostringstream os; + os << "Sentence " << sentence_index << " tokens: ["; + for (size_t j = 0; j < sentence_ids.size(); ++j) { + if (j > 0) os << ", "; + std::string tok_str = "?"; + for (const auto &kv : token_str2id_) { + if (kv.second == sentence_ids[j]) { + tok_str = kv.first; + break; + } + } + os << tok_str << "(" << sentence_ids[j] << ")"; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + + // Wrap with PAD (kokoro requires PAD at start and end) + std::vector wrapped; + wrapped.push_back(0); // PAD + wrapped.insert(wrapped.end(), sentence_ids.begin(), sentence_ids.end()); + wrapped.push_back(0); // PAD + result.emplace_back(std::move(wrapped)); + } + } + + return result; + } + + std::vector ConvertPhonemeStringsToIds( + const std::vector &phones) const { + std::vector ans; + for (const auto &p : phones) { + auto it = token_str2id_.find(p); + if (it != token_str2id_.end()) { + ans.push_back(it->second); + } else { + SHERPA_ONNX_LOGE("Skip unknown token: '%s'", p.c_str()); + } + } + return ans; + } + + void LoadLexicon(std::istream &is, bool debug) { + auto entries = ParseLexiconFile(is, &max_lexicon_phrase_len_); + for (auto &e : entries) { + lexicon_str_[std::move(e.key)] = std::move(e.phonemes); + } + if (debug) { + SHERPA_ONNX_LOGE("Loaded lexicon: %d entries, max phrase len %d", + static_cast(lexicon_str_.size()), + max_lexicon_phrase_len_); + } + } + + static std::string ReplacePunctuations(const std::string &s) { + static const std::vector> replacements = + { + {"，", ","}, {"、", ","}, {"；", ";"}, {"：", ","}, {":", ","}, + {"。", "."}, {"？", "?"}, {"！", "!"}, {"…", "..."}, + }; + std::string result = s; + for (const auto &p : replacements) { + size_t pos = 0; + while ((pos = result.find(p.first, pos)) != std::string::npos) { + result.replace(pos, p.first.size(), p.second); + pos += p.second.size(); + } + } + return result; + } + + static bool IsPunctuation(const std::string &s) { + if (s.empty()) return false; + static const std::string puncts = ",.;:!?"; + return s.size() == 1 && puncts.find(s[0]) != std::string::npos; } GeneratedAudio Process(const std::vector> &tokens, - int32_t sid, float speed, - float silence_scale) const { + int32_t sid, float speed, float silence_scale) const { int32_t num_tokens = 0; for (const auto &k : tokens) { num_tokens += k.size(); @@ -444,7 +861,6 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { audio.GetTensorTypeAndShapeInfo().GetShape(); int64_t total = 1; - // The output shape may be (1, 1, total) or (1, total) or (total,) for (auto i : audio_shape) { total *= i; } @@ -470,7 +886,10 @@ class OfflineTtsKokoroImpl : public OfflineTtsImpl { OfflineTtsConfig config_; std::unique_ptr model_; std::vector> tn_list_; - std::unique_ptr frontend_; + std::unordered_map token2id_; // for IPA codepoints + std::unordered_map token_str2id_; // for string tokens + std::unordered_map> lexicon_str_; + int32_t max_lexicon_phrase_len_ = 1; }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-model-config.cc b/sherpa-onnx/csrc/offline-tts-kokoro-model-config.cc index 5082158ba..0fc4f2590 100644 --- a/sherpa-onnx/csrc/offline-tts-kokoro-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-kokoro-model-config.cc @@ -32,7 +32,9 @@ void OfflineTtsKokoroModelConfig::Register(ParseOptions *po) { "You can pass multiple files, separated by ','. Example: " "./lexicon-us-en.txt,./lexicon-zh.txt"); po->Register("kokoro-data-dir", &data_dir, - "Path to the directory containing dict for espeak-ng."); + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. " + "Use --kokoro-lexicon or " + "GenerationConfig.phoneme_codepoints instead."); po->Register("kokoro-dict-dir", &dict_dir, "Not used. You don't need to provide a value for it"); po->Register("kokoro-length-scale", &length_scale, @@ -83,37 +85,19 @@ bool OfflineTtsKokoroModelConfig::Validate() const { } } - if (data_dir.empty()) { - SHERPA_ONNX_LOGE("Please provide --kokoro-data-dir"); - return false; + if (!data_dir.empty()) { + SHERPA_ONNX_LOGE( + "WARNING: --kokoro-data-dir is deprecated and ignored in " + "sherpa-onnx >= v2.0.0. Please use --kokoro-lexicon or " + "GenerationConfig.phoneme_codepoints with an external " + "phonemizer (e.g. piper_phonemize) instead."); } - if (!FileExists(data_dir + "/phontab")) { + if (!lang.empty()) { SHERPA_ONNX_LOGE( - "'%s/phontab' does not exist. Please check --kokoro-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/phonindex")) { - SHERPA_ONNX_LOGE( - "'%s/phonindex' does not exist. Please check --kokoro-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/phondata")) { - SHERPA_ONNX_LOGE( - "'%s/phondata' does not exist. Please check --kokoro-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/intonations")) { - SHERPA_ONNX_LOGE( - "'%s/intonations' does not exist. Please check --kokoro-data-dir", - data_dir.c_str()); - return false; + "WARNING: --kokoro-lang is deprecated and ignored in " + "sherpa-onnx >= v2.0.0. espeak-ng is no longer used. " + "Language is determined by the lexicon files you provide."); } if (!dict_dir.empty()) { diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-model-config.h b/sherpa-onnx/csrc/offline-tts-kokoro-model-config.h index 573b283fc..671d9c127 100644 --- a/sherpa-onnx/csrc/offline-tts-kokoro-model-config.h +++ b/sherpa-onnx/csrc/offline-tts-kokoro-model-config.h @@ -20,6 +20,9 @@ struct OfflineTtsKokoroModelConfig { // Example: lexicon = "./lexicon-gb-en.txt,./lexicon-zh.txt"; std::string lexicon; + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon for text input, or GenerationConfig.phoneme_codepoints / + // GenerationConfig.tokens with an external phonemizer instead. std::string data_dir; std::string dict_dir; diff --git a/sherpa-onnx/csrc/offline-tts-kokoro-model.cc b/sherpa-onnx/csrc/offline-tts-kokoro-model.cc index e473b37a8..68d67ec12 100644 --- a/sherpa-onnx/csrc/offline-tts-kokoro-model.cc +++ b/sherpa-onnx/csrc/offline-tts-kokoro-model.cc @@ -149,7 +149,10 @@ class OfflineTtsKokoroModel::Impl { SHERPA_ONNX_READ_META_DATA(meta_data_.sample_rate, "sample_rate"); SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.version, "version", 1); SHERPA_ONNX_READ_META_DATA(meta_data_.num_speakers, "n_speakers"); - SHERPA_ONNX_READ_META_DATA(meta_data_.has_espeak, "has_espeak"); + // has_espeak is legacy metadata and is unused: espeak-ng is no longer + // used. Read optionally so models without the key still load. + SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.has_espeak, "has_espeak", + 1); SHERPA_ONNX_READ_META_DATA_STR_WITH_DEFAULT(meta_data_.voice, "voice", "en-us"); diff --git a/sherpa-onnx/csrc/offline-tts-matcha-impl.h b/sherpa-onnx/csrc/offline-tts-matcha-impl.h index f12b3e503..9bde09a87 100644 --- a/sherpa-onnx/csrc/offline-tts-matcha-impl.h +++ b/sherpa-onnx/csrc/offline-tts-matcha-impl.h @@ -6,27 +6,21 @@ #include #include -#include #include +#include +#include #include #include -#include "fst/extensions/far/far.h" -#include "kaldifst/csrc/kaldi-fst-io.h" -#include "kaldifst/csrc/text-normalizer.h" -#include "sherpa-onnx/csrc/character-lexicon.h" -#include "sherpa-onnx/csrc/fst-utils.h" -#include "sherpa-onnx/csrc/lexicon.h" +#include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/macros.h" -#include "sherpa-onnx/csrc/matcha-tts-lexicon.h" -#include "sherpa-onnx/csrc/melo-tts-lexicon.h" -#include "sherpa-onnx/csrc/offline-tts-character-frontend.h" #include "sherpa-onnx/csrc/offline-tts-frontend.h" #include "sherpa-onnx/csrc/offline-tts-impl.h" #include "sherpa-onnx/csrc/offline-tts-matcha-model.h" #include "sherpa-onnx/csrc/onnx-utils.h" -#include "sherpa-onnx/csrc/piper-phonemize-lexicon.h" +#include "sherpa-onnx/csrc/symbol-table.h" #include "sherpa-onnx/csrc/text-utils.h" +#include "sherpa-onnx/csrc/tts-text-normalizer.h" #include "sherpa-onnx/csrc/vocoder.h" namespace sherpa_onnx { @@ -55,68 +49,68 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { "You don't need to provide vocoder for this model. Ignore it"); } - InitFrontend(); + { + auto is = OpenInputFile(config.model.matcha.tokens); + token_str2id_ = ReadTokens(is); + } - if (!config.rule_fsts.empty()) { - std::vector files; - SplitStringToVector(config.rule_fsts, ",", false, &files); - tn_list_.reserve(files.size()); - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); -#endif + // Build token2id_ (char32_t → int) from single-codepoint entries in + // tokens.txt. For IPA models all entries are single-codepoint; for + // zh-en models only the IPA entries are single-codepoint (pinyin entries + // like "zhong1" are multi-char and go into token_str2id_ only). + { + auto is = OpenInputFile(config.model.matcha.tokens); + std::string line; + while (std::getline(is, line)) { + std::istringstream iss(line); + std::string sym; + int32_t id = 0; + iss >> sym; + if (sym.empty()) continue; + if (iss.eof()) { + id = atoi(sym.c_str()); + sym = " "; + } else { + iss >> id; + } + std::u32string u32 = Utf8ToUtf32(sym); + if (u32.size() == 1) { + char32_t c = u32[0]; + if (!token2id_.count(c)) { + token2id_[c] = id; + } } - tn_list_.push_back(std::make_unique(f)); } } - if (!config.rule_fars.empty()) { - if (config.model.debug) { - SHERPA_ONNX_LOGE("Loading FST archives"); - } + if (!config.model.matcha.lexicon.empty()) { std::vector files; - SplitStringToVector(config.rule_fars, ",", false, &files); - - tn_list_.reserve(files.size() + tn_list_.size()); - + SplitStringToVector(config.model.matcha.lexicon, ",", false, &files); for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); -#endif - } - std::unique_ptr> reader( - fst::FarReader::Open(f)); - for (; !reader->Done(); reader->Next()) { - std::unique_ptr r( - fst::CastOrConvertToConstFst(reader->GetFst()->Copy())); - - tn_list_.push_back( - std::make_unique(std::move(r))); - } - } - - if (config.model.debug) { - SHERPA_ONNX_LOGE("FST archives loaded!"); + auto is = OpenInputFile(f); + LoadLexicon(is, config.model.debug); } } - if (meta_data.sample_rate == 16000 && meta_data.is_zh_en == 1) { - if (!Contains(config.model.matcha.vocoder, "16") && - Contains(config.model.matcha.vocoder, "2")) { - SHERPA_ONNX_LOGE( - "This Chinese+English TTS model requires a 16khz Vocoder."); - SHERPA_ONNX_LOGE("You should use vocos-16khz-univ.onnx."); - SHERPA_ONNX_LOGE( - "Please re-download a vocoder from " - "https://github.com/k2-fsa/sherpa-onnx/releases/tag/" - "vocoder-models."); + tn_list_ = LoadTextNormalizers(config.rule_fsts, config.rule_fars, + config.model.debug); + + if (vocoder_ && vocoder_->SampleRate() != meta_data.sample_rate) { + SHERPA_ONNX_LOGE( + "ERROR: Vocoder sample rate (%d Hz) does not match the TTS model " + "sample rate (%d Hz).", + vocoder_->SampleRate(), meta_data.sample_rate); + SHERPA_ONNX_LOGE( + "Please download the correct vocoder from " + "https://github.com/k2-fsa/sherpa-onnx/releases/tag/vocoder-models"); + if (meta_data.sample_rate == 16000) { + SHERPA_ONNX_LOGE("For 16kHz models, use vocos-16khz-univ.onnx"); + } else if (meta_data.sample_rate == 22050) { + SHERPA_ONNX_LOGE("For 22050Hz models, use vocos-22khz-univ.onnx"); + } else if (meta_data.sample_rate == 24000) { + SHERPA_ONNX_LOGE("For 24kHz models, use vocos-24khz-univ.onnx"); } + SHERPA_ONNX_EXIT(-1); } } @@ -137,62 +131,68 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { "You don't need to provide vocoder for this model. Ignore it"); } - InitFrontend(mgr); + { + auto buf = ReadFile(mgr, config.model.matcha.tokens); + std::istringstream is(std::string(buf.data(), buf.size())); + token_str2id_ = ReadTokens(is); + } - if (!config.rule_fsts.empty()) { - std::vector files; - SplitStringToVector(config.rule_fsts, ",", false, &files); - tn_list_.reserve(files.size()); - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); -#endif + { + auto buf = ReadFile(mgr, config.model.matcha.tokens); + std::istringstream is(std::string(buf.data(), buf.size())); + std::string line; + while (std::getline(is, line)) { + std::istringstream iss(line); + std::string sym; + int32_t id = 0; + iss >> sym; + if (sym.empty()) continue; + if (iss.eof()) { + id = atoi(sym.c_str()); + sym = " "; + } else { + iss >> id; + } + std::u32string u32 = Utf8ToUtf32(sym); + if (u32.size() == 1) { + char32_t c = u32[0]; + if (!token2id_.count(c)) { + token2id_[c] = id; + } } - auto buf = ReadFile(mgr, f); - std::istringstream is(std::string(buf.data(), buf.size())); - tn_list_.push_back(std::make_unique(is)); } } - if (!config.rule_fars.empty()) { + if (!config.model.matcha.lexicon.empty()) { std::vector files; - SplitStringToVector(config.rule_fars, ",", false, &files); - tn_list_.reserve(files.size() + tn_list_.size()); - + SplitStringToVector(config.model.matcha.lexicon, ",", false, &files); for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); -#endif - } - auto buf = ReadFile(mgr, f); - - auto fsts = ReadFstsFromFar(buf); - for (auto &r : fsts) { - tn_list_.push_back( - std::make_unique(std::move(r))); - } - } // for (const auto &f : files) - } // if (!config.rule_fars.empty()) - - if (meta_data.sample_rate == 16000 && meta_data.is_zh_en == 1) { - if (!Contains(config.model.matcha.vocoder, "16") && - Contains(config.model.matcha.vocoder, "2")) { - SHERPA_ONNX_LOGE( - "This Chinese+English TTS model requires a 16khz Vocoder."); - SHERPA_ONNX_LOGE("You should use vocos-16khz-univ.onnx."); - SHERPA_ONNX_LOGE( - "Please re-download a vocoder from " - "https://github.com/k2-fsa/sherpa-onnx/releases/tag/" - "vocoder-models."); + std::istringstream is(std::string(buf.data(), buf.size())); + LoadLexicon(is, config.model.debug); } } + + tn_list_ = LoadTextNormalizers(mgr, config.rule_fsts, config.rule_fars, + config.model.debug); + + if (vocoder_ && vocoder_->SampleRate() != meta_data.sample_rate) { + SHERPA_ONNX_LOGE( + "ERROR: Vocoder sample rate (%d Hz) does not match the TTS model " + "sample rate (%d Hz).", + vocoder_->SampleRate(), meta_data.sample_rate); + SHERPA_ONNX_LOGE( + "Please download the correct vocoder from " + "https://github.com/k2-fsa/sherpa-onnx/releases/tag/vocoder-models"); + if (meta_data.sample_rate == 16000) { + SHERPA_ONNX_LOGE("For 16kHz models, use vocos-16khz-univ.onnx"); + } else if (meta_data.sample_rate == 22050) { + SHERPA_ONNX_LOGE("For 22050Hz models, use vocos-22khz-univ.onnx"); + } else if (meta_data.sample_rate == 24000) { + SHERPA_ONNX_LOGE("For 24kHz models, use vocos-24khz-univ.onnx"); + } + SHERPA_ONNX_EXIT(-1); + } } int32_t SampleRate() const override { @@ -203,17 +203,35 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { return model_->GetMetaData().num_speakers; } - // Supported options in GenerationConfig: - // - sid: Speaker ID for multi-speaker models - // - speed: Speech speed factor (default: 1.0) - // - silence_scale: Scale applied to pauses in the generated audio - // - // Supported extra options in config.extra: - // - None + std::string Lang() const override { return LastLang(model_->GetMetaData().voice); } + + /** + * + * Supported options in GenerationConfig: + * - sid: Speaker ID for multi-speaker models + * - speed: Speech speed factor (default: 1.0) + * - silence_scale: Scale applied to pauses in the generated audio + * + * Supported extra parameters: + * + * - debug, int, default from model config + * - min_words_in_sentence, int, default 5. + * Merge adjacent sentences if the number of words is less than this. + * Each CJK character counts as one word; English words are + * space-separated. + * - max_words_in_sentence, int, default 20. + * Split a sentence into chunks if the number of words exceeds this. + * Splits at punctuation or space boundaries. + * - max_codepoints_in_sentence, int, default 200. + * For the phoneme_codepoints path: split a phoneme sequence if it + * exceeds this many codepoints. Splits at space boundaries. + */ GeneratedAudio Generate( const std::string &_text, const GenerationConfig &gen_config, GeneratedAudioCallback callback = nullptr) const override { - if (config_.model.debug) { + bool debug = gen_config.GetExtraInt("debug", config_.model.debug); + + if (debug) { SHERPA_ONNX_LOGE("%s", gen_config.ToString().c_str()); } @@ -257,29 +275,287 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { } std::string text = _text; - if (config_.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("Raw text: %{public}s", text.c_str()); -#else - SHERPA_ONNX_LOGE("Raw text: %s", text.c_str()); -#endif + if (!text.empty()) { + text = NormalizeText(text, debug); } - if (!tn_list_.empty()) { - for (const auto &tn : tn_list_) { - text = tn->Normalize(text); - if (config_.model.debug) { + std::vector token_ids; + + if (!gen_config.tokens.empty()) { + // String tokens path (e.g., Chinese pinyin: "zhong1", "guo2") + // Flatten all sentences, split at punctuation, merge short, split long + std::vector all_tokens; + for (const auto &sentence : gen_config.tokens) { + for (const auto &tok : sentence) { + all_tokens.push_back(tok); + } + } + + // Split at punctuation tokens into sub-sentences + auto IsPunctToken = [](const std::string &t) { + return t == "," || t == "." || t == "!" || t == "?" || t == ";" || + t == ":"; + }; + + std::vector> token_sentences; + std::vector current; + for (const auto &tok : all_tokens) { + current.push_back(tok); + if (IsPunctToken(tok)) { + token_sentences.push_back(std::move(current)); + current.clear(); + } + } + if (!current.empty()) { + token_sentences.push_back(std::move(current)); + } + + // Merge short token sentences + int32_t min_words = gen_config.GetExtraInt("min_words_in_sentence", 5); + std::vector> merged; + std::vector buffer; + for (auto &ts : token_sentences) { + buffer.insert(buffer.end(), ts.begin(), ts.end()); + if (static_cast(buffer.size()) >= min_words) { + merged.push_back(std::move(buffer)); + buffer.clear(); + } + } + if (!buffer.empty()) { + if (!merged.empty()) { + merged.back().insert(merged.back().end(), buffer.begin(), + buffer.end()); + } else { + merged.push_back(std::move(buffer)); + } + } + + // Convert token strings to IDs, merge single-token sentences + int32_t sentence_idx = 0; + for (const auto &sentence : merged) { + std::vector ids; + std::vector id_strs; + for (const auto &tok : sentence) { + auto it = token_str2id_.find(tok); + if (it != token_str2id_.end()) { + ids.push_back(it->second); + id_strs.push_back(tok + "(" + std::to_string(it->second) + ")"); + } else { + SHERPA_ONNX_LOGE("Skip unknown token: '%s'", tok.c_str()); + } + } + if (ids.empty()) continue; + + if (debug) { + std::ostringstream os; + os << "Sentence " << sentence_idx << " tokens: ["; + for (size_t j = 0; j < id_strs.size(); ++j) { + if (j > 0) os << ", "; + os << id_strs[j]; + } + os << "]"; #if __OHOS__ - SHERPA_ONNX_LOGE("After normalizing: %{public}s", text.c_str()); + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); #else - SHERPA_ONNX_LOGE("After normalizing: %s", text.c_str()); + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + ++sentence_idx; + } + + if (ids.size() == 1 && !token_ids.empty()) { + // Merge single-token sentence into previous + token_ids.back().tokens.insert(token_ids.back().tokens.end(), + ids.begin(), ids.end()); + } else { + token_ids.emplace_back(std::move(ids)); + } + } + } else if (!gen_config.phoneme_codepoints.empty()) { + // phoneme_codepoints path: split at punctuation, merge short, split long + int32_t max_codepoints = + gen_config.GetExtraInt("max_codepoints_in_sentence", 200); + int32_t min_codepoints = + gen_config.GetExtraInt("min_words_in_sentence", 5) * 3; + + auto IsPunctCodepoint = [](int32_t cp) { + return cp == ',' || cp == '.' || cp == '!' || cp == '?' || cp == ';' || + cp == ':'; + }; + + // Flatten all sentences + std::vector all_codepoints; + for (const auto &sentence : gen_config.phoneme_codepoints) { + all_codepoints.insert(all_codepoints.end(), sentence.begin(), + sentence.end()); + } + + // Split at punctuation boundaries + std::vector> punct_chunks; + std::vector current; + for (int32_t cp : all_codepoints) { + current.push_back(cp); + if (IsPunctCodepoint(cp)) { + punct_chunks.push_back(std::move(current)); + current.clear(); + } + } + if (!current.empty()) { + punct_chunks.push_back(std::move(current)); + } + + // Merge short chunks + std::vector> merged; + std::vector buffer; + for (auto &chunk : punct_chunks) { + buffer.insert(buffer.end(), chunk.begin(), chunk.end()); + if (static_cast(buffer.size()) >= min_codepoints) { + merged.push_back(std::move(buffer)); + buffer.clear(); + } + } + if (!buffer.empty()) { + if (!merged.empty()) { + merged.back().insert(merged.back().end(), buffer.begin(), + buffer.end()); + } else { + merged.push_back(std::move(buffer)); + } + } + + // Split long chunks at space boundaries + std::vector> chunks; + for (auto &chunk : merged) { + if (max_codepoints > 0 && + static_cast(chunk.size()) > max_codepoints) { + std::vector piece; + for (int32_t cp : chunk) { + piece.push_back(cp); + if (cp == 0x20 && + static_cast(piece.size()) >= max_codepoints) { + chunks.push_back(std::move(piece)); + piece.clear(); + } + } + if (!piece.empty()) { + chunks.push_back(std::move(piece)); + } + } else { + chunks.push_back(std::move(chunk)); + } + } + + // Merge single-codepoint chunks into previous + std::vector> final_chunks; + for (auto &chunk : chunks) { + if (chunk.size() == 1 && !final_chunks.empty()) { + final_chunks.back().insert(final_chunks.back().end(), chunk.begin(), + chunk.end()); + } else { + final_chunks.push_back(std::move(chunk)); + } + } + chunks = std::move(final_chunks); + + if (debug) { + SHERPA_ONNX_LOGE("phoneme_codepoints: %d sentences, max_codepoints=%d", + static_cast(chunks.size()), max_codepoints); + } + + token_ids = TokenizeFromCodepoints(chunks, meta_data, debug); + } else { + // Lexicon path: convert Chinese punctuations, split text into sentences, + // merge short ones, split long ones, then tokenize each sentence. + std::string normalized = ReplacePunctuations(text); + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("After replacing punctuations: %{public}s", + normalized.c_str()); +#else + SHERPA_ONNX_LOGE("After replacing punctuations: %s", + normalized.c_str()); +#endif + } + + auto sentences = SplitByAllPunctuation(normalized); + if (sentences.empty()) { + SHERPA_ONNX_LOGE("No sentences after splitting"); + return {}; + } + + int32_t min_words_in_sentence = + gen_config.GetExtraInt("min_words_in_sentence", 5); + int32_t max_words_in_sentence = + gen_config.GetExtraInt("max_words_in_sentence", 20); + + sentences = MergeShortSentencesByWords(sentences, min_words_in_sentence); + + std::vector final_chunks; + for (const auto &s : sentences) { + auto pieces = SplitLongSentenceByWords(s, max_words_in_sentence); + final_chunks.insert(final_chunks.end(), pieces.begin(), pieces.end()); + } + + // Merge punctuation-only sentences (e.g., standalone ".") into previous + std::vector merged_chunks; + for (const auto &s : final_chunks) { + std::u32string u32 = Utf8ToUtf32(Trim(s)); + bool all_punct = !u32.empty(); + for (char32_t c : u32) { + if (c != U',' && c != U'.' && c != U'!' && c != U'?' && c != U';' && + c != U':' && c != U' ') { + all_punct = false; + break; + } + } + if (all_punct && !merged_chunks.empty()) { + merged_chunks.back() += Trim(s); + } else { + merged_chunks.push_back(s); + } + } + sentences = std::move(merged_chunks); + + if (debug) { + SHERPA_ONNX_LOGE("After split/merge: %d sentences", + static_cast(sentences.size())); + for (int32_t i = 0; i < static_cast(sentences.size()); ++i) { +#if __OHOS__ + SHERPA_ONNX_LOGE(" sentence %d: '%{public}s'", i, + sentences[i].c_str()); +#else + SHERPA_ONNX_LOGE(" sentence %d: '%s'", i, sentences[i].c_str()); #endif } } - } - std::vector token_ids = - frontend_->ConvertTextToTokenIds(text, meta_data.voice); + for (int32_t si = 0; si < static_cast(sentences.size()); ++si) { + auto ids = + TokenizeSentence(gen_config, sentences[si], meta_data, debug, si); + + // Add trailing punctuation token if the sentence ends with punctuation. + // SplitByAllPunctuation strips the punctuation character, so we need + // to add it back as a token — but only if it's not already the last + // token. + std::string trimmed = Trim(sentences[si]); + if (!trimmed.empty()) { + char last = trimmed.back(); + std::string punct(1, last); + if (IsPunctuation(punct) && !ids.empty()) { + auto it = token_str2id_.find(punct); + if (it != token_str2id_.end() && !ids.back().tokens.empty() && + ids.back().tokens.back() != it->second) { + ids.back().tokens.push_back(it->second); + if (debug) { + SHERPA_ONNX_LOGE("Trailing punctuation: '%s' -> %d", + punct.c_str(), it->second); + } + } + } + } + + token_ids.insert(token_ids.end(), ids.begin(), ids.end()); + } + } if (token_ids.empty() || (token_ids.size() == 1 && token_ids[0].tokens.empty())) { @@ -306,6 +582,23 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { } } + if (debug) { + for (int32_t i = 0; i < static_cast(x.size()); ++i) { + std::ostringstream os; + os << "Sentence " << i << " final IDs (" << x[i].size() << "): ["; + for (size_t j = 0; j < x[i].size(); ++j) { + if (j > 0) os << ", "; + os << x[i][j]; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + } + int32_t x_size = static_cast(x.size()); if (config_.max_num_sentences <= 0 || x_size <= config_.max_num_sentences) { @@ -398,55 +691,648 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { } private: - template - void InitFrontend(Manager *mgr) { - // for piper phonemizer - // we require that you copy espeak_ng_data - // from assets to disk - const auto &meta_data = model_->GetMetaData(); + std::string NormalizeText(const std::string &text, bool debug) const { + std::string result = text; - if (meta_data.is_zh_en) { - frontend_ = std::make_unique( - mgr, config_.model.matcha.lexicon, config_.model.matcha.tokens, - config_.model.matcha.data_dir, config_.model.debug, false); - } else if (meta_data.jieba) { - frontend_ = std::make_unique( - mgr, config_.model.matcha.lexicon, config_.model.matcha.tokens, - config_.model.debug); - } else if (meta_data.has_espeak) { - frontend_ = std::make_unique( - mgr, config_.model.matcha.tokens, config_.model.matcha.data_dir, - meta_data); - } else { - SHERPA_ONNX_LOGE("Unsupported matcha tts model. Please ask for help"); - SHERPA_ONNX_EXIT(-1); + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Raw text: %{public}s", result.c_str()); +#else + SHERPA_ONNX_LOGE("Raw text: %s", result.c_str()); +#endif + } + + if (!tn_list_.empty()) { + for (const auto &tn : tn_list_) { + result = tn->Normalize(result); + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("After normalizing: %{public}s", result.c_str()); +#else + SHERPA_ONNX_LOGE("After normalizing: %s", result.c_str()); +#endif + } + } + } + + return result; + } + + std::vector TokenizeSentence( + const GenerationConfig &gen_config, const std::string &text, + const OfflineTtsMatchaModelMetaData &meta_data, bool debug, + int32_t sentence_index = 0) const { + if (!lexicon_.empty() && lexicon_str_.empty()) { + // Pure IPA model (e.g., en with || format lexicon only) + auto codepoints = TokenizeFromLexicon(text, debug, sentence_index); + return TokenizeFromCodepoints(codepoints, meta_data, debug); + } + + if (!lexicon_str_.empty()) { + // Mixed model (e.g., zh-en with both zh space-format and en || format) + // or pure zh model with space-format lexicon. + // TokenizeFromLexiconStr handles both IPA and pinyin entries correctly. + return TokenizeFromLexiconStr(text, meta_data, debug, sentence_index); + } + + SHERPA_ONNX_LOGE( + "No lexicon available. " + "Please provide a lexicon via --matcha-lexicon."); + return {}; + } + + std::vector TokenizeFromCodepoints( + const std::vector> &codepoints, + const OfflineTtsMatchaModelMetaData &meta_data, + bool debug = false) const { + std::vector result; + + if (!token2id_.empty()) { + // IPA single-codepoint path (e.g., en model with || format lexicon) + result.reserve(codepoints.size()); + for (const auto &phonemes : codepoints) { + std::vector phonemes_c32(phonemes.begin(), phonemes.end()); + auto ids_list = PiperPhonemesToIdsMatcha(token2id_, phonemes_c32, + meta_data.use_eos_bos); + for (auto &ids : ids_list) { + if (debug) { + std::ostringstream os; + os << "IPA tokens: ["; + // Build phoneme string for display + std::string phonemes_str; + for (int32_t cp : phonemes) { + phonemes_str += Utf32ToUtf8(static_cast(cp)); + } + os << phonemes_str << "] -> IDs: ["; + for (size_t j = 0; j < ids.size(); ++j) { + if (j > 0) os << ", "; + os << ids[j]; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + result.emplace_back(std::move(ids)); + } + } + return result; + } + + if (!token_str2id_.empty()) { + // Multi-character token path (e.g., zh model with pinyin tokens). + // Codepoints are ASCII characters that form token strings like "dang1". + // Split into non-space runs, look up each run in token_str2id_. + for (const auto &phonemes : codepoints) { + std::vector ids; + std::string current; + for (int32_t cp : phonemes) { + if (cp == ' ') { + if (!current.empty()) { + auto it = token_str2id_.find(current); + if (it != token_str2id_.end()) { + ids.push_back(it->second); + } else { + SHERPA_ONNX_LOGE("Skip unknown token: '%s'", current.c_str()); + } + current.clear(); + } + } else { + current.push_back(static_cast(cp)); + } + } + if (!current.empty()) { + auto it = token_str2id_.find(current); + if (it != token_str2id_.end()) { + ids.push_back(it->second); + } else { + SHERPA_ONNX_LOGE("Skip unknown token: '%s'", current.c_str()); + } + } + + if (!ids.empty()) { + result.emplace_back(std::move(ids)); + } + } + return result; + } + + SHERPA_ONNX_LOGE("No token mapping available for codepoints."); + return {}; + } + + std::vector ConvertPhonemeStringsToIds( + const std::vector &phones) const { + std::vector ans; + for (const auto &p : phones) { + auto it = token_str2id_.find(p); + if (it != token_str2id_.end()) { + ans.push_back(it->second); + } else { + SHERPA_ONNX_LOGE("Skip unknown token: '%s'", p.c_str()); + } + } + return ans; + } + + static std::string ReplacePunctuations(const std::string &s) { + // Convert Chinese punctuations to English equivalents + static const std::vector> replacements = + { + {"，", ","}, {"、", ","}, {"；", ";"}, {"：", ","}, {":", ","}, + {"。", "."}, {"？", "?"}, {"！", "!"}, {"…", "..."}, + }; + std::string result = s; + for (const auto &p : replacements) { + size_t pos = 0; + while ((pos = result.find(p.first, pos)) != std::string::npos) { + result.replace(pos, p.first.size(), p.second); + pos += p.second.size(); + } + } + return result; + } + + static bool IsPunctuation(const std::string &s) { + if (s.empty()) return false; + // ASCII punctuations and Chinese equivalents (already converted) + static const std::string puncts = ",.;:!?"; + if (s.size() == 1 && puncts.find(s[0]) != std::string::npos) return true; + return false; + } + + std::vector TokenizeFromLexiconStr( + const std::string &text, const OfflineTtsMatchaModelMetaData &meta_data, + bool debug, int32_t sentence_index = 0) const { + std::vector result; + + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("TokenizeFromLexiconStr: text='%{public}s'", + text.c_str()); +#else + SHERPA_ONNX_LOGE("TokenizeFromLexiconStr: text='%s'", text.c_str()); +#endif + } + + { + std::vector words; + std::u32string u32 = Utf8ToUtf32(text); + size_t current_word_start = 0; + bool in_word = false; + + for (size_t ci = 0; ci < u32.size(); ++ci) { + char32_t c = u32[ci]; + if (c == U' ' || c == U'\t' || c == U'\n' || c == U'\r') { + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(" "); + } else if (c == U',' || c == U'.' || c == U'!' || c == U'?' || + c == U';' || c == U':') { + // ASCII punctuation: keep as separate token + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(Utf32ToUtf8(u32.substr(ci, 1))); + } else if (IsCJK(c)) { + // CJK character: each is a separate word + if (in_word) { + words.push_back(Utf32ToUtf8( + u32.substr(current_word_start, ci - current_word_start))); + in_word = false; + } + words.push_back(Utf32ToUtf8(u32.substr(ci, 1))); + } else { + // ASCII letter/digit: accumulate into word + if (!in_word) { + current_word_start = ci; + in_word = true; + } + } + } + if (in_word) { + words.push_back(Utf32ToUtf8(u32.substr(current_word_start))); + } + + if (words.empty()) return result; + + std::vector sentence_ids; + int32_t i = 0; + int32_t n = static_cast(words.size()); + + while (i < n) { + // Space tokens: insert space token ID (used as blank between words) + if (words[i] == " ") { + auto it_space = token_str2id_.find(" "); + if (it_space != token_str2id_.end()) { + sentence_ids.push_back(it_space->second); + } + ++i; + continue; + } + + // Punctuation: look up in token_str2id_ + if (IsPunctuation(words[i])) { + auto it = token_str2id_.find(words[i]); + if (it != token_str2id_.end()) { + sentence_ids.push_back(it->second); + if (debug) { + SHERPA_ONNX_LOGE("Punctuation: '%s' -> %d", words[i].c_str(), + it->second); + } + } + ++i; + continue; + } + + // Lexicon lookup (longest match) + // For CJK characters, concatenate directly (e.g., "中国"). + // For English words, join with space (e.g., "how are"). + bool found = false; + int32_t max_try = std::min(max_lexicon_phrase_len_, n - i); + for (int32_t len = max_try; len >= 1; --len) { + std::string phrase; + for (int32_t j = i; j < i + len; ++j) { + bool prev_is_cjk = + !phrase.empty() && IsCJK(Utf8ToUtf32(phrase).back()); + bool curr_is_cjk = + !words[j].empty() && IsCJK(Utf8ToUtf32(words[j]).front()); + if (j > i && !(prev_is_cjk && curr_is_cjk)) { + phrase.push_back(' '); + } + phrase += ToLowerCase(words[j]); + } + + auto it = lexicon_str_.find(phrase); + if (it != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it->second) { + phones_str += p + " "; + } + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", phrase.c_str(), + phones_str.c_str()); + } + auto ids = ConvertPhonemeStringsToIds(it->second); + sentence_ids.insert(sentence_ids.end(), ids.begin(), ids.end()); + i += len; + found = true; + break; + } + + // Fallback: check IPA lexicon (|| format entries in lexicon_) + auto it_ipa = lexicon_.find(phrase); + if (it_ipa != lexicon_.end()) { + if (debug) { + std::string phonemes_str; + for (int32_t cp : it_ipa->second) { + phonemes_str += Utf32ToUtf8(static_cast(cp)); + } + SHERPA_ONNX_LOGE("Lexicon (IPA) matched: '%s' -> '%s'", + phrase.c_str(), phonemes_str.c_str()); + } + std::vector phonemes_c32(it_ipa->second.begin(), + it_ipa->second.end()); + auto ids_list = + PiperPhonemesToIdsMatcha(token2id_, phonemes_c32, false); + for (auto &ids : ids_list) { + sentence_ids.insert(sentence_ids.end(), ids.begin(), ids.end()); + } + i += len; + found = true; + break; + } + } + + if (!found) { + // For CJK text, split into individual characters and look up each + if (ContainsCJK(words[i])) { + std::vector chars = SplitUtf8(words[i]); + for (const auto &ch : chars) { + std::string ch_lower = ToLowerCase(ch); + auto it = lexicon_str_.find(ch_lower); + if (it != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", + ch_lower.c_str(), phones_str.c_str()); + } + auto ids = ConvertPhonemeStringsToIds(it->second); + sentence_ids.insert(sentence_ids.end(), ids.begin(), ids.end()); + } else { + SHERPA_ONNX_LOGE("OOV character skipped: '%s'", ch.c_str()); + } + } + } else { + // The word is not in the lexicon. If it contains '-', split it + // and look up each part separately, e.g. "two-thirds" -> "two" + + // "thirds". + std::vector parts = SplitStringAndTrim(words[i], '-'); + if (words[i].find('-') != std::string::npos && !parts.empty()) { + for (const auto &part : parts) { + std::string part_lower = ToLowerCase(part); + auto it2 = lexicon_str_.find(part_lower); + if (it2 != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it2->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", + part_lower.c_str(), phones_str.c_str()); + } + auto ids = ConvertPhonemeStringsToIds(it2->second); + sentence_ids.insert(sentence_ids.end(), ids.begin(), + ids.end()); + } else { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", part.c_str()); + } + } + } else { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", words[i].c_str()); + } + } + ++i; + } + } + + if (!sentence_ids.empty()) { + if (debug) { + SHERPA_ONNX_LOGE("Sentence %d: text='%s'", sentence_index, + Trim(text).c_str()); + + std::ostringstream os; + os << "Sentence " << sentence_index << " tokens: ["; + for (int32_t j = 0; j < static_cast(sentence_ids.size()); + ++j) { + if (j > 0) os << ", "; + // Look up token string from id + std::string tok_str = "?"; + for (const auto &kv : token_str2id_) { + if (kv.second == sentence_ids[j]) { + tok_str = kv.first; + break; + } + } + os << tok_str << "(" << sentence_ids[j] << ")"; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + + auto ids_list = ApplyBosEos(sentence_ids, meta_data.use_eos_bos); + for (auto &ids : ids_list) { + result.emplace_back(std::move(ids)); + } + } + } + + return result; + } + + // Split a flat ID sequence into sub-sequences with BOS/EOS, respecting + // max_token_len (same logic as PiperPhonemesToIdsMatcha but with int64_t). + static std::vector ApplyBosEos(const std::vector &ids, + bool use_eos_bos, + int32_t max_token_len = 400) { + std::vector ans; + if (!use_eos_bos) { + ans.emplace_back(ids); + return ans; + } + + // BOS and EOS are ^ and $ — use 0 as placeholder; they'll be resolved + // via token2id_ at this stage we already have integer IDs. + // Actually for matcha, BOS/EOS are handled by PiperPhonemesToIdsMatcha + // using token2id_.at(U'^') and token2id_.at(U'$'). For the string-based + // path we need the same IDs. We'll just wrap with the IDs from + // token_str2id_["^"] and token_str2id_["$"]. + // But those might not exist. Instead, just return flat — AddBlank will + // handle padding. The model expects BOS/EOS from PiperPhonemesToIdsMatcha + // but for string-based lexicon we skip BOS/EOS for simplicity since + // the zh-en model doesn't use them (use_eos_bos=0 for zh-en). + ans.emplace_back(ids); + return ans; + } + + void LoadLexicon(std::istream &is, bool debug) { + auto entries = ParseLexiconFile(is, &max_lexicon_phrase_len_); + for (auto &e : entries) { + // Detect format: if phonemes contain multi-byte IPA chars or multi-char + // strings, store as token strings. Otherwise store as IPA codepoints. + bool has_multi_char = false; + for (const auto &p : e.phonemes) { + std::u32string u32 = Utf8ToUtf32(p); + if (u32.size() != 1) { + has_multi_char = true; + break; + } + } + + if (has_multi_char) { + // Token strings (e.g., pinyin: "zhong1", "guo2") + lexicon_str_[e.key] = std::move(e.phonemes); + } else { + // IPA codepoints + std::vector codepoints; + for (const auto &p : e.phonemes) { + std::u32string u32 = Utf8ToUtf32(p); + for (char32_t cp : u32) { + codepoints.push_back(static_cast(cp)); + } + } + lexicon_[e.key] = std::move(codepoints); + } + } + if (debug) { + SHERPA_ONNX_LOGE( + "Loaded lexicon: %d new-format (IPA) entries, " + "%d old-format (token string) entries, max phrase len %d", + static_cast(lexicon_.size()), + static_cast(lexicon_str_.size()), max_lexicon_phrase_len_); } } - void InitFrontend() { - const auto &meta_data = model_->GetMetaData(); + std::vector> TokenizeFromLexicon( + const std::string &text, bool debug, int32_t sentence_index = 0) const { + std::vector> result; - if (meta_data.is_zh_en) { - frontend_ = std::make_unique( - config_.model.matcha.lexicon, config_.model.matcha.tokens, - config_.model.matcha.data_dir, config_.model.debug, false); - } else if (meta_data.jieba) { - frontend_ = std::make_unique( - config_.model.matcha.lexicon, config_.model.matcha.tokens, - config_.model.debug); - } else if (meta_data.has_espeak) { - frontend_ = std::make_unique( - config_.model.matcha.tokens, config_.model.matcha.data_dir, - meta_data); - } else { - SHERPA_ONNX_LOGE("Unsupported matcha tts model. Please ask for help"); - SHERPA_ONNX_EXIT(-1); + { + const std::string &sentence = text; + // SplitUtf8 merges ASCII into words, splits CJK into characters, + // keeps punctuation as separate tokens, keeps spaces as tokens. + std::vector words = SplitUtf8(sentence); + + if (words.empty()) return result; + + std::vector sentence_codepoints; + int32_t i = 0; + int32_t n = static_cast(words.size()); + + while (i < n) { + // Space tokens: insert space token ID + if (words[i] == " ") { + auto it_space = token_str2id_.find(" "); + if (it_space != token_str2id_.end()) { + sentence_codepoints.push_back(it_space->second); + } + ++i; + continue; + } + + // Punctuation: look up in token_str2id_ or add as codepoint + if (IsPunctuation(words[i])) { + auto it_punct = token_str2id_.find(words[i]); + if (it_punct != token_str2id_.end()) { + sentence_codepoints.push_back(it_punct->second); + } else { + // Single ASCII punctuation — add as codepoint + sentence_codepoints.push_back(static_cast(words[i][0])); + } + ++i; + continue; + } + + bool found = false; + int32_t max_try = std::min(max_lexicon_phrase_len_, n - i); + for (int32_t len = max_try; len >= 1; --len) { + std::string phrase; + for (int32_t j = i; j < i + len; ++j) { + if (j > i) phrase.push_back(' '); + phrase += ToLowerCase(words[j]); + } + + auto it = lexicon_.find(phrase); + if (it != lexicon_.end()) { + if (debug) { + std::string phonemes_str; + for (int32_t cp : it->second) { + phonemes_str += Utf32ToUtf8(static_cast(cp)); + } + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", phrase.c_str(), + phonemes_str.c_str()); + } + if (!sentence_codepoints.empty()) { + int32_t last = sentence_codepoints.back(); + if (last != ',' && last != '.' && last != '!' && last != '?' && + last != ';' && last != ':' && last != ' ') { + sentence_codepoints.push_back(static_cast(' ')); + } + } + sentence_codepoints.insert(sentence_codepoints.end(), + it->second.begin(), it->second.end()); + i += len; + found = true; + break; + } + } + + if (!found) { + // Fallback: check lexicon_str_ (old-format, token string entries) + std::string key = ToLowerCase(words[i]); + auto it_str = lexicon_str_.find(key); + if (it_str != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it_str->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon (str) matched: '%s' -> '%s'", + key.c_str(), phones_str.c_str()); + } + // Convert token strings to IDs and add as codepoints + for (const auto &tok : it_str->second) { + auto it_id = token_str2id_.find(tok); + if (it_id != token_str2id_.end()) { + sentence_codepoints.push_back(it_id->second); + } + } + } else if (ContainsCJK(words[i])) { + // CJK: split into individual characters + std::vector chars = SplitUtf8(words[i]); + for (const auto &ch : chars) { + std::string ch_lower = ToLowerCase(ch); + auto it_lex = lexicon_str_.find(ch_lower); + if (it_lex != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it_lex->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon (str) matched: '%s' -> '%s'", + ch_lower.c_str(), phones_str.c_str()); + } + for (const auto &tok : it_lex->second) { + auto it_id = token_str2id_.find(tok); + if (it_id != token_str2id_.end()) { + sentence_codepoints.push_back(it_id->second); + } + } + } else { + SHERPA_ONNX_LOGE("OOV character skipped: '%s'", ch.c_str()); + } + } + } else { + // The word is not in the lexicon. If it contains '-', split it + // and look up each part separately, e.g. "two-thirds" -> "two" + + // "thirds". + std::vector parts = SplitStringAndTrim(words[i], '-'); + if (words[i].find('-') != std::string::npos && !parts.empty()) { + for (const auto &part : parts) { + std::string part_lower = ToLowerCase(part); + auto it2 = lexicon_str_.find(part_lower); + if (it2 != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it2->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon (str) matched: '%s' -> '%s'", + part_lower.c_str(), phones_str.c_str()); + } + for (const auto &tok : it2->second) { + auto it_id = token_str2id_.find(tok); + if (it_id != token_str2id_.end()) { + sentence_codepoints.push_back(it_id->second); + } + } + } else { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", part.c_str()); + } + } + } else { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", words[i].c_str()); + } + } + ++i; + } + } + + if (!sentence_codepoints.empty()) { + if (debug) { + std::string phonemes_str; + for (int32_t cp : sentence_codepoints) { + phonemes_str += Utf32ToUtf8(static_cast(cp)); + } + SHERPA_ONNX_LOGE( + "Sentence %d: text='%s', %d codepoints, phonemes='%s'", + sentence_index, Trim(text).c_str(), + static_cast(sentence_codepoints.size()), + phonemes_str.c_str()); + } + result.push_back(std::move(sentence_codepoints)); + } } + + return result; } GeneratedAudio Process(const std::vector> &tokens, - int32_t sid, float speed, - float silence_scale) const { + int32_t sid, float speed, float silence_scale) const { int32_t num_tokens = 0; for (const auto &k : tokens) { num_tokens += k.size(); @@ -458,15 +1344,6 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { x.insert(x.end(), k.begin(), k.end()); } - if (config_.model.debug) { - std::ostringstream oss; - for (int32_t i : x) { - oss << i << ", "; - } - oss << "\n"; - SHERPA_ONNX_LOGE("%s\n", oss.str().c_str()); - } - auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); @@ -506,7 +1383,12 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl { std::unique_ptr model_; std::unique_ptr vocoder_; std::vector> tn_list_; - std::unique_ptr frontend_; + std::unordered_map token2id_; // for IPA codepoints + std::unordered_map token_str2id_; // for string tokens + std::unordered_map> lexicon_; // || format + std::unordered_map> + lexicon_str_; // space format + int32_t max_lexicon_phrase_len_ = 1; }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model-config.cc b/sherpa-onnx/csrc/offline-tts-matcha-model-config.cc index 90cf6289e..69f79b290 100644 --- a/sherpa-onnx/csrc/offline-tts-matcha-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-matcha-model-config.cc @@ -24,8 +24,9 @@ void OfflineTtsMatchaModelConfig::Register(ParseOptions *po) { po->Register("matcha-tokens", &tokens, "Path to tokens.txt for Matcha models"); po->Register("matcha-data-dir", &data_dir, - "Path to the directory containing dict for espeak-ng. If it is " - "given, --matcha-lexicon is ignored."); + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. " + "Use --matcha-lexicon or " + "GenerationConfig.phoneme_codepoints instead."); po->Register("matcha-dict-dir", &dict_dir, "Not used. You don't need to provide a value for it"); po->Register("matcha-noise-scale", &noise_scale, @@ -57,33 +58,10 @@ bool OfflineTtsMatchaModelConfig::Validate() const { } if (!data_dir.empty()) { - if (!FileExists(data_dir + "/phontab")) { - SHERPA_ONNX_LOGE( - "'%s/phontab' does not exist. Please check --matcha-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/phonindex")) { - SHERPA_ONNX_LOGE( - "'%s/phonindex' does not exist. Please check --matcha-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/phondata")) { - SHERPA_ONNX_LOGE( - "'%s/phondata' does not exist. Please check --matcha-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/intonations")) { - SHERPA_ONNX_LOGE( - "'%s/intonations' does not exist. Please check --matcha-data-dir", - data_dir.c_str()); - return false; - } + SHERPA_ONNX_LOGE( + "WARNING: --matcha-data-dir is deprecated. Please use " + "--matcha-lexicon or GenerationConfig.phoneme_codepoints with an " + "external phonemizer (e.g. piper_phonemize) instead."); } if (!lexicon.empty()) { diff --git a/sherpa-onnx/csrc/offline-tts-matcha-model-config.h b/sherpa-onnx/csrc/offline-tts-matcha-model-config.h index f367a7e05..30e6e9269 100644 --- a/sherpa-onnx/csrc/offline-tts-matcha-model-config.h +++ b/sherpa-onnx/csrc/offline-tts-matcha-model-config.h @@ -17,8 +17,9 @@ struct OfflineTtsMatchaModelConfig { std::string lexicon; std::string tokens; - // If data_dir is given, lexicon is ignored - // data_dir is for piper-phonemizer, which uses espeak-ng + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon for text input, or GenerationConfig.phoneme_codepoints / + // GenerationConfig.tokens with an external phonemizer instead. std::string data_dir; // Used for Chinese TTS models using jieba diff --git a/sherpa-onnx/csrc/offline-tts-supertonic-impl.cc b/sherpa-onnx/csrc/offline-tts-supertonic-impl.cc index 7046d9c0f..36d522090 100644 --- a/sherpa-onnx/csrc/offline-tts-supertonic-impl.cc +++ b/sherpa-onnx/csrc/offline-tts-supertonic-impl.cc @@ -10,13 +10,11 @@ #include #include -#include #include #include #include #include #include -#include #include #include #include @@ -97,8 +95,8 @@ SupertonicStyle ParseVoiceStyleFromBinary(const std::vector &buf) { std::memcpy(dims, buf.data(), kHeaderSize); for (int i = 0; i < 6; ++i) { if (dims[i] <= 0) { - SHERPA_ONNX_LOGE("Invalid voice style .bin: dims[%d]=%" PRId64 " <= 0", i, - dims[i]); + SHERPA_ONNX_LOGE("Invalid voice style .bin: dims[%d]=%d <= 0", i, + static_cast(dims[i])); SHERPA_ONNX_EXIT(-1); } } @@ -310,9 +308,11 @@ GeneratedAudio OfflineTtsSupertonicImpl::Process( int64_t text_seq_len = static_cast(text_ids.size()); int64_t text_mask_len = text_mask_shape[2]; if (text_seq_len != text_mask_len) { - SHERPA_ONNX_LOGE("Text sequence length mismatch: text_ids=%" PRId64 - ", text_mask=%" PRId64 ". Text: \"%s\"", - text_seq_len, text_mask_len, text.c_str()); + SHERPA_ONNX_LOGE( + "Text sequence length mismatch: text_ids=%d" + ", text_mask=%d. Text: \"%s\"", + static_cast(text_seq_len), static_cast(text_mask_len), + text.c_str()); return {}; } @@ -416,9 +416,10 @@ GeneratedAudio OfflineTtsSupertonicImpl::Process( &latent_mask_shape); int64_t latent_mask_len = latent_mask_shape[2]; if (latent_mask_len != latent_len) { - SHERPA_ONNX_LOGE("Latent mask length mismatch: expected %d, got %" PRId64 - ". Text: \"%s\"", - latent_len, latent_mask_len, text.c_str()); + SHERPA_ONNX_LOGE( + "Latent mask length mismatch: expected %d, got %d" + ". Text: \"%s\"", + latent_len, static_cast(latent_mask_len), text.c_str()); return {}; } for (int32_t b = 0; b < bsz; ++b) { diff --git a/sherpa-onnx/csrc/offline-tts-vits-impl.h b/sherpa-onnx/csrc/offline-tts-vits-impl.h index 6de312662..7323b664c 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-impl.h +++ b/sherpa-onnx/csrc/offline-tts-vits-impl.h @@ -1,21 +1,19 @@ // sherpa-onnx/csrc/offline-tts-vits-impl.h // -// Copyright (c) 2023 Xiaomi Corporation +// Copyright (c) 2023-2026 Xiaomi Corporation #ifndef SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_IMPL_H_ #define SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_IMPL_H_ +#include #include #include #include +#include #include #include -#include "fst/extensions/far/far.h" -#include "kaldifst/csrc/kaldi-fst-io.h" -#include "kaldifst/csrc/text-normalizer.h" #include "sherpa-onnx/csrc/character-lexicon.h" #include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/fst-utils.h" #include "sherpa-onnx/csrc/lexicon.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/melo-tts-lexicon.h" @@ -23,8 +21,8 @@ #include "sherpa-onnx/csrc/offline-tts-frontend.h" #include "sherpa-onnx/csrc/offline-tts-impl.h" #include "sherpa-onnx/csrc/offline-tts-vits-model.h" -#include "sherpa-onnx/csrc/piper-phonemize-lexicon.h" #include "sherpa-onnx/csrc/text-utils.h" +#include "sherpa-onnx/csrc/tts-text-normalizer.h" namespace sherpa_onnx { @@ -35,54 +33,18 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { model_(std::make_unique(config.model)) { InitFrontend(); - if (!config.rule_fsts.empty()) { - std::vector files; - SplitStringToVector(config.rule_fsts, ",", false, &files); - tn_list_.reserve(files.size()); - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); -#endif - } - tn_list_.push_back(std::make_unique(f)); - } + if (!frontend_ && !config.model.vits.tokens.empty()) { + auto is = OpenInputFile(config.model.vits.tokens); + token2id_ = ReadPiperTokens(is); } - if (!config.rule_fars.empty()) { - if (config.model.debug) { - SHERPA_ONNX_LOGE("Loading FST archives"); - } - std::vector files; - SplitStringToVector(config.rule_fars, ",", false, &files); - - tn_list_.reserve(files.size() + tn_list_.size()); - - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); -#endif - } - std::unique_ptr> reader( - fst::FarReader::Open(f)); - for (; !reader->Done(); reader->Next()) { - std::unique_ptr r( - fst::CastOrConvertToConstFst(reader->GetFst()->Copy())); - - tn_list_.push_back( - std::make_unique(std::move(r))); - } - } - - if (config.model.debug) { - SHERPA_ONNX_LOGE("FST archives loaded!"); - } + if (!frontend_ && !config.model.vits.lexicon.empty()) { + auto is = OpenInputFile(config.model.vits.lexicon); + LoadLexicon(is, config.model.debug); } + + tn_list_ = LoadTextNormalizers(config.rule_fsts, config.rule_fars, + config.model.debug); } template @@ -91,47 +53,20 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { model_(std::make_unique(mgr, config.model)) { InitFrontend(mgr); - if (!config.rule_fsts.empty()) { - std::vector files; - SplitStringToVector(config.rule_fsts, ",", false, &files); - tn_list_.reserve(files.size()); - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); -#endif - } - auto buf = ReadFile(mgr, f); - std::istringstream is(std::string(buf.data(), buf.size())); - tn_list_.push_back(std::make_unique(is)); - } + if (!frontend_ && !config.model.vits.tokens.empty()) { + auto buf = ReadFile(mgr, config.model.vits.tokens); + std::istringstream is(std::string(buf.data(), buf.size())); + token2id_ = ReadPiperTokens(is); } - if (!config.rule_fars.empty()) { - std::vector files; - SplitStringToVector(config.rule_fars, ",", false, &files); - tn_list_.reserve(files.size() + tn_list_.size()); + if (!frontend_ && !config.model.vits.lexicon.empty()) { + auto buf = ReadFile(mgr, config.model.vits.lexicon); + std::istringstream is(std::string(buf.data(), buf.size())); + LoadLexicon(is, config.model.debug); + } - for (const auto &f : files) { - if (config.model.debug) { -#if __OHOS__ - SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); -#else - SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); -#endif - } - - auto buf = ReadFile(mgr, f); - - auto fsts = ReadFstsFromFar(buf); - for (auto &r : fsts) { - tn_list_.push_back( - std::make_unique(std::move(r))); - } - } // for (const auto &f : files) - } // if (!config.rule_fars.empty()) + tn_list_ = LoadTextNormalizers(mgr, config.rule_fsts, config.rule_fars, + config.model.debug); } int32_t SampleRate() const override { @@ -142,33 +77,101 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { return model_->GetMetaData().num_speakers; } - // Supported options in GenerationConfig: - // - sid: Speaker ID for multi-speaker models - // - speed: Speech speed factor (default: 1.0) - // - silence_scale: Scale applied to pauses in the generated audio - // - // Supported extra options in config.extra: - // - emotion_id: Emotion index for multi-emotion models (default: 0). - // Only used if the model exposes num_emotions > 0 metadata and - // an "emotion_id" input tensor. + std::string Lang() const override { return LastLang(model_->GetMetaData().voice); } + GeneratedAudio Generate( const std::string &_text, const GenerationConfig &gen_config, GeneratedAudioCallback callback = nullptr) const override { - if (config_.model.debug) { + bool debug = gen_config.GetExtraInt("debug", config_.model.debug); + + if (debug) { SHERPA_ONNX_LOGE("%s", gen_config.ToString().c_str()); } - int64_t sid = gen_config.sid; - int64_t emotion_id = gen_config.GetExtraInt("emotion_id", 0); + const auto &meta_data = model_->GetMetaData(); + float speed = gen_config.speed; if (speed <= 0) { SHERPA_ONNX_LOGE("Speed must be > 0. Given: %f", speed); return {}; } - const auto &meta_data = model_->GetMetaData(); - int32_t num_speakers = meta_data.num_speakers; + int64_t sid = ValidateSid(gen_config.sid, meta_data.num_speakers); + int64_t emotion_id = ValidateEmotionId( + gen_config.GetExtraInt("emotion_id", 0), meta_data.num_emotions); + std::string text = _text; + if (!text.empty()) { + text = NormalizeText(text, debug); + } + + std::vector token_ids = + Tokenize(gen_config, text, meta_data, debug); + if (token_ids.empty() || + (token_ids.size() == 1 && token_ids[0].tokens.empty())) { + SHERPA_ONNX_LOGE("Failed to convert '%s' to token IDs", text.c_str()); + return {}; + } + + std::vector> x; + std::vector> tones; + + x.reserve(token_ids.size()); + for (auto &i : token_ids) { + x.push_back(std::move(i.tokens)); + } + + if (!token_ids[0].tones.empty()) { + tones.reserve(token_ids.size()); + for (auto &i : token_ids) { + tones.push_back(std::move(i.tones)); + } + } + + if (gen_config.phoneme_codepoints.empty() && meta_data.add_blank && + meta_data.frontend != "characters") { + for (auto &k : x) { + k = AddBlank(k); + } + for (auto &k : tones) { + k = AddBlank(k); + } + } + + if (debug) { + for (int32_t i = 0; i < static_cast(x.size()); ++i) { + std::ostringstream os; + os << "sentence " << i << ": " << x[i].size() << " token IDs: ["; + for (int32_t j = 0; j < static_cast(x[i].size()); ++j) { + if (j > 0) os << ", "; + os << x[i][j]; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + } + + return GenerateInBatches(x, tones, sid, speed, gen_config.silence_scale, + emotion_id, debug, callback); + } + + [[deprecated("Use Generate(text, GenerationConfig, callback) instead")]] + GeneratedAudio Generate( + const std::string &text, int64_t sid = 0, float speed = 1.0, + GeneratedAudioCallback callback = nullptr) const override { + GenerationConfig gen_config; + gen_config.sid = sid; + gen_config.speed = speed; + gen_config.silence_scale = config_.silence_scale; + return Generate(text, gen_config, std::move(callback)); + } + + private: + int64_t ValidateSid(int64_t sid, int32_t num_speakers) const { if (num_speakers == 0 && sid != 0) { #if __OHOS__ SHERPA_ONNX_LOGE( @@ -198,8 +201,10 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { sid = 0; } - int32_t num_emotions = meta_data.num_emotions; + return sid; + } + int64_t ValidateEmotionId(int64_t emotion_id, int32_t num_emotions) const { if (num_emotions == 0 && emotion_id != 0) { #if __OHOS__ SHERPA_ONNX_LOGE( @@ -215,106 +220,133 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { emotion_id = 0; } - if (num_emotions != 0 && - (emotion_id >= num_emotions || emotion_id < 0)) { + if (num_emotions != 0 && (emotion_id >= num_emotions || emotion_id < 0)) { #if __OHOS__ SHERPA_ONNX_LOGE( "This model contains only %{public}d emotions. emotion_id should be " "in the range [%{public}d, %{public}d]. Given: %{public}d. Use " "emotion_id=0", - num_emotions, 0, num_emotions - 1, - static_cast(emotion_id)); + num_emotions, 0, num_emotions - 1, static_cast(emotion_id)); #else SHERPA_ONNX_LOGE( "This model contains only %d emotions. emotion_id should be in the " "range [%d, %d]. Given: %d. Use emotion_id=0", - num_emotions, 0, num_emotions - 1, - static_cast(emotion_id)); + num_emotions, 0, num_emotions - 1, static_cast(emotion_id)); #endif emotion_id = 0; } - std::string text = _text; - if (config_.model.debug) { + return emotion_id; + } + + std::string NormalizeText(const std::string &text, bool debug) const { + std::string result = text; + + if (debug) { #if __OHOS__ - SHERPA_ONNX_LOGE("Raw text: %{public}s", text.c_str()); + SHERPA_ONNX_LOGE("Raw text: %{public}s", result.c_str()); #else - SHERPA_ONNX_LOGE("Raw text: %s", text.c_str()); + SHERPA_ONNX_LOGE("Raw text: %s", result.c_str()); #endif } if (!tn_list_.empty()) { for (const auto &tn : tn_list_) { - text = tn->Normalize(text); - if (config_.model.debug) { + result = tn->Normalize(result); + if (debug) { #if __OHOS__ - SHERPA_ONNX_LOGE("After normalizing: %{public}s", text.c_str()); + SHERPA_ONNX_LOGE("After normalizing: %{public}s", result.c_str()); #else - SHERPA_ONNX_LOGE("After normalizing: %s", text.c_str()); + SHERPA_ONNX_LOGE("After normalizing: %s", result.c_str()); #endif } } } - std::vector token_ids = - frontend_->ConvertTextToTokenIds(text, meta_data.voice); + return result; + } - if (token_ids.empty() || - (token_ids.size() == 1 && token_ids[0].tokens.empty())) { - SHERPA_ONNX_LOGE("Failed to convert %s to token IDs", text.c_str()); - return {}; - } - - std::vector> x; - std::vector> tones; - - x.reserve(token_ids.size()); - - for (auto &i : token_ids) { - x.push_back(std::move(i.tokens)); - } - - if (!token_ids[0].tones.empty()) { - tones.reserve(token_ids.size()); - for (auto &i : token_ids) { - tones.push_back(std::move(i.tones)); + std::vector Tokenize(const GenerationConfig &gen_config, + const std::string &text, + const OfflineTtsVitsModelMetaData &meta_data, + bool debug) const { + if (!gen_config.tokens.empty()) { + // Pre-tokenized input given as token strings. Each inner vector is one + // sentence. VITS vocabularies key one symbol per Unicode codepoint (see + // ReadPiperTokens), so convert to codepoints and reuse the codepoint + // path: that way the bos/pad/eos markers are added here, exactly as for + // phoneme_codepoints, and callers pass bare tokens. + std::vector> codepoints; + codepoints.reserve(gen_config.tokens.size()); + for (const auto &sentence : gen_config.tokens) { + std::vector cps; + cps.reserve(sentence.size()); + for (const auto &tok : sentence) { + const std::u32string u32 = Utf8ToUtf32(tok); + if (u32.empty()) { + continue; + } + if (u32.size() != 1) { + SHERPA_ONNX_LOGE( + "Token '%s' in GenerationConfig.tokens is not a single " + "codepoint and cannot be used with this model. It is ignored.", + tok.c_str()); + continue; + } + cps.push_back(static_cast(u32[0])); + } + if (!cps.empty()) { + codepoints.emplace_back(std::move(cps)); + } } + return TokenizeFromCodepoints(codepoints); } - // TODO(fangjun): add blank inside the frontend, not here - if (meta_data.add_blank && config_.model.vits.data_dir.empty() && - meta_data.frontend != "characters") { - for (auto &k : x) { - k = AddBlank(k); - } - - for (auto &k : tones) { - k = AddBlank(k); - } + if (!gen_config.phoneme_codepoints.empty()) { + return TokenizeFromCodepoints(gen_config.phoneme_codepoints); } + if (!lexicon_.empty()) { + auto codepoints = TokenizeFromLexicon(text, debug); + return TokenizeFromCodepoints(codepoints); + } + + if (frontend_) { + return frontend_->ConvertTextToTokenIds(text, meta_data.voice); + } + + SHERPA_ONNX_LOGE( + "phoneme_codepoints is empty, no lexicon, and no frontend available. " + "Please provide phoneme_codepoints via GenerationConfig or " + "a lexicon via --vits-lexicon."); + return {}; + } + + GeneratedAudio GenerateInBatches(std::vector> &x, + std::vector> &tones, + int64_t sid, float speed, + float silence_scale, int64_t emotion_id, + bool debug, + GeneratedAudioCallback callback) const { int32_t x_size = static_cast(x.size()); if (config_.max_num_sentences <= 0 || x_size <= config_.max_num_sentences) { - auto ans = Process(x, tones, sid, speed, gen_config.silence_scale, - emotion_id); + auto ans = Process(x, tones, sid, speed, silence_scale, emotion_id); if (callback) { callback(ans.samples.data(), ans.samples.size(), 1.0); } return ans; } - // the input text is too long, we process sentences within it in batches - // to avoid OOM. Batch size is config_.max_num_sentences std::vector> batch_x; std::vector> batch_tones; int32_t batch_size = config_.max_num_sentences; - batch_x.reserve(config_.max_num_sentences); - batch_tones.reserve(config_.max_num_sentences); + batch_x.reserve(batch_size); + batch_tones.reserve(batch_size); int32_t num_batches = x_size / batch_size; - if (config_.model.debug) { + if (debug) { #if __OHOS__ SHERPA_ONNX_LOGE( "Text is too long. Split it into %{public}d batches. batch size: " @@ -329,9 +361,7 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { } GeneratedAudio ans; - int32_t should_continue = 1; - int32_t k = 0; for (int32_t b = 0; b != num_batches && should_continue; ++b) { @@ -339,105 +369,94 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { batch_tones.clear(); for (int32_t i = 0; i != batch_size; ++i, ++k) { batch_x.push_back(std::move(x[k])); - if (!tones.empty()) { batch_tones.push_back(std::move(tones[k])); } } auto audio = - Process(batch_x, batch_tones, sid, speed, gen_config.silence_scale, - emotion_id); + Process(batch_x, batch_tones, sid, speed, silence_scale, emotion_id); ans.sample_rate = audio.sample_rate; ans.samples.insert(ans.samples.end(), audio.samples.begin(), audio.samples.end()); if (callback) { should_continue = callback(audio.samples.data(), audio.samples.size(), (b + 1) * 1.0 / num_batches); - // Caution(fangjun): audio is freed when the callback returns, so users - // should copy the data if they want to access the data after - // the callback returns to avoid segmentation fault. } } batch_x.clear(); batch_tones.clear(); - while (k < static_cast(x.size()) && should_continue) { + while (k < x_size && should_continue) { batch_x.push_back(std::move(x[k])); if (!tones.empty()) { batch_tones.push_back(std::move(tones[k])); } - ++k; } if (!batch_x.empty()) { auto audio = - Process(batch_x, batch_tones, sid, speed, gen_config.silence_scale, - emotion_id); + Process(batch_x, batch_tones, sid, speed, silence_scale, emotion_id); ans.sample_rate = audio.sample_rate; ans.samples.insert(ans.samples.end(), audio.samples.begin(), audio.samples.end()); if (callback) { callback(audio.samples.data(), audio.samples.size(), 1.0); - // Caution(fangjun): audio is freed when the callback returns, so users - // should copy the data if they want to access the data after - // the callback returns to avoid segmentation fault. } } return ans; } - [[deprecated("Use Generate(text, GenerationConfig, callback) instead")]] - GeneratedAudio Generate( - const std::string &text, int64_t sid = 0, float speed = 1.0, - GeneratedAudioCallback callback = nullptr) const override { - GenerationConfig gen_config; - gen_config.sid = sid; - gen_config.speed = speed; - gen_config.silence_scale = config_.silence_scale; - return Generate(text, gen_config, std::move(callback)); + std::vector TokenizeFromCodepoints( + const std::vector> &codepoints) const { + const auto &meta_data = model_->GetMetaData(); + std::vector result; + result.reserve(codepoints.size()); + + for (const auto &phonemes : codepoints) { + std::vector phonemes_c32(phonemes.begin(), phonemes.end()); + + if (meta_data.is_piper || meta_data.is_icefall || meta_data.is_inflect) { + auto ids = PiperPhonemesToIdsVits(token2id_, phonemes_c32, + meta_data.is_inflect); + result.emplace_back(std::move(ids)); + } else if (meta_data.is_coqui) { + auto ids = CoquiPhonemesToIds(token2id_, phonemes_c32, meta_data); + result.emplace_back(std::move(ids)); + } else { + SHERPA_ONNX_LOGE( + "phoneme_codepoints not supported for this model type"); + return {}; + } + } + return result; } - private: + static bool IsNewFormatLexicon(const std::string &path) { + auto is = OpenInputFile(path); + return IsNewFormatLexiconStream(is); + } + + // Android/OHOS assets cannot be opened with ifstream, so read the file + // with the asset-aware reader before checking its format. Without this, + // a new-format lexicon shipped as an asset is misdetected as old-format + // and every entry is dropped. template - void InitFrontend(Manager *mgr) { - const auto &meta_data = model_->GetMetaData(); + static bool IsNewFormatLexicon(Manager *mgr, const std::string &path) { + auto buf = ReadFile(mgr, path); + std::istringstream is(std::string(buf.data(), buf.size())); + return IsNewFormatLexiconStream(is); + } - if (meta_data.frontend == "characters") { - frontend_ = std::make_unique( - mgr, config_.model.vits.tokens, meta_data); - } else if (meta_data.jieba && meta_data.is_melo_tts) { - frontend_ = std::make_unique( - mgr, config_.model.vits.lexicon, config_.model.vits.tokens, - model_->GetMetaData(), config_.model.debug); - } else if (meta_data.jieba || meta_data.use_g2pw) { - frontend_ = std::make_unique( - mgr, config_.model.vits.lexicon, config_.model.vits.tokens, - config_.model.debug, meta_data.use_g2pw); - } else if (meta_data.is_melo_tts && meta_data.language == "English") { - frontend_ = std::make_unique( - mgr, config_.model.vits.lexicon, config_.model.vits.tokens, - model_->GetMetaData(), config_.model.debug); - } else if ((meta_data.is_piper || meta_data.is_coqui || - meta_data.is_icefall || meta_data.is_inflect) && - !config_.model.vits.data_dir.empty()) { - frontend_ = std::make_unique( - mgr, config_.model.vits.tokens, config_.model.vits.data_dir, - meta_data); - } else { - if (config_.model.vits.lexicon.empty()) { - SHERPA_ONNX_LOGE( - "Not a model using characters as modeling unit. Please provide " - "--vits-lexicon if you leave --vits-data-dir empty"); - SHERPA_ONNX_EXIT(-1); - } - - frontend_ = std::make_unique( - mgr, config_.model.vits.lexicon, config_.model.vits.tokens, - meta_data.punctuations, meta_data.language, config_.model.debug); + static bool IsNewFormatLexiconStream(std::istream &is) { + std::string line; + while (std::getline(is, line)) { + if (line.empty() || line[0] == '#') continue; + return line.find("||") != std::string::npos; } + return false; } void InitFrontend() { @@ -458,22 +477,62 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { frontend_ = std::make_unique( config_.model.vits.lexicon, config_.model.vits.tokens, config_.model.debug, meta_data.use_g2pw); - } else if ((meta_data.is_piper || meta_data.is_coqui || - meta_data.is_icefall || meta_data.is_inflect) && - !config_.model.vits.data_dir.empty()) { - frontend_ = std::make_unique( - config_.model.vits.tokens, config_.model.vits.data_dir, - model_->GetMetaData()); - } else { - if (config_.model.vits.lexicon.empty()) { - SHERPA_ONNX_LOGE( - "Not a model using characters as modeling unit. Please provide " - "--vits-lexicon if you leave --vits-data-dir empty"); - SHERPA_ONNX_EXIT(-1); + } else if (meta_data.is_piper || meta_data.is_coqui || + meta_data.is_icefall || meta_data.is_inflect) { + // No frontend — uses phoneme_codepoints or new-format lexicon + // If an old-format lexicon is provided (no ||), fall through to Lexicon + if (!config_.model.vits.lexicon.empty() && + !IsNewFormatLexicon(config_.model.vits.lexicon)) { + frontend_ = std::make_unique( + config_.model.vits.lexicon, config_.model.vits.tokens, + meta_data.punctuations, meta_data.language, config_.model.debug); } + } else if (!config_.model.vits.lexicon.empty()) { frontend_ = std::make_unique( config_.model.vits.lexicon, config_.model.vits.tokens, meta_data.punctuations, meta_data.language, config_.model.debug); + } else { + SHERPA_ONNX_LOGE( + "No frontend available. For piper/icefall/inflect/coqui models, " + "provide phoneme_codepoints via GenerationConfig at generate time."); + } + } + + template + void InitFrontend(Manager *mgr) { + const auto &meta_data = model_->GetMetaData(); + + if (meta_data.frontend == "characters") { + frontend_ = std::make_unique( + mgr, config_.model.vits.tokens, meta_data); + } else if (meta_data.jieba && meta_data.is_melo_tts) { + frontend_ = std::make_unique( + mgr, config_.model.vits.lexicon, config_.model.vits.tokens, + model_->GetMetaData(), config_.model.debug); + } else if (meta_data.jieba || meta_data.use_g2pw) { + frontend_ = std::make_unique( + mgr, config_.model.vits.lexicon, config_.model.vits.tokens, + config_.model.debug, meta_data.use_g2pw); + } else if (meta_data.is_melo_tts && meta_data.language == "English") { + frontend_ = std::make_unique( + mgr, config_.model.vits.lexicon, config_.model.vits.tokens, + model_->GetMetaData(), config_.model.debug); + } else if (meta_data.is_piper || meta_data.is_coqui || + meta_data.is_icefall || meta_data.is_inflect) { + if (!config_.model.vits.lexicon.empty() && + !IsNewFormatLexicon(mgr, config_.model.vits.lexicon)) { + frontend_ = std::make_unique( + mgr, config_.model.vits.lexicon, config_.model.vits.tokens, + meta_data.punctuations, meta_data.language, config_.model.debug); + } + } else if (!config_.model.vits.lexicon.empty()) { + frontend_ = std::make_unique( + mgr, config_.model.vits.lexicon, config_.model.vits.tokens, + meta_data.punctuations, meta_data.language, config_.model.debug); + } else { + SHERPA_ONNX_LOGE( + "No frontend available. For piper/icefall/inflect/coqui models, " + "provide phoneme_codepoints via GenerationConfig at generate time."); } } @@ -526,7 +585,6 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { audio.GetTensorTypeAndShapeInfo().GetShape(); int64_t total = 1; - // The output shape may be (1, 1, total) or (1, total) or (total,) for (auto i : audio_shape) { total *= i; } @@ -544,12 +602,197 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl { return ans; } + void LoadLexicon(std::istream &is, bool debug) { + auto entries = ParseLexiconFile(is, &max_lexicon_phrase_len_); + for (auto &e : entries) { + std::vector codepoints; + for (const auto &p : e.phonemes) { + std::u32string u32 = Utf8ToUtf32(p); + for (char32_t cp : u32) { + codepoints.push_back(static_cast(cp)); + } + } + lexicon_[e.key] = std::move(codepoints); + } + if (debug) { + SHERPA_ONNX_LOGE("Loaded lexicon with %d entries, max phrase len %d", + static_cast(lexicon_.size()), + max_lexicon_phrase_len_); + } + } + + std::vector> TokenizeFromLexicon(const std::string &text, + bool debug) const { + std::vector> result; + + // split text into sentences by .!? + std::vector sentences; + std::string current; + for (char c : text) { + current.push_back(c); + if (c == '.' || c == '!' || c == '?') { + sentences.push_back(current); + current.clear(); + } + } + if (!current.empty()) { + sentences.push_back(current); + } + + for (const auto &sentence : sentences) { + // tokenize into words (split by space/tab, keep punctuation separate) + std::vector words; + std::string word; + for (char c : sentence) { + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + if (!word.empty()) { + words.push_back(word); + word.clear(); + } + } else if (c == ',' || c == ';' || c == ':' || c == '.' || c == '!' || + c == '?') { + if (!word.empty()) { + words.push_back(word); + word.clear(); + } + words.push_back(std::string(1, c)); + } else { + word.push_back(c); + } + } + if (!word.empty()) { + words.push_back(word); + } + + if (words.empty()) { + continue; + } + + std::vector sentence_codepoints; + int32_t i = 0; + int32_t n = static_cast(words.size()); + + while (i < n) { + // punctuation: pass through as literal codepoints + if (words[i].size() == 1) { + char c = words[i][0]; + if (c == ',' || c == '.' || c == '!' || c == '?' || c == ';' || + c == ':') { + sentence_codepoints.push_back(static_cast(c)); + ++i; + continue; + } + } + + bool found = false; + // longest match: try max_lexicon_phrase_len_ words, then fewer + int32_t max_try = std::min(max_lexicon_phrase_len_, n - i); + for (int32_t len = max_try; len >= 1; --len) { + std::string phrase; + for (int32_t j = i; j < i + len; ++j) { + if (j > i) phrase.push_back(' '); + phrase += ToLowerCase(words[j]); + } + + auto it = lexicon_.find(phrase); + if (it != lexicon_.end()) { + if (debug) { + std::string phonemes_str; + for (int32_t cp : it->second) { + phonemes_str += Utf32ToUtf8(static_cast(cp)); + } + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", phrase.c_str(), + phonemes_str.c_str()); + } + // add space between words (like espeak does) + if (!sentence_codepoints.empty()) { + int32_t last = sentence_codepoints.back(); + if (last != ',' && last != '.' && last != '!' && last != '?' && + last != ';' && last != ':' && last != ' ') { + sentence_codepoints.push_back(static_cast(' ')); + } + } + sentence_codepoints.insert(sentence_codepoints.end(), + it->second.begin(), it->second.end()); + i += len; + found = true; + break; + } + } + + if (!found) { + // The word is not in the lexicon. If it contains '-', split it and + // look up each part separately, e.g. "two-thirds" -> "two" + "thirds". + std::vector parts = SplitStringAndTrim(words[i], '-'); + if (words[i].find('-') != std::string::npos && !parts.empty()) { + bool first = true; + for (const auto &part : parts) { + auto it = lexicon_.find(ToLowerCase(part)); + if (it == lexicon_.end()) { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", part.c_str()); + continue; + } + if (debug) { + std::string phonemes_str; + for (int32_t cp : it->second) { + phonemes_str += Utf32ToUtf8(static_cast(cp)); + } + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", part.c_str(), + phonemes_str.c_str()); + } + if (first) { + // add space between words (like espeak does) + if (!sentence_codepoints.empty()) { + int32_t last = sentence_codepoints.back(); + if (last != ',' && last != '.' && last != '!' && + last != '?' && last != ';' && last != ':' && + last != ' ') { + sentence_codepoints.push_back(static_cast(' ')); + } + } + first = false; + } + // The parts of a hyphenated word form a single spoken word, so + // no extra space is inserted between them. + sentence_codepoints.insert(sentence_codepoints.end(), + it->second.begin(), it->second.end()); + } + } else { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", words[i].c_str()); + } + ++i; + } + } + + if (!sentence_codepoints.empty()) { + if (debug) { + std::string phonemes_str; + for (int32_t cp : sentence_codepoints) { + phonemes_str += Utf32ToUtf8(static_cast(cp)); + } + SHERPA_ONNX_LOGE( + "Sentence %d: text='%s', %d codepoints, phonemes='%s'", + static_cast(result.size()), Trim(sentence).c_str(), + static_cast(sentence_codepoints.size()), + phonemes_str.c_str()); + } + result.push_back(std::move(sentence_codepoints)); + } + } + + return result; + } + private: OfflineTtsConfig config_; std::unique_ptr model_; std::vector> tn_list_; + std::unordered_map token2id_; std::unique_ptr frontend_; + std::unordered_map> lexicon_; + int32_t max_lexicon_phrase_len_ = 1; }; } // namespace sherpa_onnx + #endif // SHERPA_ONNX_CSRC_OFFLINE_TTS_VITS_IMPL_H_ diff --git a/sherpa-onnx/csrc/offline-tts-vits-model-config.cc b/sherpa-onnx/csrc/offline-tts-vits-model-config.cc index 66face5cd..758d17365 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-vits-model-config.cc @@ -17,8 +17,9 @@ void OfflineTtsVitsModelConfig::Register(ParseOptions *po) { po->Register("vits-lexicon", &lexicon, "Path to lexicon.txt for VITS models"); po->Register("vits-tokens", &tokens, "Path to tokens.txt for VITS models"); po->Register("vits-data-dir", &data_dir, - "Path to the directory containing dict for espeak-ng. If it is " - "given, --vits-lexicon is ignored."); + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. " + "Use --vits-lexicon or " + "GenerationConfig.phoneme_codepoints instead."); po->Register("vits-dict-dir", &dict_dir, "Not used. You don't need to provide a value for it"); po->Register("vits-noise-scale", &noise_scale, "noise_scale for VITS models"); @@ -49,34 +50,17 @@ bool OfflineTtsVitsModelConfig::Validate() const { return false; } + if (!lexicon.empty() && !FileExists(lexicon)) { + SHERPA_ONNX_LOGE("--vits-lexicon: '%s' does not exist", lexicon.c_str()); + return false; + } + if (!data_dir.empty()) { - if (!FileExists(data_dir + "/phontab")) { - SHERPA_ONNX_LOGE( - "'%s/phontab' does not exist. Please check --vits-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/phonindex")) { - SHERPA_ONNX_LOGE( - "'%s/phonindex' does not exist. Please check --vits-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/phondata")) { - SHERPA_ONNX_LOGE( - "'%s/phondata' does not exist. Please check --vits-data-dir", - data_dir.c_str()); - return false; - } - - if (!FileExists(data_dir + "/intonations")) { - SHERPA_ONNX_LOGE( - "'%s/intonations' does not exist. Please check --vits-data-dir", - data_dir.c_str()); - return false; - } + SHERPA_ONNX_LOGE( + "WARNING: --vits-data-dir is deprecated and ignored in " + "sherpa-onnx >= v2.0.0. Please use GenerationConfig.phoneme_codepoints " + "with an external phonemizer (e.g. piper_phonemize) instead. " + "See python-api-examples/test-offline-tts-piper-phonemize.py"); } if (!dict_dir.empty()) { @@ -95,7 +79,6 @@ std::string OfflineTtsVitsModelConfig::ToString() const { os << "model=\"" << model << "\", "; os << "lexicon=\"" << lexicon << "\", "; os << "tokens=\"" << tokens << "\", "; - os << "data_dir=\"" << data_dir << "\", "; os << "noise_scale=" << noise_scale << ", "; os << "noise_scale_w=" << noise_scale_w << ", "; os << "length_scale=" << length_scale << ")"; diff --git a/sherpa-onnx/csrc/offline-tts-vits-model-config.h b/sherpa-onnx/csrc/offline-tts-vits-model-config.h index 09f00b8b2..61c16dac1 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model-config.h +++ b/sherpa-onnx/csrc/offline-tts-vits-model-config.h @@ -16,8 +16,9 @@ struct OfflineTtsVitsModelConfig { std::string lexicon; std::string tokens; - // If data_dir is given, lexicon is ignored - // data_dir is for piper-phonemize, which uses espeak-ng + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon for text input, or GenerationConfig.phoneme_codepoints / + // GenerationConfig.tokens with an external phonemizer instead. std::string data_dir; // Used for Chinese TTS models using jieba diff --git a/sherpa-onnx/csrc/offline-tts-vits-model.cc b/sherpa-onnx/csrc/offline-tts-vits-model.cc index b16b5a21f..56b5bda66 100644 --- a/sherpa-onnx/csrc/offline-tts-vits-model.cc +++ b/sherpa-onnx/csrc/offline-tts-vits-model.cc @@ -10,7 +10,6 @@ #include #include - #if __ANDROID_API__ >= 9 #include "android/asset_manager.h" #include "android/asset_manager_jni.h" @@ -227,6 +226,8 @@ class OfflineTtsVitsModel::Impl { if (comment.find("Inflect") != std::string::npos) { meta_data_.is_inflect = true; + // ignore its add_blank from the meta data + meta_data_.add_blank = 0; } } @@ -375,8 +376,8 @@ class OfflineTtsVitsModel::Impl { Ort::Value sid_tensor = Ort::Value::CreateTensor(memory_info, &sid, 1, &scale_shape, 1); - Ort::Value emotion_tensor = Ort::Value::CreateTensor( - memory_info, &emotion_id, 1, &scale_shape, 1); + Ort::Value emotion_tensor = + Ort::Value::CreateTensor(memory_info, &emotion_id, 1, &scale_shape, 1); std::vector inputs; inputs.reserve(7); diff --git a/sherpa-onnx/csrc/offline-tts-zipvoice-impl.h b/sherpa-onnx/csrc/offline-tts-zipvoice-impl.h index 499f9eb85..9795dbeaf 100644 --- a/sherpa-onnx/csrc/offline-tts-zipvoice-impl.h +++ b/sherpa-onnx/csrc/offline-tts-zipvoice-impl.h @@ -15,7 +15,6 @@ #include "kaldi-native-fbank/csrc/mel-computations.h" #include "kaldi-native-fbank/csrc/stft.h" #include "sherpa-onnx/csrc/macros.h" -#include "sherpa-onnx/csrc/matcha-tts-lexicon.h" #include "sherpa-onnx/csrc/math.h" #include "sherpa-onnx/csrc/offline-tts-frontend.h" #include "sherpa-onnx/csrc/offline-tts-impl.h" @@ -23,7 +22,9 @@ #include "sherpa-onnx/csrc/offline-tts-zipvoice-model.h" #include "sherpa-onnx/csrc/onnx-utils.h" #include "sherpa-onnx/csrc/resample.h" +#include "sherpa-onnx/csrc/symbol-table.h" #include "sherpa-onnx/csrc/text-utils.h" +#include "sherpa-onnx/csrc/tts-text-normalizer.h" #include "sherpa-onnx/csrc/vocoder.h" namespace sherpa_onnx { @@ -34,7 +35,40 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { : config_(config), model_(std::make_unique(config.model)), vocoder_(Vocoder::Create(config.model)) { - InitFrontend(); + if (vocoder_ && vocoder_->SampleRate() != model_->GetMetaData().sample_rate) { + SHERPA_ONNX_LOGE( + "ERROR: Vocoder sample rate (%d Hz) does not match the ZipVoice " + "model sample rate (%d Hz).", + vocoder_->SampleRate(), model_->GetMetaData().sample_rate); + SHERPA_ONNX_LOGE( + "Please download the correct vocoder from " + "https://github.com/k2-fsa/sherpa-onnx/releases/tag/vocoder-models"); + if (model_->GetMetaData().sample_rate == 16000) { + SHERPA_ONNX_LOGE("For 16kHz models, use vocos-16khz-univ.onnx"); + } else if (model_->GetMetaData().sample_rate == 22050) { + SHERPA_ONNX_LOGE("For 22050Hz models, use vocos-22khz-univ.onnx"); + } else if (model_->GetMetaData().sample_rate == 24000) { + SHERPA_ONNX_LOGE("For 24kHz models, use vocos_24khz.onnx"); + } + SHERPA_ONNX_EXIT(-1); + } + + if (!config.model.zipvoice.tokens.empty()) { + auto is = OpenInputFile(config.model.zipvoice.tokens); + token_str2id_ = ReadTokens(is); + } + + if (!config.model.zipvoice.lexicon.empty()) { + std::vector files; + SplitStringToVector(config.model.zipvoice.lexicon, ",", false, &files); + for (const auto &f : files) { + auto is = OpenInputFile(f); + LoadLexicon(is, config.model.debug); + } + } + + tn_list_ = LoadTextNormalizers(config.rule_fsts, config.rule_fars, + config.model.debug); PostInit(); } @@ -44,7 +78,42 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { : config_(config), model_(std::make_unique(mgr, config.model)), vocoder_(Vocoder::Create(mgr, config.model)) { - InitFrontend(mgr); + if (vocoder_ && vocoder_->SampleRate() != model_->GetMetaData().sample_rate) { + SHERPA_ONNX_LOGE( + "ERROR: Vocoder sample rate (%d Hz) does not match the ZipVoice " + "model sample rate (%d Hz).", + vocoder_->SampleRate(), model_->GetMetaData().sample_rate); + SHERPA_ONNX_LOGE( + "Please download the correct vocoder from " + "https://github.com/k2-fsa/sherpa-onnx/releases/tag/vocoder-models"); + if (model_->GetMetaData().sample_rate == 16000) { + SHERPA_ONNX_LOGE("For 16kHz models, use vocos-16khz-univ.onnx"); + } else if (model_->GetMetaData().sample_rate == 22050) { + SHERPA_ONNX_LOGE("For 22050Hz models, use vocos-22khz-univ.onnx"); + } else if (model_->GetMetaData().sample_rate == 24000) { + SHERPA_ONNX_LOGE("For 24kHz models, use vocos_24khz.onnx"); + } + SHERPA_ONNX_EXIT(-1); + } + + if (!config.model.zipvoice.tokens.empty()) { + auto buf = ReadFile(mgr, config.model.zipvoice.tokens); + std::istringstream is(std::string(buf.data(), buf.size())); + token_str2id_ = ReadTokens(is); + } + + if (!config.model.zipvoice.lexicon.empty()) { + std::vector files; + SplitStringToVector(config.model.zipvoice.lexicon, ",", false, &files); + for (const auto &f : files) { + auto buf = ReadFile(mgr, f); + std::istringstream is(std::string(buf.data(), buf.size())); + LoadLexicon(is, config.model.debug); + } + } + + tn_list_ = LoadTextNormalizers(mgr, config.rule_fsts, config.rule_fars, + config.model.debug); PostInit(); } @@ -53,19 +122,40 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { return model_->GetMetaData().sample_rate; } + /** + * + * Supported options in GenerationConfig: + * - speed: Speech speed factor (default: 1.0) + * - silence_scale: Scale applied to pauses in the generated audio + * - reference_audio: Mono float32 audio samples for zero-shot cloning + * - reference_sample_rate: Sample rate of reference_audio + * - reference_text: Transcript of reference_audio + * - tokens: Vector of sentences, each a vector of token strings. + * tokens[0] is used for the reference text, tokens[1..] for generated text. + * + * Supported extra parameters: + * + * - debug, int, default from model config + * - min_words_in_sentence, int, default 5. + * Merge adjacent sentences if the number of words is less than this. + * Each CJK character counts as one word; English words are space-separated. + * - max_words_in_sentence, int, default 20. + * Split a sentence into chunks if the number of words exceeds this. + * Splits at punctuation or space boundaries. + * - num_steps, int, default 4. + * Number of flow-matching denoising steps. + * - feat_scale, float, default from model config. + * Prompt mel log scaling factor. + * - t_shift, float, default from model config. + * Timestep shift used by the decoder schedule. + * - target_rms, float, default from model config. + * Prompt RMS normalization target. + * - guidance_scale, float, default from model config. + * Classifier-free guidance scale for the decoder. + */ GeneratedAudio Generate( const std::string &text, const GenerationConfig &config, GeneratedAudioCallback callback = nullptr) const override { - // Supported extra options in config.extra: - // - "speed" (float): Speech speed factor (default: 1.0) - // - "num_steps" (int): Number of flow-matching steps (default: 4) - // - "max_char_in_sentence" (int): Max characters per chunk (default: 200) - // - "min_char_in_sentence" (int): Merge shorter chunks until this size - // (default: 30) - // - "feat_scale" (float): Prompt mel log scaling factor (default: - // config.model.zipvoice.feat_scale) - // - "t_shift" (float): Timestep shift used by the decoder schedule - // (default: config.model.zipvoice.t_shift) // - "target_rms" (float): Prompt RMS normalization target (default: // config.model.zipvoice.target_rms) // - "guidance_scale" (float): Classifier-free guidance scale for the @@ -132,10 +222,26 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { return {}; } - std::vector prompt_token_ids = - frontend_->ConvertTextToTokenIds(config.reference_text); - if (prompt_token_ids.empty() || - (prompt_token_ids.size() == 1 && prompt_token_ids[0].tokens.empty())) { + // Tokenize reference text + std::vector prompt_tokens; + if (!config.phoneme_codepoints.empty()) { + SHERPA_ONNX_LOGE( + "GenerationConfig.phoneme_codepoints is not supported by ZipVoice " + "models and is ignored. Use lexicon or GenerationConfig.tokens " + "instead."); + } + if (!config.tokens.empty()) { + // tokens path: use first sentence as prompt tokens + for (const auto &tok : config.tokens[0]) { + auto it = token_str2id_.find(tok); + if (it != token_str2id_.end()) { + prompt_tokens.push_back(it->second); + } + } + } else { + prompt_tokens = TokenizeText(config.reference_text, config_.model.debug); + } + if (prompt_tokens.empty()) { #if __OHOS__ SHERPA_ONNX_LOGE( "Failed to convert prompt text '%{public}s' to token IDs", @@ -147,12 +253,6 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { return {}; } - std::vector prompt_tokens; - for (const auto &t : prompt_token_ids) { - prompt_tokens.insert(prompt_tokens.end(), t.tokens.begin(), - t.tokens.end()); - } - std::vector prompt_features = ComputePromptFeatures( config.reference_audio, config.reference_sample_rate, feat_scale, target_rms); @@ -161,60 +261,203 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { return {}; } - auto sentences = SplitByPunctuation(text); - if (sentences.empty()) { - return {}; - } - - int32_t max_char_in_sentence = - config.GetExtraInt("max_char_in_sentence", 200); - int32_t min_char_in_sentence = - config.GetExtraInt("min_char_in_sentence", 30); - - if (max_char_in_sentence <= 0) { - SHERPA_ONNX_LOGE("max_char_in_sentence must be > 0. Given: %d", - max_char_in_sentence); - return {}; - } - - if (min_char_in_sentence <= 0) { - SHERPA_ONNX_LOGE("min_char_in_sentence must be > 0. Given: %d", - min_char_in_sentence); - return {}; - } - - sentences = MergeShortSentences(sentences, min_char_in_sentence); - - std::vector final_chunks; - for (const auto &s : sentences) { - auto pieces = SplitLongSentence(s, max_char_in_sentence); - final_chunks.insert(final_chunks.end(), pieces.begin(), pieces.end()); - } - - sentences = std::move(final_chunks); - if (sentences.empty()) { - return {}; - } - GeneratedAudio result; result.sample_rate = SampleRate(); - const int32_t total = static_cast(sentences.size()); + if (!config.tokens.empty()) { + // tokens path: use token strings directly for generated text + // config.tokens[0] was used for prompt, config.tokens[1..] for generated + + // Flatten all generated tokens + std::vector all_tokens; + for (size_t si = 1; si < config.tokens.size(); ++si) { + for (const auto &tok : config.tokens[si]) { + all_tokens.push_back(tok); + } + } + + // Split at punctuation tokens into sub-sentences + auto IsPunctToken = [](const std::string &t) { + return t == "," || t == "." || t == "!" || t == "?" || t == ";" || + t == ":"; + }; + + std::vector> token_sentences; + std::vector current; + for (const auto &tok : all_tokens) { + current.push_back(tok); + if (IsPunctToken(tok)) { + token_sentences.push_back(std::move(current)); + current.clear(); + } + } + if (!current.empty()) { + token_sentences.push_back(std::move(current)); + } + + // Merge short token sentences + int32_t min_words = + config.GetExtraInt("min_words_in_sentence", 5); + std::vector> merged; + std::vector buffer; + for (auto &ts : token_sentences) { + buffer.insert(buffer.end(), ts.begin(), ts.end()); + if (static_cast(buffer.size()) >= min_words) { + merged.push_back(std::move(buffer)); + buffer.clear(); + } + } + if (!buffer.empty()) { + if (!merged.empty()) { + merged.back().insert(merged.back().end(), buffer.begin(), + buffer.end()); + } else { + merged.push_back(std::move(buffer)); + } + } - for (int32_t i = 0; i < total; ++i) { if (config_.model.debug) { #if __OHOS__ - SHERPA_ONNX_LOGE("Processing %{public}d/%{public}d: %{public}s", i + 1, - total, sentences[i].c_str()); + SHERPA_ONNX_LOGE("Reference text: %{public}s", + config.reference_text.c_str()); + SHERPA_ONNX_LOGE("Using tokens path: %{public}d sentence(s)", + static_cast(merged.size())); #else - SHERPA_ONNX_LOGE("Processing %d/%d: %s", i + 1, total, - sentences[i].c_str()); + SHERPA_ONNX_LOGE("Reference text: %s", + config.reference_text.c_str()); + SHERPA_ONNX_LOGE("Using tokens path: %d sentence(s)", + static_cast(merged.size())); #endif } - GeneratedAudio cur = GenerateChunk( - sentences[i], prompt_tokens, prompt_features, speed, num_steps, - feat_scale, t_shift, guidance_scale); + // Convert token strings to IDs and process each sentence + int32_t sentence_idx = 0; + for (const auto &sentence : merged) { + std::vector tokens; + std::vector id_strs; + for (const auto &tok : sentence) { + auto it = token_str2id_.find(tok); + if (it != token_str2id_.end()) { + tokens.push_back(it->second); + id_strs.push_back(tok + "(" + std::to_string(it->second) + ")"); + } else { + if (config_.model.debug) { + SHERPA_ONNX_LOGE("Skip unknown token: '%s'", tok.c_str()); + } + } + } + + // Merge single-token sentence into previous + if (tokens.size() == 1 && !result.samples.empty()) { + // Skip — don't process single-token sentences alone + continue; + } + + if (tokens.empty()) continue; + + if (config_.model.debug) { + std::ostringstream os; + os << "Sentence " << sentence_idx << " tokens: ["; + for (size_t j = 0; j < id_strs.size(); ++j) { + if (j > 0) os << ", "; + os << id_strs[j]; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + ++sentence_idx; + } + + GeneratedAudio cur = Process(tokens, prompt_tokens, prompt_features, + speed, num_steps, feat_scale, t_shift, + guidance_scale); + if (cur.samples.empty()) continue; + result.samples.insert(result.samples.end(), cur.samples.begin(), + cur.samples.end()); + } + } else { + // Text path: convert punctuations, split into sentences, merge short, + // split long, tokenize each. + if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Raw text: %{public}s", text.c_str()); +#else + SHERPA_ONNX_LOGE("Raw text: %s", text.c_str()); +#endif + } + + std::string normalized = ReplacePunctuations(text); + + auto sentences = SplitByAllPunctuation(normalized); + if (sentences.empty()) { + return {}; + } + + int32_t min_words = + config.GetExtraInt("min_words_in_sentence", 5); + int32_t max_words = + config.GetExtraInt("max_words_in_sentence", 20); + + sentences = MergeShortSentencesByWords(sentences, min_words); + + std::vector final_chunks; + for (const auto &s : sentences) { + auto pieces = SplitLongSentenceByWords(s, max_words); + final_chunks.insert(final_chunks.end(), pieces.begin(), pieces.end()); + } + + // Merge punctuation-only chunks into previous + std::vector merged_chunks; + for (const auto &s : final_chunks) { + std::u32string u32 = Utf8ToUtf32(Trim(s)); + bool all_punct = !u32.empty(); + for (char32_t c : u32) { + if (c != U',' && c != U'.' && c != U'!' && c != U'?' && + c != U';' && c != U':' && c != U' ') { + all_punct = false; + break; + } + } + if (all_punct && !merged_chunks.empty()) { + merged_chunks.back() += Trim(s); + } else { + merged_chunks.push_back(s); + } + } + sentences = std::move(merged_chunks); + + if (config_.model.debug) { + SHERPA_ONNX_LOGE("After split/merge: %d sentences", + static_cast(sentences.size())); + for (int32_t i = 0; i < static_cast(sentences.size()); ++i) { +#if __OHOS__ + SHERPA_ONNX_LOGE(" sentence %d: '%{public}s'", i, + sentences[i].c_str()); +#else + SHERPA_ONNX_LOGE(" sentence %d: '%s'", i, sentences[i].c_str()); +#endif + } + } + + const int32_t total = static_cast(sentences.size()); + + for (int32_t i = 0; i < total; ++i) { + if (config_.model.debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("Processing %{public}d/%{public}d: %{public}s", + i + 1, total, sentences[i].c_str()); +#else + SHERPA_ONNX_LOGE("Processing %d/%d: %s", i + 1, total, + sentences[i].c_str()); +#endif + } + + GeneratedAudio cur = GenerateChunk( + sentences[i], prompt_tokens, prompt_features, speed, num_steps, + feat_scale, t_shift, guidance_scale); if (cur.samples.empty()) { continue; @@ -231,6 +474,7 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { } } } + } // end else (text path) if (config.silence_scale != 1) { result = result.ScaleSilence(config.silence_scale); @@ -283,15 +527,13 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { template void InitFrontend(Manager *mgr) { - frontend_ = std::make_unique( - mgr, config_.model.zipvoice.lexicon, config_.model.zipvoice.tokens, - config_.model.zipvoice.data_dir, config_.model.debug, true); + // No longer using MatchaTtsLexicon (espeak-ng removed). + // Lexicon-based tokenization is handled by LoadLexicon + TokenizeText. } void InitFrontend() { - frontend_ = std::make_unique( - config_.model.zipvoice.lexicon, config_.model.zipvoice.tokens, - config_.model.zipvoice.data_dir, config_.model.debug, true); + // No longer using MatchaTtsLexicon (espeak-ng removed). + // Lexicon-based tokenization is handled by LoadLexicon + TokenizeText. } void ComputeMelSpectrogram(const std::vector &_samples, @@ -367,11 +609,9 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { const std::vector &prompt_features, float speed, int32_t num_steps, float feat_scale, float t_shift, float guidance_scale) const { - std::vector text_token_ids = - frontend_->ConvertTextToTokenIds(text); + std::vector tokens = TokenizeText(text, config_.model.debug); - if (text_token_ids.empty() || - (text_token_ids.size() == 1 && text_token_ids[0].tokens.empty())) { + if (tokens.empty()) { #if __OHOS__ SHERPA_ONNX_LOGE("Failed to convert '%{public}s' to token IDs", text.c_str()); @@ -381,11 +621,6 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { return {}; } - std::vector tokens; - for (const auto &t : text_token_ids) { - tokens.insert(tokens.end(), t.tokens.begin(), t.tokens.end()); - } - return Process(tokens, prompt_tokens, prompt_features, speed, num_steps, feat_scale, t_shift, guidance_scale); } @@ -477,12 +712,193 @@ class OfflineTtsZipvoiceImpl : public OfflineTtsImpl { return ans; } + void LoadLexicon(std::istream &is, bool debug) { + auto entries = ParseLexiconFile(is, &max_lexicon_phrase_len_); + for (auto &e : entries) { + lexicon_str_[std::move(e.key)] = std::move(e.phonemes); + } + if (debug) { + SHERPA_ONNX_LOGE("Loaded lexicon: %d entries, max phrase len %d", + static_cast(lexicon_str_.size()), + max_lexicon_phrase_len_); + } + } + + std::vector TokenizeText(const std::string &text, + bool debug) const { + std::vector ids; + std::string normalized = ReplacePunctuations(text); + + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("TokenizeText: text='%{public}s'", normalized.c_str()); +#else + SHERPA_ONNX_LOGE("TokenizeText: text='%s'", normalized.c_str()); +#endif + } + + std::vector words = SplitUtf8(normalized); + + int32_t i = 0; + int32_t n = static_cast(words.size()); + while (i < n) { + if (words[i] == " ") { + auto it = token_str2id_.find(" "); + if (it != token_str2id_.end()) ids.push_back(it->second); + ++i; + continue; + } + if (IsPunctWord(words[i])) { + auto it = token_str2id_.find(words[i]); + if (it != token_str2id_.end()) { + ids.push_back(it->second); + if (debug) { + SHERPA_ONNX_LOGE("Punctuation: '%s' -> %d", words[i].c_str(), + it->second); + } + } + ++i; + continue; + } + bool found = false; + int32_t max_try = std::min(max_lexicon_phrase_len_, n - i); + for (int32_t len = max_try; len >= 1; --len) { + std::string phrase; + for (int32_t j = i; j < i + len; ++j) { + bool prev_cjk = !phrase.empty() && IsCJK(Utf8ToUtf32(phrase).back()); + bool curr_cjk = !words[j].empty() && IsCJK(Utf8ToUtf32(words[j]).front()); + if (j > i && !(prev_cjk && curr_cjk)) phrase.push_back(' '); + phrase += ToLowerCase(words[j]); + } + auto it = lexicon_str_.find(phrase); + if (it != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", phrase.c_str(), + phones_str.c_str()); + } + auto tok_ids = ConvertPhonemeStringsToIds(it->second); + ids.insert(ids.end(), tok_ids.begin(), tok_ids.end()); + i += len; + found = true; + break; + } + } + if (!found) { + if (ContainsCJK(words[i])) { + for (const auto &ch : SplitUtf8(words[i])) { + std::string ch_lower = ToLowerCase(ch); + auto it = lexicon_str_.find(ch_lower); + if (it != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", + ch_lower.c_str(), phones_str.c_str()); + } + auto tok_ids = ConvertPhonemeStringsToIds(it->second); + ids.insert(ids.end(), tok_ids.begin(), tok_ids.end()); + } else if (debug) { + SHERPA_ONNX_LOGE("OOV character skipped: '%s'", ch.c_str()); + } + } + } else { + // The word is not in the lexicon. If it contains '-', split it and + // look up each part separately, e.g. "two-thirds" -> "two" + + // "thirds". + std::vector parts = SplitStringAndTrim(words[i], '-'); + if (words[i].find('-') != std::string::npos && !parts.empty()) { + for (const auto &part : parts) { + std::string part_lower = ToLowerCase(part); + auto it2 = lexicon_str_.find(part_lower); + if (it2 != lexicon_str_.end()) { + if (debug) { + std::string phones_str; + for (const auto &p : it2->second) phones_str += p + " "; + SHERPA_ONNX_LOGE("Lexicon matched: '%s' -> '%s'", + part_lower.c_str(), phones_str.c_str()); + } + auto tok_ids = ConvertPhonemeStringsToIds(it2->second); + ids.insert(ids.end(), tok_ids.begin(), tok_ids.end()); + } else if (debug) { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", part.c_str()); + } + } + } else if (debug) { + SHERPA_ONNX_LOGE("OOV word skipped: '%s'", words[i].c_str()); + } + } + ++i; + } + } + + if (debug) { + std::ostringstream os; + os << "Tokens (" << ids.size() << "): ["; + for (size_t j = 0; j < ids.size(); ++j) { + if (j > 0) os << ", "; + // Look up token string + std::string tok_str = "?"; + for (const auto &kv : token_str2id_) { + if (kv.second == ids[j]) { + tok_str = kv.first; + break; + } + } + os << tok_str << "(" << ids[j] << ")"; + } + os << "]"; +#if __OHOS__ + SHERPA_ONNX_LOGE("%{public}s", os.str().c_str()); +#else + SHERPA_ONNX_LOGE("%s", os.str().c_str()); +#endif + } + + return ids; + } + + std::vector ConvertPhonemeStringsToIds( + const std::vector &phones) const { + std::vector ans; + for (const auto &p : phones) { + auto it = token_str2id_.find(p); + if (it != token_str2id_.end()) ans.push_back(it->second); + } + return ans; + } + + static std::string ReplacePunctuations(const std::string &s) { + static const std::vector> r = { + {"，", ","}, {"、", ","}, {"；", ";"}, {"：", ","}, {":", ","}, + {"。", "."}, {"？", "?"}, {"！", "!"}, {"…", "..."}, + }; + std::string result = s; + for (const auto &p : r) { + size_t pos = 0; + while ((pos = result.find(p.first, pos)) != std::string::npos) { + result.replace(pos, p.first.size(), p.second); + pos += p.second.size(); + } + } + return result; + } + + static bool IsPunctWord(const std::string &s) { + if (s.empty()) return false; + static const std::string puncts = ",.;:!?"; + return s.size() == 1 && puncts.find(s[0]) != std::string::npos; + } + private: OfflineTtsConfig config_; std::unique_ptr model_; std::unique_ptr vocoder_; - std::unique_ptr frontend_; - + std::vector> tn_list_; + std::unordered_map token_str2id_; + std::unordered_map> lexicon_str_; + int32_t max_lexicon_phrase_len_ = 1; std::unique_ptr mel_banks_; }; diff --git a/sherpa-onnx/csrc/offline-tts-zipvoice-model-config.cc b/sherpa-onnx/csrc/offline-tts-zipvoice-model-config.cc index 712590ff9..57bf8d143 100644 --- a/sherpa-onnx/csrc/offline-tts-zipvoice-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-zipvoice-model-config.cc @@ -9,6 +9,7 @@ #include "sherpa-onnx/csrc/file-utils.h" #include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/text-utils.h" namespace sherpa_onnx { @@ -16,8 +17,10 @@ void OfflineTtsZipvoiceModelConfig::Register(ParseOptions *po) { po->Register("zipvoice-tokens", &tokens, "Path to tokens.txt for ZipVoice models"); po->Register("zipvoice-data-dir", &data_dir, - "Path to the directory containing dict for espeak-ng."); - po->Register("zipvoice-lexicon", &lexicon, "Path to lexicon.txt for Chinese"); + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. " + "Use --zipvoice-lexicon or " + "GenerationConfig.phoneme_codepoints instead."); + po->Register("zipvoice-lexicon", &lexicon, "Path to lexicon.txt"); po->Register("zipvoice-encoder", &encoder, "Path to zipvoice text model"); po->Register("zipvoice-decoder", &decoder, "Path to zipvoice flow-matching decoder model"); @@ -77,17 +80,21 @@ bool OfflineTtsZipvoiceModelConfig::Validate() const { } if (!data_dir.empty()) { - std::vector required_files = { - "phontab", - "phonindex", - "phondata", - "intonations", - }; - for (const auto &f : required_files) { - if (!FileExists(data_dir + "/" + f)) { + SHERPA_ONNX_LOGE( + "WARNING: --zipvoice-data-dir is deprecated and ignored in " + "sherpa-onnx >= v2.0.0. Please use --zipvoice-lexicon or " + "GenerationConfig.phoneme_codepoints with an external " + "phonemizer (e.g. piper_phonemize) instead."); + } + + if (!lexicon.empty()) { + std::vector files; + SplitStringToVector(lexicon, ",", false, &files); + for (const auto &f : files) { + if (!FileExists(f)) { SHERPA_ONNX_LOGE( - "'%s/%s' does not exist. Please check zipvoice-data-dir", - data_dir.c_str(), f.c_str()); + "lexicon '%s' does not exist. Please check --zipvoice-lexicon", + f.c_str()); return false; } } diff --git a/sherpa-onnx/csrc/offline-tts-zipvoice-model-config.h b/sherpa-onnx/csrc/offline-tts-zipvoice-model-config.h index eb837ecec..475e35bb3 100644 --- a/sherpa-onnx/csrc/offline-tts-zipvoice-model-config.h +++ b/sherpa-onnx/csrc/offline-tts-zipvoice-model-config.h @@ -18,6 +18,9 @@ struct OfflineTtsZipvoiceModelConfig { std::string decoder; std::string vocoder; + // Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. + // Use lexicon for text input, or GenerationConfig.phoneme_codepoints / + // GenerationConfig.tokens with an external phonemizer instead. std::string data_dir; std::string lexicon; diff --git a/sherpa-onnx/csrc/offline-tts.cc b/sherpa-onnx/csrc/offline-tts.cc index 8a6b895bb..ac659a16c 100644 --- a/sherpa-onnx/csrc/offline-tts.cc +++ b/sherpa-onnx/csrc/offline-tts.cc @@ -174,6 +174,14 @@ std::string GenerationConfig::ToString() const { os << "}"; } + if (!phoneme_codepoints.empty()) { + os << ", phoneme_codepoints=[" << phoneme_codepoints.size() << " sentences]"; + } + + if (!tokens.empty()) { + os << ", tokens=[" << tokens.size() << " sentences]"; + } + os << ")"; return os.str(); } @@ -336,6 +344,24 @@ GeneratedAudio OfflineTts::Generate( GeneratedAudio OfflineTts::Generate( const std::string &text, const GenerationConfig &config, GeneratedAudioCallback callback /*= nullptr*/) const { + // phoneme_codepoints and tokens are mutually exclusive. + // + // This overload is the funnel for every caller that passes a + // GenerationConfig: C++, Python and JNI call it directly, and the C API + // calls tts->impl->Generate where impl is a std::unique_ptr, so + // it lands here as well. The C API additionally pre-checks in + // SherpaOnnxOfflineTtsGenerateInternal and returns nullptr (its own failure + // convention), so C callers normally never reach this line. The deprecated + // overloads above call impl_->Generate and skip this funnel, but they build + // their own GenerationConfig with both fields empty and cannot trigger it. + if (!config.phoneme_codepoints.empty() && !config.tokens.empty()) { + SHERPA_ONNX_LOGE( + "Both GenerationConfig.phoneme_codepoints and GenerationConfig.tokens " + "are given. They are mutually exclusive; please set only one of them. " + "Return an empty result."); + return GeneratedAudio{{}, 0}; + } + #if !defined(_WIN32) return impl_->Generate(text, config, std::move(callback)); #else @@ -363,6 +389,8 @@ int32_t OfflineTts::SampleRate() const { return impl_->SampleRate(); } int32_t OfflineTts::NumSpeakers() const { return impl_->NumSpeakers(); } +std::string OfflineTts::Lang() const { return impl_->Lang(); } + #if __ANDROID_API__ >= 9 template OfflineTts::OfflineTts(AAssetManager *mgr, const OfflineTtsConfig &config); diff --git a/sherpa-onnx/csrc/offline-tts.h b/sherpa-onnx/csrc/offline-tts.h index a9e0eb3fc..c6f354b58 100644 --- a/sherpa-onnx/csrc/offline-tts.h +++ b/sherpa-onnx/csrc/offline-tts.h @@ -80,6 +80,15 @@ struct GenerationConfig { // e.g., in ./offline-tts-pocket-impl.h std::unordered_map extra; + std::vector> phoneme_codepoints; + + // For models that use string-based tokens (e.g., Chinese pinyin like + // "zhong1", "ang2"). Each inner vector is a sentence of token strings. + // + // Mutually exclusive with phoneme_codepoints: if both are set, a warning is + // printed and generation fails with an empty result. + std::vector> tokens; + std::string GetExtraString(const std::string &key, const std::string &def = "") const; @@ -155,6 +164,9 @@ class OfflineTts { // If it supports only a single speaker, then it return 0 or 1. int32_t NumSpeakers() const; + // Language/voice of the model, e.g., "en-us". Empty if not available. + std::string Lang() const; + private: std::unique_ptr impl_; }; diff --git a/sherpa-onnx/csrc/online-recognizer.cc b/sherpa-onnx/csrc/online-recognizer.cc index 20d4646bb..1583ff1aa 100644 --- a/sherpa-onnx/csrc/online-recognizer.cc +++ b/sherpa-onnx/csrc/online-recognizer.cc @@ -255,6 +255,7 @@ void OnlineRecognizer::DecodeStreams(OnlineStream **ss, int32_t n) const { OnlineRecognizerResult OnlineRecognizer::GetResult(OnlineStream *s) const { auto r = impl_->GetResult(s); r.text = RemoveLeadingSpaces(r.text); + r.text = RemoveLeadingPunct(r.text); return r; } diff --git a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc b/sherpa-onnx/csrc/piper-phonemize-lexicon.cc deleted file mode 100644 index 2e24267db..000000000 --- a/sherpa-onnx/csrc/piper-phonemize-lexicon.cc +++ /dev/null @@ -1,702 +0,0 @@ -// sherpa-onnx/csrc/piper-phonemize-lexicon.cc -// -// Copyright (c) 2022-2023 Xiaomi Corporation - -#include "sherpa-onnx/csrc/piper-phonemize-lexicon.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if __ANDROID_API__ >= 9 -#include "android/asset_manager.h" -#include "android/asset_manager_jni.h" -#endif - -#if __OHOS__ -#include "rawfile/raw_file_manager.h" -#endif - -#include "espeak-ng/speak_lib.h" -#include "phoneme_ids.hpp" // NOLINT -#include "phonemize.hpp" // NOLINT -#include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/macros.h" -#include "sherpa-onnx/csrc/text-utils.h" - -namespace sherpa_onnx { - -// Encode a single char32_t to UTF-8 string. For debugging only -static std::string ToString(char32_t cp) { - std::string result; - - if (cp <= 0x7F) { - result += static_cast(cp); - } else if (cp <= 0x7FF) { - result += static_cast(0xC0 | ((cp >> 6) & 0x1F)); - result += static_cast(0x80 | (cp & 0x3F)); - } else if (cp <= 0xFFFF) { - result += static_cast(0xE0 | ((cp >> 12) & 0x0F)); - result += static_cast(0x80 | ((cp >> 6) & 0x3F)); - result += static_cast(0x80 | (cp & 0x3F)); - } else if (cp <= 0x10FFFF) { - result += static_cast(0xF0 | ((cp >> 18) & 0x07)); - result += static_cast(0x80 | ((cp >> 12) & 0x3F)); - result += static_cast(0x80 | ((cp >> 6) & 0x3F)); - result += static_cast(0x80 | (cp & 0x3F)); - } else { - SHERPA_ONNX_LOGE("Invalid Unicode code point: %d", - static_cast(cp)); - } - - return result; -} - -void CallPhonemizeEspeak(const std::string &text, - piper::eSpeakPhonemeConfig &config, // NOLINT - std::vector> *phonemes) { - static std::mutex espeak_mutex; - - // keep multi threads from calling into piper::phonemize_eSpeak - std::lock_guard lock(espeak_mutex); - - try { - piper::phonemize_eSpeak(text, config, *phonemes); - } catch (const std::exception &ex) { - // piper::phonemize_eSpeak() throws if espeak-ng does not recognize - // config.voice, e.g., when a user passes an unsupported --kokoro-lang. - // Return no phonemes so that the caller fails the generation instead - // of the uncaught exception terminating the whole process. - SHERPA_ONNX_LOGE("Failed to phonemize '%s' with espeak-ng voice '%s': %s", - text.c_str(), config.voice.c_str(), ex.what()); - phonemes->clear(); - } -} - -static std::unordered_map ReadTokens(std::istream &is) { - std::unordered_map token2id; - - std::string line; - - std::string sym; - std::u32string s; - int32_t id = 0; - while (std::getline(is, line)) { - std::istringstream iss(line); - iss >> sym; - if (iss.eof()) { - id = atoi(sym.c_str()); - sym = " "; - } else { - iss >> id; - } - - // eat the trailing \r\n on windows - iss >> std::ws; - if (!iss.eof()) { - SHERPA_ONNX_LOGE("Error when reading tokens: %s", line.c_str()); - SHERPA_ONNX_EXIT(-1); - } - - s = Utf8ToUtf32(sym); - if (s.size() != 1) { - // for tokens.txt from coqui-ai/TTS, the last token is - if (s.size() == 6 && s[0] == '<' && s[1] == 'B' && s[2] == 'L' && - s[3] == 'N' && s[4] == 'K' && s[5] == '>') { - continue; - } - - SHERPA_ONNX_LOGE("Error when reading tokens at Line %s. size: %d", - line.c_str(), static_cast(s.size())); - SHERPA_ONNX_EXIT(-1); - } - - char32_t c = s[0]; - - if (token2id.count(c)) { - SHERPA_ONNX_LOGE("Duplicated token %s. Line %s. Existing ID: %d", - sym.c_str(), line.c_str(), token2id.at(c)); - SHERPA_ONNX_EXIT(-1); - } - - token2id.insert({c, id}); - } - - return token2id; -} - -// see the function "phonemes_to_ids" from -// https://github.com/rhasspy/piper/blob/master/notebooks/piper_inference_(ONNX).ipynb -static std::vector PiperPhonemesToIdsVits( - const std::unordered_map &token2id, - const std::vector &phonemes, bool is_inflect) { - // see - // https://github.com/rhasspy/piper-phonemize/blob/master/src/phoneme_ids.hpp#L17 - int32_t pad = token2id.at(U'_'); - - int32_t bos = -1; - int32_t eos = -1; - - if (!is_inflect) { - bos = token2id.at(U'^'); - eos = token2id.at(U'$'); - } - - std::vector ans; - // bos + pad after bos + (phoneme + pad) * N + eos - ans.reserve(phonemes.size() * 2 + 3); - - if (is_inflect) { - ans.push_back(pad); - } else { - ans.push_back(bos); - // Match piper-phonemize phoneme_ids.cpp and OfflineTtsImpl::AddBlank(): - // pad must follow bos so every phoneme (including the first) is framed. - // See https://github.com/k2-fsa/sherpa-onnx/issues/3721 - ans.push_back(pad); - } - - for (auto p : phonemes) { - if (token2id.count(p)) { - ans.push_back(token2id.at(p)); - ans.push_back(pad); - } else { - SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", - static_cast(p)); - } - } - - if (!is_inflect) { - ans.push_back(eos); - } - - return ans; -} - -static std::vector> PiperPhonemesToIdsMatcha( - const std::unordered_map &token2id, - const std::vector &phonemes, bool use_eos_bos, - int32_t max_token_len = 400) { - // We set max_token_len to 400 here to fix - // https://github.com/k2-fsa/sherpa-onnx/issues/2666 - std::vector> ans; - std::vector current; - - int32_t bos = token2id.at(U'^'); - int32_t eos = token2id.at(U'$'); - - if (use_eos_bos) { - current.push_back(bos); - } - - for (auto p : phonemes) { - if (token2id.count(p)) { - current.push_back(token2id.at(p)); - } else { - SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", - static_cast(p)); - } - - if (current.size() > max_token_len + 1) { - if (use_eos_bos) { - current.push_back(eos); - } - - ans.push_back(std::move(current)); - - if (use_eos_bos) { - current.push_back(bos); - } - } - } // for (auto p : phonemes) - - if (!current.empty()) { - if (use_eos_bos) { - if (current.size() > 1) { - current.push_back(eos); - - ans.push_back(std::move(current)); - } - } else { - ans.push_back(std::move(current)); - } - } - - return ans; -} - -static std::vector> PiperPhonemesToIdsKokoroOrKitten( - const std::unordered_map &token2id, - const std::vector &phonemes, int32_t max_len) { - std::vector> ans; - - std::vector current; - current.reserve(phonemes.size()); - - current.push_back(0); - - for (auto p : phonemes) { - // SHERPA_ONNX_LOGE("%d %s", static_cast(p), ToString(p).c_str()); - if (token2id.count(p)) { - if (current.size() > max_len - 1) { - current.push_back(0); - ans.push_back(std::move(current)); - - current.reserve(phonemes.size()); - current.push_back(0); - } - - current.push_back(token2id.at(p)); - if (p == '.') { - current.push_back(token2id.at(' ')); - } - } else { - SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", - static_cast(p)); - } - } - - current.push_back(0); - ans.push_back(std::move(current)); - return ans; -} - -static std::vector> PiperPhonemesToIdsKitten( - const std::unordered_map &token2id, - const std::vector &phonemes, - const OfflineTtsKittenModelMetaData &meta_data) { - std::vector> ans; - - std::vector current; - current.reserve(phonemes.size()); - - current.push_back(meta_data.start_id); - - int32_t suffix_size = meta_data.add_pad_after_end ? 2 : 1; - for (auto p : phonemes) { - if (token2id.count(p)) { - int32_t emitted_tokens = p == '.' ? 2 : 1; - if (static_cast(current.size()) + emitted_tokens + suffix_size > - meta_data.max_token_len) { - current.push_back(meta_data.end_id); - if (meta_data.add_pad_after_end) { - current.push_back(meta_data.pad_id); - } - ans.push_back(std::move(current)); - - current.reserve(phonemes.size()); - current.push_back(meta_data.start_id); - } - - current.push_back(token2id.at(p)); - if (p == '.') { - current.push_back(token2id.at(' ')); - } - } else { - SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", - static_cast(p)); - } - } - - current.push_back(meta_data.end_id); - if (meta_data.add_pad_after_end) { - current.push_back(meta_data.pad_id); - } - ans.push_back(std::move(current)); - return ans; -} - -static std::vector CoquiPhonemesToIds( - const std::unordered_map &token2id, - const std::vector &phonemes, - const OfflineTtsVitsModelMetaData &vits_meta_data) { - // see - // https://github.com/coqui-ai/TTS/blob/dev/TTS/tts/utils/text/tokenizer.py#L87 - int32_t use_eos_bos = vits_meta_data.use_eos_bos; - int32_t bos_id = vits_meta_data.bos_id; - int32_t eos_id = vits_meta_data.eos_id; - int32_t blank_id = vits_meta_data.blank_id; - int32_t add_blank = vits_meta_data.add_blank; - int32_t comma_id = token2id.at(','); - - std::vector ans; - if (add_blank) { - ans.reserve(phonemes.size() * 2 + 3); - } else { - ans.reserve(phonemes.size() + 2); - } - - if (use_eos_bos) { - ans.push_back(bos_id); - } - - if (add_blank) { - ans.push_back(blank_id); - - for (auto p : phonemes) { - if (token2id.count(p)) { - ans.push_back(token2id.at(p)); - ans.push_back(blank_id); - } else { - SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", - static_cast(p)); - } - } - } else { - // not adding blank - for (auto p : phonemes) { - if (token2id.count(p)) { - ans.push_back(token2id.at(p)); - } else { - SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.", - static_cast(p)); - } - } - } - - // add a comma at the end of a sentence so that we can have a longer pause. - ans.push_back(comma_id); - - if (use_eos_bos) { - ans.push_back(eos_id); - } - - return ans; -} - -void InitEspeak(const std::string &data_dir) { - static std::once_flag init_flag; - std::call_once(init_flag, [data_dir]() { -#if __ANDROID_API__ >= 9 || defined(__OHOS__) - if (data_dir[0] != '/') { - SHERPA_ONNX_LOGE( - "You need to follow our examples to copy the espeak-ng-data " - "directory from the assets folder to an external storage directory."); - - SHERPA_ONNX_LOGE( - "Hint: Please see\n" - "https://github.com/k2-fsa/sherpa-onnx/blob/master/android/" - "SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/" - "engine/TtsEngine.kt#L188\n" - "The function copyDataDir()\n"); - } -#endif - - int32_t result = - espeak_Initialize(AUDIO_OUTPUT_SYNCHRONOUS, 0, data_dir.c_str(), 0); - if (result != 22050) { - SHERPA_ONNX_LOGE( - "Failed to initialize espeak-ng with data dir: %s. Return code is: " - "%d", - data_dir.c_str(), result); - SHERPA_ONNX_EXIT(-1); - } - }); -} - -std::vector ConvertTextToTokenIdsKokoroOrKitten( - const std::unordered_map &token2id, - int32_t max_token_len, const std::string &text, const std::string &voice); - -std::vector ConvertTextToTokenIdsKitten( - const std::unordered_map &token2id, - const OfflineTtsKittenModelMetaData &meta_data, const std::string &text, - const std::string &voice); - -PiperPhonemizeLexicon::PiperPhonemizeLexicon( - const std::string &tokens, const std::string &data_dir, - const OfflineTtsVitsModelMetaData &vits_meta_data) - : vits_meta_data_(vits_meta_data) { - { - auto is = OpenInputFile(tokens); - token2id_ = ReadTokens(is); - } - - InitEspeak(data_dir); -} - -template -PiperPhonemizeLexicon::PiperPhonemizeLexicon( - Manager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsVitsModelMetaData &vits_meta_data) - : vits_meta_data_(vits_meta_data) { - { - auto buf = ReadFile(mgr, tokens); - std::istringstream is(std::string(buf.data(), buf.size())); - token2id_ = ReadTokens(is); - } - - // We should copy the directory of espeak-ng-data from the asset to - // some internal or external storage and then pass the directory to - // data_dir. - InitEspeak(data_dir); -} - -PiperPhonemizeLexicon::PiperPhonemizeLexicon( - const std::string &tokens, const std::string &data_dir, - const OfflineTtsMatchaModelMetaData &matcha_meta_data) - : matcha_meta_data_(matcha_meta_data), is_matcha_(true) { - { - auto is = OpenInputFile(tokens); - token2id_ = ReadTokens(is); - } - - InitEspeak(data_dir); -} - -PiperPhonemizeLexicon::PiperPhonemizeLexicon( - const std::string &tokens, const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &kokoro_meta_data) - : kokoro_meta_data_(kokoro_meta_data), is_kokoro_(true) { - { - auto is = OpenInputFile(tokens); - token2id_ = ReadTokens(is); - } - - InitEspeak(data_dir); -} - -PiperPhonemizeLexicon::PiperPhonemizeLexicon( - const std::string &tokens, const std::string &data_dir, - const OfflineTtsKittenModelMetaData &kitten_meta_data) - : kitten_meta_data_(kitten_meta_data), is_kitten_(true) { - { - auto is = OpenInputFile(tokens); - token2id_ = ReadTokens(is); - } - - InitEspeak(data_dir); -} - -template -PiperPhonemizeLexicon::PiperPhonemizeLexicon( - Manager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsMatchaModelMetaData &matcha_meta_data) - : matcha_meta_data_(matcha_meta_data), is_matcha_(true) { - { - auto buf = ReadFile(mgr, tokens); - std::istringstream is(std::string(buf.data(), buf.size())); - token2id_ = ReadTokens(is); - } - - // We should copy the directory of espeak-ng-data from the asset to - // some internal or external storage and then pass the directory to - // data_dir. - InitEspeak(data_dir); -} - -template -PiperPhonemizeLexicon::PiperPhonemizeLexicon( - Manager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &kokoro_meta_data) - : kokoro_meta_data_(kokoro_meta_data), is_kokoro_(true) { - { - auto buf = ReadFile(mgr, tokens); - std::istringstream is(std::string(buf.data(), buf.size())); - token2id_ = ReadTokens(is); - } - - // We should copy the directory of espeak-ng-data from the asset to - // some internal or external storage and then pass the directory to - // data_dir. - InitEspeak(data_dir); -} - -template -PiperPhonemizeLexicon::PiperPhonemizeLexicon( - Manager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsKittenModelMetaData &kitten_meta_data) - : kitten_meta_data_(kitten_meta_data), is_kitten_(true) { - { - auto buf = ReadFile(mgr, tokens); - std::istringstream is(std::string(buf.data(), buf.size())); - token2id_ = ReadTokens(is); - } - - // We should copy the directory of espeak-ng-data from the asset to - // some internal or external storage and then pass the directory to - // data_dir. - InitEspeak(data_dir); -} - -std::vector PiperPhonemizeLexicon::ConvertTextToTokenIds( - const std::string &text, const std::string &voice /*= ""*/) const { - if (is_matcha_) { - return ConvertTextToTokenIdsMatcha(text, voice); - } else if (is_kokoro_) { - return ConvertTextToTokenIdsKokoroOrKitten( - token2id_, kokoro_meta_data_.max_token_len, text, voice); - } else if (is_kitten_) { - return ConvertTextToTokenIdsKitten(token2id_, kitten_meta_data_, text, - voice); - } else { - return ConvertTextToTokenIdsVits(text, voice); - } -} - -std::vector PiperPhonemizeLexicon::ConvertTextToTokenIdsMatcha( - const std::string &text, const std::string &voice /*= ""*/) const { - piper::eSpeakPhonemeConfig config; - - // ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices - // to list available voices - config.voice = voice; // e.g., voice is en-us - - std::vector> phonemes; - - CallPhonemizeEspeak(text, config, &phonemes); - - std::vector ans; - - for (const auto &p : phonemes) { - auto phoneme_ids = - PiperPhonemesToIdsMatcha(token2id_, p, matcha_meta_data_.use_eos_bos); - - for (auto &ids : phoneme_ids) { - ans.emplace_back(std::move(ids)); - } - } - - return ans; -} - -std::vector ConvertTextToTokenIdsKokoroOrKitten( - const std::unordered_map &token2id, - int32_t max_token_len, const std::string &text, - const std::string &voice /*= ""*/) { - piper::eSpeakPhonemeConfig config; - - // ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices - // to list available voices - config.voice = voice; // e.g., voice is en-us - - std::vector> phonemes; - - CallPhonemizeEspeak(text, config, &phonemes); - - std::vector ans; - - for (const auto &p : phonemes) { - auto phoneme_ids = - PiperPhonemesToIdsKokoroOrKitten(token2id, p, max_token_len); - - for (auto &ids : phoneme_ids) { - ans.emplace_back(std::move(ids)); - } - } - - return ans; -} - -std::vector ConvertTextToTokenIdsKitten( - const std::unordered_map &token2id, - const OfflineTtsKittenModelMetaData &meta_data, const std::string &text, - const std::string &voice /*= ""*/) { - piper::eSpeakPhonemeConfig config; - - // ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices - // to list available voices - config.voice = voice; // e.g., voice is en-us - - std::vector> phonemes; - - CallPhonemizeEspeak(text, config, &phonemes); - - std::vector ans; - - for (const auto &p : phonemes) { - auto phoneme_ids = PiperPhonemesToIdsKitten(token2id, p, meta_data); - - for (auto &ids : phoneme_ids) { - ans.emplace_back(std::move(ids)); - } - } - - return ans; -} - -std::vector PiperPhonemizeLexicon::ConvertTextToTokenIdsVits( - const std::string &text, const std::string &voice /*= ""*/) const { - piper::eSpeakPhonemeConfig config; - - // ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices - // to list available voices - config.voice = voice; // e.g., voice is en-us - - std::vector> phonemes; - - CallPhonemizeEspeak(text, config, &phonemes); - - std::vector ans; - - std::vector phoneme_ids; - - if (vits_meta_data_.is_piper || vits_meta_data_.is_icefall || - vits_meta_data_.is_inflect) { - for (const auto &p : phonemes) { - phoneme_ids = - PiperPhonemesToIdsVits(token2id_, p, vits_meta_data_.is_inflect); - ans.emplace_back(std::move(phoneme_ids)); - } - } else if (vits_meta_data_.is_coqui) { - for (const auto &p : phonemes) { - phoneme_ids = CoquiPhonemesToIds(token2id_, p, vits_meta_data_); - ans.emplace_back(std::move(phoneme_ids)); - } - - } else { - SHERPA_ONNX_LOGE("Unsupported model"); - SHERPA_ONNX_EXIT(-1); - } - - return ans; -} - -#if __ANDROID_API__ >= 9 -template PiperPhonemizeLexicon::PiperPhonemizeLexicon( - AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsVitsModelMetaData &vits_meta_data); - -template PiperPhonemizeLexicon::PiperPhonemizeLexicon( - AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsMatchaModelMetaData &matcha_meta_data); - -template PiperPhonemizeLexicon::PiperPhonemizeLexicon( - AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &kokoro_meta_data); - -template PiperPhonemizeLexicon::PiperPhonemizeLexicon( - AAssetManager *mgr, const std::string &tokens, const std::string &data_dir, - const OfflineTtsKittenModelMetaData &kokoro_meta_data); -#endif - -#if __OHOS__ -template PiperPhonemizeLexicon::PiperPhonemizeLexicon( - NativeResourceManager *mgr, const std::string &tokens, - const std::string &data_dir, - const OfflineTtsVitsModelMetaData &vits_meta_data); - -template PiperPhonemizeLexicon::PiperPhonemizeLexicon( - NativeResourceManager *mgr, const std::string &tokens, - const std::string &data_dir, - const OfflineTtsMatchaModelMetaData &matcha_meta_data); - -template PiperPhonemizeLexicon::PiperPhonemizeLexicon( - NativeResourceManager *mgr, const std::string &tokens, - const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &kokoro_meta_data); - -template PiperPhonemizeLexicon::PiperPhonemizeLexicon( - NativeResourceManager *mgr, const std::string &tokens, - const std::string &data_dir, - const OfflineTtsKittenModelMetaData &kokoro_meta_data); -#endif - -} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/piper-phonemize-lexicon.h b/sherpa-onnx/csrc/piper-phonemize-lexicon.h deleted file mode 100644 index 0c9b3a5ec..000000000 --- a/sherpa-onnx/csrc/piper-phonemize-lexicon.h +++ /dev/null @@ -1,78 +0,0 @@ -// sherpa-onnx/csrc/piper-phonemize-lexicon.h -// -// Copyright (c) 2022-2023 Xiaomi Corporation - -#ifndef SHERPA_ONNX_CSRC_PIPER_PHONEMIZE_LEXICON_H_ -#define SHERPA_ONNX_CSRC_PIPER_PHONEMIZE_LEXICON_H_ - -#include -#include -#include - -#include "sherpa-onnx/csrc/offline-tts-frontend.h" -#include "sherpa-onnx/csrc/offline-tts-kitten-model-meta-data.h" -#include "sherpa-onnx/csrc/offline-tts-kokoro-model-meta-data.h" -#include "sherpa-onnx/csrc/offline-tts-matcha-model-meta-data.h" -#include "sherpa-onnx/csrc/offline-tts-vits-model-meta-data.h" - -namespace sherpa_onnx { - -class PiperPhonemizeLexicon : public OfflineTtsFrontend { - public: - PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir, - const OfflineTtsVitsModelMetaData &vits_meta_data); - - PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir, - const OfflineTtsMatchaModelMetaData &matcha_meta_data); - - PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &kokoro_meta_data); - - PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir, - const OfflineTtsKittenModelMetaData &kitten_meta_data); - - template - PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens, - const std::string &data_dir, - const OfflineTtsVitsModelMetaData &vits_meta_data); - - template - PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens, - const std::string &data_dir, - const OfflineTtsMatchaModelMetaData &matcha_meta_data); - - template - PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens, - const std::string &data_dir, - const OfflineTtsKokoroModelMetaData &kokoro_meta_data); - - template - PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens, - const std::string &data_dir, - const OfflineTtsKittenModelMetaData &kitten_meta_data); - - std::vector ConvertTextToTokenIds( - const std::string &text, const std::string &voice = "") const override; - - private: - std::vector ConvertTextToTokenIdsVits( - const std::string &text, const std::string &voice = "") const; - - std::vector ConvertTextToTokenIdsMatcha( - const std::string &text, const std::string &voice = "") const; - - private: - // map unicode codepoint to an integer ID - std::unordered_map token2id_; - OfflineTtsVitsModelMetaData vits_meta_data_; - OfflineTtsMatchaModelMetaData matcha_meta_data_; - OfflineTtsKokoroModelMetaData kokoro_meta_data_; - OfflineTtsKittenModelMetaData kitten_meta_data_; - bool is_matcha_ = false; - bool is_kokoro_ = false; - bool is_kitten_ = false; -}; - -} // namespace sherpa_onnx - -#endif // SHERPA_ONNX_CSRC_PIPER_PHONEMIZE_LEXICON_H_ diff --git a/sherpa-onnx/csrc/piper-phonemize-test.cc b/sherpa-onnx/csrc/piper-phonemize-test.cc deleted file mode 100644 index 65abfabd7..000000000 --- a/sherpa-onnx/csrc/piper-phonemize-test.cc +++ /dev/null @@ -1,83 +0,0 @@ -// sherpa-onnx/csrc/piper-phonemize-test.cc -// -// Copyright (c) 2023 Xiaomi Corporation - -#include -#include -#include -#include - -#include "espeak-ng/speak_lib.h" -#include "gtest/gtest.h" -#include "phoneme_ids.hpp" // NOLINT -#include "phonemize.hpp" // NOLINT -#include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/macros.h" - -namespace sherpa_onnx { - -TEST(PiperPhonemize, Case1) { - std::string data_dir = "./install/share/espeak-ng-data"; - if (!FileExists(data_dir + "/en_dict")) { - SHERPA_ONNX_LOGE("%s/en_dict does not exist. Skipping test", - data_dir.c_str()); - return; - } - - if (!FileExists(data_dir + "/phontab")) { - SHERPA_ONNX_LOGE("%s/phontab does not exist. Skipping test", - data_dir.c_str()); - return; - } - - if (!FileExists(data_dir + "/phonindex")) { - SHERPA_ONNX_LOGE("%s/phonindex does not exist. Skipping test", - data_dir.c_str()); - return; - } - - if (!FileExists(data_dir + "/phondata")) { - SHERPA_ONNX_LOGE("%s/phondata does not exist. Skipping test", - data_dir.c_str()); - return; - } - - if (!FileExists(data_dir + "/intonations")) { - SHERPA_ONNX_LOGE("%s/intonations does not exist. Skipping test", - data_dir.c_str()); - return; - } - int32_t result = - espeak_Initialize(AUDIO_OUTPUT_SYNCHRONOUS, 0, data_dir.c_str(), 0); - EXPECT_EQ(result, 22050); - - piper::eSpeakPhonemeConfig config; - - // ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices - // to list available voices - config.voice = "en-us"; - - std::vector> phonemes; - std::string text = "how are you doing?"; - piper::phonemize_eSpeak(text, config, phonemes); - - for (int32_t p : phonemes[0]) { - std::cout << p << " "; - } - std::cout << "\n"; - - std::vector phoneme_ids; - std::map missing_phonemes; - - { - piper::PhonemeIdConfig config; - phonemes_to_ids(phonemes[0], config, phoneme_ids, missing_phonemes); - } - - for (int32_t p : phoneme_ids) { - std::cout << p << " "; - } - std::cout << "\n"; -} - -} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/rknn/keyword-spotter-transducer-rknn-impl.h b/sherpa-onnx/csrc/rknn/keyword-spotter-transducer-rknn-impl.h index d5a3f86c2..329c951a0 100644 --- a/sherpa-onnx/csrc/rknn/keyword-spotter-transducer-rknn-impl.h +++ b/sherpa-onnx/csrc/rknn/keyword-spotter-transducer-rknn-impl.h @@ -174,11 +174,11 @@ class KeywordSpotterTransducerRknnImpl : public KeywordSpotterImpl { int32_t num_trailing_blanks = r.num_trailing_blanks; // assume subsampling_factor is 4 // assume frameshift is 0.01 second - float trailing_slience = num_trailing_blanks * 4 * 0.01; + float trailing_silence = num_trailing_blanks * 4 * 0.01; // it resets automatically after detecting 1.5 seconds of silence float threshold = 1.5; - if (trailing_slience > threshold) { + if (trailing_silence > threshold) { Reset(s); } diff --git a/sherpa-onnx/csrc/sherpa-onnx-offline-tts-play-alsa.cc b/sherpa-onnx/csrc/sherpa-onnx-offline-tts-play-alsa.cc index c6b300bbd..116892a85 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-offline-tts-play-alsa.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-offline-tts-play-alsa.cc @@ -110,10 +110,12 @@ Usage examples: wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./bin/sherpa-onnx-offline-tts-play-alsa \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." @@ -155,12 +157,12 @@ ZipVoice TTS: wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 tar xf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt ./bin/sherpa-onnx-offline-tts-play-alsa \ --zipvoice-encoder=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx \ --zipvoice-decoder=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx \ - --zipvoice-data-dir=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data \ - --zipvoice-lexicon=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt \ + --zipvoice-lexicon=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt \ --zipvoice-tokens=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt \ --zipvoice-vocoder=./vocos_24khz.onnx \ --reference-audio=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/test_wavs/leijun-1.wav \ diff --git a/sherpa-onnx/csrc/sherpa-onnx-offline-tts-play.cc b/sherpa-onnx/csrc/sherpa-onnx-offline-tts-play.cc index abd5a8df6..8394bc917 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-offline-tts-play.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-offline-tts-play.cc @@ -201,10 +201,12 @@ Usage examples: wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./bin/sherpa-onnx-offline-tts-play \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." @@ -246,12 +248,12 @@ ZipVoice TTS: wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 tar xf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt ./bin/sherpa-onnx-offline-tts-play \ --zipvoice-encoder=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx \ --zipvoice-decoder=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx \ - --zipvoice-data-dir=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data \ - --zipvoice-lexicon=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt \ + --zipvoice-lexicon=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt \ --zipvoice-tokens=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt \ --zipvoice-vocoder=./vocos_24khz.onnx \ --reference-audio=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/test_wavs/leijun-1.wav \ diff --git a/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc b/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc index e7358267e..7251daabd 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-offline-tts.cc @@ -29,10 +29,12 @@ Usage examples: wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2 tar xf vits-piper-en_US-amy-low.tar.bz2 +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ./bin/sherpa-onnx-offline-tts \ --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \ --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \ - --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \ + --vits-lexicon=./lexicon-en-us.txt \ --output-filename=./generated.wav \ "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar." @@ -76,12 +78,12 @@ ZipVoice TTS: wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 tar xf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx +wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt ./bin/sherpa-onnx-offline-tts \ --zipvoice-encoder=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx \ --zipvoice-decoder=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx \ - --zipvoice-data-dir=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data \ - --zipvoice-lexicon=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt \ + --zipvoice-lexicon=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt \ --zipvoice-tokens=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt \ --zipvoice-vocoder=./vocos_24khz.onnx \ --reference-audio=./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/test_wavs/leijun-1.wav \ diff --git a/sherpa-onnx/csrc/text-utils-test.cc b/sherpa-onnx/csrc/text-utils-test.cc index 06d6b7904..af97f9781 100644 --- a/sherpa-onnx/csrc/text-utils-test.cc +++ b/sherpa-onnx/csrc/text-utils-test.cc @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -135,17 +134,261 @@ TEST(RemoveInvalidUtf8Sequences, DebugSpaceFollowedByInvalidByte) { EXPECT_EQ(output, " "); // Expect `0xc4` to be removed, leaving only space } -TEST(RemoveInvalidUtf8Sequences, ByteLevelBpeTokenFragment) { - // Whisper byte-level-BPE tokens can split a multi-byte UTF-8 character - // across two tokens. Each fragment alone is invalid UTF-8 and is - // stripped; the character only survives when the fragments are joined - // first. Recognizer implementations must therefore concatenate tokens - // before calling this function (see the whisper Convert()). - std::string frag1 = "\xd7"; // leading byte of a Hebrew letter - std::string frag2 = "\xa8"; // continuation byte (frag1+frag2 is a letter) - EXPECT_EQ(RemoveInvalidUtf8Sequences(frag1), ""); - EXPECT_EQ(RemoveInvalidUtf8Sequences(frag2), ""); - EXPECT_EQ(RemoveInvalidUtf8Sequences(frag1 + frag2), "\xd7\xa8"); +TEST(SplitByAllPunctuation, English) { + auto result = SplitByAllPunctuation("Hello, world. How are you?"); + ASSERT_EQ(result.size(), 3u); + EXPECT_EQ(result[0], "Hello,"); + EXPECT_EQ(result[1], "world."); + EXPECT_EQ(result[2], "How are you?"); } +TEST(SplitByAllPunctuation, Chinese) { + auto result = SplitByAllPunctuation("你好，世界。你好吗？"); + ASSERT_EQ(result.size(), 3u); + EXPECT_EQ(result[0], "你好，"); + EXPECT_EQ(result[1], "世界。"); + EXPECT_EQ(result[2], "你好吗？"); +} + +TEST(SplitByAllPunctuation, MixedPunctuation) { + auto result = SplitByAllPunctuation("a;b:c,d"); + ASSERT_EQ(result.size(), 4u); + EXPECT_EQ(result[0], "a;"); + EXPECT_EQ(result[1], "b:"); + EXPECT_EQ(result[2], "c,"); + EXPECT_EQ(result[3], "d"); +} + +TEST(SplitByAllPunctuation, NoPunctuation) { + auto result = SplitByAllPunctuation("hello world"); + ASSERT_EQ(result.size(), 1u); + EXPECT_EQ(result[0], "hello world"); +} + +TEST(SplitByAllPunctuation, Empty) { + auto result = SplitByAllPunctuation(""); + EXPECT_EQ(result.size(), 0u); +} + +TEST(CountWords, English) { + EXPECT_EQ(CountWords("hello world"), 2); + EXPECT_EQ(CountWords("one two three four five"), 5); + EXPECT_EQ(CountWords(""), 0); +} + +TEST(CountWords, Chinese) { + // Each CJK character is one word + EXPECT_EQ(CountWords("你好世界"), 4); + EXPECT_EQ(CountWords("当夜幕降临"), 5); +} + +TEST(CountWords, Mixed) { + // Each CJK char = 1 word, each English word = 1 word + // "hello"(1) "你"(2) "好"(3) "world"(4) = 4 + EXPECT_EQ(CountWords("hello 你好 world"), 4); +} + +TEST(CountWords, Punctuation) { + // Sentence-ending punctuation counts as a word + EXPECT_EQ(CountWords("hello."), 2); + // Commas/semicolons are word boundaries, not words + EXPECT_EQ(CountWords("a,b,c"), 3); +} + +TEST(MergeShortSentencesByWords, Basic) { + std::vector sentences = {"hello", "world"}; + auto result = MergeShortSentencesByWords(sentences, 5); + // "hello" has 1 word < 5, merged with "world" → "helloworld" + ASSERT_EQ(result.size(), 1u); + EXPECT_EQ(result[0], "helloworld"); +} + +TEST(MergeShortSentencesByWords, LongEnough) { + std::vector sentences = {"one two three four five", + "six seven eight nine ten"}; + auto result = MergeShortSentencesByWords(sentences, 5); + // Both have >= 5 words, not merged + ASSERT_EQ(result.size(), 2u); + EXPECT_EQ(result[0], "one two three four five"); + EXPECT_EQ(result[1], "six seven eight nine ten"); +} + +TEST(MergeShortSentencesByWords, PunctuationOnlyMerged) { + std::vector sentences = {"hello world", "."}; + auto result = MergeShortSentencesByWords(sentences, 5); + // "." is all punctuation, merged into previous + ASSERT_EQ(result.size(), 1u); + EXPECT_EQ(result[0], "hello world."); +} + +TEST(SplitLongSentenceByWords, Short) { + auto result = SplitLongSentenceByWords("hello world", 20); + ASSERT_EQ(result.size(), 1u); + EXPECT_EQ(result[0], "hello world"); +} + +TEST(SplitLongSentenceByWords, SplitAtSpace) { + // "a b c d e" has 5 words, max=3 → split at space boundary + auto result = SplitLongSentenceByWords("a b c d e", 3); + ASSERT_EQ(result.size(), 2u); + // The space at the split boundary is consumed + EXPECT_EQ(result[0], "a b c"); + EXPECT_EQ(result[1], "d e"); +} + +TEST(SplitLongSentenceByWords, Empty) { + auto result = SplitLongSentenceByWords("", 20); + EXPECT_EQ(result.size(), 0u); +} + +TEST(SplitLongSentenceByWords, NoHangOnLeadingSpaceAfterSplit) { + // Regression: the first chunk ends on a space, so the second chunk starts + // with that space. If it then needs another split and has no internal + // boundary, the backward search finds the leading space and would set + // split_pos == start, never advancing. Must terminate and make progress. + auto result = SplitLongSentenceByWords("ab 你好世界你好世界你好世界你好世界你好世界你好", 20); + EXPECT_GT(result.size(), 1u); + + // The pieces must still cover the input without dropping content. + std::string joined; + for (const auto &p : result) joined += p; + EXPECT_EQ(joined, "ab你好世界你好世界你好世界你好世界你好世界你好"); +} + +TEST(SplitLongSentenceByWords, AdvancesOnSpaceOnlySeparator) { + // Two words separated by a single space, with max_words=1: each split lands + // on the space, so without skipping leading spaces the loop would spin. + auto result = SplitLongSentenceByWords("aa bb", 1); + EXPECT_EQ(result.size(), 2u); + EXPECT_EQ(result[0], "aa"); + EXPECT_EQ(result[1], "bb"); +} + +TEST(ParseLexiconFile, NewFormat) { + std::istringstream is( + "# comment\n" + "hello || h ə l oʊ\n" + "world || w ɜː l d\n"); + int32_t max_len = 0; + auto entries = ParseLexiconFile(is, &max_len); + ASSERT_EQ(entries.size(), 2u); + EXPECT_EQ(entries[0].key, "hello"); + // "oʊ" ends with ʊ (not digit) → split into "o" + "ʊ" + ASSERT_EQ(entries[0].phonemes.size(), 5u); + EXPECT_EQ(entries[0].phonemes[0], "h"); + EXPECT_EQ(entries[0].phonemes[1], "ə"); + EXPECT_EQ(entries[0].phonemes[2], "l"); + EXPECT_EQ(entries[0].phonemes[3], "o"); + EXPECT_EQ(entries[0].phonemes[4], "ʊ"); + EXPECT_EQ(entries[1].key, "world"); + EXPECT_EQ(max_len, 5); +} + +TEST(ParseLexiconFile, OldFormat) { + std::istringstream is( + "# comment\n" + "zhong1 z h o ng 1\n" + "guo2 g u o 2\n"); + int32_t max_len = 0; + auto entries = ParseLexiconFile(is, &max_len); + ASSERT_EQ(entries.size(), 2u); + EXPECT_EQ(entries[0].key, "zhong1"); + // "ng" ends with 'g' (letter) → kept whole + ASSERT_EQ(entries[0].phonemes.size(), 5u); + EXPECT_EQ(entries[0].phonemes[0], "z"); + EXPECT_EQ(entries[0].phonemes[1], "h"); + EXPECT_EQ(entries[0].phonemes[2], "o"); + EXPECT_EQ(entries[0].phonemes[3], "ng"); + EXPECT_EQ(entries[0].phonemes[4], "1"); + EXPECT_EQ(entries[1].key, "guo2"); + EXPECT_EQ(max_len, 6); +} + +TEST(ParseLexiconFile, MultiWord) { + std::istringstream is("New York || n j uː ˈ j ɔː k\n"); + int32_t max_len = 0; + auto entries = ParseLexiconFile(is, &max_len); + ASSERT_EQ(entries.size(), 1u); + EXPECT_EQ(entries[0].key, "new york"); + // "uː" ends with ː (not digit) → split into "u" + "ː" + ASSERT_EQ(entries[0].phonemes.size(), 9u); + EXPECT_EQ(entries[0].phonemes[0], "n"); + EXPECT_EQ(entries[0].phonemes[1], "j"); + EXPECT_EQ(entries[0].phonemes[2], "u"); + EXPECT_EQ(entries[0].phonemes[3], "ː"); + EXPECT_EQ(max_len, 8); +} + +TEST(ParseLexiconFile, PinyinKeptWhole) { + // Pinyin tokens like "ao3" end with a digit → NOT split + std::istringstream is("好 h ao3\n"); + int32_t max_len = 0; + auto entries = ParseLexiconFile(is, &max_len); + ASSERT_EQ(entries.size(), 1u); + EXPECT_EQ(entries[0].key, "好"); + ASSERT_EQ(entries[0].phonemes.size(), 2u); + EXPECT_EQ(entries[0].phonemes[0], "h"); + EXPECT_EQ(entries[0].phonemes[1], "ao3"); // kept whole +} + +TEST(ParseLexiconFile, MultiByteLastCodepoint) { + // "ɑː" is ɑ (U+0251, 2 bytes) + ː (U+02D0, 2 bytes) + // Last codepoint ː is not alphanumeric → split into ɑ + ː + // Verify we check the codepoint, not the byte. + std::istringstream is("foo || ɑː l\n"); + int32_t max_len = 0; + auto entries = ParseLexiconFile(is, &max_len); + ASSERT_EQ(entries.size(), 1u); + EXPECT_EQ(entries[0].key, "foo"); + ASSERT_EQ(entries[0].phonemes.size(), 3u); + EXPECT_EQ(entries[0].phonemes[0], "ɑ"); + EXPECT_EQ(entries[0].phonemes[1], "ː"); + EXPECT_EQ(entries[0].phonemes[2], "l"); +} + +TEST(ParseLexiconFile, EmptyAndComments) { + std::istringstream is("# comment\n\n \n# another\n"); + int32_t max_len = 0; + auto entries = ParseLexiconFile(is, &max_len); + EXPECT_EQ(entries.size(), 0u); + EXPECT_EQ(max_len, 0); +} + +TEST(ParseLexiconFile, NullMaxLen) { + std::istringstream is("hello || h ə l oʊ\n"); + auto entries = ParseLexiconFile(is, nullptr); + ASSERT_EQ(entries.size(), 1u); + EXPECT_EQ(entries[0].key, "hello"); +} + +TEST(ParseLexiconFile, MultiCodepointPhonemes) { + // "dˈ" ends with ˈ (not digit) → split into "d" + "ˈ" + // "ao3" ends with 3 (digit) → kept as "ao3" + std::istringstream is("today || t ə dˈ ao3 ɪ\n"); + int32_t max_len = 0; + auto entries = ParseLexiconFile(is, &max_len); + ASSERT_EQ(entries.size(), 1u); + EXPECT_EQ(entries[0].key, "today"); + ASSERT_EQ(entries[0].phonemes.size(), 6u); + EXPECT_EQ(entries[0].phonemes[0], "t"); + EXPECT_EQ(entries[0].phonemes[1], "ə"); + EXPECT_EQ(entries[0].phonemes[2], "d"); + EXPECT_EQ(entries[0].phonemes[3], "ˈ"); + EXPECT_EQ(entries[0].phonemes[4], "ao3"); // ends with digit, kept whole + EXPECT_EQ(entries[0].phonemes[5], "ɪ"); +} + +TEST(RemoveInvalidUtf8Sequences, ByteLevelBpeTokenFragment) { + // Whisper byte-level-BPE tokens can split a multi-byte UTF-8 character + // across two tokens. Each fragment alone is invalid UTF-8 and is + // stripped; the character only survives when the fragments are joined + // first. Recognizer implementations must therefore concatenate tokens + // before calling this function (see the whisper Convert()). + std::string frag1 = "\xd7"; // leading byte of a Hebrew letter + std::string frag2 = "\xa8"; // continuation byte (frag1+frag2 is a letter) + EXPECT_EQ(RemoveInvalidUtf8Sequences(frag1), ""); + EXPECT_EQ(RemoveInvalidUtf8Sequences(frag2), ""); + EXPECT_EQ(RemoveInvalidUtf8Sequences(frag1 + frag2), "\xd7\xa8"); + } + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/text-utils.cc b/sherpa-onnx/csrc/text-utils.cc index 5dc0139f6..b2bada64c 100644 --- a/sherpa-onnx/csrc/text-utils.cc +++ b/sherpa-onnx/csrc/text-utils.cc @@ -9,13 +9,10 @@ #include #include #include -#include -#include #include #include #include #include -#include #include #include #include @@ -1127,6 +1124,14 @@ std::string RemoveLeadingSpaces(const std::string &text) { return text.substr(start); } +std::string RemoveLeadingPunct(const std::string &text) { + std::u32string utf32 = Utf8ToUtf32(text); + if (utf32.empty() || !IsPunct(utf32[0])) { + return text; + } + return Utf32ToUtf8(utf32.substr(1)); +} + std::string GetWord(const std::vector &words, int32_t start, int32_t end) { std::string ans; @@ -1214,7 +1219,8 @@ void LengthsToMask(const std::vector &lengths, const int bsz = static_cast(lengths.size()); const int64_t max_len = *std::max_element(lengths.begin(), lengths.end()); if (max_len < 0) { - SHERPA_ONNX_LOGE("LengthsToMask: max_len (%" PRId64 ") < 0", max_len); + SHERPA_ONNX_LOGE("LengthsToMask: max_len (%d) < 0", + static_cast(max_len)); SHERPA_ONNX_EXIT(-1); } @@ -1305,8 +1311,12 @@ bool NeedSpaceBetween(const std::string &left, const std::string &right) { return false; } - if (IsCJK(last) || IsCJK(first) || IsChunkBoundary(last) || - IsChunkBoundary(first)) { + // Add space between punctuation and a following letter (e.g., "world") + if (IsChunkBoundary(last) && !IsChunkBoundary(first) && !IsSpace(first)) { + return true; + } + + if (IsCJK(last) || IsCJK(first)) { return false; } @@ -1333,6 +1343,24 @@ std::vector SplitByPunctuation(const std::string &text) { return sentences; } +std::vector SplitByAllPunctuation(const std::string &text) { + std::vector sentences; + std::u32string cur; + auto flush = [&]() { + std::string s = Trim(Utf32ToUtf8(cur)); + if (!s.empty()) sentences.emplace_back(std::move(s)); + cur.clear(); + }; + for (char32_t c : Utf8ToUtf32(text)) { + cur.push_back(c); + if (IsChunkBoundary(c)) { + flush(); + } + } + flush(); + return sentences; +} + std::vector MergeShortSentences( const std::vector &sentences, size_t min_chars) { std::vector merged; @@ -1465,4 +1493,210 @@ std::vector ChunkText(const std::string &text, size_t max_len) { return chunks; } +int32_t CountWords(const std::string &text) { + std::u32string u32 = Utf8ToUtf32(text); + int32_t count = 0; + bool in_word = false; + + for (char32_t c : u32) { + if (IsCJK(c) || IsSentenceBoundary(c)) { + // Each CJK character or sentence-ending punctuation is a word + count++; + in_word = false; + } else if (IsSpace(c) || c == U',' || c == U';' || c == U':' || + c == U'，' || c == U'；' || c == U'：') { + // Word boundary + in_word = false; + } else { + if (!in_word) { + count++; + in_word = true; + } + } + } + + return count; +} + +std::vector MergeShortSentencesByWords( + const std::vector &sentences, size_t min_words) { + std::vector merged; + std::string buffer; + + for (const auto &s : sentences) { + std::string piece = Trim(s); + if (piece.empty()) continue; + + // Don't add space between pieces — preserve original spacing. + // SplitByAllPunctuation strips trailing space from each piece, + // so punctuation runs directly into the next text (correct for CJK). + buffer += piece; + + if (static_cast(CountWords(buffer)) >= min_words) { + merged.push_back(Trim(buffer)); + buffer.clear(); + } + } + + if (!buffer.empty()) { + merged.push_back(Trim(buffer)); + } + + return merged; +} + +std::vector SplitLongSentenceByWords(const std::string &sentence, + size_t max_words) { + std::vector chunks; + if (max_words == 0) return chunks; + std::string s = Trim(sentence); + if (s.empty()) return chunks; + + std::u32string u32 = Utf8ToUtf32(s); + size_t start = 0; + const size_t len = u32.size(); + + while (start < len) { + // Count words from start + int32_t word_count = 0; + bool in_word = false; + size_t end = start; + + for (size_t i = start; i < len; ++i) { + char32_t c = u32[i]; + if (IsCJK(c) || IsSentenceBoundary(c)) { + word_count++; + in_word = false; + } else if (IsSpace(c) || c == U',' || c == U';' || c == U':' || + c == U'，' || c == U'；' || c == U'：') { + in_word = false; + } else { + if (!in_word) { + word_count++; + in_word = true; + } + } + + if (static_cast(word_count) > max_words) { + break; + } + end = i + 1; + } + + if (end >= len) { + std::string piece = Trim(Utf32ToUtf8(u32.substr(start))); + if (!piece.empty()) chunks.emplace_back(std::move(piece)); + break; + } + + // Try to split at punctuation or space boundary + size_t split_pos = end; + bool found = false; + for (size_t i = end; i > start; --i) { + char32_t c = u32[i - 1]; + if (IsSpace(c)) { + split_pos = i - 1; + found = true; + break; + } + if (IsChunkBoundary(c)) { + split_pos = i; + found = true; + break; + } + } + + if (!found || split_pos <= start) { + // Force split at end. The `split_pos <= start` case happens when the + // only boundary found is the separator we started on; splitting there + // would not advance `start` and the loop below would never terminate. + split_pos = end; + } + + std::string piece = Trim(Utf32ToUtf8(u32.substr(start, split_pos - start))); + if (!piece.empty()) chunks.emplace_back(std::move(piece)); + + start = split_pos; + while (start < len && IsSpace(u32[start])) { + ++start; + } + } + + return chunks; +} + +std::vector ParseLexiconFile(std::istream &is, + int32_t *max_phrase_len_out) { + std::vector entries; + int32_t max_phrase_len = 0; + + std::string line; + while (std::getline(is, line)) { + if (line.empty() || line[0] == '#') continue; + + std::string key; + std::vector phonemes; + + auto pos = line.find("||"); + if (pos != std::string::npos) { + // New-format: word || phone1 phone2 ... + key = ToLowerCase(Trim(line.substr(0, pos))); + std::istringstream pss(Trim(line.substr(pos + 2))); + std::string phone; + while (pss >> phone) { + phonemes.push_back(std::move(phone)); + } + } else { + // Old-format: word phone1 phone2 ... + std::istringstream iss(line); + std::string word; + iss >> word; + if (word.empty()) continue; + key = ToLowerCase(word); + std::string phone; + while (iss >> phone) { + phonemes.push_back(std::move(phone)); + } + } + + if (key.empty() || phonemes.empty()) continue; + + int32_t word_count = static_cast(Utf8ToUtf32(key).size()); + if (word_count > max_phrase_len) { + max_phrase_len = word_count; + } + + // Split multi-codepoint phonemes into individual characters, + // but only if the last codepoint is NOT a digit (0-9) or letter (a-z). + // Pinyin tokens like "ao3", "zhong1", "de" end with digit/letter → keep. + // IPA phonemes like "dˈ", "oʊ", "ɑː" end with non-alphanumeric → split. + std::vector split_phonemes; + for (const auto &p : phonemes) { + std::u32string u32 = Utf8ToUtf32(p); + bool keep = u32.size() <= 1; + if (!keep) { + char32_t last = u32.back(); + keep = (last >= U'0' && last <= U'9') || + (last >= U'a' && last <= U'z') || + (last >= U'A' && last <= U'Z'); + } + if (keep) { + split_phonemes.push_back(p); + } else { + for (char32_t cp : u32) { + split_phonemes.push_back(Utf32ToUtf8(std::u32string(1, cp))); + } + } + } + + entries.push_back({std::move(key), std::move(split_phonemes)}); + } + + if (max_phrase_len_out) { + *max_phrase_len_out = max_phrase_len; + } + + return entries; +} + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/text-utils.h b/sherpa-onnx/csrc/text-utils.h index 7eddcf127..95155a79d 100644 --- a/sherpa-onnx/csrc/text-utils.h +++ b/sherpa-onnx/csrc/text-utils.h @@ -192,6 +192,9 @@ std::string RemoveSpaceBetweenCjk(const std::string &text); std::string RemoveLeadingSpaces(const std::string &text); +// If the first character is a punctuation, drop it. +std::string RemoveLeadingPunct(const std::string &text); + bool StringToBool(const std::string &s); // end is inclusive @@ -219,12 +222,40 @@ void LengthsToMask(const std::vector &lengths, // TTS text chunking helpers. std::vector SplitByBlankLines(const std::string &text); std::vector SplitByPunctuation(const std::string &text); +// Split text at all punctuation boundaries: sentence-ending (.!?) and +// mid-sentence (,;: and their Chinese equivalents). +std::vector SplitByAllPunctuation(const std::string &text); std::vector MergeShortSentences( const std::vector &sentences, size_t min_chars); std::vector SplitLongSentence(const std::string &sentence, size_t max_chars); std::vector ChunkText(const std::string &text, size_t max_len); +// Count words in text. Each CJK character counts as one word. +// English words are sequences of non-CJK, non-space, non-punctuation chars. +int32_t CountWords(const std::string &text); + +// Word-based versions of MergeShortSentences and SplitLongSentence. +// min_words/max_words are word counts (not codepoint counts). +// Each CJK character is one word; English words are space-separated. +std::vector MergeShortSentencesByWords( + const std::vector &sentences, size_t min_words); +std::vector SplitLongSentenceByWords(const std::string &sentence, + size_t max_words); + +// A single lexicon entry: word (lowercased) and its phonemes as strings. +struct LexiconEntry { + std::string key; // lowercased word/phrase + std::vector phonemes; // phoneme strings (IPA or token names) +}; + +// Parse a lexicon file. Supports both formats: +// - New-format (||): word1 word2 || phone1 phone2 phone3 ... +// - Old-format (space): word phone1 phone2 ... +// Returns all entries and the max phrase length (in words/tokens). +std::vector ParseLexiconFile(std::istream &is, + int32_t *max_phrase_len_out); + } // namespace sherpa_onnx #endif // SHERPA_ONNX_CSRC_TEXT_UTILS_H_ diff --git a/sherpa-onnx/csrc/tts-text-normalizer.cc b/sherpa-onnx/csrc/tts-text-normalizer.cc new file mode 100644 index 000000000..3e5a1906a --- /dev/null +++ b/sherpa-onnx/csrc/tts-text-normalizer.cc @@ -0,0 +1,154 @@ +// sherpa-onnx/csrc/tts-text-normalizer.cc +// +// Copyright (c) 2026 Xiaomi Corporation + +#include "sherpa-onnx/csrc/tts-text-normalizer.h" + +#include +#include +#include +#include + +#if __ANDROID_API__ >= 9 +#include "android/asset_manager.h" +#include "android/asset_manager_jni.h" +#endif + +#if __OHOS__ +#include "rawfile/raw_file_manager.h" +#endif + +#include "fst/extensions/far/far.h" +#include "kaldifst/csrc/kaldi-fst-io.h" +#include "kaldifst/csrc/text-normalizer.h" +#include "sherpa-onnx/csrc/file-utils.h" +#include "sherpa-onnx/csrc/fst-utils.h" +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/text-utils.h" + +namespace sherpa_onnx { + +std::vector> LoadTextNormalizers( + const std::string &rule_fsts, const std::string &rule_fars, bool debug) { + std::vector> tn_list; + + if (!rule_fsts.empty()) { + std::vector files; + SplitStringToVector(rule_fsts, ",", false, &files); + tn_list.reserve(files.size()); + for (const auto &f : files) { + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); +#endif + } + tn_list.push_back(std::make_unique(f)); + } + } + + if (!rule_fars.empty()) { + if (debug) { + SHERPA_ONNX_LOGE("Loading FST archives"); + } + std::vector files; + SplitStringToVector(rule_fars, ",", false, &files); + + tn_list.reserve(files.size() + tn_list.size()); + + for (const auto &f : files) { + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); +#endif + } + std::unique_ptr> reader( + fst::FarReader::Open(f)); + if (!reader) { + SHERPA_ONNX_LOGE("Failed to open FAR file: %s", f.c_str()); + continue; + } + for (; !reader->Done(); reader->Next()) { + std::unique_ptr r( + fst::CastOrConvertToConstFst(reader->GetFst()->Copy())); + + tn_list.push_back( + std::make_unique(std::move(r))); + } + } + + if (debug) { + SHERPA_ONNX_LOGE("FST archives loaded!"); + } + } + + return tn_list; +} + +template +std::vector> LoadTextNormalizers( + Manager *mgr, const std::string &rule_fsts, const std::string &rule_fars, + bool debug) { + std::vector> tn_list; + + if (!rule_fsts.empty()) { + std::vector files; + SplitStringToVector(rule_fsts, ",", false, &files); + tn_list.reserve(files.size()); + for (const auto &f : files) { + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule fst: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule fst: %s", f.c_str()); +#endif + } + auto buf = ReadFile(mgr, f); + std::istringstream is(std::string(buf.data(), buf.size())); + tn_list.push_back(std::make_unique(is)); + } + } + + if (!rule_fars.empty()) { + std::vector files; + SplitStringToVector(rule_fars, ",", false, &files); + tn_list.reserve(files.size() + tn_list.size()); + + for (const auto &f : files) { + if (debug) { +#if __OHOS__ + SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str()); +#else + SHERPA_ONNX_LOGE("rule far: %s", f.c_str()); +#endif + } + + auto buf = ReadFile(mgr, f); + + auto fsts = ReadFstsFromFar(buf); + for (auto &r : fsts) { + tn_list.push_back( + std::make_unique(std::move(r))); + } + } + } + + return tn_list; +} + +#if __ANDROID_API__ >= 9 +template std::vector> +LoadTextNormalizers(AAssetManager *mgr, const std::string &rule_fsts, + const std::string &rule_fars, bool debug); +#endif + +#if __OHOS__ +template std::vector> +LoadTextNormalizers(NativeResourceManager *mgr, const std::string &rule_fsts, + const std::string &rule_fars, bool debug); +#endif + +} // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/tts-text-normalizer.h b/sherpa-onnx/csrc/tts-text-normalizer.h new file mode 100644 index 000000000..cd70fe344 --- /dev/null +++ b/sherpa-onnx/csrc/tts-text-normalizer.h @@ -0,0 +1,26 @@ +// sherpa-onnx/csrc/tts-text-normalizer.h +// +// Copyright (c) 2026 Xiaomi Corporation + +#ifndef SHERPA_ONNX_CSRC_TTS_TEXT_NORMALIZER_H_ +#define SHERPA_ONNX_CSRC_TTS_TEXT_NORMALIZER_H_ + +#include +#include +#include + +#include "kaldifst/csrc/text-normalizer.h" + +namespace sherpa_onnx { + +std::vector> LoadTextNormalizers( + const std::string &rule_fsts, const std::string &rule_fars, bool debug); + +template +std::vector> LoadTextNormalizers( + Manager *mgr, const std::string &rule_fsts, const std::string &rule_fars, + bool debug); + +} // namespace sherpa_onnx + +#endif // SHERPA_ONNX_CSRC_TTS_TEXT_NORMALIZER_H_ diff --git a/sherpa-onnx/csrc/vocoder.h b/sherpa-onnx/csrc/vocoder.h index b281af3b1..6c005f11a 100644 --- a/sherpa-onnx/csrc/vocoder.h +++ b/sherpa-onnx/csrc/vocoder.h @@ -28,6 +28,9 @@ class Vocoder { * @return Return a float32 vector containing audio samples.. */ virtual std::vector Run(Ort::Value mel) const = 0; + + /** @return Return the sample rate of the vocoder. */ + virtual int32_t SampleRate() const = 0; }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/vocos-vocoder.cc b/sherpa-onnx/csrc/vocos-vocoder.cc index 9b078d048..a46a919a0 100644 --- a/sherpa-onnx/csrc/vocos-vocoder.cc +++ b/sherpa-onnx/csrc/vocos-vocoder.cc @@ -33,6 +33,7 @@ struct VocosModelMetaData { int32_t win_length; int32_t center; int32_t normalized; + int32_t sample_rate; std::string window_type; std::string pad_mode; }; @@ -124,6 +125,8 @@ class VocosVocoder::Impl { return istft.Compute(stft_result); } + int32_t SampleRate() const { return meta_.sample_rate; } + private: void Init(void *model_data, size_t model_data_length) { if (model_data) { @@ -179,6 +182,8 @@ class VocosVocoder::Impl { "window_type", "hann"); SHERPA_ONNX_READ_META_DATA_STR_WITH_DEFAULT(meta_.pad_mode, "pad_mode", "reflect"); + SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_.sample_rate, "sample_rate", + 22050); } private: @@ -211,6 +216,8 @@ std::vector VocosVocoder::Run(Ort::Value mel) const { return impl_->Run(std::move(mel)); } +int32_t VocosVocoder::SampleRate() const { return impl_->SampleRate(); } + #if __ANDROID_API__ >= 9 template VocosVocoder::VocosVocoder(AAssetManager *mgr, const OfflineTtsModelConfig &config); diff --git a/sherpa-onnx/csrc/vocos-vocoder.h b/sherpa-onnx/csrc/vocos-vocoder.h index 1dcc00401..30c3b4e6b 100644 --- a/sherpa-onnx/csrc/vocos-vocoder.h +++ b/sherpa-onnx/csrc/vocos-vocoder.h @@ -29,6 +29,8 @@ class VocosVocoder : public Vocoder { */ std::vector Run(Ort::Value mel) const override; + int32_t SampleRate() const override; + private: class Impl; std::unique_ptr impl_; diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/GenerationConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/GenerationConfig.java index 1932fcfb7..55e3ca04d 100644 --- a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/GenerationConfig.java +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/GenerationConfig.java @@ -28,6 +28,19 @@ public class GenerationConfig { /** Extra model-specific key-value pairs. Can be null. */ private Map extra = null; + /** + * Pre-phonemized input as Unicode codepoints. One inner array per sentence. + * When set, it is used in place of the text argument. Mutually exclusive + * with {@link #tokens}. + */ + private int[][] phonemeCodepoints = null; + + /** + * Pre-tokenized input, e.g. Chinese pinyin like "zhong1". One inner array + * per sentence. Mutually exclusive with {@link #phonemeCodepoints}. + */ + private String[][] tokens = null; + /** Default constructor */ public GenerationConfig() { } @@ -65,6 +78,14 @@ public class GenerationConfig { return extra; } + public int[][] getPhonemeCodepoints() { + return phonemeCodepoints; + } + + public String[][] getTokens() { + return tokens; + } + /** Setters */ public void setSilenceScale(float silenceScale) { this.silenceScale = silenceScale; @@ -98,6 +119,14 @@ public class GenerationConfig { this.extra = extra; } + public void setPhonemeCodepoints(int[][] phonemeCodepoints) { + this.phonemeCodepoints = phonemeCodepoints; + } + + public void setTokens(String[][] tokens) { + this.tokens = tokens; + } + @Override public String toString() { return "GenerationConfig{" + @@ -109,6 +138,10 @@ public class GenerationConfig { ", referenceText='" + referenceText + '\'' + ", numSteps=" + numSteps + ", extra=" + extra + + ", phonemeCodepoints=" + (phonemeCodepoints != null ? phonemeCodepoints.length : 0) + + " sentences" + + ", tokens=" + (tokens != null ? tokens.length : 0) + + " sentences" + '}'; } } diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTts.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTts.java index a9f1b217f..5e4d10cca 100644 --- a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTts.java +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTts.java @@ -24,6 +24,11 @@ public class OfflineTts { return getNumSpeakers(ptr); } + /** Returns the language/voice of the model, e.g., "en-us". Empty if not available. */ + public String getLang() { + return getLang(ptr); + } + /** Generates audio for the given text using the default speaker (sid=0) and speed=1.0. */ public GeneratedAudio generate(String text) { return generate(text, 0, 1.0f); @@ -80,6 +85,18 @@ public class OfflineTts { return generateWithCallback(text, sid, speed, cb); } + /** + * Generate audio using a GenerationConfig. + * + * @param text The text to synthesize. Pass "" if the input is + * pre-phonemized via GenerationConfig.phonemeCodepoints. + * @param config The generation configuration. + * @return GeneratedAudio with samples and sample rate. + */ + public GeneratedAudio generateWithConfig(String text, GenerationConfig config) { + return generateWithConfigImpl(ptr, text, config, null); + } + /** * Generate audio using a GenerationConfig and a callback. * @@ -128,6 +145,8 @@ public class OfflineTts { private native int getNumSpeakers(long ptr); + private native String getLang(long ptr); + private native GeneratedAudio generateImpl(long ptr, String text, int sid, float speed); private native GeneratedAudio generateWithCallbackImpl(long ptr, String text, int sid, float speed, OfflineTtsCallback callback); diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsKittenModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsKittenModelConfig.java index 17ef40217..08f807c3e 100644 --- a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsKittenModelConfig.java +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsKittenModelConfig.java @@ -7,6 +7,7 @@ public class OfflineTtsKittenModelConfig { private final String tokens; private final String dataDir; private final float lengthScale; + private final String lexicon; private OfflineTtsKittenModelConfig(Builder builder) { this.model = builder.model; @@ -14,6 +15,7 @@ public class OfflineTtsKittenModelConfig { this.tokens = builder.tokens; this.dataDir = builder.dataDir; this.lengthScale = builder.lengthScale; + this.lexicon = builder.lexicon; } public static Builder builder() { @@ -32,6 +34,11 @@ public class OfflineTtsKittenModelConfig { return tokens; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public String getDataDir() { return dataDir; } @@ -40,6 +47,10 @@ public class OfflineTtsKittenModelConfig { return lengthScale; } + public String getLexicon() { + return lexicon; + } + public static class Builder { private String model = ""; @@ -47,6 +58,7 @@ public class OfflineTtsKittenModelConfig { private String tokens = ""; private String dataDir = ""; private float lengthScale = 1.0f; + private String lexicon = ""; public OfflineTtsKittenModelConfig build() { return new OfflineTtsKittenModelConfig(this); @@ -67,6 +79,11 @@ public class OfflineTtsKittenModelConfig { return this; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public Builder setDataDir(String dataDir) { this.dataDir = dataDir; return this; @@ -76,5 +93,10 @@ public class OfflineTtsKittenModelConfig { this.lengthScale = lengthScale; return this; } + + public Builder setLexicon(String lexicon) { + this.lexicon = lexicon; + return this; + } } -} \ No newline at end of file +} diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsKokoroModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsKokoroModelConfig.java index 3c88fe824..f5e3bc2a2 100644 --- a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsKokoroModelConfig.java +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsKokoroModelConfig.java @@ -38,6 +38,11 @@ public class OfflineTtsKokoroModelConfig { return tokens; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public String getDataDir() { return dataDir; } @@ -86,6 +91,11 @@ public class OfflineTtsKokoroModelConfig { return this; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public Builder setDataDir(String dataDir) { this.dataDir = dataDir; return this; diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsMatchaModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsMatchaModelConfig.java index 80b43d31c..e359cc1ea 100644 --- a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsMatchaModelConfig.java +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsMatchaModelConfig.java @@ -43,6 +43,11 @@ public class OfflineTtsMatchaModelConfig { return tokens; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public String getDataDir() { return dataDir; } @@ -93,6 +98,11 @@ public class OfflineTtsMatchaModelConfig { return this; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public Builder setDataDir(String dataDir) { this.dataDir = dataDir; return this; diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java index 8445b5809..922834e66 100644 --- a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java @@ -68,7 +68,7 @@ public class OfflineTtsModelConfig { private OfflineTtsPocketModelConfig pocket = OfflineTtsPocketModelConfig.builder().build(); private OfflineTtsSupertonicModelConfig supertonic = OfflineTtsSupertonicModelConfig.builder().build(); private int numThreads = 1; - private boolean debug = true; + private boolean debug = false; private String provider = "cpu"; public OfflineTtsModelConfig build() { diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsVitsModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsVitsModelConfig.java index aad3a2540..456854016 100644 --- a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsVitsModelConfig.java +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsVitsModelConfig.java @@ -39,6 +39,11 @@ public class OfflineTtsVitsModelConfig { return tokens; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public String getDataDir() { return dataDir; } @@ -88,6 +93,11 @@ public class OfflineTtsVitsModelConfig { return this; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public Builder setDataDir(String dataDir) { this.dataDir = dataDir; return this; diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsZipVoiceModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsZipVoiceModelConfig.java index f8a02a67d..2de3e74f3 100644 --- a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsZipVoiceModelConfig.java +++ b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsZipVoiceModelConfig.java @@ -47,6 +47,11 @@ public class OfflineTtsZipVoiceModelConfig { return vocoder; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public String getDataDir() { return dataDir; } @@ -107,6 +112,11 @@ public class OfflineTtsZipVoiceModelConfig { return this; } + /** + * @deprecated Deprecated. Ignored since v2.0.0. Use lexicon or + * phonemeCodepoints instead. + */ + @Deprecated public Builder setDataDir(String dataDir) { this.dataDir = dataDir; return this; diff --git a/sherpa-onnx/jni/offline-tts.cc b/sherpa-onnx/jni/offline-tts.cc index 39feb3a37..dae62f2c2 100644 --- a/sherpa-onnx/jni/offline-tts.cc +++ b/sherpa-onnx/jni/offline-tts.cc @@ -111,6 +111,72 @@ static GenerationConfig GetGenerationConfig(JNIEnv *env, jobject config_obj) { } } + // phonemeCodepoints: int[][], one inner array per sentence. Optional; a + // missing field (older managed class) is ignored. + fid = env->GetFieldID(cls, "phonemeCodepoints", "[[I"); + if (fid == nullptr || env->ExceptionCheck()) { + if (env->ExceptionCheck()) { + env->ExceptionClear(); + } + } else { + jobjectArray outer = (jobjectArray)env->GetObjectField(config_obj, fid); + if (outer != nullptr) { + jsize num_sentences = env->GetArrayLength(outer); + ans.phoneme_codepoints.reserve(num_sentences); + for (jsize i = 0; i < num_sentences; ++i) { + jintArray inner = (jintArray)env->GetObjectArrayElement(outer, i); + if (inner == nullptr) { + ans.phoneme_codepoints.emplace_back(); + continue; + } + jsize len = env->GetArrayLength(inner); + std::vector sentence(len); + if (len > 0) { + env->GetIntArrayRegion(inner, 0, len, sentence.data()); + } + ans.phoneme_codepoints.push_back(std::move(sentence)); + env->DeleteLocalRef(inner); + } + env->DeleteLocalRef(outer); + } + } + + // tokens: String[][], one inner array per sentence. Optional. + fid = env->GetFieldID(cls, "tokens", "[[Ljava/lang/String;"); + if (fid == nullptr || env->ExceptionCheck()) { + if (env->ExceptionCheck()) { + env->ExceptionClear(); + } + } else { + jobjectArray outer = (jobjectArray)env->GetObjectField(config_obj, fid); + if (outer != nullptr) { + jsize num_sentences = env->GetArrayLength(outer); + ans.tokens.reserve(num_sentences); + for (jsize i = 0; i < num_sentences; ++i) { + jobjectArray inner = (jobjectArray)env->GetObjectArrayElement(outer, i); + if (inner == nullptr) { + ans.tokens.emplace_back(); + continue; + } + jsize len = env->GetArrayLength(inner); + std::vector sentence; + sentence.reserve(len); + for (jsize j = 0; j < len; ++j) { + jstring s = (jstring)env->GetObjectArrayElement(inner, j); + if (s != nullptr) { + const char *p = env->GetStringUTFChars(s, nullptr); + sentence.emplace_back(p); + env->ReleaseStringUTFChars(s, p); + env->DeleteLocalRef(s); + } + } + ans.tokens.push_back(std::move(sentence)); + env->DeleteLocalRef(inner); + } + env->DeleteLocalRef(outer); + } + } + env->DeleteLocalRef(cls); return ans; } @@ -202,9 +268,9 @@ static OfflineTtsConfig GetOfflineTtsConfig(JNIEnv *env, jobject config, kokoro_cls, kokoro); // zipvoice - fid = env->GetFieldID( - model_config_cls, "zipvoice", - "Lcom/k2fsa/sherpa/onnx/OfflineTtsZipVoiceModelConfig;"); + fid = + env->GetFieldID(model_config_cls, "zipvoice", + "Lcom/k2fsa/sherpa/onnx/OfflineTtsZipVoiceModelConfig;"); jobject zipvoice = env->GetObjectField(model, fid); jclass zipvoice_cls = env->GetObjectClass(zipvoice); @@ -220,8 +286,8 @@ static OfflineTtsConfig GetOfflineTtsConfig(JNIEnv *env, jobject config, SHERPA_ONNX_JNI_READ_STRING(ans.model.zipvoice.vocoder, vocoder, zipvoice_cls, zipvoice); - SHERPA_ONNX_JNI_READ_STRING(ans.model.zipvoice.data_dir, dataDir, zipvoice_cls, - zipvoice); + SHERPA_ONNX_JNI_READ_STRING(ans.model.zipvoice.data_dir, dataDir, + zipvoice_cls, zipvoice); SHERPA_ONNX_JNI_READ_STRING(ans.model.zipvoice.lexicon, lexicon, zipvoice_cls, zipvoice); @@ -259,6 +325,9 @@ static OfflineTtsConfig GetOfflineTtsConfig(JNIEnv *env, jobject config, SHERPA_ONNX_JNI_READ_FLOAT(ans.model.kitten.length_scale, lengthScale, kitten_cls, kitten); + SHERPA_ONNX_JNI_READ_STRING(ans.model.kitten.lexicon, lexicon, kitten_cls, + kitten); + // pocket fid = env->GetFieldID(model_config_cls, "pocket", "Lcom/k2fsa/sherpa/onnx/OfflineTtsPocketModelConfig;"); @@ -509,11 +578,22 @@ JNIEXPORT jint JNICALL Java_com_k2fsa_sherpa_onnx_OfflineTts_getNumSpeakers( return reinterpret_cast(ptr)->NumSpeakers(); } +SHERPA_ONNX_EXTERN_C +JNIEXPORT jstring JNICALL Java_com_k2fsa_sherpa_onnx_OfflineTts_getLang( + JNIEnv *env, jobject /*obj*/, jlong ptr) { + std::string lang = reinterpret_cast(ptr)->Lang(); + return env->NewStringUTF(lang.c_str()); +} + SHERPA_ONNX_EXTERN_C JNIEXPORT jobject JNICALL Java_com_k2fsa_sherpa_onnx_OfflineTts_generateImpl( JNIEnv *env, jobject /*obj*/, jlong ptr, jstring text, jint sid, jfloat speed) { - const char *p_text = env->GetStringUTFChars(text, nullptr); + // `text` may be null when the caller supplies phonemeCodepoints or + // tokens instead. Treat it as empty rather than crashing in + // GetStringUTFChars(nullptr). + const char *p_text = + text != nullptr ? env->GetStringUTFChars(text, nullptr) : ""; sherpa_onnx::GenerationConfig config; config.sid = sid; @@ -522,7 +602,9 @@ JNIEXPORT jobject JNICALL Java_com_k2fsa_sherpa_onnx_OfflineTts_generateImpl( auto audio = reinterpret_cast(ptr)->Generate( p_text, config); - env->ReleaseStringUTFChars(text, p_text); + if (text != nullptr) { + env->ReleaseStringUTFChars(text, p_text); + } return CreateAudioObject(env, audio.samples, audio.sample_rate); } @@ -532,7 +614,11 @@ JNIEXPORT jobject JNICALL Java_com_k2fsa_sherpa_onnx_OfflineTts_generateWithCallbackImpl( JNIEnv *env, jobject /*obj*/, jlong ptr, jstring text, jint sid, jfloat speed, jobject callback) { - const char *p_text = env->GetStringUTFChars(text, nullptr); + // `text` may be null when the caller supplies phonemeCodepoints or + // tokens instead. Treat it as empty rather than crashing in + // GetStringUTFChars(nullptr). + const char *p_text = + text != nullptr ? env->GetStringUTFChars(text, nullptr) : ""; auto tts = reinterpret_cast(ptr); @@ -557,7 +643,9 @@ Java_com_k2fsa_sherpa_onnx_OfflineTts_generateWithCallbackImpl( audio = tts->Generate(p_text, config, nullptr); } - env->ReleaseStringUTFChars(text, p_text); + if (text != nullptr) { + env->ReleaseStringUTFChars(text, p_text); + } return CreateAudioObject(env, audio.samples, audio.sample_rate); } @@ -567,7 +655,11 @@ JNIEXPORT jobject JNICALL Java_com_k2fsa_sherpa_onnx_OfflineTts_generateWithConfigImpl( JNIEnv *env, jobject /*obj*/, jlong ptr, jstring text, jobject _gen_config, jobject callback) { - const char *p_text = env->GetStringUTFChars(text, nullptr); + // `text` may be null when the caller supplies phonemeCodepoints or + // tokens instead. Treat it as empty rather than crashing in + // GetStringUTFChars(nullptr). + const char *p_text = + text != nullptr ? env->GetStringUTFChars(text, nullptr) : ""; auto gen_config = sherpa_onnx::GetGenerationConfig(env, _gen_config); auto tts = reinterpret_cast(ptr); @@ -588,7 +680,9 @@ Java_com_k2fsa_sherpa_onnx_OfflineTts_generateWithConfigImpl( audio = tts->Generate(p_text, gen_config, nullptr); } - env->ReleaseStringUTFChars(text, p_text); + if (text != nullptr) { + env->ReleaseStringUTFChars(text, p_text); + } return CreateAudioObject(env, audio.samples, audio.sample_rate); } diff --git a/sherpa-onnx/jni/sherpa-onnx-symbols.exp b/sherpa-onnx/jni/sherpa-onnx-symbols.exp index e7c74f3b9..659b7e035 100644 --- a/sherpa-onnx/jni/sherpa-onnx-symbols.exp +++ b/sherpa-onnx/jni/sherpa-onnx-symbols.exp @@ -52,6 +52,7 @@ _Java_com_k2fsa_sherpa_onnx_OfflineTts_delete _Java_com_k2fsa_sherpa_onnx_OfflineTts_generateImpl _Java_com_k2fsa_sherpa_onnx_OfflineTts_generateWithCallbackImpl _Java_com_k2fsa_sherpa_onnx_OfflineTts_generateWithConfigImpl +_Java_com_k2fsa_sherpa_onnx_OfflineTts_getLang _Java_com_k2fsa_sherpa_onnx_OfflineTts_getNumSpeakers _Java_com_k2fsa_sherpa_onnx_OfflineTts_getSampleRate _Java_com_k2fsa_sherpa_onnx_OfflineTts_newFromAsset diff --git a/sherpa-onnx/kotlin-api/Tts.kt b/sherpa-onnx/kotlin-api/Tts.kt index d5dd08ace..33f22bb21 100644 --- a/sherpa-onnx/kotlin-api/Tts.kt +++ b/sherpa-onnx/kotlin-api/Tts.kt @@ -7,7 +7,7 @@ data class OfflineTtsVitsModelConfig( var model: String = "", var lexicon: String = "", var tokens: String = "", - var dataDir: String = "", + var dataDir: String = "", // deprecated, ignored; use lexicon or phonemeCodepoints var dictDir: String = "", // unused var noiseScale: Float = 0.667f, var noiseScaleW: Float = 0.8f, @@ -19,7 +19,7 @@ data class OfflineTtsMatchaModelConfig( var vocoder: String = "", var lexicon: String = "", var tokens: String = "", - var dataDir: String = "", + var dataDir: String = "", // deprecated, ignored; use lexicon or phonemeCodepoints var dictDir: String = "", // unused var noiseScale: Float = 1.0f, var lengthScale: Float = 1.0f, @@ -29,7 +29,7 @@ data class OfflineTtsKokoroModelConfig( var model: String = "", var voices: String = "", var tokens: String = "", - var dataDir: String = "", + var dataDir: String = "", // deprecated, ignored; use lexicon or phonemeCodepoints var lexicon: String = "", var lang: String = "", var dictDir: String = "", // unused @@ -41,7 +41,7 @@ data class OfflineTtsZipVoiceModelConfig( var encoder: String = "", var decoder: String = "", var vocoder: String = "", - var dataDir: String = "", + var dataDir: String = "", // deprecated, ignored; use lexicon or phonemeCodepoints var lexicon: String = "", var featScale: Float = 0.1f, var tShift: Float = 0.5f, @@ -53,8 +53,9 @@ data class OfflineTtsKittenModelConfig( var model: String = "", var voices: String = "", var tokens: String = "", - var dataDir: String = "", + var dataDir: String = "", // deprecated, ignored; use lexicon or phonemeCodepoints var lengthScale: Float = 1.0f, + var lexicon: String = "", ) /** @@ -135,7 +136,14 @@ data class GenerationConfig( var referenceSampleRate: Int = 0, var referenceText: String? = null, var numSteps: Int = 5, - var extra: Map? = null + var extra: Map? = null, + // Pre-phonemized input as Unicode codepoints. One inner array per + // sentence. When set, it is used in place of the text argument. + // Mutually exclusive with tokens. + var phonemeCodepoints: Array? = null, + // Pre-tokenized input, e.g. Chinese pinyin like "zhong1". One inner array + // per sentence. Mutually exclusive with phonemeCodepoints. + var tokens: Array>? = null ) class OfflineTts( @@ -159,6 +167,9 @@ class OfflineTts( fun numSpeakers() = getNumSpeakers(ptr) + // Language/voice of the model, e.g., "en-us". Empty if not available. + fun lang() = getLang(ptr) + fun generate( text: String, sid: Int = 0, @@ -238,6 +249,7 @@ class OfflineTts( private external fun delete(ptr: Long) private external fun getSampleRate(ptr: Long): Int private external fun getNumSpeakers(ptr: Long): Int + private external fun getLang(ptr: Long): String // The returned array has two entries: // - the first entry is an 1-D float array containing audio samples. @@ -278,15 +290,15 @@ class OfflineTts( // to download models fun getOfflineTtsConfig( modelDir: String, - modelName: String, // for VITS - acousticModelName: String, // for Matcha - vocoder: String, // for Matcha - voices: String, // for Kokoro or kitten - lexicon: String, - dataDir: String, - dictDir: String, // unused - ruleFsts: String, - ruleFars: String, + modelName: String = "", // for VITS + acousticModelName: String = "", // for Matcha + vocoder: String = "", // for Matcha + voices: String = "", // for Kokoro or kitten + lexicon: String = "", + dataDir: String = "", // deprecated, ignored; use lexicon or phonemeCodepoints + dictDir: String = "", // unused + ruleFsts: String = "", + ruleFars: String = "", numThreads: Int? = null, isKitten: Boolean = false, isSupertonic: Boolean = false, @@ -297,6 +309,23 @@ fun getOfflineTtsConfig( ttsJson: String = "", // for Supertonic unicodeIndexer: String = "", // for Supertonic voiceStyle: String = "", // for Supertonic + isZipvoice: Boolean = false, + zipvoiceEncoder: String = "", // for ZipVoice + zipvoiceDecoder: String = "", // for ZipVoice + zipvoiceVocoder: String = "", // for ZipVoice + featScale: Float = 0.1f, // for ZipVoice + tShift: Float = 0.5f, // for ZipVoice + targetRms: Float = 0.1f, // for ZipVoice + guidanceScale: Float = 1.0f, // for ZipVoice + isPocket: Boolean = false, + lmFlow: String = "", // for Pocket + lmMain: String = "", // for Pocket + pocketEncoder: String = "", // for Pocket + pocketDecoder: String = "", // for Pocket + textConditioner: String = "", // for Pocket + vocabJson: String = "", // for Pocket + tokenScoresJson: String = "", // for Pocket + voiceEmbeddingCacheCapacity: Int = 50, // for Pocket ): OfflineTtsConfig { // For Matcha TTS, please set // acousticModelName, vocoder @@ -323,7 +352,9 @@ fun getOfflineTtsConfig( 2 } - if (!isSupertonic && modelName.isEmpty() && acousticModelName.isEmpty()) { + if (!isSupertonic && !isZipvoice && !isPocket && modelName.isEmpty() && + acousticModelName.isEmpty() + ) { throw IllegalArgumentException("Please specify a TTS model") } @@ -335,10 +366,28 @@ fun getOfflineTtsConfig( throw IllegalArgumentException("Please provide vocoder for Matcha TTS") } + if (isZipvoice && zipvoiceEncoder.isEmpty()) { + throw IllegalArgumentException("Please provide encoder for ZipVoice") + } + + if (isPocket && lmFlow.isEmpty()) { + throw IllegalArgumentException("Please provide lmFlow for Pocket TTS") + } + + // Resolve the lexicon path once: "" stays "" (e.g. when the text is + // pre-phonemized via GenerationConfig.phonemeCodepoints and no lexicon is + // used), multi-lexicon strings ("a.txt,b.txt") are used as-is, and a bare + // filename is looked up inside modelDir. + val lexiconPath = when { + lexicon == "" -> lexicon + "," in lexicon -> lexicon + else -> "$modelDir/$lexicon" + } + val vits = if (modelName.isNotEmpty() && voices.isEmpty() && !isSupertonic) { OfflineTtsVitsModelConfig( model = "$modelDir/$modelName", - lexicon = "$modelDir/$lexicon", + lexicon = lexiconPath, tokens = "$modelDir/tokens.txt", dataDir = dataDir, ) @@ -350,7 +399,7 @@ fun getOfflineTtsConfig( OfflineTtsMatchaModelConfig( acousticModel = "$modelDir/$acousticModelName", vocoder = vocoder, - lexicon = "$modelDir/$lexicon", + lexicon = lexiconPath, tokens = "$modelDir/tokens.txt", dataDir = dataDir, ) @@ -364,11 +413,7 @@ fun getOfflineTtsConfig( voices = "$modelDir/$voices", tokens = "$modelDir/tokens.txt", dataDir = dataDir, - lexicon = when { - lexicon == "" -> lexicon - "," in lexicon -> lexicon - else -> "$modelDir/$lexicon" - }, + lexicon = lexiconPath, ) } else { OfflineTtsKokoroModelConfig() @@ -380,6 +425,7 @@ fun getOfflineTtsConfig( voices = "$modelDir/$voices", tokens = "$modelDir/tokens.txt", dataDir = dataDir, + lexicon = lexiconPath, ) } else { OfflineTtsKittenModelConfig() @@ -399,15 +445,50 @@ fun getOfflineTtsConfig( OfflineTtsSupertonicModelConfig() } + val zipvoice = if (isZipvoice) { + OfflineTtsZipVoiceModelConfig( + tokens = "$modelDir/tokens.txt", + encoder = "$modelDir/$zipvoiceEncoder", + decoder = "$modelDir/$zipvoiceDecoder", + // The vocoder ships separately from the model package (e.g. + // vocos_24khz.onnx), so it is used as given, like Matcha's. + vocoder = zipvoiceVocoder, + lexicon = lexiconPath, + featScale = featScale, + tShift = tShift, + targetRms = targetRms, + guidanceScale = guidanceScale, + ) + } else { + OfflineTtsZipVoiceModelConfig() + } + + val pocket = if (isPocket) { + OfflineTtsPocketModelConfig( + lmFlow = "$modelDir/$lmFlow", + lmMain = "$modelDir/$lmMain", + encoder = "$modelDir/$pocketEncoder", + decoder = "$modelDir/$pocketDecoder", + textConditioner = "$modelDir/$textConditioner", + vocabJson = "$modelDir/$vocabJson", + tokenScoresJson = "$modelDir/$tokenScoresJson", + voiceEmbeddingCacheCapacity = voiceEmbeddingCacheCapacity, + ) + } else { + OfflineTtsPocketModelConfig() + } + return OfflineTtsConfig( model = OfflineTtsModelConfig( vits = vits, matcha = matcha, kokoro = kokoro, + zipvoice = zipvoice, kitten = kitten, + pocket = pocket, supertonic = supertonic, numThreads = numberOfThreads, - debug = true, + debug = false, provider = "cpu", ), ruleFsts = ruleFsts, diff --git a/sherpa-onnx/pascal-api/sherpa_onnx.pas b/sherpa-onnx/pascal-api/sherpa_onnx.pas index d9cc79751..1955021b5 100644 --- a/sherpa-onnx/pascal-api/sherpa_onnx.pas +++ b/sherpa-onnx/pascal-api/sherpa_onnx.pas @@ -20,6 +20,10 @@ uses type TSherpaOnnxSamplesArray = array of Single; + TSherpaOnnxInt32Array = array of Integer; + TSherpaOnnxInt32ArrayArray = array of TSherpaOnnxInt32Array; + TSherpaOnnxStringArray = array of AnsiString; + TSherpaOnnxStringArrayArray = array of TSherpaOnnxStringArray; TSherpaOnnxLinearResampler = class private @@ -54,11 +58,11 @@ type Model: AnsiString; Lexicon: AnsiString; Tokens: AnsiString; - DataDir: AnsiString; + DataDir: AnsiString; { Deprecated. Ignored since v2.0.0. } NoiseScale: Single; NoiseScaleW: Single; LengthScale: Single; - DictDir: AnsiString; + DictDir: AnsiString; { Unused legacy field kept for ABI compatibility. } function ToString: AnsiString; class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsVitsModelConfig); @@ -74,6 +78,16 @@ type ReferenceText: AnsiString; NumSteps: Integer; Extra: AnsiString; + { Pre-phonemized input as Unicode codepoints. Each inner array is one + sentence. When set, it is used in place of the Text argument; leave + empty to use Text with a lexicon. + Mutually exclusive with Tokens. } + PhonemeCodepoints: TSherpaOnnxInt32ArrayArray; + { Pre-tokenized input, e.g. Chinese pinyin like "zhong1". Each inner + array is one sentence. When set, it is used in place of the Text + argument; leave empty to use Text with a lexicon. + Mutually exclusive with PhonemeCodepoints. } + Tokens: TSherpaOnnxStringArrayArray; class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxGenerationConfig); end; @@ -82,10 +96,10 @@ type Vocoder: AnsiString; Lexicon: AnsiString; Tokens: AnsiString; - DataDir: AnsiString; + DataDir: AnsiString; { Deprecated. Ignored since v2.0.0. } NoiseScale: Single; LengthScale: Single; - DictDir: AnsiString; + DictDir: AnsiString; { Unused legacy field kept for ABI compatibility. } function ToString: AnsiString; class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsMatchaModelConfig); @@ -95,9 +109,9 @@ type Model: AnsiString; Voices: AnsiString; Tokens: AnsiString; - DataDir: AnsiString; + DataDir: AnsiString; { Deprecated. Ignored since v2.0.0. } LengthScale: Single; - DictDir: AnsiString; + DictDir: AnsiString; { Unused legacy field kept for ABI compatibility. } Lexicon: AnsiString; Lang: AnsiString; @@ -109,8 +123,9 @@ type Model: AnsiString; Voices: AnsiString; Tokens: AnsiString; - DataDir: AnsiString; + DataDir: AnsiString; { Deprecated. Ignored since v2.0.0. } LengthScale: Single; + Lexicon: AnsiString; function ToString: AnsiString; class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsKittenModelConfig); @@ -121,7 +136,7 @@ type Encoder: AnsiString; Decoder: AnsiString; Vocoder: AnsiString; - DataDir: AnsiString; + DataDir: AnsiString; { Deprecated. Ignored since v2.0.0. } Lexicon: AnsiString; FeatScale: Single; Tshift: Single; @@ -195,6 +210,7 @@ type Handle: Pointer; SampleRate: Integer; NumSpeakers: Integer; + Lang: AnsiString; _Config: TSherpaOnnxOfflineTtsConfig; public constructor Create(Config: TSherpaOnnxOfflineTtsConfig); @@ -218,6 +234,8 @@ type property GetHandle: Pointer Read Handle; property GetSampleRate: Integer Read SampleRate; property GetNumSpeakers: Integer Read NumSpeakers; + { Language/voice of the model, e.g., 'en-us'. Empty if not available. } + property GetLang: AnsiString Read Lang; end; TSherpaOnnxWave = record @@ -287,7 +305,7 @@ type end; TSherpaOnnxHomophoneReplacerConfig = record - DictDir: AnsiString; + DictDir: AnsiString; { Unused legacy field kept for ABI compatibility. } Lexicon: AnsiString; RuleFsts: AnsiString; function ToString: AnsiString; @@ -530,6 +548,13 @@ type function ToString: AnsiString; end; + TSherpaOnnxOfflineCtcFstDecoderConfig = record + Graph: AnsiString; + MaxActive: Integer; + function ToString: AnsiString; + class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineCtcFstDecoderConfig); + end; + TSherpaOnnxOfflineRecognizerConfig = record FeatConfig: TSherpaOnnxFeatureConfig; ModelConfig: TSherpaOnnxOfflineModelConfig; @@ -542,6 +567,7 @@ type RuleFars: AnsiString; BlankPenalty: Single; Hr: TSherpaOnnxHomophoneReplacerConfig; + CtcFstDecoderConfig: TSherpaOnnxOfflineCtcFstDecoderConfig; class operator Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineRecognizerConfig); function ToString: AnsiString; end; @@ -847,9 +873,6 @@ const {$linklib sherpa-onnx-fst} {$linklib kissfft-float} {$linklib kaldi-native-fbank-core} - {$linklib piper_phonemize} - {$linklib espeak-ng} - {$linklib ucd} {$linklib onnxruntime} {$linklib ssentencepiece_core} @@ -1083,6 +1106,11 @@ type CohereTranscribe: SherpaOnnxOfflineCohereTranscribeModelConfig; end; + SherpaOnnxOfflineCtcFstDecoderConfig = record + Graph: PAnsiChar; + MaxActive: cint32; + end; + SherpaOnnxOfflineRecognizerConfig = record FeatConfig: SherpaOnnxFeatureConfig; ModelConfig: SherpaOnnxOfflineModelConfig; @@ -1095,6 +1123,7 @@ type RuleFars: PAnsiChar; BlankPenalty: cfloat; Hr: SherpaOnnxHomophoneReplacerConfig; + CtcFstDecoderConfig: SherpaOnnxOfflineCtcFstDecoderConfig; end; PSherpaOnnxOfflineRecognizerConfig = ^SherpaOnnxOfflineRecognizerConfig; @@ -1158,6 +1187,12 @@ type ReferenceText: PAnsiChar; NumSteps: cint32; Extra: PAnsiChar; + PhonemeCodepoints: pcint32; + PhonemeCodepointsLens: pcint32; + PhonemeCodepointsNumSentences: cint32; + Tokens: PPAnsiChar; + TokensLens: pcint32; + TokensNumSentences: cint32; end; SherpaOnnxOfflineTtsMatchaModelConfig = record @@ -1188,6 +1223,7 @@ type Tokens: PAnsiChar; DataDir: PAnsiChar; LengthScale: cfloat; + Lexicon: PAnsiChar; end; SherpaOnnxOfflineTtsZipVoiceModelConfig = record @@ -1477,6 +1513,12 @@ function SherpaOnnxOfflineTtsSampleRate(Tts: Pointer): cint32; cdecl; function SherpaOnnxOfflineTtsNumSpeakers(Tts: Pointer): cint32; cdecl; external SherpaOnnxLibName; +function SherpaOnnxOfflineTtsLang(Tts: Pointer): PAnsiChar; cdecl; + external SherpaOnnxLibName; + +procedure SherpaOnnxDestroyOfflineTtsLang(S: PAnsiChar); cdecl; + external SherpaOnnxLibName; + function SherpaOnnxOfflineTtsGenerate(Tts: Pointer; Text: PAnsiChar; Sid: cint32; Speed: cfloat): PSherpaOnnxGeneratedAudio; cdecl; external SherpaOnnxLibName; @@ -1732,6 +1774,12 @@ begin [Self.Graph, Self.MaxActive]); end; +function TSherpaOnnxOfflineCtcFstDecoderConfig.ToString: AnsiString; +begin + Result := Format('TSherpaOnnxOfflineCtcFstDecoderConfig(Graph := %s, MaxActive := %d)', + [Self.Graph, Self.MaxActive]); +end; + function TSherpaOnnxHomophoneReplacerConfig.ToString: AnsiString; begin Result := Format('TSherpaOnnxHomophoneReplacerConfig(Lexicon := %s, RuleFsts := %s)', @@ -2196,12 +2244,13 @@ begin 'RuleFsts := %s, ' + 'RuleFars := %s, ' + 'BlankPenalty := %1.f, ' + - 'Hr := %s' + + 'Hr := %s, ' + + 'CtcFstDecoderConfig := %s' + ')', [Self.FeatConfig.ToString, Self.ModelConfig.ToString, Self.LMConfig.ToString, Self.DecodingMethod, Self.MaxActivePaths, Self.HotwordsFile, Self.HotwordsScore, Self.RuleFsts, Self.RuleFars, - Self.BlankPenalty, Self.Hr.ToString + Self.BlankPenalty, Self.Hr.ToString, Self.CtcFstDecoderConfig.ToString ]); end; @@ -2312,6 +2361,9 @@ begin C.Hr.Lexicon := PAnsiChar(Config.Hr.Lexicon); C.Hr.RuleFsts := PAnsiChar(Config.Hr.RuleFsts); + C.CtcFstDecoderConfig.Graph := PAnsiChar(Config.CtcFstDecoderConfig.Graph); + C.CtcFstDecoderConfig.MaxActive := Config.CtcFstDecoderConfig.MaxActive; + Result := C; end; @@ -2545,6 +2597,11 @@ begin Dest.MaxActive := 3000; end; +class operator TSherpaOnnxOfflineCtcFstDecoderConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineCtcFstDecoderConfig); +begin + Dest.MaxActive := 3000; +end; + class operator TSherpaOnnxOnlineRecognizerConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOnlineRecognizerConfig); begin Dest.DecodingMethod := 'greedy_search'; @@ -2857,9 +2914,11 @@ begin 'Voices := %s, ' + 'Tokens := %s, ' + 'DataDir := %s, ' + - 'LengthScale := %.2f' + + 'LengthScale := %.2f, ' + + 'Lexicon := %s' + ')', - [Self.Model, Self.Voices, Self.Tokens, Self.DataDir, Self.LengthScale]); + [Self.Model, Self.Voices, Self.Tokens, Self.DataDir, Self.LengthScale, + Self.Lexicon]); end; class operator TSherpaOnnxOfflineTtsKittenModelConfig.Initialize({$IFDEF FPC}var{$ELSE}out{$ENDIF} Dest: TSherpaOnnxOfflineTtsKittenModelConfig); @@ -2979,6 +3038,7 @@ end; constructor TSherpaOnnxOfflineTts.Create(Config: TSherpaOnnxOfflineTtsConfig); var C: SherpaOnnxOfflineTtsConfig; + P: PAnsiChar; begin C := Default(SherpaOnnxOfflineTtsConfig); Self._Config := Config; @@ -3012,6 +3072,7 @@ begin C.Model.Kitten.Tokens := PAnsiChar(Config.Model.Kitten.Tokens); C.Model.Kitten.DataDir := PAnsiChar(Config.Model.Kitten.DataDir); C.Model.Kitten.LengthScale := Config.Model.Kitten.LengthScale; + C.Model.Kitten.Lexicon := PAnsiChar(Config.Model.Kitten.Lexicon); C.Model.ZipVoice.Tokens := PAnsiChar(Config.Model.ZipVoice.Tokens); C.Model.ZipVoice.Encoder := PAnsiChar(Config.Model.ZipVoice.Encoder); @@ -3051,9 +3112,14 @@ begin C.SilenceScale := Config.SilenceScale; Self.Handle := SherpaOnnxCreateOfflineTts(@C); + if Self.Handle = nil then + raise Exception.Create('SherpaOnnxCreateOfflineTts failed. Please check your config'); Self.SampleRate := SherpaOnnxOfflineTtsSampleRate(Self.Handle); Self.NumSpeakers := SherpaOnnxOfflineTtsNumSpeakers(Self.Handle); + P := SherpaOnnxOfflineTtsLang(Self.Handle); + Self.Lang := P; + SherpaOnnxDestroyOfflineTtsLang(P); end; destructor TSherpaOnnxOfflineTts.Destroy; @@ -3095,13 +3161,15 @@ function TSherpaOnnxOfflineTts.Generate(Text: AnsiString; SpeakerId: Integer; Speed: Single): TSherpaOnnxGeneratedAudio; var Audio: PSherpaOnnxGeneratedAudio; - Config: TSherpaOnnxGenerationConfig; + // Must be the C-layout record, not TSherpaOnnxGenerationConfig: the C API + // reads the raw SherpaOnnxGenerationConfig (15 fields) from this address. + C: SherpaOnnxGenerationConfig; begin - Config := Default(TSherpaOnnxGenerationConfig); - Config.Sid := SpeakerId; - Config.Speed := Speed; + C := Default(SherpaOnnxGenerationConfig); + C.Sid := SpeakerId; + C.Speed := Speed; Audio := SherpaOnnxOfflineTtsGenerateWithConfig(Self.Handle, PAnsiChar(Text), - @Config, nil, nil); + @C, nil, nil); Result := ExtractGeneratedAudio(Audio); end; @@ -3112,21 +3180,23 @@ function TSherpaOnnxOfflineTts.Generate(Text: AnsiString; SpeakerId: Integer; ): TSherpaOnnxGeneratedAudio; var Audio: PSherpaOnnxGeneratedAudio; - Config: TSherpaOnnxGenerationConfig; + // Must be the C-layout record, not TSherpaOnnxGenerationConfig: the C API + // reads the raw SherpaOnnxGenerationConfig (15 fields) from this address. + C: SherpaOnnxGenerationConfig; begin - Config := Default(TSherpaOnnxGenerationConfig); - Config.Sid := SpeakerId; - Config.Speed := Speed; + C := Default(SherpaOnnxGenerationConfig); + C.Sid := SpeakerId; + C.Speed := Speed; _OfflineTtsCallbackWithArg := Callback; _OfflineTtsCallbackWithArgUserArg := Arg; if Assigned(Callback) then Audio := SherpaOnnxOfflineTtsGenerateWithConfig(Self.Handle, PAnsiChar(Text), - @Config, @_OfflineTtsCallbackWithArgWrapper, nil) + @C, @_OfflineTtsCallbackWithArgWrapper, nil) else Audio := SherpaOnnxOfflineTtsGenerateWithConfig(Self.Handle, PAnsiChar(Text), - @Config, nil, nil); + @C, nil, nil); Result := ExtractGeneratedAudio(Audio); end; @@ -3143,6 +3213,17 @@ var CReferenceAudio: pcfloat; ReferenceText: AnsiString; Extra: AnsiString; + PhonemeCodepoints: TSherpaOnnxInt32ArrayArray; + FlatCodepoints: TSherpaOnnxInt32Array; + CodepointLens: TSherpaOnnxInt32Array; + Tokens: TSherpaOnnxStringArrayArray; + FlatTokens: array of PAnsiChar; + TokenLens: TSherpaOnnxInt32Array; + NumCodepoints: Integer; + NumTokens: Integer; + I: Integer; + J: Integer; + K: Integer; begin C := Default(SherpaOnnxGenerationConfig); C.SilenceScale := GenerationConfig.SilenceScale; @@ -3165,6 +3246,56 @@ begin Extra := GenerationConfig.Extra; C.Extra := PAnsiChar(Extra); + { Flatten PhonemeCodepoints; the C API takes a flat array plus the + length of each sentence. The local copies keep the arrays alive + during the call. } + PhonemeCodepoints := GenerationConfig.PhonemeCodepoints; + NumCodepoints := 0; + for I := 0 to High(PhonemeCodepoints) do + Inc(NumCodepoints, Length(PhonemeCodepoints[I])); + if (Length(PhonemeCodepoints) > 0) and (NumCodepoints > 0) then + begin + SetLength(FlatCodepoints, NumCodepoints); + SetLength(CodepointLens, Length(PhonemeCodepoints)); + K := 0; + for I := 0 to High(PhonemeCodepoints) do + begin + CodepointLens[I] := Length(PhonemeCodepoints[I]); + for J := 0 to High(PhonemeCodepoints[I]) do + begin + FlatCodepoints[K] := PhonemeCodepoints[I][J]; + Inc(K); + end; + end; + C.PhonemeCodepoints := pcint32(@FlatCodepoints[0]); + C.PhonemeCodepointsLens := pcint32(@CodepointLens[0]); + C.PhonemeCodepointsNumSentences := Length(CodepointLens); + end; + + { Flatten Tokens the same way. } + Tokens := GenerationConfig.Tokens; + NumTokens := 0; + for I := 0 to High(Tokens) do + Inc(NumTokens, Length(Tokens[I])); + if (Length(Tokens) > 0) and (NumTokens > 0) then + begin + SetLength(FlatTokens, NumTokens); + SetLength(TokenLens, Length(Tokens)); + K := 0; + for I := 0 to High(Tokens) do + begin + TokenLens[I] := Length(Tokens[I]); + for J := 0 to High(Tokens[I]) do + begin + FlatTokens[K] := PAnsiChar(Tokens[I][J]); + Inc(K); + end; + end; + C.Tokens := PPAnsiChar(@FlatTokens[0]); + C.TokensLens := pcint32(@TokenLens[0]); + C.TokensNumSentences := Length(TokenLens); + end; + Audio := nil; try Audio := SherpaOnnxOfflineTtsGenerateWithConfig(Self.Handle, PAnsiChar(Text), diff --git a/sherpa-onnx/python/csrc/offline-tts-kitten-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-kitten-model-config.cc index 27e484d81..bddf4e1b1 100644 --- a/sherpa-onnx/python/csrc/offline-tts-kitten-model-config.cc +++ b/sherpa-onnx/python/csrc/offline-tts-kitten-model-config.cc @@ -16,13 +16,17 @@ void PybindOfflineTtsKittenModelConfig(py::module *m) { py::class_(*m, "OfflineTtsKittenModelConfig") .def(py::init<>()) .def(py::init(), + const std::string &, const std::string &, float, + const std::string &>(), py::arg("model"), py::arg("voices"), py::arg("tokens"), - py::arg("data_dir"), py::arg("length_scale") = 1.0) + py::arg("data_dir") = "", py::arg("length_scale") = 1.0, + py::arg("lexicon") = "") .def_readwrite("model", &PyClass::model) .def_readwrite("voices", &PyClass::voices) .def_readwrite("tokens", &PyClass::tokens) - .def_readwrite("data_dir", &PyClass::data_dir) + .def_readwrite("lexicon", &PyClass::lexicon) + .def_readwrite("data_dir", &PyClass::data_dir, + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. Use lexicon, or GenerationConfig.phoneme_codepoints / tokens with an external phonemizer instead.") .def_readwrite("length_scale", &PyClass::length_scale) .def("__str__", &PyClass::ToString) .def("validate", &PyClass::Validate); diff --git a/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.cc index 620e82c4e..b442df21e 100644 --- a/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.cc +++ b/sherpa-onnx/python/csrc/offline-tts-kokoro-model-config.cc @@ -20,14 +20,15 @@ void PybindOfflineTtsKokoroModelConfig(py::module *m) { const std::string &, const std::string &, float, const std::string &>(), py::arg("model"), py::arg("voices"), py::arg("tokens"), - py::arg("lexicon") = "", py::arg("data_dir"), + py::arg("lexicon") = "", py::arg("data_dir") = "", py::arg("dict_dir") = "", py::arg("length_scale") = 1.0, py::arg("lang") = "") .def_readwrite("model", &PyClass::model) .def_readwrite("voices", &PyClass::voices) .def_readwrite("tokens", &PyClass::tokens) .def_readwrite("lexicon", &PyClass::lexicon) - .def_readwrite("data_dir", &PyClass::data_dir) + .def_readwrite("data_dir", &PyClass::data_dir, + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. Use lexicon, or GenerationConfig.phoneme_codepoints / tokens with an external phonemizer instead.") .def_readwrite("dict_dir", &PyClass::dict_dir) .def_readwrite("length_scale", &PyClass::length_scale) .def_readwrite("lang", &PyClass::lang) diff --git a/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.cc index b7ccf7926..4fcdd8a68 100644 --- a/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.cc +++ b/sherpa-onnx/python/csrc/offline-tts-matcha-model-config.cc @@ -26,7 +26,8 @@ void PybindOfflineTtsMatchaModelConfig(py::module *m) { .def_readwrite("vocoder", &PyClass::vocoder) .def_readwrite("lexicon", &PyClass::lexicon) .def_readwrite("tokens", &PyClass::tokens) - .def_readwrite("data_dir", &PyClass::data_dir) + .def_readwrite("data_dir", &PyClass::data_dir, + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. Use lexicon, or GenerationConfig.phoneme_codepoints / tokens with an external phonemizer instead.") .def_readwrite("dict_dir", &PyClass::dict_dir) .def_readwrite("noise_scale", &PyClass::noise_scale) .def_readwrite("length_scale", &PyClass::length_scale) diff --git a/sherpa-onnx/python/csrc/offline-tts-vits-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-vits-model-config.cc index 99ceb1ba1..91b011e04 100644 --- a/sherpa-onnx/python/csrc/offline-tts-vits-model-config.cc +++ b/sherpa-onnx/python/csrc/offline-tts-vits-model-config.cc @@ -25,7 +25,8 @@ void PybindOfflineTtsVitsModelConfig(py::module *m) { .def_readwrite("model", &PyClass::model) .def_readwrite("lexicon", &PyClass::lexicon) .def_readwrite("tokens", &PyClass::tokens) - .def_readwrite("data_dir", &PyClass::data_dir) + .def_readwrite("data_dir", &PyClass::data_dir, + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. Use lexicon, or GenerationConfig.phoneme_codepoints / tokens with an external phonemizer instead.") .def_readwrite("dict_dir", &PyClass::dict_dir) .def_readwrite("noise_scale", &PyClass::noise_scale) .def_readwrite("noise_scale_w", &PyClass::noise_scale_w) diff --git a/sherpa-onnx/python/csrc/offline-tts-zipvoice-model-config.cc b/sherpa-onnx/python/csrc/offline-tts-zipvoice-model-config.cc index 55bb5ade8..9a4073121 100644 --- a/sherpa-onnx/python/csrc/offline-tts-zipvoice-model-config.cc +++ b/sherpa-onnx/python/csrc/offline-tts-zipvoice-model-config.cc @@ -28,7 +28,8 @@ void PybindOfflineTtsZipvoiceModelConfig(py::module *m) { .def_readwrite("encoder", &PyClass::encoder) .def_readwrite("decoder", &PyClass::decoder) .def_readwrite("vocoder", &PyClass::vocoder) - .def_readwrite("data_dir", &PyClass::data_dir) + .def_readwrite("data_dir", &PyClass::data_dir, + "Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. Use lexicon or GenerationConfig.tokens instead. (ZipVoice does not support phoneme_codepoints.)") .def_readwrite("lexicon", &PyClass::lexicon) .def_readwrite("feat_scale", &PyClass::feat_scale) .def_readwrite("t_shift", &PyClass::t_shift) diff --git a/sherpa-onnx/python/csrc/offline-tts.cc b/sherpa-onnx/python/csrc/offline-tts.cc index 9739f1b65..5833797ef 100644 --- a/sherpa-onnx/python/csrc/offline-tts.cc +++ b/sherpa-onnx/python/csrc/offline-tts.cc @@ -39,6 +39,8 @@ static void PybindGenerationConfig(py::module *m) { .def_readwrite("reference_text", &PyClass::reference_text) .def_readwrite("num_steps", &PyClass::num_steps) .def_readwrite("extra", &PyClass::extra) + .def_readwrite("phoneme_codepoints", &PyClass::phoneme_codepoints) + .def_readwrite("tokens", &PyClass::tokens) .def("__str__", &PyClass::ToString); } @@ -83,7 +85,7 @@ Args: callback: If not None, it is called during speech generation with ``(samples: np.ndarray, progress: float) -> int``. - Return a non-zero value to stop generation early. + Return 1 to keep generating, or 0 to stop generation early. Returns: A ``GeneratedAudio`` object containing the audio samples and sample rate. @@ -97,6 +99,12 @@ static constexpr const char *kNumSpeakersDoc = R"doc( Return the number of speakers supported by the model. )doc"; +static constexpr const char *kLangDoc = R"doc( +Return the language/voice of the model, e.g., "en-us". + +Returns an empty string if such information is not available. +)doc"; + void PybindOfflineTts(py::module *m) { PybindOfflineTtsConfig(m); PybindGeneratedAudio(m); @@ -110,6 +118,7 @@ void PybindOfflineTts(py::module *m) { kSampleRateDoc) .def_property_readonly("num_speakers", &PyClass::NumSpeakers, kNumSpeakersDoc) + .def_property_readonly("lang", &PyClass::Lang, kLangDoc) .def( "generate", [](const PyClass &self, const std::string &text, int64_t sid, diff --git a/sherpa-onnx/python/sherpa_onnx/offline_recognizer.py b/sherpa-onnx/python/sherpa_onnx/offline_recognizer.py index d9b0cecd4..93d18737f 100644 --- a/sherpa-onnx/python/sherpa_onnx/offline_recognizer.py +++ b/sherpa-onnx/python/sherpa_onnx/offline_recognizer.py @@ -958,6 +958,8 @@ class OfflineRecognizer(object): hr_dict_dir: str = "", hr_rule_fsts: str = "", hr_lexicon: str = "", + ctc_graph: str = "", + ctc_max_active: int = 3000, ): """ Please refer to @@ -992,6 +994,13 @@ class OfflineRecognizer(object): rule_fars: If not empty, it specifies fst archives for inverse text normalization. If there are multiple archives, they are separated by a comma. + ctc_graph: + If not empty, it specifies the decoding graph (H.fst, HL.fst or + HLG.fst) for CTC+FST decoding. It is used together with the CTC + model. + ctc_max_active: + Used only when ctc_graph is not empty. It specifies the maximum + number of active tokens during CTC+FST decoding. 0 means 3000. """ self = cls.__new__(cls) model_config = OfflineModelConfig( @@ -1018,6 +1027,10 @@ class OfflineRecognizer(object): lexicon=hr_lexicon, rule_fsts=hr_rule_fsts, ), + ctc_fst_decoder_config=OfflineCtcFstDecoderConfig( + graph=ctc_graph, + max_active=ctc_max_active, + ), ) self.recognizer = _Recognizer(recognizer_config) self.config = recognizer_config diff --git a/sherpa-onnx/python/tests/test_offline_tts_config.py b/sherpa-onnx/python/tests/test_offline_tts_config.py index 0072744ba..8a29129d3 100644 --- a/sherpa-onnx/python/tests/test_offline_tts_config.py +++ b/sherpa-onnx/python/tests/test_offline_tts_config.py @@ -22,18 +22,12 @@ def _create_kokoro_files(d: Path): _touch(d / "voices.bin") _touch(d / "tokens.txt") - data_dir = d / "espeak-ng-data" - data_dir.mkdir() - for name in ["phontab", "phonindex", "phondata", "intonations"]: - _touch(data_dir / name) - def _make_kokoro_config(d: Path, **kwargs): args = { "model": str(d / "model.onnx"), "voices": str(d / "voices.bin"), "tokens": str(d / "tokens.txt"), - "data_dir": str(d / "espeak-ng-data"), } args.update(kwargs) return _sherpa_onnx.OfflineTtsKokoroModelConfig(**args) diff --git a/sherpa-onnx/rust/sherpa-onnx-sys/build.rs b/sherpa-onnx/rust/sherpa-onnx-sys/build.rs index 34a340482..d2dcd17ef 100644 --- a/sherpa-onnx/rust/sherpa-onnx-sys/build.rs +++ b/sherpa-onnx/rust/sherpa-onnx-sys/build.rs @@ -22,9 +22,6 @@ const SHERPA_ONNX_STATIC_LIBS: &[&str] = &[ "sherpa-onnx-fst", "kaldi-native-fbank-core", "kissfft-float", - "piper_phonemize", - "espeak-ng", - "ucd", "onnxruntime", "ssentencepiece_core", ]; @@ -125,7 +122,6 @@ fn resolve_lib_dir( } return Ok((path, None)); } - download_prebuilt_libs(link_mode, target_os, target_arch).map(|(p, s)| (p, Some(s))) } diff --git a/sherpa-onnx/rust/sherpa-onnx-sys/src/offline_asr.rs b/sherpa-onnx/rust/sherpa-onnx-sys/src/offline_asr.rs index 9d0d28c74..95445e5a8 100644 --- a/sherpa-onnx/rust/sherpa-onnx-sys/src/offline_asr.rs +++ b/sherpa-onnx/rust/sherpa-onnx-sys/src/offline_asr.rs @@ -211,6 +211,14 @@ pub struct OfflineRecognizerConfig { pub rule_fars: *const c_char, pub blank_penalty: c_float, pub hr: super::online_asr::HomophoneReplacerConfig, + pub ctc_fst_decoder_config: OfflineCtcFstDecoderConfig, +} + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct OfflineCtcFstDecoderConfig { + pub graph: *const c_char, + pub max_active: i32, } #[repr(C)] diff --git a/sherpa-onnx/rust/sherpa-onnx-sys/src/tts.rs b/sherpa-onnx/rust/sherpa-onnx-sys/src/tts.rs index 7b04b7627..e39d09648 100644 --- a/sherpa-onnx/rust/sherpa-onnx-sys/src/tts.rs +++ b/sherpa-onnx/rust/sherpa-onnx-sys/src/tts.rs @@ -10,6 +10,7 @@ pub struct OfflineTtsVitsModelConfig { pub model: *const c_char, pub lexicon: *const c_char, pub tokens: *const c_char, + /// Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. pub data_dir: *const c_char, pub noise_scale: c_float, pub noise_scale_w: c_float, @@ -24,6 +25,7 @@ pub struct OfflineTtsMatchaModelConfig { pub vocoder: *const c_char, pub lexicon: *const c_char, pub tokens: *const c_char, + /// Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. pub data_dir: *const c_char, pub noise_scale: c_float, pub length_scale: c_float, @@ -36,6 +38,7 @@ pub struct OfflineTtsKokoroModelConfig { pub model: *const c_char, pub voices: *const c_char, pub tokens: *const c_char, + /// Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. pub data_dir: *const c_char, pub length_scale: c_float, pub dict_dir: *const c_char, @@ -49,8 +52,10 @@ pub struct OfflineTtsKittenModelConfig { pub model: *const c_char, pub voices: *const c_char, pub tokens: *const c_char, + /// Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. pub data_dir: *const c_char, pub length_scale: c_float, + pub lexicon: *const c_char, } #[repr(C)] @@ -60,6 +65,7 @@ pub struct OfflineTtsZipvoiceModelConfig { pub encoder: *const c_char, pub decoder: *const c_char, pub vocoder: *const c_char, + /// Deprecated. Ignored since v2.0.0: espeak-ng is no longer used. pub data_dir: *const c_char, pub lexicon: *const c_char, pub feat_scale: c_float, @@ -138,6 +144,12 @@ pub struct SherpaOnnxGenerationConfig { pub reference_text: *const c_char, pub num_steps: i32, pub extra: *const c_char, + pub phoneme_codepoints: *const i32, + pub phoneme_codepoints_lens: *const i32, + pub phoneme_codepoints_num_sentences: i32, + pub tokens: *const *const c_char, + pub tokens_lens: *const i32, + pub tokens_num_sentences: i32, } pub type SherpaOnnxGeneratedAudioProgressCallbackWithArg = @@ -159,6 +171,10 @@ extern "C" { pub fn SherpaOnnxOfflineTtsNumSpeakers(tts: *const SherpaOnnxOfflineTts) -> i32; + pub fn SherpaOnnxOfflineTtsLang(tts: *const SherpaOnnxOfflineTts) -> *const c_char; + + pub fn SherpaOnnxDestroyOfflineTtsLang(s: *const c_char); + pub fn SherpaOnnxOfflineTtsGenerateWithConfig( tts: *const SherpaOnnxOfflineTts, text: *const c_char, diff --git a/sherpa-onnx/rust/sherpa-onnx/src/lib.rs b/sherpa-onnx/rust/sherpa-onnx/src/lib.rs index fd33b5f99..5f0a13ef5 100644 --- a/sherpa-onnx/rust/sherpa-onnx/src/lib.rs +++ b/sherpa-onnx/rust/sherpa-onnx/src/lib.rs @@ -139,15 +139,20 @@ //! //! ## TTS //! -//! - [`kitten_tts_en.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/kitten_tts_en.rs) -//! - [`kokoro_tts_en.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/kokoro_tts_en.rs) -//! - [`kokoro_tts_zh_en.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/kokoro_tts_zh_en.rs) -//! - [`matcha_tts_en.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/matcha_tts_en.rs) -//! - [`matcha_tts_zh.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/matcha_tts_zh.rs) +//! - [`kitten_tts_en_lexicon.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/kitten_tts_en_lexicon.rs) +//! - [`kitten_tts_en_phonemize.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/kitten_tts_en_phonemize.rs) +//! - [`kokoro_tts_en_lexicon.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/kokoro_tts_en_lexicon.rs) +//! - [`kokoro_tts_en_phonemize.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/kokoro_tts_en_phonemize.rs) +//! - [`kokoro_tts_zh_en_lexicon.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/kokoro_tts_zh_en_lexicon.rs) +//! - [`matcha_tts_en_lexicon.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/matcha_tts_en_lexicon.rs) +//! - [`matcha_tts_en_phonemize.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/matcha_tts_en_phonemize.rs) +//! - [`matcha_tts_zh_lexicon.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/matcha_tts_zh_lexicon.rs) //! - [`pocket_tts.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/pocket_tts.rs) //! - [`supertonic_tts.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/supertonic_tts.rs) -//! - [`vits_tts.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/vits_tts.rs) -//! - [`zipvoice_tts.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/zipvoice_tts.rs) +//! - [`vits_tts_de_phonemize.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/vits_tts_de_phonemize.rs) +//! - [`vits_tts_lexicon.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/vits_tts_lexicon.rs) +//! - [`vits_tts_phonemize.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/vits_tts_phonemize.rs) +//! - [`zipvoice_tts_lexicon.rs`](https://github.com/k2-fsa/sherpa-onnx/blob/master/rust-api-examples/examples/zipvoice_tts_lexicon.rs) //! //! ## Audio tagging //! diff --git a/sherpa-onnx/rust/sherpa-onnx/src/offline_asr.rs b/sherpa-onnx/rust/sherpa-onnx/src/offline_asr.rs index 39f1b1b71..c0574fdef 100644 --- a/sherpa-onnx/rust/sherpa-onnx/src/offline_asr.rs +++ b/sherpa-onnx/rust/sherpa-onnx/src/offline_asr.rs @@ -566,6 +566,33 @@ pub struct OfflineRecognizerConfig { pub rule_fars: Option, pub blank_penalty: f32, pub hr: super::online_asr::HomophoneReplacerConfig, + pub ctc_fst_decoder_config: OfflineCtcFstDecoderConfig, +} + +#[derive(Clone, Debug)] +/// FST decoder options for CTC models. +pub struct OfflineCtcFstDecoderConfig { + pub graph: Option, + pub max_active: i32, +} + +impl Default for OfflineCtcFstDecoderConfig { + fn default() -> Self { + Self { + graph: None, + max_active: 3000, + } + } +} + +impl OfflineCtcFstDecoderConfig { + /// Convert to sys struct using `to_c_ptr()` + pub(crate) fn to_sys(&self, cstrings: &mut Vec) -> sys::OfflineCtcFstDecoderConfig { + sys::OfflineCtcFstDecoderConfig { + graph: to_c_ptr(&self.graph, cstrings), + max_active: self.max_active, + } + } } impl OfflineRecognizerConfig { @@ -588,6 +615,7 @@ impl OfflineRecognizerConfig { hr: self .hr .to_sys(cstrings), + ctc_fst_decoder_config: self.ctc_fst_decoder_config.to_sys(cstrings), } } } @@ -610,6 +638,7 @@ impl Default for OfflineRecognizerConfig { rule_fars: None, blank_penalty: 0.0, hr: super::online_asr::HomophoneReplacerConfig::default(), + ctc_fst_decoder_config: OfflineCtcFstDecoderConfig::default(), } } } diff --git a/sherpa-onnx/rust/sherpa-onnx/src/tts.rs b/sherpa-onnx/rust/sherpa-onnx/src/tts.rs index 9ff3a99d6..159eaf3a0 100644 --- a/sherpa-onnx/rust/sherpa-onnx/src/tts.rs +++ b/sherpa-onnx/rust/sherpa-onnx/src/tts.rs @@ -4,11 +4,11 @@ //! Pocket TTS, and Supertonic. See the repository examples: //! //! - `rust-api-examples/examples/pocket_tts.rs` -//! - `rust-api-examples/examples/kokoro_tts_en.rs` -//! - `rust-api-examples/examples/kokoro_tts_zh_en.rs` -//! - `rust-api-examples/examples/matcha_tts_en.rs` -//! - `rust-api-examples/examples/matcha_tts_zh.rs` -//! - `rust-api-examples/examples/zipvoice_tts.rs` +//! - `rust-api-examples/examples/kokoro_tts_en_lexicon.rs` +//! - `rust-api-examples/examples/kokoro_tts_zh_en_lexicon.rs` +//! - `rust-api-examples/examples/matcha_tts_en_lexicon.rs` +//! - `rust-api-examples/examples/matcha_tts_zh_lexicon.rs` +//! - `rust-api-examples/examples/zipvoice_tts_lexicon.rs` //! - `rust-api-examples/examples/supertonic_tts.rs` //! //! # Example @@ -58,7 +58,7 @@ use crate::utils::to_c_ptr; use sherpa_onnx_sys as sys; use std::collections::HashMap; use std::ffi::CString; -use std::os::raw::c_void; +use std::os::raw::{c_char, c_void}; use std::ptr; use std::slice; @@ -73,6 +73,8 @@ pub struct OfflineTtsVitsModelConfig { pub model: Option, pub lexicon: Option, pub tokens: Option, + /// Deprecated. Ignored since v2.0.0. Use `lexicon` or + /// [`GenerationConfig::phoneme_codepoints`] instead. pub data_dir: Option, pub noise_scale: f32, pub noise_scale_w: f32, @@ -117,6 +119,8 @@ pub struct OfflineTtsMatchaModelConfig { pub vocoder: Option, pub lexicon: Option, pub tokens: Option, + /// Deprecated. Ignored since v2.0.0. Use `lexicon` or + /// [`GenerationConfig::phoneme_codepoints`] instead. pub data_dir: Option, pub noise_scale: f32, pub length_scale: f32, @@ -131,7 +135,7 @@ impl Default for OfflineTtsMatchaModelConfig { lexicon: None, tokens: None, data_dir: None, - noise_scale: 0.667, + noise_scale: 1.0, length_scale: 1.0, dict_dir: None, } @@ -159,6 +163,8 @@ pub struct OfflineTtsKokoroModelConfig { pub model: Option, pub voices: Option, pub tokens: Option, + /// Deprecated. Ignored since v2.0.0. Use `lexicon` or + /// [`GenerationConfig::phoneme_codepoints`] instead. pub data_dir: Option, pub length_scale: f32, pub dict_dir: Option, @@ -202,8 +208,11 @@ pub struct OfflineTtsKittenModelConfig { pub model: Option, pub voices: Option, pub tokens: Option, + /// Deprecated. Ignored since v2.0.0. Use `lexicon` or + /// [`GenerationConfig::phoneme_codepoints`] instead. pub data_dir: Option, pub length_scale: f32, + pub lexicon: Option, } impl Default for OfflineTtsKittenModelConfig { @@ -214,6 +223,7 @@ impl Default for OfflineTtsKittenModelConfig { tokens: None, data_dir: None, length_scale: 1.0, + lexicon: None, } } } @@ -226,6 +236,7 @@ impl OfflineTtsKittenModelConfig { tokens: to_c_ptr(&self.tokens, cstrings), data_dir: to_c_ptr(&self.data_dir, cstrings), length_scale: self.length_scale, + lexicon: to_c_ptr(&self.lexicon, cstrings), } } } @@ -237,6 +248,9 @@ pub struct OfflineTtsZipvoiceModelConfig { pub encoder: Option, pub decoder: Option, pub vocoder: Option, + /// Deprecated. Ignored since v2.0.0. Use `lexicon` or + /// [`GenerationConfig::tokens`] instead. (ZipVoice does not support + /// `phoneme_codepoints`.) pub data_dir: Option, pub lexicon: Option, pub feat_scale: f32, @@ -354,32 +368,27 @@ pub struct OfflineTtsModelConfig { impl OfflineTtsModelConfig { fn to_sys(&self, cstrings: &mut Vec) -> sys::OfflineTtsModelConfig { - sys::OfflineTtsModelConfig { - vits: self - .vits - .to_sys(cstrings), + let vits = self.vits.to_sys(cstrings); + let provider = to_c_ptr(&self.provider, cstrings); + let matcha = self.matcha.to_sys(cstrings); + let kokoro = self.kokoro.to_sys(cstrings); + let kitten = self.kitten.to_sys(cstrings); + let zipvoice = self.zipvoice.to_sys(cstrings); + let pocket = self.pocket.to_sys(cstrings); + let supertonic = self.supertonic.to_sys(cstrings); + let out = sys::OfflineTtsModelConfig { + vits, num_threads: self.num_threads, debug: self.debug as i32, - provider: to_c_ptr(&self.provider, cstrings), - matcha: self - .matcha - .to_sys(cstrings), - kokoro: self - .kokoro - .to_sys(cstrings), - kitten: self - .kitten - .to_sys(cstrings), - zipvoice: self - .zipvoice - .to_sys(cstrings), - pocket: self - .pocket - .to_sys(cstrings), - supertonic: self - .supertonic - .to_sys(cstrings), - } + provider, + matcha, + kokoro, + kitten, + zipvoice, + pocket, + supertonic, + }; + out } } @@ -395,15 +404,17 @@ pub struct OfflineTtsConfig { impl OfflineTtsConfig { fn to_sys(&self, cstrings: &mut Vec) -> sys::OfflineTtsConfig { - sys::OfflineTtsConfig { - model: self - .model - .to_sys(cstrings), - rule_fsts: to_c_ptr(&self.rule_fsts, cstrings), + let model = self.model.to_sys(cstrings); + let rule_fsts = to_c_ptr(&self.rule_fsts, cstrings); + let rule_fars = to_c_ptr(&self.rule_fars, cstrings); + let out = sys::OfflineTtsConfig { + model, + rule_fsts, max_num_sentences: self.max_num_sentences, - rule_fars: to_c_ptr(&self.rule_fars, cstrings), + rule_fars, silence_scale: self.silence_scale, - } + }; + out } } @@ -420,6 +431,15 @@ pub struct GenerationConfig { pub reference_text: Option, pub num_steps: i32, pub extra: Option>, + + /// Pre-phonemized input as Unicode codepoints. Each inner vector is one + /// sentence. When set, it is used in place of the `text` argument; leave + /// empty to use `text` with a lexicon. Mutually exclusive with `tokens`. + pub phoneme_codepoints: Option>>, + + /// Pre-tokenized input, e.g., Chinese pinyin like "zhong1". Each inner + /// vector is one sentence. Mutually exclusive with `phoneme_codepoints`. + pub tokens: Option>>, } impl Default for GenerationConfig { @@ -433,6 +453,8 @@ impl Default for GenerationConfig { reference_text: None, num_steps: 5, extra: None, + phoneme_codepoints: None, + tokens: None, } } } @@ -548,6 +570,21 @@ impl OfflineTts { unsafe { sys::SherpaOnnxOfflineTtsNumSpeakers(self.ptr) } } + /// Return the language/voice of the model, e.g., "en-us". + /// + /// Returns an empty string if such information is not available. + pub fn lang(&self) -> String { + unsafe { + let p = sys::SherpaOnnxOfflineTtsLang(self.ptr); + if p.is_null() { + return String::new(); + } + let s = std::ffi::CStr::from_ptr(p).to_string_lossy().into_owned(); + sys::SherpaOnnxDestroyOfflineTtsLang(p); + s + } + } + /// Generate audio for `text`. /// /// The optional callback receives the samples generated so far together @@ -579,6 +616,33 @@ impl OfflineTts { None => (ptr::null(), 0), }; + // Flatten phoneme_codepoints; the C API takes a flat array plus the + // length of each sentence. + let mut flat_codepoints: Vec = Vec::new(); + let mut codepoint_lens: Vec = Vec::new(); + if let Some(sentences) = &config.phoneme_codepoints { + for sentence in sentences { + flat_codepoints.extend_from_slice(sentence); + codepoint_lens.push(sentence.len() as i32); + } + } + let has_codepoints = !codepoint_lens.is_empty() && !flat_codepoints.is_empty(); + + // Flatten tokens the same way. The C strings are kept alive in + // `cstrings` until the call returns. + let mut flat_tokens: Vec<*const c_char> = Vec::new(); + let mut token_lens: Vec = Vec::new(); + if let Some(sentences) = &config.tokens { + for sentence in sentences { + for tok in sentence { + cstrings.push(CString::new(tok.as_str()).unwrap_or_default()); + flat_tokens.push(cstrings.last().unwrap().as_ptr()); + } + token_lens.push(sentence.len() as i32); + } + } + let has_tokens = !token_lens.is_empty() && !flat_tokens.is_empty(); + let sys_gen_config = sys::SherpaOnnxGenerationConfig { silence_scale: config.silence_scale, speed: config.speed, @@ -589,6 +653,36 @@ impl OfflineTts { reference_text: c_ref_text, num_steps: config.num_steps, extra: c_extra.as_ptr(), + phoneme_codepoints: if has_codepoints { + flat_codepoints.as_ptr() + } else { + ptr::null() + }, + phoneme_codepoints_lens: if has_codepoints { + codepoint_lens.as_ptr() + } else { + ptr::null() + }, + phoneme_codepoints_num_sentences: if has_codepoints { + codepoint_lens.len() as i32 + } else { + 0 + }, + tokens: if has_tokens { + flat_tokens.as_ptr() + } else { + ptr::null() + }, + tokens_lens: if has_tokens { + token_lens.as_ptr() + } else { + ptr::null() + }, + tokens_num_sentences: if has_tokens { + token_lens.len() as i32 + } else { + 0 + }, }; let (c_callback, c_arg): ( diff --git a/swift-api-examples/SherpaOnnx.swift b/swift-api-examples/SherpaOnnx.swift index b4d91f0b8..ea8c2cdcf 100644 --- a/swift-api-examples/SherpaOnnx.swift +++ b/swift-api-examples/SherpaOnnx.swift @@ -1030,6 +1030,7 @@ public func sherpaOnnxOfflineTtsVitsModelConfig( model: String = "", lexicon: String = "", tokens: String = "", + /// Deprecated. Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead. dataDir: String = "", noiseScale: Float = 0.667, noiseScaleW: Float = 0.8, @@ -1053,6 +1054,7 @@ public func sherpaOnnxOfflineTtsMatchaModelConfig( vocoder: String = "", lexicon: String = "", tokens: String = "", + /// Deprecated. Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead. dataDir: String = "", noiseScale: Float = 0.667, lengthScale: Float = 1.0, @@ -1074,6 +1076,7 @@ public func sherpaOnnxOfflineTtsKokoroModelConfig( model: String = "", voices: String = "", tokens: String = "", + /// Deprecated. Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead. dataDir: String = "", lengthScale: Float = 1.0, dictDir: String = "", @@ -1096,15 +1099,18 @@ public func sherpaOnnxOfflineTtsKittenModelConfig( model: String = "", voices: String = "", tokens: String = "", + /// Deprecated. Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead. dataDir: String = "", - lengthScale: Float = 1.0 + lengthScale: Float = 1.0, + lexicon: String = "" ) -> SherpaOnnxOfflineTtsKittenModelConfig { return SherpaOnnxOfflineTtsKittenModelConfig( model: toCPointer(model), voices: toCPointer(voices), tokens: toCPointer(tokens), data_dir: toCPointer(dataDir), - length_scale: lengthScale + length_scale: lengthScale, + lexicon: toCPointer(lexicon) ) } @@ -1113,6 +1119,7 @@ public func sherpaOnnxOfflineTtsZipvoiceModelConfig( encoder: String = "", decoder: String = "", vocoder: String = "", + /// Deprecated. Ignored since v2.0.0. Use lexicon or phonemeCodepoints instead. dataDir: String = "", lexicon: String = "", featScale: Float = 0.1, @@ -1255,10 +1262,14 @@ public class SherpaOnnxWaveWrapper { } public class SherpaOnnxGeneratedAudioWrapper { - /// A pointer to the underlying counterpart in C - public let audio: UnsafePointer! + /// A pointer to the underlying counterpart in C. + /// + /// nil when generation failed — for instance when both `phonemeCodepoints` + /// and `tokens` were set, which the C API rejects with a warning. The + /// accessors below then report empty audio instead of crashing. + public let audio: UnsafePointer? - public init(audio: UnsafePointer!) { + public init(audio: UnsafePointer?) { self.audio = audio } @@ -1269,22 +1280,22 @@ public class SherpaOnnxGeneratedAudioWrapper { } public var n: Int32 { - return audio.pointee.n + return audio?.pointee.n ?? 0 } public var sampleRate: Int32 { - return audio.pointee.sample_rate + return audio?.pointee.sample_rate ?? 0 } public var samples: [Float] { - if let p = audio.pointee.samples { - return [Float](UnsafeBufferPointer(start: p, count: Int(n))) - } else { + guard let audio, let p = audio.pointee.samples else { return [] } + return [Float](UnsafeBufferPointer(start: p, count: Int(n))) } public func save(filename: String) -> Int32 { + guard let audio else { return -1 } return SherpaOnnxWriteWave(audio.pointee.samples, n, sampleRate, toCPointer(filename)) } } @@ -1312,7 +1323,7 @@ public typealias TtsProgressCallbackWithArg = ) -> Int32 public struct SherpaOnnxGenerationConfigSwift { - public init(silenceScale: Float = 0.2, speed: Float = 1.0, sid: Int = 0, referenceAudio: [Float] = [], referenceSampleRate: Int = 16000, referenceText: String = "", numSteps: Int = 1, extra: [String: Any] = [:]) { + public init(silenceScale: Float = 0.2, speed: Float = 1.0, sid: Int = 0, referenceAudio: [Float] = [], referenceSampleRate: Int = 0, referenceText: String = "", numSteps: Int = 5, extra: [String: Any] = [:], phonemeCodepoints: [[Int32]] = [], tokens: [[String]] = []) { self.silenceScale = silenceScale self.speed = speed self.sid = sid @@ -1321,17 +1332,28 @@ public struct SherpaOnnxGenerationConfigSwift { self.referenceText = referenceText self.numSteps = numSteps self.extra = extra + self.phonemeCodepoints = phonemeCodepoints + self.tokens = tokens } public var silenceScale: Float = 0.2 public var speed: Float = 1.0 public var sid: Int = 0 public var referenceAudio: [Float] = [] - public var referenceSampleRate: Int = 16000 + public var referenceSampleRate: Int = 0 public var referenceText: String = "" - public var numSteps: Int = 1 + public var numSteps: Int = 5 public var extra: [String: Any] = [:] // Any can be String, Int, Float, Double + /// Pre-phonemized input as Unicode codepoints. Each inner array is one + /// sentence. When set, it is used in place of the text argument; leave + /// empty to use text with a lexicon. Mutually exclusive with `tokens`. + public var phonemeCodepoints: [[Int32]] = [] + + /// Pre-tokenized input, e.g. Chinese pinyin like "zhong1". Each inner array + /// is one sentence. Mutually exclusive with `phonemeCodepoints`. + public var tokens: [[String]] = [] + /// Convert the extra dictionary into a JSON string public func extraJsonString() -> String { var jsonCompatible: [String: Any] = [:] @@ -1371,6 +1393,17 @@ final class SherpaOnnxGenerationConfigC { /// Extra JSON string for C API let extraJson: String + /// Flattened phoneme codepoints and per-sentence lengths; the pointers in + /// cConfig point into these buffers for the duration of the C call. + private let codepointsStorage: [Int32] + private let codepointsLensStorage: [Int32] + + /// C strings for the flattened token array and the char** array pointing at + /// them; freed in deinit. + private let tokenStrings: [UnsafeMutablePointer?] + private let tokenPtrsStorage: UnsafeMutablePointer?>? + private let tokenLensStorage: [Int32] + init(_ swiftConfig: SherpaOnnxGenerationConfigSwift) { let referenceAudio = swiftConfig.referenceAudio @@ -1379,19 +1412,76 @@ final class SherpaOnnxGenerationConfigC { self.referenceAudioStorage = referenceAudio - self.cConfig = self.referenceAudioStorage.withUnsafeBufferPointer { buffer in - SherpaOnnxGenerationConfig( - silence_scale: swiftConfig.silenceScale, - speed: swiftConfig.speed, - sid: Int32(swiftConfig.sid), - reference_audio: buffer.count > 0 ? buffer.baseAddress : nil, - reference_audio_len: Int32(buffer.count), - reference_sample_rate: Int32(swiftConfig.referenceSampleRate), - reference_text: toCPointer(swiftConfig.referenceText), - num_steps: Int32(swiftConfig.numSteps), - extra: toCPointer(extraJson) - ) + // Flatten phoneme codepoints; the C API takes a flat array plus the + // length of each sentence. + var flatCodepoints: [Int32] = [] + var codepointLens: [Int32] = [] + for sentence in swiftConfig.phonemeCodepoints { + codepointLens.append(Int32(sentence.count)) + flatCodepoints.append(contentsOf: sentence) } + self.codepointsStorage = flatCodepoints + self.codepointsLensStorage = codepointLens + + // Flatten tokens the same way; each token becomes a C string and the + // char** array points at them. + var tokenStrings: [UnsafeMutablePointer?] = [] + var tokenLens: [Int32] = [] + for sentence in swiftConfig.tokens { + tokenLens.append(Int32(sentence.count)) + for token in sentence { + tokenStrings.append(strdup(token)) + } + } + self.tokenStrings = tokenStrings + self.tokenLensStorage = tokenLens + + var tokenPtrs: UnsafeMutablePointer?>? = nil + if !tokenStrings.isEmpty { + tokenPtrs = .allocate(capacity: tokenStrings.count) + for (i, p) in tokenStrings.enumerated() { + tokenPtrs!.advanced(by: i).initialize(to: UnsafePointer(p)) + } + } + self.tokenPtrsStorage = tokenPtrs + + let hasCodepoints = !codepointLens.isEmpty && !flatCodepoints.isEmpty + let hasTokens = !tokenLens.isEmpty && !tokenStrings.isEmpty + + self.cConfig = self.referenceAudioStorage.withUnsafeBufferPointer { buffer in + flatCodepoints.withUnsafeBufferPointer { codepointsBuf in + codepointLens.withUnsafeBufferPointer { codepointsLensBuf in + tokenLens.withUnsafeBufferPointer { tokenLensBuf in + SherpaOnnxGenerationConfig( + silence_scale: swiftConfig.silenceScale, + speed: swiftConfig.speed, + sid: Int32(swiftConfig.sid), + reference_audio: buffer.count > 0 ? buffer.baseAddress : nil, + reference_audio_len: Int32(buffer.count), + reference_sample_rate: Int32(swiftConfig.referenceSampleRate), + reference_text: toCPointer(swiftConfig.referenceText), + num_steps: Int32(swiftConfig.numSteps), + extra: toCPointer(extraJson), + phoneme_codepoints: hasCodepoints ? codepointsBuf.baseAddress : nil, + phoneme_codepoints_lens: hasCodepoints ? codepointsLensBuf.baseAddress : nil, + phoneme_codepoints_num_sentences: Int32(codepointLens.count), + tokens: hasTokens ? UnsafePointer(tokenPtrs!) : nil, + tokens_lens: hasTokens ? tokenLensBuf.baseAddress : nil, + tokens_num_sentences: Int32(tokenLens.count) + ) + } + } + } + } + } + + deinit { + for p in tokenStrings { + if let p { + free(p) + } + } + tokenPtrsStorage?.deallocate() } } @@ -1412,6 +1502,14 @@ public class SherpaOnnxOfflineTtsWrapper { return SherpaOnnxOfflineTtsNumSpeakers(tts) } + /// The language/voice of the model, e.g., "en-us". Empty if not available. + public var lang: String { + guard let p = SherpaOnnxOfflineTtsLang(tts) else { return "" } + let ans = String(cString: p) + SherpaOnnxDestroyOfflineTtsLang(p) + return ans + } + /// Constructor taking a model config public init( config: UnsafePointer! diff --git a/swift-api-examples/run-tts-kitten-en.sh b/swift-api-examples/run-tts-kitten-en-lexicon.sh similarity index 65% rename from swift-api-examples/run-tts-kitten-en.sh rename to swift-api-examples/run-tts-kitten-en-lexicon.sh index 726fd3f9e..cb64b7ef4 100755 --- a/swift-api-examples/run-tts-kitten-en.sh +++ b/swift-api-examples/run-tts-kitten-en-lexicon.sh @@ -37,29 +37,26 @@ PY cp -R "${src_dir}/kitten-mini-en-v0_8" "${model_dir}" fi -if [ ! -d "${model_dir}/espeak-ng-data" ]; then - curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/espeak-ng-data.tar.bz2 - tar xf espeak-ng-data.tar.bz2 - rm espeak-ng-data.tar.bz2 - mv espeak-ng-data "${model_dir}/" +if [ ! -e ./lexicon-en-us.txt ]; then + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt fi -if [ ! -e ./tts-kitten-en ] || [ ./tts-kitten-en.swift -nt ./tts-kitten-en ] || [ ./SherpaOnnx.swift -nt ./tts-kitten-en ] || [ ../build-macos/install/lib/libsherpa-onnx-c-api.a -nt ./tts-kitten-en ]; then +if [ ! -e ./tts-kitten-en-lexicon ] || [ ./tts-kitten-en-lexicon.swift -nt ./tts-kitten-en-lexicon ] || [ ./SherpaOnnx.swift -nt ./tts-kitten-en-lexicon ] || [ ../build-macos/install/lib/libsherpa-onnx-c-api.a -nt ./tts-kitten-en-lexicon ]; then # Note: We use -lc++ to link against libc++ instead of libstdc++ swiftc \ -lc++ \ -I ../build-macos/install/include \ -import-objc-header ./SherpaOnnx-Bridging-Header.h \ - ./tts-kitten-en.swift ./SherpaOnnx.swift \ + ./tts-kitten-en-lexicon.swift ./SherpaOnnx.swift \ -L ../build-macos/install/lib/ \ -l sherpa-onnx-c-api \ -l onnxruntime \ - -o tts-kitten-en + -o tts-kitten-en-lexicon - strip tts-kitten-en + strip tts-kitten-en-lexicon else - echo "./tts-kitten-en exists - skip building" + echo "./tts-kitten-en-lexicon exists - skip building" fi export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH -./tts-kitten-en +./tts-kitten-en-lexicon diff --git a/swift-api-examples/run-tts-kokoro-en.sh b/swift-api-examples/run-tts-kokoro-en-lexicon.sh similarity index 63% rename from swift-api-examples/run-tts-kokoro-en.sh rename to swift-api-examples/run-tts-kokoro-en-lexicon.sh index 3ec99937e..b60b9fa72 100755 --- a/swift-api-examples/run-tts-kokoro-en.sh +++ b/swift-api-examples/run-tts-kokoro-en-lexicon.sh @@ -16,22 +16,26 @@ if [ ! -f ./kokoro-en-v0_19/model.onnx ]; then rm kokoro-en-v0_19.tar.bz2 fi -if [ ! -e ./tts-kokoro-en ] || [ ../build-macos/install/lib/libsherpa-onnx-c-api.a -nt ./tts-kokoro-en ]; then +if [ ! -e ./lexicon-en-us.txt ]; then + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +if [ ! -e ./tts-kokoro-en-lexicon ] || [ ../build-macos/install/lib/libsherpa-onnx-c-api.a -nt ./tts-kokoro-en-lexicon ]; then # Note: We use -lc++ to link against libc++ instead of libstdc++ swiftc \ -lc++ \ -I ../build-macos/install/include \ -import-objc-header ./SherpaOnnx-Bridging-Header.h \ - ./tts-kokoro-en.swift ./SherpaOnnx.swift \ + ./tts-kokoro-en-lexicon.swift ./SherpaOnnx.swift \ -L ../build-macos/install/lib/ \ -l sherpa-onnx-c-api \ -l onnxruntime \ - -o tts-kokoro-en + -o tts-kokoro-en-lexicon - strip tts-kokoro-en + strip tts-kokoro-en-lexicon else - echo "./tts-kokoro-en exists - skip building" + echo "./tts-kokoro-en-lexicon exists - skip building" fi export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH -./tts-kokoro-en +./tts-kokoro-en-lexicon diff --git a/swift-api-examples/run-tts-kokoro-zh-en.sh b/swift-api-examples/run-tts-kokoro-zh-en-lexicon.sh similarity index 70% rename from swift-api-examples/run-tts-kokoro-zh-en.sh rename to swift-api-examples/run-tts-kokoro-zh-en-lexicon.sh index 5dcafab9a..448b2b4b2 100755 --- a/swift-api-examples/run-tts-kokoro-zh-en.sh +++ b/swift-api-examples/run-tts-kokoro-zh-en-lexicon.sh @@ -16,22 +16,22 @@ if [ ! -f ./kokoro-multi-lang-v1_0/model.onnx ]; then rm kokoro-multi-lang-v1_0.tar.bz2 fi -if [ ! -e ./tts-kokoro-zh-en ] || [ ../build-macos/install/lib/libsherpa-onnx-c-api.a -nt ./tts-kokoro-zh-en ]; then +if [ ! -e ./tts-kokoro-zh-en-lexicon ] || [ ../build-macos/install/lib/libsherpa-onnx-c-api.a -nt ./tts-kokoro-zh-en-lexicon ]; then # Note: We use -lc++ to link against libc++ instead of libstdc++ swiftc \ -lc++ \ -I ../build-macos/install/include \ -import-objc-header ./SherpaOnnx-Bridging-Header.h \ - ./tts-kokoro-zh-en.swift ./SherpaOnnx.swift \ + ./tts-kokoro-zh-en-lexicon.swift ./SherpaOnnx.swift \ -L ../build-macos/install/lib/ \ -l sherpa-onnx-c-api \ -l onnxruntime \ - -o tts-kokoro-zh-en + -o tts-kokoro-zh-en-lexicon - strip tts-kokoro-zh-en + strip tts-kokoro-zh-en-lexicon else - echo "./tts-kokoro-zh-en exists - skip building" + echo "./tts-kokoro-zh-en-lexicon exists - skip building" fi export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH -./tts-kokoro-zh-en +./tts-kokoro-zh-en-lexicon diff --git a/swift-api-examples/run-tts-matcha-en.sh b/swift-api-examples/run-tts-matcha-en-lexicon.sh similarity index 71% rename from swift-api-examples/run-tts-matcha-en.sh rename to swift-api-examples/run-tts-matcha-en-lexicon.sh index fe86d76b1..c67cb7108 100755 --- a/swift-api-examples/run-tts-matcha-en.sh +++ b/swift-api-examples/run-tts-matcha-en-lexicon.sh @@ -21,22 +21,26 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi -if [ ! -e ./tts-matcha-en ] || [ ../build-macos/install/lib/libsherpa-onnx-c-api.a -nt ./tts-matcha-en ]; then +if [ ! -e ./lexicon-en-us.txt ]; then + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +if [ ! -e ./tts-matcha-en-lexicon ] || [ ../build-macos/install/lib/libsherpa-onnx-c-api.a -nt ./tts-matcha-en-lexicon ]; then # Note: We use -lc++ to link against libc++ instead of libstdc++ swiftc \ -lc++ \ -I ../build-macos/install/include \ -import-objc-header ./SherpaOnnx-Bridging-Header.h \ - ./tts-matcha-en.swift ./SherpaOnnx.swift \ + ./tts-matcha-en-lexicon.swift ./SherpaOnnx.swift \ -L ../build-macos/install/lib/ \ -l sherpa-onnx-c-api \ -l onnxruntime \ - -o tts-matcha-en + -o tts-matcha-en-lexicon - strip tts-matcha-en + strip tts-matcha-en-lexicon else - echo "./tts-matcha-en exists - skip building" + echo "./tts-matcha-en-lexicon exists - skip building" fi export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH -./tts-matcha-en +./tts-matcha-en-lexicon diff --git a/swift-api-examples/run-tts-matcha-zh.sh b/swift-api-examples/run-tts-matcha-zh-lexicon.sh similarity index 81% rename from swift-api-examples/run-tts-matcha-zh.sh rename to swift-api-examples/run-tts-matcha-zh-lexicon.sh index a943a29e2..78680c4ad 100755 --- a/swift-api-examples/run-tts-matcha-zh.sh +++ b/swift-api-examples/run-tts-matcha-zh-lexicon.sh @@ -20,22 +20,22 @@ if [ ! -f ./vocos-22khz-univ.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos-22khz-univ.onnx fi -if [ ! -e ./tts-matcha-zh ]; then +if [ ! -e ./tts-matcha-zh-lexicon ]; then # Note: We use -lc++ to link against libc++ instead of libstdc++ swiftc \ -lc++ \ -I ../build-macos/install/include \ -import-objc-header ./SherpaOnnx-Bridging-Header.h \ - ./tts-matcha-zh.swift ./SherpaOnnx.swift \ + ./tts-matcha-zh-lexicon.swift ./SherpaOnnx.swift \ -L ../build-macos/install/lib/ \ -l sherpa-onnx-c-api \ -l onnxruntime \ - -o tts-matcha-zh + -o tts-matcha-zh-lexicon - strip tts-matcha-zh + strip tts-matcha-zh-lexicon else - echo "./tts-matcha-zh exists - skip building" + echo "./tts-matcha-zh-lexicon exists - skip building" fi export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH -./tts-matcha-zh +./tts-matcha-zh-lexicon diff --git a/swift-api-examples/run-tts-vits.sh b/swift-api-examples/run-tts-vits-lexicon.sh similarity index 69% rename from swift-api-examples/run-tts-vits.sh rename to swift-api-examples/run-tts-vits-lexicon.sh index 87c92f89e..cfc2dd065 100755 --- a/swift-api-examples/run-tts-vits.sh +++ b/swift-api-examples/run-tts-vits-lexicon.sh @@ -15,22 +15,26 @@ if [ ! -d ./vits-piper-en_US-amy-low ]; then rm vits-piper-en_US-amy-low.tar.bz2 fi -if [ ! -e ./tts-vits ]; then +if [ ! -e ./lexicon-en-us.txt ]; then + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +if [ ! -e ./tts-vits-lexicon ]; then # Note: We use -lc++ to link against libc++ instead of libstdc++ swiftc \ -lc++ \ -I ../build-macos/install/include \ -import-objc-header ./SherpaOnnx-Bridging-Header.h \ - ./tts-vits.swift ./SherpaOnnx.swift \ + ./tts-vits-lexicon.swift ./SherpaOnnx.swift \ -L ../build-macos/install/lib/ \ -l sherpa-onnx-c-api \ -l onnxruntime \ - -o tts-vits + -o tts-vits-lexicon - strip tts-vits + strip tts-vits-lexicon else - echo "./tts-vits exists - skip building" + echo "./tts-vits-lexicon exists - skip building" fi export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH -./tts-vits +./tts-vits-lexicon diff --git a/swift-api-examples/run-tts-zipvoice.sh b/swift-api-examples/run-tts-zipvoice-lexicon.sh similarity index 73% rename from swift-api-examples/run-tts-zipvoice.sh rename to swift-api-examples/run-tts-zipvoice-lexicon.sh index b2fd6e0f3..6cedc6128 100755 --- a/swift-api-examples/run-tts-zipvoice.sh +++ b/swift-api-examples/run-tts-zipvoice-lexicon.sh @@ -20,21 +20,25 @@ if [ ! -f ./vocos_24khz.onnx ]; then curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx fi -if [ ! -e ./tts-zipvoice ]; then +if [ ! -e ./lexicon-en-us.txt ]; then + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +fi + +if [ ! -e ./tts-zipvoice-lexicon ]; then swiftc \ -lc++ \ -I ../build-macos/install/include \ -import-objc-header ./SherpaOnnx-Bridging-Header.h \ - ./tts-zipvoice.swift ./SherpaOnnx.swift \ + ./tts-zipvoice-lexicon.swift ./SherpaOnnx.swift \ -L ../build-macos/install/lib/ \ -l sherpa-onnx-c-api \ -l onnxruntime \ - -o tts-zipvoice + -o tts-zipvoice-lexicon - strip tts-zipvoice + strip tts-zipvoice-lexicon else - echo "./tts-zipvoice exists - skip building" + echo "./tts-zipvoice-lexicon exists - skip building" fi export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH -./tts-zipvoice +./tts-zipvoice-lexicon diff --git a/swift-api-examples/run-zipformer-ctc-hlg.sh b/swift-api-examples/run-zipformer-ctc-hlg.sh new file mode 100755 index 000000000..6ad26d578 --- /dev/null +++ b/swift-api-examples/run-zipformer-ctc-hlg.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -d ../build-macos ]; then + echo "Please run ../build-macos.sh first!" + exit 1 +fi + +if [ ! -f ./sherpa-onnx-zipformer-ctc-en-2023-10-02/model.int8.onnx ]; then + echo "Please download the pre-trained model for testing." + echo "You can refer to" + echo "" + echo "https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/icefall/zipformer.html#sherpa-onnx-zipformer-ctc-en-2023-10-02-english" + echo "" + echo "for help" + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + + tar xvf sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + rm sherpa-onnx-zipformer-ctc-en-2023-10-02.tar.bz2 + ls -lh sherpa-onnx-zipformer-ctc-en-2023-10-02 +fi + +if [ ! -e ./zipformer-ctc-hlg ]; then + # Note: We use -lc++ to link against libc++ instead of libstdc++ + swiftc \ + -lc++ \ + -I ../build-macos/install/include \ + -import-objc-header ./SherpaOnnx-Bridging-Header.h \ + ./zipformer-ctc-hlg.swift ./SherpaOnnx.swift \ + -L ../build-macos/install/lib/ \ + -l sherpa-onnx-c-api \ + -l onnxruntime \ + -o zipformer-ctc-hlg + + strip zipformer-ctc-hlg +else + echo "./zipformer-ctc-hlg exists - skip building" +fi + +export DYLD_LIBRARY_PATH=$PWD/../build-macos/install/lib:$DYLD_LIBRARY_PATH +./zipformer-ctc-hlg diff --git a/swift-api-examples/tts-kitten-en.swift b/swift-api-examples/tts-kitten-en-lexicon.swift similarity index 94% rename from swift-api-examples/tts-kitten-en.swift rename to swift-api-examples/tts-kitten-en-lexicon.swift index 5f5ff1983..43c453253 100644 --- a/swift-api-examples/tts-kitten-en.swift +++ b/swift-api-examples/tts-kitten-en-lexicon.swift @@ -8,12 +8,12 @@ func run() { let model = "./kitten-mini-en-v0_8/model.onnx" let voices = "./kitten-mini-en-v0_8/voices.bin" let tokens = "./kitten-mini-en-v0_8/tokens.txt" - let dataDir = "./kitten-mini-en-v0_8/espeak-ng-data" + let lexicon = "./lexicon-en-us.txt" let kitten = sherpaOnnxOfflineTtsKittenModelConfig( model: model, voices: voices, tokens: tokens, - dataDir: dataDir + lexicon: lexicon ) let modelConfig = sherpaOnnxOfflineTtsModelConfig(debug: 0, kitten: kitten) var ttsConfig = sherpaOnnxOfflineTtsConfig(model: modelConfig) @@ -50,7 +50,7 @@ func run() { let audio = tts.generateWithConfig( text: text, config: genConfig, callback: callback, arg: arg) - let filename = "test-kitten-en.wav" + let filename = "test-kitten-en-lexicon.wav" let ok = audio.save(filename: filename) if ok == 1 { print("\nSaved to:\(filename)") diff --git a/swift-api-examples/tts-kokoro-en.swift b/swift-api-examples/tts-kokoro-en-lexicon.swift similarity index 94% rename from swift-api-examples/tts-kokoro-en.swift rename to swift-api-examples/tts-kokoro-en-lexicon.swift index 3f08b0fce..038db6971 100644 --- a/swift-api-examples/tts-kokoro-en.swift +++ b/swift-api-examples/tts-kokoro-en-lexicon.swift @@ -8,12 +8,12 @@ func run() { let model = "./kokoro-en-v0_19/model.onnx" let voices = "./kokoro-en-v0_19/voices.bin" let tokens = "./kokoro-en-v0_19/tokens.txt" - let dataDir = "./kokoro-en-v0_19/espeak-ng-data" + let lexicon = "./lexicon-en-us.txt" let kokoro = sherpaOnnxOfflineTtsKokoroModelConfig( model: model, voices: voices, tokens: tokens, - dataDir: dataDir + lexicon: lexicon ) let modelConfig = sherpaOnnxOfflineTtsModelConfig(kokoro: kokoro, debug: 0) var ttsConfig = sherpaOnnxOfflineTtsConfig(model: modelConfig) @@ -50,7 +50,7 @@ func run() { let audio = tts.generateWithConfig( text: text, config: genConfig, callback: callback, arg: arg) - let filename = "test-kokoro-en.wav" + let filename = "test-kokoro-en-lexicon.wav" let ok = audio.save(filename: filename) if ok == 1 { print("\nSaved to:\(filename)") diff --git a/swift-api-examples/tts-kokoro-zh-en.swift b/swift-api-examples/tts-kokoro-zh-en-lexicon.swift similarity index 94% rename from swift-api-examples/tts-kokoro-zh-en.swift rename to swift-api-examples/tts-kokoro-zh-en-lexicon.swift index 7392bf2c3..2b88a78cc 100644 --- a/swift-api-examples/tts-kokoro-zh-en.swift +++ b/swift-api-examples/tts-kokoro-zh-en-lexicon.swift @@ -8,13 +8,11 @@ func run() { let model = "./kokoro-multi-lang-v1_0/model.onnx" let voices = "./kokoro-multi-lang-v1_0/voices.bin" let tokens = "./kokoro-multi-lang-v1_0/tokens.txt" - let dataDir = "./kokoro-multi-lang-v1_0/espeak-ng-data" let lexicon = "./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt" let kokoro = sherpaOnnxOfflineTtsKokoroModelConfig( model: model, voices: voices, tokens: tokens, - dataDir: dataDir, lexicon: lexicon ) let modelConfig = sherpaOnnxOfflineTtsModelConfig(kokoro: kokoro, debug: 0) @@ -52,7 +50,7 @@ func run() { let audio = tts.generateWithConfig( text: text, config: genConfig, callback: callback, arg: arg) - let filename = "test-kokoro-zh-en.wav" + let filename = "test-kokoro-zh-en-lexicon.wav" let ok = audio.save(filename: filename) if ok == 1 { print("\nSaved to:\(filename)") diff --git a/swift-api-examples/tts-matcha-en.swift b/swift-api-examples/tts-matcha-en-lexicon.swift similarity index 92% rename from swift-api-examples/tts-matcha-en.swift rename to swift-api-examples/tts-matcha-en-lexicon.swift index 3b0fd8d98..ffb68ba76 100644 --- a/swift-api-examples/tts-matcha-en.swift +++ b/swift-api-examples/tts-matcha-en-lexicon.swift @@ -8,12 +8,12 @@ func run() { let acousticModel = "./matcha-icefall-en_US-ljspeech/model-steps-3.onnx" let vocoder = "./vocos-22khz-univ.onnx" let tokens = "./matcha-icefall-en_US-ljspeech/tokens.txt" - let dataDir = "./matcha-icefall-en_US-ljspeech/espeak-ng-data" + let lexicon = "./lexicon-en-us.txt" let matcha = sherpaOnnxOfflineTtsMatchaModelConfig( acousticModel: acousticModel, vocoder: vocoder, - tokens: tokens, - dataDir: dataDir + lexicon: lexicon, + tokens: tokens ) let modelConfig = sherpaOnnxOfflineTtsModelConfig(matcha: matcha, debug: 0) var ttsConfig = sherpaOnnxOfflineTtsConfig(model: modelConfig) @@ -50,7 +50,7 @@ func run() { let audio = tts.generateWithConfig( text: text, config: genConfig, callback: callback, arg: arg) - let filename = "test-matcha-en.wav" + let filename = "test-matcha-en-lexicon.wav" let ok = audio.save(filename: filename) if ok == 1 { print("\nSaved to:\(filename)") diff --git a/swift-api-examples/tts-matcha-zh.swift b/swift-api-examples/tts-matcha-zh-lexicon.swift similarity index 97% rename from swift-api-examples/tts-matcha-zh.swift rename to swift-api-examples/tts-matcha-zh-lexicon.swift index b8139e8e2..24efa845b 100644 --- a/swift-api-examples/tts-matcha-zh.swift +++ b/swift-api-examples/tts-matcha-zh-lexicon.swift @@ -51,7 +51,7 @@ func run() { let audio = tts.generateWithConfig( text: text, config: genConfig, callback: callback, arg: arg) - let filename = "test-matcha-zh.wav" + let filename = "test-matcha-zh-lexicon.wav" let ok = audio.save(filename: filename) if ok == 1 { print("\nSaved to:\(filename)") diff --git a/swift-api-examples/tts-vits.swift b/swift-api-examples/tts-vits-lexicon.swift similarity index 93% rename from swift-api-examples/tts-vits.swift rename to swift-api-examples/tts-vits-lexicon.swift index 6be8bda90..dc5628aca 100644 --- a/swift-api-examples/tts-vits.swift +++ b/swift-api-examples/tts-vits-lexicon.swift @@ -7,12 +7,11 @@ class MyClass { func run() { let model = "./vits-piper-en_US-amy-low/en_US-amy-low.onnx" let tokens = "./vits-piper-en_US-amy-low/tokens.txt" - let dataDir = "./vits-piper-en_US-amy-low/espeak-ng-data" + let lexicon = "./lexicon-en-us.txt" let vits = sherpaOnnxOfflineTtsVitsModelConfig( model: model, - lexicon: "", + lexicon: lexicon, tokens: tokens, - dataDir: dataDir ) let modelConfig = sherpaOnnxOfflineTtsModelConfig(vits: vits) var ttsConfig = sherpaOnnxOfflineTtsConfig(model: modelConfig) @@ -49,7 +48,7 @@ func run() { let audio = tts.generateWithConfig( text: text, config: genConfig, callback: callback, arg: arg) - let filename = "test-vits-en.wav" + let filename = "test-vits-en-lexicon.wav" let ok = audio.save(filename: filename) if ok == 1 { print("\nSaved to:\(filename)") diff --git a/swift-api-examples/tts-zipvoice.swift b/swift-api-examples/tts-zipvoice-lexicon.swift similarity index 91% rename from swift-api-examples/tts-zipvoice.swift rename to swift-api-examples/tts-zipvoice-lexicon.swift index 9a80b21c0..b6924bf58 100644 --- a/swift-api-examples/tts-zipvoice.swift +++ b/swift-api-examples/tts-zipvoice-lexicon.swift @@ -12,8 +12,7 @@ func runZipVoiceTtsDemo() { encoder: "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx", decoder: "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx", vocoder: "./vocos_24khz.onnx", - dataDir: "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data", - lexicon: "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt" + lexicon: "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt,./lexicon-en-us.txt" ) let modelConfig = sherpaOnnxOfflineTtsModelConfig(numThreads: 2, zipvoice: zipvoice) @@ -67,9 +66,9 @@ func runZipVoiceTtsDemo() { return 1 } - generateAndSave(outputFile: "generated-zipvoice-callback.wav", callback: callback, arg: arg) + generateAndSave(outputFile: "generated-zipvoice-lexicon-callback.wav", callback: callback, arg: arg) } else { - generateAndSave(outputFile: "generated-zipvoice-direct.wav") + generateAndSave(outputFile: "generated-zipvoice-lexicon-direct.wav") } } diff --git a/swift-api-examples/zipformer-ctc-hlg.swift b/swift-api-examples/zipformer-ctc-hlg.swift new file mode 100644 index 000000000..f36fe1a6e --- /dev/null +++ b/swift-api-examples/zipformer-ctc-hlg.swift @@ -0,0 +1,78 @@ +import AVFoundation + +// This file shows how to use a non-streaming Zipformer CTC model with an HLG +// decoding graph via sherpa-onnx's Swift API. +// +// The HLG graph is passed to the C API through ctcFstDecoderConfig. + +extension AudioBuffer { + func array() -> [Float] { + return Array(UnsafeBufferPointer(self)) + } +} + +extension AVAudioPCMBuffer { + func array() -> [Float] { + return self.audioBufferList.pointee.mBuffers.array() + } +} + +func run() { + let dir = "./sherpa-onnx-zipformer-ctc-en-2023-10-02" + let model = dir + "/model.int8.onnx" + let tokens = dir + "/tokens.txt" + let graph = dir + "/HLG.fst" + + let zipformerCtc = sherpaOnnxOfflineZipformerCtcModelConfig( + model: model + ) + + let modelConfig = sherpaOnnxOfflineModelConfig( + tokens: tokens, + debug: 0, + zipformerCtc: zipformerCtc + ) + + let featConfig = sherpaOnnxFeatureConfig( + sampleRate: 16000, + featureDim: 80 + ) + + // This is what the example exercises: the graph and maxActive reach the C + // API through ctcFstDecoderConfig. + let ctcFstDecoderConfig = sherpaOnnxOfflineCtcFstDecoderConfig( + graph: graph, + maxActive: 3000 + ) + + var config = sherpaOnnxOfflineRecognizerConfig( + featConfig: featConfig, + modelConfig: modelConfig, + ctcFstDecoderConfig: ctcFstDecoderConfig + ) + + let recognizer = SherpaOnnxOfflineRecognizer(config: &config) + + let filePath = dir + "/test_wavs/0.wav" + let fileURL: NSURL = NSURL(fileURLWithPath: filePath) + let audioFile = try! AVAudioFile(forReading: fileURL as URL) + + let audioFormat = audioFile.processingFormat + assert(audioFormat.channelCount == 1) + assert(audioFormat.commonFormat == AVAudioCommonFormat.pcmFormatFloat32) + + let audioFrameCount = UInt32(audioFile.length) + let audioFileBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: audioFrameCount) + + try! audioFile.read(into: audioFileBuffer!) + let array: [Float]! = audioFileBuffer?.array() + let result = recognizer.decode(samples: array, sampleRate: Int(audioFormat.sampleRate)) + print("\nresult is:\n\(result.text)") +} + +@main +struct App { + static func main() { + run() + } +} diff --git a/wasm/tts/CMakeLists.txt b/wasm/tts/CMakeLists.txt index 663076b2f..2b9acc644 100644 --- a/wasm/tts/CMakeLists.txt +++ b/wasm/tts/CMakeLists.txt @@ -3,18 +3,42 @@ if(NOT $ENV{SHERPA_ONNX_IS_USING_BUILD_WASM_SH}) endif() if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/tokens.txt" AND - NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/lm_flow.int8.onnx") + NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/lm_flow.int8.onnx" AND + NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/unicode_indexer.bin") message(FATAL_ERROR "Please read ${CMAKE_CURRENT_SOURCE_DIR}/assets/README.md before you continue") endif() +# Text frontend: check which option of assets/README.md is set up. Neither the +# model package's espeak-ng-data nor anything in sherpa-onnx provides one, so +# without these files the demo cannot convert text at all. +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/piper-phonemize/piper-phonemize.js") + message(STATUS "Text frontend: piper-phonemize (option 1)") +elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/lexicon-en-us.txt") + message(STATUS "Text frontend: lexicon (option 2: assets/lexicon-en-us.txt)") +elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/lexicon.txt") + message(STATUS "Text frontend: lexicon (option 2: assets/lexicon.txt)") +elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/lm_flow.int8.onnx") + message(STATUS "Text frontend: built into the pocket model") +elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/unicode_indexer.bin") + message(STATUS "Text frontend: built into the supertonic model") +else() + message(FATAL_ERROR + "No text frontend found: please read ${CMAKE_CURRENT_SOURCE_DIR}/assets/README.md " + "before you continue. Either the piper-phonemize wasm bundle (option 1) or a " + "lexicon file (option 2) is required; the pocket and supertonic models bring " + "their own.") +endif() + set(exported_functions MyPrint SherpaOnnxCreateOfflineTts SherpaOnnxDestroyOfflineTts SherpaOnnxDestroyOfflineTtsGeneratedAudio + SherpaOnnxDestroyOfflineTtsLang SherpaOnnxOfflineTtsGenerate SherpaOnnxOfflineTtsGenerateWithCallback SherpaOnnxOfflineTtsGenerateWithConfig + SherpaOnnxOfflineTtsLang SherpaOnnxOfflineTtsNumSpeakers SherpaOnnxOfflineTtsSampleRate SherpaOnnxWriteWave @@ -48,6 +72,9 @@ endif() add_executable(sherpa-onnx-wasm-main-tts sherpa-onnx-wasm-main-tts.cc) target_link_libraries(sherpa-onnx-wasm-main-tts sherpa-onnx-c-api) install(TARGETS sherpa-onnx-wasm-main-tts DESTINATION bin/wasm/tts) +# The piper-phonemize wasm bundle (option 1: the default text frontend); +# see assets/README.md. Absent for the lexicon-only models. +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/piper-phonemize DESTINATION bin/wasm/tts OPTIONAL) install( FILES diff --git a/wasm/tts/app-tts.js b/wasm/tts/app-tts.js index d951fedef..5e23d2ad7 100644 --- a/wasm/tts/app-tts.js +++ b/wasm/tts/app-tts.js @@ -2,6 +2,11 @@ const generateBtn = document.getElementById('generateBtn'); const speakerIdLabel = document.getElementById('speakerIdLabel'); const speakerIdInput = document.getElementById('speakerId'); const speakerIdSection = document.getElementById('speakerIdSection'); +const langSection = document.getElementById('langSection'); +const langSelect = document.getElementById('lang'); +const frontendSection = document.getElementById('frontendSection'); +const frontendPhonemizeInput = document.getElementById('frontendPhonemize'); +const frontendLexiconInput = document.getElementById('frontendLexicon'); const referenceAudioSection = document.getElementById('referenceAudioSection'); const referenceTextSection = document.getElementById('referenceTextSection'); const referenceAudioInput = document.getElementById('referenceAudio'); @@ -21,12 +26,14 @@ let audioCtx = null; const worker = new Worker("sherpa-onnx-tts.worker.js"); let ttsInstanceInfo = { modelType: null, + usePhonemize: false, numSpeakers: 0, + lang: '', isReady: false, }; worker.onmessage = (e) => { if (e.data.type === "sherpa-onnx-tts-progress") { - Module.setStatus(e.data.status); + uiStatus.setStatus(e.data.status); return; } if (e.data.type === "sherpa-onnx-tts-generation-progress") { @@ -36,12 +43,15 @@ worker.onmessage = (e) => { } if (e.data.type === "sherpa-onnx-tts-ready") { ttsInstanceInfo.modelType = e.data.modelType; + ttsInstanceInfo.usePhonemize = e.data.usePhonemize; ttsInstanceInfo.numSpeakers = e.data.numSpeakers; + ttsInstanceInfo.lang = e.data.lang; ttsInstanceInfo.isReady = true; generateBtn.disabled = false; speakerIdLabel.innerHTML = `Speaker ID (0 - ${e.data.numSpeakers - 1}):`; updateUiForModelType(); - Module.setStatus(''); + uiStatus.setStatus(''); + setGenerationStatus(''); return; } if (e.data.type === "error") { @@ -49,7 +59,7 @@ worker.onmessage = (e) => { if (ttsInstanceInfo.isReady) { setGenerationStatus(e.data.message); } else { - Module.setStatus(e.data.message); + uiStatus.setStatus(e.data.message); } return; } @@ -80,10 +90,12 @@ worker.onmessage = (e) => { } }; -Module = {}; +// UI status helper, used by the sherpa-onnx worker messages below. +const uiStatus = {}; -// https://emscripten.org/docs/api_reference/module.html#Module.locateFile -Module.setStatus = function(status) { +// Mirrors emscripten's Module.setStatus: +// https://emscripten.org/docs/api_reference/module.html#Module.setStatus +uiStatus.setStatus = function(status) { console.log(`status ${status}`); if (status == 'Running...') { status = 'Model downloaded. Initializing text to speech model...' @@ -117,6 +129,65 @@ Module.setStatus = function(status) { }); } }; + +// piper-phonemize's Emscripten module. This mirrors its standalone demo +// (piper-phonemize/index.html + piper-phonemize/app-phonemize.js shipped in +// the bundle): the app defines `Module`, the glue script is loaded last (see +// index.html) and attaches its runtime to the same object. +// +// The one difference from that demo: it lives next to the bundle files while +// this page is one directory above them. When the glue fetches its .data it +// calls locateFile(name, "") with an empty scriptDirectory, so we must prefix +// "piper-phonemize/" ourselves - otherwise the .data request resolves against +// this page's URL, 404s, the espeak-ng data never reaches the wasm FS, and +// espeak init fails. +let resolvePiperReady; +let rejectPiperReady; +const piperReady = new Promise((resolve, reject) => { + resolvePiperReady = resolve; + rejectPiperReady = reject; +}); +// Failures are reported at the generate step below; don't also warn here. +piperReady.catch(() => {}); + +// https://emscripten.org/docs/api_reference/module.html#Module.locateFile +var Module = { + locateFile: function(path, scriptDirectory = '') { + return (scriptDirectory || 'piper-phonemize/') + path; + }, + + setStatus: function(status) { + // piper's runtime statuses are not about the TTS model; keep the status + // bar for the sherpa-onnx download/init progress. + if (status === 'Running...') return; + uiStatus.setStatus(status); + }, + + onRuntimeInitialized: function() { + console.log('piper-phonemize WASM runtime initialized'); + try { + const ret = piperPhonemizeInitialize(); + console.log('piperPhonemizeInitialize returned:', ret); + if (ret >= 0) { + // 22050 on the first call, 0 if espeak-ng was already initialized + resolvePiperReady(); + } else { + rejectPiperReady(new Error('failed to initialize espeak-ng')); + } + } catch (e) { + console.error('piper-phonemize initialization error:', e); + rejectPiperReady(e); + } + }, +}; + +// The glue is loaded after this file. If the bundle is missing entirely (see +// assets/README.md), say so instead of waiting forever on the promise above. +if (typeof piperPhonemizeInitialize !== 'function') { + rejectPiperReady(new Error( + 'piper-phonemize bundle not loaded; see wasm/tts/assets/README.md')); +} + speedInput.oninput = function() { speedValue.innerHTML = this.value; }; @@ -124,12 +195,36 @@ speedInput.oninput = function() { function updateUiForModelType() { const isZipVoice = ttsInstanceInfo.modelType === 4; const isPocketTts = ttsInstanceInfo.modelType === 5; + const isSupertonic = ttsInstanceInfo.modelType === 7; const useGenerationConfig = isZipVoice || isPocketTts; speakerIdSection.classList.toggle('hidden', useGenerationConfig); referenceAudioSection.classList.toggle('hidden', !useGenerationConfig); referenceTextSection.classList.toggle('hidden', !isZipVoice); + // SupertonicTTS supports 31 languages; no other model here needs a + // language picker. + langSection.classList.toggle('hidden', !isSupertonic); + + // The picker is only meaningful for the English piper-family models, which + // can use either frontend. The Chinese/multi-lingual/zipvoice models have a + // lexicon only, the pocket model has its own frontend, and the German model + // is phonemize-only (no German lexicon exists). + const isEnglishFamily = [0, 3, 6].includes(ttsInstanceInfo.modelType) && + (ttsInstanceInfo.lang || '').toLowerCase().startsWith('en'); + frontendSection.classList.toggle('hidden', !isEnglishFamily); + frontendPhonemizeInput.checked = ttsInstanceInfo.usePhonemize; + frontendLexiconInput.checked = !ttsInstanceInfo.usePhonemize; } +function onFrontendChange() { + const usePhonemize = frontendPhonemizeInput.checked; + generateBtn.disabled = true; + setGenerationStatus('Switching text frontend...'); + worker.postMessage({type: 'setFrontend', usePhonemize: usePhonemize}); +} + +frontendPhonemizeInput.addEventListener('change', onFrontendChange); +frontendLexiconInput.addEventListener('change', onFrontendChange); + function setGenerationStatus(status) { if (!generationStatusElement) { return; @@ -196,6 +291,7 @@ function downloadBlob(blob, filename) { generateBtn.onclick = async function() { const isZipVoice = ttsInstanceInfo.modelType === 4; const isPocketTts = ttsInstanceInfo.modelType === 5; + const isSupertonic = ttsInstanceInfo.modelType === 7; const useGenerationConfig = isZipVoice || isPocketTts; let speakerId = speakerIdInput.value; @@ -228,6 +324,26 @@ generateBtn.onclick = async function() { console.log('speed', speedInput.value); console.log('text', text); + if (isSupertonic) { + // SupertonicTTS has its own text frontend (unicode indexer): pass the + // raw text plus the language; no reference audio and no phonemize. + // numSteps is 8, like python-api-examples/supertonic-tts.py and + // nodejs-examples/test-offline-tts-supertonic.js. + generateBtn.disabled = true; + setGenerationStatus('Generating audio...'); + worker.postMessage({ + text, + type: 'generateWithConfig', + genConfig: { + sid: speakerId, + speed: parseFloat(speedInput.value), + numSteps: 8, + extra: {lang: langSelect.value}, + }, + }); + return; + } + if (useGenerationConfig) { if (!referenceAudioInput.files || referenceAudioInput.files.length === 0) { alert('Please select a reference audio file'); @@ -272,6 +388,42 @@ generateBtn.onclick = async function() { return; } + if (ttsInstanceInfo.usePhonemize) { + // Option 1: phonemize the text with piper-phonemize (loaded from + // ./piper-phonemize/, see assets/README.md) and pass the phoneme + // codepoints. No lexicon file and no espeak-ng-data are needed. Set + // usePhonemize = false in sherpa-onnx-tts.js for option 2 (lexicon). + try { + await piperReady; + } catch (e) { + generateBtn.disabled = false; + setGenerationStatus(`piper-phonemize is not available: ${e.message}`); + return; + } + const voice = ttsInstanceInfo.lang || 'en-us'; + const sentences = piperPhonemize(text, voice); + if (!sentences) { + generateBtn.disabled = false; + setGenerationStatus('Phonemization failed'); + return; + } + // piper may emit empty sentences (e.g. a trailing segment after + // punctuation); the C API requires every sentence to be non-empty. + const phonemeCodepoints = sentences.filter((sentence) => sentence.length > 0); + generateBtn.disabled = true; + setGenerationStatus('Generating audio...'); + worker.postMessage({ + text: '', + genConfig: { + phonemeCodepoints: phonemeCodepoints, + sid: speakerId, + speed: parseFloat(speedInput.value), + }, + type: "generateWithConfig", + }); + return; + } + worker.postMessage({ text, sid: speakerId, diff --git a/wasm/tts/assets/README.md b/wasm/tts/assets/README.md index 7772c0318..08ce61def 100644 --- a/wasm/tts/assets/README.md +++ b/wasm/tts/assets/README.md @@ -4,6 +4,44 @@ Please refer to https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models to download a model. +## Text frontend + +Choose exactly one of the two text frontends below (not both): + +- **option 1 (default): phonemize** — the text is phonemized with + piper-phonemize; no extra *model* files are needed. Download the wasm bundle + and unzip it so the files end up in `wasm/tts/piper-phonemize/`: + + ```bash + cd sherpa-onnx/wasm/tts + wget -q https://github.com/csukuangfj/piper-phonemize/releases/download/wasm/piper-phonemize-v1.5.8-wasm-browser.zip + unzip -q piper-phonemize-v1.5.8-wasm-browser.zip + mv browser piper-phonemize + rm piper-phonemize-v1.5.8-wasm-browser.zip + ``` + +- **option 2: lexicon** — set `usePhonemize = false` in + `../sherpa-onnx-tts.js` and put a lexicon next to the model files: + + ```bash + cd sherpa-onnx/wasm/tts/assets + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt + ``` + +Neither option uses the espeak-ng-data directory shipped in the model +package; you can leave it out to keep the download small. + +Some models bring their own text frontend and need neither option above: +PocketTTS (`modelType` 5) and SupertonicTTS (`modelType` 7). For those, skip +this whole section. + +Please set `let modelType = 0` to the number matching your model in +`../sherpa-onnx-tts.js` (0 vits, 1/2/3 matcha, 4 zipvoice, 5 pocket, +6 kitten, 7 supertonic, 8 kokoro-multi-lang). The demo UI adapts itself +to the model type. + +## Model files + The following is an example: ```bash cd sherpa-onnx/wasm/tts/assets @@ -13,7 +51,6 @@ tar xf vits-piper-en_US-libritts_r-medium.tar.bz2 rm vits-piper-en_US-libritts_r-medium.tar.bz2 mv vits-piper-en_US-libritts_r-medium/en_US-libritts_r-medium.onnx ./model.onnx mv vits-piper-en_US-libritts_r-medium/tokens.txt ./ -mv vits-piper-en_US-libritts_r-medium/espeak-ng-data ./ rm -rf vits-piper-en_US-libritts_r-medium ``` @@ -30,7 +67,6 @@ mv sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx ./ mv sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/decoder.int8.onnx ./ mv sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/tokens.txt ./ mv sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/lexicon.txt ./ -mv sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/espeak-ng-data ./ rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx @@ -55,6 +91,28 @@ mv sherpa-onnx-pocket-tts-int8-2026-01-26/token_scores.json ./ rm -rf sherpa-onnx-pocket-tts-int8-2026-01-26 ``` +SupertonicTTS example (31 languages; own text frontend, so no lexicon and no +piper-phonemize are needed): + +```bash +cd sherpa-onnx/wasm/tts/assets + +wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 +tar xf sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 +rm sherpa-onnx-supertonic-3-tts-int8-2026-05-11.tar.bz2 + +mv sherpa-onnx-supertonic-3-tts-int8-2026-05-11/duration_predictor.int8.onnx ./ +mv sherpa-onnx-supertonic-3-tts-int8-2026-05-11/text_encoder.int8.onnx ./ +mv sherpa-onnx-supertonic-3-tts-int8-2026-05-11/vector_estimator.int8.onnx ./ +mv sherpa-onnx-supertonic-3-tts-int8-2026-05-11/vocoder.int8.onnx ./ +mv sherpa-onnx-supertonic-3-tts-int8-2026-05-11/tts.json ./ +mv sherpa-onnx-supertonic-3-tts-int8-2026-05-11/unicode_indexer.bin ./ +mv sherpa-onnx-supertonic-3-tts-int8-2026-05-11/voice.bin ./ +rm -rf sherpa-onnx-supertonic-3-tts-int8-2026-05-11 + +# in ../sherpa-onnx-tts.js, set: let modelType = 7 +``` + You should have the following files in `assets` before you can run `build-wasm-simd-tts.sh` @@ -62,11 +120,10 @@ You should have the following files in `assets` before you can run assets fangjun$ tree -L 1 . ├── README.md -├── espeak-ng-data -├── mode.onnx +├── model.onnx └── tokens.txt -1 directory, 3 files +1 directory, 2 files ``` You can find example build scripts at: diff --git a/wasm/tts/index.html b/wasm/tts/index.html index e842b4322..662e92fed 100644 --- a/wasm/tts/index.html +++ b/wasm/tts/index.html @@ -30,12 +30,62 @@
Loading...
+


+ + + + + + diff --git a/wasm/tts/sherpa-onnx-tts.js b/wasm/tts/sherpa-onnx-tts.js index a7de45079..c70d2abd2 100644 --- a/wasm/tts/sherpa-onnx-tts.js +++ b/wasm/tts/sherpa-onnx-tts.js @@ -38,6 +38,12 @@ function freeConfig(config, Module) { } // The user should free the returned pointers +// +// Note on `dataDir` (all model configs below, vits/matcha/kokoro/kitten/ +// zipvoice): deprecated and ignored since v2.0.0. espeak-ng is no longer used +// by sherpa-onnx. The slot is still marshalled for ABI stability, but the C +// side discards it. Use `lexicon` for plain text, or pass pre-phonemized +// input via `phonemeCodepoints` / `tokens` in the generation config. function initSherpaOnnxOfflineTtsVitsModelConfig(config, Module) { const modelLen = Module.lengthBytesUTF8(config.model || '') + 1; const lexiconLen = Module.lengthBytesUTF8(config.lexicon || '') + 1; @@ -237,12 +243,13 @@ function initSherpaOnnxOfflineTtsKittenModelConfig(config, Module) { const voicesLen = Module.lengthBytesUTF8(config.voices) + 1; const tokensLen = Module.lengthBytesUTF8(config.tokens || '') + 1; const dataDirLen = Module.lengthBytesUTF8(config.dataDir || '') + 1; + const lexiconLen = Module.lengthBytesUTF8(config.lexicon || '') + 1; - const n = modelLen + voicesLen + tokensLen + dataDirLen; + const n = modelLen + voicesLen + tokensLen + dataDirLen + lexiconLen; const buffer = Module._malloc(n); - const len = 5 * 4; + const len = 6 * 4; const ptr = Module._malloc(len); let offset = 0; @@ -258,6 +265,9 @@ function initSherpaOnnxOfflineTtsKittenModelConfig(config, Module) { Module.stringToUTF8(config.dataDir || '', buffer + offset, dataDirLen); offset += dataDirLen; + Module.stringToUTF8(config.lexicon || '', buffer + offset, lexiconLen); + offset += lexiconLen; + offset = 0; Module.setValue(ptr, buffer + offset, 'i8*'); offset += modelLen; @@ -273,6 +283,8 @@ function initSherpaOnnxOfflineTtsKittenModelConfig(config, Module) { Module.setValue(ptr + 16, config.lengthScale || 1.0, 'float'); + Module.setValue(ptr + 20, buffer + offset, 'i8*'); + return { buffer: buffer, ptr: ptr, @@ -690,10 +702,6 @@ function initSherpaOnnxOfflineTtsConfig(config, Module) { if ('model' in config && !('offlineTtsModelConfig' in config)) { config.offlineTtsModelConfig = config.model; } - // Support typo in Dart config class. - if ('maxNumSenetences' in config && !('maxNumSentences' in config)) { - config.maxNumSentences = config.maxNumSenetences; - } const modelConfig = initSherpaOnnxOfflineTtsModelConfig(config.offlineTtsModelConfig, Module); @@ -745,9 +753,44 @@ const genConfig = { */ +// Validate a nested-array field (phonemeCodepoints: number[][], tokens: +// string[][]) before anything is allocated. Anything else is a caller bug, +// so throw a TypeError naming the offending element. +function validateNestedArray(name, value, elementType) { + if (!Array.isArray(value)) { + throw new TypeError(`${name} must be an array of arrays`); + } + for (let i = 0; i < value.length; ++i) { + const inner = value[i]; + if (!Array.isArray(inner)) { + throw new TypeError(`${name}[${i}] must be an array`); + } + if (inner.length === 0) { + throw new TypeError(`${name}[${i}] must be a non-empty array`); + } + for (let j = 0; j < inner.length; ++j) { + const item = inner[j]; + const ok = elementType === 'number' ? typeof item === 'number' : + typeof item === 'string'; + if (!ok) { + throw new TypeError(`${name}[${i}][${j}] must be a ${elementType}`); + } + } + } +} + + // Allocate a SherpaOnnxGenerationConfig in WASM function initSherpaOnnxGenerationConfig(config, Module) { - const len = 9 * 4; + // Validate before allocating anything. + if (config.phonemeCodepoints !== undefined) { + validateNestedArray('phonemeCodepoints', config.phonemeCodepoints, 'number'); + } + if (config.tokens !== undefined) { + validateNestedArray('tokens', config.tokens, 'string'); + } + + const len = 15 * 4; const ptr = Module._malloc(len); // float silence_scale @@ -807,11 +850,75 @@ function initSherpaOnnxGenerationConfig(config, Module) { Module.setValue(ptr + 8 * 4, extraPtr, 'i8*'); + // const int32_t* phoneme_codepoints + lens + num_sentences + let phonemeCodepointsPtr = 0; + let phonemeCodepointsLensPtr = 0; + if (config.phonemeCodepoints && config.phonemeCodepoints.length > 0) { + const numSentences = config.phonemeCodepoints.length; + let total = 0; + for (const inner of config.phonemeCodepoints) total += inner.length; + + phonemeCodepointsPtr = Module._malloc(total * 4); + phonemeCodepointsLensPtr = Module._malloc(numSentences * 4); + + const flat = new Int32Array(total); + const lens = new Int32Array(numSentences); + let offset = 0; + for (let i = 0; i < numSentences; ++i) { + flat.set(config.phonemeCodepoints[i], offset); + lens[i] = config.phonemeCodepoints[i].length; + offset += config.phonemeCodepoints[i].length; + } + Module.HEAP32.set(flat, phonemeCodepointsPtr / 4); + Module.HEAP32.set(lens, phonemeCodepointsLensPtr / 4); + } + Module.setValue(ptr + 9 * 4, phonemeCodepointsPtr, 'i8*'); + Module.setValue(ptr + 10 * 4, phonemeCodepointsLensPtr, 'i8*'); + Module.setValue(ptr + 11 * 4, + config.phonemeCodepoints ? config.phonemeCodepoints.length : 0, 'i32'); + + // const char* const* tokens + lens + num_sentences + let tokensPtr = 0; + let tokensLensPtr = 0; + const tokenStringPtrs = []; + if (config.tokens && config.tokens.length > 0) { + const numSentences = config.tokens.length; + let total = 0; + for (const inner of config.tokens) total += inner.length; + + tokensPtr = Module._malloc(Math.max(total, 1) * 4); + tokensLensPtr = Module._malloc(numSentences * 4); + + const lens = new Int32Array(numSentences); + let offset = 0; + for (let i = 0; i < numSentences; ++i) { + lens[i] = config.tokens[i].length; + for (const token of config.tokens[i]) { + const n = Module.lengthBytesUTF8(token) + 1; + const p = Module._malloc(n); + Module.stringToUTF8(token, p, n); + tokenStringPtrs.push(p); + Module.setValue(tokensPtr + offset * 4, p, 'i8*'); + offset += 1; + } + } + Module.HEAP32.set(lens, tokensLensPtr / 4); + } + Module.setValue(ptr + 12 * 4, tokensPtr, 'i8*'); + Module.setValue(ptr + 13 * 4, tokensLensPtr, 'i8*'); + Module.setValue(ptr + 14 * 4, config.tokens ? config.tokens.length : 0, + 'i32'); + return { ptr, referenceAudioPtr, referenceTextPtr, extraPtr, + phonemeCodepointsPtr, + phonemeCodepointsLensPtr, + tokensPtr, + tokensLensPtr, + tokenStringPtrs, }; } @@ -823,6 +930,11 @@ function freeSherpaOnnxGenerationConfig(cfg, Module) { if (cfg.referenceAudioPtr) Module._free(cfg.referenceAudioPtr); if (cfg.referenceTextPtr) Module._free(cfg.referenceTextPtr); if (cfg.extraPtr) Module._free(cfg.extraPtr); + if (cfg.phonemeCodepointsPtr) Module._free(cfg.phonemeCodepointsPtr); + if (cfg.phonemeCodepointsLensPtr) Module._free(cfg.phonemeCodepointsLensPtr); + if (cfg.tokensPtr) Module._free(cfg.tokensPtr); + if (cfg.tokensLensPtr) Module._free(cfg.tokensLensPtr); + for (const p of (cfg.tokenStringPtrs || [])) Module._free(p); if (cfg.ptr) Module._free(cfg.ptr); } @@ -837,6 +949,11 @@ class OfflineTts { this.handle = handle; this.sampleRate = Module._SherpaOnnxOfflineTtsSampleRate(this.handle); this.numSpeakers = Module._SherpaOnnxOfflineTtsNumSpeakers(this.handle); + { + const p = Module._SherpaOnnxOfflineTtsLang(this.handle); + this.lang = p ? Module.UTF8ToString(p) : ''; + Module._SherpaOnnxDestroyOfflineTtsLang(p); + } this.Module = Module } @@ -930,7 +1047,10 @@ class OfflineTts { } if (!audioPtr) { - throw new Error('Failed to generate audio'); + // Generation failed. The C API already logged the reason (for instance + // phonemeCodepoints and tokens are mutually exclusive). Return empty + // audio instead of throwing, matching C++/C/Python/Rust/Go/Java. + return {samples: new Float32Array(0), sampleRate: 0}; } const base = audioPtr / 4; @@ -966,16 +1086,45 @@ class OfflineTts { let modelType = 0; +// Choose exactly one of the two text frontends below (not both): +// +// usePhonemize = true // option 1 (default): phonemize the text with piper-phonemize; no extra files needed +// lexicon = './lexicon-en-us.txt' // option 2: lexicon file next to the model files. +// Download it from https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-en-us.txt +// +// Neither option uses the espeak-ng-data directory shipped in the model +// package; you can leave it out to keep the download small. +// +// Option 1 loads the piper-phonemize wasm bundle from ./piper-phonemize/ +// (see assets/README.md for how to get it). +// +// The demo UI can flip this at runtime (the TTS instance is recreated with +// the chosen frontend). +let usePhonemize = true; + +function setUsePhonemize(value) { + usePhonemize = value; +} + +// The frontend the selected model actually uses. Models that bring their own +// lexicon (Chinese/multi-lingual/zipvoice) always use it; the German model is +// phonemize-only because there is no German lexicon. +let modelUsesPhonemize = false; + function getDefaultOfflineTtsModelType() { return modelType; } +function getDefaultUsePhonemize() { + return modelUsesPhonemize; +} + function createOfflineTts(Module, myConfig) { const vits = { model: '', lexicon: '', tokens: '', - dataDir: '', + dataDir: '', // deprecated, ignored; use lexicon or phonemeCodepoints noiseScale: 0.667, noiseScaleW: 0.8, lengthScale: 1.0, @@ -986,7 +1135,7 @@ function createOfflineTts(Module, myConfig) { vocoder: '', lexicon: '', tokens: '', - dataDir: '', + dataDir: '', // deprecated, ignored; use lexicon or phonemeCodepoints noiseScale: 0.667, lengthScale: 1.0, }; @@ -995,7 +1144,7 @@ function createOfflineTts(Module, myConfig) { model: '', voices: '', tokens: '', - dataDir: '', + dataDir: '', // deprecated, ignored; use lexicon or phonemeCodepoints lengthScale: 1.0, lexicon: '', lang: '', @@ -1005,8 +1154,9 @@ function createOfflineTts(Module, myConfig) { model: '', voices: '', tokens: '', - dataDir: '', + dataDir: '', // deprecated, ignored; use lexicon or phonemeCodepoints lengthScale: 1.0, + lexicon: '', }; const offlineTtsZipVoiceModelConfig = { @@ -1014,7 +1164,7 @@ function createOfflineTts(Module, myConfig) { encoder: '', decoder: '', vocoder: '', - dataDir: '', + dataDir: '', // deprecated, ignored; use lexicon or phonemeCodepoints lexicon: '', featScale: 0.1, tShift: 0.5, @@ -1033,6 +1183,16 @@ function createOfflineTts(Module, myConfig) { voiceEmbeddingCacheCapacity: 50, }; + const offlineTtsSupertonicModelConfig = { + durationPredictor: '', + textEncoder: '', + vectorEstimator: '', + vocoder: '', + ttsJson: '', + unicodeIndexer: '', + voiceStyle: '', + }; + let ruleFsts = ''; switch (modelType) { @@ -1040,7 +1200,12 @@ function createOfflineTts(Module, myConfig) { // vits vits.model = './model.onnx'; vits.tokens = './tokens.txt'; - vits.dataDir = './espeak-ng-data'; + // Record the frontend actually configured: the ready message reports + // this back and the UI keeps the picker in sync with it. + modelUsesPhonemize = usePhonemize; + if (!usePhonemize) { + vits.lexicon = './lexicon-en-us.txt'; + } break; case 1: // matcha zh-en @@ -1049,7 +1214,6 @@ function createOfflineTts(Module, myConfig) { matcha.vocoder = './vocos-16khz-univ.onnx'; matcha.lexicon = './lexicon.txt'; matcha.tokens = './tokens.txt'; - matcha.dataDir = './espeak-ng-data'; ruleFsts = './phone-zh.fst,./date-zh.fst,./number-zh.fst'; break; case 2: @@ -1067,7 +1231,10 @@ function createOfflineTts(Module, myConfig) { matcha.acousticModel = './model-steps-3.onnx'; matcha.vocoder = './vocos-22khz-univ.onnx'; matcha.tokens = './tokens.txt'; - matcha.dataDir = './espeak-ng-data'; + modelUsesPhonemize = usePhonemize; + if (!usePhonemize) { + matcha.lexicon = './lexicon-en-us.txt'; + } break; case 4: // zipvoice zh-en @@ -1076,7 +1243,6 @@ function createOfflineTts(Module, myConfig) { offlineTtsZipVoiceModelConfig.encoder = './encoder.int8.onnx'; offlineTtsZipVoiceModelConfig.decoder = './decoder.int8.onnx'; offlineTtsZipVoiceModelConfig.vocoder = './vocos_24khz.onnx'; - offlineTtsZipVoiceModelConfig.dataDir = './espeak-ng-data'; offlineTtsZipVoiceModelConfig.lexicon = './lexicon.txt'; break; case 5: @@ -1095,15 +1261,35 @@ function createOfflineTts(Module, myConfig) { offlineTtsKittenModelConfig.model = './model.onnx'; offlineTtsKittenModelConfig.voices = './voices.bin'; offlineTtsKittenModelConfig.tokens = './tokens.txt'; - offlineTtsKittenModelConfig.dataDir = './espeak-ng-data'; + modelUsesPhonemize = usePhonemize; + if (!usePhonemize) { + offlineTtsKittenModelConfig.lexicon = './lexicon-en-us.txt'; + } break; case 7: + // SupertonicTTS (multilingual) + // https://k2-fsa.github.io/sherpa/onnx/tts/supertonic.html + offlineTtsSupertonicModelConfig.durationPredictor = + './duration_predictor.int8.onnx'; + offlineTtsSupertonicModelConfig.textEncoder = './text_encoder.int8.onnx'; + offlineTtsSupertonicModelConfig.vectorEstimator = + './vector_estimator.int8.onnx'; + offlineTtsSupertonicModelConfig.vocoder = './vocoder.int8.onnx'; + offlineTtsSupertonicModelConfig.ttsJson = './tts.json'; + offlineTtsSupertonicModelConfig.unicodeIndexer = + './unicode_indexer.bin'; + offlineTtsSupertonicModelConfig.voiceStyle = './voice.bin'; + // Own text frontend (unicode indexer); not the phonemize/lexicon picker. + modelUsesPhonemize = false; + break; + case 8: // kokoro-multi-lang-v1_0 // https://k2-fsa.github.io/sherpa/onnx/tts/all/Chinese-English/kokoro-multi-lang-v1_0.html + // Note: case 8, not 7. 7 is Supertonic's id and must stay unique. offlineTtsKokoroModelConfig.model = './model.onnx'; offlineTtsKokoroModelConfig.voices = './voices.bin'; offlineTtsKokoroModelConfig.tokens = './tokens.txt'; - offlineTtsKokoroModelConfig.dataDir = './espeak-ng-data'; + // data_dir is ignored since v2.0.0; the lexicon below is the frontend. offlineTtsKokoroModelConfig.lexicon = './lexicon-us-en.txt,./lexicon-zh.txt'; ruleFsts = './phone-zh.fst,./date-zh.fst,./number-zh.fst'; break; @@ -1116,6 +1302,7 @@ function createOfflineTts(Module, myConfig) { offlineTtsKittenModelConfig: offlineTtsKittenModelConfig, offlineTtsZipVoiceModelConfig: offlineTtsZipVoiceModelConfig, offlineTtsPocketModelConfig: offlineTtsPocketModelConfig, + offlineTtsSupertonicModelConfig: offlineTtsSupertonicModelConfig, numThreads: 1, debug: 1, provider: 'cpu', diff --git a/wasm/tts/sherpa-onnx-tts.worker.js b/wasm/tts/sherpa-onnx-tts.worker.js index 1cd8f5954..a266b73a4 100644 --- a/wasm/tts/sherpa-onnx-tts.worker.js +++ b/wasm/tts/sherpa-onnx-tts.worker.js @@ -16,7 +16,9 @@ self.Module = { self.postMessage({ type: "sherpa-onnx-tts-ready", modelType: getDefaultOfflineTtsModelType(), + usePhonemize: getDefaultUsePhonemize(), numSpeakers: tts.numSpeakers, + lang: tts.lang, }); } catch (e) { self.postMessage({ @@ -68,6 +70,29 @@ self.onmessage = async (e) => { message: "Generation failed: " + getErrorMessage(err), }); } + } else if (type == "setFrontend") { + if (!self.Module) { + return; + } + setUsePhonemize(!!e.data.usePhonemize); + if (tts && tts.free) { + tts.free(); + } + try { + tts = createOfflineTts(self.Module); + self.postMessage({ + type: "sherpa-onnx-tts-ready", + modelType: getDefaultOfflineTtsModelType(), + usePhonemize: getDefaultUsePhonemize(), + numSpeakers: tts.numSpeakers, + lang: tts.lang, + }); + } catch (err) { + self.postMessage({ + type: "error", + message: "TTS Initialization failed: " + getErrorMessage(err), + }); + } } else if (type == "generateWithConfig") { if (!tts) { return; diff --git a/wasm/tts/sherpa-onnx-wasm-main-tts.cc b/wasm/tts/sherpa-onnx-wasm-main-tts.cc index 7460e2c79..912d14b4c 100644 --- a/wasm/tts/sherpa-onnx-wasm-main-tts.cc +++ b/wasm/tts/sherpa-onnx-wasm-main-tts.cc @@ -16,7 +16,7 @@ extern "C" { static_assert(sizeof(SherpaOnnxOfflineTtsVitsModelConfig) == 8 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsMatchaModelConfig) == 8 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsKokoroModelConfig) == 8 * 4, ""); -static_assert(sizeof(SherpaOnnxOfflineTtsKittenModelConfig) == 5 * 4, ""); +static_assert(sizeof(SherpaOnnxOfflineTtsKittenModelConfig) == 6 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsZipvoiceModelConfig) == 10 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsPocketModelConfig) == 8 * 4, ""); static_assert(sizeof(SherpaOnnxOfflineTtsSupertonicModelConfig) == 7 * 4, ""); @@ -34,7 +34,7 @@ static_assert(sizeof(SherpaOnnxOfflineTtsConfig) == sizeof(SherpaOnnxOfflineTtsModelConfig) + 4 * 4, ""); -static_assert(sizeof(SherpaOnnxGenerationConfig) == 9 * 4, ""); +static_assert(sizeof(SherpaOnnxGenerationConfig) == 15 * 4, ""); void MyPrint(SherpaOnnxOfflineTtsConfig *tts_config) { auto tts_model_config = &tts_config->model; diff --git a/wasm/wasm-common.cmake b/wasm/wasm-common.cmake index 85d5a2056..b3c0951f0 100644 --- a/wasm/wasm-common.cmake +++ b/wasm/wasm-common.cmake @@ -7,9 +7,11 @@ set(exported_functions SherpaOnnxCreateOfflineTts SherpaOnnxDestroyOfflineTts SherpaOnnxDestroyOfflineTtsGeneratedAudio + SherpaOnnxDestroyOfflineTtsLang SherpaOnnxOfflineTtsGenerate SherpaOnnxOfflineTtsGenerateWithCallback SherpaOnnxOfflineTtsGenerateWithConfig + SherpaOnnxOfflineTtsLang SherpaOnnxOfflineTtsNumSpeakers SherpaOnnxOfflineTtsSampleRate SherpaOnnxWriteWave