mirror of
https://github.com/k2-fsa/sherpa-onnx.git
synced 2026-11-10 21:11:03 +08:00
48 lines
1.1 KiB
Python
Executable File
48 lines
1.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (c) 2026 Xiaomi Corporation
|
|
|
|
"""
|
|
Please download test files
|
|
- vocab.json
|
|
- token_scores.json
|
|
from
|
|
https://huggingface.co/csukuangfj/sherpa-onnx-test-data/tree/main
|
|
|
|
They are generated by ../scripts/pocket-tts/convert_tokenizer.py
|
|
using the BPE model from
|
|
https://huggingface.co/KevinAHM/pocket-tts-onnx/blob/main/tokenizer.model
|
|
|
|
See also ../scripts/pocket-tts/test_tokenizer.py
|
|
"""
|
|
|
|
from pathlib import Path
|
|
|
|
import sherpa_onnx
|
|
|
|
|
|
def main():
|
|
vocab_json = "./vocab.json"
|
|
token_scores_json = "./token_scores.json"
|
|
|
|
if not Path(vocab_json).is_file() or not Path(token_scores_json).is_file():
|
|
print("Please download test files first")
|
|
return
|
|
|
|
sp = sherpa_onnx.SentencePieceTokenizer(
|
|
vocab_json=vocab_json,
|
|
token_scores_json=token_scores_json,
|
|
)
|
|
|
|
text = "Yesterday, I bought 3 apples, 2 bananas, and a dozen oranges. Wow! That's amazing—did you see it too? I can't believe it's already 10:30 p.m."
|
|
|
|
ids = sp.encode(text, out_type=int)
|
|
tokens = sp.encode(text, out_type=str)
|
|
print(text)
|
|
print(tokens)
|
|
print(ids)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|