diff --git a/.github/scripts/test-nodejs-addon-npm.sh b/.github/scripts/test-nodejs-addon-npm.sh index c588a485d..7695c6581 100755 --- a/.github/scripts/test-nodejs-addon-npm.sh +++ b/.github/scripts/test-nodejs-addon-npm.sh @@ -16,21 +16,13 @@ node ./test_version.js echo "----------Source separation----------" -if [ -f ./sherpa-onnx.js ]; then - source_separation_module="$PWD/sherpa-onnx.js" -else - source_separation_module=$(node -p "require.resolve('sherpa-onnx-node')") -fi - download_and_extract https://github.com/k2-fsa/sherpa-onnx/releases/download/source-separation-models/sherpa-onnx-spleeter-2stems-fp16.tar.bz2 download https://github.com/k2-fsa/sherpa-onnx/releases/download/source-separation-models/qi-feng-le-zh.wav node ./test_offline_source_separation_spleeter.js -node --expose-gc ../scripts/node-addon-api/test/test_source_separation.js spleeter "$source_separation_module" download https://github.com/k2-fsa/sherpa-onnx/releases/download/source-separation-models/UVR-MDX-NET-Voc_FT.onnx node ./test_offline_source_separation_uvr.js -node --expose-gc ../scripts/node-addon-api/test/test_source_separation.js uvr "$source_separation_module" rm -rf sherpa-onnx-spleeter-2stems-fp16 rm -f UVR-MDX-NET-Voc_FT.onnx qi-feng-le-zh.wav diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets index 48be3afed..96a6d803e 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/Index.ets @@ -1,4 +1,4 @@ -export { listRawfileDir, readWave, readWaveFromBinary, } from "libsherpa_onnx.so"; +export { listRawfileDir, readWave, readWaveFromBinary, readWaveMultiChannel, writeWaveMultiChannel, } from "libsherpa_onnx.so"; export { CircularBuffer, SileroVadConfig, TenVadConfig, SpeechSegment, Vad, VadConfig, } from './src/main/ets/components/Vad'; @@ -88,3 +88,12 @@ export { KeywordSpotterConfig, KeywordSpotterResult, KeywordSpotter, } from './src/main/ets/components/KeywordSpotting'; + +export { OfflineSourceSeparationSpleeterModelConfig, + OfflineSourceSeparationUvrModelConfig, + OfflineSourceSeparationModelConfig, + OfflineSourceSeparationConfig, + SourceSeparationInput, + SourceSeparationOutput, + OfflineSourceSeparation, +} from './src/main/ets/components/OfflineSourceSeparation'; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts index 1710c2f35..5192c4f6c 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/types/libsherpa_onnx/Index.d.ts @@ -2,6 +2,8 @@ export const listRawfileDir: (mgr: object, dir: string) => Array; export const readWave: (filename: string, enableExternalBuffer: boolean = true) => {samples: Float32Array, sampleRate: number}; export const readWaveFromBinary: (data: Uint8Array, enableExternalBuffer: boolean = true) => {samples: Float32Array, sampleRate: number}; +export const readWaveMultiChannel: (filename: string, enableExternalBuffer: boolean = true) => {samples: Float32Array[], sampleRate: number}; +export const writeWaveMultiChannel: (filename: string, obj: {samples: Float32Array[], sampleRate: number}) => boolean; export const createCircularBuffer: (capacity: number) => object; export const circularBufferPush: (handle: object, samples: Float32Array) => void; export const circularBufferGet: (handle: object, index: number, n: number, enableExternalBuffer: boolean = true) => Float32Array; diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc index b08a3465b..9598f6ccb 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-reader.cc @@ -164,10 +164,109 @@ static Napi::Object ReadWaveFromBinaryWrapper(const Napi::CallbackInfo &info) { } } +static Napi::Object ReadWaveMultiChannelWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + if (info.Length() > 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsString()) { + Napi::TypeError::New(env, "Argument 0 should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + std::string filename = info[0].As().Utf8Value(); + + bool enable_external_buffer = true; + if (info.Length() == 2) { + if (info[1].IsBoolean()) { + enable_external_buffer = info[1].As().Value(); + } else { + Napi::TypeError::New(env, "Argument 1 should be a boolean") + .ThrowAsJavaScriptException(); + + return {}; + } + } + + const SherpaOnnxMultiChannelWave *wave = + SherpaOnnxReadWaveMultiChannel(filename.c_str()); + if (!wave) { + std::ostringstream os; + os << "Failed to read '" << filename << "'"; + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + int32_t num_channels = wave->num_channels; + int32_t num_samples = wave->num_samples; + int32_t sample_rate = wave->sample_rate; + size_t total = static_cast(num_channels) * num_samples; + + Napi::Array channels = Napi::Array::New(env, num_channels); + + if (num_samples <= 0) { + // An empty data chunk leaves no samples to share or copy. + SherpaOnnxFreeMultiChannelWave(wave); + for (int32_t c = 0; c != num_channels; ++c) { + channels.Set(c, Napi::Float32Array::New(env, 0)); + } + } else if (enable_external_buffer) { + // Samples of all channels live in one contiguous memory block; create one + // ArrayBuffer for the block and a view for each channel. + Napi::ArrayBuffer arrayBuffer = Napi::ArrayBuffer::New( + env, const_cast(wave->samples[0]), sizeof(float) * total, + [](Napi::Env /*env*/, void * /*data*/, + const SherpaOnnxMultiChannelWave *hint) { + SherpaOnnxFreeMultiChannelWave(hint); + }, + wave); + for (int32_t c = 0; c != num_channels; ++c) { + channels.Set( + c, Napi::Float32Array::New(env, num_samples, arrayBuffer, + sizeof(float) * num_samples * c)); + } + } else { + // don't use external buffer + Napi::ArrayBuffer arrayBuffer = + Napi::ArrayBuffer::New(env, sizeof(float) * total); + Napi::Float32Array float32Array = + Napi::Float32Array::New(env, total, arrayBuffer, 0); + + std::copy(wave->samples[0], wave->samples[0] + total, + float32Array.Data()); + + for (int32_t c = 0; c != num_channels; ++c) { + channels.Set( + c, Napi::Float32Array::New(env, num_samples, arrayBuffer, + sizeof(float) * num_samples * c)); + } + + SherpaOnnxFreeMultiChannelWave(wave); + } + + Napi::Object obj = Napi::Object::New(env); + obj.Set(Napi::String::New(env, "samples"), channels); + obj.Set(Napi::String::New(env, "sampleRate"), sample_rate); + return obj; +} + void InitWaveReader(Napi::Env env, Napi::Object exports) { exports.Set(Napi::String::New(env, "readWave"), Napi::Function::New(env, ReadWaveWrapper)); exports.Set(Napi::String::New(env, "readWaveFromBinary"), Napi::Function::New(env, ReadWaveFromBinaryWrapper)); + + exports.Set(Napi::String::New(env, "readWaveMultiChannel"), + Napi::Function::New(env, ReadWaveMultiChannelWrapper)); } diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc index 8f6d7bcab..b08c970c3 100644 --- a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/wave-writer.cc @@ -2,7 +2,10 @@ // // Copyright (c) 2024 Xiaomi Corporation +#include +#include #include +#include #include "napi.h" // NOLINT #include "sherpa-onnx/c-api/c-api.h" @@ -80,7 +83,114 @@ static Napi::Boolean WriteWaveWrapper(const Napi::CallbackInfo &info) { return Napi::Boolean::New(env, ok); } +// (filename, {samples: [samples0, samples1, ...], sampleRate: sampleRate} +static Napi::Boolean WriteWaveMultiChannelWrapper( + const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 2) { + std::ostringstream os; + os << "Expect only 2 arguments. Given: " << info.Length(); + + Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[0].IsString()) { + Napi::TypeError::New(env, "Argument 0 should be a string") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!info[1].IsObject()) { + Napi::TypeError::New(env, "Argument 1 should be an object") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Object obj = info[1].As(); + + if (!obj.Has("samples") || !obj.Get("samples").IsArray()) { + Napi::TypeError::New(env, "The object['samples'] should be an array") + .ThrowAsJavaScriptException(); + + return {}; + } + + if (!obj.Has("sampleRate") || !obj.Get("sampleRate").IsNumber()) { + Napi::TypeError::New(env, "The object['sampleRate'] should be a number") + .ThrowAsJavaScriptException(); + + return {}; + } + + Napi::Array channels = obj.Get("samples").As(); + uint32_t num_channels = channels.Length(); + if (num_channels == 0) { + Napi::RangeError::New(env, "The object['samples'] should not be empty") + .ThrowAsJavaScriptException(); + + return {}; + } + + double sample_rate = obj.Get("sampleRate").As().DoubleValue(); + if (!std::isfinite(sample_rate) || sample_rate <= 0 || + sample_rate > std::numeric_limits::max() || + sample_rate != std::floor(sample_rate)) { + Napi::RangeError::New( + env, "The object['sampleRate'] should be a positive integer") + .ThrowAsJavaScriptException(); + + return {}; + } + + // Read pointers after all property getters have run: a later channel getter + // may detach an earlier channel's ArrayBuffer. + std::vector channel_values; + for (uint32_t c = 0; c != num_channels; ++c) { + Napi::Value value = channels.Get(c); + if (!value.IsTypedArray() || + value.As().TypedArrayType() != napi_float32_array) { + Napi::TypeError::New(env, "Each channel should be a Float32Array") + .ThrowAsJavaScriptException(); + + return {}; + } + channel_values.push_back(value); + } + + std::vector channel_ptrs; + int32_t n = 0; + for (const auto &value : channel_values) { + Napi::Float32Array samples = value.As(); + int32_t length = samples.ElementLength(); +#if __OHOS__ + length /= sizeof(float); +#endif + if (length <= 0 || (n != 0 && length != n)) { + Napi::RangeError::New(env, "Channels should have equal, nonzero lengths") + .ThrowAsJavaScriptException(); + + return {}; + } + n = length; + channel_ptrs.push_back(samples.Data()); + } + + int32_t ok = SherpaOnnxWriteWaveMultiChannel( + channel_ptrs.data(), n, static_cast(sample_rate), num_channels, + info[0].As().Utf8Value().c_str()); + + return Napi::Boolean::New(env, ok); +} + void InitWaveWriter(Napi::Env env, Napi::Object exports) { exports.Set(Napi::String::New(env, "writeWave"), Napi::Function::New(env, WriteWaveWrapper)); + + exports.Set(Napi::String::New(env, "writeWaveMultiChannel"), + Napi::Function::New(env, WriteWaveMultiChannelWrapper)); } diff --git a/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/OfflineSourceSeparation.ets b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/OfflineSourceSeparation.ets new file mode 100644 index 000000000..41fd57a85 --- /dev/null +++ b/harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/OfflineSourceSeparation.ets @@ -0,0 +1,73 @@ +import { + createOfflineSourceSeparation, + offlineSourceSeparationGetNumberOfStems, + offlineSourceSeparationGetOutputSampleRate, + offlineSourceSeparationProcess, +} from 'libsherpa_onnx.so'; + +export class OfflineSourceSeparationSpleeterModelConfig { + public vocals: string = ''; + public accompaniment: string = ''; +} + +export class OfflineSourceSeparationUvrModelConfig { + public model: string = ''; +} + +export class OfflineSourceSeparationModelConfig { + public spleeter: OfflineSourceSeparationSpleeterModelConfig = new OfflineSourceSeparationSpleeterModelConfig(); + public uvr: OfflineSourceSeparationUvrModelConfig = new OfflineSourceSeparationUvrModelConfig(); + public numThreads: number = 1; + public debug: boolean = false; + public provider: string = 'cpu'; +} + +export class OfflineSourceSeparationConfig { + public model: OfflineSourceSeparationModelConfig = new OfflineSourceSeparationModelConfig(); +} + +export class SourceSeparationInput { + /** + * One or two channels of equal, nonzero length. Each element of a channel + * should be in the range [-1, 1]. Mono input is duplicated to stereo + * internally by sherpa-onnx; you don't need to duplicate it yourself. + */ + public samples: Float32Array[] = []; + public sampleRate: number = 0; + public enableExternalBuffer: boolean = true; +} + +export class SourceSeparationOutput { + /** + * stems[s][c] is channel c of stem s. For the models above, stems[0] is + * vocals and stems[1] is accompaniment (non-vocals). + */ + public stems: Float32Array[][] = []; + public sampleRate: number = 0; +} + +export class OfflineSourceSeparation { + public config: OfflineSourceSeparationConfig; + // Output sample rate of the model. Input is resampled to it if needed. + public sampleRate: number; + public numStems: number; + private handle: object; + + constructor(config: OfflineSourceSeparationConfig, mgr?: object) { + this.handle = createOfflineSourceSeparation(config, mgr); + this.config = config; + + this.sampleRate = offlineSourceSeparationGetOutputSampleRate(this.handle); + this.numStems = offlineSourceSeparationGetNumberOfStems(this.handle); + } + + /** + * Run source separation synchronously. + * + * Returns an object {stems, sampleRate}, where stems is an array of stems + * and each stem is an array of Float32Array channels. + */ + process(input: SourceSeparationInput): SourceSeparationOutput { + return offlineSourceSeparationProcess(this.handle, input); + } +} diff --git a/nodejs-addon-examples/README.md b/nodejs-addon-examples/README.md index 29815f60f..5508e320c 100644 --- a/nodejs-addon-examples/README.md +++ b/nodejs-addon-examples/README.md @@ -52,9 +52,6 @@ The following tables list the examples in this folder. Download the models and sample audio before running these examples: -Until a published release includes `OfflineSourceSeparation`, these examples -require a source-built `sherpa-onnx-node` addon. - ```bash curl -LO https://github.com/k2-fsa/sherpa-onnx/releases/download/source-separation-models/sherpa-onnx-spleeter-2stems-fp16.tar.bz2 tar xjf sherpa-onnx-spleeter-2stems-fp16.tar.bz2 @@ -64,9 +61,15 @@ node test_offline_source_separation_spleeter.js node test_offline_source_separation_uvr.js ``` +The examples read the stereo test file with `readWaveMultiChannel` (`readWave` +keeps only the first channel) and write each stem to a single wave file with +`writeWaveMultiChannel`, e.g., `spleeter-vocals.wav` and +`spleeter-non-vocals.wav`. + `OfflineSourceSeparation.process({samples, sampleRate, enableExternalBuffer})` runs synchronously. `samples` is an array containing one or two `Float32Array` -channels of equal, nonzero length. Mono input is duplicated to stereo. The +channels of equal, nonzero length. Mono input is duplicated to stereo +internally by sherpa-onnx; you don't need to duplicate it yourself. The native engine resamples input to the model's output sample rate if needed. The result is `{stems, sampleRate}`. For the models above, `stems[0]` is vocals diff --git a/nodejs-addon-examples/test_offline_source_separation_spleeter.js b/nodejs-addon-examples/test_offline_source_separation_spleeter.js index 50ced9345..d67e2314c 100644 --- a/nodejs-addon-examples/test_offline_source_separation_spleeter.js +++ b/nodejs-addon-examples/test_offline_source_separation_spleeter.js @@ -15,19 +15,32 @@ const separator = new sherpa_onnx.OfflineSourceSeparation({ }, }); -const wave = sherpa_onnx.readWave('./qi-feng-le-zh.wav'); +// The model expects stereo; readWave() would keep only the first channel. +const wave = sherpa_onnx.readWaveMultiChannel('./qi-feng-le-zh.wav'); + +const start = Date.now(); const output = separator.process({ - samples: [wave.samples], + samples: wave.samples, sampleRate: wave.sampleRate, enableExternalBuffer: false, }); +const end = Date.now(); -// writeWave writes mono audio. Each stem contains separate stereo channels. +// Each stem contains one channel per input channel; writeWaveMultiChannel +// writes a stem to a single wave file. for (let i = 0; i < output.stems.length; ++i) { - const name = ['vocals', 'accompaniment'][i]; - output.stems[i].forEach((samples, channel) => { - const filename = `spleeter-${name}-${channel}.wav`; - sherpa_onnx.writeWave(filename, {samples, sampleRate: output.sampleRate}); - console.log(`Saved ${filename}`); - }); + const filename = ['spleeter-vocals.wav', 'spleeter-non-vocals.wav'][i]; + sherpa_onnx.writeWaveMultiChannel( + filename, {samples: output.stems[i], sampleRate: output.sampleRate}); + console.log(`Saved ${filename}`); } + +const elapsed_seconds = (end - start) / 1000; +const audio_duration = wave.samples[0].length / wave.sampleRate; +const real_time_factor = elapsed_seconds / audio_duration; + +console.log(`Elapsed seconds: ${elapsed_seconds.toFixed(3)}`); +console.log(`Audio duration in seconds: ${audio_duration.toFixed(3)}`); +console.log( + `RTF: ${elapsed_seconds.toFixed(3)}/${audio_duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); diff --git a/nodejs-addon-examples/test_offline_source_separation_uvr.js b/nodejs-addon-examples/test_offline_source_separation_uvr.js index 4d0e8ddde..69f73e946 100644 --- a/nodejs-addon-examples/test_offline_source_separation_uvr.js +++ b/nodejs-addon-examples/test_offline_source_separation_uvr.js @@ -11,19 +11,32 @@ const separator = new sherpa_onnx.OfflineSourceSeparation({ }, }); -const wave = sherpa_onnx.readWave('./qi-feng-le-zh.wav'); +// The model expects stereo; readWave() would keep only the first channel. +const wave = sherpa_onnx.readWaveMultiChannel('./qi-feng-le-zh.wav'); + +const start = Date.now(); const output = separator.process({ - samples: [wave.samples], + samples: wave.samples, sampleRate: wave.sampleRate, enableExternalBuffer: false, }); +const end = Date.now(); -// writeWave writes mono audio. Each stem contains separate stereo channels. +// Each stem contains one channel per input channel; writeWaveMultiChannel +// writes a stem to a single wave file. for (let i = 0; i < output.stems.length; ++i) { - const name = ['vocals', 'accompaniment'][i]; - output.stems[i].forEach((samples, channel) => { - const filename = `uvr-${name}-${channel}.wav`; - sherpa_onnx.writeWave(filename, {samples, sampleRate: output.sampleRate}); - console.log(`Saved ${filename}`); - }); + const filename = ['uvr-vocals.wav', 'uvr-non-vocals.wav'][i]; + sherpa_onnx.writeWaveMultiChannel( + filename, {samples: output.stems[i], sampleRate: output.sampleRate}); + console.log(`Saved ${filename}`); } + +const elapsed_seconds = (end - start) / 1000; +const audio_duration = wave.samples[0].length / wave.sampleRate; +const real_time_factor = elapsed_seconds / audio_duration; + +console.log(`Elapsed seconds: ${elapsed_seconds.toFixed(3)}`); +console.log(`Audio duration in seconds: ${audio_duration.toFixed(3)}`); +console.log( + `RTF: ${elapsed_seconds.toFixed(3)}/${audio_duration.toFixed(3)} =`, + real_time_factor.toFixed(3)); diff --git a/scripts/node-addon-api/lib/addon.js b/scripts/node-addon-api/lib/addon.js index 6e65955b0..f4760c5ca 100644 --- a/scripts/node-addon-api/lib/addon.js +++ b/scripts/node-addon-api/lib/addon.js @@ -1,4 +1,5 @@ /** @typedef {import('./types').WaveObject} WaveObject */ +/** @typedef {import('./types').MultiChannelWaveObject} MultiChannelWaveObject */ const os = require('os'); const path = require('path'); @@ -96,3 +97,19 @@ if (!addon) { * @param {WaveObject} obj - { samples: Float32Array, sampleRate: number } * @returns {boolean} */ + +/** + * Read a multi-channel wave file from disk. + * @function module.exports.readWaveMultiChannel + * @param {string} filename + * @param {boolean} [enableExternalBuffer=true] + * @returns {MultiChannelWaveObject} + */ + +/** + * Write a multi-channel wave file to disk. + * @function module.exports.writeWaveMultiChannel + * @param {string} filename + * @param {MultiChannelWaveObject} obj - { samples: Float32Array[], sampleRate: number } + * @returns {boolean} + */ diff --git a/scripts/node-addon-api/lib/offline-source-separation.js b/scripts/node-addon-api/lib/offline-source-separation.js index adec56641..8cd7c9c0c 100644 --- a/scripts/node-addon-api/lib/offline-source-separation.js +++ b/scripts/node-addon-api/lib/offline-source-separation.js @@ -20,7 +20,8 @@ class OfflineSourceSeparation { } /** - * Separate audio synchronously. Mono input is duplicated to stereo. + * Separate audio synchronously. Mono input is duplicated to stereo + * internally by sherpa-onnx; you don't need to duplicate it yourself. * @param {SourceSeparationRequest} obj * @returns {SourceSeparationOutput} */ diff --git a/scripts/node-addon-api/lib/sherpa-onnx.js b/scripts/node-addon-api/lib/sherpa-onnx.js index 81b2076f0..ac681df20 100644 --- a/scripts/node-addon-api/lib/sherpa-onnx.js +++ b/scripts/node-addon-api/lib/sherpa-onnx.js @@ -1,4 +1,5 @@ /** @typedef {import('./types').WaveObject} WaveObject */ +/** @typedef {import('./types').MultiChannelWaveObject} MultiChannelWaveObject */ /** * @typedef {import('./types').OnlineRecognizerResult} OnlineRecognizerResult */ @@ -29,6 +30,8 @@ module.exports = { GenerationConfig : non_streaming_tts.GenerationConfig, readWave : addon.readWave, writeWave : addon.writeWave, + readWaveMultiChannel : addon.readWaveMultiChannel, + writeWaveMultiChannel : addon.writeWaveMultiChannel, Display : streaming_asr.Display, Vad : vad.Vad, CircularBuffer : vad.CircularBuffer, diff --git a/scripts/node-addon-api/lib/types.js b/scripts/node-addon-api/lib/types.js index 0e731ef17..376a3cc76 100644 --- a/scripts/node-addon-api/lib/types.js +++ b/scripts/node-addon-api/lib/types.js @@ -424,6 +424,16 @@ * @see src/wave-reader.cc */ +/** + * Multi-channel wave object returned by readWaveMultiChannel and used by + * writeWaveMultiChannel. + * @typedef {Object} MultiChannelWaveObject + * @property {Float32Array[]} samples - samples[c] holds channel c 1-D float32 + * samples in [-1, 1]. All channels have the same number of samples. + * @property {number} sampleRate - Sample rate as an integer (e.g., 44100). + * @see src/wave-reader.cc + */ + /** * Speech segment returned by Vad.front(). * @typedef {Object} SpeechSegment @@ -862,7 +872,8 @@ /** * @typedef {Object} SourceSeparationRequest * @property {Float32Array[]} samples - One or two equal-length channels. - * Mono input is duplicated to stereo. Values should be in [-1, 1]. + * Mono input is duplicated to stereo internally by sherpa-onnx; you don't + * need to duplicate it yourself. Values should be in [-1, 1]. * @property {number} sampleRate - Positive integer input sample rate in Hz, * no greater than 2147483647. * @property {boolean} [enableExternalBuffer=true] - Set false for Electron. diff --git a/scripts/node-addon-api/test/test_source_separation.js b/scripts/node-addon-api/test/test_source_separation.js index 5bbc1b3ce..8c70fce39 100644 --- a/scripts/node-addon-api/test/test_source_separation.js +++ b/scripts/node-addon-api/test/test_source_separation.js @@ -1,6 +1,7 @@ // Copyright (c) 2026 Nexmoe const assert = require('assert'); +const fs = require('fs'); const {createRequire} = require('module'); const {MessageChannel} = require('worker_threads'); const modulePath = process.argv[3] || require.resolve('../lib/sherpa-onnx.js'); @@ -54,6 +55,128 @@ function testValidation() { assert.throws(() => addon.offlineSourceSeparationProcess(), TypeError); } +/** Round-trip multi-channel wave IO without loading a model. */ +function testWaveIO() { + assert.strictEqual(typeof sherpa_onnx.readWaveMultiChannel, 'function'); + assert.strictEqual(typeof sherpa_onnx.writeWaveMultiChannel, 'function'); + + const left = new Float32Array(1600); + const right = new Float32Array(1600); + for (let i = 0; i < left.length; ++i) { + left[i] = 0.9 * Math.sin(i / 10); + right[i] = -0.5 * Math.cos(i / 7); + } + + const filename = 'test-wave-io-roundtrip.wav'; + try { + assert.strictEqual( + sherpa_onnx.writeWaveMultiChannel( + filename, {samples: [left, right], sampleRate: 16000}), + true); + for (const enableExternalBuffer of [true, false]) { + const wave = + sherpa_onnx.readWaveMultiChannel(filename, enableExternalBuffer); + assert.strictEqual(wave.sampleRate, 16000); + assert.deepStrictEqual( + wave.samples.map(channel => channel.length), [1600, 1600]); + // 16-bit PCM quantization + for (const [actual, expected] of [[wave.samples[0], left], + [wave.samples[1], right]]) { + for (let i = 0; i < expected.length; ++i) { + assert(Math.abs(actual[i] - expected[i]) <= 2 / 32768); + } + } + assert.notDeepStrictEqual(wave.samples[0], wave.samples[1]); + } + + assert.strictEqual( + sherpa_onnx.writeWaveMultiChannel( + filename, {samples: [left], sampleRate: 16000}), + true); + const mono = sherpa_onnx.readWaveMultiChannel(filename, false); + assert.strictEqual(mono.samples.length, 1); + assert.strictEqual(mono.samples[0].length, 1600); + + // Any channel count is fine, not only 1 or 2. + assert.strictEqual( + sherpa_onnx.writeWaveMultiChannel( + filename, {samples: Array(3).fill(left), sampleRate: 16000}), + true); + const three = sherpa_onnx.readWaveMultiChannel(filename, false); + assert.strictEqual(three.samples.length, 3); + + // A wave with an empty data chunk has no samples per channel. + assert.strictEqual( + sherpa_onnx.writeWave( + filename, {samples: new Float32Array(0), sampleRate: 16000}), + true); + for (const enableExternalBuffer of [true, false]) { + const empty = + sherpa_onnx.readWaveMultiChannel(filename, enableExternalBuffer); + assert.strictEqual(empty.samples.length, 1); + assert.strictEqual(empty.samples[0].length, 0); + } + } finally { + fs.unlinkSync(filename); + } + + for (const obj of [undefined, null, 1, [], {samples: [left]}, + {samples: [left], sampleRate: '16000'}]) { + assert.throws( + () => sherpa_onnx.writeWaveMultiChannel(filename, obj), TypeError); + } + for (const samples of [[], [new Float32Array(0)], + [new Float32Array(10), new Float32Array(11)]]) { + assert.throws( + () => sherpa_onnx.writeWaveMultiChannel( + filename, {samples, sampleRate: 16000}), + RangeError); + } + for (const samples of [[new Int16Array(10)], [left, 1], ['a']]) { + assert.throws( + () => sherpa_onnx.writeWaveMultiChannel( + filename, {samples, sampleRate: 16000}), + TypeError); + } + for (const sampleRate of [0, -1, NaN, Infinity, 44100.5, 2 ** 31]) { + assert.throws( + () => sherpa_onnx.writeWaveMultiChannel( + filename, {samples: [left], sampleRate}), + RangeError); + } + + // A channel getter may detach an earlier channel's ArrayBuffer; the saved + // pointers must not be read after any property getter has run. + { + const first = new Float32Array(10); + const channels = [first]; + const {port1, port2} = new MessageChannel(); + try { + Object.defineProperty(channels, 1, { + get() { + port1.postMessage(first.buffer, [first.buffer]); + return new Float32Array(10); + }, + }); + assert.throws( + () => sherpa_onnx.writeWaveMultiChannel( + filename, {samples: channels, sampleRate: 16000}), + RangeError); + } finally { + port1.close(); + port2.close(); + } + } + assert.throws(() => sherpa_onnx.readWaveMultiChannel(), TypeError); + assert.throws(() => sherpa_onnx.readWaveMultiChannel(1), TypeError); + assert.throws( + () => sherpa_onnx.readWaveMultiChannel('no-such-file.wav'), TypeError); + assert.throws( + () => sherpa_onnx.readWaveMultiChannel(filename, 1), TypeError); + assert.throws( + () => sherpa_onnx.readWaveMultiChannel(filename, true, true), TypeError); +} + /** Reject malformed audio before it reaches native inference. */ function testInvalidAudio(separator) { for (const request of [undefined, null, 1, [], {}]) { @@ -185,6 +308,23 @@ async function testModel(kind) { let stereo = separator.process({...request, samples: [left, left]}); assert.deepStrictEqual(mono, stereo); + // The test wav is real stereo; readWaveMultiChannel must keep both channels + // while readWave keeps only the first one. + const multi = sherpa_onnx.readWaveMultiChannel('./qi-feng-le-zh.wav', false); + assert.strictEqual(multi.sampleRate, wave.sampleRate); + assert.deepStrictEqual( + multi.samples.map(channel => channel.length), + [wave.samples.length, wave.samples.length]); + assert.deepStrictEqual(multi.samples[0], wave.samples); + assert.notDeepStrictEqual(multi.samples[0], multi.samples[1]); + checkOutput( + separator.process({ + samples: multi.samples.map(channel => channel.subarray(0, n)), + sampleRate: multi.sampleRate, + enableExternalBuffer: false, + }), + separator, n / multi.sampleRate); + const resampler = new sherpa_onnx.LinearResampler(wave.sampleRate, 16000); const resampled = resampler.flush(left); let resampledOutput = separator.process({ @@ -206,6 +346,7 @@ async function testModel(kind) { /** Run model-free checks, then optional real-model regression checks. */ async function main() { testValidation(); + testWaveIO(); if (process.argv[2]) await testModel(process.argv[2]); console.log('Source separation tests passed'); }