From 57b5415e66042cb8c2531b605554198fe283183a Mon Sep 17 00:00:00 2001 From: robin Date: Sun, 5 Oct 2025 12:56:40 +0000 Subject: [PATCH] worklet fix --- .../src/lib/components/voicebotDemo/helper.ts | 169 +++++++++++++----- 1 file changed, 126 insertions(+), 43 deletions(-) diff --git a/frontend/src/lib/components/voicebotDemo/helper.ts b/frontend/src/lib/components/voicebotDemo/helper.ts index 25ea51e..fa92616 100644 --- a/frontend/src/lib/components/voicebotDemo/helper.ts +++ b/frontend/src/lib/components/voicebotDemo/helper.ts @@ -9,26 +9,80 @@ export const WS_URL = isBrowser && window.location.protocol === "http:" ? `ws://${WS_HOST}/api/v1/voicebot/ws` : `wss://${WS_HOST}/api/v1/voicebot/ws` -const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProcessor { +const playbackWorkletCode = `/** + * @typedef {Object} AppendDeltaMessage + * @property {"appendDelta"} type + * @property {ResponseAudioDelta} delta + */ + +/** + * @typedef {Object} DeleteItemMessage + * @property {"deleteItem"} type + * @property {string} item_id + */ + +/** + * @typedef {Object} ClearMessage + * @property {"clear"} type + */ + +/** + * @typedef {Object} SetSourceRateMessage + * @property {"setSourceRate"} type + * @property {number} hz + */ + +/** + * @typedef {Object} NowPlayingMessage + * @property {"nowPlaying"} type + * @property {string|null} item_id + * @property {number} played_ms + */ + +/** + * @typedef {Object} MuteMessage + * @property {"mute"} type + */ + +/** + * @typedef {Object} UnmuteMessage + * @property {"unmute"} type + */ + +/** + * @typedef {AppendDeltaMessage | DeleteItemMessage | ClearMessage | SetSourceRateMessage | MuteMessage | UnmuteMessage} PlaybackMessage + */ + +/** + * @typedef {Object} Chunk + * @property {string} item_id + * @property {Int16Array} data + * @property {number} off + */ + +class AudioPlaybackWorklet extends AudioWorkletProcessor { constructor() { super() - this.srcRate = ${SAMPLE_RATE} - this.dstRate = sampleRate - this.step = this.srcRate / this.dstRate - this.queue = [] - this.cur = null - this.hold = 0 - this.phase = 0 - this._x0 = undefined - this._x1 = undefined - this._nextItemId = null - this.nowItemId = null - this.nowItemSamples = 0 - this._notifyFrames = 0 - this.muted = false - this.port.onmessage = (e) => this._onMessage(e.data) + /** @type {number} */ this.srcRate = 24000 + /** @type {number} */ this.dstRate = sampleRate + /** @type {number} */ this.step = this.srcRate / this.dstRate + /** @type {Chunk[]} */ this.queue = [] + /** @type {Chunk|null} */ this.cur = null + /** @type {number} */ this.hold = 0 + /** @type {number} */ this.phase = 0 + /** @type {number|undefined} */ this._x0 = undefined + /** @type {number|undefined} */ this._x1 = undefined + /** @type {string|null} */ this._x0ItemId = null + /** @type {string|null} */ this._x1ItemId = null + /** @type {string|null} */ this._nextItemId = null + /** @type {string|null} */ this.nowItemId = null + /** @type {number} */ this.nowItemSamples = 0 + /** @type {number} */ this._notifyFrames = 0 + /** @type {boolean} */ this.muted = false + this.port.onmessage = (e) => this._onMessage(/** @type {PlaybackMessage} */ (e.data)) } + /** @param {PlaybackMessage} msg */ _onMessage(msg) { if (!msg || !msg.type) return if (msg.type === "appendDelta" && msg.delta && msg.delta.pcmInt16 instanceof Int16Array) { @@ -45,7 +99,7 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc if (this.nowItemId === id) { this.nowItemId = null this.nowItemSamples = 0 - this._notifyFrames = 0 + this._postNowPlaying() } return } @@ -56,10 +110,13 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc this.phase = 0 this._x0 = undefined this._x1 = undefined + this._x0ItemId = null + this._x1ItemId = null this._nextItemId = null this.nowItemId = null this.nowItemSamples = 0 this._notifyFrames = 0 + this._postNowPlaying() return } if (msg.type === "setSourceRate" && Number.isFinite(msg.hz) && msg.hz > 0) { @@ -73,9 +130,11 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc } if (msg.type === "unmute") { this.muted = false + return } } + /** @returns {boolean} */ _ensureCurrent() { if (this.cur == null) { if (this.queue.length === 0) return false @@ -85,6 +144,7 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc return true } + /** @returns {number} */ _nextInt16() { for (;;) { if (!this._ensureCurrent()) { @@ -105,7 +165,23 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc } } - process(_inputs, outputs) { + _postNowPlaying() { + /** @type {NowPlayingMessage} */ + const m = { + type: "nowPlaying", + item_id: this.nowItemId, + played_ms: Math.max(0, Math.floor((this.nowItemSamples * 1000) / this.srcRate)), + } + this.port.postMessage(m) + } + + /** + * @param {Float32Array[][]} _inputs + * @param {Float32Array[][]} outputs + * @param {Record} _parameters + * @returns {boolean} + */ + process(_inputs, outputs, _parameters) { const out = outputs[0] if (!out || out.length === 0) return true const ch0 = out[0] @@ -113,45 +189,45 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc if (this._x1 === undefined) { this._x1 = this._nextInt16() + this._x1ItemId = this._nextItemId this._x0 = this._x1 + this._x0ItemId = this._x1ItemId this.phase = 0 - this.nowItemId = this._nextItemId + this.nowItemId = this._x0ItemId this.nowItemSamples = 0 + this._postNowPlaying() } - const advance = () => { + const common = () => { this.phase += this.step while (this.phase >= 1) { this.phase -= 1 this._x0 = this._x1 + this._x0ItemId = this._x1ItemId this._x1 = this._nextInt16() - if (this.nowItemId !== this._nextItemId) { - this.nowItemId = this._nextItemId + this._x1ItemId = this._nextItemId + + if (this.nowItemId !== this._x0ItemId) { + this.nowItemId = this._x0ItemId this.nowItemSamples = 0 + this._postNowPlaying() } - if (this.nowItemId) this.nowItemSamples += 1 } } - if (this.muted) { - for (let i = 0; i < N; i++) { - ch0[i] = 0 - for (let c = 1; c < out.length; c++) out[c][i] = 0 - advance() - } - } else { - for (let i = 0; i < N; i++) { - const x0 = this._x0 ?? 0 - const x1 = this._x1 ?? x0 - const value = x0 + (x1 - x0) * this.phase - const sample = value <= -32768 ? -1 : value >= 32767 ? 1 : value / 32768 - ch0[i] = sample - for (let c = 1; c < out.length; c++) out[c][i] = sample - advance() - this._notifyFrames += 1 - if (this._notifyFrames >= this.dstRate / 10) { - this._notifyFrames = 0 - } + for (let i = 0; i < N; i++) { + const x0 = this._x0 ?? 0 + const x1 = this._x1 ?? x0 + const value = this.muted ? 0 : x0 + (x1 - x0) * this.phase + const sample = value <= -32768 ? -1 : value >= 32767 ? 1 : value / 32768 + out[0][i] = sample + if (out[1]) out[1][i] = sample + this.nowItemSamples += 1 + this._notifyFrames += 1 + common() + if (this._notifyFrames >= this.dstRate / 10) { + this._notifyFrames = 0 + this._postNowPlaying() } } @@ -168,12 +244,13 @@ const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProces this._inRate = sampleRate this._pos = 0 this._carry = null - this._outRate = ${SAMPLE_RATE} + this._outRate = 24000 } process(inputs) { const chs = inputs[0] if (!chs || chs.length === 0) return true + const inF32 = chs[0] const step = this._inRate / this._outRate @@ -187,24 +264,30 @@ const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProces const avail = src.length - 1 - this._pos const outLen = avail > 0 ? Math.ceil(avail / step) : 0 + const outI16 = new Int16Array(outLen) let pos = this._pos for (let k = 0; k < outLen; k++) { const i = Math.floor(pos) const frac = pos - i + const x0 = src[i] const x1 = src[i + 1] + let y = x0 + frac * (x1 - x0) if (y > 1) y = 1 else if (y < -1) y = -1 + const s = y <= -1 ? -0x8000 : Math.round(y * 0x7fff) outI16[k] = s + pos += step } this._pos = pos - (src.length - 1) if (this._pos < 0) this._pos = 0 + this._carry = src[src.length - 1] try {