robin/kontextwerk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

worklet fix

Browse Source
This commit is contained in:
Robin Grenzdörfer
2025-10-05 12:56:40 +00:00
parent 2be72187ec
commit 57b5415e66
1 changed files with 126 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
frontend/src/lib/components/voicebotDemo/helper.ts
View File

@@ -9,26 +9,80 @@ export const WS_URL = isBrowser && window.location.protocol === "http:"
? `ws://${WS_HOST}/api/v1/voicebot/ws` ? `ws://${WS_HOST}/api/v1/voicebot/ws`
: `wss://${WS_HOST}/api/v1/voicebot/ws` : `wss://${WS_HOST}/api/v1/voicebot/ws`
const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProcessor { const playbackWorkletCode = `/**
* @typedef {Object} AppendDeltaMessage
* @property {"appendDelta"} type
* @property {ResponseAudioDelta} delta
*/
/**
* @typedef {Object} DeleteItemMessage
* @property {"deleteItem"} type
* @property {string} item_id
*/
/**
* @typedef {Object} ClearMessage
* @property {"clear"} type
*/
/**
* @typedef {Object} SetSourceRateMessage
* @property {"setSourceRate"} type
* @property {number} hz
*/
/**
* @typedef {Object} NowPlayingMessage
* @property {"nowPlaying"} type
* @property {string|null} item_id
* @property {number} played_ms
*/
/**
* @typedef {Object} MuteMessage
* @property {"mute"} type
*/
/**
* @typedef {Object} UnmuteMessage
* @property {"unmute"} type
*/
/**
* @typedef {AppendDeltaMessage | DeleteItemMessage | ClearMessage | SetSourceRateMessage | MuteMessage | UnmuteMessage} PlaybackMessage
*/
/**
* @typedef {Object} Chunk
* @property {string} item_id
* @property {Int16Array} data
* @property {number} off
*/
class AudioPlaybackWorklet extends AudioWorkletProcessor {
constructor() { constructor() {
super() super()
this.srcRate = ${SAMPLE_RATE} /** @type {number} */ this.srcRate = 24000
this.dstRate = sampleRate /** @type {number} */ this.dstRate = sampleRate
this.step = this.srcRate / this.dstRate /** @type {number} */ this.step = this.srcRate / this.dstRate
this.queue = [] /** @type {Chunk[]} */ this.queue = []
this.cur = null /** @type {Chunk|null} */ this.cur = null
this.hold = 0 /** @type {number} */ this.hold = 0
this.phase = 0 /** @type {number} */ this.phase = 0
this._x0 = undefined /** @type {number|undefined} */ this._x0 = undefined
this._x1 = undefined /** @type {number|undefined} */ this._x1 = undefined
this._nextItemId = null /** @type {string|null} */ this._x0ItemId = null
this.nowItemId = null /** @type {string|null} */ this._x1ItemId = null
this.nowItemSamples = 0 /** @type {string|null} */ this._nextItemId = null
this._notifyFrames = 0 /** @type {string|null} */ this.nowItemId = null
this.muted = false /** @type {number} */ this.nowItemSamples = 0
this.port.onmessage = (e) => this._onMessage(e.data) /** @type {number} */ this._notifyFrames = 0
/** @type {boolean} */ this.muted = false
this.port.onmessage = (e) => this._onMessage(/** @type {PlaybackMessage} */ (e.data))
} }
/** @param {PlaybackMessage} msg */
_onMessage(msg) { _onMessage(msg) {
if (!msg || !msg.type) return if (!msg || !msg.type) return
if (msg.type === "appendDelta" && msg.delta && msg.delta.pcmInt16 instanceof Int16Array) { if (msg.type === "appendDelta" && msg.delta && msg.delta.pcmInt16 instanceof Int16Array) {
@@ -45,7 +99,7 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
if (this.nowItemId === id) { if (this.nowItemId === id) {
this.nowItemId = null this.nowItemId = null
this.nowItemSamples = 0 this.nowItemSamples = 0
this._notifyFrames = 0 this._postNowPlaying()
} }
return return
} }
@@ -56,10 +110,13 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
this.phase = 0 this.phase = 0
this._x0 = undefined this._x0 = undefined
this._x1 = undefined this._x1 = undefined
this._x0ItemId = null
this._x1ItemId = null
this._nextItemId = null this._nextItemId = null
this.nowItemId = null this.nowItemId = null
this.nowItemSamples = 0 this.nowItemSamples = 0
this._notifyFrames = 0 this._notifyFrames = 0
this._postNowPlaying()
return return
} }
if (msg.type === "setSourceRate" && Number.isFinite(msg.hz) && msg.hz > 0) { if (msg.type === "setSourceRate" && Number.isFinite(msg.hz) && msg.hz > 0) {
@@ -73,9 +130,11 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
} }
if (msg.type === "unmute") { if (msg.type === "unmute") {
this.muted = false this.muted = false
return
} }
} }
/** @returns {boolean} */
_ensureCurrent() { _ensureCurrent() {
if (this.cur == null) { if (this.cur == null) {
if (this.queue.length === 0) return false if (this.queue.length === 0) return false
@@ -85,6 +144,7 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
return true return true
} }
/** @returns {number} */
_nextInt16() { _nextInt16() {
for (;;) { for (;;) {
if (!this._ensureCurrent()) { if (!this._ensureCurrent()) {
@@ -105,7 +165,23 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
} }
} }
process(_inputs, outputs) { _postNowPlaying() {
/** @type {NowPlayingMessage} */
const m = {
type: "nowPlaying",
item_id: this.nowItemId,
played_ms: Math.max(0, Math.floor((this.nowItemSamples * 1000) / this.srcRate)),
}
this.port.postMessage(m)
}
/**
* @param {Float32Array[][]} _inputs
* @param {Float32Array[][]} outputs
* @param {Record<string, Float32Array>} _parameters
* @returns {boolean}
*/
process(_inputs, outputs, _parameters) {
const out = outputs[0] const out = outputs[0]
if (!out || out.length === 0) return true if (!out || out.length === 0) return true
const ch0 = out[0] const ch0 = out[0]
@@ -113,45 +189,45 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
if (this._x1 === undefined) { if (this._x1 === undefined) {
this._x1 = this._nextInt16() this._x1 = this._nextInt16()
this._x1ItemId = this._nextItemId
this._x0 = this._x1 this._x0 = this._x1
this._x0ItemId = this._x1ItemId
this.phase = 0 this.phase = 0
this.nowItemId = this._nextItemId this.nowItemId = this._x0ItemId
this.nowItemSamples = 0 this.nowItemSamples = 0
this._postNowPlaying()
} }
const advance = () => { const common = () => {
this.phase += this.step this.phase += this.step
while (this.phase >= 1) { while (this.phase >= 1) {
this.phase -= 1 this.phase -= 1
this._x0 = this._x1 this._x0 = this._x1
this._x0ItemId = this._x1ItemId
this._x1 = this._nextInt16() this._x1 = this._nextInt16()
if (this.nowItemId !== this._nextItemId) { this._x1ItemId = this._nextItemId
this.nowItemId = this._nextItemId
if (this.nowItemId !== this._x0ItemId) {
this.nowItemId = this._x0ItemId
this.nowItemSamples = 0 this.nowItemSamples = 0
this._postNowPlaying()
} }
if (this.nowItemId) this.nowItemSamples += 1
} }
} }
if (this.muted) { for (let i = 0; i < N; i++) {
for (let i = 0; i < N; i++) { const x0 = this._x0 ?? 0
ch0[i] = 0 const x1 = this._x1 ?? x0
for (let c = 1; c < out.length; c++) out[c][i] = 0 const value = this.muted ? 0 : x0 + (x1 - x0) * this.phase
advance() const sample = value <= -32768 ? -1 : value >= 32767 ? 1 : value / 32768
} out[0][i] = sample
} else { if (out[1]) out[1][i] = sample
for (let i = 0; i < N; i++) { this.nowItemSamples += 1
const x0 = this._x0 ?? 0 this._notifyFrames += 1
const x1 = this._x1 ?? x0 common()
const value = x0 + (x1 - x0) * this.phase if (this._notifyFrames >= this.dstRate / 10) {
const sample = value <= -32768 ? -1 : value >= 32767 ? 1 : value / 32768 this._notifyFrames = 0
ch0[i] = sample this._postNowPlaying()
for (let c = 1; c < out.length; c++) out[c][i] = sample
advance()
this._notifyFrames += 1
if (this._notifyFrames >= this.dstRate / 10) {
this._notifyFrames = 0
}
} }
} }
@@ -168,12 +244,13 @@ const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProces
this._inRate = sampleRate this._inRate = sampleRate
this._pos = 0 this._pos = 0
this._carry = null this._carry = null
this._outRate = ${SAMPLE_RATE} this._outRate = 24000
} }
process(inputs) { process(inputs) {
const chs = inputs[0] const chs = inputs[0]
if (!chs || chs.length === 0) return true if (!chs || chs.length === 0) return true
const inF32 = chs[0] const inF32 = chs[0]
const step = this._inRate / this._outRate const step = this._inRate / this._outRate
@@ -187,24 +264,30 @@ const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProces
const avail = src.length - 1 - this._pos const avail = src.length - 1 - this._pos
const outLen = avail > 0 ? Math.ceil(avail / step) : 0 const outLen = avail > 0 ? Math.ceil(avail / step) : 0
const outI16 = new Int16Array(outLen) const outI16 = new Int16Array(outLen)
let pos = this._pos let pos = this._pos
for (let k = 0; k < outLen; k++) { for (let k = 0; k < outLen; k++) {
const i = Math.floor(pos) const i = Math.floor(pos)
const frac = pos - i const frac = pos - i
const x0 = src[i] const x0 = src[i]
const x1 = src[i + 1] const x1 = src[i + 1]
let y = x0 + frac * (x1 - x0) let y = x0 + frac * (x1 - x0)
if (y > 1) y = 1 if (y > 1) y = 1
else if (y < -1) y = -1 else if (y < -1) y = -1
const s = y <= -1 ? -0x8000 : Math.round(y * 0x7fff) const s = y <= -1 ? -0x8000 : Math.round(y * 0x7fff)
outI16[k] = s outI16[k] = s
pos += step pos += step
} }
this._pos = pos - (src.length - 1) this._pos = pos - (src.length - 1)
if (this._pos < 0) this._pos = 0 if (this._pos < 0) this._pos = 0
this._carry = src[src.length - 1] this._carry = src[src.length - 1]
try { try {