robin/kontextwerk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

worklet fix

Browse Source
This commit is contained in:
Robin Grenzdörfer
2025-10-05 12:56:40 +00:00
parent 2be72187ec
commit 57b5415e66
1 changed files with 126 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
frontend/src/lib/components/voicebotDemo/helper.ts
View File

@@ -9,26 +9,80 @@ export const WS_URL = isBrowser && window.location.protocol === "http:"
? `ws://${WS_HOST}/api/v1/voicebot/ws`
: `wss://${WS_HOST}/api/v1/voicebot/ws`
const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProcessor {
const playbackWorkletCode = `/**
* @typedef {Object} AppendDeltaMessage
* @property {"appendDelta"} type
* @property {ResponseAudioDelta} delta
*/
/**
* @typedef {Object} DeleteItemMessage
* @property {"deleteItem"} type
* @property {string} item_id
*/
/**
* @typedef {Object} ClearMessage
* @property {"clear"} type
*/
/**
* @typedef {Object} SetSourceRateMessage
* @property {"setSourceRate"} type
* @property {number} hz
*/
/**
* @typedef {Object} NowPlayingMessage
* @property {"nowPlaying"} type
* @property {string|null} item_id
* @property {number} played_ms
*/
/**
* @typedef {Object} MuteMessage
* @property {"mute"} type
*/
/**
* @typedef {Object} UnmuteMessage
* @property {"unmute"} type
*/
/**
* @typedef {AppendDeltaMessage | DeleteItemMessage | ClearMessage | SetSourceRateMessage | MuteMessage | UnmuteMessage} PlaybackMessage
*/
/**
* @typedef {Object} Chunk
* @property {string} item_id
* @property {Int16Array} data
* @property {number} off
*/
class AudioPlaybackWorklet extends AudioWorkletProcessor {
constructor() {
super()
this.srcRate = ${SAMPLE_RATE}
this.dstRate = sampleRate
this.step = this.srcRate / this.dstRate
this.queue = []
this.cur = null
this.hold = 0
this.phase = 0
this._x0 = undefined
this._x1 = undefined
this._nextItemId = null
this.nowItemId = null
this.nowItemSamples = 0
this._notifyFrames = 0
this.muted = false
this.port.onmessage = (e) => this._onMessage(e.data)
/** @type {number} */ this.srcRate = 24000
/** @type {number} */ this.dstRate = sampleRate
/** @type {number} */ this.step = this.srcRate / this.dstRate
/** @type {Chunk[]} */ this.queue = []
/** @type {Chunk|null} */ this.cur = null
/** @type {number} */ this.hold = 0
/** @type {number} */ this.phase = 0
/** @type {number|undefined} */ this._x0 = undefined
/** @type {number|undefined} */ this._x1 = undefined
/** @type {string|null} */ this._x0ItemId = null
/** @type {string|null} */ this._x1ItemId = null
/** @type {string|null} */ this._nextItemId = null
/** @type {string|null} */ this.nowItemId = null
/** @type {number} */ this.nowItemSamples = 0
/** @type {number} */ this._notifyFrames = 0
/** @type {boolean} */ this.muted = false
this.port.onmessage = (e) => this._onMessage(/** @type {PlaybackMessage} */ (e.data))
}
/** @param {PlaybackMessage} msg */
_onMessage(msg) {
if (!msg || !msg.type) return
if (msg.type === "appendDelta" && msg.delta && msg.delta.pcmInt16 instanceof Int16Array) {
@@ -45,7 +99,7 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
if (this.nowItemId === id) {
this.nowItemId = null
this.nowItemSamples = 0
this._notifyFrames = 0
this._postNowPlaying()
}
return
}
@@ -56,10 +110,13 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
this.phase = 0
this._x0 = undefined
this._x1 = undefined
this._x0ItemId = null
this._x1ItemId = null
this._nextItemId = null
this.nowItemId = null
this.nowItemSamples = 0
this._notifyFrames = 0
this._postNowPlaying()
return
}
if (msg.type === "setSourceRate" && Number.isFinite(msg.hz) && msg.hz > 0) {
@@ -73,9 +130,11 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
}
if (msg.type === "unmute") {
this.muted = false
return
}
}
/** @returns {boolean} */
_ensureCurrent() {
if (this.cur == null) {
if (this.queue.length === 0) return false
@@ -85,6 +144,7 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
return true
}
/** @returns {number} */
_nextInt16() {
for (;;) {
if (!this._ensureCurrent()) {
@@ -105,7 +165,23 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
}
}
process(_inputs, outputs) {
_postNowPlaying() {
/** @type {NowPlayingMessage} */
const m = {
type: "nowPlaying",
item_id: this.nowItemId,
played_ms: Math.max(0, Math.floor((this.nowItemSamples * 1000) / this.srcRate)),
}
this.port.postMessage(m)
}
/**
* @param {Float32Array[][]} _inputs
* @param {Float32Array[][]} outputs
* @param {Record<string, Float32Array>} _parameters
* @returns {boolean}
*/
process(_inputs, outputs, _parameters) {
const out = outputs[0]
if (!out || out.length === 0) return true
const ch0 = out[0]
@@ -113,45 +189,45 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
if (this._x1 === undefined) {
this._x1 = this._nextInt16()
this._x1ItemId = this._nextItemId
this._x0 = this._x1
this._x0ItemId = this._x1ItemId
this.phase = 0
this.nowItemId = this._nextItemId
this.nowItemId = this._x0ItemId
this.nowItemSamples = 0
this._postNowPlaying()
}
const advance = () => {
const common = () => {
this.phase += this.step
while (this.phase >= 1) {
this.phase -= 1
this._x0 = this._x1
this._x0ItemId = this._x1ItemId
this._x1 = this._nextInt16()
if (this.nowItemId !== this._nextItemId) {
this.nowItemId = this._nextItemId
this._x1ItemId = this._nextItemId
if (this.nowItemId !== this._x0ItemId) {
this.nowItemId = this._x0ItemId
this.nowItemSamples = 0
this._postNowPlaying()
}
if (this.nowItemId) this.nowItemSamples += 1
}
}
if (this.muted) {
for (let i = 0; i < N; i++) {
ch0[i] = 0
for (let c = 1; c < out.length; c++) out[c][i] = 0
advance()
}
} else {
for (let i = 0; i < N; i++) {
const x0 = this._x0 ?? 0
const x1 = this._x1 ?? x0
const value = x0 + (x1 - x0) * this.phase
const value = this.muted ? 0 : x0 + (x1 - x0) * this.phase
const sample = value <= -32768 ? -1 : value >= 32767 ? 1 : value / 32768
ch0[i] = sample
for (let c = 1; c < out.length; c++) out[c][i] = sample
advance()
out[0][i] = sample
if (out[1]) out[1][i] = sample
this.nowItemSamples += 1
this._notifyFrames += 1
common()
if (this._notifyFrames >= this.dstRate / 10) {
this._notifyFrames = 0
}
this._postNowPlaying()
}
}
@@ -168,12 +244,13 @@ const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProces
this._inRate = sampleRate
this._pos = 0
this._carry = null
this._outRate = ${SAMPLE_RATE}
this._outRate = 24000
}
process(inputs) {
const chs = inputs[0]
if (!chs || chs.length === 0) return true
const inF32 = chs[0]
const step = this._inRate / this._outRate
@@ -187,24 +264,30 @@ const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProces
const avail = src.length - 1 - this._pos
const outLen = avail > 0 ? Math.ceil(avail / step) : 0
const outI16 = new Int16Array(outLen)
let pos = this._pos
for (let k = 0; k < outLen; k++) {
const i = Math.floor(pos)
const frac = pos - i
const x0 = src[i]
const x1 = src[i + 1]
let y = x0 + frac * (x1 - x0)
if (y > 1) y = 1
else if (y < -1) y = -1
const s = y <= -1 ? -0x8000 : Math.round(y * 0x7fff)
outI16[k] = s
pos += step
}
this._pos = pos - (src.length - 1)
if (this._pos < 0) this._pos = 0
this._carry = src[src.length - 1]
try {