worklet fix

frontend/src/lib/components/voicebotDemo/helper.ts

@@ -9,26 +9,80 @@ export const WS_URL = isBrowser && window.location.protocol === "http:"
     ? `ws://${WS_HOST}/api/v1/voicebot/ws`
     : `wss://${WS_HOST}/api/v1/voicebot/ws`
 
-const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProcessor {
+const playbackWorkletCode = `/**
+ * @typedef {Object} AppendDeltaMessage
+ * @property {"appendDelta"} type
+ * @property {ResponseAudioDelta} delta
+ */
+
+/**
+ * @typedef {Object} DeleteItemMessage
+ * @property {"deleteItem"} type
+ * @property {string} item_id
+ */
+
+/**
+ * @typedef {Object} ClearMessage
+ * @property {"clear"} type
+ */
+
+/**
+ * @typedef {Object} SetSourceRateMessage
+ * @property {"setSourceRate"} type
+ * @property {number} hz
+ */
+
+/**
+ * @typedef {Object} NowPlayingMessage
+ * @property {"nowPlaying"} type
+ * @property {string|null} item_id
+ * @property {number} played_ms
+ */
+
+/**
+ * @typedef {Object} MuteMessage
+ * @property {"mute"} type
+ */
+
+/**
+ * @typedef {Object} UnmuteMessage
+ * @property {"unmute"} type
+ */
+
+/**
+ * @typedef {AppendDeltaMessage | DeleteItemMessage | ClearMessage | SetSourceRateMessage | MuteMessage | UnmuteMessage} PlaybackMessage
+ */
+
+/**
+ * @typedef {Object} Chunk
+ * @property {string} item_id
+ * @property {Int16Array} data
+ * @property {number} off
+ */
+
+class AudioPlaybackWorklet extends AudioWorkletProcessor {
     constructor() {
         super()
-        this.srcRate = ${SAMPLE_RATE}
+        /** @type {number} */ this.srcRate = 24000
-        this.dstRate = sampleRate
+        /** @type {number} */ this.dstRate = sampleRate
-        this.step = this.srcRate / this.dstRate
+        /** @type {number} */ this.step = this.srcRate / this.dstRate
-        this.queue = []
+        /** @type {Chunk[]} */ this.queue = []
-        this.cur = null
+        /** @type {Chunk|null} */ this.cur = null
-        this.hold = 0
+        /** @type {number} */ this.hold = 0
-        this.phase = 0
+        /** @type {number} */ this.phase = 0
-        this._x0 = undefined
+        /** @type {number|undefined} */ this._x0 = undefined
-        this._x1 = undefined
+        /** @type {number|undefined} */ this._x1 = undefined
-        this._nextItemId = null
+        /** @type {string|null} */ this._x0ItemId = null
-        this.nowItemId = null
+        /** @type {string|null} */ this._x1ItemId = null
-        this.nowItemSamples = 0
+        /** @type {string|null} */ this._nextItemId = null
-        this._notifyFrames = 0
+        /** @type {string|null} */ this.nowItemId = null
-        this.muted = false
+        /** @type {number} */ this.nowItemSamples = 0
-        this.port.onmessage = (e) => this._onMessage(e.data)
+        /** @type {number} */ this._notifyFrames = 0
+        /** @type {boolean} */ this.muted = false
+        this.port.onmessage = (e) => this._onMessage(/** @type {PlaybackMessage} */ (e.data))
     }
 
+    /** @param {PlaybackMessage} msg */
     _onMessage(msg) {
         if (!msg || !msg.type) return
         if (msg.type === "appendDelta" && msg.delta && msg.delta.pcmInt16 instanceof Int16Array) {
@@ -45,7 +99,7 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
             if (this.nowItemId === id) {
                 this.nowItemId = null
                 this.nowItemSamples = 0
-                this._notifyFrames = 0
+                this._postNowPlaying()
             }
             return
         }
@@ -56,10 +110,13 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
             this.phase = 0
             this._x0 = undefined
             this._x1 = undefined
+            this._x0ItemId = null
+            this._x1ItemId = null
             this._nextItemId = null
             this.nowItemId = null
             this.nowItemSamples = 0
             this._notifyFrames = 0
+            this._postNowPlaying()
             return
         }
         if (msg.type === "setSourceRate" && Number.isFinite(msg.hz) && msg.hz > 0) {
@@ -73,9 +130,11 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
         }
         if (msg.type === "unmute") {
             this.muted = false
+            return
         }
     }
 
+    /** @returns {boolean} */
     _ensureCurrent() {
         if (this.cur == null) {
             if (this.queue.length === 0) return false
@@ -85,6 +144,7 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
         return true
     }
 
+    /** @returns {number} */
     _nextInt16() {
         for (;;) {
             if (!this._ensureCurrent()) {
@@ -105,7 +165,23 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
         }
     }
 
-    process(_inputs, outputs) {
+    _postNowPlaying() {
+        /** @type {NowPlayingMessage} */
+        const m = {
+            type: "nowPlaying",
+            item_id: this.nowItemId,
+            played_ms: Math.max(0, Math.floor((this.nowItemSamples * 1000) / this.srcRate)),
+        }
+        this.port.postMessage(m)
+    }
+
+    /**
+     * @param {Float32Array[][]} _inputs
+     * @param {Float32Array[][]} outputs
+     * @param {Record<string, Float32Array>} _parameters
+     * @returns {boolean}
+     */
+    process(_inputs, outputs, _parameters) {
         const out = outputs[0]
         if (!out || out.length === 0) return true
         const ch0 = out[0]
@@ -113,45 +189,45 @@ const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProc
 
         if (this._x1 === undefined) {
             this._x1 = this._nextInt16()
+            this._x1ItemId = this._nextItemId
             this._x0 = this._x1
+            this._x0ItemId = this._x1ItemId
             this.phase = 0
-            this.nowItemId = this._nextItemId
+            this.nowItemId = this._x0ItemId
             this.nowItemSamples = 0
+            this._postNowPlaying()
         }
 
-        const advance = () => {
+        const common = () => {
             this.phase += this.step
             while (this.phase >= 1) {
                 this.phase -= 1
                 this._x0 = this._x1
+                this._x0ItemId = this._x1ItemId
                 this._x1 = this._nextInt16()
-                if (this.nowItemId !== this._nextItemId) {
+                this._x1ItemId = this._nextItemId
-                    this.nowItemId = this._nextItemId
+
+                if (this.nowItemId !== this._x0ItemId) {
+                    this.nowItemId = this._x0ItemId
                     this.nowItemSamples = 0
+                    this._postNowPlaying()
                 }
-                if (this.nowItemId) this.nowItemSamples += 1
             }
         }
 
-        if (this.muted) {
-            for (let i = 0; i < N; i++) {
-                ch0[i] = 0
-                for (let c = 1; c < out.length; c++) out[c][i] = 0
-                advance()
-            }
-        } else {
         for (let i = 0; i < N; i++) {
             const x0 = this._x0 ?? 0
             const x1 = this._x1 ?? x0
-                const value = x0 + (x1 - x0) * this.phase
+            const value = this.muted ? 0 : x0 + (x1 - x0) * this.phase
             const sample = value <= -32768 ? -1 : value >= 32767 ? 1 : value / 32768
-                ch0[i] = sample
+            out[0][i] = sample
-                for (let c = 1; c < out.length; c++) out[c][i] = sample
+            if (out[1]) out[1][i] = sample
-                advance()
+            this.nowItemSamples += 1
             this._notifyFrames += 1
+            common()
             if (this._notifyFrames >= this.dstRate / 10) {
                 this._notifyFrames = 0
-                }
+                this._postNowPlaying()
             }
         }
 
@@ -168,12 +244,13 @@ const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProces
         this._inRate = sampleRate
         this._pos = 0
         this._carry = null
-        this._outRate = ${SAMPLE_RATE}
+        this._outRate = 24000
     }
 
     process(inputs) {
         const chs = inputs[0]
         if (!chs || chs.length === 0) return true
+
         const inF32 = chs[0]
         const step = this._inRate / this._outRate
 
@@ -187,24 +264,30 @@ const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProces
 
         const avail = src.length - 1 - this._pos
         const outLen = avail > 0 ? Math.ceil(avail / step) : 0
+
         const outI16 = new Int16Array(outLen)
 
         let pos = this._pos
         for (let k = 0; k < outLen; k++) {
             const i = Math.floor(pos)
             const frac = pos - i
+
             const x0 = src[i]
             const x1 = src[i + 1]
+
             let y = x0 + frac * (x1 - x0)
             if (y > 1) y = 1
             else if (y < -1) y = -1
+
             const s = y <= -1 ? -0x8000 : Math.round(y * 0x7fff)
             outI16[k] = s
+
             pos += step
         }
 
         this._pos = pos - (src.length - 1)
         if (this._pos < 0) this._pos = 0
+
         this._carry = src[src.length - 1]
 
         try {