sdk zwischenstand
This commit is contained in:
@@ -1,10 +1,9 @@
|
||||
<script lang="ts">
|
||||
import { onDestroy } from "svelte"
|
||||
import { onMount } from "svelte"
|
||||
import { mdiBookAccountOutline, mdiCreation, mdiFaceAgent, mdiHours24 } from "@mdi/js"
|
||||
import ProductCategoryFrame from "../widgets/ProductCategoryFrame.svelte"
|
||||
import CrinkledSection from "../CrinkledSection.svelte"
|
||||
import { base64ToUint8, createPlayer, createRecorder, SAMPLE_RATE, uint8ToBase64 } from "../voicebotDemo/helper"
|
||||
import { RealtimeServerEvent as RSE } from "../voicebotDemo/events"
|
||||
import { createVoicebotPreviewController } from "./voicebotPreviewController"
|
||||
|
||||
const voiceProperties: Array<{ title: string; icon: string; color: string }> = [
|
||||
{
|
||||
@@ -29,300 +28,18 @@
|
||||
},
|
||||
]
|
||||
|
||||
const browser = typeof window !== "undefined"
|
||||
const VOICE_WS_URL =
|
||||
browser && window.location.protocol === "http:"
|
||||
? "ws://2svoice-server.kontextwerk.info/api/v1/voicebot/ws"
|
||||
: "wss://2svoice-server.kontextwerk.info/api/v1/voicebot/ws"
|
||||
const CHUNK_DURATION_MS = 200
|
||||
const CHUNK_SIZE_BYTES = Math.round((SAMPLE_RATE * CHUNK_DURATION_MS) / 1000) * 2
|
||||
|
||||
type VoiceStatus = "idle" | "connecting" | "connected" | "error"
|
||||
|
||||
let status: VoiceStatus = "idle"
|
||||
let errorMessage = ""
|
||||
|
||||
let ws: WebSocket | null = null
|
||||
let recorder: ReturnType<typeof createRecorder> | null = null
|
||||
let player: ReturnType<typeof createPlayer> | null = null
|
||||
let outboundBuffer = new Uint8Array(0)
|
||||
let closing = false
|
||||
let cleanupPromise: Promise<void> | null = null
|
||||
let startPromise: Promise<void> | null = null
|
||||
|
||||
$: statusHint =
|
||||
status === "idle"
|
||||
? "Tippen, um die Voice-Demo zu starten"
|
||||
: status === "connecting"
|
||||
? "Verbindung wird aufgebaut …"
|
||||
: status === "connected"
|
||||
? "Live – sprechen Sie jetzt"
|
||||
: errorMessage || "Verbindung fehlgeschlagen"
|
||||
|
||||
const toggleVoiceDemo = async () => {
|
||||
if (status === "connecting") return
|
||||
if (status === "connected") {
|
||||
await stopVoiceDemo()
|
||||
return
|
||||
}
|
||||
await startVoiceDemo()
|
||||
}
|
||||
const controller = createVoicebotPreviewController()
|
||||
const { status, statusHint, toggle, setup, teardown } = controller
|
||||
|
||||
const handleKeydown = (event: KeyboardEvent) => {
|
||||
if (event.key !== "Enter" && event.key !== " ") return
|
||||
event.preventDefault()
|
||||
void toggleVoiceDemo()
|
||||
void toggle()
|
||||
}
|
||||
|
||||
const startVoiceDemo = async () => {
|
||||
if (!browser) {
|
||||
status = "error"
|
||||
errorMessage = "Die Sprach-Demo steht nur im Browser zur Verfügung."
|
||||
return
|
||||
}
|
||||
if (startPromise || status === "connecting" || status === "connected") return
|
||||
|
||||
startPromise = (async () => {
|
||||
await stopVoiceDemo({ resetStatus: false })
|
||||
status = "connecting"
|
||||
errorMessage = ""
|
||||
outboundBuffer = new Uint8Array(0)
|
||||
closing = false
|
||||
|
||||
try {
|
||||
const newPlayer = createPlayer()
|
||||
await newPlayer.init()
|
||||
player = newPlayer
|
||||
|
||||
const handleChunk = (pcm: Int16Array) => {
|
||||
if (pcm.length === 0) return
|
||||
const bytes = new Uint8Array(pcm.byteLength)
|
||||
bytes.set(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength))
|
||||
appendToOutboundBuffer(bytes)
|
||||
}
|
||||
|
||||
const newRecorder = createRecorder(handleChunk)
|
||||
await newRecorder.start()
|
||||
recorder = newRecorder
|
||||
} catch (err) {
|
||||
const message = extractErrorMessage(err, "Mikrofon konnte nicht gestartet werden.")
|
||||
handleConnectionError(message, err)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
ws = new WebSocket(VOICE_WS_URL)
|
||||
} catch (err) {
|
||||
const message = extractErrorMessage(err, "WebSocket-Verbindung konnte nicht aufgebaut werden.")
|
||||
handleConnectionError(message, err)
|
||||
return
|
||||
}
|
||||
|
||||
if (!ws) return
|
||||
|
||||
ws.onopen = () => {
|
||||
status = "connected"
|
||||
flushOutboundBuffer(true)
|
||||
}
|
||||
|
||||
ws.onmessage = (event) => handleServerMessage(event)
|
||||
|
||||
ws.onerror = (event) => {
|
||||
handleConnectionError("WebSocket-Fehler – bitte später erneut versuchen.", event)
|
||||
}
|
||||
|
||||
ws.onclose = () => {
|
||||
if (!closing && status === "connected") {
|
||||
status = "idle"
|
||||
errorMessage = ""
|
||||
}
|
||||
}
|
||||
})()
|
||||
|
||||
try {
|
||||
await startPromise
|
||||
} finally {
|
||||
startPromise = null
|
||||
}
|
||||
}
|
||||
|
||||
const stopVoiceDemo = async ({ resetStatus = true }: { resetStatus?: boolean } = {}) => {
|
||||
if (cleanupPromise) {
|
||||
await cleanupPromise
|
||||
if (resetStatus && status !== "error") {
|
||||
status = "idle"
|
||||
errorMessage = ""
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
closing = true
|
||||
|
||||
cleanupPromise = (async () => {
|
||||
try {
|
||||
flushOutboundBuffer(true)
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
if (recorder) {
|
||||
try {
|
||||
await recorder.stop()
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
recorder = null
|
||||
|
||||
if (player) {
|
||||
try {
|
||||
player.stop()
|
||||
await player.destroy()
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
player = null
|
||||
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
ws.close(1000, "client-stop")
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
ws = null
|
||||
outboundBuffer = new Uint8Array(0)
|
||||
})()
|
||||
|
||||
try {
|
||||
await cleanupPromise
|
||||
} finally {
|
||||
cleanupPromise = null
|
||||
closing = false
|
||||
if (resetStatus && status !== "error") {
|
||||
status = "idle"
|
||||
errorMessage = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const handleServerMessage = (event: MessageEvent) => {
|
||||
if (!player) return
|
||||
let payload: unknown = event.data
|
||||
|
||||
if (typeof payload !== "string") return
|
||||
|
||||
try {
|
||||
payload = JSON.parse(payload)
|
||||
} catch (err) {
|
||||
console.warn("VoiceBot Preview: Konnte Nachricht nicht parsen.", err)
|
||||
return
|
||||
}
|
||||
|
||||
const message = payload as Record<string, unknown>
|
||||
const type = typeof message.type === "string" ? message.type : "<unbekannt>"
|
||||
if (type === RSE.INPUT_AUDIO_BUFFER_SPEECH_STARTED) {
|
||||
const { item_id, played_ms } = player.getNowPlaying()
|
||||
if (item_id) {
|
||||
player.stop()
|
||||
ws?.send(
|
||||
JSON.stringify({
|
||||
type: "last_item_played_ms.truncate",
|
||||
details: { item_id, played_ms: played_ms || 0 },
|
||||
})
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (type === RSE.RESPONSE_AUDIO_DELTA) {
|
||||
const bytes = base64ToUint8((message as any).delta)
|
||||
const pcm = new Int16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2)
|
||||
player.play({
|
||||
response_id: message.response_id,
|
||||
item_id: message.item_id,
|
||||
delta: message.delta,
|
||||
pcmInt16: pcm,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (type === "error") {
|
||||
console.log("VoiceBot Preview: Server hat einen Fehler gemeldet.", message)
|
||||
}
|
||||
}
|
||||
|
||||
const appendToOutboundBuffer = (chunk: Uint8Array) => {
|
||||
if (!chunk.length) return
|
||||
const combined = new Uint8Array(outboundBuffer.length + chunk.length)
|
||||
combined.set(outboundBuffer)
|
||||
combined.set(chunk, outboundBuffer.length)
|
||||
outboundBuffer = combined
|
||||
flushOutboundBuffer()
|
||||
}
|
||||
|
||||
const flushOutboundBuffer = (force = false) => {
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN || outboundBuffer.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const chunkSize = CHUNK_SIZE_BYTES > 0 ? CHUNK_SIZE_BYTES : outboundBuffer.length
|
||||
let buffer = outboundBuffer
|
||||
outboundBuffer = new Uint8Array(0)
|
||||
|
||||
while (buffer.length >= chunkSize && chunkSize > 0) {
|
||||
const part = buffer.slice(0, chunkSize)
|
||||
buffer = buffer.slice(chunkSize)
|
||||
sendChunk(part)
|
||||
}
|
||||
|
||||
if (force && buffer.length > 0) {
|
||||
sendChunk(buffer)
|
||||
} else if (buffer.length > 0) {
|
||||
outboundBuffer = buffer
|
||||
}
|
||||
}
|
||||
|
||||
const sendChunk = (chunk: Uint8Array) => {
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN) return
|
||||
|
||||
try {
|
||||
ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: uint8ToBase64(chunk) }))
|
||||
} catch (err) {
|
||||
if (!closing) {
|
||||
handleConnectionError("Senden des Audiostreams fehlgeschlagen.", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const handleConnectionError = (message: string, err?: unknown) => {
|
||||
console.error("VoiceBot Preview Fehler:", err ?? message)
|
||||
errorMessage = message
|
||||
status = "error"
|
||||
void stopVoiceDemo({ resetStatus: false })
|
||||
}
|
||||
|
||||
const extractErrorMessage = (err: unknown, fallback: string) => {
|
||||
if (err instanceof DOMException) {
|
||||
if (err.name === "NotAllowedError") {
|
||||
return "Zugriff auf das Mikrofon wurde verweigert."
|
||||
}
|
||||
if (err.name === "NotFoundError") {
|
||||
return "Kein Mikrofon gefunden oder verfügbar."
|
||||
}
|
||||
if (err.name === "NotReadableError") {
|
||||
return "Auf das Mikrofon konnte nicht zugegriffen werden (ggf. bereits in Verwendung)."
|
||||
}
|
||||
if (err.name === "SecurityError") {
|
||||
return "Der Browser blockiert den Zugriff – bitte die Seite über HTTPS öffnen."
|
||||
}
|
||||
}
|
||||
if (err instanceof Error && err.message) return err.message
|
||||
return fallback
|
||||
}
|
||||
|
||||
onDestroy(() => {
|
||||
void stopVoiceDemo({ resetStatus: false })
|
||||
onMount(() => {
|
||||
setup()
|
||||
return () => teardown()
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -341,26 +58,27 @@
|
||||
{#snippet primaryContent()}
|
||||
<div
|
||||
class="img"
|
||||
class:connected={status === "connected"}
|
||||
class:errored={status === "error"}
|
||||
class:connected={$status === "connected"}
|
||||
class:errored={$status === "error"}
|
||||
role="button"
|
||||
tabindex="0"
|
||||
aria-pressed={status === "connected"}
|
||||
aria-busy={status === "connecting"}
|
||||
aria-pressed={$status === "connected"}
|
||||
aria-busy={$status === "connecting"}
|
||||
aria-label="Voicebot Demo starten"
|
||||
on:click={() => void toggleVoiceDemo()}
|
||||
on:click={() => void toggle()}
|
||||
on:keydown={handleKeydown}
|
||||
>
|
||||
<img
|
||||
src="/media/iphone.png"
|
||||
alt="Kontextwerk is calling"
|
||||
/>
|
||||
<div class="shadow"></div>
|
||||
<div
|
||||
class="voice-overlay"
|
||||
data-status={status}
|
||||
data-status={$status}
|
||||
aria-live="polite"
|
||||
>
|
||||
<span>{statusHint}</span>
|
||||
<span>{$statusHint}</span>
|
||||
</div>
|
||||
</div>
|
||||
{/snippet}
|
||||
@@ -432,11 +150,9 @@
|
||||
}
|
||||
|
||||
&.connected {
|
||||
border-color: rgba(76, 175, 80, 0.4);
|
||||
}
|
||||
|
||||
&.errored {
|
||||
border-color: rgba(235, 87, 87, 0.45);
|
||||
}
|
||||
|
||||
&:focus-visible {
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import { derived, get, writable, type Readable } from "svelte/store"
|
||||
import {
|
||||
ConnectorLifecycleEvents,
|
||||
createVoiceConnector,
|
||||
WS_URL,
|
||||
} from "../voicebotDemo/helper"
|
||||
import type { WerkRealtimeConnector } from "../voicebotDemo/helper"
|
||||
|
||||
export type VoiceStatus = "idle" | "connecting" | "connected" | "error"
|
||||
|
||||
interface VoicebotPreviewController {
|
||||
status: Readable<VoiceStatus>
|
||||
errorMessage: Readable<string>
|
||||
statusHint: Readable<string>
|
||||
setup: () => void
|
||||
teardown: () => void
|
||||
start: () => Promise<void>
|
||||
stop: (silent?: boolean) => Promise<void>
|
||||
toggle: () => Promise<void>
|
||||
}
|
||||
|
||||
const isBrowser = typeof window !== "undefined"
|
||||
|
||||
const extractErrorMessage = (err: unknown, fallback: string) => {
|
||||
if (err instanceof DOMException) {
|
||||
if (err.name === "NotAllowedError") return "Zugriff auf das Mikrofon wurde verweigert."
|
||||
if (err.name === "NotFoundError") return "Kein Mikrofon gefunden oder verfügbar."
|
||||
if (err.name === "NotReadableError") return "Auf das Mikrofon konnte nicht zugegriffen werden."
|
||||
if (err.name === "SecurityError") return "Der Browser blockiert den Zugriff – bitte die Seite über HTTPS öffnen."
|
||||
}
|
||||
if (err instanceof Error && err.message) return err.message
|
||||
return fallback
|
||||
}
|
||||
|
||||
export const createVoicebotPreviewController = (): VoicebotPreviewController => {
|
||||
const statusStore = writable<VoiceStatus>("idle")
|
||||
const errorStore = writable("")
|
||||
|
||||
const statusHint = derived([statusStore, errorStore], ([$status, $error]) => {
|
||||
switch ($status) {
|
||||
case "idle":
|
||||
return "Tippen, um die Voice-Demo zu starten"
|
||||
case "connecting":
|
||||
return "Verbindung wird aufgebaut …"
|
||||
case "connected":
|
||||
return "Live – sprechen Sie jetzt"
|
||||
case "error":
|
||||
return $error || "Verbindung fehlgeschlagen"
|
||||
default:
|
||||
return "Voice-Demo"
|
||||
}
|
||||
})
|
||||
|
||||
let connector: WerkRealtimeConnector | null = null
|
||||
let detachHandlers: Array<() => void> = []
|
||||
let startPromise: Promise<void> | null = null
|
||||
let stopPromise: Promise<void> | null = null
|
||||
let closing = false
|
||||
const handleRealtimeEvent = (rawType: string, msg: any) => {
|
||||
const type = rawType
|
||||
if (!type) return
|
||||
|
||||
const now = new Date()
|
||||
|
||||
}
|
||||
const ensureConnector = () => {
|
||||
if (!isBrowser || connector) return
|
||||
|
||||
const instance = createVoiceConnector()
|
||||
|
||||
const detachConnecting = instance.onLifecycle(ConnectorLifecycleEvents.CONNECTING, () => {
|
||||
statusStore.set("connecting")
|
||||
errorStore.set("")
|
||||
})
|
||||
const detachConnected = instance.onLifecycle(ConnectorLifecycleEvents.CONNECTED, () => {
|
||||
statusStore.set("connected")
|
||||
closing = false
|
||||
})
|
||||
const detachDisconnected = instance.onLifecycle(ConnectorLifecycleEvents.DISCONNECTED, () => {
|
||||
if (closing) {
|
||||
closing = false
|
||||
return
|
||||
}
|
||||
if (get(statusStore) !== "error") {
|
||||
statusStore.set("idle")
|
||||
errorStore.set("")
|
||||
}
|
||||
})
|
||||
const detachError = instance.onLifecycle(ConnectorLifecycleEvents.ERROR, (evt) => {
|
||||
const message =
|
||||
typeof evt?.message === "string" && evt.message.trim().length
|
||||
? evt.message
|
||||
: "Verbindung fehlgeschlagen"
|
||||
errorStore.set(`${message} (${WS_URL})`)
|
||||
statusStore.set("error")
|
||||
closing = false
|
||||
})
|
||||
|
||||
detachHandlers = [detachConnecting, detachConnected, detachDisconnected, detachError]
|
||||
connector = instance
|
||||
}
|
||||
|
||||
const cleanupConnector = () => {
|
||||
detachHandlers.forEach((fn) => fn())
|
||||
detachHandlers = []
|
||||
const instance = connector
|
||||
connector = null
|
||||
if (instance) {
|
||||
void instance.stop()
|
||||
}
|
||||
startPromise = null
|
||||
stopPromise = null
|
||||
closing = false
|
||||
}
|
||||
|
||||
const stop = async (silent = false) => {
|
||||
if (!connector) return
|
||||
if (stopPromise) {
|
||||
await stopPromise
|
||||
return
|
||||
}
|
||||
|
||||
closing = true
|
||||
if (!silent && get(statusStore) !== "error") {
|
||||
statusStore.set("idle")
|
||||
errorStore.set("")
|
||||
}
|
||||
|
||||
stopPromise = connector.stop()
|
||||
|
||||
try {
|
||||
await stopPromise
|
||||
} catch (err) {
|
||||
console.error("VoicebotPreview stop error", err)
|
||||
if (!silent) {
|
||||
errorStore.set(extractErrorMessage(err, "Verbindung konnte nicht beendet werden."))
|
||||
statusStore.set("error")
|
||||
}
|
||||
} finally {
|
||||
stopPromise = null
|
||||
closing = false
|
||||
}
|
||||
}
|
||||
|
||||
const start = async () => {
|
||||
if (!isBrowser) {
|
||||
statusStore.set("error")
|
||||
errorStore.set("Die Sprach-Demo steht nur im Browser zur Verfügung.")
|
||||
return
|
||||
}
|
||||
|
||||
ensureConnector()
|
||||
if (!connector || startPromise) return
|
||||
|
||||
await stop(true)
|
||||
|
||||
statusStore.set("connecting")
|
||||
errorStore.set("")
|
||||
|
||||
startPromise = (async () => {
|
||||
await connector!.start()
|
||||
connector!.setInputMuted(false)
|
||||
connector!.setOutputMuted(false)
|
||||
})()
|
||||
|
||||
try {
|
||||
await startPromise
|
||||
} catch (err) {
|
||||
console.error("VoicebotPreview start error", err)
|
||||
errorStore.set(extractErrorMessage(err, "Verbindung konnte nicht aufgebaut werden."))
|
||||
statusStore.set("error")
|
||||
closing = false
|
||||
} finally {
|
||||
startPromise = null
|
||||
}
|
||||
}
|
||||
|
||||
const toggle = async () => {
|
||||
const current = get(statusStore)
|
||||
if (current === "connecting") return
|
||||
if (current === "connected") {
|
||||
await stop()
|
||||
return
|
||||
}
|
||||
await start()
|
||||
}
|
||||
|
||||
const setup = () => {
|
||||
ensureConnector()
|
||||
}
|
||||
|
||||
const teardown = () => {
|
||||
void stop(true)
|
||||
cleanupConnector()
|
||||
}
|
||||
|
||||
return {
|
||||
status: { subscribe: statusStore.subscribe },
|
||||
errorMessage: { subscribe: errorStore.subscribe },
|
||||
statusHint,
|
||||
setup,
|
||||
teardown,
|
||||
start,
|
||||
stop,
|
||||
toggle,
|
||||
}
|
||||
}
|
||||
@@ -1,148 +1,236 @@
|
||||
const SAMPLE_RATE = 24_000
|
||||
const BUFFER_SIZE = 4_800
|
||||
import { ConnectorLifecycleEvents, WerkRealtimeConnector, inlineWorklet } from "@kontextwerk/web-sdk"
|
||||
|
||||
const AUDIO_PLAYBACK_WORKLET_URL = "/assets/audio-playback-worklet.js"
|
||||
const AUDIO_PROCESSOR_WORKLET_URL = "/assets/audio-processor-worklet.js"
|
||||
const isBrowser = typeof window !== "undefined"
|
||||
const WS_HOST = "2svoice-server.kontextwerk.info"
|
||||
|
||||
const uint8ToBase64 = (u8: Uint8Array): string => {
|
||||
let bin = ""
|
||||
for (let i = 0; i < u8.length; i++) bin += String.fromCharCode(u8[i])
|
||||
return btoa(bin)
|
||||
}
|
||||
export const SAMPLE_RATE = 24_000
|
||||
export const BUFFER_SIZE = 4_800
|
||||
export const WS_URL = isBrowser && window.location.protocol === "http:"
|
||||
? `ws://${WS_HOST}/api/v1/voicebot/ws`
|
||||
: `wss://${WS_HOST}/api/v1/voicebot/ws`
|
||||
|
||||
const base64ToUint8 = (b64: string): Uint8Array => {
|
||||
const bin = atob(b64)
|
||||
const out = new Uint8Array(bin.length)
|
||||
for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i)
|
||||
return out
|
||||
}
|
||||
|
||||
interface NowPlayingMessage {
|
||||
type: "nowPlaying"
|
||||
item_id: string
|
||||
played_ms: number
|
||||
}
|
||||
|
||||
interface NowPlayingState {
|
||||
item_id: string | null
|
||||
played_ms: number
|
||||
}
|
||||
|
||||
interface Player {
|
||||
init: (sampleRate?: number) => Promise<void>
|
||||
play: (delta) => void
|
||||
deleteItem: (item_id: string) => void
|
||||
stop: () => void
|
||||
setSourceRate: (hz: number) => void
|
||||
getNowPlaying: () => NowPlayingState
|
||||
destroy: () => Promise<void>
|
||||
mute: () => void
|
||||
unmute: () => void
|
||||
node?: AudioWorkletNode | null
|
||||
}
|
||||
|
||||
const createPlayer = (defaultSampleRate = 48000): Player => {
|
||||
let ctx: AudioContext | null = null
|
||||
let node: AudioWorkletNode | null = null
|
||||
let nowItemId: string | null = null
|
||||
let playedMs = 0
|
||||
|
||||
const isNowPlayingMessage = (m: unknown): m is NowPlayingMessage => {
|
||||
if (!m || typeof m !== "object") return false
|
||||
const x = m as Record<string, unknown>
|
||||
return x["type"] === "nowPlaying" && "played_ms" in x
|
||||
const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super()
|
||||
this.srcRate = ${SAMPLE_RATE}
|
||||
this.dstRate = sampleRate
|
||||
this.step = this.srcRate / this.dstRate
|
||||
this.queue = []
|
||||
this.cur = null
|
||||
this.hold = 0
|
||||
this.phase = 0
|
||||
this._x0 = undefined
|
||||
this._x1 = undefined
|
||||
this._nextItemId = null
|
||||
this.nowItemId = null
|
||||
this.nowItemSamples = 0
|
||||
this._notifyFrames = 0
|
||||
this.muted = false
|
||||
this.port.onmessage = (e) => this._onMessage(e.data)
|
||||
}
|
||||
|
||||
const init = async (sampleRate = defaultSampleRate): Promise<void> => {
|
||||
ctx = new AudioContext({ sampleRate })
|
||||
await ctx.audioWorklet.addModule(AUDIO_PLAYBACK_WORKLET_URL)
|
||||
node = new AudioWorkletNode(ctx, "audio-playback-worklet")
|
||||
node.port.onmessage = (e: MessageEvent) => {
|
||||
const m = e.data
|
||||
if (isNowPlayingMessage(m)) {
|
||||
nowItemId = m.item_id
|
||||
playedMs = m.played_ms | 0
|
||||
_onMessage(msg) {
|
||||
if (!msg || !msg.type) return
|
||||
if (msg.type === "appendDelta" && msg.delta && msg.delta.pcmInt16 instanceof Int16Array) {
|
||||
this.queue.push({ item_id: msg.delta.item_id, data: msg.delta.pcmInt16, off: 0 })
|
||||
return
|
||||
}
|
||||
if (msg.type === "deleteItem") {
|
||||
const id = msg.item_id
|
||||
this.queue = this.queue.filter((ch) => ch.item_id !== id)
|
||||
if (this.cur && this.cur.item_id === id) {
|
||||
this.cur = null
|
||||
this.hold = 0
|
||||
}
|
||||
if (this.nowItemId === id) {
|
||||
this.nowItemId = null
|
||||
this.nowItemSamples = 0
|
||||
this._notifyFrames = 0
|
||||
}
|
||||
return
|
||||
}
|
||||
if (msg.type === "clear") {
|
||||
this.queue.length = 0
|
||||
this.cur = null
|
||||
this.hold = 0
|
||||
this.phase = 0
|
||||
this._x0 = undefined
|
||||
this._x1 = undefined
|
||||
this._nextItemId = null
|
||||
this.nowItemId = null
|
||||
this.nowItemSamples = 0
|
||||
this._notifyFrames = 0
|
||||
return
|
||||
}
|
||||
if (msg.type === "setSourceRate" && Number.isFinite(msg.hz) && msg.hz > 0) {
|
||||
this.srcRate = msg.hz | 0
|
||||
this.step = this.srcRate / this.dstRate
|
||||
return
|
||||
}
|
||||
if (msg.type === "mute") {
|
||||
this.muted = true
|
||||
return
|
||||
}
|
||||
if (msg.type === "unmute") {
|
||||
this.muted = false
|
||||
}
|
||||
}
|
||||
|
||||
_ensureCurrent() {
|
||||
if (this.cur == null) {
|
||||
if (this.queue.length === 0) return false
|
||||
this.cur = this.queue.shift() || null
|
||||
if (this.cur == null) return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
_nextInt16() {
|
||||
for (;;) {
|
||||
if (!this._ensureCurrent()) {
|
||||
this._nextItemId = null
|
||||
this.hold = 0
|
||||
return 0
|
||||
}
|
||||
const d = this.cur.data
|
||||
const o = this.cur.off | 0
|
||||
if (o < d.length) {
|
||||
const s = d[o]
|
||||
this.cur.off = o + 1
|
||||
this.hold = s
|
||||
this._nextItemId = this.cur.item_id
|
||||
return s
|
||||
}
|
||||
this.cur = null
|
||||
}
|
||||
}
|
||||
|
||||
process(_inputs, outputs) {
|
||||
const out = outputs[0]
|
||||
if (!out || out.length === 0) return true
|
||||
const ch0 = out[0]
|
||||
const N = ch0.length
|
||||
|
||||
if (this._x1 === undefined) {
|
||||
this._x1 = this._nextInt16()
|
||||
this._x0 = this._x1
|
||||
this.phase = 0
|
||||
this.nowItemId = this._nextItemId
|
||||
this.nowItemSamples = 0
|
||||
}
|
||||
|
||||
const advance = () => {
|
||||
this.phase += this.step
|
||||
while (this.phase >= 1) {
|
||||
this.phase -= 1
|
||||
this._x0 = this._x1
|
||||
this._x1 = this._nextInt16()
|
||||
if (this.nowItemId !== this._nextItemId) {
|
||||
this.nowItemId = this._nextItemId
|
||||
this.nowItemSamples = 0
|
||||
}
|
||||
if (this.nowItemId) this.nowItemSamples += 1
|
||||
}
|
||||
}
|
||||
node.connect(ctx.destination)
|
||||
|
||||
if (this.muted) {
|
||||
for (let i = 0; i < N; i++) {
|
||||
ch0[i] = 0
|
||||
for (let c = 1; c < out.length; c++) out[c][i] = 0
|
||||
advance()
|
||||
}
|
||||
} else {
|
||||
for (let i = 0; i < N; i++) {
|
||||
const x0 = this._x0 ?? 0
|
||||
const x1 = this._x1 ?? x0
|
||||
const value = x0 + (x1 - x0) * this.phase
|
||||
const sample = value <= -32768 ? -1 : value >= 32767 ? 1 : value / 32768
|
||||
ch0[i] = sample
|
||||
for (let c = 1; c < out.length; c++) out[c][i] = sample
|
||||
advance()
|
||||
this._notifyFrames += 1
|
||||
if (this._notifyFrames >= this.dstRate / 10) {
|
||||
this._notifyFrames = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("audio-playback-worklet", AudioPlaybackWorklet)
|
||||
`
|
||||
|
||||
const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super()
|
||||
this._inRate = sampleRate
|
||||
this._pos = 0
|
||||
this._carry = null
|
||||
this._outRate = ${SAMPLE_RATE}
|
||||
}
|
||||
|
||||
const play = (delta: ResponseAudioDelta): void => {
|
||||
if (!node) return
|
||||
const buf = delta.pcmInt16.buffer
|
||||
node.port.postMessage({ type: "appendDelta", delta }, [buf])
|
||||
}
|
||||
const deleteItem = (item_id: string): void => {
|
||||
node?.port.postMessage({ type: "deleteItem", item_id })
|
||||
}
|
||||
process(inputs) {
|
||||
const chs = inputs[0]
|
||||
if (!chs || chs.length === 0) return true
|
||||
const inF32 = chs[0]
|
||||
const step = this._inRate / this._outRate
|
||||
|
||||
const stop = (): void => {
|
||||
node?.port.postMessage({ type: "clear" })
|
||||
}
|
||||
let src = inF32
|
||||
if (this._carry !== null) {
|
||||
const tmp = new Float32Array(1 + inF32.length)
|
||||
tmp[0] = this._carry
|
||||
tmp.set(inF32, 1)
|
||||
src = tmp
|
||||
}
|
||||
|
||||
const setSourceRate = (hz: number): void => {
|
||||
node?.port.postMessage({ type: "setSourceRate", hz })
|
||||
}
|
||||
const avail = src.length - 1 - this._pos
|
||||
const outLen = avail > 0 ? Math.ceil(avail / step) : 0
|
||||
const outI16 = new Int16Array(outLen)
|
||||
|
||||
const getNowPlaying = (): NowPlayingState => {
|
||||
return { item_id: nowItemId, played_ms: playedMs }
|
||||
}
|
||||
const mute = (): void => {
|
||||
node?.port.postMessage({ type: "mute" })
|
||||
}
|
||||
const unmute = (): void => {
|
||||
node?.port.postMessage({ type: "unmute" })
|
||||
}
|
||||
let pos = this._pos
|
||||
for (let k = 0; k < outLen; k++) {
|
||||
const i = Math.floor(pos)
|
||||
const frac = pos - i
|
||||
const x0 = src[i]
|
||||
const x1 = src[i + 1]
|
||||
let y = x0 + frac * (x1 - x0)
|
||||
if (y > 1) y = 1
|
||||
else if (y < -1) y = -1
|
||||
const s = y <= -1 ? -0x8000 : Math.round(y * 0x7fff)
|
||||
outI16[k] = s
|
||||
pos += step
|
||||
}
|
||||
|
||||
this._pos = pos - (src.length - 1)
|
||||
if (this._pos < 0) this._pos = 0
|
||||
this._carry = src[src.length - 1]
|
||||
|
||||
const destroy = async (): Promise<void> => {
|
||||
if (!ctx) return
|
||||
try {
|
||||
await ctx.close()
|
||||
} finally {
|
||||
ctx = null
|
||||
node = null
|
||||
nowItemId = null
|
||||
playedMs = 0
|
||||
this.port.postMessage(outI16, [outI16.buffer])
|
||||
} catch {
|
||||
this.port.postMessage(outI16)
|
||||
}
|
||||
}
|
||||
|
||||
return { init, play, deleteItem, stop, setSourceRate, getNowPlaying, destroy, mute, unmute }
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
const createRecorder = (onChunk: (pcm: Int16Array) => void) => {
|
||||
let ctx: AudioContext | null = null
|
||||
let stream: MediaStream | null = null
|
||||
let source: MediaStreamAudioSourceNode | null = null
|
||||
let worklet: AudioWorkletNode | null = null
|
||||
registerProcessor("audio-processor-worklet", PCMAudioProcessor)
|
||||
`
|
||||
|
||||
const start = async () => {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
if (ctx) await ctx.close()
|
||||
ctx = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: SAMPLE_RATE })
|
||||
await ctx.audioWorklet.addModule(AUDIO_PROCESSOR_WORKLET_URL)
|
||||
export const playbackWorklet = inlineWorklet(playbackWorkletCode)
|
||||
export const processorWorklet = inlineWorklet(processorWorkletCode)
|
||||
|
||||
source = ctx.createMediaStreamSource(stream)
|
||||
worklet = new AudioWorkletNode(ctx, "audio-processor-worklet")
|
||||
worklet.port.onmessage = (ev: MessageEvent<Int16Array>) => onChunk(ev.data)
|
||||
export const createVoiceConnector = () =>
|
||||
new WerkRealtimeConnector({
|
||||
wsUrl: WS_URL,
|
||||
playbackWorklet,
|
||||
processorWorklet,
|
||||
sampleRate: SAMPLE_RATE,
|
||||
bufferSize: BUFFER_SIZE,
|
||||
})
|
||||
|
||||
source.connect(worklet)
|
||||
worklet.connect(ctx.destination)
|
||||
}
|
||||
const stop = async () => {
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((t) => t.stop())
|
||||
stream = null
|
||||
}
|
||||
if (ctx) {
|
||||
try {
|
||||
await ctx.close()
|
||||
} finally {
|
||||
ctx = null
|
||||
}
|
||||
}
|
||||
source = null
|
||||
worklet = null
|
||||
}
|
||||
return { start, stop }
|
||||
}
|
||||
export { uint8ToBase64, base64ToUint8, createPlayer, createRecorder,SAMPLE_RATE }
|
||||
export { ConnectorLifecycleEvents }
|
||||
export type { WerkRealtimeConnector }
|
||||
|
||||
Reference in New Issue
Block a user