sdk zwischenstand
This commit is contained in:
@@ -1,10 +1,9 @@
|
||||
<script lang="ts">
|
||||
import { onDestroy } from "svelte"
|
||||
import { onMount } from "svelte"
|
||||
import { mdiBookAccountOutline, mdiCreation, mdiFaceAgent, mdiHours24 } from "@mdi/js"
|
||||
import ProductCategoryFrame from "../widgets/ProductCategoryFrame.svelte"
|
||||
import CrinkledSection from "../CrinkledSection.svelte"
|
||||
import { base64ToUint8, createPlayer, createRecorder, SAMPLE_RATE, uint8ToBase64 } from "../voicebotDemo/helper"
|
||||
import { RealtimeServerEvent as RSE } from "../voicebotDemo/events"
|
||||
import { createVoicebotPreviewController } from "./voicebotPreviewController"
|
||||
|
||||
const voiceProperties: Array<{ title: string; icon: string; color: string }> = [
|
||||
{
|
||||
@@ -29,300 +28,18 @@
|
||||
},
|
||||
]
|
||||
|
||||
const browser = typeof window !== "undefined"
|
||||
const VOICE_WS_URL =
|
||||
browser && window.location.protocol === "http:"
|
||||
? "ws://2svoice-server.kontextwerk.info/api/v1/voicebot/ws"
|
||||
: "wss://2svoice-server.kontextwerk.info/api/v1/voicebot/ws"
|
||||
const CHUNK_DURATION_MS = 200
|
||||
const CHUNK_SIZE_BYTES = Math.round((SAMPLE_RATE * CHUNK_DURATION_MS) / 1000) * 2
|
||||
|
||||
type VoiceStatus = "idle" | "connecting" | "connected" | "error"
|
||||
|
||||
let status: VoiceStatus = "idle"
|
||||
let errorMessage = ""
|
||||
|
||||
let ws: WebSocket | null = null
|
||||
let recorder: ReturnType<typeof createRecorder> | null = null
|
||||
let player: ReturnType<typeof createPlayer> | null = null
|
||||
let outboundBuffer = new Uint8Array(0)
|
||||
let closing = false
|
||||
let cleanupPromise: Promise<void> | null = null
|
||||
let startPromise: Promise<void> | null = null
|
||||
|
||||
$: statusHint =
|
||||
status === "idle"
|
||||
? "Tippen, um die Voice-Demo zu starten"
|
||||
: status === "connecting"
|
||||
? "Verbindung wird aufgebaut …"
|
||||
: status === "connected"
|
||||
? "Live – sprechen Sie jetzt"
|
||||
: errorMessage || "Verbindung fehlgeschlagen"
|
||||
|
||||
const toggleVoiceDemo = async () => {
|
||||
if (status === "connecting") return
|
||||
if (status === "connected") {
|
||||
await stopVoiceDemo()
|
||||
return
|
||||
}
|
||||
await startVoiceDemo()
|
||||
}
|
||||
const controller = createVoicebotPreviewController()
|
||||
const { status, statusHint, toggle, setup, teardown } = controller
|
||||
|
||||
const handleKeydown = (event: KeyboardEvent) => {
|
||||
if (event.key !== "Enter" && event.key !== " ") return
|
||||
event.preventDefault()
|
||||
void toggleVoiceDemo()
|
||||
void toggle()
|
||||
}
|
||||
|
||||
const startVoiceDemo = async () => {
|
||||
if (!browser) {
|
||||
status = "error"
|
||||
errorMessage = "Die Sprach-Demo steht nur im Browser zur Verfügung."
|
||||
return
|
||||
}
|
||||
if (startPromise || status === "connecting" || status === "connected") return
|
||||
|
||||
startPromise = (async () => {
|
||||
await stopVoiceDemo({ resetStatus: false })
|
||||
status = "connecting"
|
||||
errorMessage = ""
|
||||
outboundBuffer = new Uint8Array(0)
|
||||
closing = false
|
||||
|
||||
try {
|
||||
const newPlayer = createPlayer()
|
||||
await newPlayer.init()
|
||||
player = newPlayer
|
||||
|
||||
const handleChunk = (pcm: Int16Array) => {
|
||||
if (pcm.length === 0) return
|
||||
const bytes = new Uint8Array(pcm.byteLength)
|
||||
bytes.set(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength))
|
||||
appendToOutboundBuffer(bytes)
|
||||
}
|
||||
|
||||
const newRecorder = createRecorder(handleChunk)
|
||||
await newRecorder.start()
|
||||
recorder = newRecorder
|
||||
} catch (err) {
|
||||
const message = extractErrorMessage(err, "Mikrofon konnte nicht gestartet werden.")
|
||||
handleConnectionError(message, err)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
ws = new WebSocket(VOICE_WS_URL)
|
||||
} catch (err) {
|
||||
const message = extractErrorMessage(err, "WebSocket-Verbindung konnte nicht aufgebaut werden.")
|
||||
handleConnectionError(message, err)
|
||||
return
|
||||
}
|
||||
|
||||
if (!ws) return
|
||||
|
||||
ws.onopen = () => {
|
||||
status = "connected"
|
||||
flushOutboundBuffer(true)
|
||||
}
|
||||
|
||||
ws.onmessage = (event) => handleServerMessage(event)
|
||||
|
||||
ws.onerror = (event) => {
|
||||
handleConnectionError("WebSocket-Fehler – bitte später erneut versuchen.", event)
|
||||
}
|
||||
|
||||
ws.onclose = () => {
|
||||
if (!closing && status === "connected") {
|
||||
status = "idle"
|
||||
errorMessage = ""
|
||||
}
|
||||
}
|
||||
})()
|
||||
|
||||
try {
|
||||
await startPromise
|
||||
} finally {
|
||||
startPromise = null
|
||||
}
|
||||
}
|
||||
|
||||
const stopVoiceDemo = async ({ resetStatus = true }: { resetStatus?: boolean } = {}) => {
|
||||
if (cleanupPromise) {
|
||||
await cleanupPromise
|
||||
if (resetStatus && status !== "error") {
|
||||
status = "idle"
|
||||
errorMessage = ""
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
closing = true
|
||||
|
||||
cleanupPromise = (async () => {
|
||||
try {
|
||||
flushOutboundBuffer(true)
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
if (recorder) {
|
||||
try {
|
||||
await recorder.stop()
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
recorder = null
|
||||
|
||||
if (player) {
|
||||
try {
|
||||
player.stop()
|
||||
await player.destroy()
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
player = null
|
||||
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
try {
|
||||
ws.close(1000, "client-stop")
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
ws = null
|
||||
outboundBuffer = new Uint8Array(0)
|
||||
})()
|
||||
|
||||
try {
|
||||
await cleanupPromise
|
||||
} finally {
|
||||
cleanupPromise = null
|
||||
closing = false
|
||||
if (resetStatus && status !== "error") {
|
||||
status = "idle"
|
||||
errorMessage = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const handleServerMessage = (event: MessageEvent) => {
|
||||
if (!player) return
|
||||
let payload: unknown = event.data
|
||||
|
||||
if (typeof payload !== "string") return
|
||||
|
||||
try {
|
||||
payload = JSON.parse(payload)
|
||||
} catch (err) {
|
||||
console.warn("VoiceBot Preview: Konnte Nachricht nicht parsen.", err)
|
||||
return
|
||||
}
|
||||
|
||||
const message = payload as Record<string, unknown>
|
||||
const type = typeof message.type === "string" ? message.type : "<unbekannt>"
|
||||
if (type === RSE.INPUT_AUDIO_BUFFER_SPEECH_STARTED) {
|
||||
const { item_id, played_ms } = player.getNowPlaying()
|
||||
if (item_id) {
|
||||
player.stop()
|
||||
ws?.send(
|
||||
JSON.stringify({
|
||||
type: "last_item_played_ms.truncate",
|
||||
details: { item_id, played_ms: played_ms || 0 },
|
||||
})
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (type === RSE.RESPONSE_AUDIO_DELTA) {
|
||||
const bytes = base64ToUint8((message as any).delta)
|
||||
const pcm = new Int16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2)
|
||||
player.play({
|
||||
response_id: message.response_id,
|
||||
item_id: message.item_id,
|
||||
delta: message.delta,
|
||||
pcmInt16: pcm,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (type === "error") {
|
||||
console.log("VoiceBot Preview: Server hat einen Fehler gemeldet.", message)
|
||||
}
|
||||
}
|
||||
|
||||
const appendToOutboundBuffer = (chunk: Uint8Array) => {
|
||||
if (!chunk.length) return
|
||||
const combined = new Uint8Array(outboundBuffer.length + chunk.length)
|
||||
combined.set(outboundBuffer)
|
||||
combined.set(chunk, outboundBuffer.length)
|
||||
outboundBuffer = combined
|
||||
flushOutboundBuffer()
|
||||
}
|
||||
|
||||
const flushOutboundBuffer = (force = false) => {
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN || outboundBuffer.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const chunkSize = CHUNK_SIZE_BYTES > 0 ? CHUNK_SIZE_BYTES : outboundBuffer.length
|
||||
let buffer = outboundBuffer
|
||||
outboundBuffer = new Uint8Array(0)
|
||||
|
||||
while (buffer.length >= chunkSize && chunkSize > 0) {
|
||||
const part = buffer.slice(0, chunkSize)
|
||||
buffer = buffer.slice(chunkSize)
|
||||
sendChunk(part)
|
||||
}
|
||||
|
||||
if (force && buffer.length > 0) {
|
||||
sendChunk(buffer)
|
||||
} else if (buffer.length > 0) {
|
||||
outboundBuffer = buffer
|
||||
}
|
||||
}
|
||||
|
||||
const sendChunk = (chunk: Uint8Array) => {
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN) return
|
||||
|
||||
try {
|
||||
ws.send(JSON.stringify({ type: "input_audio_buffer.append", audio: uint8ToBase64(chunk) }))
|
||||
} catch (err) {
|
||||
if (!closing) {
|
||||
handleConnectionError("Senden des Audiostreams fehlgeschlagen.", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const handleConnectionError = (message: string, err?: unknown) => {
|
||||
console.error("VoiceBot Preview Fehler:", err ?? message)
|
||||
errorMessage = message
|
||||
status = "error"
|
||||
void stopVoiceDemo({ resetStatus: false })
|
||||
}
|
||||
|
||||
const extractErrorMessage = (err: unknown, fallback: string) => {
|
||||
if (err instanceof DOMException) {
|
||||
if (err.name === "NotAllowedError") {
|
||||
return "Zugriff auf das Mikrofon wurde verweigert."
|
||||
}
|
||||
if (err.name === "NotFoundError") {
|
||||
return "Kein Mikrofon gefunden oder verfügbar."
|
||||
}
|
||||
if (err.name === "NotReadableError") {
|
||||
return "Auf das Mikrofon konnte nicht zugegriffen werden (ggf. bereits in Verwendung)."
|
||||
}
|
||||
if (err.name === "SecurityError") {
|
||||
return "Der Browser blockiert den Zugriff – bitte die Seite über HTTPS öffnen."
|
||||
}
|
||||
}
|
||||
if (err instanceof Error && err.message) return err.message
|
||||
return fallback
|
||||
}
|
||||
|
||||
onDestroy(() => {
|
||||
void stopVoiceDemo({ resetStatus: false })
|
||||
onMount(() => {
|
||||
setup()
|
||||
return () => teardown()
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -341,26 +58,27 @@
|
||||
{#snippet primaryContent()}
|
||||
<div
|
||||
class="img"
|
||||
class:connected={status === "connected"}
|
||||
class:errored={status === "error"}
|
||||
class:connected={$status === "connected"}
|
||||
class:errored={$status === "error"}
|
||||
role="button"
|
||||
tabindex="0"
|
||||
aria-pressed={status === "connected"}
|
||||
aria-busy={status === "connecting"}
|
||||
aria-pressed={$status === "connected"}
|
||||
aria-busy={$status === "connecting"}
|
||||
aria-label="Voicebot Demo starten"
|
||||
on:click={() => void toggleVoiceDemo()}
|
||||
on:click={() => void toggle()}
|
||||
on:keydown={handleKeydown}
|
||||
>
|
||||
<img
|
||||
src="/media/iphone.png"
|
||||
alt="Kontextwerk is calling"
|
||||
/>
|
||||
<div class="shadow"></div>
|
||||
<div
|
||||
class="voice-overlay"
|
||||
data-status={status}
|
||||
data-status={$status}
|
||||
aria-live="polite"
|
||||
>
|
||||
<span>{statusHint}</span>
|
||||
<span>{$statusHint}</span>
|
||||
</div>
|
||||
</div>
|
||||
{/snippet}
|
||||
@@ -432,11 +150,9 @@
|
||||
}
|
||||
|
||||
&.connected {
|
||||
border-color: rgba(76, 175, 80, 0.4);
|
||||
}
|
||||
|
||||
&.errored {
|
||||
border-color: rgba(235, 87, 87, 0.45);
|
||||
}
|
||||
|
||||
&:focus-visible {
|
||||
|
||||
Reference in New Issue
Block a user