From 2be72187ecca60e866c1cec23ce5f404ddff0b3f Mon Sep 17 00:00:00 2001 From: robin Date: Sun, 5 Oct 2025 12:55:59 +0000 Subject: [PATCH] sdk zwischenstand --- api/collections/contact.yml | 60 +-- api/hooks/config.js | 8 +- api/hooks/contact/post_create.js | 7 +- api/hooks/contact/post_return.js | 2 +- .../staticPageRows/VoicebotPreview.svelte | 316 +--------------- .../voicebotPreviewController.ts | 207 +++++++++++ .../src/lib/components/voicebotDemo/helper.ts | 342 +++++++++++------- package.json | 1 + yarn.lock | 17 + 9 files changed, 468 insertions(+), 492 deletions(-) create mode 100644 frontend/src/lib/components/staticPageRows/voicebotPreviewController.ts diff --git a/api/collections/contact.yml b/api/collections/contact.yml index 0584605..a5fe7c4 100644 --- a/api/collections/contact.yml +++ b/api/collections/contact.yml @@ -21,7 +21,6 @@ meta: - type: simpleList mediaQuery: "(max-width:599px)" primaryText: status - secondaryText: request.email # Desktop - type: table @@ -32,21 +31,7 @@ meta: de: Status en: Status filter: true - - source: request.email - label: - de: E-Mail - en: E-Mail - filter: true - - source: request.name - label: - de: Name - en: Name - filter: true - - source: request.description - label: - de: Beschreibung - en: Description - filter: true + - source: insertTime label: de: Erstellt am @@ -138,48 +123,15 @@ fields: id: inProgress - name: abgeschlossen id: done - - name: request type: object meta: - label: - de: Anfrage - en: Request - subFields: - - name: email - type: string - meta: - label: - de: E-Mail - en: E-Mail - containerProps: - layout: - size: - default: "col-6" - small: "col-12" - large: "col-6" + label: + de: Anfrage + en: Request + widget: jsonField - - name: name - type: string - meta: - label: - de: Name - en: Name - containerProps: - layout: - size: - default: "col-6" - small: "col-12" - large: "col-6" - - - name: description - type: string - meta: - label: - de: Beschreibung - en: Description - inputProps: - multiline: true + indexes: - name: fulltext # Ein eindeutiger Name für den Index. Es ist optional, wird jedoch empfohlen, um den Index später leicht identifizieren zu können. key: # Bestimmt, auf welche Felder der Index angewendet werden soll. Dies kann ein einfacher String sein, wenn der Index nur ein Feld umfasst, oder ein Array von Strings, wenn der Index mehrere Felder umfasst. diff --git a/api/hooks/config.js b/api/hooks/config.js index 2115e60..7eb3f94 100644 --- a/api/hooks/config.js +++ b/api/hooks/config.js @@ -1,10 +1,10 @@ const apiSsrBaseURL = "http://localhost:8080/api/v1/_/bkdf_tibi_2024/" const { frontendBase, tibiUrl } = require("./config-client") module.exports = { - operatorEmail: "info@binkrassdufass.de", - operatorName: "BinKrassDuFass", - contactEmail: "support@binkrassdufass.de", - noReplyEmail: "noreply@binkrassdufass.de", + operatorEmail: "about@kontextwerk.info", + operatorName: "KontextWerk", + contactEmail: "about@kontextwerk.info", + noReplyEmail: "about@kontextwerk.info", frontendBase, apiBase: frontendBase + "/api/", tibiUrl, diff --git a/api/hooks/contact/post_create.js b/api/hooks/contact/post_create.js index 0306679..caabf14 100644 --- a/api/hooks/contact/post_create.js +++ b/api/hooks/contact/post_create.js @@ -1,8 +1,3 @@ -const { cryptchaCheck } = require("../lib/utils") ;(function () { - throw { - status: 500, - data: "Hello, World!", - } - cryptchaCheck() + })() diff --git a/api/hooks/contact/post_return.js b/api/hooks/contact/post_return.js index fe6cbe4..d071346 100644 --- a/api/hooks/contact/post_return.js +++ b/api/hooks/contact/post_return.js @@ -3,7 +3,7 @@ const { noReplyEmail, contactEmail } = require("../config") context.smtp.sendMail({ to: contactEmail, from: noReplyEmail, - fromName: "BinKrassDuFass", + fromName: "Kontextwerk", replyTo: noReplyEmail, subject: "New Contact Request", plain: "Neue Kontaktanfrage", diff --git a/frontend/src/lib/components/staticPageRows/VoicebotPreview.svelte b/frontend/src/lib/components/staticPageRows/VoicebotPreview.svelte index f02f7f0..b1ba328 100644 --- a/frontend/src/lib/components/staticPageRows/VoicebotPreview.svelte +++ b/frontend/src/lib/components/staticPageRows/VoicebotPreview.svelte @@ -1,10 +1,9 @@ @@ -341,26 +58,27 @@ {#snippet primaryContent()}
void toggleVoiceDemo()} + on:click={() => void toggle()} on:keydown={handleKeydown} > Kontextwerk is calling +
- {statusHint} + {$statusHint}
{/snippet} @@ -432,11 +150,9 @@ } &.connected { - border-color: rgba(76, 175, 80, 0.4); } &.errored { - border-color: rgba(235, 87, 87, 0.45); } &:focus-visible { diff --git a/frontend/src/lib/components/staticPageRows/voicebotPreviewController.ts b/frontend/src/lib/components/staticPageRows/voicebotPreviewController.ts new file mode 100644 index 0000000..0778aac --- /dev/null +++ b/frontend/src/lib/components/staticPageRows/voicebotPreviewController.ts @@ -0,0 +1,207 @@ +import { derived, get, writable, type Readable } from "svelte/store" +import { + ConnectorLifecycleEvents, + createVoiceConnector, + WS_URL, +} from "../voicebotDemo/helper" +import type { WerkRealtimeConnector } from "../voicebotDemo/helper" + +export type VoiceStatus = "idle" | "connecting" | "connected" | "error" + +interface VoicebotPreviewController { + status: Readable + errorMessage: Readable + statusHint: Readable + setup: () => void + teardown: () => void + start: () => Promise + stop: (silent?: boolean) => Promise + toggle: () => Promise +} + +const isBrowser = typeof window !== "undefined" + +const extractErrorMessage = (err: unknown, fallback: string) => { + if (err instanceof DOMException) { + if (err.name === "NotAllowedError") return "Zugriff auf das Mikrofon wurde verweigert." + if (err.name === "NotFoundError") return "Kein Mikrofon gefunden oder verfügbar." + if (err.name === "NotReadableError") return "Auf das Mikrofon konnte nicht zugegriffen werden." + if (err.name === "SecurityError") return "Der Browser blockiert den Zugriff – bitte die Seite über HTTPS öffnen." + } + if (err instanceof Error && err.message) return err.message + return fallback +} + +export const createVoicebotPreviewController = (): VoicebotPreviewController => { + const statusStore = writable("idle") + const errorStore = writable("") + + const statusHint = derived([statusStore, errorStore], ([$status, $error]) => { + switch ($status) { + case "idle": + return "Tippen, um die Voice-Demo zu starten" + case "connecting": + return "Verbindung wird aufgebaut …" + case "connected": + return "Live – sprechen Sie jetzt" + case "error": + return $error || "Verbindung fehlgeschlagen" + default: + return "Voice-Demo" + } + }) + + let connector: WerkRealtimeConnector | null = null + let detachHandlers: Array<() => void> = [] + let startPromise: Promise | null = null + let stopPromise: Promise | null = null + let closing = false + const handleRealtimeEvent = (rawType: string, msg: any) => { + const type = rawType + if (!type) return + + const now = new Date() + + } + const ensureConnector = () => { + if (!isBrowser || connector) return + + const instance = createVoiceConnector() + + const detachConnecting = instance.onLifecycle(ConnectorLifecycleEvents.CONNECTING, () => { + statusStore.set("connecting") + errorStore.set("") + }) + const detachConnected = instance.onLifecycle(ConnectorLifecycleEvents.CONNECTED, () => { + statusStore.set("connected") + closing = false + }) + const detachDisconnected = instance.onLifecycle(ConnectorLifecycleEvents.DISCONNECTED, () => { + if (closing) { + closing = false + return + } + if (get(statusStore) !== "error") { + statusStore.set("idle") + errorStore.set("") + } + }) + const detachError = instance.onLifecycle(ConnectorLifecycleEvents.ERROR, (evt) => { + const message = + typeof evt?.message === "string" && evt.message.trim().length + ? evt.message + : "Verbindung fehlgeschlagen" + errorStore.set(`${message} (${WS_URL})`) + statusStore.set("error") + closing = false + }) + + detachHandlers = [detachConnecting, detachConnected, detachDisconnected, detachError] + connector = instance + } + + const cleanupConnector = () => { + detachHandlers.forEach((fn) => fn()) + detachHandlers = [] + const instance = connector + connector = null + if (instance) { + void instance.stop() + } + startPromise = null + stopPromise = null + closing = false + } + + const stop = async (silent = false) => { + if (!connector) return + if (stopPromise) { + await stopPromise + return + } + + closing = true + if (!silent && get(statusStore) !== "error") { + statusStore.set("idle") + errorStore.set("") + } + + stopPromise = connector.stop() + + try { + await stopPromise + } catch (err) { + console.error("VoicebotPreview stop error", err) + if (!silent) { + errorStore.set(extractErrorMessage(err, "Verbindung konnte nicht beendet werden.")) + statusStore.set("error") + } + } finally { + stopPromise = null + closing = false + } + } + + const start = async () => { + if (!isBrowser) { + statusStore.set("error") + errorStore.set("Die Sprach-Demo steht nur im Browser zur Verfügung.") + return + } + + ensureConnector() + if (!connector || startPromise) return + + await stop(true) + + statusStore.set("connecting") + errorStore.set("") + + startPromise = (async () => { + await connector!.start() + connector!.setInputMuted(false) + connector!.setOutputMuted(false) + })() + + try { + await startPromise + } catch (err) { + console.error("VoicebotPreview start error", err) + errorStore.set(extractErrorMessage(err, "Verbindung konnte nicht aufgebaut werden.")) + statusStore.set("error") + closing = false + } finally { + startPromise = null + } + } + + const toggle = async () => { + const current = get(statusStore) + if (current === "connecting") return + if (current === "connected") { + await stop() + return + } + await start() + } + + const setup = () => { + ensureConnector() + } + + const teardown = () => { + void stop(true) + cleanupConnector() + } + + return { + status: { subscribe: statusStore.subscribe }, + errorMessage: { subscribe: errorStore.subscribe }, + statusHint, + setup, + teardown, + start, + stop, + toggle, + } +} diff --git a/frontend/src/lib/components/voicebotDemo/helper.ts b/frontend/src/lib/components/voicebotDemo/helper.ts index af66a22..25ea51e 100644 --- a/frontend/src/lib/components/voicebotDemo/helper.ts +++ b/frontend/src/lib/components/voicebotDemo/helper.ts @@ -1,148 +1,236 @@ -const SAMPLE_RATE = 24_000 -const BUFFER_SIZE = 4_800 +import { ConnectorLifecycleEvents, WerkRealtimeConnector, inlineWorklet } from "@kontextwerk/web-sdk" -const AUDIO_PLAYBACK_WORKLET_URL = "/assets/audio-playback-worklet.js" -const AUDIO_PROCESSOR_WORKLET_URL = "/assets/audio-processor-worklet.js" +const isBrowser = typeof window !== "undefined" +const WS_HOST = "2svoice-server.kontextwerk.info" -const uint8ToBase64 = (u8: Uint8Array): string => { - let bin = "" - for (let i = 0; i < u8.length; i++) bin += String.fromCharCode(u8[i]) - return btoa(bin) -} +export const SAMPLE_RATE = 24_000 +export const BUFFER_SIZE = 4_800 +export const WS_URL = isBrowser && window.location.protocol === "http:" + ? `ws://${WS_HOST}/api/v1/voicebot/ws` + : `wss://${WS_HOST}/api/v1/voicebot/ws` -const base64ToUint8 = (b64: string): Uint8Array => { - const bin = atob(b64) - const out = new Uint8Array(bin.length) - for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i) - return out -} - -interface NowPlayingMessage { - type: "nowPlaying" - item_id: string - played_ms: number -} - -interface NowPlayingState { - item_id: string | null - played_ms: number -} - -interface Player { - init: (sampleRate?: number) => Promise - play: (delta) => void - deleteItem: (item_id: string) => void - stop: () => void - setSourceRate: (hz: number) => void - getNowPlaying: () => NowPlayingState - destroy: () => Promise - mute: () => void - unmute: () => void - node?: AudioWorkletNode | null -} - -const createPlayer = (defaultSampleRate = 48000): Player => { - let ctx: AudioContext | null = null - let node: AudioWorkletNode | null = null - let nowItemId: string | null = null - let playedMs = 0 - - const isNowPlayingMessage = (m: unknown): m is NowPlayingMessage => { - if (!m || typeof m !== "object") return false - const x = m as Record - return x["type"] === "nowPlaying" && "played_ms" in x +const playbackWorkletCode = `class AudioPlaybackWorklet extends AudioWorkletProcessor { + constructor() { + super() + this.srcRate = ${SAMPLE_RATE} + this.dstRate = sampleRate + this.step = this.srcRate / this.dstRate + this.queue = [] + this.cur = null + this.hold = 0 + this.phase = 0 + this._x0 = undefined + this._x1 = undefined + this._nextItemId = null + this.nowItemId = null + this.nowItemSamples = 0 + this._notifyFrames = 0 + this.muted = false + this.port.onmessage = (e) => this._onMessage(e.data) } - const init = async (sampleRate = defaultSampleRate): Promise => { - ctx = new AudioContext({ sampleRate }) - await ctx.audioWorklet.addModule(AUDIO_PLAYBACK_WORKLET_URL) - node = new AudioWorkletNode(ctx, "audio-playback-worklet") - node.port.onmessage = (e: MessageEvent) => { - const m = e.data - if (isNowPlayingMessage(m)) { - nowItemId = m.item_id - playedMs = m.played_ms | 0 + _onMessage(msg) { + if (!msg || !msg.type) return + if (msg.type === "appendDelta" && msg.delta && msg.delta.pcmInt16 instanceof Int16Array) { + this.queue.push({ item_id: msg.delta.item_id, data: msg.delta.pcmInt16, off: 0 }) + return + } + if (msg.type === "deleteItem") { + const id = msg.item_id + this.queue = this.queue.filter((ch) => ch.item_id !== id) + if (this.cur && this.cur.item_id === id) { + this.cur = null + this.hold = 0 + } + if (this.nowItemId === id) { + this.nowItemId = null + this.nowItemSamples = 0 + this._notifyFrames = 0 + } + return + } + if (msg.type === "clear") { + this.queue.length = 0 + this.cur = null + this.hold = 0 + this.phase = 0 + this._x0 = undefined + this._x1 = undefined + this._nextItemId = null + this.nowItemId = null + this.nowItemSamples = 0 + this._notifyFrames = 0 + return + } + if (msg.type === "setSourceRate" && Number.isFinite(msg.hz) && msg.hz > 0) { + this.srcRate = msg.hz | 0 + this.step = this.srcRate / this.dstRate + return + } + if (msg.type === "mute") { + this.muted = true + return + } + if (msg.type === "unmute") { + this.muted = false + } + } + + _ensureCurrent() { + if (this.cur == null) { + if (this.queue.length === 0) return false + this.cur = this.queue.shift() || null + if (this.cur == null) return false + } + return true + } + + _nextInt16() { + for (;;) { + if (!this._ensureCurrent()) { + this._nextItemId = null + this.hold = 0 + return 0 + } + const d = this.cur.data + const o = this.cur.off | 0 + if (o < d.length) { + const s = d[o] + this.cur.off = o + 1 + this.hold = s + this._nextItemId = this.cur.item_id + return s + } + this.cur = null + } + } + + process(_inputs, outputs) { + const out = outputs[0] + if (!out || out.length === 0) return true + const ch0 = out[0] + const N = ch0.length + + if (this._x1 === undefined) { + this._x1 = this._nextInt16() + this._x0 = this._x1 + this.phase = 0 + this.nowItemId = this._nextItemId + this.nowItemSamples = 0 + } + + const advance = () => { + this.phase += this.step + while (this.phase >= 1) { + this.phase -= 1 + this._x0 = this._x1 + this._x1 = this._nextInt16() + if (this.nowItemId !== this._nextItemId) { + this.nowItemId = this._nextItemId + this.nowItemSamples = 0 + } + if (this.nowItemId) this.nowItemSamples += 1 } } - node.connect(ctx.destination) + + if (this.muted) { + for (let i = 0; i < N; i++) { + ch0[i] = 0 + for (let c = 1; c < out.length; c++) out[c][i] = 0 + advance() + } + } else { + for (let i = 0; i < N; i++) { + const x0 = this._x0 ?? 0 + const x1 = this._x1 ?? x0 + const value = x0 + (x1 - x0) * this.phase + const sample = value <= -32768 ? -1 : value >= 32767 ? 1 : value / 32768 + ch0[i] = sample + for (let c = 1; c < out.length; c++) out[c][i] = sample + advance() + this._notifyFrames += 1 + if (this._notifyFrames >= this.dstRate / 10) { + this._notifyFrames = 0 + } + } + } + + return true + } +} + +registerProcessor("audio-playback-worklet", AudioPlaybackWorklet) +` + +const processorWorkletCode = `class PCMAudioProcessor extends AudioWorkletProcessor { + constructor() { + super() + this._inRate = sampleRate + this._pos = 0 + this._carry = null + this._outRate = ${SAMPLE_RATE} } - const play = (delta: ResponseAudioDelta): void => { - if (!node) return - const buf = delta.pcmInt16.buffer - node.port.postMessage({ type: "appendDelta", delta }, [buf]) - } - const deleteItem = (item_id: string): void => { - node?.port.postMessage({ type: "deleteItem", item_id }) - } + process(inputs) { + const chs = inputs[0] + if (!chs || chs.length === 0) return true + const inF32 = chs[0] + const step = this._inRate / this._outRate - const stop = (): void => { - node?.port.postMessage({ type: "clear" }) - } + let src = inF32 + if (this._carry !== null) { + const tmp = new Float32Array(1 + inF32.length) + tmp[0] = this._carry + tmp.set(inF32, 1) + src = tmp + } - const setSourceRate = (hz: number): void => { - node?.port.postMessage({ type: "setSourceRate", hz }) - } + const avail = src.length - 1 - this._pos + const outLen = avail > 0 ? Math.ceil(avail / step) : 0 + const outI16 = new Int16Array(outLen) - const getNowPlaying = (): NowPlayingState => { - return { item_id: nowItemId, played_ms: playedMs } - } - const mute = (): void => { - node?.port.postMessage({ type: "mute" }) - } - const unmute = (): void => { - node?.port.postMessage({ type: "unmute" }) - } + let pos = this._pos + for (let k = 0; k < outLen; k++) { + const i = Math.floor(pos) + const frac = pos - i + const x0 = src[i] + const x1 = src[i + 1] + let y = x0 + frac * (x1 - x0) + if (y > 1) y = 1 + else if (y < -1) y = -1 + const s = y <= -1 ? -0x8000 : Math.round(y * 0x7fff) + outI16[k] = s + pos += step + } + + this._pos = pos - (src.length - 1) + if (this._pos < 0) this._pos = 0 + this._carry = src[src.length - 1] - const destroy = async (): Promise => { - if (!ctx) return try { - await ctx.close() - } finally { - ctx = null - node = null - nowItemId = null - playedMs = 0 + this.port.postMessage(outI16, [outI16.buffer]) + } catch { + this.port.postMessage(outI16) } - } - return { init, play, deleteItem, stop, setSourceRate, getNowPlaying, destroy, mute, unmute } + return true + } } -const createRecorder = (onChunk: (pcm: Int16Array) => void) => { - let ctx: AudioContext | null = null - let stream: MediaStream | null = null - let source: MediaStreamAudioSourceNode | null = null - let worklet: AudioWorkletNode | null = null +registerProcessor("audio-processor-worklet", PCMAudioProcessor) +` - const start = async () => { - stream = await navigator.mediaDevices.getUserMedia({ audio: true }) - if (ctx) await ctx.close() - ctx = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: SAMPLE_RATE }) - await ctx.audioWorklet.addModule(AUDIO_PROCESSOR_WORKLET_URL) +export const playbackWorklet = inlineWorklet(playbackWorkletCode) +export const processorWorklet = inlineWorklet(processorWorkletCode) - source = ctx.createMediaStreamSource(stream) - worklet = new AudioWorkletNode(ctx, "audio-processor-worklet") - worklet.port.onmessage = (ev: MessageEvent) => onChunk(ev.data) +export const createVoiceConnector = () => + new WerkRealtimeConnector({ + wsUrl: WS_URL, + playbackWorklet, + processorWorklet, + sampleRate: SAMPLE_RATE, + bufferSize: BUFFER_SIZE, + }) - source.connect(worklet) - worklet.connect(ctx.destination) - } - const stop = async () => { - if (stream) { - stream.getTracks().forEach((t) => t.stop()) - stream = null - } - if (ctx) { - try { - await ctx.close() - } finally { - ctx = null - } - } - source = null - worklet = null - } - return { start, stop } -} -export { uint8ToBase64, base64ToUint8, createPlayer, createRecorder,SAMPLE_RATE } +export { ConnectorLifecycleEvents } +export type { WerkRealtimeConnector } diff --git a/package.json b/package.json index 169397b..dd0d071 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "typescript": "^5.8.3" }, "dependencies": { + "@kontextwerk/web-sdk": "0.1.0", "@mdi/js": "^7.4.47", "@microsoft/fetch-event-source": "^2.0.1", "@okrad/svelte-progressbar": "^2.2.0", diff --git a/yarn.lock b/yarn.lock index e623b9a..d85b2f0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1779,6 +1779,15 @@ __metadata: languageName: node linkType: hard +"@kontextwerk/web-sdk@npm:0.1.0": + version: 0.1.0 + resolution: "@kontextwerk/web-sdk@npm:0.1.0" + dependencies: + zod: ^3.23.8 + checksum: b99d6a71584c1db40ab2eb83f5b269bca6a70c4eec4c644eed0873e909c08fbae9600556599b7012ff13fb6150086687612c59c98c2f03699ad8559302090f83 + languageName: node + linkType: hard + "@mdi/js@npm:^7.0.96, @mdi/js@npm:^7.4.47": version: 7.4.47 resolution: "@mdi/js@npm:7.4.47" @@ -6087,6 +6096,7 @@ __metadata: "@babel/core": ^7.27.1 "@babel/plugin-transform-async-to-generator": ^7.27.1 "@babel/preset-env": ^7.27.2 + "@kontextwerk/web-sdk": 0.1.0 "@mdi/js": ^7.4.47 "@microsoft/fetch-event-source": ^2.0.1 "@okrad/svelte-progressbar": ^2.2.0 @@ -7364,3 +7374,10 @@ __metadata: checksum: f7917916db73ad09c4870dc7045fdefb9f0122257878ec53e75ff6ea633718369b99185a21aae1fed1d258e7d66d95080169ef1a386c599b8b912467f17932bc languageName: node linkType: hard + +"zod@npm:^3.23.8": + version: 3.25.76 + resolution: "zod@npm:3.25.76" + checksum: c9a403a62b329188a5f6bd24d5d935d2bba345f7ab8151d1baa1505b5da9f227fb139354b043711490c798e91f3df75991395e40142e6510a4b16409f302b849 + languageName: node + linkType: hard