diff --git a/env-example/.env.local.example b/env-example/.env.local.example index e3b67c2a..098cccae 100644 --- a/env-example/.env.local.example +++ b/env-example/.env.local.example @@ -5,6 +5,7 @@ NEXTAUTH_URL=https://frontend-test.banlv-ai.com NEXTAUTH_URL_INTERNAL=http://localhost:3000 NEXT_PUBLIC_API_BASE_URL=https://proapi.banlv-ai.com NEXT_PUBLIC_WS_BASE_URL=wss://proapi.banlv-ai.com/ws +NEXT_PUBLIC_ENABLE_VOICE_CALL=false NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_test_51Te8ybEJDtUGxQHo0UMySJ2hW0vVryynzllmIUI3UQdQCFgxNLci0Jmm2lQkRsTaIP7C7IQlFzfFyBJ5rOhr1ML800G8P8DF5R NEXT_PUBLIC_API_CONNECT_TIMEOUT=30000 diff --git a/env-example/.env.production.example b/env-example/.env.production.example index 8538f57b..bb0c3fdc 100644 --- a/env-example/.env.production.example +++ b/env-example/.env.production.example @@ -10,6 +10,7 @@ NEXT_PUBLIC_API_BASE_URL=https://api.banlv-ai.com # WebSocket URL（生产环境） NEXT_PUBLIC_WS_BASE_URL=wss://api.banlv-ai.com/ws +NEXT_PUBLIC_ENABLE_VOICE_CALL=false # Stripe 支付（publishable key 可暴露在浏览器端） NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY=pk_live_51Te8ybEJDtUGxQHouKlEMfMuCZWifzlhBIReeKvANOtyCXS4zL0SMTWJXiFHPKz7azpF6OnIKXUPFSLs82fevVjr00w1bw5mvl diff --git a/src/app/call/__tests__/voice-call-player.test.ts b/src/app/call/__tests__/voice-call-player.test.ts new file mode 100644 index 00000000..0afcc63c --- /dev/null +++ b/src/app/call/__tests__/voice-call-player.test.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { VoiceCallPlayer } from "@/app/call/voice-call-player"; + +describe("VoiceCallPlayer", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("schedules decoded sentence audio on one continuous Web Audio timeline", async () => { + const starts: number[] = []; + const sources: Array<{ onended: (() => void) | null }> = []; + const context = { + state: "running", + currentTime: 0, + destination: {}, + resume: vi.fn(async () => undefined), + close: vi.fn(async () => undefined), + decodeAudioData: vi.fn(async () => ({ duration: 1 })), + createBufferSource: vi.fn(() => { + const source = { + buffer: null, + onended: null as (() => void) | null, + connect: vi.fn(), + disconnect: vi.fn(), + stop: vi.fn(), + start: vi.fn((at: number) => starts.push(at)), + }; + sources.push(source); + return source; + }), + }; + vi.stubGlobal("AudioContext", function AudioContextMock() { + return context; + }); + + const player = new VoiceCallPlayer(); + const gaps: number[] = []; + player.onGap = (milliseconds) => gaps.push(milliseconds); + const meta = { + callId: "call-1", + turnId: "turn-1", + index: 0, + mimeType: "audio/mpeg", + byteLength: 3, + }; + + player.enqueue(meta, new Uint8Array([1, 2, 3]).buffer); + player.enqueue({ ...meta, index: 1 }, new Uint8Array([4, 5, 6]).buffer); + + await vi.waitFor(() => expect(starts).toHaveLength(2)); + expect(starts).toEqual([0.025, 1.025]); + expect(gaps).toEqual([0]); + expect(sources).toHaveLength(2); + player.dispose(); + }); +}); diff --git a/src/app/call/page.tsx b/src/app/call/page.tsx new file mode 100644 index 00000000..2b9f0d0b --- /dev/null +++ b/src/app/call/page.tsx @@ -0,0 +1,8 @@ +import { redirect } from "next/navigation"; + +import { DEFAULT_CHARACTER_SLUG } from "@/data/constants/character"; +import { getCharacterRoutes } from "@/router/routes"; + +export default function LegacyCallPage() { + redirect(getCharacterRoutes(DEFAULT_CHARACTER_SLUG).call); +} diff --git a/src/app/call/voice-call-player.ts b/src/app/call/voice-call-player.ts new file mode 100644 index 00000000..76dd6c56 --- /dev/null +++ b/src/app/call/voice-call-player.ts @@ -0,0 +1,101 @@ +import type { CallAudioChunkMeta } from "@/core/net/voice-call-transport"; + +export class VoiceCallPlayer { + private context: AudioContext | null = null; + private sources = new Set(); + private decodeChain: Promise = Promise.resolve(); + private scheduledUntil = 0; + private pendingDecodes = 0; + private generation = 0; + onPlaying: (() => void) | null = null; + onIdle: (() => void) | null = null; + onGap: ((milliseconds: number) => void) | null = null; + onError: ((error: Error) => void) | null = null; + + get isPlaying(): boolean { + return this.sources.size > 0 || this.pendingDecodes > 0; + } + + async prepare(): Promise { + const context = this.getContext(); + if (context.state === "suspended") await context.resume(); + } + + enqueue(_meta: CallAudioChunkMeta, bytes: ArrayBuffer): void { + const generation = this.generation; + this.pendingDecodes += 1; + this.decodeChain = this.decodeChain + .then(async () => { + const context = this.getContext(); + if (context.state === "suspended") await context.resume(); + const audioBuffer = await context.decodeAudioData(bytes.slice(0)); + if (generation !== this.generation) return; + this.schedule(context, audioBuffer); + }) + .catch((error: unknown) => { + this.onError?.(error instanceof Error ? error : new Error(String(error))); + }) + .finally(() => { + this.pendingDecodes = Math.max(0, this.pendingDecodes - 1); + this.notifyIdleIfNeeded(); + }); + } + + stop(): void { + this.generation += 1; + for (const source of this.sources) { + source.onended = null; + try { + source.stop(); + } catch { + // 已自然结束的 source 无需重复停止。 + } + source.disconnect(); + } + this.sources.clear(); + this.scheduledUntil = 0; + this.onIdle?.(); + } + + dispose(): void { + this.stop(); + const context = this.context; + this.context = null; + if (context && context.state !== "closed") void context.close(); + } + + private getContext(): AudioContext { + if (!this.context || this.context.state === "closed") { + this.context = new AudioContext({ latencyHint: "interactive" }); + } + return this.context; + } + + private schedule(context: AudioContext, buffer: AudioBuffer): void { + const leadSeconds = 0.025; + const earliestStart = context.currentTime + leadSeconds; + const hadPreviousSegment = this.scheduledUntil > 0; + const startAt = Math.max(earliestStart, this.scheduledUntil); + if (hadPreviousSegment) { + this.onGap?.(Math.max(0, (startAt - this.scheduledUntil) * 1000)); + } + const source = context.createBufferSource(); + source.buffer = buffer; + source.connect(context.destination); + this.sources.add(source); + this.scheduledUntil = startAt + buffer.duration; + source.onended = () => { + source.disconnect(); + this.sources.delete(source); + this.notifyIdleIfNeeded(); + }; + source.start(startAt); + this.onPlaying?.(); + } + + private notifyIdleIfNeeded(): void { + if (this.sources.size || this.pendingDecodes) return; + this.scheduledUntil = 0; + this.onIdle?.(); + } +} diff --git a/src/app/call/voice-call-screen.module.css b/src/app/call/voice-call-screen.module.css new file mode 100644 index 00000000..4c02fe73 --- /dev/null +++ b/src/app/call/voice-call-screen.module.css @@ -0,0 +1,47 @@ +.screen { + position: relative; + display: flex; + min-height: var(--app-viewport-height, 100dvh); + overflow: hidden; + flex-direction: column; + color: #fff; + background: #090810; +} + +.background { object-fit: cover; opacity: .48; } +.scrim { position: absolute; inset: 0; background: radial-gradient(circle at 50% 32%, rgba(239,83,155,.18), transparent 36%), linear-gradient(180deg, rgba(9,8,16,.28), rgba(9,8,16,.94)); } +.header { position: relative; z-index: 1; display: flex; align-items: center; justify-content: space-between; padding: calc(18px + var(--app-safe-top, 0px)) 20px 12px; } +.iconButton { display: inline-flex; width: 42px; height: 42px; cursor: pointer; align-items: center; justify-content: center; border: 1px solid rgba(255,255,255,.16); border-radius: 999px; color: #fff; background: rgba(10,9,18,.55); backdrop-filter: blur(14px); } +.balances { display: flex; gap: 8px; font-size: 12px; font-weight: 650; } +.balances span { padding: 7px 10px; border: 1px solid rgba(255,255,255,.12); border-radius: 999px; background: rgba(10,9,18,.58); backdrop-filter: blur(14px); } +.content { position: relative; z-index: 1; display: flex; min-height: 0; flex: 1; flex-direction: column; align-items: center; justify-content: center; padding: 24px 28px; text-align: center; } +.avatarRing { position: relative; display: grid; width: 174px; height: 174px; place-items: center; border-radius: 50%; background: linear-gradient(135deg, rgba(255,255,255,.24), rgba(239,83,155,.42)); box-shadow: 0 28px 72px rgba(0,0,0,.42); transition: transform .25s ease, box-shadow .25s ease; } +.avatarRing::after { position: absolute; inset: -10px; border: 1px solid rgba(255,255,255,.16); border-radius: inherit; content: ""; } +.avatarRing.listening, .avatarRing.speaking { animation: callPulse 1.8s ease-in-out infinite; box-shadow: 0 0 0 12px rgba(239,83,155,.08), 0 28px 72px rgba(0,0,0,.42); } +.avatar { width: 158px; height: 158px; border: 4px solid rgba(9,8,16,.8); border-radius: 50%; object-fit: cover; } +.content h1 { margin: 30px 0 8px; font-size: clamp(27px, 7vw, 36px); letter-spacing: -.025em; } +.state { display: flex; min-height: 28px; align-items: center; gap: 7px; color: rgba(255,255,255,.78); font-size: 15px; } +.lowBalance { margin: 12px 0 0; padding: 7px 11px; border-radius: 999px; color: #ffd6e8; background: rgba(210,48,121,.2); font-size: 12px; } +.transcript { width: min(100%, 440px); min-height: 105px; margin-top: 24px; text-align: left; } +.transcript p { margin: 0 0 10px; padding: 11px 13px; border: 1px solid rgba(255,255,255,.09); border-radius: 15px; color: rgba(255,255,255,.9); background: rgba(14,12,24,.48); font-size: 14px; line-height: 1.45; backdrop-filter: blur(12px); } +.transcript span { display: block; margin-bottom: 4px; color: #f3a3c8; font-size: 11px; font-weight: 700; text-transform: uppercase; } +.error { max-width: 360px; margin: 8px 0 0; color: #ffc2ca; font-size: 13px; } +.footer { position: relative; z-index: 1; display: flex; flex-direction: column; align-items: center; padding: 10px 24px calc(24px + var(--app-safe-bottom, 0px)); } +.footer p { margin: 12px 0 0; color: rgba(255,255,255,.52); font-size: 12px; } +.startButton { display: inline-flex; min-width: 170px; cursor: pointer; align-items: center; justify-content: center; gap: 9px; border: 0; border-radius: 999px; padding: 15px 24px; color: #fff; background: linear-gradient(135deg, #ee579d, #ce357c); box-shadow: 0 16px 36px rgba(212,54,127,.32); font-size: 15px; font-weight: 750; } +.endButton { display: inline-flex; width: 66px; height: 66px; cursor: pointer; align-items: center; justify-content: center; border: 0; border-radius: 50%; color: #fff; background: #e24456; box-shadow: 0 15px 34px rgba(226,68,86,.34); } +.topUpBackdrop { position: absolute; z-index: 10; inset: 0; display: flex; align-items: flex-end; justify-content: center; padding: 18px; background: rgba(5,4,9,.68); backdrop-filter: blur(8px); } +.topUpDialog { width: min(100%, 460px); padding: 24px; border: 1px solid rgba(255,255,255,.12); border-radius: 24px; background: #17131f; box-shadow: 0 24px 70px rgba(0,0,0,.5); text-align: center; } +.topUpDialog h2 { margin: 0; font-size: 21px; } +.topUpDialog p { margin: 10px 0 20px; color: rgba(255,255,255,.66); font-size: 14px; line-height: 1.5; } +.topUpDialog button { width: 100%; cursor: pointer; border: 0; border-radius: 999px; padding: 13px 18px; color: #fff; background: #df478d; font-weight: 750; } +.topUpDialog .secondaryButton { margin-top: 8px; color: rgba(255,255,255,.72); background: transparent; } + +@keyframes callPulse { 0%,100% { transform: scale(1); } 50% { transform: scale(1.025); } } + +@media (max-height: 700px) { + .avatarRing { width: 138px; height: 138px; } + .avatar { width: 124px; height: 124px; } + .content h1 { margin-top: 20px; } + .transcript { min-height: 72px; margin-top: 16px; } +} diff --git a/src/app/call/voice-call-screen.tsx b/src/app/call/voice-call-screen.tsx new file mode 100644 index 00000000..1fcb36cc --- /dev/null +++ b/src/app/call/voice-call-screen.tsx @@ -0,0 +1,280 @@ +"use client"; + +import Image from "next/image"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { Mic, PhoneOff, RotateCcw, Volume2, X } from "lucide-react"; + +import { MobileShell } from "@/app/_components/core"; +import { getApiConfig } from "@/core/net/config/api_config"; +import { + VoiceCallTransport, + type CallBillingStatusPayload, +} from "@/core/net/voice-call-transport"; +import { VoiceCallAudioCapture } from "@/integrations/voice-call-audio"; +import { getSessionAccessToken } from "@/lib/auth/auth_session"; +import { useActiveCharacter, useActiveCharacterRoutes } from "@/providers/character-provider"; +import { useAppNavigator } from "@/router/use-app-navigator"; + +import { VoiceCallPlayer } from "./voice-call-player"; +import styles from "./voice-call-screen.module.css"; + +type CallUiState = "idle" | "connecting" | "listening" | "thinking" | "speaking" | "ended" | "error"; + +const STATE_COPY: Record = { + idle: "Ready to call", + connecting: "Connecting…", + listening: "Listening…", + thinking: "Thinking…", + speaking: "Speaking…", + ended: "Call ended", + error: "Call unavailable", +}; + +async function getSessionToken(): Promise { + const token = await getSessionAccessToken(); + if (token) return token; + throw new Error("Please sign in before starting a call"); +} + +export function VoiceCallScreen() { + const character = useActiveCharacter(); + const routes = useActiveCharacterRoutes(); + const navigator = useAppNavigator(); + const [uiState, setUiState] = useState("idle"); + const [userTranscript, setUserTranscript] = useState(""); + const [aiTranscript, setAiTranscript] = useState(""); + const [freeTurns, setFreeTurns] = useState(0); + const [balance, setBalance] = useState(0); + const [cost, setCost] = useState(2); + const [error, setError] = useState(null); + const [showTopUp, setShowTopUp] = useState(false); + const transportRef = useRef(null); + const captureRef = useRef(null); + const playerRef = useRef(null); + const callIdRef = useRef(""); + const activeTurnRef = useRef(""); + const reconnectTurnRef = useRef(""); + const resumeVersionRef = useRef(0); + const endingRef = useRef(false); + const pendingTopUpRef = useRef(false); + + const finishCall = useCallback((navigate = true) => { + if (endingRef.current) return; + endingRef.current = true; + captureRef.current?.dispose(); + playerRef.current?.dispose(); + if (callIdRef.current) transportRef.current?.endCall(callIdRef.current); + transportRef.current?.disconnect(); + reconnectTurnRef.current = ""; + resumeVersionRef.current = 0; + setUiState("ended"); + if (navigate) navigator.push(routes.chat); + }, [navigator, routes.chat]); + + const beginListening = useCallback((turnId: string) => { + const capture = captureRef.current; + const transport = transportRef.current; + const callId = callIdRef.current; + if (!capture || !transport || !callId || !turnId) return; + activeTurnRef.current = turnId; + reconnectTurnRef.current = turnId; + setUiState(playerRef.current?.isPlaying ? "speaking" : "listening"); + const preRoll: ArrayBuffer[] = []; + let streamOpened = false; + capture.startTurn({ + onChunk: (chunk) => { + if (streamOpened) { + transport.sendAudioChunk(chunk); + return; + } + preRoll.push(chunk); + if (preRoll.length > 5) preRoll.shift(); + }, + onSpeechStart: () => { + setError(null); + if (playerRef.current?.isPlaying) { + playerRef.current.stop(); + transport.interrupt(callId); + } + streamOpened = transport.startAudio( + callId, + turnId, + capture.mimeType, + capture.sampleRate, + ); + if (streamOpened) { + for (const chunk of preRoll) transport.sendAudioChunk(chunk); + preRoll.length = 0; + setUserTranscript(""); + setAiTranscript(""); + setUiState("listening"); + } + }, + onTurnEnd: (vadLatencyMs) => { + if (streamOpened) { + transport.endAudio(callId, turnId, vadLatencyMs); + setUiState("thinking"); + } + }, + onError: (captureError) => { + setError(captureError.message); + setUiState("error"); + }, + }); + }, []); + + const applyBilling = useCallback((status: CallBillingStatusPayload) => { + setBalance(status.creditBalance); + setFreeTurns(status.freeTurnsRemaining); + setCost(status.requiredCredits || 2); + if (status.canContinue && status.nextTurnId) { + beginListening(status.nextTurnId); + return; + } + captureRef.current?.stopTurn(); + pendingTopUpRef.current = true; + if (!playerRef.current?.isPlaying) setShowTopUp(true); + }, [beginListening]); + + const startCall = useCallback(async () => { + if (uiState !== "idle" && uiState !== "error") return; + endingRef.current = false; + pendingTopUpRef.current = false; + setError(null); + setUiState("connecting"); + try { + const player = new VoiceCallPlayer(); + playerRef.current = player; + const playerReady = player.prepare(); + const capture = new VoiceCallAudioCapture(); + await Promise.all([capture.prepare(), playerReady]); + captureRef.current = capture; + const token = await getSessionToken(); + const transport = new VoiceCallTransport(getApiConfig().wsUrl, token); + transportRef.current = transport; + player.onPlaying = () => setUiState("speaking"); + player.onIdle = () => { + if (pendingTopUpRef.current) setShowTopUp(true); + else if (activeTurnRef.current) setUiState("listening"); + }; + player.onError = (playbackError) => setError(playbackError.message); + player.onGap = (milliseconds) => { + transport.sendClientMetric("audioChunkGap", milliseconds); + }; + transport.onOpen = () => { + if (!reconnectTurnRef.current) reconnectTurnRef.current = crypto.randomUUID(); + transport.startCall( + character.id, + reconnectTurnRef.current, + resumeVersionRef.current || undefined, + ); + }; + transport.onReconnecting = () => { + capture.stopTurn(); + player.stop(); + callIdRef.current = ""; + setUiState("connecting"); + }; + transport.onStarted = (started) => { + callIdRef.current = started.callId; + activeTurnRef.current = started.turnId; + reconnectTurnRef.current = started.turnId; + resumeVersionRef.current = started.resumeVersion; + setFreeTurns(started.freeTurnsRemaining); + setBalance(started.creditBalance); + setCost(started.costPerPaidTurn || 2); + beginListening(started.turnId); + }; + transport.onState = setUiState; + transport.onTranscript = (text) => setUserTranscript(text); + transport.onAiSentence = (text) => { + setAiTranscript((current) => `${current}${current ? " " : ""}${text}`); + }; + transport.onAudioChunk = (meta, bytes) => player.enqueue(meta, bytes); + transport.onBillingStatus = applyBilling; + transport.onEnded = () => setUiState("ended"); + transport.onError = (code, message) => { + setError(message); + if ( + activeTurnRef.current + && (code === "CALL_TURN_FAILED" || code === "CALL_STT_FAILED") + ) { + setUiState("listening"); + } else { + setUiState("error"); + } + }; + transport.connect(); + } catch (startError) { + captureRef.current?.dispose(); + playerRef.current?.dispose(); + setError(startError instanceof Error ? startError.message : String(startError)); + setUiState("error"); + } + }, [applyBilling, beginListening, character.id, uiState]); + + useEffect(() => () => finishCall(false), [finishCall]); + + const lowBalance = freeTurns === 0 && balance < cost * 2 && balance >= cost; + + return ( + +
+ +
+
+ +
+ {freeTurns} free + {balance} credits +
+
+ +
+
+ {character.displayName} +
+

{character.displayName}

+
+ {uiState === "listening" ?
+ {lowBalance ?

Your balance covers fewer than two paid turns.

: null} +
+ {userTranscript ?

You{userTranscript}

: null} + {aiTranscript ?

{character.shortName}{aiTranscript}

: null} +
+ {error ?

{error}

: null} +
+ +
+ {uiState === "idle" || uiState === "error" ? ( + + ) : ( + + )} +

Speak naturally — you can interrupt at any time.

+
+ + {showTopUp ? ( +
+
+

Not enough credits for another turn

+

Your current reply has finished. Top up to keep the conversation going.

+ + +
+
+ ) : null} +
+ + ); +} diff --git a/src/app/characters/[characterSlug]/call/page.tsx b/src/app/characters/[characterSlug]/call/page.tsx new file mode 100644 index 00000000..854e23ec --- /dev/null +++ b/src/app/characters/[characterSlug]/call/page.tsx @@ -0,0 +1,5 @@ +import { VoiceCallScreen } from "@/app/call/voice-call-screen"; + +export default function CharacterCallPage() { + return ; +} diff --git a/src/app/chat/components/chat-header.tsx b/src/app/chat/components/chat-header.tsx index f2a9350e..1bd22de4 100644 --- a/src/app/chat/components/chat-header.tsx +++ b/src/app/chat/components/chat-header.tsx @@ -5,7 +5,7 @@ * 顶部保持返回、首充优惠、收藏入口三段结构。 */ import type { ReactNode } from "react"; -import { Lock } from "lucide-react"; +import { Lock, Phone } from "lucide-react"; import { BackButton } from "@/app/_components"; import { FavoriteEntryButton } from "@/app/_components/core"; @@ -27,6 +27,7 @@ export function ChatHeader({ const navigator = useAppNavigator(); const character = useActiveCharacter(); const characterRoutes = useActiveCharacterRoutes(); + const voiceCallEnabled = process.env.NEXT_PUBLIC_ENABLE_VOICE_CALL === "true"; return (
@@ -58,10 +59,24 @@ export function ChatHeader({
{offerBanner}
- +
+ {voiceCallEnabled ? ( + + ) : null} + +
); diff --git a/src/core/net/__tests__/voice-call-transport.test.ts b/src/core/net/__tests__/voice-call-transport.test.ts new file mode 100644 index 00000000..8333ad8f --- /dev/null +++ b/src/core/net/__tests__/voice-call-transport.test.ts @@ -0,0 +1,139 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { VoiceCallTransport } from "@/core/net/voice-call-transport"; + +class FakeWebSocket { + static readonly CONNECTING = 0; + static readonly OPEN = 1; + static readonly CLOSED = 3; + readyState = FakeWebSocket.CONNECTING; + binaryType = "blob"; + sent: unknown[] = []; + onopen: (() => void) | null = null; + onmessage: ((event: { data: unknown }) => void) | null = null; + onclose: ((event: { code: number; reason: string; wasClean: boolean }) => void) | null = null; + onerror: (() => void) | null = null; + + constructor(readonly url: string) {} + send(data: unknown) { this.sent.push(data); } + close() { this.readyState = FakeWebSocket.CLOSED; } + open() { this.readyState = FakeWebSocket.OPEN; this.onopen?.(); } + receive(data: unknown) { this.onmessage?.({ data }); } +} + +describe("VoiceCallTransport", () => { + let sockets: FakeWebSocket[]; + + beforeEach(() => { + sockets = []; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + function create() { + return new VoiceCallTransport("wss://api.example/ws", "secret-token", { + webSocketFactory: (url) => { + const socket = new FakeWebSocket(url); + sockets.push(socket); + return socket as unknown as WebSocket; + }, + }); + } + + it("uploads 100-200ms chunks as binary frames instead of base64 JSON", () => { + const transport = create(); + transport.connect(); + sockets[0].open(); + + transport.startCall("elio", "turn-1"); + transport.startAudio("call-1", "turn-1", "audio/webm;codecs=opus", 48000); + const frame = new Uint8Array([1, 2, 3]).buffer; + transport.sendAudioChunk(frame); + transport.endAudio("call-1", "turn-1"); + + expect(JSON.parse(sockets[0].sent[0] as string)).toMatchObject({ + type: "call_start", + protocolVersion: 2, + characterId: "elio", + turnId: "turn-1", + }); + expect(JSON.parse(sockets[0].sent[1] as string)).toMatchObject({ + type: "call_audio_start", + encoding: "opus", + }); + expect(sockets[0].sent[2]).toBe(frame); + expect(JSON.parse(sockets[0].sent[3] as string).type).toBe("call_audio_end"); + }); + + it("associates the next binary payload with its JSON audio metadata", () => { + const transport = create(); + const received = vi.fn(); + transport.onAudioChunk = received; + transport.connect(); + sockets[0].open(); + sockets[0].receive(JSON.stringify({ + type: "call_audio_chunk_start", + protocolVersion: 2, + callId: "call-1", + turnId: "turn-1", + index: 0, + mimeType: "audio/mpeg", + byteLength: 3, + })); + const bytes = new Uint8Array([4, 5, 6]).buffer; + sockets[0].receive(bytes); + + expect(received).toHaveBeenCalledWith( + expect.objectContaining({ turnId: "turn-1", index: 0, mimeType: "audio/mpeg" }), + bytes, + ); + }); + + it("surfaces billing fields without treating a normal charge as an error", () => { + const transport = create(); + const billing = vi.fn(); + transport.onBillingStatus = billing; + transport.connect(); + sockets[0].open(); + sockets[0].receive(JSON.stringify({ + type: "call_billing_status", + status: "settled", + creditsCharged: 2, + creditBalance: 8, + requiredCredits: 2, + shortfallCredits: 0, + canContinue: true, + nextTurnId: "turn-2", + })); + + expect(billing).toHaveBeenCalledWith(expect.objectContaining({ + creditsCharged: 2, + creditBalance: 8, + canContinue: true, + nextTurnId: "turn-2", + })); + }); + + it("starts a reconnect attempt within the two-second release gate", () => { + vi.useFakeTimers(); + const transport = create(); + const reconnecting = vi.fn(); + transport.onReconnecting = reconnecting; + transport.connect(); + sockets[0].open(); + sockets[0].onclose?.({ code: 1006, reason: "network", wasClean: false }); + + expect(reconnecting).toHaveBeenCalledWith(400); + vi.advanceTimersByTime(400); + expect(sockets).toHaveLength(2); + sockets[1].open(); + expect(JSON.parse(sockets[1].sent[0] as string)).toMatchObject({ + type: "call_client_metric", + name: "reconnectLatency", + milliseconds: 400, + }); + transport.disconnect(); + }); +}); diff --git a/src/core/net/voice-call-transport.ts b/src/core/net/voice-call-transport.ts new file mode 100644 index 00000000..93a67771 --- /dev/null +++ b/src/core/net/voice-call-transport.ts @@ -0,0 +1,210 @@ +import { Logger } from "@/utils/logger"; + +const log = new Logger("VoiceCallTransport"); +const PROTOCOL_VERSION = 2 as const; +const WS_OPEN = 1; +const WS_CLOSING = 2; + +export interface CallStartedPayload { + callId: string; + turnId: string; + characterId: string; + costPerPaidTurn: number; + freeTurnsRemaining: number; + creditBalance: number; + canContinue: boolean; + resumeVersion: number; +} + +export interface CallBillingStatusPayload { + callId?: string; + turnId?: string; + nextTurnId?: string; + status: string; + creditsCharged: number; + creditBalance: number; + freeTurnsRemaining: number; + requiredCredits: number; + shortfallCredits: number; + canContinue: boolean; +} + +export interface CallAudioChunkMeta { + callId: string; + turnId: string; + index: number; + mimeType: string; + byteLength: number; +} + +export interface VoiceCallTransportOptions { + webSocketFactory?: (url: string) => WebSocket; + reconnectBaseMs?: number; +} + +export class VoiceCallTransport { + private socket: WebSocket | null = null; + private disposed = false; + private reconnectTimer: ReturnType | null = null; + private reconnectAttempt = 0; + private disconnectedAt = 0; + private pendingAudioMeta: CallAudioChunkMeta | null = null; + private readonly factory: (url: string) => WebSocket; + private readonly reconnectBaseMs: number; + + onOpen: (() => void) | null = null; + onReconnecting: ((delayMs: number) => void) | null = null; + onStarted: ((payload: CallStartedPayload) => void) | null = null; + onState: ((state: "listening" | "thinking" | "speaking") => void) | null = null; + onTranscript: ((text: string, final: boolean) => void) | null = null; + onAiSentence: ((text: string, index: number) => void) | null = null; + onBillingStatus: ((payload: CallBillingStatusPayload) => void) | null = null; + onAudioChunk: ((meta: CallAudioChunkMeta, bytes: ArrayBuffer) => void) | null = null; + onInterrupted: (() => void) | null = null; + onEnded: (() => void) | null = null; + onError: ((code: string, message: string) => void) | null = null; + + constructor( + private readonly serverUrl: string, + private readonly token: string, + options: VoiceCallTransportOptions = {}, + ) { + this.factory = options.webSocketFactory ?? ((url) => new WebSocket(url)); + this.reconnectBaseMs = options.reconnectBaseMs ?? 400; + } + + get isConnected(): boolean { + return this.socket?.readyState === WS_OPEN; + } + + connect(): void { + if (this.disposed || (this.socket && this.socket.readyState < WS_CLOSING)) return; + const separator = this.serverUrl.includes("?") ? "&" : "?"; + const url = `${this.serverUrl}${separator}token=${encodeURIComponent(this.token)}`; + const socket = this.factory(url); + socket.binaryType = "arraybuffer"; + this.socket = socket; + socket.onopen = () => { + this.reconnectAttempt = 0; + if (this.disconnectedAt) { + this.sendClientMetric("reconnectLatency", Date.now() - this.disconnectedAt); + this.disconnectedAt = 0; + } + this.onOpen?.(); + }; + socket.onmessage = (event) => this.handleMessage(event.data); + socket.onerror = () => this.onError?.("CALL_NETWORK_ERROR", "通话网络连接异常"); + socket.onclose = () => { + this.socket = null; + this.disconnectedAt = Date.now(); + if (!this.disposed) this.scheduleReconnect(); + }; + } + + startCall(characterId: string, turnId: string, resumeVersion?: number): boolean { + return this.sendJson({ + type: "call_start", protocolVersion: PROTOCOL_VERSION, characterId, turnId, + ...(resumeVersion ? { resumeVersion } : {}), + }); + } + + startAudio(callId: string, turnId: string, mimeType: string, sampleRate: number): boolean { + return this.sendJson({ + type: "call_audio_start", protocolVersion: PROTOCOL_VERSION, + callId, turnId, encoding: "opus", mimeType, sampleRate, + }); + } + + sendAudioChunk(bytes: ArrayBuffer): boolean { + if (!this.isConnected) return false; + this.socket?.send(bytes); + return true; + } + + endAudio(callId: string, turnId: string, vadLatency = 0): boolean { + return this.sendJson({ + type: "call_audio_end", protocolVersion: PROTOCOL_VERSION, callId, turnId, + vadLatency, + }); + } + + sendClientMetric(name: "audioChunkGap" | "reconnectLatency", milliseconds: number): boolean { + return this.sendJson({ + type: "call_client_metric", + protocolVersion: PROTOCOL_VERSION, + name, + milliseconds: Math.max(0, Math.min(60_000, milliseconds)), + }); + } + + interrupt(callId: string): boolean { + return this.sendJson({ + type: "call_interrupt", protocolVersion: PROTOCOL_VERSION, callId, + }); + } + + endCall(callId: string): boolean { + return this.sendJson({ + type: "call_end", protocolVersion: PROTOCOL_VERSION, callId, + }); + } + + disconnect(): void { + this.disposed = true; + if (this.reconnectTimer) clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + this.socket?.close(); + this.socket = null; + } + + private sendJson(payload: Record): boolean { + if (!this.isConnected) return false; + this.socket?.send(JSON.stringify(payload)); + return true; + } + + private scheduleReconnect(): void { + const delayMs = Math.min(1600, this.reconnectBaseMs * 2 ** this.reconnectAttempt); + this.reconnectAttempt += 1; + this.onReconnecting?.(delayMs); + this.reconnectTimer = setTimeout(() => this.connect(), delayMs); + } + + private handleMessage(data: unknown): void { + if (data instanceof ArrayBuffer) { + const meta = this.pendingAudioMeta; + this.pendingAudioMeta = null; + if (meta && meta.byteLength === data.byteLength) this.onAudioChunk?.(meta, data); + else if (meta) this.onError?.("CALL_AUDIO_FRAME_INVALID", "通话音频帧不完整"); + return; + } + if (typeof data !== "string") return; + let payload: Record; + try { + payload = JSON.parse(data) as Record; + } catch (error) { + log.warn("Ignored malformed call event", error); + return; + } + const type = String(payload.type ?? ""); + if (type === "call_started") { + this.onStarted?.(payload as unknown as CallStartedPayload); + } else if (type === "call_state") { + this.onState?.(String(payload.state) as "listening" | "thinking" | "speaking"); + } else if (type === "call_stt_partial" || type === "call_stt_final") { + this.onTranscript?.(String(payload.text ?? ""), type === "call_stt_final"); + } else if (type === "call_ai_sentence") { + this.onAiSentence?.(String(payload.text ?? ""), Number(payload.index ?? 0)); + } else if (type === "call_billing_status") { + this.onBillingStatus?.(payload as unknown as CallBillingStatusPayload); + } else if (type === "call_audio_chunk_start") { + this.pendingAudioMeta = payload as unknown as CallAudioChunkMeta; + } else if (type === "call_interrupted") { + this.onInterrupted?.(); + } else if (type === "call_ended") { + this.onEnded?.(); + } else if (type === "call_error") { + this.onError?.(String(payload.errorCode ?? "CALL_ERROR"), String(payload.message ?? "通话失败")); + } + } +} diff --git a/src/data/schemas/user/__tests__/user.test.ts b/src/data/schemas/user/__tests__/user.test.ts index 76872be3..d0caf976 100644 --- a/src/data/schemas/user/__tests__/user.test.ts +++ b/src/data/schemas/user/__tests__/user.test.ts @@ -134,11 +134,26 @@ describe("User", () => { photo: 40, }, }, + voiceCallBilling: { + enabled: true, + protocolVersion: 2, + chargeMode: "per_turn", + sharesNormalChatFreeDaily: true, + freeTurnsDaily: 30, + costPerPaidTurn: 2, + usageType: "voice_call_turn", + legacyMinuteCostUsed: false, + }, }); expect(entitlements.isVip).toBe(true); expect(entitlements.creditBalance).toBe(120); expect(entitlements.historyUnlock.costs.photo).toBe(40); + expect(entitlements.voiceCallBilling).toMatchObject({ + protocolVersion: 2, + chargeMode: "per_turn", + costPerPaidTurn: 2, + }); }); it("defaults nullable entitlement sections through schema helpers", () => { diff --git a/src/data/schemas/user/user_entitlements.ts b/src/data/schemas/user/user_entitlements.ts index 86588d32..ba11bcab 100644 --- a/src/data/schemas/user/user_entitlements.ts +++ b/src/data/schemas/user/user_entitlements.ts @@ -49,6 +49,17 @@ const HISTORY_UNLOCK_DEFAULTS = { costs: HISTORY_UNLOCK_COSTS_DEFAULTS, } as const; +const VOICE_CALL_BILLING_DEFAULTS = { + enabled: false, + protocolVersion: 2, + chargeMode: "per_turn", + sharesNormalChatFreeDaily: true, + freeTurnsDaily: 0, + costPerPaidTurn: 0, + usageType: "voice_call_turn", + legacyMinuteCostUsed: false, +} as const; + export const UserEntitlementsPolicySchema = z .object({ membershipState: stringOrEmpty, @@ -105,6 +116,20 @@ export const UserEntitlementsHistoryUnlockSchema = z .passthrough() .readonly(); +export const VoiceCallBillingSchema = z + .object({ + enabled: booleanOrFalse, + protocolVersion: numberOrZero, + chargeMode: stringOrEmpty, + sharesNormalChatFreeDaily: booleanOrFalse, + freeTurnsDaily: numberOrZero, + costPerPaidTurn: numberOrZero, + usageType: stringOrEmpty, + legacyMinuteCostUsed: booleanOrFalse, + }) + .passthrough() + .readonly(); + export const UserEntitlementsSchema = z .object({ userId: stringOrEmpty, @@ -120,6 +145,10 @@ export const UserEntitlementsSchema = z UserEntitlementsHistoryUnlockSchema, HISTORY_UNLOCK_DEFAULTS, ), + voiceCallBilling: schemaOr( + VoiceCallBillingSchema, + VOICE_CALL_BILLING_DEFAULTS, + ), }) .readonly(); diff --git a/src/integrations/__tests__/voice-call-audio.test.ts b/src/integrations/__tests__/voice-call-audio.test.ts new file mode 100644 index 00000000..7bf4fbae --- /dev/null +++ b/src/integrations/__tests__/voice-call-audio.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; + +import { AdaptiveVad } from "@/integrations/voice-call-audio"; + +describe("AdaptiveVad", () => { + it("adapts its silence window between 400 and 600ms", () => { + const quiet = new AdaptiveVad(); + for (let i = 0; i < 20; i += 1) quiet.observe(0.005, i * 50); + expect(quiet.silenceWindowMs).toBeGreaterThanOrEqual(400); + expect(quiet.silenceWindowMs).toBeLessThanOrEqual(600); + + const noisy = new AdaptiveVad(); + for (let i = 0; i < 20; i += 1) noisy.observe(0.04, i * 50); + expect(noisy.silenceWindowMs).toBeGreaterThanOrEqual(quiet.silenceWindowMs); + expect(noisy.silenceWindowMs).toBeLessThanOrEqual(600); + }); + + it("ends a turn after speech followed by adaptive silence", () => { + const vad = new AdaptiveVad(); + for (let i = 0; i < 10; i += 1) vad.observe(0.004, i * 50); + expect(vad.observe(0.2, 500).speechStarted).toBe(true); + let ended = false; + let silenceDurationMs = 0; + for (let t = 550; t <= 1200; t += 50) { + const observation = vad.observe(0.001, t); + ended ||= observation.turnEnded; + silenceDurationMs ||= observation.silenceDurationMs; + } + expect(ended).toBe(true); + expect(silenceDurationMs).toBeGreaterThanOrEqual(400); + expect(silenceDurationMs).toBeLessThanOrEqual(600); + }); +}); diff --git a/src/integrations/voice-call-audio.ts b/src/integrations/voice-call-audio.ts new file mode 100644 index 00000000..86ef2c2a --- /dev/null +++ b/src/integrations/voice-call-audio.ts @@ -0,0 +1,142 @@ +export interface VadObservation { + speechStarted: boolean; + turnEnded: boolean; + silenceDurationMs: number; +} + +export class AdaptiveVad { + private noiseFloor = 0.008; + private speaking = false; + private lastSpeechAt = 0; + + get silenceWindowMs(): number { + return Math.max(400, Math.min(600, Math.round(400 + this.noiseFloor * 5000))); + } + + observe(rms: number, nowMs: number): VadObservation { + const level = Math.max(0, Number.isFinite(rms) ? rms : 0); + if (!this.speaking) { + this.noiseFloor = this.noiseFloor * 0.9 + Math.min(level, 0.06) * 0.1; + } + const speechThreshold = Math.max(0.018, this.noiseFloor * 2.8); + const isSpeech = level >= speechThreshold; + let speechStarted = false; + let turnEnded = false; + let silenceDurationMs = 0; + if (isSpeech) { + speechStarted = !this.speaking; + this.speaking = true; + this.lastSpeechAt = nowMs; + } else if (this.speaking && nowMs - this.lastSpeechAt >= this.silenceWindowMs) { + this.speaking = false; + turnEnded = true; + silenceDurationMs = nowMs - this.lastSpeechAt; + } + return { speechStarted, turnEnded, silenceDurationMs }; + } + + reset(): void { + this.speaking = false; + this.lastSpeechAt = 0; + } +} + +export interface VoiceCaptureCallbacks { + onChunk: (chunk: ArrayBuffer) => void; + onSpeechStart?: () => void; + onTurnEnd: (vadLatencyMs: number) => void; + onError: (error: Error) => void; +} + +export class VoiceCallAudioCapture { + private stream: MediaStream | null = null; + private audioContext: AudioContext | null = null; + private analyser: AnalyserNode | null = null; + private recorder: MediaRecorder | null = null; + private vadTimer: number | null = null; + private pendingChunkReads: Promise = Promise.resolve(); + private readonly vad = new AdaptiveVad(); + + get mimeType(): string { + if (typeof MediaRecorder === "undefined") return "audio/webm;codecs=opus"; + if (MediaRecorder.isTypeSupported("audio/webm;codecs=opus")) return "audio/webm;codecs=opus"; + if (MediaRecorder.isTypeSupported("audio/ogg;codecs=opus")) return "audio/ogg;codecs=opus"; + return "audio/webm"; + } + + get sampleRate(): number { + return this.audioContext?.sampleRate ?? 48000; + } + + async prepare(): Promise { + if (this.stream) return; + this.stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + channelCount: 1, + }, + }); + this.audioContext = new AudioContext({ latencyHint: "interactive" }); + await this.audioContext.resume(); + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 1024; + this.audioContext.createMediaStreamSource(this.stream).connect(this.analyser); + } + + startTurn(callbacks: VoiceCaptureCallbacks): void { + if (!this.stream || !this.analyser) throw new Error("Microphone is not prepared"); + if (this.recorder?.state === "recording") return; + this.vad.reset(); + this.pendingChunkReads = Promise.resolve(); + const recorder = new MediaRecorder(this.stream, { mimeType: this.mimeType }); + this.recorder = recorder; + recorder.ondataavailable = (event) => { + if (event.data.size === 0) return; + this.pendingChunkReads = this.pendingChunkReads + .then(() => event.data.arrayBuffer()) + .then(callbacks.onChunk) + .catch((error: unknown) => { + callbacks.onError(error instanceof Error ? error : new Error(String(error))); + }); + }; + recorder.onerror = () => callbacks.onError(new Error("Microphone recorder failed")); + recorder.start(120); + const samples = new Float32Array(this.analyser.fftSize); + this.vadTimer = window.setInterval(() => { + this.analyser?.getFloatTimeDomainData(samples); + let squareSum = 0; + for (const sample of samples) squareSum += sample * sample; + const rms = Math.sqrt(squareSum / samples.length); + const observation = this.vad.observe(rms, performance.now()); + if (observation.speechStarted) callbacks.onSpeechStart?.(); + if (observation.turnEnded) { + this.stopTurn(() => callbacks.onTurnEnd(observation.silenceDurationMs)); + } + }, 50); + } + + stopTurn(onStopped?: () => void): void { + if (this.vadTimer !== null) window.clearInterval(this.vadTimer); + this.vadTimer = null; + if (this.recorder?.state === "recording") { + this.recorder.onstop = () => { + void this.pendingChunkReads.then(() => onStopped?.()); + }; + this.recorder.stop(); + } else { + onStopped?.(); + } + this.recorder = null; + } + + dispose(): void { + this.stopTurn(); + this.stream?.getTracks().forEach((track) => track.stop()); + this.stream = null; + void this.audioContext?.close(); + this.audioContext = null; + this.analyser = null; + } +} diff --git a/src/lib/auth/auth_session.ts b/src/lib/auth/auth_session.ts index 4f4d9b9c..dbe7f186 100644 --- a/src/lib/auth/auth_session.ts +++ b/src/lib/auth/auth_session.ts @@ -14,3 +14,15 @@ export async function hasCompleteBusinessAuthSession( const refreshTokenResult = await storage.getRefreshToken(); return refreshTokenResult.success && Boolean(refreshTokenResult.data); } + +/** 返回当前业务会话 token；真实用户优先，游客作为通话降级身份。 */ +export async function getSessionAccessToken( + storage: IAuthStorage = AuthStorage.getInstance(), +): Promise { + const loginTokenResult = await storage.getLoginToken(); + if (loginTokenResult.success && loginTokenResult.data) { + return loginTokenResult.data; + } + const guestTokenResult = await storage.getGuestToken(); + return guestTokenResult.success ? guestTokenResult.data : null; +} diff --git a/src/router/__tests__/route-meta.test.ts b/src/router/__tests__/route-meta.test.ts index 2c42bef1..eca42708 100644 --- a/src/router/__tests__/route-meta.test.ts +++ b/src/router/__tests__/route-meta.test.ts @@ -26,11 +26,13 @@ describe("route meta", () => { expect(routes).toEqual({ splash: "/characters/maya/splash", chat: "/characters/maya/chat", + call: "/characters/maya/call", privateZone: "/characters/maya/private-zone", tip: "/characters/maya/tip", }); expect(getRouteAccess(routes.splash)).toBe("authOnly"); expect(getRouteAccess(routes.chat)).toBe("guestEntry"); + expect(getRouteAccess(routes.call)).toBe("guestEntry"); expect(getRouteAccess(routes.privateZone)).toBe("guestEntry"); expect(getRouteAccess(routes.tip)).toBe("public"); }); diff --git a/src/router/route-meta.ts b/src/router/route-meta.ts index b0730d1d..c1cd1ab0 100644 --- a/src/router/route-meta.ts +++ b/src/router/route-meta.ts @@ -12,6 +12,7 @@ const GLOBAL_ROUTE_ACCESS: Partial> = { [ROUTES.splash]: "authOnly", [ROUTES.auth]: "authOnly", [ROUTES.chat]: "guestEntry", + [ROUTES.call]: "guestEntry", [ROUTES.privateZone]: "guestEntry", [ROUTES.tip]: "public", [ROUTES.externalEntry]: "public", @@ -24,6 +25,7 @@ const GLOBAL_ROUTE_ACCESS: Partial> = { const CHARACTER_ROUTE_ACCESS: Readonly> = { splash: "authOnly", chat: "guestEntry", + call: "guestEntry", "private-zone": "guestEntry", tip: "public", }; diff --git a/src/router/routes.ts b/src/router/routes.ts index ae43d7ca..6cca83c6 100644 --- a/src/router/routes.ts +++ b/src/router/routes.ts @@ -21,6 +21,7 @@ export const ROUTES = { root: "/", splash: "/splash", chat: "/chat", + call: "/call", privateZone: "/private-zone", tip: "/tip", externalEntry: "/external-entry", @@ -36,6 +37,7 @@ export type StaticRoute = (typeof ROUTES)[keyof typeof ROUTES]; export interface CharacterRoutes { readonly splash: string; readonly chat: string; + readonly call: string; readonly privateZone: string; readonly tip: string; } @@ -47,6 +49,7 @@ export function getCharacterRoutes( return { splash: `${root}/splash`, chat: `${root}/chat`, + call: `${root}/call`, privateZone: `${root}/private-zone`, tip: `${root}/tip`, }; @@ -116,6 +119,7 @@ export const ROUTE_BUILDERS = { export const ALL_STATIC_ROUTES: readonly StaticRoute[] = [ ROUTES.splash, ROUTES.chat, + ROUTES.call, ROUTES.privateZone, ROUTES.tip, ROUTES.externalEntry,