feat(call): add streaming voice call experience
Docker Image / Build and Push Docker Image (push) Successful in 2m3s

This commit is contained in:
Codex
2026-07-24 16:36:36 +08:00
parent 5e0361a199
commit 17236bd14e
19 changed files with 1108 additions and 5 deletions
@@ -0,0 +1,57 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { VoiceCallPlayer } from "@/app/call/voice-call-player";
describe("VoiceCallPlayer", () => {
afterEach(() => {
vi.unstubAllGlobals();
});
it("schedules decoded sentence audio on one continuous Web Audio timeline", async () => {
const starts: number[] = [];
const sources: Array<{ onended: (() => void) | null }> = [];
const context = {
state: "running",
currentTime: 0,
destination: {},
resume: vi.fn(async () => undefined),
close: vi.fn(async () => undefined),
decodeAudioData: vi.fn(async () => ({ duration: 1 })),
createBufferSource: vi.fn(() => {
const source = {
buffer: null,
onended: null as (() => void) | null,
connect: vi.fn(),
disconnect: vi.fn(),
stop: vi.fn(),
start: vi.fn((at: number) => starts.push(at)),
};
sources.push(source);
return source;
}),
};
vi.stubGlobal("AudioContext", function AudioContextMock() {
return context;
});
const player = new VoiceCallPlayer();
const gaps: number[] = [];
player.onGap = (milliseconds) => gaps.push(milliseconds);
const meta = {
callId: "call-1",
turnId: "turn-1",
index: 0,
mimeType: "audio/mpeg",
byteLength: 3,
};
player.enqueue(meta, new Uint8Array([1, 2, 3]).buffer);
player.enqueue({ ...meta, index: 1 }, new Uint8Array([4, 5, 6]).buffer);
await vi.waitFor(() => expect(starts).toHaveLength(2));
expect(starts).toEqual([0.025, 1.025]);
expect(gaps).toEqual([0]);
expect(sources).toHaveLength(2);
player.dispose();
});
});
+8
View File
@@ -0,0 +1,8 @@
import { redirect } from "next/navigation";
import { DEFAULT_CHARACTER_SLUG } from "@/data/constants/character";
import { getCharacterRoutes } from "@/router/routes";
export default function LegacyCallPage() {
redirect(getCharacterRoutes(DEFAULT_CHARACTER_SLUG).call);
}
+101
View File
@@ -0,0 +1,101 @@
import type { CallAudioChunkMeta } from "@/core/net/voice-call-transport";
export class VoiceCallPlayer {
private context: AudioContext | null = null;
private sources = new Set<AudioBufferSourceNode>();
private decodeChain: Promise<void> = Promise.resolve();
private scheduledUntil = 0;
private pendingDecodes = 0;
private generation = 0;
onPlaying: (() => void) | null = null;
onIdle: (() => void) | null = null;
onGap: ((milliseconds: number) => void) | null = null;
onError: ((error: Error) => void) | null = null;
get isPlaying(): boolean {
return this.sources.size > 0 || this.pendingDecodes > 0;
}
async prepare(): Promise<void> {
const context = this.getContext();
if (context.state === "suspended") await context.resume();
}
enqueue(_meta: CallAudioChunkMeta, bytes: ArrayBuffer): void {
const generation = this.generation;
this.pendingDecodes += 1;
this.decodeChain = this.decodeChain
.then(async () => {
const context = this.getContext();
if (context.state === "suspended") await context.resume();
const audioBuffer = await context.decodeAudioData(bytes.slice(0));
if (generation !== this.generation) return;
this.schedule(context, audioBuffer);
})
.catch((error: unknown) => {
this.onError?.(error instanceof Error ? error : new Error(String(error)));
})
.finally(() => {
this.pendingDecodes = Math.max(0, this.pendingDecodes - 1);
this.notifyIdleIfNeeded();
});
}
stop(): void {
this.generation += 1;
for (const source of this.sources) {
source.onended = null;
try {
source.stop();
} catch {
// 已自然结束的 source 无需重复停止。
}
source.disconnect();
}
this.sources.clear();
this.scheduledUntil = 0;
this.onIdle?.();
}
dispose(): void {
this.stop();
const context = this.context;
this.context = null;
if (context && context.state !== "closed") void context.close();
}
private getContext(): AudioContext {
if (!this.context || this.context.state === "closed") {
this.context = new AudioContext({ latencyHint: "interactive" });
}
return this.context;
}
private schedule(context: AudioContext, buffer: AudioBuffer): void {
const leadSeconds = 0.025;
const earliestStart = context.currentTime + leadSeconds;
const hadPreviousSegment = this.scheduledUntil > 0;
const startAt = Math.max(earliestStart, this.scheduledUntil);
if (hadPreviousSegment) {
this.onGap?.(Math.max(0, (startAt - this.scheduledUntil) * 1000));
}
const source = context.createBufferSource();
source.buffer = buffer;
source.connect(context.destination);
this.sources.add(source);
this.scheduledUntil = startAt + buffer.duration;
source.onended = () => {
source.disconnect();
this.sources.delete(source);
this.notifyIdleIfNeeded();
};
source.start(startAt);
this.onPlaying?.();
}
private notifyIdleIfNeeded(): void {
if (this.sources.size || this.pendingDecodes) return;
this.scheduledUntil = 0;
this.onIdle?.();
}
}
+47
View File
@@ -0,0 +1,47 @@
.screen {
position: relative;
display: flex;
min-height: var(--app-viewport-height, 100dvh);
overflow: hidden;
flex-direction: column;
color: #fff;
background: #090810;
}
.background { object-fit: cover; opacity: .48; }
.scrim { position: absolute; inset: 0; background: radial-gradient(circle at 50% 32%, rgba(239,83,155,.18), transparent 36%), linear-gradient(180deg, rgba(9,8,16,.28), rgba(9,8,16,.94)); }
.header { position: relative; z-index: 1; display: flex; align-items: center; justify-content: space-between; padding: calc(18px + var(--app-safe-top, 0px)) 20px 12px; }
.iconButton { display: inline-flex; width: 42px; height: 42px; cursor: pointer; align-items: center; justify-content: center; border: 1px solid rgba(255,255,255,.16); border-radius: 999px; color: #fff; background: rgba(10,9,18,.55); backdrop-filter: blur(14px); }
.balances { display: flex; gap: 8px; font-size: 12px; font-weight: 650; }
.balances span { padding: 7px 10px; border: 1px solid rgba(255,255,255,.12); border-radius: 999px; background: rgba(10,9,18,.58); backdrop-filter: blur(14px); }
.content { position: relative; z-index: 1; display: flex; min-height: 0; flex: 1; flex-direction: column; align-items: center; justify-content: center; padding: 24px 28px; text-align: center; }
.avatarRing { position: relative; display: grid; width: 174px; height: 174px; place-items: center; border-radius: 50%; background: linear-gradient(135deg, rgba(255,255,255,.24), rgba(239,83,155,.42)); box-shadow: 0 28px 72px rgba(0,0,0,.42); transition: transform .25s ease, box-shadow .25s ease; }
.avatarRing::after { position: absolute; inset: -10px; border: 1px solid rgba(255,255,255,.16); border-radius: inherit; content: ""; }
.avatarRing.listening, .avatarRing.speaking { animation: callPulse 1.8s ease-in-out infinite; box-shadow: 0 0 0 12px rgba(239,83,155,.08), 0 28px 72px rgba(0,0,0,.42); }
.avatar { width: 158px; height: 158px; border: 4px solid rgba(9,8,16,.8); border-radius: 50%; object-fit: cover; }
.content h1 { margin: 30px 0 8px; font-size: clamp(27px, 7vw, 36px); letter-spacing: -.025em; }
.state { display: flex; min-height: 28px; align-items: center; gap: 7px; color: rgba(255,255,255,.78); font-size: 15px; }
.lowBalance { margin: 12px 0 0; padding: 7px 11px; border-radius: 999px; color: #ffd6e8; background: rgba(210,48,121,.2); font-size: 12px; }
.transcript { width: min(100%, 440px); min-height: 105px; margin-top: 24px; text-align: left; }
.transcript p { margin: 0 0 10px; padding: 11px 13px; border: 1px solid rgba(255,255,255,.09); border-radius: 15px; color: rgba(255,255,255,.9); background: rgba(14,12,24,.48); font-size: 14px; line-height: 1.45; backdrop-filter: blur(12px); }
.transcript span { display: block; margin-bottom: 4px; color: #f3a3c8; font-size: 11px; font-weight: 700; text-transform: uppercase; }
.error { max-width: 360px; margin: 8px 0 0; color: #ffc2ca; font-size: 13px; }
.footer { position: relative; z-index: 1; display: flex; flex-direction: column; align-items: center; padding: 10px 24px calc(24px + var(--app-safe-bottom, 0px)); }
.footer p { margin: 12px 0 0; color: rgba(255,255,255,.52); font-size: 12px; }
.startButton { display: inline-flex; min-width: 170px; cursor: pointer; align-items: center; justify-content: center; gap: 9px; border: 0; border-radius: 999px; padding: 15px 24px; color: #fff; background: linear-gradient(135deg, #ee579d, #ce357c); box-shadow: 0 16px 36px rgba(212,54,127,.32); font-size: 15px; font-weight: 750; }
.endButton { display: inline-flex; width: 66px; height: 66px; cursor: pointer; align-items: center; justify-content: center; border: 0; border-radius: 50%; color: #fff; background: #e24456; box-shadow: 0 15px 34px rgba(226,68,86,.34); }
.topUpBackdrop { position: absolute; z-index: 10; inset: 0; display: flex; align-items: flex-end; justify-content: center; padding: 18px; background: rgba(5,4,9,.68); backdrop-filter: blur(8px); }
.topUpDialog { width: min(100%, 460px); padding: 24px; border: 1px solid rgba(255,255,255,.12); border-radius: 24px; background: #17131f; box-shadow: 0 24px 70px rgba(0,0,0,.5); text-align: center; }
.topUpDialog h2 { margin: 0; font-size: 21px; }
.topUpDialog p { margin: 10px 0 20px; color: rgba(255,255,255,.66); font-size: 14px; line-height: 1.5; }
.topUpDialog button { width: 100%; cursor: pointer; border: 0; border-radius: 999px; padding: 13px 18px; color: #fff; background: #df478d; font-weight: 750; }
.topUpDialog .secondaryButton { margin-top: 8px; color: rgba(255,255,255,.72); background: transparent; }
@keyframes callPulse { 0%,100% { transform: scale(1); } 50% { transform: scale(1.025); } }
@media (max-height: 700px) {
.avatarRing { width: 138px; height: 138px; }
.avatar { width: 124px; height: 124px; }
.content h1 { margin-top: 20px; }
.transcript { min-height: 72px; margin-top: 16px; }
}
+280
View File
@@ -0,0 +1,280 @@
"use client";
import Image from "next/image";
import { useCallback, useEffect, useRef, useState } from "react";
import { Mic, PhoneOff, RotateCcw, Volume2, X } from "lucide-react";
import { MobileShell } from "@/app/_components/core";
import { getApiConfig } from "@/core/net/config/api_config";
import {
VoiceCallTransport,
type CallBillingStatusPayload,
} from "@/core/net/voice-call-transport";
import { VoiceCallAudioCapture } from "@/integrations/voice-call-audio";
import { getSessionAccessToken } from "@/lib/auth/auth_session";
import { useActiveCharacter, useActiveCharacterRoutes } from "@/providers/character-provider";
import { useAppNavigator } from "@/router/use-app-navigator";
import { VoiceCallPlayer } from "./voice-call-player";
import styles from "./voice-call-screen.module.css";
type CallUiState = "idle" | "connecting" | "listening" | "thinking" | "speaking" | "ended" | "error";
const STATE_COPY: Record<CallUiState, string> = {
idle: "Ready to call",
connecting: "Connecting…",
listening: "Listening…",
thinking: "Thinking…",
speaking: "Speaking…",
ended: "Call ended",
error: "Call unavailable",
};
async function getSessionToken(): Promise<string> {
const token = await getSessionAccessToken();
if (token) return token;
throw new Error("Please sign in before starting a call");
}
export function VoiceCallScreen() {
const character = useActiveCharacter();
const routes = useActiveCharacterRoutes();
const navigator = useAppNavigator();
const [uiState, setUiState] = useState<CallUiState>("idle");
const [userTranscript, setUserTranscript] = useState("");
const [aiTranscript, setAiTranscript] = useState("");
const [freeTurns, setFreeTurns] = useState(0);
const [balance, setBalance] = useState(0);
const [cost, setCost] = useState(2);
const [error, setError] = useState<string | null>(null);
const [showTopUp, setShowTopUp] = useState(false);
const transportRef = useRef<VoiceCallTransport | null>(null);
const captureRef = useRef<VoiceCallAudioCapture | null>(null);
const playerRef = useRef<VoiceCallPlayer | null>(null);
const callIdRef = useRef("");
const activeTurnRef = useRef("");
const reconnectTurnRef = useRef("");
const resumeVersionRef = useRef(0);
const endingRef = useRef(false);
const pendingTopUpRef = useRef(false);
const finishCall = useCallback((navigate = true) => {
if (endingRef.current) return;
endingRef.current = true;
captureRef.current?.dispose();
playerRef.current?.dispose();
if (callIdRef.current) transportRef.current?.endCall(callIdRef.current);
transportRef.current?.disconnect();
reconnectTurnRef.current = "";
resumeVersionRef.current = 0;
setUiState("ended");
if (navigate) navigator.push(routes.chat);
}, [navigator, routes.chat]);
const beginListening = useCallback((turnId: string) => {
const capture = captureRef.current;
const transport = transportRef.current;
const callId = callIdRef.current;
if (!capture || !transport || !callId || !turnId) return;
activeTurnRef.current = turnId;
reconnectTurnRef.current = turnId;
setUiState(playerRef.current?.isPlaying ? "speaking" : "listening");
const preRoll: ArrayBuffer[] = [];
let streamOpened = false;
capture.startTurn({
onChunk: (chunk) => {
if (streamOpened) {
transport.sendAudioChunk(chunk);
return;
}
preRoll.push(chunk);
if (preRoll.length > 5) preRoll.shift();
},
onSpeechStart: () => {
setError(null);
if (playerRef.current?.isPlaying) {
playerRef.current.stop();
transport.interrupt(callId);
}
streamOpened = transport.startAudio(
callId,
turnId,
capture.mimeType,
capture.sampleRate,
);
if (streamOpened) {
for (const chunk of preRoll) transport.sendAudioChunk(chunk);
preRoll.length = 0;
setUserTranscript("");
setAiTranscript("");
setUiState("listening");
}
},
onTurnEnd: (vadLatencyMs) => {
if (streamOpened) {
transport.endAudio(callId, turnId, vadLatencyMs);
setUiState("thinking");
}
},
onError: (captureError) => {
setError(captureError.message);
setUiState("error");
},
});
}, []);
const applyBilling = useCallback((status: CallBillingStatusPayload) => {
setBalance(status.creditBalance);
setFreeTurns(status.freeTurnsRemaining);
setCost(status.requiredCredits || 2);
if (status.canContinue && status.nextTurnId) {
beginListening(status.nextTurnId);
return;
}
captureRef.current?.stopTurn();
pendingTopUpRef.current = true;
if (!playerRef.current?.isPlaying) setShowTopUp(true);
}, [beginListening]);
const startCall = useCallback(async () => {
if (uiState !== "idle" && uiState !== "error") return;
endingRef.current = false;
pendingTopUpRef.current = false;
setError(null);
setUiState("connecting");
try {
const player = new VoiceCallPlayer();
playerRef.current = player;
const playerReady = player.prepare();
const capture = new VoiceCallAudioCapture();
await Promise.all([capture.prepare(), playerReady]);
captureRef.current = capture;
const token = await getSessionToken();
const transport = new VoiceCallTransport(getApiConfig().wsUrl, token);
transportRef.current = transport;
player.onPlaying = () => setUiState("speaking");
player.onIdle = () => {
if (pendingTopUpRef.current) setShowTopUp(true);
else if (activeTurnRef.current) setUiState("listening");
};
player.onError = (playbackError) => setError(playbackError.message);
player.onGap = (milliseconds) => {
transport.sendClientMetric("audioChunkGap", milliseconds);
};
transport.onOpen = () => {
if (!reconnectTurnRef.current) reconnectTurnRef.current = crypto.randomUUID();
transport.startCall(
character.id,
reconnectTurnRef.current,
resumeVersionRef.current || undefined,
);
};
transport.onReconnecting = () => {
capture.stopTurn();
player.stop();
callIdRef.current = "";
setUiState("connecting");
};
transport.onStarted = (started) => {
callIdRef.current = started.callId;
activeTurnRef.current = started.turnId;
reconnectTurnRef.current = started.turnId;
resumeVersionRef.current = started.resumeVersion;
setFreeTurns(started.freeTurnsRemaining);
setBalance(started.creditBalance);
setCost(started.costPerPaidTurn || 2);
beginListening(started.turnId);
};
transport.onState = setUiState;
transport.onTranscript = (text) => setUserTranscript(text);
transport.onAiSentence = (text) => {
setAiTranscript((current) => `${current}${current ? " " : ""}${text}`);
};
transport.onAudioChunk = (meta, bytes) => player.enqueue(meta, bytes);
transport.onBillingStatus = applyBilling;
transport.onEnded = () => setUiState("ended");
transport.onError = (code, message) => {
setError(message);
if (
activeTurnRef.current
&& (code === "CALL_TURN_FAILED" || code === "CALL_STT_FAILED")
) {
setUiState("listening");
} else {
setUiState("error");
}
};
transport.connect();
} catch (startError) {
captureRef.current?.dispose();
playerRef.current?.dispose();
setError(startError instanceof Error ? startError.message : String(startError));
setUiState("error");
}
}, [applyBilling, beginListening, character.id, uiState]);
useEffect(() => () => finishCall(false), [finishCall]);
const lowBalance = freeTurns === 0 && balance < cost * 2 && balance >= cost;
return (
<MobileShell background="#090810">
<main className={styles.screen}>
<Image src={character.assets.chatBackground} alt="" fill priority className={styles.background} />
<div className={styles.scrim} />
<header className={styles.header}>
<button type="button" className={styles.iconButton} onClick={() => finishCall()} aria-label="Close call">
<X size={22} aria-hidden="true" />
</button>
<div className={styles.balances}>
<span>{freeTurns} free</span>
<span>{balance} credits</span>
</div>
</header>
<section className={styles.content} aria-live="polite">
<div className={`${styles.avatarRing} ${styles[uiState]}`}>
<Image src={character.assets.avatar} alt={character.displayName} width={148} height={148} className={styles.avatar} priority />
</div>
<h1>{character.displayName}</h1>
<div className={styles.state}>
{uiState === "listening" ? <Mic size={18} aria-hidden="true" /> : null}
{uiState === "speaking" ? <Volume2 size={18} aria-hidden="true" /> : null}
<span>{STATE_COPY[uiState]}</span>
</div>
{lowBalance ? <p className={styles.lowBalance}>Your balance covers fewer than two paid turns.</p> : null}
<div className={styles.transcript}>
{userTranscript ? <p><span>You</span>{userTranscript}</p> : null}
{aiTranscript ? <p><span>{character.shortName}</span>{aiTranscript}</p> : null}
</div>
{error ? <p className={styles.error}>{error}</p> : null}
</section>
<footer className={styles.footer}>
{uiState === "idle" || uiState === "error" ? (
<button type="button" className={styles.startButton} onClick={() => void startCall()}>
{uiState === "error" ? <RotateCcw size={20} aria-hidden="true" /> : <Mic size={20} aria-hidden="true" />}
{uiState === "error" ? "Try again" : "Start call"}
</button>
) : (
<button type="button" className={styles.endButton} onClick={() => finishCall()} aria-label="End call">
<PhoneOff size={27} aria-hidden="true" />
</button>
)}
<p>Speak naturally you can interrupt at any time.</p>
</footer>
{showTopUp ? (
<div className={styles.topUpBackdrop} role="presentation">
<section className={styles.topUpDialog} role="dialog" aria-modal="true" aria-labelledby="call-topup-title">
<h2 id="call-topup-title">Not enough credits for another turn</h2>
<p>Your current reply has finished. Top up to keep the conversation going.</p>
<button type="button" onClick={() => navigator.openSubscription({ type: "topup", returnTo: "chat" })}>Top up credits</button>
<button type="button" className={styles.secondaryButton} onClick={() => finishCall()}>End call</button>
</section>
</div>
) : null}
</main>
</MobileShell>
);
}
@@ -0,0 +1,5 @@
import { VoiceCallScreen } from "@/app/call/voice-call-screen";
export default function CharacterCallPage() {
return <VoiceCallScreen />;
}
+20 -5
View File
@@ -5,7 +5,7 @@
* 顶部保持返回、首充优惠、收藏入口三段结构。
*/
import type { ReactNode } from "react";
import { Lock } from "lucide-react";
import { Lock, Phone } from "lucide-react";
import { BackButton } from "@/app/_components";
import { FavoriteEntryButton } from "@/app/_components/core";
@@ -27,6 +27,7 @@ export function ChatHeader({
const navigator = useAppNavigator();
const character = useActiveCharacter();
const characterRoutes = useActiveCharacterRoutes();
const voiceCallEnabled = process.env.NEXT_PUBLIC_ENABLE_VOICE_CALL === "true";
return (
<header className="flex shrink-0 flex-col items-stretch bg-transparent p-0">
@@ -58,10 +59,24 @@ export function ChatHeader({
<div className="flex min-w-0 justify-center">{offerBanner}</div>
<FavoriteEntryButton
characterSlug={character.slug}
tone="dark"
/>
<div className="flex items-center gap-2">
{voiceCallEnabled ? (
<button
type="button"
className="inline-flex size-(--responsive-icon-button-size,42px) cursor-pointer items-center justify-center rounded-full border border-[rgba(255,255,255,0.12)] bg-[rgba(13,11,20,0.7)] text-white shadow-[0_10px_24px_rgba(0,0,0,0.2)] backdrop-blur-md transition hover:bg-[rgba(28,24,39,0.82)] active:scale-96"
aria-label={`Call ${character.shortName}`}
data-analytics-key="chat.start_voice_call"
data-analytics-label="Start voice call"
onClick={() => navigator.push(characterRoutes.call)}
>
<Phone size={19} strokeWidth={2.3} aria-hidden="true" />
</button>
) : null}
<FavoriteEntryButton
characterSlug={character.slug}
tone="dark"
/>
</div>
</div>
</header>
);