feat(call): add streaming voice call experience

src/integrations/__tests__/voice-call-audio.test.ts

@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vitest";
+
+import { AdaptiveVad } from "@/integrations/voice-call-audio";
+
+describe("AdaptiveVad", () => {
+  it("adapts its silence window between 400 and 600ms", () => {
+    const quiet = new AdaptiveVad();
+    for (let i = 0; i < 20; i += 1) quiet.observe(0.005, i * 50);
+    expect(quiet.silenceWindowMs).toBeGreaterThanOrEqual(400);
+    expect(quiet.silenceWindowMs).toBeLessThanOrEqual(600);
+
+    const noisy = new AdaptiveVad();
+    for (let i = 0; i < 20; i += 1) noisy.observe(0.04, i * 50);
+    expect(noisy.silenceWindowMs).toBeGreaterThanOrEqual(quiet.silenceWindowMs);
+    expect(noisy.silenceWindowMs).toBeLessThanOrEqual(600);
+  });
+
+  it("ends a turn after speech followed by adaptive silence", () => {
+    const vad = new AdaptiveVad();
+    for (let i = 0; i < 10; i += 1) vad.observe(0.004, i * 50);
+    expect(vad.observe(0.2, 500).speechStarted).toBe(true);
+    let ended = false;
+    let silenceDurationMs = 0;
+    for (let t = 550; t <= 1200; t += 50) {
+      const observation = vad.observe(0.001, t);
+      ended ||= observation.turnEnded;
+      silenceDurationMs ||= observation.silenceDurationMs;
+    }
+    expect(ended).toBe(true);
+    expect(silenceDurationMs).toBeGreaterThanOrEqual(400);
+    expect(silenceDurationMs).toBeLessThanOrEqual(600);
+  });
+});

src/integrations/voice-call-audio.ts

@@ -0,0 +1,142 @@
+export interface VadObservation {
+  speechStarted: boolean;
+  turnEnded: boolean;
+  silenceDurationMs: number;
+}
+
+export class AdaptiveVad {
+  private noiseFloor = 0.008;
+  private speaking = false;
+  private lastSpeechAt = 0;
+
+  get silenceWindowMs(): number {
+    return Math.max(400, Math.min(600, Math.round(400 + this.noiseFloor * 5000)));
+  }
+
+  observe(rms: number, nowMs: number): VadObservation {
+    const level = Math.max(0, Number.isFinite(rms) ? rms : 0);
+    if (!this.speaking) {
+      this.noiseFloor = this.noiseFloor * 0.9 + Math.min(level, 0.06) * 0.1;
+    }
+    const speechThreshold = Math.max(0.018, this.noiseFloor * 2.8);
+    const isSpeech = level >= speechThreshold;
+    let speechStarted = false;
+    let turnEnded = false;
+    let silenceDurationMs = 0;
+    if (isSpeech) {
+      speechStarted = !this.speaking;
+      this.speaking = true;
+      this.lastSpeechAt = nowMs;
+    } else if (this.speaking && nowMs - this.lastSpeechAt >= this.silenceWindowMs) {
+      this.speaking = false;
+      turnEnded = true;
+      silenceDurationMs = nowMs - this.lastSpeechAt;
+    }
+    return { speechStarted, turnEnded, silenceDurationMs };
+  }
+
+  reset(): void {
+    this.speaking = false;
+    this.lastSpeechAt = 0;
+  }
+}
+
+export interface VoiceCaptureCallbacks {
+  onChunk: (chunk: ArrayBuffer) => void;
+  onSpeechStart?: () => void;
+  onTurnEnd: (vadLatencyMs: number) => void;
+  onError: (error: Error) => void;
+}
+
+export class VoiceCallAudioCapture {
+  private stream: MediaStream | null = null;
+  private audioContext: AudioContext | null = null;
+  private analyser: AnalyserNode | null = null;
+  private recorder: MediaRecorder | null = null;
+  private vadTimer: number | null = null;
+  private pendingChunkReads: Promise<void> = Promise.resolve();
+  private readonly vad = new AdaptiveVad();
+
+  get mimeType(): string {
+    if (typeof MediaRecorder === "undefined") return "audio/webm;codecs=opus";
+    if (MediaRecorder.isTypeSupported("audio/webm;codecs=opus")) return "audio/webm;codecs=opus";
+    if (MediaRecorder.isTypeSupported("audio/ogg;codecs=opus")) return "audio/ogg;codecs=opus";
+    return "audio/webm";
+  }
+
+  get sampleRate(): number {
+    return this.audioContext?.sampleRate ?? 48000;
+  }
+
+  async prepare(): Promise<void> {
+    if (this.stream) return;
+    this.stream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+        channelCount: 1,
+      },
+    });
+    this.audioContext = new AudioContext({ latencyHint: "interactive" });
+    await this.audioContext.resume();
+    this.analyser = this.audioContext.createAnalyser();
+    this.analyser.fftSize = 1024;
+    this.audioContext.createMediaStreamSource(this.stream).connect(this.analyser);
+  }
+
+  startTurn(callbacks: VoiceCaptureCallbacks): void {
+    if (!this.stream || !this.analyser) throw new Error("Microphone is not prepared");
+    if (this.recorder?.state === "recording") return;
+    this.vad.reset();
+    this.pendingChunkReads = Promise.resolve();
+    const recorder = new MediaRecorder(this.stream, { mimeType: this.mimeType });
+    this.recorder = recorder;
+    recorder.ondataavailable = (event) => {
+      if (event.data.size === 0) return;
+      this.pendingChunkReads = this.pendingChunkReads
+        .then(() => event.data.arrayBuffer())
+        .then(callbacks.onChunk)
+        .catch((error: unknown) => {
+          callbacks.onError(error instanceof Error ? error : new Error(String(error)));
+        });
+    };
+    recorder.onerror = () => callbacks.onError(new Error("Microphone recorder failed"));
+    recorder.start(120);
+    const samples = new Float32Array(this.analyser.fftSize);
+    this.vadTimer = window.setInterval(() => {
+      this.analyser?.getFloatTimeDomainData(samples);
+      let squareSum = 0;
+      for (const sample of samples) squareSum += sample * sample;
+      const rms = Math.sqrt(squareSum / samples.length);
+      const observation = this.vad.observe(rms, performance.now());
+      if (observation.speechStarted) callbacks.onSpeechStart?.();
+      if (observation.turnEnded) {
+        this.stopTurn(() => callbacks.onTurnEnd(observation.silenceDurationMs));
+      }
+    }, 50);
+  }
+
+  stopTurn(onStopped?: () => void): void {
+    if (this.vadTimer !== null) window.clearInterval(this.vadTimer);
+    this.vadTimer = null;
+    if (this.recorder?.state === "recording") {
+      this.recorder.onstop = () => {
+        void this.pendingChunkReads.then(() => onStopped?.());
+      };
+      this.recorder.stop();
+    } else {
+      onStopped?.();
+    }
+    this.recorder = null;
+  }
+
+  dispose(): void {
+    this.stopTurn();
+    this.stream?.getTracks().forEach((track) => track.stop());
+    this.stream = null;
+    void this.audioContext?.close();
+    this.audioContext = null;
+    this.analyser = null;
+  }
+}