Restored to '84d7de2da64f8f2fd1e2d64959817c64c8fb7c04'

Replit-Restored-To: 84d7de2da6
2026-02-28 20:36:50 +00:00
parent 802282187e
commit c441cc1594
76 changed files with 4644 additions and 212 deletions
@@ -0,0 +1,112 @@
+/**
+ * Reusable AudioWorklet for streaming PCM16 audio playback.
+ * Place in public/ folder and load via audioContext.audioWorklet.addModule()
+ */
+class RingBuffer {
+  constructor(initialCapacity) {
+    this.capacity = initialCapacity;
+    this.buffer = new Float32Array(initialCapacity);
+    this.readIndex = 0;
+    this.writeIndex = 0;
+    this.availableData = 0;
+  }
+
+  push(data) {
+    const len = data.length;
+    // Auto-grow if needed
+    while (this.availableData + len > this.capacity) {
+      this.grow();
+    }
+    for (let i = 0; i < len; i++) {
+      this.buffer[this.writeIndex] = data[i];
+      this.writeIndex = (this.writeIndex + 1) % this.capacity;
+      this.availableData++;
+    }
+  }
+
+  grow() {
+    const newCapacity = this.capacity * 2;
+    const newBuffer = new Float32Array(newCapacity);
+    // Copy existing data maintaining order
+    for (let i = 0; i < this.availableData; i++) {
+      const srcIndex = (this.readIndex + i) % this.capacity;
+      newBuffer[i] = this.buffer[srcIndex];
+    }
+    this.buffer = newBuffer;
+    this.readIndex = 0;
+    this.writeIndex = this.availableData;
+    this.capacity = newCapacity;
+  }
+
+  pull(outputBuffer) {
+    const len = outputBuffer.length;
+    const available = Math.min(len, this.availableData);
+    for (let i = 0; i < available; i++) {
+      outputBuffer[i] = this.buffer[this.readIndex];
+      this.readIndex = (this.readIndex + 1) % this.capacity;
+    }
+    // Pad remaining with silence
+    for (let i = available; i < len; i++) {
+      outputBuffer[i] = 0;
+    }
+    this.availableData -= available;
+    return available > 0;
+  }
+
+  available() {
+    return this.availableData;
+  }
+
+  clear() {
+    this.readIndex = 0;
+    this.writeIndex = 0;
+    this.availableData = 0;
+  }
+}
+
+class AudioPlaybackProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.ringBuffer = new RingBuffer(24000 * 30); // 30s initial capacity
+    this.isPlaying = false;
+    this.streamComplete = false;
+
+    this.port.onmessage = (event) => {
+      const { type, samples } = event.data;
+      if (type === "audio") {
+        this.ringBuffer.push(samples);
+        this.isPlaying = true;
+      } else if (type === "clear") {
+        this.ringBuffer.clear();
+        this.isPlaying = false;
+        this.streamComplete = false;
+      } else if (type === "streamComplete") {
+        this.streamComplete = true;
+      } else if (type === "stop") {
+        this.isPlaying = false;
+        this.streamComplete = false;
+      }
+    };
+  }
+
+  process(inputs, outputs) {
+    const output = outputs[0];
+    if (!output || output.length === 0) return true;
+
+    const channel = output[0];
+    if (this.isPlaying) {
+      this.ringBuffer.pull(channel);
+      if (this.streamComplete && this.ringBuffer.available() === 0) {
+        this.isPlaying = false;
+        this.streamComplete = false;
+        this.port.postMessage({ type: "ended" });
+      }
+    } else {
+      channel.fill(0);
+    }
+    return true;
+  }
+}
+
+registerProcessor("audio-playback-processor", AudioPlaybackProcessor);
+
@@ -0,0 +1,36 @@
+/**
+ * Audio utility functions for voice chat.
+ * Handles PCM16 decoding and AudioContext initialization.
+ */
+
+/**
+ * Decode base64 PCM16 audio to Float32Array for Web Audio API
+ */
+export function decodePCM16ToFloat32(base64Audio: string): Float32Array {
+  const raw = atob(base64Audio);
+  const bytes = new Uint8Array(raw.length);
+  for (let i = 0; i < raw.length; i++) {
+    bytes[i] = raw.charCodeAt(i);
+  }
+  const pcm16 = new Int16Array(bytes.buffer);
+  const float32 = new Float32Array(pcm16.length);
+  for (let i = 0; i < pcm16.length; i++) {
+    float32[i] = pcm16[i] / 32768;
+  }
+  return float32;
+}
+
+/**
+ * Create and initialize AudioContext with worklet
+ */
+export async function createAudioPlaybackContext(
+  workletPath = "/audio-playback-worklet.js",
+  sampleRate = 24000
+): Promise<{ ctx: AudioContext; worklet: AudioWorkletNode }> {
+  const ctx = new AudioContext({ sampleRate });
+  await ctx.audioWorklet.addModule(workletPath);
+  const worklet = new AudioWorkletNode(ctx, "audio-playback-processor");
+  worklet.connect(ctx.destination);
+  return { ctx, worklet };
+}
+
@@ -0,0 +1,45 @@
+/**
+ * Voice chat client utilities for Replit AI Integrations.
+ * 
+ * Usage:
+ * 1. Copy audio-playback-worklet.js to your public/ folder
+ * 2. Import and use the React hooks in your components
+ * 
+ * Example:
+ * ```tsx
+ * import { useVoiceRecorder, useVoiceStream } from "./audio";
+ * 
+ * function VoiceChat() {
+ *   const [transcript, setTranscript] = useState("");
+ *   const recorder = useVoiceRecorder();
+ *   const stream = useVoiceStream({
+ *     onTranscript: (_, full) => setTranscript(full),
+ *     onComplete: (text) => console.log("Done:", text),
+ *   });
+ * 
+ *   const handleClick = async () => {
+ *     if (recorder.state === "recording") {
+ *       const blob = await recorder.stopRecording();
+ *       await stream.streamVoiceResponse("/api/voice-conversations/1/messages", blob);
+ *     } else {
+ *       await recorder.startRecording();
+ *     }
+ *   };
+ * 
+ *   return (
+ *     <div>
+ *       <button onClick={handleClick}>
+ *         {recorder.state === "recording" ? "Stop" : "Record"}
+ *       </button>
+ *       <p>{transcript}</p>
+ *     </div>
+ *   );
+ * }
+ * ```
+ */
+
+export { decodePCM16ToFloat32, createAudioPlaybackContext } from "./audio-utils";
+export { useVoiceRecorder, type RecordingState } from "./useVoiceRecorder";
+export { useAudioPlayback, type PlaybackState } from "./useAudioPlayback";
+export { useVoiceStream } from "./useVoiceStream";
+
@@ -0,0 +1,105 @@
+/**
+ * React hook for streaming audio playback using AudioWorklet.
+ * Supports real-time PCM16 audio streaming from SSE responses.
+ * Includes sequence buffer for reordering out-of-order chunks.
+ */
+import { useRef, useCallback, useState } from "react";
+import { decodePCM16ToFloat32 } from "./audio-utils";
+
+export type PlaybackState = "idle" | "playing" | "ended";
+
+/**
+ * Reorders audio chunks that may arrive out of sequence.
+ * Buffers chunks until they can be played in correct order.
+ *
+ * Example: If chunks arrive as seq 2, seq 0, seq 1:
+ * - seq 2 arrives → buffered (waiting for seq 0)
+ * - seq 0 arrives → played immediately, then check buffer
+ * - seq 1 arrives → played immediately (seq 0 done), seq 2 now plays
+ */
+class SequenceBuffer {
+  private pending = new Map<number, string[]>();
+  private nextSeq = 0;
+
+  /** Add chunk with sequence number, returns chunks ready to play in order */
+  push(seq: number, data: string): string[] {
+    // Store the chunk under its sequence number
+    if (!this.pending.has(seq)) {
+      this.pending.set(seq, []);
+    }
+    this.pending.get(seq)!.push(data);
+
+    // Drain consecutive ready sequences
+    const ready: string[] = [];
+    while (this.pending.has(this.nextSeq)) {
+      ready.push(...this.pending.get(this.nextSeq)!);
+      this.pending.delete(this.nextSeq);
+      this.nextSeq++;
+    }
+    return ready;
+  }
+
+  reset() {
+    this.pending.clear();
+    this.nextSeq = 0;
+  }
+}
+
+export function useAudioPlayback(workletPath = "/audio-playback-worklet.js") {
+  const [state, setState] = useState<PlaybackState>("idle");
+  const ctxRef = useRef<AudioContext | null>(null);
+  const workletRef = useRef<AudioWorkletNode | null>(null);
+  const readyRef = useRef(false);
+  const seqBufferRef = useRef(new SequenceBuffer());
+
+  const init = useCallback(async () => {
+    if (readyRef.current) return;
+
+    const ctx = new AudioContext({ sampleRate: 24000 });
+    await ctx.audioWorklet.addModule(workletPath);
+    const worklet = new AudioWorkletNode(ctx, "audio-playback-processor");
+    worklet.connect(ctx.destination);
+
+    worklet.port.onmessage = (e) => {
+      if (e.data.type === "ended") setState("idle");
+    };
+
+    ctxRef.current = ctx;
+    workletRef.current = worklet;
+    readyRef.current = true;
+  }, [workletPath]);
+
+  /** Push audio directly (no sequencing) - for simple streaming */
+  const pushAudio = useCallback((base64Audio: string) => {
+    if (!workletRef.current) return;
+    const samples = decodePCM16ToFloat32(base64Audio);
+    workletRef.current.port.postMessage({ type: "audio", samples });
+    setState("playing");
+  }, []);
+
+  /** Push audio with sequence number - reorders before playback */
+  const pushSequencedAudio = useCallback((seq: number, base64Audio: string) => {
+    if (!workletRef.current) return;
+
+    const readyChunks = seqBufferRef.current.push(seq, base64Audio);
+    for (const chunk of readyChunks) {
+      const samples = decodePCM16ToFloat32(chunk);
+      workletRef.current.port.postMessage({ type: "audio", samples });
+    }
+    if (readyChunks.length > 0) {
+      setState("playing");
+    }
+  }, []);
+
+  const signalComplete = useCallback(() => {
+    workletRef.current?.port.postMessage({ type: "streamComplete" });
+  }, []);
+
+  const clear = useCallback(() => {
+    workletRef.current?.port.postMessage({ type: "clear" });
+    seqBufferRef.current.reset();
+    setState("idle");
+  }, []);
+
+  return { state, init, pushAudio, pushSequencedAudio, signalComplete, clear };
+}
@@ -0,0 +1,52 @@
+/**
+ * React hook for voice recording using MediaRecorder API.
+ * Records audio in WebM/Opus format for efficient streaming.
+ */
+import { useRef, useCallback, useState } from "react";
+
+export type RecordingState = "idle" | "recording" | "stopped";
+
+export function useVoiceRecorder() {
+  const [state, setState] = useState<RecordingState>("idle");
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const chunksRef = useRef<Blob[]>([]);
+
+  const startRecording = useCallback(async (): Promise<void> => {
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    const recorder = new MediaRecorder(stream, {
+      mimeType: "audio/webm;codecs=opus",
+    });
+
+    mediaRecorderRef.current = recorder;
+    chunksRef.current = [];
+
+    recorder.ondataavailable = (e) => {
+      if (e.data.size > 0) chunksRef.current.push(e.data);
+    };
+
+    recorder.start(100); // Collect chunks every 100ms
+    setState("recording");
+  }, []);
+
+  const stopRecording = useCallback((): Promise<Blob> => {
+    return new Promise((resolve) => {
+      const recorder = mediaRecorderRef.current;
+      if (!recorder || recorder.state !== "recording") {
+        resolve(new Blob());
+        return;
+      }
+
+      recorder.onstop = () => {
+        const blob = new Blob(chunksRef.current, { type: "audio/webm" });
+        recorder.stream.getTracks().forEach((t) => t.stop());
+        setState("stopped");
+        resolve(blob);
+      };
+
+      recorder.stop();
+    });
+  }, []);
+
+  return { state, startRecording, stopRecording };
+}
+
@@ -0,0 +1,91 @@
+/**
+ * React hook for handling SSE voice streaming responses.
+ * Converts audio blob to base64 and sends as JSON to match server expectations.
+ */
+import { useCallback } from "react";
+import { useAudioPlayback } from "./useAudioPlayback";
+
+interface StreamCallbacks {
+  onUserTranscript?: (text: string) => void;
+  onTranscript?: (text: string, full: string) => void;
+  onComplete?: (transcript: string) => void;
+  onError?: (error: Error) => void;
+}
+
+export function useVoiceStream(callbacks: StreamCallbacks = {}) {
+  const playback = useAudioPlayback();
+
+  const streamVoiceResponse = useCallback(
+    async (url: string, audioBlob: Blob) => {
+      await playback.init();
+      playback.clear();
+
+      // Convert blob to base64 for JSON body (server expects express.json())
+      const base64Audio = await new Promise<string>((resolve) => {
+        const fileReader = new FileReader();
+        fileReader.onload = () => {
+          const result = fileReader.result as string;
+          resolve(result.split(",")[1]); // Remove data URL prefix
+        };
+        fileReader.readAsDataURL(audioBlob);
+      });
+
+      const response = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ audio: base64Audio }),
+      });
+      if (!response.ok) throw new Error("Voice request failed");
+
+      const streamReader = response.body?.getReader();
+      if (!streamReader) throw new Error("No response body");
+
+      const decoder = new TextDecoder();
+      let buffer = "";
+      let fullTranscript = "";
+
+      while (true) {
+        const { done, value } = await streamReader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() || "";
+
+        for (const line of lines) {
+          if (!line.startsWith("data: ")) continue;
+
+          try {
+            const event = JSON.parse(line.slice(6));
+
+            switch (event.type) {
+              case "user_transcript":
+                callbacks.onUserTranscript?.(event.data);
+                break;
+              case "transcript":
+                fullTranscript += event.data;
+                callbacks.onTranscript?.(event.data, fullTranscript);
+                break;
+              case "audio":
+                playback.pushAudio(event.data);
+                break;
+              case "done":
+                playback.signalComplete();
+                callbacks.onComplete?.(fullTranscript);
+                break;
+              case "error":
+                throw new Error(event.error);
+            }
+          } catch (e) {
+            if (!(e instanceof SyntaxError)) {
+              callbacks.onError?.(e as Error);
+            }
+          }
+        }
+      }
+    },
+    [playback, callbacks]
+  );
+
+  return { streamVoiceResponse, playbackState: playback.state };
+}