Fix hero card aspect ratio and add horoscope generation functionality
Introduce horoscope generation via OpenAI API, including new API endpoints and database schema. Adjust card components in `home.tsx` to use `aspect-[16/9]` for consistent image sizing, resolving previous height stretching issues. Update dependencies in `package.json`. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 413891e8-d784-4bea-b9f5-91a5a68316b4 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: ca1aa952-242c-43c1-9e28-47aed39cee1b Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/f209e72a-0939-48fa-84fc-57854de71967/413891e8-d784-4bea-b9f5-91a5a68316b4/nTLKCC5 Replit-Helium-Checkpoint-Created: true
This commit is contained in:
sebastjanartic
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Reusable AudioWorklet for streaming PCM16 audio playback.
|
||||
* Place in public/ folder and load via audioContext.audioWorklet.addModule()
|
||||
*/
|
||||
class RingBuffer {
|
||||
constructor(initialCapacity) {
|
||||
this.capacity = initialCapacity;
|
||||
this.buffer = new Float32Array(initialCapacity);
|
||||
this.readIndex = 0;
|
||||
this.writeIndex = 0;
|
||||
this.availableData = 0;
|
||||
}
|
||||
|
||||
push(data) {
|
||||
const len = data.length;
|
||||
// Auto-grow if needed
|
||||
while (this.availableData + len > this.capacity) {
|
||||
this.grow();
|
||||
}
|
||||
for (let i = 0; i < len; i++) {
|
||||
this.buffer[this.writeIndex] = data[i];
|
||||
this.writeIndex = (this.writeIndex + 1) % this.capacity;
|
||||
this.availableData++;
|
||||
}
|
||||
}
|
||||
|
||||
grow() {
|
||||
const newCapacity = this.capacity * 2;
|
||||
const newBuffer = new Float32Array(newCapacity);
|
||||
// Copy existing data maintaining order
|
||||
for (let i = 0; i < this.availableData; i++) {
|
||||
const srcIndex = (this.readIndex + i) % this.capacity;
|
||||
newBuffer[i] = this.buffer[srcIndex];
|
||||
}
|
||||
this.buffer = newBuffer;
|
||||
this.readIndex = 0;
|
||||
this.writeIndex = this.availableData;
|
||||
this.capacity = newCapacity;
|
||||
}
|
||||
|
||||
pull(outputBuffer) {
|
||||
const len = outputBuffer.length;
|
||||
const available = Math.min(len, this.availableData);
|
||||
for (let i = 0; i < available; i++) {
|
||||
outputBuffer[i] = this.buffer[this.readIndex];
|
||||
this.readIndex = (this.readIndex + 1) % this.capacity;
|
||||
}
|
||||
// Pad remaining with silence
|
||||
for (let i = available; i < len; i++) {
|
||||
outputBuffer[i] = 0;
|
||||
}
|
||||
this.availableData -= available;
|
||||
return available > 0;
|
||||
}
|
||||
|
||||
available() {
|
||||
return this.availableData;
|
||||
}
|
||||
|
||||
clear() {
|
||||
this.readIndex = 0;
|
||||
this.writeIndex = 0;
|
||||
this.availableData = 0;
|
||||
}
|
||||
}
|
||||
|
||||
class AudioPlaybackProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.ringBuffer = new RingBuffer(24000 * 30); // 30s initial capacity
|
||||
this.isPlaying = false;
|
||||
this.streamComplete = false;
|
||||
|
||||
this.port.onmessage = (event) => {
|
||||
const { type, samples } = event.data;
|
||||
if (type === "audio") {
|
||||
this.ringBuffer.push(samples);
|
||||
this.isPlaying = true;
|
||||
} else if (type === "clear") {
|
||||
this.ringBuffer.clear();
|
||||
this.isPlaying = false;
|
||||
this.streamComplete = false;
|
||||
} else if (type === "streamComplete") {
|
||||
this.streamComplete = true;
|
||||
} else if (type === "stop") {
|
||||
this.isPlaying = false;
|
||||
this.streamComplete = false;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
process(inputs, outputs) {
|
||||
const output = outputs[0];
|
||||
if (!output || output.length === 0) return true;
|
||||
|
||||
const channel = output[0];
|
||||
if (this.isPlaying) {
|
||||
this.ringBuffer.pull(channel);
|
||||
if (this.streamComplete && this.ringBuffer.available() === 0) {
|
||||
this.isPlaying = false;
|
||||
this.streamComplete = false;
|
||||
this.port.postMessage({ type: "ended" });
|
||||
}
|
||||
} else {
|
||||
channel.fill(0);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("audio-playback-processor", AudioPlaybackProcessor);
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Audio utility functions for voice chat.
|
||||
* Handles PCM16 decoding and AudioContext initialization.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Decode base64 PCM16 audio to Float32Array for Web Audio API
|
||||
*/
|
||||
export function decodePCM16ToFloat32(base64Audio: string): Float32Array {
|
||||
const raw = atob(base64Audio);
|
||||
const bytes = new Uint8Array(raw.length);
|
||||
for (let i = 0; i < raw.length; i++) {
|
||||
bytes[i] = raw.charCodeAt(i);
|
||||
}
|
||||
const pcm16 = new Int16Array(bytes.buffer);
|
||||
const float32 = new Float32Array(pcm16.length);
|
||||
for (let i = 0; i < pcm16.length; i++) {
|
||||
float32[i] = pcm16[i] / 32768;
|
||||
}
|
||||
return float32;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create and initialize AudioContext with worklet
|
||||
*/
|
||||
export async function createAudioPlaybackContext(
|
||||
workletPath = "/audio-playback-worklet.js",
|
||||
sampleRate = 24000
|
||||
): Promise<{ ctx: AudioContext; worklet: AudioWorkletNode }> {
|
||||
const ctx = new AudioContext({ sampleRate });
|
||||
await ctx.audioWorklet.addModule(workletPath);
|
||||
const worklet = new AudioWorkletNode(ctx, "audio-playback-processor");
|
||||
worklet.connect(ctx.destination);
|
||||
return { ctx, worklet };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Voice chat client utilities for Replit AI Integrations.
|
||||
*
|
||||
* Usage:
|
||||
* 1. Copy audio-playback-worklet.js to your public/ folder
|
||||
* 2. Import and use the React hooks in your components
|
||||
*
|
||||
* Example:
|
||||
* ```tsx
|
||||
* import { useVoiceRecorder, useVoiceStream } from "./audio";
|
||||
*
|
||||
* function VoiceChat() {
|
||||
* const [transcript, setTranscript] = useState("");
|
||||
* const recorder = useVoiceRecorder();
|
||||
* const stream = useVoiceStream({
|
||||
* onTranscript: (_, full) => setTranscript(full),
|
||||
* onComplete: (text) => console.log("Done:", text),
|
||||
* });
|
||||
*
|
||||
* const handleClick = async () => {
|
||||
* if (recorder.state === "recording") {
|
||||
* const blob = await recorder.stopRecording();
|
||||
* await stream.streamVoiceResponse("/api/voice-conversations/1/messages", blob);
|
||||
* } else {
|
||||
* await recorder.startRecording();
|
||||
* }
|
||||
* };
|
||||
*
|
||||
* return (
|
||||
* <div>
|
||||
* <button onClick={handleClick}>
|
||||
* {recorder.state === "recording" ? "Stop" : "Record"}
|
||||
* </button>
|
||||
* <p>{transcript}</p>
|
||||
* </div>
|
||||
* );
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
|
||||
export { decodePCM16ToFloat32, createAudioPlaybackContext } from "./audio-utils";
|
||||
export { useVoiceRecorder, type RecordingState } from "./useVoiceRecorder";
|
||||
export { useAudioPlayback, type PlaybackState } from "./useAudioPlayback";
|
||||
export { useVoiceStream } from "./useVoiceStream";
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* React hook for streaming audio playback using AudioWorklet.
|
||||
* Supports real-time PCM16 audio streaming from SSE responses.
|
||||
* Includes sequence buffer for reordering out-of-order chunks.
|
||||
*/
|
||||
import { useRef, useCallback, useState } from "react";
|
||||
import { decodePCM16ToFloat32 } from "./audio-utils";
|
||||
|
||||
export type PlaybackState = "idle" | "playing" | "ended";
|
||||
|
||||
/**
|
||||
* Reorders audio chunks that may arrive out of sequence.
|
||||
* Buffers chunks until they can be played in correct order.
|
||||
*
|
||||
* Example: If chunks arrive as seq 2, seq 0, seq 1:
|
||||
* - seq 2 arrives → buffered (waiting for seq 0)
|
||||
* - seq 0 arrives → played immediately, then check buffer
|
||||
* - seq 1 arrives → played immediately (seq 0 done), seq 2 now plays
|
||||
*/
|
||||
class SequenceBuffer {
|
||||
private pending = new Map<number, string[]>();
|
||||
private nextSeq = 0;
|
||||
|
||||
/** Add chunk with sequence number, returns chunks ready to play in order */
|
||||
push(seq: number, data: string): string[] {
|
||||
// Store the chunk under its sequence number
|
||||
if (!this.pending.has(seq)) {
|
||||
this.pending.set(seq, []);
|
||||
}
|
||||
this.pending.get(seq)!.push(data);
|
||||
|
||||
// Drain consecutive ready sequences
|
||||
const ready: string[] = [];
|
||||
while (this.pending.has(this.nextSeq)) {
|
||||
ready.push(...this.pending.get(this.nextSeq)!);
|
||||
this.pending.delete(this.nextSeq);
|
||||
this.nextSeq++;
|
||||
}
|
||||
return ready;
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.pending.clear();
|
||||
this.nextSeq = 0;
|
||||
}
|
||||
}
|
||||
|
||||
export function useAudioPlayback(workletPath = "/audio-playback-worklet.js") {
|
||||
const [state, setState] = useState<PlaybackState>("idle");
|
||||
const ctxRef = useRef<AudioContext | null>(null);
|
||||
const workletRef = useRef<AudioWorkletNode | null>(null);
|
||||
const readyRef = useRef(false);
|
||||
const seqBufferRef = useRef(new SequenceBuffer());
|
||||
|
||||
const init = useCallback(async () => {
|
||||
if (readyRef.current) return;
|
||||
|
||||
const ctx = new AudioContext({ sampleRate: 24000 });
|
||||
await ctx.audioWorklet.addModule(workletPath);
|
||||
const worklet = new AudioWorkletNode(ctx, "audio-playback-processor");
|
||||
worklet.connect(ctx.destination);
|
||||
|
||||
worklet.port.onmessage = (e) => {
|
||||
if (e.data.type === "ended") setState("idle");
|
||||
};
|
||||
|
||||
ctxRef.current = ctx;
|
||||
workletRef.current = worklet;
|
||||
readyRef.current = true;
|
||||
}, [workletPath]);
|
||||
|
||||
/** Push audio directly (no sequencing) - for simple streaming */
|
||||
const pushAudio = useCallback((base64Audio: string) => {
|
||||
if (!workletRef.current) return;
|
||||
const samples = decodePCM16ToFloat32(base64Audio);
|
||||
workletRef.current.port.postMessage({ type: "audio", samples });
|
||||
setState("playing");
|
||||
}, []);
|
||||
|
||||
/** Push audio with sequence number - reorders before playback */
|
||||
const pushSequencedAudio = useCallback((seq: number, base64Audio: string) => {
|
||||
if (!workletRef.current) return;
|
||||
|
||||
const readyChunks = seqBufferRef.current.push(seq, base64Audio);
|
||||
for (const chunk of readyChunks) {
|
||||
const samples = decodePCM16ToFloat32(chunk);
|
||||
workletRef.current.port.postMessage({ type: "audio", samples });
|
||||
}
|
||||
if (readyChunks.length > 0) {
|
||||
setState("playing");
|
||||
}
|
||||
}, []);
|
||||
|
||||
const signalComplete = useCallback(() => {
|
||||
workletRef.current?.port.postMessage({ type: "streamComplete" });
|
||||
}, []);
|
||||
|
||||
const clear = useCallback(() => {
|
||||
workletRef.current?.port.postMessage({ type: "clear" });
|
||||
seqBufferRef.current.reset();
|
||||
setState("idle");
|
||||
}, []);
|
||||
|
||||
return { state, init, pushAudio, pushSequencedAudio, signalComplete, clear };
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* React hook for voice recording using MediaRecorder API.
|
||||
* Records audio in WebM/Opus format for efficient streaming.
|
||||
*/
|
||||
import { useRef, useCallback, useState } from "react";
|
||||
|
||||
export type RecordingState = "idle" | "recording" | "stopped";
|
||||
|
||||
export function useVoiceRecorder() {
|
||||
const [state, setState] = useState<RecordingState>("idle");
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const chunksRef = useRef<Blob[]>([]);
|
||||
|
||||
const startRecording = useCallback(async (): Promise<void> => {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
const recorder = new MediaRecorder(stream, {
|
||||
mimeType: "audio/webm;codecs=opus",
|
||||
});
|
||||
|
||||
mediaRecorderRef.current = recorder;
|
||||
chunksRef.current = [];
|
||||
|
||||
recorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunksRef.current.push(e.data);
|
||||
};
|
||||
|
||||
recorder.start(100); // Collect chunks every 100ms
|
||||
setState("recording");
|
||||
}, []);
|
||||
|
||||
const stopRecording = useCallback((): Promise<Blob> => {
|
||||
return new Promise((resolve) => {
|
||||
const recorder = mediaRecorderRef.current;
|
||||
if (!recorder || recorder.state !== "recording") {
|
||||
resolve(new Blob());
|
||||
return;
|
||||
}
|
||||
|
||||
recorder.onstop = () => {
|
||||
const blob = new Blob(chunksRef.current, { type: "audio/webm" });
|
||||
recorder.stream.getTracks().forEach((t) => t.stop());
|
||||
setState("stopped");
|
||||
resolve(blob);
|
||||
};
|
||||
|
||||
recorder.stop();
|
||||
});
|
||||
}, []);
|
||||
|
||||
return { state, startRecording, stopRecording };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* React hook for handling SSE voice streaming responses.
|
||||
* Converts audio blob to base64 and sends as JSON to match server expectations.
|
||||
*/
|
||||
import { useCallback } from "react";
|
||||
import { useAudioPlayback } from "./useAudioPlayback";
|
||||
|
||||
interface StreamCallbacks {
|
||||
onUserTranscript?: (text: string) => void;
|
||||
onTranscript?: (text: string, full: string) => void;
|
||||
onComplete?: (transcript: string) => void;
|
||||
onError?: (error: Error) => void;
|
||||
}
|
||||
|
||||
export function useVoiceStream(callbacks: StreamCallbacks = {}) {
|
||||
const playback = useAudioPlayback();
|
||||
|
||||
const streamVoiceResponse = useCallback(
|
||||
async (url: string, audioBlob: Blob) => {
|
||||
await playback.init();
|
||||
playback.clear();
|
||||
|
||||
// Convert blob to base64 for JSON body (server expects express.json())
|
||||
const base64Audio = await new Promise<string>((resolve) => {
|
||||
const fileReader = new FileReader();
|
||||
fileReader.onload = () => {
|
||||
const result = fileReader.result as string;
|
||||
resolve(result.split(",")[1]); // Remove data URL prefix
|
||||
};
|
||||
fileReader.readAsDataURL(audioBlob);
|
||||
});
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ audio: base64Audio }),
|
||||
});
|
||||
if (!response.ok) throw new Error("Voice request failed");
|
||||
|
||||
const streamReader = response.body?.getReader();
|
||||
if (!streamReader) throw new Error("No response body");
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
let fullTranscript = "";
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await streamReader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split("\n");
|
||||
buffer = lines.pop() || "";
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.startsWith("data: ")) continue;
|
||||
|
||||
try {
|
||||
const event = JSON.parse(line.slice(6));
|
||||
|
||||
switch (event.type) {
|
||||
case "user_transcript":
|
||||
callbacks.onUserTranscript?.(event.data);
|
||||
break;
|
||||
case "transcript":
|
||||
fullTranscript += event.data;
|
||||
callbacks.onTranscript?.(event.data, fullTranscript);
|
||||
break;
|
||||
case "audio":
|
||||
playback.pushAudio(event.data);
|
||||
break;
|
||||
case "done":
|
||||
playback.signalComplete();
|
||||
callbacks.onComplete?.(fullTranscript);
|
||||
break;
|
||||
case "error":
|
||||
throw new Error(event.error);
|
||||
}
|
||||
} catch (e) {
|
||||
if (!(e instanceof SyntaxError)) {
|
||||
callbacks.onError?.(e as Error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
[playback, callbacks]
|
||||
);
|
||||
|
||||
return { streamVoiceResponse, playbackState: playback.state };
|
||||
}
|
||||
Reference in New Issue
Block a user