mirror of
https://github.com/Purple-Horizons/openclaw-voice.git
synced 2026-08-14 00:58:01 +00:00
Complete roadmap: VAD, streaming, Docker, React component
New features: - VAD integration in WebSocket flow (sends speech_detected status) - Streaming response module (sentence-by-sentence TTS) - Dockerfile with NVIDIA GPU support (CUDA 12.1) - docker-compose.yml (GPU + CPU profiles) - React component package (@openclaw/voice-widget-react) Roadmap progress: ✅ VAD ✅ Streaming responses ✅ Docker GPU support ✅ React component ✅ API key auth ✅ Continuous mode Remaining: - WebRTC - Vue component - Kubernetes Helm - RunPod template Tests: 25 passing
This commit is contained in:
gianni-dalerta
+62
@@ -0,0 +1,62 @@
|
||||
# OpenClaw Voice - GPU-enabled Docker image
|
||||
# Supports NVIDIA GPUs for fast Whisper + TTS inference
|
||||
|
||||
FROM nvidia/cuda:12.1-cudnn8-runtime-ubuntu22.04
|
||||
|
||||
# Prevent interactive prompts
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Install Python and dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
python3.11 \
|
||||
python3.11-venv \
|
||||
python3-pip \
|
||||
ffmpeg \
|
||||
libsndfile1 \
|
||||
git \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set Python 3.11 as default
|
||||
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 \
|
||||
&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
|
||||
|
||||
# Create app directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install uv for fast package management
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
ENV PATH="/root/.cargo/bin:$PATH"
|
||||
|
||||
# Copy requirements first for caching
|
||||
COPY requirements.txt pyproject.toml ./
|
||||
|
||||
# Create venv and install dependencies
|
||||
RUN uv venv && \
|
||||
. .venv/bin/activate && \
|
||||
uv pip install -e ".[stt]" && \
|
||||
uv pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu121
|
||||
|
||||
# Copy application code
|
||||
COPY src/ ./src/
|
||||
COPY .env.example ./.env.example
|
||||
|
||||
# Create directories for models (will be mounted or downloaded)
|
||||
RUN mkdir -p /app/models /app/voices
|
||||
|
||||
# Environment variables
|
||||
ENV OPENCLAW_HOST=0.0.0.0
|
||||
ENV OPENCLAW_PORT=8765
|
||||
ENV OPENCLAW_STT_MODEL=large-v3-turbo
|
||||
ENV OPENCLAW_STT_DEVICE=cuda
|
||||
ENV OPENCLAW_REQUIRE_AUTH=true
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8765
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
|
||||
CMD curl -f http://localhost:8765/ || exit 1
|
||||
|
||||
# Run server
|
||||
CMD [".venv/bin/python", "-m", "uvicorn", "src.server.main:app", "--host", "0.0.0.0", "--port", "8765"]
|
||||
@@ -154,14 +154,17 @@ Connect to `ws://localhost:8765/ws` and send/receive JSON messages:
|
||||
- [x] Basic WebSocket voice gateway
|
||||
- [x] Whisper STT integration
|
||||
- [x] Chatterbox TTS integration
|
||||
- [x] Voice Activity Detection (VAD)
|
||||
- [x] Streaming responses
|
||||
- [x] Docker GPU support
|
||||
- [x] React component (`@openclaw/voice-widget-react`)
|
||||
- [x] API key authentication
|
||||
- [x] Continuous conversation mode
|
||||
- [ ] WebRTC for lower latency
|
||||
- [ ] Voice Activity Detection (VAD)
|
||||
- [ ] Streaming responses
|
||||
- [ ] Voice cloning UI
|
||||
- [ ] Browser widget npm package
|
||||
- [ ] React/Vue components
|
||||
- [ ] Docker GPU support
|
||||
- [ ] Vue component
|
||||
- [ ] Kubernetes Helm chart
|
||||
- [ ] RunPod template
|
||||
|
||||
## Hosted Service (Coming Soon)
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
openclaw-voice:
|
||||
build: .
|
||||
ports:
|
||||
- "8765:8765"
|
||||
environment:
|
||||
- OPENCLAW_HOST=0.0.0.0
|
||||
- OPENCLAW_PORT=8765
|
||||
- OPENCLAW_STT_MODEL=${OPENCLAW_STT_MODEL:-base}
|
||||
- OPENCLAW_STT_DEVICE=${OPENCLAW_STT_DEVICE:-cuda}
|
||||
- OPENCLAW_REQUIRE_AUTH=${OPENCLAW_REQUIRE_AUTH:-false}
|
||||
- OPENCLAW_MASTER_KEY=${OPENCLAW_MASTER_KEY:-}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY:-}
|
||||
volumes:
|
||||
# Persist downloaded models
|
||||
- whisper-models:/root/.cache/huggingface
|
||||
- ./voices:/app/voices
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8765/"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
# CPU-only version (no GPU required)
|
||||
openclaw-voice-cpu:
|
||||
build: .
|
||||
ports:
|
||||
- "8766:8765"
|
||||
environment:
|
||||
- OPENCLAW_HOST=0.0.0.0
|
||||
- OPENCLAW_PORT=8765
|
||||
- OPENCLAW_STT_MODEL=${OPENCLAW_STT_MODEL:-tiny}
|
||||
- OPENCLAW_STT_DEVICE=cpu
|
||||
- OPENCLAW_REQUIRE_AUTH=${OPENCLAW_REQUIRE_AUTH:-false}
|
||||
- OPENCLAW_MASTER_KEY=${OPENCLAW_MASTER_KEY:-}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY:-}
|
||||
volumes:
|
||||
- whisper-models:/root/.cache/huggingface
|
||||
- ./voices:/app/voices
|
||||
profiles:
|
||||
- cpu
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
whisper-models:
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"name": "@openclaw/voice-widget-react",
|
||||
"version": "0.1.0",
|
||||
"description": "React component for OpenClaw Voice - self-hosted voice AI interface",
|
||||
"main": "dist/index.js",
|
||||
"module": "dist/index.mjs",
|
||||
"types": "dist/index.d.ts",
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsup src/index.tsx --format cjs,esm --dts",
|
||||
"dev": "tsup src/index.tsx --format cjs,esm --dts --watch"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": ">=17.0.0",
|
||||
"react-dom": ">=17.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^18.2.0",
|
||||
"react": "^18.2.0",
|
||||
"tsup": "^8.0.0",
|
||||
"typescript": "^5.0.0"
|
||||
},
|
||||
"keywords": [
|
||||
"voice",
|
||||
"ai",
|
||||
"react",
|
||||
"speech",
|
||||
"whisper",
|
||||
"tts",
|
||||
"openclaw"
|
||||
],
|
||||
"author": "Purple Horizons <hello@purplehorizons.io>",
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/Purple-Horizons/openclaw-voice.git",
|
||||
"directory": "packages/react"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
import React, { useState, useEffect, useRef, useCallback } from 'react';
|
||||
|
||||
export interface VoiceWidgetProps {
|
||||
/** WebSocket server URL (e.g., wss://voice.example.com/ws) */
|
||||
serverUrl: string;
|
||||
/** API key for authentication */
|
||||
apiKey?: string;
|
||||
/** Enable continuous conversation mode */
|
||||
continuousMode?: boolean;
|
||||
/** Callback when transcript is received */
|
||||
onTranscript?: (text: string) => void;
|
||||
/** Callback when AI responds */
|
||||
onResponse?: (text: string) => void;
|
||||
/** Callback on error */
|
||||
onError?: (error: string) => void;
|
||||
/** Custom button style */
|
||||
buttonStyle?: React.CSSProperties;
|
||||
/** Custom container style */
|
||||
style?: React.CSSProperties;
|
||||
/** Button size in pixels */
|
||||
size?: number;
|
||||
/** Primary color */
|
||||
color?: string;
|
||||
}
|
||||
|
||||
type ConnectionStatus = 'disconnected' | 'connecting' | 'connected' | 'error';
|
||||
|
||||
export function VoiceWidget({
|
||||
serverUrl,
|
||||
apiKey,
|
||||
continuousMode = false,
|
||||
onTranscript,
|
||||
onResponse,
|
||||
onError,
|
||||
buttonStyle,
|
||||
style,
|
||||
size = 80,
|
||||
color = '#ff6b35',
|
||||
}: VoiceWidgetProps) {
|
||||
const [status, setStatus] = useState<ConnectionStatus>('disconnected');
|
||||
const [isListening, setIsListening] = useState(false);
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const mediaStreamRef = useRef<MediaStream | null>(null);
|
||||
const processorRef = useRef<ScriptProcessorNode | null>(null);
|
||||
|
||||
// Connect to WebSocket
|
||||
const connect = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) return;
|
||||
|
||||
setStatus('connecting');
|
||||
|
||||
const url = apiKey
|
||||
? `${serverUrl}?api_key=${apiKey}`
|
||||
: serverUrl;
|
||||
|
||||
const ws = new WebSocket(url);
|
||||
|
||||
ws.onopen = () => {
|
||||
setStatus('connected');
|
||||
};
|
||||
|
||||
ws.onclose = (event) => {
|
||||
setStatus('disconnected');
|
||||
if (event.code === 4001) {
|
||||
onError?.('API key required');
|
||||
} else if (event.code === 4002) {
|
||||
onError?.('Invalid API key');
|
||||
} else if (event.code === 4003) {
|
||||
onError?.('Rate limit exceeded');
|
||||
}
|
||||
};
|
||||
|
||||
ws.onerror = () => {
|
||||
setStatus('error');
|
||||
onError?.('Connection failed');
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
const msg = JSON.parse(event.data);
|
||||
handleMessage(msg);
|
||||
};
|
||||
|
||||
wsRef.current = ws;
|
||||
}, [serverUrl, apiKey, onError]);
|
||||
|
||||
// Handle incoming messages
|
||||
const handleMessage = useCallback((msg: any) => {
|
||||
switch (msg.type) {
|
||||
case 'listening_started':
|
||||
setIsListening(true);
|
||||
break;
|
||||
case 'listening_stopped':
|
||||
setIsListening(false);
|
||||
break;
|
||||
case 'transcript':
|
||||
onTranscript?.(msg.text);
|
||||
break;
|
||||
case 'response_text':
|
||||
onResponse?.(msg.text);
|
||||
break;
|
||||
case 'audio_response':
|
||||
playAudio(msg.data, msg.sample_rate);
|
||||
if (continuousMode) {
|
||||
// Auto-start listening after response
|
||||
setTimeout(() => startListening(), 500);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}, [onTranscript, onResponse, continuousMode]);
|
||||
|
||||
// Play audio response
|
||||
const playAudio = useCallback((base64Data: string, sampleRate: number) => {
|
||||
setIsSpeaking(true);
|
||||
|
||||
const audioCtx = new AudioContext({ sampleRate });
|
||||
const binary = atob(base64Data);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
const audioData = new Float32Array(bytes.buffer);
|
||||
|
||||
const buffer = audioCtx.createBuffer(1, audioData.length, sampleRate);
|
||||
buffer.getChannelData(0).set(audioData);
|
||||
|
||||
const source = audioCtx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(audioCtx.destination);
|
||||
source.onended = () => setIsSpeaking(false);
|
||||
source.start();
|
||||
}, []);
|
||||
|
||||
// Start listening
|
||||
const startListening = useCallback(async () => {
|
||||
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
|
||||
connect();
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { sampleRate: 16000, channelCount: 1 }
|
||||
});
|
||||
|
||||
mediaStreamRef.current = stream;
|
||||
audioContextRef.current = new AudioContext({ sampleRate: 16000 });
|
||||
|
||||
const source = audioContextRef.current.createMediaStreamSource(stream);
|
||||
const processor = audioContextRef.current.createScriptProcessor(4096, 1, 1);
|
||||
|
||||
processor.onaudioprocess = (e) => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
const audioData = e.inputBuffer.getChannelData(0);
|
||||
const base64 = btoa(String.fromCharCode(...new Uint8Array(audioData.buffer)));
|
||||
wsRef.current.send(JSON.stringify({ type: 'audio', data: base64 }));
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(audioContextRef.current.destination);
|
||||
processorRef.current = processor;
|
||||
|
||||
wsRef.current.send(JSON.stringify({ type: 'start_listening' }));
|
||||
|
||||
} catch (err) {
|
||||
onError?.('Microphone access denied');
|
||||
}
|
||||
}, [connect, onError]);
|
||||
|
||||
// Stop listening
|
||||
const stopListening = useCallback(() => {
|
||||
if (processorRef.current) {
|
||||
processorRef.current.disconnect();
|
||||
processorRef.current = null;
|
||||
}
|
||||
if (audioContextRef.current) {
|
||||
audioContextRef.current.close();
|
||||
audioContextRef.current = null;
|
||||
}
|
||||
if (mediaStreamRef.current) {
|
||||
mediaStreamRef.current.getTracks().forEach(t => t.stop());
|
||||
mediaStreamRef.current = null;
|
||||
}
|
||||
|
||||
wsRef.current?.send(JSON.stringify({ type: 'stop_listening' }));
|
||||
}, []);
|
||||
|
||||
// Connect on mount
|
||||
useEffect(() => {
|
||||
connect();
|
||||
return () => {
|
||||
wsRef.current?.close();
|
||||
};
|
||||
}, [connect]);
|
||||
|
||||
// Button handlers
|
||||
const handleMouseDown = () => {
|
||||
if (!continuousMode) startListening();
|
||||
};
|
||||
|
||||
const handleMouseUp = () => {
|
||||
if (!continuousMode) stopListening();
|
||||
};
|
||||
|
||||
const handleClick = () => {
|
||||
if (continuousMode) {
|
||||
if (isListening) {
|
||||
stopListening();
|
||||
} else {
|
||||
startListening();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const buttonColor = isListening ? color : isSpeaking ? '#4fc3f7' : color;
|
||||
|
||||
return (
|
||||
<div style={{ textAlign: 'center', ...style }}>
|
||||
<button
|
||||
onMouseDown={handleMouseDown}
|
||||
onMouseUp={handleMouseUp}
|
||||
onMouseLeave={handleMouseUp}
|
||||
onClick={handleClick}
|
||||
disabled={status !== 'connected'}
|
||||
style={{
|
||||
width: size,
|
||||
height: size,
|
||||
borderRadius: '50%',
|
||||
border: 'none',
|
||||
background: buttonColor,
|
||||
color: 'white',
|
||||
fontSize: size / 5,
|
||||
fontWeight: 600,
|
||||
cursor: status === 'connected' ? 'pointer' : 'not-allowed',
|
||||
opacity: status === 'connected' ? 1 : 0.5,
|
||||
transition: 'all 0.2s',
|
||||
boxShadow: isListening ? `0 0 20px ${color}` : 'none',
|
||||
...buttonStyle,
|
||||
}}
|
||||
>
|
||||
{isListening ? '🎙️' : isSpeaking ? '🔊' : '🎤'}
|
||||
</button>
|
||||
<div style={{ marginTop: 8, fontSize: 12, color: '#888' }}>
|
||||
{status === 'connecting' && 'Connecting...'}
|
||||
{status === 'connected' && (isListening ? 'Listening...' : continuousMode ? 'Tap to talk' : 'Hold to talk')}
|
||||
{status === 'disconnected' && 'Disconnected'}
|
||||
{status === 'error' && 'Connection error'}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default VoiceWidget;
|
||||
@@ -283,6 +283,14 @@ async def websocket_endpoint(websocket: WebSocket):
|
||||
audio_np = np.frombuffer(audio_bytes, dtype=np.float32)
|
||||
audio_buffer.append(audio_np)
|
||||
|
||||
# VAD check - notify client if speech detected
|
||||
if vad and len(audio_np) > 0:
|
||||
has_speech = vad.is_speech(audio_np)
|
||||
await websocket.send_json({
|
||||
"type": "vad_status",
|
||||
"speech_detected": has_speech,
|
||||
})
|
||||
|
||||
elif msg["type"] == "ping":
|
||||
await websocket.send_json({"type": "pong"})
|
||||
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
"""
|
||||
Streaming response utilities.
|
||||
|
||||
Enables lower perceived latency by:
|
||||
1. Streaming AI responses sentence-by-sentence
|
||||
2. Starting TTS while AI is still generating
|
||||
3. Sending audio chunks as they're ready
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from typing import AsyncGenerator, Optional
|
||||
from loguru import logger
|
||||
|
||||
|
||||
async def stream_sentences(text: str) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Split text into sentences for streaming.
|
||||
|
||||
Yields sentences as they're "ready" (simulated for non-streaming backends).
|
||||
"""
|
||||
# Split on sentence boundaries
|
||||
sentences = re.split(r'(?<=[.!?])\s+', text)
|
||||
|
||||
for sentence in sentences:
|
||||
sentence = sentence.strip()
|
||||
if sentence:
|
||||
yield sentence
|
||||
|
||||
|
||||
async def stream_openai_response(
|
||||
client,
|
||||
messages: list,
|
||||
model: str = "gpt-4o-mini",
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Stream OpenAI response chunk by chunk.
|
||||
|
||||
Yields text as it arrives from the API.
|
||||
"""
|
||||
try:
|
||||
response = await client.chat.completions.create(
|
||||
model=model,
|
||||
messages=messages,
|
||||
max_tokens=150,
|
||||
temperature=0.7,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
buffer = ""
|
||||
async for chunk in response:
|
||||
if chunk.choices[0].delta.content:
|
||||
text = chunk.choices[0].delta.content
|
||||
buffer += text
|
||||
|
||||
# Yield complete sentences
|
||||
while True:
|
||||
match = re.search(r'^(.*?[.!?])\s*', buffer)
|
||||
if match:
|
||||
sentence = match.group(1)
|
||||
buffer = buffer[match.end():]
|
||||
yield sentence
|
||||
else:
|
||||
break
|
||||
|
||||
# Yield any remaining text
|
||||
if buffer.strip():
|
||||
yield buffer.strip()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Streaming error: {e}")
|
||||
yield "Sorry, I had trouble processing that."
|
||||
|
||||
|
||||
class StreamingTTS:
|
||||
"""
|
||||
Wrapper for TTS that supports streaming output.
|
||||
|
||||
For models that don't support native streaming,
|
||||
we synthesize sentence-by-sentence.
|
||||
"""
|
||||
|
||||
def __init__(self, tts):
|
||||
self.tts = tts
|
||||
|
||||
async def synthesize_streaming(
|
||||
self,
|
||||
text_stream: AsyncGenerator[str, None],
|
||||
) -> AsyncGenerator[bytes, None]:
|
||||
"""
|
||||
Synthesize audio from a stream of text chunks.
|
||||
|
||||
Yields audio bytes as each chunk is ready.
|
||||
"""
|
||||
async for sentence in text_stream:
|
||||
if sentence.strip():
|
||||
logger.debug(f"Synthesizing: {sentence[:50]}...")
|
||||
audio = await self.tts.synthesize(sentence)
|
||||
yield audio.tobytes()
|
||||
|
||||
|
||||
async def process_with_streaming(
|
||||
transcript: str,
|
||||
backend,
|
||||
tts,
|
||||
websocket,
|
||||
) -> None:
|
||||
"""
|
||||
Process user input with streaming responses.
|
||||
|
||||
Flow:
|
||||
1. Send transcript to AI (streaming)
|
||||
2. As sentences arrive, synthesize TTS
|
||||
3. Send audio chunks to client immediately
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
|
||||
full_response = ""
|
||||
|
||||
# Check if backend supports streaming
|
||||
if hasattr(backend, '_client') and backend._client:
|
||||
# Stream from OpenAI
|
||||
messages = [
|
||||
{"role": "system", "content": backend.system_prompt},
|
||||
*backend.conversation_history[-10:],
|
||||
{"role": "user", "content": transcript},
|
||||
]
|
||||
|
||||
sentence_buffer = ""
|
||||
|
||||
async for chunk in stream_openai_response(
|
||||
backend._client,
|
||||
messages,
|
||||
backend.model
|
||||
):
|
||||
full_response += chunk + " "
|
||||
|
||||
# Send text chunk to client
|
||||
await websocket.send_json({
|
||||
"type": "response_chunk",
|
||||
"text": chunk,
|
||||
})
|
||||
|
||||
# Synthesize and send audio
|
||||
audio = await tts.synthesize(chunk)
|
||||
audio_b64 = base64.b64encode(audio.tobytes()).decode()
|
||||
|
||||
await websocket.send_json({
|
||||
"type": "audio_chunk",
|
||||
"data": audio_b64,
|
||||
"sample_rate": 24000,
|
||||
})
|
||||
|
||||
# Update conversation history
|
||||
backend.conversation_history.append({"role": "user", "content": transcript})
|
||||
backend.conversation_history.append({"role": "assistant", "content": full_response.strip()})
|
||||
|
||||
# Send completion signal
|
||||
await websocket.send_json({
|
||||
"type": "response_complete",
|
||||
"text": full_response.strip(),
|
||||
})
|
||||
|
||||
else:
|
||||
# Fallback to non-streaming
|
||||
response = await backend.chat(transcript)
|
||||
|
||||
await websocket.send_json({
|
||||
"type": "response_text",
|
||||
"text": response,
|
||||
})
|
||||
|
||||
audio = await tts.synthesize(response)
|
||||
audio_b64 = base64.b64encode(audio.tobytes()).decode()
|
||||
|
||||
await websocket.send_json({
|
||||
"type": "audio_response",
|
||||
"data": audio_b64,
|
||||
"sample_rate": 24000,
|
||||
"text": response,
|
||||
})
|
||||
Reference in New Issue
Block a user