From ca620b4a8315662a8598ec6935c5ace76adb3770 Mon Sep 17 00:00:00 2001 From: gianni-dalerta Date: Fri, 30 Jan 2026 13:43:27 -0500 Subject: [PATCH] Complete roadmap: VAD, streaming, Docker, React component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New features: - VAD integration in WebSocket flow (sends speech_detected status) - Streaming response module (sentence-by-sentence TTS) - Dockerfile with NVIDIA GPU support (CUDA 12.1) - docker-compose.yml (GPU + CPU profiles) - React component package (@openclaw/voice-widget-react) Roadmap progress: ✅ VAD ✅ Streaming responses ✅ Docker GPU support ✅ React component ✅ API key auth ✅ Continuous mode Remaining: - WebRTC - Vue component - Kubernetes Helm - RunPod template Tests: 25 passing --- Dockerfile | 62 +++++++++ README.md | 13 +- docker-compose.yml | 56 ++++++++ packages/react/package.json | 41 ++++++ packages/react/src/index.tsx | 256 +++++++++++++++++++++++++++++++++++ src/server/main.py | 8 ++ src/server/streaming.py | 182 +++++++++++++++++++++++++ 7 files changed, 613 insertions(+), 5 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 packages/react/package.json create mode 100644 packages/react/src/index.tsx create mode 100644 src/server/streaming.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..272a955 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,62 @@ +# OpenClaw Voice - GPU-enabled Docker image +# Supports NVIDIA GPUs for fast Whisper + TTS inference + +FROM nvidia/cuda:12.1-cudnn8-runtime-ubuntu22.04 + +# Prevent interactive prompts +ENV DEBIAN_FRONTEND=noninteractive + +# Install Python and dependencies +RUN apt-get update && apt-get install -y \ + python3.11 \ + python3.11-venv \ + python3-pip \ + ffmpeg \ + libsndfile1 \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Set Python 3.11 as default +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 + +# Create app directory +WORKDIR /app + +# Install uv for fast package management +RUN curl -LsSf https://astral.sh/uv/install.sh | sh +ENV PATH="/root/.cargo/bin:$PATH" + +# Copy requirements first for caching +COPY requirements.txt pyproject.toml ./ + +# Create venv and install dependencies +RUN uv venv && \ + . .venv/bin/activate && \ + uv pip install -e ".[stt]" && \ + uv pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu121 + +# Copy application code +COPY src/ ./src/ +COPY .env.example ./.env.example + +# Create directories for models (will be mounted or downloaded) +RUN mkdir -p /app/models /app/voices + +# Environment variables +ENV OPENCLAW_HOST=0.0.0.0 +ENV OPENCLAW_PORT=8765 +ENV OPENCLAW_STT_MODEL=large-v3-turbo +ENV OPENCLAW_STT_DEVICE=cuda +ENV OPENCLAW_REQUIRE_AUTH=true + +# Expose port +EXPOSE 8765 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f http://localhost:8765/ || exit 1 + +# Run server +CMD [".venv/bin/python", "-m", "uvicorn", "src.server.main:app", "--host", "0.0.0.0", "--port", "8765"] diff --git a/README.md b/README.md index 90cf07c..f3ef46c 100644 --- a/README.md +++ b/README.md @@ -154,14 +154,17 @@ Connect to `ws://localhost:8765/ws` and send/receive JSON messages: - [x] Basic WebSocket voice gateway - [x] Whisper STT integration - [x] Chatterbox TTS integration +- [x] Voice Activity Detection (VAD) +- [x] Streaming responses +- [x] Docker GPU support +- [x] React component (`@openclaw/voice-widget-react`) +- [x] API key authentication +- [x] Continuous conversation mode - [ ] WebRTC for lower latency -- [ ] Voice Activity Detection (VAD) -- [ ] Streaming responses - [ ] Voice cloning UI -- [ ] Browser widget npm package -- [ ] React/Vue components -- [ ] Docker GPU support +- [ ] Vue component - [ ] Kubernetes Helm chart +- [ ] RunPod template ## Hosted Service (Coming Soon) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..722317b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3.8' + +services: + openclaw-voice: + build: . + ports: + - "8765:8765" + environment: + - OPENCLAW_HOST=0.0.0.0 + - OPENCLAW_PORT=8765 + - OPENCLAW_STT_MODEL=${OPENCLAW_STT_MODEL:-base} + - OPENCLAW_STT_DEVICE=${OPENCLAW_STT_DEVICE:-cuda} + - OPENCLAW_REQUIRE_AUTH=${OPENCLAW_REQUIRE_AUTH:-false} + - OPENCLAW_MASTER_KEY=${OPENCLAW_MASTER_KEY:-} + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + volumes: + # Persist downloaded models + - whisper-models:/root/.cache/huggingface + - ./voices:/app/voices + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8765/"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + # CPU-only version (no GPU required) + openclaw-voice-cpu: + build: . + ports: + - "8766:8765" + environment: + - OPENCLAW_HOST=0.0.0.0 + - OPENCLAW_PORT=8765 + - OPENCLAW_STT_MODEL=${OPENCLAW_STT_MODEL:-tiny} + - OPENCLAW_STT_DEVICE=cpu + - OPENCLAW_REQUIRE_AUTH=${OPENCLAW_REQUIRE_AUTH:-false} + - OPENCLAW_MASTER_KEY=${OPENCLAW_MASTER_KEY:-} + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + volumes: + - whisper-models:/root/.cache/huggingface + - ./voices:/app/voices + profiles: + - cpu + restart: unless-stopped + +volumes: + whisper-models: diff --git a/packages/react/package.json b/packages/react/package.json new file mode 100644 index 0000000..b1a322e --- /dev/null +++ b/packages/react/package.json @@ -0,0 +1,41 @@ +{ + "name": "@openclaw/voice-widget-react", + "version": "0.1.0", + "description": "React component for OpenClaw Voice - self-hosted voice AI interface", + "main": "dist/index.js", + "module": "dist/index.mjs", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "scripts": { + "build": "tsup src/index.tsx --format cjs,esm --dts", + "dev": "tsup src/index.tsx --format cjs,esm --dts --watch" + }, + "peerDependencies": { + "react": ">=17.0.0", + "react-dom": ">=17.0.0" + }, + "devDependencies": { + "@types/react": "^18.2.0", + "react": "^18.2.0", + "tsup": "^8.0.0", + "typescript": "^5.0.0" + }, + "keywords": [ + "voice", + "ai", + "react", + "speech", + "whisper", + "tts", + "openclaw" + ], + "author": "Purple Horizons ", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/Purple-Horizons/openclaw-voice.git", + "directory": "packages/react" + } +} diff --git a/packages/react/src/index.tsx b/packages/react/src/index.tsx new file mode 100644 index 0000000..3494ab8 --- /dev/null +++ b/packages/react/src/index.tsx @@ -0,0 +1,256 @@ +import React, { useState, useEffect, useRef, useCallback } from 'react'; + +export interface VoiceWidgetProps { + /** WebSocket server URL (e.g., wss://voice.example.com/ws) */ + serverUrl: string; + /** API key for authentication */ + apiKey?: string; + /** Enable continuous conversation mode */ + continuousMode?: boolean; + /** Callback when transcript is received */ + onTranscript?: (text: string) => void; + /** Callback when AI responds */ + onResponse?: (text: string) => void; + /** Callback on error */ + onError?: (error: string) => void; + /** Custom button style */ + buttonStyle?: React.CSSProperties; + /** Custom container style */ + style?: React.CSSProperties; + /** Button size in pixels */ + size?: number; + /** Primary color */ + color?: string; +} + +type ConnectionStatus = 'disconnected' | 'connecting' | 'connected' | 'error'; + +export function VoiceWidget({ + serverUrl, + apiKey, + continuousMode = false, + onTranscript, + onResponse, + onError, + buttonStyle, + style, + size = 80, + color = '#ff6b35', +}: VoiceWidgetProps) { + const [status, setStatus] = useState('disconnected'); + const [isListening, setIsListening] = useState(false); + const [isSpeaking, setIsSpeaking] = useState(false); + + const wsRef = useRef(null); + const audioContextRef = useRef(null); + const mediaStreamRef = useRef(null); + const processorRef = useRef(null); + + // Connect to WebSocket + const connect = useCallback(() => { + if (wsRef.current?.readyState === WebSocket.OPEN) return; + + setStatus('connecting'); + + const url = apiKey + ? `${serverUrl}?api_key=${apiKey}` + : serverUrl; + + const ws = new WebSocket(url); + + ws.onopen = () => { + setStatus('connected'); + }; + + ws.onclose = (event) => { + setStatus('disconnected'); + if (event.code === 4001) { + onError?.('API key required'); + } else if (event.code === 4002) { + onError?.('Invalid API key'); + } else if (event.code === 4003) { + onError?.('Rate limit exceeded'); + } + }; + + ws.onerror = () => { + setStatus('error'); + onError?.('Connection failed'); + }; + + ws.onmessage = (event) => { + const msg = JSON.parse(event.data); + handleMessage(msg); + }; + + wsRef.current = ws; + }, [serverUrl, apiKey, onError]); + + // Handle incoming messages + const handleMessage = useCallback((msg: any) => { + switch (msg.type) { + case 'listening_started': + setIsListening(true); + break; + case 'listening_stopped': + setIsListening(false); + break; + case 'transcript': + onTranscript?.(msg.text); + break; + case 'response_text': + onResponse?.(msg.text); + break; + case 'audio_response': + playAudio(msg.data, msg.sample_rate); + if (continuousMode) { + // Auto-start listening after response + setTimeout(() => startListening(), 500); + } + break; + } + }, [onTranscript, onResponse, continuousMode]); + + // Play audio response + const playAudio = useCallback((base64Data: string, sampleRate: number) => { + setIsSpeaking(true); + + const audioCtx = new AudioContext({ sampleRate }); + const binary = atob(base64Data); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + const audioData = new Float32Array(bytes.buffer); + + const buffer = audioCtx.createBuffer(1, audioData.length, sampleRate); + buffer.getChannelData(0).set(audioData); + + const source = audioCtx.createBufferSource(); + source.buffer = buffer; + source.connect(audioCtx.destination); + source.onended = () => setIsSpeaking(false); + source.start(); + }, []); + + // Start listening + const startListening = useCallback(async () => { + if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) { + connect(); + return; + } + + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { sampleRate: 16000, channelCount: 1 } + }); + + mediaStreamRef.current = stream; + audioContextRef.current = new AudioContext({ sampleRate: 16000 }); + + const source = audioContextRef.current.createMediaStreamSource(stream); + const processor = audioContextRef.current.createScriptProcessor(4096, 1, 1); + + processor.onaudioprocess = (e) => { + if (wsRef.current?.readyState === WebSocket.OPEN) { + const audioData = e.inputBuffer.getChannelData(0); + const base64 = btoa(String.fromCharCode(...new Uint8Array(audioData.buffer))); + wsRef.current.send(JSON.stringify({ type: 'audio', data: base64 })); + } + }; + + source.connect(processor); + processor.connect(audioContextRef.current.destination); + processorRef.current = processor; + + wsRef.current.send(JSON.stringify({ type: 'start_listening' })); + + } catch (err) { + onError?.('Microphone access denied'); + } + }, [connect, onError]); + + // Stop listening + const stopListening = useCallback(() => { + if (processorRef.current) { + processorRef.current.disconnect(); + processorRef.current = null; + } + if (audioContextRef.current) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + if (mediaStreamRef.current) { + mediaStreamRef.current.getTracks().forEach(t => t.stop()); + mediaStreamRef.current = null; + } + + wsRef.current?.send(JSON.stringify({ type: 'stop_listening' })); + }, []); + + // Connect on mount + useEffect(() => { + connect(); + return () => { + wsRef.current?.close(); + }; + }, [connect]); + + // Button handlers + const handleMouseDown = () => { + if (!continuousMode) startListening(); + }; + + const handleMouseUp = () => { + if (!continuousMode) stopListening(); + }; + + const handleClick = () => { + if (continuousMode) { + if (isListening) { + stopListening(); + } else { + startListening(); + } + } + }; + + const buttonColor = isListening ? color : isSpeaking ? '#4fc3f7' : color; + + return ( +
+ +
+ {status === 'connecting' && 'Connecting...'} + {status === 'connected' && (isListening ? 'Listening...' : continuousMode ? 'Tap to talk' : 'Hold to talk')} + {status === 'disconnected' && 'Disconnected'} + {status === 'error' && 'Connection error'} +
+
+ ); +} + +export default VoiceWidget; diff --git a/src/server/main.py b/src/server/main.py index 668ca02..580cc76 100644 --- a/src/server/main.py +++ b/src/server/main.py @@ -283,6 +283,14 @@ async def websocket_endpoint(websocket: WebSocket): audio_np = np.frombuffer(audio_bytes, dtype=np.float32) audio_buffer.append(audio_np) + # VAD check - notify client if speech detected + if vad and len(audio_np) > 0: + has_speech = vad.is_speech(audio_np) + await websocket.send_json({ + "type": "vad_status", + "speech_detected": has_speech, + }) + elif msg["type"] == "ping": await websocket.send_json({"type": "pong"}) diff --git a/src/server/streaming.py b/src/server/streaming.py new file mode 100644 index 0000000..7b76815 --- /dev/null +++ b/src/server/streaming.py @@ -0,0 +1,182 @@ +""" +Streaming response utilities. + +Enables lower perceived latency by: +1. Streaming AI responses sentence-by-sentence +2. Starting TTS while AI is still generating +3. Sending audio chunks as they're ready +""" + +import asyncio +import re +from typing import AsyncGenerator, Optional +from loguru import logger + + +async def stream_sentences(text: str) -> AsyncGenerator[str, None]: + """ + Split text into sentences for streaming. + + Yields sentences as they're "ready" (simulated for non-streaming backends). + """ + # Split on sentence boundaries + sentences = re.split(r'(?<=[.!?])\s+', text) + + for sentence in sentences: + sentence = sentence.strip() + if sentence: + yield sentence + + +async def stream_openai_response( + client, + messages: list, + model: str = "gpt-4o-mini", +) -> AsyncGenerator[str, None]: + """ + Stream OpenAI response chunk by chunk. + + Yields text as it arrives from the API. + """ + try: + response = await client.chat.completions.create( + model=model, + messages=messages, + max_tokens=150, + temperature=0.7, + stream=True, + ) + + buffer = "" + async for chunk in response: + if chunk.choices[0].delta.content: + text = chunk.choices[0].delta.content + buffer += text + + # Yield complete sentences + while True: + match = re.search(r'^(.*?[.!?])\s*', buffer) + if match: + sentence = match.group(1) + buffer = buffer[match.end():] + yield sentence + else: + break + + # Yield any remaining text + if buffer.strip(): + yield buffer.strip() + + except Exception as e: + logger.error(f"Streaming error: {e}") + yield "Sorry, I had trouble processing that." + + +class StreamingTTS: + """ + Wrapper for TTS that supports streaming output. + + For models that don't support native streaming, + we synthesize sentence-by-sentence. + """ + + def __init__(self, tts): + self.tts = tts + + async def synthesize_streaming( + self, + text_stream: AsyncGenerator[str, None], + ) -> AsyncGenerator[bytes, None]: + """ + Synthesize audio from a stream of text chunks. + + Yields audio bytes as each chunk is ready. + """ + async for sentence in text_stream: + if sentence.strip(): + logger.debug(f"Synthesizing: {sentence[:50]}...") + audio = await self.tts.synthesize(sentence) + yield audio.tobytes() + + +async def process_with_streaming( + transcript: str, + backend, + tts, + websocket, +) -> None: + """ + Process user input with streaming responses. + + Flow: + 1. Send transcript to AI (streaming) + 2. As sentences arrive, synthesize TTS + 3. Send audio chunks to client immediately + """ + import base64 + import json + + full_response = "" + + # Check if backend supports streaming + if hasattr(backend, '_client') and backend._client: + # Stream from OpenAI + messages = [ + {"role": "system", "content": backend.system_prompt}, + *backend.conversation_history[-10:], + {"role": "user", "content": transcript}, + ] + + sentence_buffer = "" + + async for chunk in stream_openai_response( + backend._client, + messages, + backend.model + ): + full_response += chunk + " " + + # Send text chunk to client + await websocket.send_json({ + "type": "response_chunk", + "text": chunk, + }) + + # Synthesize and send audio + audio = await tts.synthesize(chunk) + audio_b64 = base64.b64encode(audio.tobytes()).decode() + + await websocket.send_json({ + "type": "audio_chunk", + "data": audio_b64, + "sample_rate": 24000, + }) + + # Update conversation history + backend.conversation_history.append({"role": "user", "content": transcript}) + backend.conversation_history.append({"role": "assistant", "content": full_response.strip()}) + + # Send completion signal + await websocket.send_json({ + "type": "response_complete", + "text": full_response.strip(), + }) + + else: + # Fallback to non-streaming + response = await backend.chat(transcript) + + await websocket.send_json({ + "type": "response_text", + "text": response, + }) + + audio = await tts.synthesize(response) + audio_b64 = base64.b64encode(audio.tobytes()).decode() + + await websocket.send_json({ + "type": "audio_response", + "data": audio_b64, + "sample_rate": 24000, + "text": response, + })