diff --git a/api/server/services/WebSocket/WebSocketServer.js b/api/server/services/WebSocket/WebSocketServer.js index 268073bb7c..816f77e355 100644 --- a/api/server/services/WebSocket/WebSocketServer.js +++ b/api/server/services/WebSocket/WebSocketServer.js @@ -21,7 +21,6 @@ class WebRTCConnection { await this.peerConnection.setRemoteDescription(offer); - // Create MediaStream instance properly const mediaStream = new MediaStream(); this.audioTransceiver = this.peerConnection.addTransceiver('audio', { @@ -34,7 +33,6 @@ class WebRTCConnection { this.socket.emit('webrtc-answer', answer); } catch (error) { this.log(`Error handling offer: ${error}`, 'error'); - // Don't throw, handle gracefully this.socket.emit('webrtc-error', { message: error.message, code: 'OFFER_ERROR', @@ -47,13 +45,11 @@ class WebRTCConnection { return; } - // Handle incoming audio tracks - this.peerConnection.ontrack = ({ track, streams }) => { + this.peerConnection.ontrack = ({ track }) => { this.log(`Received ${track.kind} track from client`); - // For testing: Echo the audio back after a delay if (track.kind === 'audio') { - this.handleIncomingAudio(track, streams[0]); + this.handleIncomingAudio(track); } track.onended = () => { @@ -71,27 +67,22 @@ class WebRTCConnection { if (!this.peerConnection) { return; } + const state = this.peerConnection.connectionState; this.log(`Connection state changed to ${state}`); this.state = state; + if (state === 'failed' || state === 'closed') { this.cleanup(); } }; - - this.peerConnection.oniceconnectionstatechange = () => { - if (this.peerConnection) { - this.log(`ICE connection state: ${this.peerConnection.iceConnectionState}`); - } - }; } - handleIncomingAudio(inputTrack) { - // For testing: Echo back the input track directly - this.peerConnection.addTrack(inputTrack); - - // Log the track info for debugging - this.log(`Audio track added: ${inputTrack.id}, enabled: ${inputTrack.enabled}`); + handleIncomingAudio(track) { + if (this.peerConnection) { + const stream = new MediaStream([track]); + this.peerConnection.addTrack(track, stream); + } } async addIceCandidate(candidate) { @@ -119,6 +110,7 @@ class WebRTCConnection { } this.peerConnection = null; } + this.audioTransceiver = null; this.pendingCandidates = []; this.state = 'idle'; @@ -153,16 +145,10 @@ class SocketIOService { this.setupSocketHandlers(); } - log(message, level = 'info') { - const timestamp = new Date().toISOString(); - console.log(`[WebRTC ${timestamp}] [${level.toUpperCase()}] ${message}`); - } - setupSocketHandlers() { this.io.on('connection', (socket) => { this.log(`Client connected: ${socket.id}`); - // Create a new WebRTC connection for this socket const rtcConnection = new WebRTCConnection(socket, { ...this.config, log: this.log.bind(this), @@ -178,6 +164,10 @@ class SocketIOService { rtcConnection.addIceCandidate(candidate); }); + socket.on('vad-status', (status) => { + this.log(`VAD status from ${socket.id}: ${JSON.stringify(status)}`); + }); + socket.on('disconnect', () => { this.log(`Client disconnected: ${socket.id}`); rtcConnection.cleanup(); @@ -186,6 +176,11 @@ class SocketIOService { }); } + log(message, level = 'info') { + const timestamp = new Date().toISOString(); + console.log(`[WebRTC ${timestamp}] [${level.toUpperCase()}] ${message}`); + } + shutdown() { for (const connection of this.connections.values()) { connection.cleanup(); diff --git a/client/package.json b/client/package.json index 08d46adc29..ba5ed555e6 100644 --- a/client/package.json +++ b/client/package.json @@ -51,6 +51,7 @@ "@radix-ui/react-switch": "^1.0.3", "@radix-ui/react-tabs": "^1.0.3", "@radix-ui/react-toast": "^1.1.5", + "@ricky0123/vad-react": "^0.0.28", "@tanstack/react-query": "^4.28.0", "@tanstack/react-table": "^8.11.7", "class-variance-authority": "^0.6.0", diff --git a/client/src/common/types.ts b/client/src/common/types.ts index 6474f9b352..e4f8e57048 100644 --- a/client/src/common/types.ts +++ b/client/src/common/types.ts @@ -64,7 +64,7 @@ export interface RTCMessage { export type MessagePayload = | RTCSessionDescriptionInit | RTCIceCandidateInit - | Record; + | { speaking: boolean }; export enum CallState { IDLE = 'idle', diff --git a/client/src/components/Chat/Input/Call.tsx b/client/src/components/Chat/Input/Call.tsx index c60900a68a..921ca8b17b 100644 --- a/client/src/components/Chat/Input/Call.tsx +++ b/client/src/components/Chat/Input/Call.tsx @@ -26,11 +26,12 @@ export const Call: React.FC = () => { localStream, remoteStream, connectionQuality, + isMuted, + toggleMute, } = useCall(); const [open, setOpen] = useRecoilState(store.callDialogOpen(0)); const [eventLog, setEventLog] = React.useState([]); - const [isMuted, setIsMuted] = React.useState(false); const [isAudioEnabled, setIsAudioEnabled] = React.useState(true); const remoteAudioRef = useRef(null); @@ -84,8 +85,8 @@ export const Call: React.FC = () => { hangUp(); }; - const toggleMute = () => { - setIsMuted((prev) => !prev); + const handleToggleMute = () => { + toggleMute(); logEvent(`Microphone ${isMuted ? 'unmuted' : 'muted'}`); }; @@ -176,7 +177,7 @@ export const Call: React.FC = () => { {isActive && ( <>