✨ feat: Add WebSocket functionality and integrate call features in the chat component

client/src/hooks/index.ts

@@ -18,10 +18,13 @@ export * from './AuthContext';
 export * from './ThemeContext';
 export * from './ScreenshotContext';
 export * from './ApiErrorBoundaryContext';
+export { default as useCall } from './useCall';
 export { default as useToast } from './useToast';
+export { default as useWebRTC } from './useWebRTC';
 export { default as useTimeout } from './useTimeout';
 export { default as useNewConvo } from './useNewConvo';
 export { default as useLocalize } from './useLocalize';
+export { default as useWebSocket } from './useWebSocket';
 export type { TranslationKeys } from './useLocalize';
 export { default as useMediaQuery } from './useMediaQuery';
 export { default as useScrollToRef } from './useScrollToRef';

client/src/hooks/useCall.ts

@@ -0,0 +1,67 @@
+import { useState, useRef, useCallback } from 'react';
+import useWebSocket from './useWebSocket';
+import { WebRTCService } from '../services/WebRTC/WebRTCService';
+
+const SILENCE_THRESHOLD = -50;
+const SILENCE_DURATION = 1000;
+
+const useCall = () => {
+  const { sendMessage } = useWebSocket();
+  const [isCalling, setIsCalling] = useState(false);
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const analyserRef = useRef<AnalyserNode | null>(null);
+  const silenceStartRef = useRef<number | null>(null);
+  const intervalRef = useRef<number | null>(null);
+  const webrtcServiceRef = useRef<WebRTCService | null>(null);
+
+  const checkSilence = useCallback(() => {
+    if (!analyserRef.current || !isCalling) {
+      return;
+    }
+    const data = new Float32Array(analyserRef.current.frequencyBinCount);
+    analyserRef.current.getFloatFrequencyData(data);
+    const avg = data.reduce((a, b) => a + b) / data.length;
+    if (avg < SILENCE_THRESHOLD) {
+      if (!silenceStartRef.current) {
+        silenceStartRef.current = Date.now();
+      } else if (Date.now() - silenceStartRef.current > SILENCE_DURATION) {
+        sendMessage({ type: 'request-response' });
+        silenceStartRef.current = null;
+      }
+    } else {
+      silenceStartRef.current = null;
+    }
+  }, [isCalling, sendMessage]);
+
+  const startCall = useCallback(async () => {
+    webrtcServiceRef.current = new WebRTCService(sendMessage);
+    await webrtcServiceRef.current.initializeCall();
+
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    audioContextRef.current = new AudioContext();
+    const source = audioContextRef.current.createMediaStreamSource(stream);
+    analyserRef.current = audioContextRef.current.createAnalyser();
+    source.connect(analyserRef.current);
+
+    intervalRef.current = window.setInterval(checkSilence, 100);
+    setIsCalling(true);
+  }, [checkSilence, sendMessage]);
+
+  const hangUp = useCallback(async () => {
+    if (intervalRef.current) {
+      clearInterval(intervalRef.current);
+    }
+    analyserRef.current = null;
+    audioContextRef.current?.close();
+    audioContextRef.current = null;
+
+    await webrtcServiceRef.current?.endCall();
+    webrtcServiceRef.current = null;
+    setIsCalling(false);
+    sendMessage({ type: 'call-ended' });
+  }, [sendMessage]);
+
+  return { isCalling, startCall, hangUp };
+};
+
+export default useCall;

client/src/hooks/useWebRTC.ts

@@ -0,0 +1,77 @@
+import { useRef, useCallback } from 'react';
+import useWebSocket from './useWebSocket';
+
+const SILENCE_THRESHOLD = -50;
+const SILENCE_DURATION = 1000;
+
+const useWebRTC = () => {
+  const { sendMessage } = useWebSocket();
+  const localStreamRef = useRef<MediaStream | null>(null);
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const analyserRef = useRef<AnalyserNode | null>(null);
+  const silenceStartTime = useRef<number | null>(null);
+  const isProcessingRef = useRef(false);
+
+  const log = (msg: string) => console.log(`[WebRTC ${new Date().toISOString()}] ${msg}`);
+
+  const processAudioLevel = () => {
+    if (!analyserRef.current || !isProcessingRef.current) {
+      return;
+    }
+
+    const dataArray = new Float32Array(analyserRef.current.frequencyBinCount);
+    analyserRef.current.getFloatFrequencyData(dataArray);
+    const average = dataArray.reduce((a, b) => a + b) / dataArray.length;
+
+    if (average < SILENCE_THRESHOLD) {
+      if (!silenceStartTime.current) {
+        silenceStartTime.current = Date.now();
+        log(`Silence started: ${average}dB`);
+      } else if (Date.now() - silenceStartTime.current > SILENCE_DURATION) {
+        log('Silence threshold reached - requesting response');
+        sendMessage({ type: 'request-response' });
+        silenceStartTime.current = null;
+      }
+    } else {
+      silenceStartTime.current = null;
+    }
+
+    requestAnimationFrame(processAudioLevel);
+  };
+
+  const startLocalStream = async () => {
+    try {
+      log('Starting audio capture');
+      localStreamRef.current = await navigator.mediaDevices.getUserMedia({ audio: true });
+
+      audioContextRef.current = new AudioContext();
+      const source = audioContextRef.current.createMediaStreamSource(localStreamRef.current);
+      analyserRef.current = audioContextRef.current.createAnalyser();
+
+      source.connect(analyserRef.current);
+      isProcessingRef.current = true;
+      processAudioLevel();
+
+      log('Audio capture started');
+    } catch (error) {
+      log(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      throw error;
+    }
+  };
+
+  const stopLocalStream = useCallback(() => {
+    log('Stopping audio capture');
+    isProcessingRef.current = false;
+    audioContextRef.current?.close();
+    localStreamRef.current?.getTracks().forEach((track) => track.stop());
+
+    localStreamRef.current = null;
+    audioContextRef.current = null;
+    analyserRef.current = null;
+    silenceStartTime.current = null;
+  }, []);
+
+  return { startLocalStream, stopLocalStream };
+};
+
+export default useWebRTC;

client/src/hooks/useWebSocket.ts

@@ -0,0 +1,45 @@
+import { useEffect, useRef, useState, useCallback } from 'react';
+import { useGetWebsocketUrlQuery } from 'librechat-data-provider/react-query';
+
+const useWebSocket = () => {
+  const { data: url } = useGetWebsocketUrlQuery();
+  const [isConnected, setIsConnected] = useState(false);
+  const wsRef = useRef<WebSocket | null>(null);
+
+  console.log('wsConfig:', url?.url);
+
+  const connect = useCallback(() => {
+    if (!url?.url) {
+      return;
+    }
+
+    wsRef.current = new WebSocket(url?.url);
+    wsRef.current.onopen = () => setIsConnected(true);
+    wsRef.current.onclose = () => setIsConnected(false);
+    wsRef.current.onerror = (err) => console.error('WebSocket error:', err);
+
+    wsRef.current.onmessage = (event) => {
+      const msg = JSON.parse(event.data);
+      if (msg.type === 'audio-response') {
+        const audioData = msg.data;
+        const audio = new Audio(`data:audio/mp3;base64,${audioData}`);
+        audio.play().catch(console.error);
+      }
+    };
+  }, [url?.url]);
+
+  useEffect(() => {
+    connect();
+    return () => wsRef.current?.close();
+  }, [connect]);
+
+  const sendMessage = useCallback((message: any) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify(message));
+    }
+  }, []);
+
+  return { isConnected, sendMessage };
+};
+
+export default useWebSocket;