- {showPlusPopover && !isAssistantsEndpoint(endpoint) && (
-
+
+
+
+
);
});
diff --git a/client/src/components/Chat/Input/SendButton.tsx b/client/src/components/Chat/Input/SendButton.tsx
index c93ae016e9..0589fc3d25 100644
--- a/client/src/components/Chat/Input/SendButton.tsx
+++ b/client/src/components/Chat/Input/SendButton.tsx
@@ -1,11 +1,13 @@
import React, { forwardRef } from 'react';
import { useWatch } from 'react-hook-form';
+import { useSetRecoilState } from 'recoil';
import type { TRealtimeEphemeralTokenResponse } from 'librechat-data-provider';
import type { Control } from 'react-hook-form';
import { useRealtimeEphemeralTokenMutation } from '~/data-provider';
import { TooltipAnchor, SendIcon, CallIcon } from '~/components';
import { useToastContext } from '~/Providers/ToastContext';
import { useLocalize } from '~/hooks';
+import store from '~/store';
import { cn } from '~/utils';
type ButtonProps = {
@@ -56,25 +58,27 @@ const SendButton = forwardRef((props: ButtonProps, ref: React.ForwardedRef
{
- showToast({
- message: 'IT WORKS!!',
- status: 'success',
- });
- },
- onError: (error: unknown) => {
- showToast({
- message: localize('com_nav_audio_process_error', (error as Error).message),
- status: 'error',
- });
- },
- });
+ // const { mutate: startCall, isLoading: isProcessing } = useRealtimeEphemeralTokenMutation({
+ // onSuccess: async (data: TRealtimeEphemeralTokenResponse) => {
+ // showToast({
+ // message: 'IT WORKS!!',
+ // status: 'success',
+ // });
+ // },
+ // onError: (error: unknown) => {
+ // showToast({
+ // message: localize('com_nav_audio_process_error', (error as Error).message),
+ // status: 'error',
+ // });
+ // },
+ // });
const handleClick = () => {
if (text.trim() === '') {
- startCall({ voice: 'verse' });
+ setCallOpen(true);
+ // startCall({ voice: 'verse' });
}
};
diff --git a/client/src/hooks/index.ts b/client/src/hooks/index.ts
index 83cdb7fbc4..987f8e6329 100644
--- a/client/src/hooks/index.ts
+++ b/client/src/hooks/index.ts
@@ -21,10 +21,13 @@ export * from './Endpoint';
export type { TranslationKeys } from './useLocalize';
+export { default as useCall } from './useCall';
export { default as useToast } from './useToast';
+export { default as useWebRTC } from './useWebRTC';
export { default as useTimeout } from './useTimeout';
export { default as useNewConvo } from './useNewConvo';
export { default as useLocalize } from './useLocalize';
+export { default as useWebSocket } from './useWebSocket';
export { default as useMediaQuery } from './useMediaQuery';
export { default as useChatBadges } from './useChatBadges';
export { default as useScrollToRef } from './useScrollToRef';
diff --git a/client/src/hooks/useCall.ts b/client/src/hooks/useCall.ts
new file mode 100644
index 0000000000..91dc2effb0
--- /dev/null
+++ b/client/src/hooks/useCall.ts
@@ -0,0 +1,67 @@
+import { useState, useRef, useCallback } from 'react';
+import useWebSocket from './useWebSocket';
+import { WebRTCService } from '../services/WebRTC/WebRTCService';
+
+const SILENCE_THRESHOLD = -50;
+const SILENCE_DURATION = 1000;
+
+const useCall = () => {
+ const { sendMessage } = useWebSocket();
+ const [isCalling, setIsCalling] = useState(false);
+ const audioContextRef = useRef(null);
+ const analyserRef = useRef(null);
+ const silenceStartRef = useRef(null);
+ const intervalRef = useRef(null);
+ const webrtcServiceRef = useRef(null);
+
+ const checkSilence = useCallback(() => {
+ if (!analyserRef.current || !isCalling) {
+ return;
+ }
+ const data = new Float32Array(analyserRef.current.frequencyBinCount);
+ analyserRef.current.getFloatFrequencyData(data);
+ const avg = data.reduce((a, b) => a + b) / data.length;
+ if (avg < SILENCE_THRESHOLD) {
+ if (!silenceStartRef.current) {
+ silenceStartRef.current = Date.now();
+ } else if (Date.now() - silenceStartRef.current > SILENCE_DURATION) {
+ sendMessage({ type: 'request-response' });
+ silenceStartRef.current = null;
+ }
+ } else {
+ silenceStartRef.current = null;
+ }
+ }, [isCalling, sendMessage]);
+
+ const startCall = useCallback(async () => {
+ webrtcServiceRef.current = new WebRTCService(sendMessage);
+ await webrtcServiceRef.current.initializeCall();
+
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+ audioContextRef.current = new AudioContext();
+ const source = audioContextRef.current.createMediaStreamSource(stream);
+ analyserRef.current = audioContextRef.current.createAnalyser();
+ source.connect(analyserRef.current);
+
+ intervalRef.current = window.setInterval(checkSilence, 100);
+ setIsCalling(true);
+ }, [checkSilence, sendMessage]);
+
+ const hangUp = useCallback(async () => {
+ if (intervalRef.current) {
+ clearInterval(intervalRef.current);
+ }
+ analyserRef.current = null;
+ audioContextRef.current?.close();
+ audioContextRef.current = null;
+
+ await webrtcServiceRef.current?.endCall();
+ webrtcServiceRef.current = null;
+ setIsCalling(false);
+ sendMessage({ type: 'call-ended' });
+ }, [sendMessage]);
+
+ return { isCalling, startCall, hangUp };
+};
+
+export default useCall;
diff --git a/client/src/hooks/useWebRTC.ts b/client/src/hooks/useWebRTC.ts
new file mode 100644
index 0000000000..436b1da1ea
--- /dev/null
+++ b/client/src/hooks/useWebRTC.ts
@@ -0,0 +1,77 @@
+import { useRef, useCallback } from 'react';
+import useWebSocket from './useWebSocket';
+
+const SILENCE_THRESHOLD = -50;
+const SILENCE_DURATION = 1000;
+
+const useWebRTC = () => {
+ const { sendMessage } = useWebSocket();
+ const localStreamRef = useRef(null);
+ const audioContextRef = useRef(null);
+ const analyserRef = useRef(null);
+ const silenceStartTime = useRef(null);
+ const isProcessingRef = useRef(false);
+
+ const log = (msg: string) => console.log(`[WebRTC ${new Date().toISOString()}] ${msg}`);
+
+ const processAudioLevel = () => {
+ if (!analyserRef.current || !isProcessingRef.current) {
+ return;
+ }
+
+ const dataArray = new Float32Array(analyserRef.current.frequencyBinCount);
+ analyserRef.current.getFloatFrequencyData(dataArray);
+ const average = dataArray.reduce((a, b) => a + b) / dataArray.length;
+
+ if (average < SILENCE_THRESHOLD) {
+ if (!silenceStartTime.current) {
+ silenceStartTime.current = Date.now();
+ log(`Silence started: ${average}dB`);
+ } else if (Date.now() - silenceStartTime.current > SILENCE_DURATION) {
+ log('Silence threshold reached - requesting response');
+ sendMessage({ type: 'request-response' });
+ silenceStartTime.current = null;
+ }
+ } else {
+ silenceStartTime.current = null;
+ }
+
+ requestAnimationFrame(processAudioLevel);
+ };
+
+ const startLocalStream = async () => {
+ try {
+ log('Starting audio capture');
+ localStreamRef.current = await navigator.mediaDevices.getUserMedia({ audio: true });
+
+ audioContextRef.current = new AudioContext();
+ const source = audioContextRef.current.createMediaStreamSource(localStreamRef.current);
+ analyserRef.current = audioContextRef.current.createAnalyser();
+
+ source.connect(analyserRef.current);
+ isProcessingRef.current = true;
+ processAudioLevel();
+
+ log('Audio capture started');
+ } catch (error) {
+ log(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+ throw error;
+ }
+ };
+
+ const stopLocalStream = useCallback(() => {
+ log('Stopping audio capture');
+ isProcessingRef.current = false;
+ audioContextRef.current?.close();
+ localStreamRef.current?.getTracks().forEach((track) => track.stop());
+
+ localStreamRef.current = null;
+ audioContextRef.current = null;
+ analyserRef.current = null;
+ silenceStartTime.current = null;
+ }, []);
+
+ return { startLocalStream, stopLocalStream };
+};
+
+export default useWebRTC;
diff --git a/client/src/hooks/useWebSocket.ts b/client/src/hooks/useWebSocket.ts
new file mode 100644
index 0000000000..4d070f499f
--- /dev/null
+++ b/client/src/hooks/useWebSocket.ts
@@ -0,0 +1,45 @@
+import { useEffect, useRef, useState, useCallback } from 'react';
+import { useGetWebsocketUrlQuery } from 'librechat-data-provider/react-query';
+
+const useWebSocket = () => {
+ const { data: url } = useGetWebsocketUrlQuery();
+ const [isConnected, setIsConnected] = useState(false);
+ const wsRef = useRef(null);
+
+ console.log('wsConfig:', url?.url);
+
+ const connect = useCallback(() => {
+ if (!url?.url) {
+ return;
+ }
+
+ wsRef.current = new WebSocket(url?.url);
+ wsRef.current.onopen = () => setIsConnected(true);
+ wsRef.current.onclose = () => setIsConnected(false);
+ wsRef.current.onerror = (err) => console.error('WebSocket error:', err);
+
+ wsRef.current.onmessage = (event) => {
+ const msg = JSON.parse(event.data);
+ if (msg.type === 'audio-response') {
+ const audioData = msg.data;
+ const audio = new Audio(`data:audio/mp3;base64,${audioData}`);
+ audio.play().catch(console.error);
+ }
+ };
+ }, [url?.url]);
+
+ useEffect(() => {
+ connect();
+ return () => wsRef.current?.close();
+ }, [connect]);
+
+ const sendMessage = useCallback((message: any) => {
+ if (wsRef.current?.readyState === WebSocket.OPEN) {
+ wsRef.current.send(JSON.stringify(message));
+ }
+ }, []);
+
+ return { isConnected, sendMessage };
+};
+
+export default useWebSocket;
diff --git a/client/src/services/WebRTC/WebRTCService.test.ts b/client/src/services/WebRTC/WebRTCService.test.ts
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/client/src/services/WebRTC/WebRTCService.ts b/client/src/services/WebRTC/WebRTCService.ts
new file mode 100644
index 0000000000..6b7ec2bfd8
--- /dev/null
+++ b/client/src/services/WebRTC/WebRTCService.ts
@@ -0,0 +1,36 @@
+export class WebRTCService {
+ private peerConnection: RTCPeerConnection | null = null;
+ private mediaRecorder: MediaRecorder | null = null;
+ private sendMessage: (msg: any) => void;
+
+ constructor(sendMessage: (msg: any) => void) {
+ this.sendMessage = sendMessage;
+ }
+
+ async initializeCall() {
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+ this.peerConnection = new RTCPeerConnection();
+ stream.getTracks().forEach((track) => this.peerConnection?.addTrack(track, stream));
+
+ this.mediaRecorder = new MediaRecorder(stream);
+ this.mediaRecorder.ondataavailable = (e) => {
+ if (e.data.size > 0) {
+ const reader = new FileReader();
+ reader.onload = () => {
+ this.sendMessage({
+ type: 'audio-chunk',
+ data: reader.result,
+ });
+ };
+ reader.readAsDataURL(e.data);
+ }
+ };
+ this.mediaRecorder.start();
+ }
+
+ async endCall() {
+ this.mediaRecorder?.stop();
+ this.peerConnection?.close();
+ this.peerConnection = null;
+ }
+}
diff --git a/client/src/store/families.ts b/client/src/store/families.ts
index 786a415d46..404ece0d4c 100644
--- a/client/src/store/families.ts
+++ b/client/src/store/families.ts
@@ -368,6 +368,11 @@ const updateConversationSelector = selectorFamily({
},
});
+const callDialogOpen = atomFamily({
+ key: 'callDialogOpen',
+ default: false,
+});
+
export default {
conversationKeysAtom,
conversationByIndex,
@@ -399,4 +404,5 @@ export default {
useClearLatestMessages,
showPromptsPopoverFamily,
updateConversationSelector,
+ callDialogOpen,
};
diff --git a/package-lock.json b/package-lock.json
index 911cd7f724..874753082a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -127,6 +127,7 @@
"winston": "^3.11.0",
"winston-daily-rotate-file": "^4.7.1",
"youtube-transcript": "^1.2.1",
+ "ws": "^8.18.0",
"zod": "^3.22.4"
},
"devDependencies": {
@@ -1655,6 +1656,27 @@
"webidl-conversions": "^3.0.0"
}
},
+ "api/node_modules/ws": {
+ "version": "8.18.0",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
+ "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=10.0.0"
+ },
+ "peerDependencies": {
+ "bufferutil": "^4.0.1",
+ "utf-8-validate": ">=5.0.2"
+ },
+ "peerDependenciesMeta": {
+ "bufferutil": {
+ "optional": true
+ },
+ "utf-8-validate": {
+ "optional": true
+ }
+ }
+ },
"client": {
"name": "@librechat/frontend",
"version": "v0.7.7",
diff --git a/packages/data-provider/src/api-endpoints.ts b/packages/data-provider/src/api-endpoints.ts
index 0cd26a4da3..c262868ed1 100644
--- a/packages/data-provider/src/api-endpoints.ts
+++ b/packages/data-provider/src/api-endpoints.ts
@@ -246,4 +246,6 @@ export const verifyTwoFactor = () => '/api/auth/2fa/verify';
export const confirmTwoFactor = () => '/api/auth/2fa/confirm';
export const disableTwoFactor = () => '/api/auth/2fa/disable';
export const regenerateBackupCodes = () => '/api/auth/2fa/backup/regenerate';
-export const verifyTwoFactorTemp = () => '/api/auth/2fa/verify-temp';
\ No newline at end of file
+export const verifyTwoFactorTemp = () => '/api/auth/2fa/verify-temp';
+
+export const websocket = () => '/api/websocket';
diff --git a/packages/data-provider/src/data-service.ts b/packages/data-provider/src/data-service.ts
index b0b3372ab8..41b3b4db48 100644
--- a/packages/data-provider/src/data-service.ts
+++ b/packages/data-provider/src/data-service.ts
@@ -809,4 +809,8 @@ export function verifyTwoFactorTemp(
payload: t.TVerify2FATempRequest,
): Promise {
return request.post(endpoints.verifyTwoFactorTemp(), payload);
-}
\ No newline at end of file
+}
+
+export function getWebsocketUrl(): Promise {
+ return request.get(endpoints.websocket());
+}
diff --git a/packages/data-provider/src/keys.ts b/packages/data-provider/src/keys.ts
index 76184d0ce3..8158ed7fbb 100644
--- a/packages/data-provider/src/keys.ts
+++ b/packages/data-provider/src/keys.ts
@@ -46,6 +46,7 @@ export enum QueryKeys {
health = 'health',
userTerms = 'userTerms',
banner = 'banner',
+ websocketUrl = 'websocketUrl',
}
export enum MutationKeys {
diff --git a/packages/data-provider/src/react-query/react-query-service.ts b/packages/data-provider/src/react-query/react-query-service.ts
index 03a37d99a7..57e98c3825 100644
--- a/packages/data-provider/src/react-query/react-query-service.ts
+++ b/packages/data-provider/src/react-query/react-query-service.ts
@@ -376,3 +376,14 @@ export const useGetCustomConfigSpeechQuery = (
},
);
};
+
+export const useGetWebsocketUrlQuery = (
+ config?: UseQueryOptions,
+): QueryObserverResult => {
+ return useQuery([QueryKeys.websocketUrl], () => dataService.getWebsocketUrl(), {
+ refetchOnWindowFocus: false,
+ refetchOnReconnect: false,
+ refetchOnMount: false,
+ ...config,
+ });
+};
diff --git a/packages/data-provider/src/types.ts b/packages/data-provider/src/types.ts
index 2354bb1afc..5febb233d9 100644
--- a/packages/data-provider/src/types.ts
+++ b/packages/data-provider/src/types.ts
@@ -541,3 +541,7 @@ export type TRealtimeEphemeralTokenResponse = {
token: string;
url: string;
};
+
+export type TWebsocketUrlResponse = {
+ url: string;
+};