diff --git a/api/server/services/Files/Audio/streamAudio.js b/api/server/services/Files/Audio/streamAudio.js index eb8134e958..4d1157bd34 100644 --- a/api/server/services/Files/Audio/streamAudio.js +++ b/api/server/services/Files/Audio/streamAudio.js @@ -1,5 +1,5 @@ const WebSocket = require('ws'); -const { CacheKeys } = require('librechat-data-provider'); +const { CacheKeys, findLastSeparatorIndex, SEPARATORS } = require('librechat-data-provider'); const { getLogStores } = require('~/cache'); /** @@ -71,25 +71,6 @@ function assembleQuery(parameters) { return query; } -const SEPARATORS = ['.', '?', '!', '۔', '。', '‥', ';', '¡', '¿', '\n']; - -/** - * - * @param {string} text - * @param {string[] | undefined} [separators] - * @returns - */ -function findLastSeparatorIndex(text, separators = SEPARATORS) { - let lastIndex = -1; - for (const separator of separators) { - const index = text.lastIndexOf(separator); - if (index > lastIndex) { - lastIndex = index; - } - } - return lastIndex; -} - const MAX_NOT_FOUND_COUNT = 6; const MAX_NO_CHANGE_COUNT = 10; diff --git a/client/src/App.jsx b/client/src/App.jsx index ce2ec3b6de..e2b11b261f 100644 --- a/client/src/App.jsx +++ b/client/src/App.jsx @@ -8,6 +8,7 @@ import { QueryClient, QueryClientProvider, QueryCache } from '@tanstack/react-qu import { ScreenshotProvider, ThemeProvider, useApiErrorBoundary } from './hooks'; import { ToastProvider } from './Providers'; import Toast from './components/ui/Toast'; +import { LiveAnnouncer } from '~/a11y'; import { router } from './routes'; const App = () => { @@ -26,18 +27,20 @@ const App = () => { return ( - - - - - - - - - - - - + + + + + + + + + + + + + + ); diff --git a/client/src/Providers/AnnouncerContext.tsx b/client/src/Providers/AnnouncerContext.tsx new file mode 100644 index 0000000000..8171492d34 --- /dev/null +++ b/client/src/Providers/AnnouncerContext.tsx @@ -0,0 +1,28 @@ +// AnnouncerContext.tsx +import React from 'react'; + +export interface AnnounceOptions { + message: string; + id?: string; + isStream?: boolean; + isComplete?: boolean; +} + +interface AnnouncerContextType { + announceAssertive: (options: AnnounceOptions) => void; + announcePolite: (options: AnnounceOptions) => void; +} + +const defaultContext: AnnouncerContextType = { + announceAssertive: () => console.warn('Announcement failed, LiveAnnouncer context is missing'), + announcePolite: () => console.warn('Announcement failed, LiveAnnouncer context is missing'), +}; + +const AnnouncerContext = React.createContext(defaultContext); + +export const useLiveAnnouncer = () => { + const context = React.useContext(AnnouncerContext); + return context; +}; + +export default AnnouncerContext; diff --git a/client/src/Providers/index.ts b/client/src/Providers/index.ts index 16487ded25..3ac7e1ba05 100644 --- a/client/src/Providers/index.ts +++ b/client/src/Providers/index.ts @@ -11,3 +11,4 @@ export * from './BookmarkContext'; export * from './DashboardContext'; export * from './AssistantsContext'; export * from './AssistantsMapContext'; +export * from './AnnouncerContext'; diff --git a/client/src/a11y/Announcer.tsx b/client/src/a11y/Announcer.tsx new file mode 100644 index 0000000000..1b79bf1918 --- /dev/null +++ b/client/src/a11y/Announcer.tsx @@ -0,0 +1,54 @@ +import React, { useState, useEffect } from 'react'; +import MessageBlock from './MessageBlock'; + +interface AnnouncerProps { + politeMessage: string; + politeMessageId: string; + assertiveMessage: string; + assertiveMessageId: string; +} + +const Announcer: React.FC = ({ + politeMessage, + politeMessageId, + assertiveMessage, + assertiveMessageId, +}) => { + const [state, setState] = useState({ + assertiveMessage1: '', + assertiveMessage2: '', + politeMessage1: '', + politeMessage2: '', + setAlternatePolite: false, + setAlternateAssertive: false, + }); + + useEffect(() => { + setState((prevState) => ({ + ...prevState, + politeMessage1: prevState.setAlternatePolite ? '' : politeMessage, + politeMessage2: prevState.setAlternatePolite ? politeMessage : '', + setAlternatePolite: !prevState.setAlternatePolite, + })); + }, [politeMessage, politeMessageId]); + + useEffect(() => { + setState((prevState) => ({ + ...prevState, + assertiveMessage1: prevState.setAlternateAssertive ? '' : assertiveMessage, + assertiveMessage2: prevState.setAlternateAssertive ? assertiveMessage : '', + setAlternateAssertive: !prevState.setAlternateAssertive, + })); + }, [assertiveMessage, assertiveMessageId]); + + return ( +
+ + + + +
+ ); +}; + +export default Announcer; diff --git a/client/src/a11y/LiveAnnouncer.tsx b/client/src/a11y/LiveAnnouncer.tsx new file mode 100644 index 0000000000..7d56ebc2f5 --- /dev/null +++ b/client/src/a11y/LiveAnnouncer.tsx @@ -0,0 +1,109 @@ +import React, { useState, useCallback, useRef, useEffect } from 'react'; +import { findLastSeparatorIndex } from 'librechat-data-provider'; +import type { AnnounceOptions } from '~/Providers/AnnouncerContext'; +import AnnouncerContext from '~/Providers/AnnouncerContext'; +import Announcer from './Announcer'; + +interface LiveAnnouncerProps { + children: React.ReactNode; +} + +const LiveAnnouncer: React.FC = ({ children }) => { + const [announcePoliteMessage, setAnnouncePoliteMessage] = useState(''); + const [politeMessageId, setPoliteMessageId] = useState(''); + const [announceAssertiveMessage, setAnnounceAssertiveMessage] = useState(''); + const [assertiveMessageId, setAssertiveMessageId] = useState(''); + + const politeProcessedTextRef = useRef(''); + const politeQueueRef = useRef>([]); + const isAnnouncingRef = useRef(false); + const counterRef = useRef(0); + + const generateUniqueId = (prefix: string) => { + counterRef.current += 1; + return `${prefix}-${counterRef.current}`; + }; + + const processChunks = (text: string, processedTextRef: React.MutableRefObject) => { + const remainingText = text.slice(processedTextRef.current.length); + const separatorIndex = findLastSeparatorIndex(remainingText); + if (separatorIndex !== -1) { + const chunkText = remainingText.slice(0, separatorIndex + 1); + processedTextRef.current += chunkText; + return chunkText.trim(); + } + return ''; + }; + + const announceNextInQueue = useCallback(() => { + if (politeQueueRef.current.length > 0 && !isAnnouncingRef.current) { + isAnnouncingRef.current = true; + const nextAnnouncement = politeQueueRef.current.shift(); + if (nextAnnouncement) { + setAnnouncePoliteMessage(nextAnnouncement.message); + setPoliteMessageId(nextAnnouncement.id); + setTimeout(() => { + isAnnouncingRef.current = false; + announceNextInQueue(); + }, 100); + } + } + }, []); + + const announcePolite = useCallback( + ({ message, id, isStream = false, isComplete = false }: AnnounceOptions) => { + const announcementId = id ?? generateUniqueId('polite'); + if (isStream) { + const chunk = processChunks(message, politeProcessedTextRef); + if (chunk) { + politeQueueRef.current.push({ message: chunk, id: announcementId }); + announceNextInQueue(); + } + } else if (isComplete) { + const remainingText = message.slice(politeProcessedTextRef.current.length); + if (remainingText.trim()) { + politeQueueRef.current.push({ message: remainingText.trim(), id: announcementId }); + announceNextInQueue(); + } + politeProcessedTextRef.current = ''; + } else { + politeQueueRef.current.push({ message, id: announcementId }); + announceNextInQueue(); + politeProcessedTextRef.current = ''; + } + }, + [announceNextInQueue], + ); + + const announceAssertive = useCallback(({ message, id }: AnnounceOptions) => { + const announcementId = id ?? generateUniqueId('assertive'); + setAnnounceAssertiveMessage(message); + setAssertiveMessageId(announcementId); + }, []); + + const contextValue = { + announcePolite, + announceAssertive, + }; + + useEffect(() => { + return () => { + politeQueueRef.current = []; + isAnnouncingRef.current = false; + }; + }, []); + + return ( + + {children} + + + ); +}; + +export default LiveAnnouncer; diff --git a/client/src/a11y/LiveMessage.tsx b/client/src/a11y/LiveMessage.tsx new file mode 100644 index 0000000000..b773deae53 --- /dev/null +++ b/client/src/a11y/LiveMessage.tsx @@ -0,0 +1,37 @@ +import React, { useEffect, useContext } from 'react'; +import AnnouncerContext from '~/Providers/AnnouncerContext'; + +interface LiveMessageProps { + message: string; + 'aria-live': 'polite' | 'assertive'; + clearOnUnmount?: boolean | 'true' | 'false'; +} + +const LiveMessage: React.FC = ({ + message, + 'aria-live': ariaLive, + clearOnUnmount, +}) => { + const { announceAssertive, announcePolite } = useContext(AnnouncerContext); + + useEffect(() => { + if (ariaLive === 'assertive') { + announceAssertive(message); + } else if (ariaLive === 'polite') { + announcePolite(message); + } + }, [message, ariaLive, announceAssertive, announcePolite]); + + useEffect(() => { + return () => { + if (clearOnUnmount === true || clearOnUnmount === 'true') { + announceAssertive(''); + announcePolite(''); + } + }; + }, [clearOnUnmount, announceAssertive, announcePolite]); + + return null; +}; + +export default LiveMessage; diff --git a/client/src/a11y/LiveMessenger.tsx b/client/src/a11y/LiveMessenger.tsx new file mode 100644 index 0000000000..3dd82232aa --- /dev/null +++ b/client/src/a11y/LiveMessenger.tsx @@ -0,0 +1,12 @@ +import React from 'react'; +import AnnouncerContext from '~/Providers/AnnouncerContext'; + +interface LiveMessengerProps { + children: (context: React.ContextType) => React.ReactNode; +} + +const LiveMessenger: React.FC = ({ children }) => ( + {(contextProps) => children(contextProps)} +); + +export default LiveMessenger; diff --git a/client/src/a11y/MessageBlock.tsx b/client/src/a11y/MessageBlock.tsx new file mode 100644 index 0000000000..a99157b6bd --- /dev/null +++ b/client/src/a11y/MessageBlock.tsx @@ -0,0 +1,26 @@ +import React from 'react'; + +const offScreenStyle: React.CSSProperties = { + border: 0, + clip: 'rect(0 0 0 0)', + height: '1px', + margin: '-1px', + overflow: 'hidden', + whiteSpace: 'nowrap', + padding: 0, + width: '1px', + position: 'absolute', +}; + +interface MessageBlockProps { + message: string; + 'aria-live': 'polite' | 'assertive'; +} + +const MessageBlock: React.FC = ({ message, 'aria-live': ariaLive }) => ( +
+ {message} +
+); + +export default MessageBlock; diff --git a/client/src/a11y/index.ts b/client/src/a11y/index.ts new file mode 100644 index 0000000000..60cd708b76 --- /dev/null +++ b/client/src/a11y/index.ts @@ -0,0 +1 @@ +export { default as LiveAnnouncer } from './LiveAnnouncer'; diff --git a/client/src/components/Nav/NewChat.tsx b/client/src/components/Nav/NewChat.tsx index dd8aced0f6..737e01e69a 100644 --- a/client/src/components/Nav/NewChat.tsx +++ b/client/src/components/Nav/NewChat.tsx @@ -37,7 +37,7 @@ const NewChatButtonIcon = ({ conversation }: { conversation: TConversation | nul {iconURL && iconURL.includes('http') ? ( ) : ( -
+
{endpoint && Icon && ( { + (data: string | undefined, submission: TSubmission) => { const { messages, userMessage, @@ -80,13 +82,20 @@ export default function useEventHandlers({ initialResponse, isRegenerate = false, } = submission; + const text = data ?? ''; + if (text.length > 0) { + announcePolite({ + message: text, + isStream: true, + }); + } if (isRegenerate) { setMessages([ ...messages, { ...initialResponse, - text: data, + text, plugin: plugin ?? null, plugins: plugins ?? [], // unfinished: true @@ -98,7 +107,7 @@ export default function useEventHandlers({ userMessage, { ...initialResponse, - text: data, + text, plugin: plugin ?? null, plugins: plugins ?? [], // unfinished: true @@ -106,7 +115,7 @@ export default function useEventHandlers({ ]); } }, - [setMessages], + [setMessages, announcePolite], ); const cancelHandler = useCallback( @@ -136,7 +145,7 @@ export default function useEventHandlers({ } // refresh title - if (genTitle && isNewConvo && requestMessage?.parentMessageId === Constants.NO_PARENT) { + if (genTitle && isNewConvo && requestMessage.parentMessageId === Constants.NO_PARENT) { setTimeout(() => { genTitle.mutate({ conversationId: convoUpdate.conversationId as string }); }, 2500); @@ -238,8 +247,8 @@ export default function useEventHandlers({ const { messages, userMessage, isRegenerate = false } = submission; const initialResponse = { ...submission.initialResponse, - parentMessageId: userMessage?.messageId, - messageId: userMessage?.messageId + '_', + parentMessageId: userMessage.messageId, + messageId: userMessage.messageId + '_', }; if (isRegenerate) { setMessages([...messages, initialResponse]); @@ -248,12 +257,16 @@ export default function useEventHandlers({ } const { conversationId, parentMessageId } = userMessage; + announceAssertive({ + message: 'The AI is generating a response.', + id: `ai-generating-${Date.now()}`, + }); let update = {} as TConversation; if (setConversation && !isAddedRequest) { setConversation((prevState) => { let title = prevState?.title; - const parentId = isRegenerate ? userMessage?.overrideParentMessageId : parentMessageId; + const parentId = isRegenerate ? userMessage.overrideParentMessageId : parentMessageId; if (parentId !== Constants.NO_PARENT && title?.toLowerCase()?.includes('new chat')) { const convos = queryClient.getQueryData([QueryKeys.allConversations]); const cachedConvo = getConversationById(convos, conversationId); @@ -295,7 +308,14 @@ export default function useEventHandlers({ scrollToEnd(); }, - [setMessages, setConversation, queryClient, isAddedRequest, resetLatestMessage], + [ + setMessages, + setConversation, + queryClient, + isAddedRequest, + resetLatestMessage, + announceAssertive, + ], ); const finalHandler = useCallback( @@ -304,7 +324,7 @@ export default function useEventHandlers({ const { messages, conversation: submissionConvo, isRegenerate = false } = submission; setShowStopButton(false); - setCompleted((prev) => new Set(prev.add(submission?.initialResponse?.messageId))); + setCompleted((prev) => new Set(prev.add(submission.initialResponse.messageId))); const currentMessages = getMessages(); // Early return if messages are empty; i.e., the user navigated away @@ -312,6 +332,19 @@ export default function useEventHandlers({ return setIsSubmitting(false); } + /* a11y announcements */ + announcePolite({ + message: '', + isComplete: true, + }); + + setTimeout(() => { + announcePolite({ + message: 'The AI has finished generating a response.', + id: `ai-finished-${Date.now()}`, + }); + }, 100); + // update the messages; if assistants endpoint, client doesn't receive responseMessage if (runMessages) { setMessages([...runMessages]); @@ -367,6 +400,7 @@ export default function useEventHandlers({ setMessages, setCompleted, isAddedRequest, + announcePolite, setConversation, setIsSubmitting, setShowStopButton, @@ -379,7 +413,7 @@ export default function useEventHandlers({ setCompleted((prev) => new Set(prev.add(initialResponse.messageId))); - const conversationId = userMessage?.conversationId ?? submission?.conversationId; + const conversationId = userMessage.conversationId ?? submission.conversationId; const parseErrorResponse = (data: TResData | Partial) => { const metadata = data['responseMessage'] ?? data; @@ -387,7 +421,7 @@ export default function useEventHandlers({ ...initialResponse, ...metadata, error: true, - parentMessageId: userMessage?.messageId, + parentMessageId: userMessage.messageId, }; if (!errorMessage.messageId) { @@ -408,7 +442,7 @@ export default function useEventHandlers({ if (newConversation) { newConversation({ template: { conversationId: convoId }, - preset: tPresetSchema.parse(submission?.conversation), + preset: tPresetSchema.parse(submission.conversation), }); } setIsSubmitting(false); @@ -422,7 +456,7 @@ export default function useEventHandlers({ if (newConversation) { newConversation({ template: { conversationId: convoId }, - preset: tPresetSchema.parse(submission?.conversation), + preset: tPresetSchema.parse(submission.conversation), }); } setIsSubmitting(false); @@ -438,14 +472,14 @@ export default function useEventHandlers({ const errorResponse = tMessageSchema.parse({ ...data, error: true, - parentMessageId: userMessage?.messageId, + parentMessageId: userMessage.messageId, }); setMessages([...messages, userMessage, errorResponse]); if (data.conversationId && paramId === 'new' && newConversation) { newConversation({ template: { conversationId: data.conversationId }, - preset: tPresetSchema.parse(submission?.conversation), + preset: tPresetSchema.parse(submission.conversation), }); } @@ -459,7 +493,7 @@ export default function useEventHandlers({ async (conversationId = '', submission: TSubmission, messages?: TMessage[]) => { const runAbortKey = `${conversationId}:${messages?.[messages.length - 1]?.messageId ?? ''}`; console.log({ conversationId, submission, messages, runAbortKey }); - const { endpoint: _endpoint, endpointType } = submission?.conversation || {}; + const { endpoint: _endpoint, endpointType } = submission.conversation || {}; const endpoint = endpointType ?? _endpoint; try { const response = await fetch(`${EndpointURLs[endpoint ?? '']}/abort`, { @@ -513,7 +547,7 @@ export default function useEventHandlers({ console.error(error); const convoId = conversationId ?? v4(); const text = - submission.initialResponse?.text?.length > 45 ? submission.initialResponse?.text : ''; + submission.initialResponse.text.length > 45 ? submission.initialResponse.text : ''; const errorMessage = { ...submission, ...submission.initialResponse, @@ -526,7 +560,7 @@ export default function useEventHandlers({ if (newConversation) { newConversation({ template: { conversationId: convoId }, - preset: tPresetSchema.parse(submission?.conversation), + preset: tPresetSchema.parse(submission.conversation), }); } setIsSubmitting(false); diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index eff515da62..7c824fbf98 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -1,6 +1,6 @@ { "name": "librechat-data-provider", - "version": "0.7.41", + "version": "0.7.411", "description": "data services for librechat apps", "main": "dist/index.js", "module": "dist/index.es.js", diff --git a/packages/data-provider/src/parsers.ts b/packages/data-provider/src/parsers.ts index 3406a33f65..4ddfa945cb 100644 --- a/packages/data-provider/src/parsers.ts +++ b/packages/data-provider/src/parsers.ts @@ -66,7 +66,7 @@ export function getEnabledEndpoints() { if (endpointsEnv) { enabledEndpoints = endpointsEnv .split(',') - .filter((endpoint) => endpoint?.trim()) + .filter((endpoint) => endpoint.trim()) .map((endpoint) => endpoint.trim()); } return enabledEndpoints; @@ -347,3 +347,16 @@ export function parseTextParts(contentParts: a.TMessageContentParts[]): string { return result; } + +export const SEPARATORS = ['.', '?', '!', '۔', '。', '‥', ';', '¡', '¿', '\n', '```']; + +export function findLastSeparatorIndex(text: string, separators = SEPARATORS): number { + let lastIndex = -1; + for (const separator of separators) { + const index = text.lastIndexOf(separator); + if (index > lastIndex) { + lastIndex = index; + } + } + return lastIndex; +}