🎙️ a11y: Screen Reader Support for Dynamic Content Updates (#3625)

* WIP: first pass, hooks

* wip: isStream arg

* feat: first pass, dynamic content updates, screen reader announcements

* chore: unrelated, styling redundancy

api/server/services/Files/Audio/streamAudio.js

@@ -1,5 +1,5 @@
 const WebSocket = require('ws');
-const { CacheKeys } = require('librechat-data-provider');
+const { CacheKeys, findLastSeparatorIndex, SEPARATORS } = require('librechat-data-provider');
 const { getLogStores } = require('~/cache');
 
 /**
@@ -71,25 +71,6 @@ function assembleQuery(parameters) {
   return query;
 }
 
-const SEPARATORS = ['.', '?', '!', '۔', '。', '‥', ';', '¡', '¿', '\n'];
-
-/**
- *
- * @param {string} text
- * @param {string[] | undefined} [separators]
- * @returns
- */
-function findLastSeparatorIndex(text, separators = SEPARATORS) {
-  let lastIndex = -1;
-  for (const separator of separators) {
-    const index = text.lastIndexOf(separator);
-    if (index > lastIndex) {
-      lastIndex = index;
-    }
-  }
-  return lastIndex;
-}
-
 const MAX_NOT_FOUND_COUNT = 6;
 const MAX_NO_CHANGE_COUNT = 10;
 

client/src/App.jsx

@@ -8,6 +8,7 @@ import { QueryClient, QueryClientProvider, QueryCache } from '@tanstack/react-qu
 import { ScreenshotProvider, ThemeProvider, useApiErrorBoundary } from './hooks';
 import { ToastProvider } from './Providers';
 import Toast from './components/ui/Toast';
+import { LiveAnnouncer } from '~/a11y';
 import { router } from './routes';
 
 const App = () => {
@@ -26,6 +27,7 @@ const App = () => {
   return (
     <QueryClientProvider client={queryClient}>
       <RecoilRoot>
+        <LiveAnnouncer>
           <ThemeProvider>
             <RadixToast.Provider>
               <ToastProvider>
@@ -38,6 +40,7 @@ const App = () => {
               </ToastProvider>
             </RadixToast.Provider>
           </ThemeProvider>
+        </LiveAnnouncer>
       </RecoilRoot>
     </QueryClientProvider>
   );

client/src/Providers/AnnouncerContext.tsx

@@ -0,0 +1,28 @@
+// AnnouncerContext.tsx
+import React from 'react';
+
+export interface AnnounceOptions {
+  message: string;
+  id?: string;
+  isStream?: boolean;
+  isComplete?: boolean;
+}
+
+interface AnnouncerContextType {
+  announceAssertive: (options: AnnounceOptions) => void;
+  announcePolite: (options: AnnounceOptions) => void;
+}
+
+const defaultContext: AnnouncerContextType = {
+  announceAssertive: () => console.warn('Announcement failed, LiveAnnouncer context is missing'),
+  announcePolite: () => console.warn('Announcement failed, LiveAnnouncer context is missing'),
+};
+
+const AnnouncerContext = React.createContext<AnnouncerContextType>(defaultContext);
+
+export const useLiveAnnouncer = () => {
+  const context = React.useContext(AnnouncerContext);
+  return context;
+};
+
+export default AnnouncerContext;

client/src/Providers/index.ts

@@ -11,3 +11,4 @@ export * from './BookmarkContext';
 export * from './DashboardContext';
 export * from './AssistantsContext';
 export * from './AssistantsMapContext';
+export * from './AnnouncerContext';

client/src/a11y/Announcer.tsx

@@ -0,0 +1,54 @@
+import React, { useState, useEffect } from 'react';
+import MessageBlock from './MessageBlock';
+
+interface AnnouncerProps {
+  politeMessage: string;
+  politeMessageId: string;
+  assertiveMessage: string;
+  assertiveMessageId: string;
+}
+
+const Announcer: React.FC<AnnouncerProps> = ({
+  politeMessage,
+  politeMessageId,
+  assertiveMessage,
+  assertiveMessageId,
+}) => {
+  const [state, setState] = useState({
+    assertiveMessage1: '',
+    assertiveMessage2: '',
+    politeMessage1: '',
+    politeMessage2: '',
+    setAlternatePolite: false,
+    setAlternateAssertive: false,
+  });
+
+  useEffect(() => {
+    setState((prevState) => ({
+      ...prevState,
+      politeMessage1: prevState.setAlternatePolite ? '' : politeMessage,
+      politeMessage2: prevState.setAlternatePolite ? politeMessage : '',
+      setAlternatePolite: !prevState.setAlternatePolite,
+    }));
+  }, [politeMessage, politeMessageId]);
+
+  useEffect(() => {
+    setState((prevState) => ({
+      ...prevState,
+      assertiveMessage1: prevState.setAlternateAssertive ? '' : assertiveMessage,
+      assertiveMessage2: prevState.setAlternateAssertive ? assertiveMessage : '',
+      setAlternateAssertive: !prevState.setAlternateAssertive,
+    }));
+  }, [assertiveMessage, assertiveMessageId]);
+
+  return (
+    <div>
+      <MessageBlock aria-live="assertive" aria-atomic="true" message={state.assertiveMessage1} />
+      <MessageBlock aria-live="assertive" aria-atomic="true" message={state.assertiveMessage2} />
+      <MessageBlock aria-live="polite" aria-atomic="false" message={state.politeMessage1} />
+      <MessageBlock aria-live="polite" aria-atomic="false" message={state.politeMessage2} />
+    </div>
+  );
+};
+
+export default Announcer;

client/src/a11y/LiveAnnouncer.tsx

@@ -0,0 +1,109 @@
+import React, { useState, useCallback, useRef, useEffect } from 'react';
+import { findLastSeparatorIndex } from 'librechat-data-provider';
+import type { AnnounceOptions } from '~/Providers/AnnouncerContext';
+import AnnouncerContext from '~/Providers/AnnouncerContext';
+import Announcer from './Announcer';
+
+interface LiveAnnouncerProps {
+  children: React.ReactNode;
+}
+
+const LiveAnnouncer: React.FC<LiveAnnouncerProps> = ({ children }) => {
+  const [announcePoliteMessage, setAnnouncePoliteMessage] = useState('');
+  const [politeMessageId, setPoliteMessageId] = useState('');
+  const [announceAssertiveMessage, setAnnounceAssertiveMessage] = useState('');
+  const [assertiveMessageId, setAssertiveMessageId] = useState('');
+
+  const politeProcessedTextRef = useRef('');
+  const politeQueueRef = useRef<Array<{ message: string; id: string }>>([]);
+  const isAnnouncingRef = useRef(false);
+  const counterRef = useRef(0);
+
+  const generateUniqueId = (prefix: string) => {
+    counterRef.current += 1;
+    return `${prefix}-${counterRef.current}`;
+  };
+
+  const processChunks = (text: string, processedTextRef: React.MutableRefObject<string>) => {
+    const remainingText = text.slice(processedTextRef.current.length);
+    const separatorIndex = findLastSeparatorIndex(remainingText);
+    if (separatorIndex !== -1) {
+      const chunkText = remainingText.slice(0, separatorIndex + 1);
+      processedTextRef.current += chunkText;
+      return chunkText.trim();
+    }
+    return '';
+  };
+
+  const announceNextInQueue = useCallback(() => {
+    if (politeQueueRef.current.length > 0 && !isAnnouncingRef.current) {
+      isAnnouncingRef.current = true;
+      const nextAnnouncement = politeQueueRef.current.shift();
+      if (nextAnnouncement) {
+        setAnnouncePoliteMessage(nextAnnouncement.message);
+        setPoliteMessageId(nextAnnouncement.id);
+        setTimeout(() => {
+          isAnnouncingRef.current = false;
+          announceNextInQueue();
+        }, 100);
+      }
+    }
+  }, []);
+
+  const announcePolite = useCallback(
+    ({ message, id, isStream = false, isComplete = false }: AnnounceOptions) => {
+      const announcementId = id ?? generateUniqueId('polite');
+      if (isStream) {
+        const chunk = processChunks(message, politeProcessedTextRef);
+        if (chunk) {
+          politeQueueRef.current.push({ message: chunk, id: announcementId });
+          announceNextInQueue();
+        }
+      } else if (isComplete) {
+        const remainingText = message.slice(politeProcessedTextRef.current.length);
+        if (remainingText.trim()) {
+          politeQueueRef.current.push({ message: remainingText.trim(), id: announcementId });
+          announceNextInQueue();
+        }
+        politeProcessedTextRef.current = '';
+      } else {
+        politeQueueRef.current.push({ message, id: announcementId });
+        announceNextInQueue();
+        politeProcessedTextRef.current = '';
+      }
+    },
+    [announceNextInQueue],
+  );
+
+  const announceAssertive = useCallback(({ message, id }: AnnounceOptions) => {
+    const announcementId = id ?? generateUniqueId('assertive');
+    setAnnounceAssertiveMessage(message);
+    setAssertiveMessageId(announcementId);
+  }, []);
+
+  const contextValue = {
+    announcePolite,
+    announceAssertive,
+  };
+
+  useEffect(() => {
+    return () => {
+      politeQueueRef.current = [];
+      isAnnouncingRef.current = false;
+    };
+  }, []);
+
+  return (
+    <AnnouncerContext.Provider value={contextValue}>
+      {children}
+      <Announcer
+        assertiveMessage={announceAssertiveMessage}
+        assertiveMessageId={assertiveMessageId}
+        politeMessage={announcePoliteMessage}
+        politeMessageId={politeMessageId}
+      />
+    </AnnouncerContext.Provider>
+  );
+};
+
+export default LiveAnnouncer;

client/src/a11y/LiveMessage.tsx

@@ -0,0 +1,37 @@
+import React, { useEffect, useContext } from 'react';
+import AnnouncerContext from '~/Providers/AnnouncerContext';
+
+interface LiveMessageProps {
+  message: string;
+  'aria-live': 'polite' | 'assertive';
+  clearOnUnmount?: boolean | 'true' | 'false';
+}
+
+const LiveMessage: React.FC<LiveMessageProps> = ({
+  message,
+  'aria-live': ariaLive,
+  clearOnUnmount,
+}) => {
+  const { announceAssertive, announcePolite } = useContext(AnnouncerContext);
+
+  useEffect(() => {
+    if (ariaLive === 'assertive') {
+      announceAssertive(message);
+    } else if (ariaLive === 'polite') {
+      announcePolite(message);
+    }
+  }, [message, ariaLive, announceAssertive, announcePolite]);
+
+  useEffect(() => {
+    return () => {
+      if (clearOnUnmount === true || clearOnUnmount === 'true') {
+        announceAssertive('');
+        announcePolite('');
+      }
+    };
+  }, [clearOnUnmount, announceAssertive, announcePolite]);
+
+  return null;
+};
+
+export default LiveMessage;

client/src/a11y/LiveMessenger.tsx

@@ -0,0 +1,12 @@
+import React from 'react';
+import AnnouncerContext from '~/Providers/AnnouncerContext';
+
+interface LiveMessengerProps {
+  children: (context: React.ContextType<typeof AnnouncerContext>) => React.ReactNode;
+}
+
+const LiveMessenger: React.FC<LiveMessengerProps> = ({ children }) => (
+  <AnnouncerContext.Consumer>{(contextProps) => children(contextProps)}</AnnouncerContext.Consumer>
+);
+
+export default LiveMessenger;

client/src/a11y/MessageBlock.tsx

@@ -0,0 +1,26 @@
+import React from 'react';
+
+const offScreenStyle: React.CSSProperties = {
+  border: 0,
+  clip: 'rect(0 0 0 0)',
+  height: '1px',
+  margin: '-1px',
+  overflow: 'hidden',
+  whiteSpace: 'nowrap',
+  padding: 0,
+  width: '1px',
+  position: 'absolute',
+};
+
+interface MessageBlockProps {
+  message: string;
+  'aria-live': 'polite' | 'assertive';
+}
+
+const MessageBlock: React.FC<MessageBlockProps> = ({ message, 'aria-live': ariaLive }) => (
+  <div style={offScreenStyle} role="log" aria-live={ariaLive}>
+    {message}
+  </div>
+);
+
+export default MessageBlock;

client/src/a11y/index.ts

@@ -0,0 +1 @@
+export { default as LiveAnnouncer } from './LiveAnnouncer';

client/src/components/Nav/NewChat.tsx

@@ -37,7 +37,7 @@ const NewChatButtonIcon = ({ conversation }: { conversation: TConversation | nul
       {iconURL && iconURL.includes('http') ? (
         <ConvoIconURL preset={conversation} endpointIconURL={iconURL} context="nav" />
       ) : (
-        <div className="shadow-stroke relative flex h-full items-center justify-center rounded-full bg-white text-black dark:bg-white">
+        <div className="shadow-stroke relative flex h-full items-center justify-center rounded-full bg-white text-black">
           {endpoint && Icon && (
             <Icon
               size={41}

client/src/hooks/SSE/useEventHandlers.ts

@@ -28,6 +28,7 @@ import {
 import useContentHandler from '~/hooks/SSE/useContentHandler';
 import type { TGenTitleMutation } from '~/data-provider';
 import { useAuthContext } from '~/hooks/AuthContext';
+import { useLiveAnnouncer } from '~/Providers';
 
 type TSyncData = {
   sync: boolean;
@@ -64,6 +65,7 @@ export default function useEventHandlers({
   resetLatestMessage,
 }: EventHandlerParams) {
   const queryClient = useQueryClient();
+  const { announcePolite, announceAssertive } = useLiveAnnouncer();
 
   const { conversationId: paramId } = useParams();
   const { token } = useAuthContext();
@@ -71,7 +73,7 @@ export default function useEventHandlers({
   const contentHandler = useContentHandler({ setMessages, getMessages });
 
   const messageHandler = useCallback(
-    (data: string, submission: TSubmission) => {
+    (data: string | undefined, submission: TSubmission) => {
       const {
         messages,
         userMessage,
@@ -80,13 +82,20 @@ export default function useEventHandlers({
         initialResponse,
         isRegenerate = false,
       } = submission;
+      const text = data ?? '';
+      if (text.length > 0) {
+        announcePolite({
+          message: text,
+          isStream: true,
+        });
+      }
 
       if (isRegenerate) {
         setMessages([
           ...messages,
           {
             ...initialResponse,
-            text: data,
+            text,
             plugin: plugin ?? null,
             plugins: plugins ?? [],
             // unfinished: true
@@ -98,7 +107,7 @@ export default function useEventHandlers({
           userMessage,
           {
             ...initialResponse,
-            text: data,
+            text,
             plugin: plugin ?? null,
             plugins: plugins ?? [],
             // unfinished: true
@@ -106,7 +115,7 @@ export default function useEventHandlers({
         ]);
       }
     },
-    [setMessages],
+    [setMessages, announcePolite],
   );
 
   const cancelHandler = useCallback(
@@ -136,7 +145,7 @@ export default function useEventHandlers({
       }
 
       // refresh title
-      if (genTitle && isNewConvo && requestMessage?.parentMessageId === Constants.NO_PARENT) {
+      if (genTitle && isNewConvo && requestMessage.parentMessageId === Constants.NO_PARENT) {
         setTimeout(() => {
           genTitle.mutate({ conversationId: convoUpdate.conversationId as string });
         }, 2500);
@@ -238,8 +247,8 @@ export default function useEventHandlers({
       const { messages, userMessage, isRegenerate = false } = submission;
       const initialResponse = {
         ...submission.initialResponse,
-        parentMessageId: userMessage?.messageId,
-        messageId: userMessage?.messageId + '_',
+        parentMessageId: userMessage.messageId,
+        messageId: userMessage.messageId + '_',
       };
       if (isRegenerate) {
         setMessages([...messages, initialResponse]);
@@ -248,12 +257,16 @@ export default function useEventHandlers({
       }
 
       const { conversationId, parentMessageId } = userMessage;
+      announceAssertive({
+        message: 'The AI is generating a response.',
+        id: `ai-generating-${Date.now()}`,
+      });
 
       let update = {} as TConversation;
       if (setConversation && !isAddedRequest) {
         setConversation((prevState) => {
           let title = prevState?.title;
-          const parentId = isRegenerate ? userMessage?.overrideParentMessageId : parentMessageId;
+          const parentId = isRegenerate ? userMessage.overrideParentMessageId : parentMessageId;
           if (parentId !== Constants.NO_PARENT && title?.toLowerCase()?.includes('new chat')) {
             const convos = queryClient.getQueryData<ConversationData>([QueryKeys.allConversations]);
             const cachedConvo = getConversationById(convos, conversationId);
@@ -295,7 +308,14 @@ export default function useEventHandlers({
 
       scrollToEnd();
     },
-    [setMessages, setConversation, queryClient, isAddedRequest, resetLatestMessage],
+    [
+      setMessages,
+      setConversation,
+      queryClient,
+      isAddedRequest,
+      resetLatestMessage,
+      announceAssertive,
+    ],
   );
 
   const finalHandler = useCallback(
@@ -304,7 +324,7 @@ export default function useEventHandlers({
       const { messages, conversation: submissionConvo, isRegenerate = false } = submission;
 
       setShowStopButton(false);
-      setCompleted((prev) => new Set(prev.add(submission?.initialResponse?.messageId)));
+      setCompleted((prev) => new Set(prev.add(submission.initialResponse.messageId)));
 
       const currentMessages = getMessages();
       // Early return if messages are empty; i.e., the user navigated away
@@ -312,6 +332,19 @@ export default function useEventHandlers({
         return setIsSubmitting(false);
       }
 
+      /* a11y announcements */
+      announcePolite({
+        message: '',
+        isComplete: true,
+      });
+
+      setTimeout(() => {
+        announcePolite({
+          message: 'The AI has finished generating a response.',
+          id: `ai-finished-${Date.now()}`,
+        });
+      }, 100);
+
       // update the messages; if assistants endpoint, client doesn't receive responseMessage
       if (runMessages) {
         setMessages([...runMessages]);
@@ -367,6 +400,7 @@ export default function useEventHandlers({
       setMessages,
       setCompleted,
       isAddedRequest,
+      announcePolite,
       setConversation,
       setIsSubmitting,
       setShowStopButton,
@@ -379,7 +413,7 @@ export default function useEventHandlers({
 
       setCompleted((prev) => new Set(prev.add(initialResponse.messageId)));
 
-      const conversationId = userMessage?.conversationId ?? submission?.conversationId;
+      const conversationId = userMessage.conversationId ?? submission.conversationId;
 
       const parseErrorResponse = (data: TResData | Partial<TMessage>) => {
         const metadata = data['responseMessage'] ?? data;
@@ -387,7 +421,7 @@ export default function useEventHandlers({
           ...initialResponse,
           ...metadata,
           error: true,
-          parentMessageId: userMessage?.messageId,
+          parentMessageId: userMessage.messageId,
         };
 
         if (!errorMessage.messageId) {
@@ -408,7 +442,7 @@ export default function useEventHandlers({
         if (newConversation) {
           newConversation({
             template: { conversationId: convoId },
-            preset: tPresetSchema.parse(submission?.conversation),
+            preset: tPresetSchema.parse(submission.conversation),
           });
         }
         setIsSubmitting(false);
@@ -422,7 +456,7 @@ export default function useEventHandlers({
         if (newConversation) {
           newConversation({
             template: { conversationId: convoId },
-            preset: tPresetSchema.parse(submission?.conversation),
+            preset: tPresetSchema.parse(submission.conversation),
           });
         }
         setIsSubmitting(false);
@@ -438,14 +472,14 @@ export default function useEventHandlers({
       const errorResponse = tMessageSchema.parse({
         ...data,
         error: true,
-        parentMessageId: userMessage?.messageId,
+        parentMessageId: userMessage.messageId,
       });
 
       setMessages([...messages, userMessage, errorResponse]);
       if (data.conversationId && paramId === 'new' && newConversation) {
         newConversation({
           template: { conversationId: data.conversationId },
-          preset: tPresetSchema.parse(submission?.conversation),
+          preset: tPresetSchema.parse(submission.conversation),
         });
       }
 
@@ -459,7 +493,7 @@ export default function useEventHandlers({
     async (conversationId = '', submission: TSubmission, messages?: TMessage[]) => {
       const runAbortKey = `${conversationId}:${messages?.[messages.length - 1]?.messageId ?? ''}`;
       console.log({ conversationId, submission, messages, runAbortKey });
-      const { endpoint: _endpoint, endpointType } = submission?.conversation || {};
+      const { endpoint: _endpoint, endpointType } = submission.conversation || {};
       const endpoint = endpointType ?? _endpoint;
       try {
         const response = await fetch(`${EndpointURLs[endpoint ?? '']}/abort`, {
@@ -513,7 +547,7 @@ export default function useEventHandlers({
         console.error(error);
         const convoId = conversationId ?? v4();
         const text =
-          submission.initialResponse?.text?.length > 45 ? submission.initialResponse?.text : '';
+          submission.initialResponse.text.length > 45 ? submission.initialResponse.text : '';
         const errorMessage = {
           ...submission,
           ...submission.initialResponse,
@@ -526,7 +560,7 @@ export default function useEventHandlers({
         if (newConversation) {
           newConversation({
             template: { conversationId: convoId },
-            preset: tPresetSchema.parse(submission?.conversation),
+            preset: tPresetSchema.parse(submission.conversation),
           });
         }
         setIsSubmitting(false);

packages/data-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "librechat-data-provider",
-  "version": "0.7.41",
+  "version": "0.7.411",
   "description": "data services for librechat apps",
   "main": "dist/index.js",
   "module": "dist/index.es.js",

packages/data-provider/src/parsers.ts

@@ -66,7 +66,7 @@ export function getEnabledEndpoints() {
   if (endpointsEnv) {
     enabledEndpoints = endpointsEnv
       .split(',')
-      .filter((endpoint) => endpoint?.trim())
+      .filter((endpoint) => endpoint.trim())
       .map((endpoint) => endpoint.trim());
   }
   return enabledEndpoints;
@@ -347,3 +347,16 @@ export function parseTextParts(contentParts: a.TMessageContentParts[]): string {
 
   return result;
 }
+
+export const SEPARATORS = ['.', '?', '!', '۔', '。', '‥', ';', '¡', '¿', '\n', '```'];
+
+export function findLastSeparatorIndex(text: string, separators = SEPARATORS): number {
+  let lastIndex = -1;
+  for (const separator of separators) {
+    const index = text.lastIndexOf(separator);
+    if (index > lastIndex) {
+      lastIndex = index;
+    }
+  }
+  return lastIndex;
+}