📣 a11y: Better Screen Reader Announcements (#3693)

* refactor: Improve LiveAnnouncer component

The LiveAnnouncer component in the client/src/a11y/LiveAnnouncer.tsx file has been refactored to improve its functionality. The component now processes text in chunks for better performance and adds a minimum announcement delay to prevent overlapping announcements. Additionally, the component now properly clears the announcement message and ID before setting a new one. These changes enhance the accessibility and user experience of the LiveAnnouncer component.

* refactor: manage only 2 LiveAnnouncer aria-live elements, queue assertive/polite together

* refactor: use localizations for event announcements

* refactor: update minimum announcement delay in LiveAnnouncer component

* refactor: replace *`_

* chore(useContentHandler): typing

* chore: more type fixes and safely announce final message

client/src/a11y/Announcer.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useEffect } from 'react';
+import React from 'react';
 import MessageBlock from './MessageBlock';
 
 interface AnnouncerProps {
@@ -8,45 +8,11 @@ interface AnnouncerProps {
   assertiveMessageId: string;
 }
 
-const Announcer: React.FC<AnnouncerProps> = ({
+const Announcer: React.FC<AnnouncerProps> = ({ politeMessage, assertiveMessage }) => {
-  politeMessage,
-  politeMessageId,
-  assertiveMessage,
-  assertiveMessageId,
-}) => {
-  const [state, setState] = useState({
-    assertiveMessage1: '',
-    assertiveMessage2: '',
-    politeMessage1: '',
-    politeMessage2: '',
-    setAlternatePolite: false,
-    setAlternateAssertive: false,
-  });
-
-  useEffect(() => {
-    setState((prevState) => ({
-      ...prevState,
-      politeMessage1: prevState.setAlternatePolite ? '' : politeMessage,
-      politeMessage2: prevState.setAlternatePolite ? politeMessage : '',
-      setAlternatePolite: !prevState.setAlternatePolite,
-    }));
-  }, [politeMessage, politeMessageId]);
-
-  useEffect(() => {
-    setState((prevState) => ({
-      ...prevState,
-      assertiveMessage1: prevState.setAlternateAssertive ? '' : assertiveMessage,
-      assertiveMessage2: prevState.setAlternateAssertive ? assertiveMessage : '',
-      setAlternateAssertive: !prevState.setAlternateAssertive,
-    }));
-  }, [assertiveMessage, assertiveMessageId]);
-
   return (
     <div>
-      <MessageBlock aria-live="assertive" aria-atomic="true" message={state.assertiveMessage1} />
+      <MessageBlock aria-live="assertive" aria-atomic="true" message={assertiveMessage} />
-      <MessageBlock aria-live="assertive" aria-atomic="true" message={state.assertiveMessage2} />
+      <MessageBlock aria-live="polite" aria-atomic="false" message={politeMessage} />
-      <MessageBlock aria-live="polite" aria-atomic="false" message={state.politeMessage1} />
-      <MessageBlock aria-live="polite" aria-atomic="false" message={state.politeMessage2} />
     </div>
   );
 };

client/src/a11y/LiveAnnouncer.tsx

@@ -1,23 +1,38 @@
-import React, { useState, useCallback, useRef, useEffect } from 'react';
+import React, { useState, useCallback, useRef, useEffect, useMemo } from 'react';
 import { findLastSeparatorIndex } from 'librechat-data-provider';
 import type { AnnounceOptions } from '~/Providers/AnnouncerContext';
 import AnnouncerContext from '~/Providers/AnnouncerContext';
+import useLocalize from '~/hooks/useLocalize';
 import Announcer from './Announcer';
 
 interface LiveAnnouncerProps {
   children: React.ReactNode;
 }
 
-const LiveAnnouncer: React.FC<LiveAnnouncerProps> = ({ children }) => {
+interface AnnouncementItem {
-  const [announcePoliteMessage, setAnnouncePoliteMessage] = useState('');
+  message: string;
-  const [politeMessageId, setPoliteMessageId] = useState('');
+  id: string;
-  const [announceAssertiveMessage, setAnnounceAssertiveMessage] = useState('');
+  isAssertive: boolean;
-  const [assertiveMessageId, setAssertiveMessageId] = useState('');
+}
+
+const CHUNK_SIZE = 50;
+const MIN_ANNOUNCEMENT_DELAY = 400;
+/** Regex to remove *, `, and _ from message text */
+const replacementRegex = /[*`_]/g;
+
+const LiveAnnouncer: React.FC<LiveAnnouncerProps> = ({ children }) => {
+  const [politeMessageId, setPoliteMessageId] = useState('');
+  const [assertiveMessageId, setAssertiveMessageId] = useState('');
+  const [announcePoliteMessage, setAnnouncePoliteMessage] = useState('');
+  const [announceAssertiveMessage, setAnnounceAssertiveMessage] = useState('');
 
-  const politeProcessedTextRef = useRef('');
-  const politeQueueRef = useRef<Array<{ message: string; id: string }>>([]);
-  const isAnnouncingRef = useRef(false);
   const counterRef = useRef(0);
+  const isAnnouncingRef = useRef(false);
+  const politeProcessedTextRef = useRef('');
+  const queueRef = useRef<AnnouncementItem[]>([]);
+  const timeoutRef = useRef<NodeJS.Timeout | null>(null);
+
+  const localize = useLocalize();
 
   const generateUniqueId = (prefix: string) => {
     counterRef.current += 1;
@@ -26,29 +41,76 @@ const LiveAnnouncer: React.FC<LiveAnnouncerProps> = ({ children }) => {
 
   const processChunks = (text: string, processedTextRef: React.MutableRefObject<string>) => {
     const remainingText = text.slice(processedTextRef.current.length);
-    const separatorIndex = findLastSeparatorIndex(remainingText);
+
+    if (remainingText.length < CHUNK_SIZE) {
+      return ''; /* Not enough characters to process */
+    }
+
+    let separatorIndex = -1;
+    let startIndex = CHUNK_SIZE;
+
+    while (separatorIndex === -1 && startIndex <= remainingText.length) {
+      separatorIndex = findLastSeparatorIndex(remainingText.slice(startIndex));
       if (separatorIndex !== -1) {
+        separatorIndex += startIndex; /* Adjust the index to account for the starting position */
+      } else {
+        startIndex += CHUNK_SIZE; /* Move the starting position by another CHUNK_SIZE characters */
+      }
+    }
+
+    if (separatorIndex === -1) {
+      return ''; /* No separator found, wait for more text */
+    }
+
     const chunkText = remainingText.slice(0, separatorIndex + 1);
     processedTextRef.current += chunkText;
     return chunkText.trim();
-    }
-    return '';
   };
 
+  /** Localized event announcements, i.e., "the AI is replying, finished, etc." */
+  const events: Record<string, string | undefined> = useMemo(
+    () => ({ start: localize('com_a11y_start'), end: localize('com_a11y_end') }),
+    [localize],
+  );
+
   const announceNextInQueue = useCallback(() => {
-    if (politeQueueRef.current.length > 0 && !isAnnouncingRef.current) {
+    if (queueRef.current.length > 0 && !isAnnouncingRef.current) {
       isAnnouncingRef.current = true;
-      const nextAnnouncement = politeQueueRef.current.shift();
+      const nextAnnouncement = queueRef.current.shift();
       if (nextAnnouncement) {
-        setAnnouncePoliteMessage(nextAnnouncement.message);
+        const { message: _msg, id, isAssertive } = nextAnnouncement;
-        setPoliteMessageId(nextAnnouncement.id);
+        const setMessage = isAssertive ? setAnnounceAssertiveMessage : setAnnouncePoliteMessage;
+        const setMessageId = isAssertive ? setAssertiveMessageId : setPoliteMessageId;
+
+        setMessage('');
+        setMessageId('');
+
+        /* Force a re-render before setting the new message */
         setTimeout(() => {
+          const message = (events[_msg] ?? _msg).replace(replacementRegex, '');
+          setMessage(message);
+          setMessageId(id);
+
+          if (timeoutRef.current) {
+            clearTimeout(timeoutRef.current);
+          }
+
+          timeoutRef.current = setTimeout(() => {
             isAnnouncingRef.current = false;
             announceNextInQueue();
-        }, 100);
+          }, MIN_ANNOUNCEMENT_DELAY);
+        }, 0);
       }
     }
-  }, []);
+  }, [events]);
+
+  const addToQueue = useCallback(
+    (item: AnnouncementItem) => {
+      queueRef.current.push(item);
+      announceNextInQueue();
+    },
+    [announceNextInQueue],
+  );
 
   const announcePolite = useCallback(
     ({ message, id, isStream = false, isComplete = false }: AnnounceOptions) => {
@@ -56,30 +118,29 @@ const LiveAnnouncer: React.FC<LiveAnnouncerProps> = ({ children }) => {
       if (isStream) {
         const chunk = processChunks(message, politeProcessedTextRef);
         if (chunk) {
-          politeQueueRef.current.push({ message: chunk, id: announcementId });
+          addToQueue({ message: chunk, id: announcementId, isAssertive: false });
-          announceNextInQueue();
         }
       } else if (isComplete) {
         const remainingText = message.slice(politeProcessedTextRef.current.length);
         if (remainingText.trim()) {
-          politeQueueRef.current.push({ message: remainingText.trim(), id: announcementId });
+          addToQueue({ message: remainingText.trim(), id: announcementId, isAssertive: false });
-          announceNextInQueue();
         }
         politeProcessedTextRef.current = '';
       } else {
-        politeQueueRef.current.push({ message, id: announcementId });
+        addToQueue({ message, id: announcementId, isAssertive: false });
-        announceNextInQueue();
         politeProcessedTextRef.current = '';
       }
     },
-    [announceNextInQueue],
+    [addToQueue],
   );
 
-  const announceAssertive = useCallback(({ message, id }: AnnounceOptions) => {
+  const announceAssertive = useCallback(
+    ({ message, id }: AnnounceOptions) => {
       const announcementId = id ?? generateUniqueId('assertive');
-    setAnnounceAssertiveMessage(message);
+      addToQueue({ message, id: announcementId, isAssertive: true });
-    setAssertiveMessageId(announcementId);
+    },
-  }, []);
+    [addToQueue],
+  );
 
   const contextValue = {
     announcePolite,
@@ -88,8 +149,11 @@ const LiveAnnouncer: React.FC<LiveAnnouncerProps> = ({ children }) => {
 
   useEffect(() => {
     return () => {
-      politeQueueRef.current = [];
+      queueRef.current = [];
       isAnnouncingRef.current = false;
+      if (timeoutRef.current) {
+        clearTimeout(timeoutRef.current);
+      }
     };
   }, []);
 

client/src/common/types.ts

@@ -461,17 +461,26 @@ export type NewConversationParams = {
 
 export type ConvoGenerator = (params: NewConversationParams) => void | TConversation;
 
-export type TResData = {
+export type TBaseResData = {
   plugin?: TResPlugin;
   final?: boolean;
   initial?: boolean;
   previousMessages?: TMessage[];
-  requestMessage: TMessage;
-  responseMessage: TMessage;
   conversation: TConversation;
   conversationId?: string;
   runMessages?: TMessage[];
 };
+
+export type TResData = TBaseResData & {
+  requestMessage: TMessage;
+  responseMessage: TMessage;
+};
+
+export type TFinalResData = TBaseResData & {
+  requestMessage?: TMessage;
+  responseMessage?: TMessage;
+};
+
 export type TVectorStore = {
   _id: string;
   object: 'vector_store';

client/src/hooks/SSE/useContentHandler.ts

@@ -36,7 +36,7 @@ export default function useContentHandler({ setMessages, getMessages }: TUseCont
         _messages
           ?.filter((m) => m.messageId !== messageId)
           ?.map((msg) => ({ ...msg, thread_id })) ?? [];
-      const userMessage = messages[messages.length - 1];
+      const userMessage = messages[messages.length - 1] as TMessage | undefined;
 
       const { initialResponse } = submission;
 
@@ -44,7 +44,7 @@ export default function useContentHandler({ setMessages, getMessages }: TUseCont
       if (!response) {
         response = {
           ...initialResponse,
-          parentMessageId: userMessage?.messageId,
+          parentMessageId: userMessage?.messageId ?? '',
           conversationId,
           messageId,
           thread_id,
@@ -55,7 +55,7 @@ export default function useContentHandler({ setMessages, getMessages }: TUseCont
       // TODO: handle streaming for non-text
       const textPart: Text | string | undefined = data[ContentTypes.TEXT];
       const part: ContentPart =
-        textPart && typeof textPart === 'string' ? { value: textPart } : data[type];
+        textPart != null && typeof textPart === 'string' ? { value: textPart } : data[type];
 
       if (type === ContentTypes.IMAGE_FILE) {
         addFileToCache(queryClient, part as ImageFile & PartMetadata);

client/src/hooks/SSE/useEventHandlers.ts

@@ -18,7 +18,7 @@ import type {
   ConversationData,
 } from 'librechat-data-provider';
 import type { SetterOrUpdater, Resetter } from 'recoil';
-import type { TResData, ConvoGenerator } from '~/common';
+import type { TResData, TFinalResData, ConvoGenerator } from '~/common';
 import {
   scrollToEnd,
   addConversation,
@@ -186,6 +186,11 @@ export default function useEventHandlers({
         },
       ]);
 
+      announceAssertive({
+        message: 'start',
+        id: `start-${Date.now()}`,
+      });
+
       let update = {} as TConversation;
       if (setConversation && !isAddedRequest) {
         setConversation((prevState) => {
@@ -236,10 +241,11 @@ export default function useEventHandlers({
       }
     },
     [
-      setMessages,
-      setConversation,
       queryClient,
+      setMessages,
       isAddedRequest,
+      setConversation,
+      announceAssertive,
       setShowStopButton,
       resetLatestMessage,
     ],
@@ -261,8 +267,8 @@ export default function useEventHandlers({
 
       const { conversationId, parentMessageId } = userMessage;
       announceAssertive({
-        message: 'The AI is generating a response.',
+        message: 'start',
-        id: `ai-generating-${Date.now()}`,
+        id: `start-${Date.now()}`,
       });
 
       let update = {} as TConversation;
@@ -323,7 +329,7 @@ export default function useEventHandlers({
   );
 
   const finalHandler = useCallback(
-    (data: TResData, submission: TSubmission) => {
+    (data: TFinalResData, submission: TSubmission) => {
       const { requestMessage, responseMessage, conversation, runMessages } = data;
       const { messages, conversation: submissionConvo, isRegenerate = false } = submission;
 
@@ -331,30 +337,30 @@ export default function useEventHandlers({
       setCompleted((prev) => new Set(prev.add(submission.initialResponse.messageId)));
 
       const currentMessages = getMessages();
-      // Early return if messages are empty; i.e., the user navigated away
+      /* Early return if messages are empty; i.e., the user navigated away */
-      if (!currentMessages?.length) {
+      if (!currentMessages || currentMessages.length === 0) {
         return setIsSubmitting(false);
       }
 
       /* a11y announcements */
       announcePolite({
-        message: '',
+        message: responseMessage?.text ?? '',
         isComplete: true,
       });
 
       setTimeout(() => {
         announcePolite({
-          message: 'The AI has finished generating a response.',
+          message: 'end',
-          id: `ai-finished-${Date.now()}`,
+          id: `end-${Date.now()}`,
         });
       }, 100);
 
-      // update the messages; if assistants endpoint, client doesn't receive responseMessage
+      /* Update messages; if assistants endpoint, client doesn't receive responseMessage */
       if (runMessages) {
         setMessages([...runMessages]);
       } else if (isRegenerate && responseMessage) {
         setMessages([...messages, responseMessage]);
-      } else if (responseMessage) {
+      } else if (requestMessage != null && responseMessage != null) {
         setMessages([...messages, requestMessage, responseMessage]);
       }
 
@@ -368,7 +374,7 @@ export default function useEventHandlers({
         });
       }
 
-      // refresh title
+      /* Refresh title */
       if (
         genTitle &&
         isNewConvo &&
@@ -380,14 +386,14 @@ export default function useEventHandlers({
         }, 2500);
       }
 
-      if (setConversation && !isAddedRequest) {
+      if (setConversation && isAddedRequest !== true) {
         setConversation((prevState) => {
           const update = {
             ...prevState,
             ...conversation,
           };
 
-          if (prevState?.model && prevState.model !== submissionConvo.model) {
+          if (prevState?.model != null && prevState.model !== submissionConvo.model) {
             update.model = prevState.model;
           }
 
@@ -421,14 +427,14 @@ export default function useEventHandlers({
 
       const parseErrorResponse = (data: TResData | Partial<TMessage>) => {
         const metadata = data['responseMessage'] ?? data;
-        const errorMessage = {
+        const errorMessage: Partial<TMessage> = {
           ...initialResponse,
           ...metadata,
           error: true,
           parentMessageId: userMessage.messageId,
         };
 
-        if (!errorMessage.messageId) {
+        if (errorMessage.messageId === undefined || errorMessage.messageId === '') {
           errorMessage.messageId = v4();
         }
 
@@ -514,7 +520,7 @@ export default function useEventHandlers({
 
         // Check if the response is JSON
         const contentType = response.headers.get('content-type');
-        if (contentType && contentType.includes('application/json')) {
+        if (contentType != null && contentType.includes('application/json')) {
           const data = await response.json();
           console.log(`[aborted] RESPONSE STATUS: ${response.status}`, data);
           if (response.status === 404) {

client/src/localization/languages/Eng.ts

@@ -3,6 +3,8 @@
 // file deepcode ignore HardcodedNonCryptoSecret: No hardcoded secrets present in this file
 
 export default {
+  com_a11y_start: 'The AI is replying.',
+  com_a11y_end: 'The AI has finished their reply.',
   com_error_moderation:
     'It appears that the content submitted has been flagged by our moderation system for not aligning with our community guidelines. We\'re unable to proceed with this specific topic. If you have any other questions or topics you\'d like to explore, please edit your message, or create a new conversation.',
   com_error_no_user_key: 'No key found. Please provide a key and try again.',