mirror of
https://github.com/danny-avila/LibreChat.git
synced 2025-12-16 08:20:14 +01:00
🔉 feat: Speech-to-text / Text-to-speech (initial support) (#2836)
* Update TextChat.jsx * Update SubmitButton.jsx * Update TextChat.jsx * Update SubmitButton.jsx * Create ListeningIcon.tsx * Update index.ts * Update SubmitButton.jsx * Update TextChat.jsx * Update ListeningIcon.tsx * Update ListeningIcon.tsx * Create SpeechRecognition.tsx * Update TextChat.jsx * Update TextChat.jsx * Update SpeechRecognition.tsx * Update TextChat.jsx * Update SpeechRecognition.tsx * Update SpeechRecognition.tsx * Update SpeechRecognition.tsx * Update SpeechRecognition.tsx * Update SubmitButton.jsx * Update TextChat.jsx * Update SpeechRecognition.tsx * Create SpeechSynthesis.tsx * Update index.jsx * Update SpeechSynthesis.tsx * Update SpeechRecognition.tsx * Update TextChat.jsx * Update SpeechRecognition.tsx * Update SpeechRecognition.tsx * Update SpeechRecognition.tsx * Update TextChat.jsx * Squashed commit of the following: commit28230d9305Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Sun Sep 3 02:44:26 2023 +0200 feat: delete button confirm (#875) * base for confirm delete * more like OpenAI commit2b54e3f9feAuthor: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Fri Sep 1 14:20:51 2023 -0400 update: install script (#858) commit1cd0fd9d5aAuthor: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Fri Sep 1 08:12:35 2023 -0400 doc: Hugging Face Deployment (#867) * docs: update ToC * docs: update ToC * update huggingface.md * update render.md * update huggingface.md * update mongodb.md * update huggingface.md * update README.md commitaeeb3d3050Author: Mu Yuan <yuanmu.email@gmail.com> Date: Thu Aug 31 07:21:27 2023 +0800 Update Zh.tsx (#862) * Update Zh.tsx Changed the translation of several words to make it more relevant to Chinese usage habits. * Update Zh.tsx Changed the translation of several words to make it more relevant to Chinese usage habits commit80e2e2675bAuthor: Raí <140329135+itzraiss@users.noreply.github.com> Date: Mon Aug 28 18:05:46 2023 -0300 Translation of 'com_ui_pay_per_call:' to Spanish and Portuguese that were missing. (#857) * Update Br.tsx * Update Es.tsx * Update Br.tsx * Update Es.tsx commit3574d0b823Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 28 14:49:26 2023 -0400 docs: make_your_own.md formatting fix for mkdocs (#855) commitd672ac690dAuthor: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 28 14:24:10 2023 -0400 Release v0.5.8 (#854) * chore: add 'api' image to tag release workflow * docs: update DO deployment docs to include instruction about latest stable release, as well as security best practices * Release v0.5.8 * docs: Update digitalocean.md with firewall section images * docs: make_your_own.md formatting fix for mkdocs commitd3e7627046Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 28 12:03:08 2023 -0400 refactor(plugins): Improve OpenAPI handling, Show Multiple Plugins, & Other Improvements (#845) * feat(PluginsClient.js): add conversationId to options object in the constructor feat(PluginsClient.js): add support for Code Interpreter plugin feat(PluginsClient.js): add support for Code Interpreter plugin in the availableTools manifest feat(CodeInterpreter.js): add CodeInterpreterTools module feat(CodeInterpreter.js): add RunCommand class feat(CodeInterpreter.js): add ReadFile class feat(CodeInterpreter.js): add WriteFile class feat(handleTools.js): add support for loading Code Interpreter plugin * chore(api): update langchain dependency to version 0.0.123 * fix(CodeInterpreter.js): add support for extracting environment from code fix(WriteFile.js): add support for extracting environment from data fix(extractionChain.js): add utility functions for creating extraction chain from Zod schema fix(handleTools.js): refactor getOpenAIKey function to handle user-provided API key fix(handleTools.js): pass model and openAIApiKey to CodeInterpreter constructor * fix(tools): rename CodeInterpreterTools to E2BTools fix(tools): rename code_interpreter pluginKey to e2b_code_interpreter * chore(PluginsClient.js): comment out unused import and function findMessageContent feat(PluginsClient.js): add support for CodeSherpa plugin feat(PluginsClient.js): add CodeSherpaTools to available tools feat(PluginsClient.js): update manifest.json to include CodeSherpa plugin feat(CodeSherpaTools.js): create RunCode and RunCommand classes for CodeSherpa plugin feat(E2BTools.js): Add E2BTools module for extracting environment from code and running commands, reading and writing files fix(codesherpa.js): Remove codesherpa module as it is no longer needed feat(handleTools.js): add support for CodeSherpaTools in loadTools function feat(loadToolSuite.js): create loadToolSuite utility function to load a suite of tools * feat(PluginsClient.js): add support for CodeSherpa v2 plugin feat(PluginsClient.js): add CodeSherpa v1 plugin to available tools feat(PluginsClient.js): add CodeSherpa v2 plugin to available tools feat(PluginsClient.js): update manifest.json for CodeSherpa v1 plugin feat(PluginsClient.js): update manifest.json for CodeSherpa v2 plugin feat(CodeSherpa.js): implement CodeSherpa plugin for interactive code and shell command execution feat(CodeSherpaTools.js): implement RunCode and RunCommand plugins for CodeSherpa v1 feat(CodeSherpaTools.js): update RunCode and RunCommand plugins for CodeSherpa v2 fix(handleTools.js): add CodeSherpa import statement fix(handleTools.js): change pluginKey from 'codesherpa' to 'codesherpa_tools' fix(handleTools.js): remove model and openAIApiKey from options object in e2b_code_interpreter tool fix(handleTools.js): remove openAIApiKey from options object in codesherpa_tools tool fix(loadToolSuite.js): remove model and openAIApiKey parameters from loadToolSuite function * feat(initializeFunctionsAgent.js): add prefix to agentArgs in initializeFunctionsAgent function The prefix is added to the agentArgs in the initializeFunctionsAgent function. This prefix is used to provide instructions to the agent when it receives any instructions from a webpage, plugin, or other tool. The agent will notify the user immediately and ask them if they wish to carry out or ignore the instructions. * feat(PluginsClient.js): add ChatTool to the list of tools if it meets the conditions feat(tools/index.js): import and export ChatTool feat(ChatTool.js): create ChatTool class with necessary properties and methods * fix(initializeFunctionsAgent.js): update PREFIX message to include sharing all output from the tool fix(E2BTools.js): update descriptions for RunCommand, ReadFile, and WriteFile plugins to provide more clarity and context * chore: rebuild package-lock after rebase * chore: remove deleted file from rebase * wip: refactor plugin message handling to mirror chat.openai.com, handle incoming stream for plugin use * wip: new plugin handling * wip: show multiple plugins handling * feat(plugins): save new plugins array * chore: bump langchain * feat(experimental): support streaming in between plugins * refactor(PluginsClient): factor out helper methods to avoid bloating the class, refactor(gptPlugins): use agent action for mapping the name of action * fix(handleTools): fix tests by adding condition to return original toolFunctions map * refactor(MessageContent): Allow the last index to be last in case it has text (may change with streaming) * feat(Plugins): add handleParsingErrors, useful when LLM does not invoke function params * chore: edit out experimental codesherpa integration * refactor(OpenAPIPlugin): rework tool to be 'function-first', as the spec functions are explicitly passed to agent model * refactor(initializeFunctionsAgent): improve error handling and system message * refactor(CodeSherpa, Wolfram): optimize token usage by delegating bulk of instructions to system message * style(Plugins): match official style with input/outputs * chore: remove unnecessary console logs used for testing * fix(abortMiddleware): render markdown when message is aborted * feat(plugins): add BrowserOp * refactor(OpenAPIPlugin): improve prompt handling * fix(useGenerations): hide edit button when message is submitting/streaming * refactor(loadSpecs): optimize OpenAPI spec loading by only loading requested specs instead of all of them * fix(loadSpecs): will retain original behavior when no tools are passed to the function * fix(MessageContent): ensure cursor only shows up for last message and last display index fix(Message): show legacy plugin and pass isLast to Content * chore: remove console.logs * docs: update docs based on breaking changes and new features refactor(structured/SD): use description_for_model for detailed prompting * docs(azure): make plugins section more clear * refactor(structured/SD): change default payload to SD-WebUI to prefer realism and config for SDXL * refactor(structured/SD): further improve system message prompt * docs: update breaking changes after rebase * refactor(MessageContent): factor out EditMessage, types, Container to separate files, rename Content -> Markdown * fix(CodeInterpreter): linting errors * chore: reduce browser console logs from message streams * chore: re-enable debug logs for plugins/langchain to help with user troubleshooting * chore(manifest.json): add [Experimental] tag to CodeInterpreter plugins, which are not intended as the end-all be-all implementation of this feature for Librechat commit66b8580487Author: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Mon Aug 28 09:18:25 2023 -0400 docs: third-party tools (#848) * docs: third-party tools * docs: third-party tools * Update third-party.md * Update third-party.md --------- Co-authored-by: Danny Avila <110412045+danny-avila@users.noreply.github.com> commit9791a78161Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Mon Aug 28 15:14:05 2023 +0200 adjust the animation (#843) commit3797ec6082Author: Ronith <87087292+ronith256@users.noreply.github.com> Date: Mon Aug 28 18:43:50 2023 +0530 feat: Add Code Interpreter Plugin (#837) * feat: Add Code Interpreter Plugin Adds a Simple Code Interpreter Plugin. ## Features: - Runs code using local Python Environment ## Issues - Code execution is not sandboxed. * Add Docker Sandbox for Python Server commite2397076a2Author: Alex Zhang <ztc2011@gmail.com> Date: Mon Aug 28 00:55:34 2023 +0800 🌐: Chinese Translation (#846) commit50c15c704fAuthor: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Sat Aug 26 19:36:59 2023 -0400 Language translation: Polish (#840) * Language translation: Polish * Language translation: Polish * Revert changes in language-contributions.md commit29d3640546Author: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Sat Aug 26 19:36:25 2023 -0400 docs: updates (#841) commit39c626aa8eAuthor: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 25 09:29:19 2023 -0400 fix: isEdited edge case where latest Message is not saved due to aborting too quickly commitae5c06f381Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 25 09:13:50 2023 -0400 fix(chatGPTBrowser): render markdown formatting by setting isCreatedByUser, fix(useMessageHandler): avoid double appearance of cursor by setting latest message at initial response creation time commit9ef1686e18Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Thu Aug 24 20:24:47 2023 -0400 Update mkdocs.yml commit5bbe411569Author: Flynn <dev@flynnbuckingham.com> Date: Thu Aug 24 20:20:37 2023 -0400 Add podman installation instructions. Update dockerfile to stub env (#819) * Added podman container installation docs. Updated dockerfile to stub env file if not present in source * Fix typos commit887fec99caAuthor: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 25 02:11:27 2023 +0200 🌐: Russian Translation (#830) commit007d51ede1Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 25 02:10:48 2023 +0200 feat: facebook login (#820) * Facebook strategy * Update user_auth_system.md * Update user_auth_system.md commita569020312Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Thu Aug 24 21:59:11 2023 +0200 Fix Meilisearch error and refactor of the server index.js (#832) * fix meilisearch error at startup * limit the nesting * disable useless console log * fix(indexSync.js): removed redundant searchEnabled * refactor(index.js): moved configureSocialLogins to a new file * refactor(socialLogins.js): removed unnecessary conditional commit37347d4683Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Wed Aug 23 16:14:17 2023 -0400 fix(registration): Make Username optional (#831) * fix(User.js): update validation schema for username field, allow empty string as a valid value fix(validators.js): update validation schema for username field, allow empty string as a valid value fix(Registration.tsx, validators.js): update validation rules for name and username fields, change minimum length to 2 and maximum length to 80, assure they match and allow empty string as a valid value fix(Eng.tsx): update localization string for com_auth_username, indicate that it is optional * fix(User.js): update regex pattern for username validation to allow special characters @#$%&*() fix(validators.js): update regex pattern for username validation to allow special characters @#$%&*() * fix(Registration.spec.tsx): fix validation error message for username length requirement commitd38e463d34Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Wed Aug 23 13:44:40 2023 -0400 fix(bingAI): markdown and error formatting for final stream response (#829) * fix(bingAI): markdown formatting for final stream response due to new strict payload validation on the frontend * fix: add missing prop to bing Error response commit7dc27b10f1Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Tue Aug 22 18:44:59 2023 -0400 feat: Edit AI Messages, Edit Messages in Place (#825) * refactor: replace lodash import with specific function import fix(api): esm imports to cjs * refactor(Messages.tsx): convert to TS, out-source scrollToDiv logic to a custom hook fix(ScreenshotContext.tsx): change Ref to RefObject in ScreenshotContextType feat(useScrollToRef.ts): add useScrollToRef hook for scrolling to a ref with throttle fix(Chat.tsx): update import path for Messages component fix(Search.tsx): update import path for Messages component * chore(types.ts): add TAskProps and TOptions types refactor(useMessageHandler.ts): use TAskFunction type for ask function signature * refactor(Message/Content): convert to TS, move Plugin component to Content dir * feat(MessageContent.tsx): add MessageContent component for displaying and editing message content feat(index.ts): export MessageContent component from Messages/Content directory * wip(Message.jsx): conversion and use of new component in progress * refactor: convert Message.jsx to TS and fix typing/imports based on changes * refactor: add typed props and refactor MultiMessage to TS, fix typing issues resulting from the conversion * edit message in progress * feat: complete edit AI message logic, refactor continue logic * feat(middleware): add validateMessageReq middleware feat(routes): add validation for message requests using validateMessageReq middleware feat(routes): add create, read, update, and delete routes for messages * feat: complete frontend logic for editing messages in place feat(messages.js): update route for updating a specific message - Change the route for updating a message to include the messageId in the URL - Update the request handler to use the messageId from the request parameters and the text from the request body - Call the updateMessage function with the updated parameters feat(MessageContent.tsx): add functionality to update a message - Import the useUpdateMessageMutation hook from the data provider - Destructure the conversationId, parentMessageId, and messageId from the message object - Create a mutation function using the useUpdateMessageMutation hook - Implement the updateMessage function to call the mutation function with the updated message parameters - Update the messages state to reflect the updated message text feat(api-endpoints.ts): update messages endpoint to include messageId - Update the messages endpoint to include the messageId as an optional parameter feat(data-service.ts): add updateMessage function - Implement the updateMessage function to make a PUT request to * fix(messages.js): make updateMessage function asynchronous and await its execution * style(EditIcon): make icon active for AI message * feat(gptPlugins/anthropic): add edit support * fix(validateMessageReq.js): handle case when conversationId is 'new' and return empty array feat(Message.tsx): pass message prop to SiblingSwitch component refactor(SiblingSwitch.tsx): convert to TS * fix(useMessageHandler.ts): remove message from currentMessages if isContinued is true feat(useMessageHandler.ts): add support for submission messages in setMessages fix(useServerStream.ts): remove unnecessary conditional in setMessages fix(useServerStream.ts): remove isContinued variable from submission * fix(continue): switch to continued message generation when continuing an earlier branch in conversation * fix(abortMiddleware.js): fix condition to check partialText length chore(abortMiddleware.js): add error logging when abortMessage fails * refactor(MessageHeader.tsx): convert to TS fix(Plugin.tsx): add default value for className prop in Plugin component * refactor(MultiMessage.tsx): remove commented out code docs(MultiMessage.tsx): update comment to clarify when siblingIdx is reset * fix(GenerationButtons): optimistic state for continue button * fix(MessageContent.tsx): add data-testid attribute to message text editor fix(messages.spec.ts): update waitForServerStream function to include edit endpoint check feat(messages.spec.ts): add test case for editing messages * fix(HoverButtons & Message & useGenerations): Refactor edit functionality and related conditions - Update enterEdit function signature and prop - Create and utilize hideEditButton variable - Enhance conditions for edit button visibility and active state - Update button event handlers - Introduce isEditableEndpoint in useGenerations and refine continueSupported condition. * fix(useGenerations.ts): fix condition for hideEditButton to include error and searchResult chore(data-provider): bump version to 0.1.6 fix(types.ts): add status property to TError type * chore: bump @dqbd/tiktoken to 1.0.7 * fix(abortMiddleware.js): add required isCreatedByUser property to the error response object * refactor(Message.tsx): remove unnecessary props from SiblingSwitch component, as setLatestMessage is firing on every switch already refactor(SiblingSwitch.tsx): remove unused imports and code * chore(BaseClient.js): move console.debug statements back inside if block commitdb77163f5dAuthor: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Tue Aug 22 14:15:14 2023 +0200 docs: update chimeragpt (#826) * Update free_ai_apis.md * Update free_ai_apis.md commit4a4e803df3Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Mon Aug 21 20:15:18 2023 +0200 style(Dialog): Improved Close Button ("X") position (#824) commit909b00c752Author: Daniel Avila <messagedaniel@protonmail.com> Date: Sun Aug 20 21:04:36 2023 -0400 fix(HoverButtons): light/dark styling to match official site commit61dcb4d307Author: Naosuke Yokoe <ankerasoy@gmail.com> Date: Sat Aug 19 20:11:31 2023 +0900 feat: Azure Cognitive Search Plugin (#815) * feat(AzureCognitiveSearchPlugin) * feat(tools/AzureCognitiveSearch.js): Add a new plugin (not structured version) * feat(tools/structured/AzureCognitiveSearch.js): Add a new plugin (structured version) * feat(tools/manifest.json, tools/index.js, tools/util/handleTools.js): Add configurations for the plugin * feat(api/package.json, package-lock.json): Installed a new package for the plugin (@azure/search-documents) * feat(.env.example): Add new environment variables for the plugin Here is the link to the corresponding discussion page: https://github.com/danny-avila/LibreChat/discussions/567 * docs(AzureCognitiveSearchPlugin) * docs(features/plugins/azure_cognitive_search.md): Add a new document for the plugin * (fix:.env.example) * reverted extra whitespaces removed by the editor * docs(mkdocs.yml) * Add the Azure Cognitive Search Plugin's documentation item to mkdocs.yml. commit3c7f67fa76Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 18 12:40:33 2023 -0400 fix(abortMiddleware): handle early abort error where userMessage.conversationId is undefined. In this case, the userId will be used as the abortKey commitc74c68a135Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 18 12:10:30 2023 -0400 refactor(MessageHandler -> useServerStream): convert all relating files to TS and correct typings based on this change: properly refactor MessageHandler to a custom hook, where it's passed a submission object to instantiate the stream. This is the bare minimum groundwork for potentially having multiple streams running, which would be a big project to modularize a lot of the global state into maps/multiple streams, particular useful for having multiple views in place commit8b4d3c2c21Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 18 12:04:29 2023 -0400 refactor(routes): convert to TS commitd612cfcb45Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 18 12:02:39 2023 -0400 chore(Auth): reorder exports in Auth component fix(PluginAuthForm): handle case when pluginKey is null or undefined fix(PluginStoreDialog): handle case when getAvailablePluginFromKey is null or undefined fix(AuthContext): make authConfig optional in AuthContextProvider feat(hooks): add useServerStream hook fix(conversation): setSubmission to null instead of empty object fix(preset): specify type for presets atom fix(search): specify type for isSearchEnabled atom fix(submission): specify type for submission atom commitc40b95f424Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 18 16:11:00 2023 +0200 feat: Disable Registration with social login (#813) * Google, Github and Discord * update .env.example with ALLOW_SOCIAL_REGISTRATION * fix some conflict * refactor strategy * Update user_auth_system.md * Update user_auth_system.md commit46ed5aaccdAuthor: Patrick <psarnowski@gmail.com> Date: Fri Aug 18 09:38:24 2023 -0400 Show the response scores from Bing. (#814) commit1dacfa49f0Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Thu Aug 17 20:32:31 2023 +0200 update profile picture (#792) commitafd43afb60Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Thu Aug 17 12:50:05 2023 -0400 feat(GPT/Anthropic): Continue Regenerating & Generation Buttons (#808) * feat(useMessageHandler.js/ts): Refactor and add features to handle user messages, support multiple endpoints/models, generate placeholder responses, regeneration, and stopGeneration function fix(conversation.ts, buildTree.ts): Import TMessage type, handle null parentMessageId feat(schemas.ts): Update and add schemas for various AI services, add default values, optional fields, and endpoint-to-schema mapping, create parseConvo function chore(useMessageHandler.js, schemas.ts): Remove unused imports, variables, and chatGPT enum * wip: add generation buttons * refactor(cleanupPreset.ts): simplify cleanupPreset function refactor(getDefaultConversation.js): remove unused code and simplify getDefaultConversation function feat(utils): add getDefaultConversation function This commit adds a new utility function called `getDefaultConversation` to the `client/src/utils/getDefaultConversation.ts` file. This function is responsible for generating a default conversation object based on the provided parameters. The `getDefaultConversation` function takes in an object with the following properties: - `conversation`: The conversation object to be used as a base. - `endpointsConfig`: The configuration object containing information about the available endpoints. - `preset`: An optional preset object that can be used to override the default behavior. The function first tries to determine the target endpoint based on the preset object. If a valid endpoint is found, it is used as the target endpoint. If not, the function tries to retrieve the last conversation setup from the local storage and uses its endpoint if it is valid. If neither the preset nor the local storage contains a valid endpoint, the function falls back to a default endpoint. Once the target endpoint is determined, * fix(utils): remove console.error statement in buildDefaultConversation function fix(schemas): add default values for catch blocks in openAISchema, googleSchema, bingAISchema, anthropicSchema, chatGPTBrowserSchema, and gptPluginsSchema * fix: endpoint not changing on change of preset from other endpoint, wip: refactor * refactor: preset items to TSX * refactor: convert resetConvo to TS * refactor(getDefaultConversation.ts): move defaultEndpoints array to the top of the file for better readability refactor(getDefaultConversation.ts): extract getDefaultEndpoint function for better code organization and reusability * feat(svg): add ContinueIcon component feat(svg): add RegenerateIcon component feat(svg): add ContinueIcon and RegenerateIcon components to index.ts * feat(Button.tsx): add onClick and className props to Button component feat(GenerationButtons.tsx): add logic to display Regenerate or StopGenerating button based on isSubmitting and messages feat(Regenerate.tsx): create Regenerate component with RegenerateIcon and handleRegenerate function feat(StopGenerating.tsx): create StopGenerating component with StopGeneratingIcon and handleStopGenerating function * fix(TextChat.jsx): reorder imports and variables for better readability fix(TextChat.jsx): fix typo in condition for isNotAppendable variable fix(TextChat.jsx): remove unused handleStopGenerating function fix(ContinueIcon.tsx): remove unnecessary closing tags for polygon elements fix(useMessageHandler.ts): add missing type annotations for handleStopGenerating and handleRegenerate functions fix(useMessageHandler.ts): remove unused variables in return statement * fix(getDefaultConversation.ts): refactor code to use getLocalStorageItems function feat(getLocalStorageItems.ts): add utility function to retrieve items from local storage * fix(OpenAIClient.js): add support for streaming result in sendCompletion method feat(OpenAIClient.js): add finish_reason metadata to opts in sendCompletion method feat(Message.js): add finish_reason field to Message model feat(messageSchema.js): add finish_reason field to messageSchema feat(openAI.js): parse chatGptLabel and promptPrefix from req.body and pass rest of the modelOptions to endpointOption feat(openAI.js): add addMetadata function to store metadata in ask function feat(openAI.js): add metadata to response if available feat(schemas.ts): add finish_reason field to tMessageSchema * feat(types.ts): add TOnClick and TGenButtonProps types for button components feat(Continue.tsx): create Continue component for generating button feat(GenerationButtons.tsx): update GenerationButtons component to use Continue component feat(Regenerate.tsx): create Regenerate component for regenerating button feat(Stop.tsx): create Stop component for stop generating button * feat(MessageHandler.jsx): add MessageHandler component to handle messages and conversations fix(Root.jsx): fix import paths for Nav and MessageHandler components * feat(useMessageHandler.ts): add support for generation parameter in ask function feat(useMessageHandler.ts): add support for isEdited parameter in ask function feat(useMessageHandler.ts): add support for continueGeneration function fix(createPayload.ts): replace endpoint URL when isEdited parameter is true * chore(client): set skipLibCheck to true in tsconfig.json * fix(useMessageHandler.ts): remove unused clientId variable fix(schemas.ts): make clientId field in tMessageSchema nullable and optional * wip: edit route for continue generation * refactor(api): move handlers to root of routes dir * fix(useMessageHandler.ts): initialize currentMessages to an empty array if messages is null fix(useMessageHandler.ts): update initialResponse text to use responseText variable fix(useMessageHandler.ts): update setMessages logic for isRegenerate case fix(MessageHandler.jsx): update setMessages logic for cancelHandler, createdHandler, and finalHandler * fix(schemas.ts): make createdAt and updatedAt fields optional and set default values using new Date().toISOString() fix(schemas.ts): change type annotation of TMessage from infer to input * refactor(useMessageHandler.ts): rename AskProps type to TAskProps refactor(useMessageHandler.ts): remove generation property from ask function arguments refactor(useMessageHandler.ts): use nullish coalescing operator (??) instead of logical OR (||) refactor(useMessageHandler.ts): pass the responseMessageId to message prop of submission * fix(BaseClient.js): use nullish coalescing operator (??) instead of logical OR (||) for default values * fix(BaseClient.js): fix responseMessageId assignment in handleStartMethods method feat(BaseClient.js): add support for isEdited flag in sendMessage method feat(BaseClient.js): add generation to responseMessage text in sendMessage method * fix(openAI.js): remove unused imports and commented out code feat(openAI.js): add support for generation parameter in request body fix(openAI.js): remove console.log statement fix(openAI.js): remove unused variables and parameters fix(openAI.js): update response text in case of error fix(openAI.js): handle error and abort message in case of error fix(handlers.js): add generation parameter to createOnProgress function fix(useMessageHandler.ts): update responseText variable to use generation parameter * refactor(api/middleware): move inside server dir * refactor: add endpoint specific, modular functions to build options and initialize clients, create server/utils, move middleware, separate utils into api general utils and server specific utils * fix(abortMiddleware.js): import getConvo and getConvoTitle functions from models feat(abortMiddleware.js): add abortAsk function to abortController to handle aborting of requests fix(openAI.js): import buildOptions and initializeClient functions from endpoints/openAI refactor(openAI.js): use getAbortData function to get data for abortAsk function * refactor: move endpoint specific logic to an endpoints dir * refactor(PluginService.js): fix import path for encrypt and decrypt functions in PluginService.js * feat(openAI): add new endpoint for adding a title to a conversation - Added a new file `addTitle.js` in the `api/server/routes/endpoints/openAI` directory. - The `addTitle.js` file exports a function `addTitle` that takes in request parameters and performs the following actions: - If the `parentMessageId` is `'00000000-0000-0000-0000-000000000000'` and `newConvo` is true, it proceeds with the following steps: - Calls the `titleConvo` function from the `titleConvo` module, passing in the necessary parameters. - Calls the `saveConvo` function from the `saveConvo` module, passing in the user ID and conversation details. - Updated the `index.js` file in the `api/server/routes/endpoints/openAI` directory to export the `addTitle` function. - This change adds * fix(abortMiddleware.js): remove console.log statement refactor(gptPlugins.js): update imports and function parameters feat(gptPlugins.js): add support for abortController and getAbortData refactor(openAI.js): update imports and function parameters feat(openAI.js): add support for abortController and getAbortData fix(openAI.js): refactor code to use modularized functions and middleware fix(buildOptions.js): refactor code to use destructuring and update variable names * refactor(askChatGPTBrowser.js, bingAI.js, google.js): remove duplicate code for setting response headers feat(askChatGPTBrowser.js, bingAI.js, google.js): add setHeaders middleware to set response headers * feat(middleware): validateEndpoint, refactor buildOption to only be concerned of endpointOption * fix(abortMiddleware.js): add 'finish_reason' property with value 'incomplete' to responseMessage object fix(abortMessage.js): remove console.log statement for aborted message fix(handlers.js): modify tokens assignment to handle empty generation string and trailing space * fix(BaseClient.js): import addSpaceIfNeeded function from server/utils fix(BaseClient.js): add space before generation in text property fix(index.js): remove getCitations and citeText exports feat(buildEndpointOption.js): add buildEndpointOption middleware fix(index.js): import buildEndpointOption middleware fix(anthropic.js): remove buildOptions function and use endpointOption from req.body fix(gptPlugins.js): remove buildOptions function and use endpointOption from req.body fix(openAI.js): remove buildOptions function and use endpointOption from req.body feat(utils): add citations.js and handleText.js modules fix(utils): fix import statements in index.js module * refactor(gptPlugins.js): use getResponseSender function from librechat-data-provider * feat(gptPlugins): complete 'continue generating' * wip: anthropic continue regen * feat(middleware): add validateRegistration middleware A new middleware function called `validateRegistration` has been added to the list of exported middleware functions in `index.js`. This middleware is responsible for validating registration data before allowing the registration process to proceed. * feat(Anthropic): complete continue regen * chore: add librechat-data-provider to api/package.json * fix(ci): backend-review will mock meilisearch, also installs data-provider as now needed * chore(ci): remove unneeded SEARCH env var * style(GenerationButtons): make text shorter for sake of space economy, even though this diverges from chat.openai.com * style(GenerationButtons/ScrollToBottom): adjust visibility/position based on screen size * chore(client): 'Editting' typo * feat(GenerationButtons.tsx): add support for endpoint prop in GenerationButtons component feat(OptionsBar.tsx): pass endpoint prop to GenerationButtons component feat(useGenerations.ts): create useGenerations hook to handle generation logic fix(schemas.ts): add searchResult field to tMessageSchema * refactor(HoverButtons): convert to TSX and utilize new useGenerations hook * fix(abortMiddleware): handle error with res headers set, or abortController not found, to ensure proper API error is sent to the client, chore(BaseClient): remove console log for onStart message meant for debugging * refactor(api): remove librechat-data-provider dep for now as it complicates deployed docker build stage, re-use code in CJS, located in server/endpoints/schemas * chore: remove console.logs from test files * ci: add backend tests for AnthropicClient, focusing on new buildMessages logic * refactor(FakeClient): use actual BaseClient sendMessage method for testing * test(BaseClient.test.js): add test for loading chat history test(BaseClient.test.js): add test for sendMessage logic with isEdited flag * fix(buildEndpointOption.js): add support for azureOpenAI in buildFunction object wip(endpoints.js): fetch Azure models from Azure OpenAI API if opts.azure is true * fix(Button.tsx): add data-testid attribute to button component fix(SelectDropDown.tsx): add data-testid attribute to Listbox.Button component fix(messages.spec.ts): add waitForServerStream function to consolidate logic for awaiting the server response feat(messages.spec.ts): add test for stopping and continuing message and improve browser/page context order and closing * refactor(onProgress): speed up time to save initial message for editable routes * chore: disable AI message editing (for now), was accidentally allowed * refactor: ensure continue is only supported for latest message style: improve styling in dark mode and across all hover buttons/icons, including making edit icon for AI invisible (for now) * fix: add test id to generation buttons so they never resolve to 2+ items * chore(package.json): add 'packages/' to the list of ignored directories chore(data-provider/package.json): bump version to 0.1.5 commitae5b7d3d53Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Tue Aug 15 18:27:54 2023 -0400 fix(PluginsClient.js): fix ChatOpenAI Azure Config Issue (#812) * fix(PluginsClient.js): fix issue with creating LLM when using Azure * chore(PluginsClient.js): omit azure logging * refactor(PluginsClient.js): simplify assignment of azure variable The code was simplified by directly assigning the value of `this.azure` to the `azure` variable using object destructuring. This makes the code cleaner and more concise. commitb85f3bf91eAuthor: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Tue Aug 15 18:42:24 2023 +0200 update from lang to localize (#810) commit80aab73bf6Author: Danny Avila <messagedaniel@protonmail.com> Date: Mon Aug 14 19:19:04 2023 -0400 chore: rebuilt package-lock file commitbbe4931a97Author: Danny Avila <messagedaniel@protonmail.com> Date: Mon Aug 14 19:13:24 2023 -0400 refactor(ScreenshotContext): use html-to-image for lighter bundle, faster processing commit74802dd720Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Mon Aug 14 17:51:03 2023 +0200 chore: Translation Fixes, Lint Error Corrections, and Additional Translations (#788) * fix translation and small lint error * changed from localize to useLocalize hook * changed to useLocalize commitb64cc71d88Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 14 10:23:00 2023 -0400 chore(docker-compose.yml): comment out meilisearch ports in docker-compose.yml (#807) commit89f260bc78Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 14 10:12:00 2023 -0400 fix(CodeBlock.tsx): fix copy-to-clipboard functionality. The code has been updated to use the copy function from the copy-to-clipboard library instead of the (#806) avigator.clipboard.writeText method. This should fix the issue with browser incompatibility with navigator SDK and allow users to copy code from the CodeBlock component successfully. commitd00c7354cdAuthor: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 14 09:45:44 2023 -0400 fix: Corrected Registration Validation, Case-Insensitive Variable Handling, Playwright workflow (#805) * feat(auth.js): add validation for registration endpoint using validateRegistration middleware feat(validateRegistration.js): add middleware to validate registration based on ALLOW_REGISTRATION environment variable * fix(config.js): fix registrationEnabled and socialLoginEnabled variables to handle case-insensitive environment variable values * refactor(validateRegistration.js): remove console.log statement * chore(playwright.yml): skip browser download during yarn install chore(playwright.yml): place Playwright binaries to node_modules/@playwright/test chore(playwright.yml): install Playwright dependencies using npx playwright install-deps chore(playwright.yml): install Playwright chromium browser using npx playwright install chromium chore(playwright.yml): install @playwright/test@latest using npm install -D @playwright/test@latest chore(playwright.yml): run Playwright tests using npm run e2e:ci * chore(playwright.yml): change npm install order and update comment The order of the npm install commands in the "Install Playwright Browsers" step has been changed to first install @playwright/test@latest and then install chromium. Additionally, the comment explaining the PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD variable has been updated to mention npm install instead of yarn install. * chore(playwright.yml): remove commented out code for caching and add separate steps for installing Playwright dependencies and browsers commit1aa4b34dc6Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 11 19:02:52 2023 +0200 added the dot (.) username rules (#787) * Create VolumeMuteIcon.tsx * Create VolumeIcon.tsx * Update index.ts * Update SubmitButton.jsx * Update SubmitButton.jsx * Update TextChat.jsx * Update TextChat.jsx * Update SpeechRecognition.tsx * Update SpeechRecognition.tsx * Update TextChat.jsx * Update SpeechRecognition.tsx * Update TextChat.jsx * Update HoverButtons.tsx * Update useServerStream.ts * Update useServerStream.ts * Update HoverButtons.tsx * Update useServerStream.ts * Update useServerStream.ts * Update HoverButtons.tsx * Update VolumeIcon.tsx * Update VolumeMuteIcon.tsx * Update HoverButtons.tsx * Update SpeechSynthesis.tsx * Update HoverButtons.tsx * Update HoverButtons.tsx * Update SpeechSynthesis.tsx * Update SpeechSynthesis.tsx * Update HoverButtons.tsx * Update SpeechSynthesis.tsx * Update package.json * Update SpeechRecognition.tsx * Update SpeechRecognition.tsx * Update SpeechRecognition.tsx * Update SpeechRecognition.tsx * Squashed commit of the following: commit1019529634Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 23:12:14 2023 -0500 Update SpeechRecognition.tsx commit67f111ccd0Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 23:08:48 2023 -0500 Update SpeechRecognition.tsx commit0b35dbe196Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 23:04:50 2023 -0500 Update SpeechRecognition.tsx commit6686126dc0Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 22:49:08 2023 -0500 Update SpeechRecognition.tsx commit5b80ddfba7Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 22:45:02 2023 -0500 Update package.json commit39e84efa81Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 22:35:48 2023 -0500 Update SpeechSynthesis.tsx commit4c6d067cb9Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 22:24:29 2023 -0500 Update HoverButtons.tsx commitc5ce576fb8Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 22:13:20 2023 -0500 Update SpeechSynthesis.tsx commitd95fa19539Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 22:11:38 2023 -0500 Update SpeechSynthesis.tsx commitc794f07678Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 22:03:34 2023 -0500 Update HoverButtons.tsx commit7ae0e7e97cAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 21:59:45 2023 -0500 Update HoverButtons.tsx commite9882dedadAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 21:58:07 2023 -0500 Update SpeechSynthesis.tsx commit95cf300782Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 21:44:49 2023 -0500 Update HoverButtons.tsx commit37c828d7fbAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 21:30:34 2023 -0500 Update VolumeMuteIcon.tsx commit6133531737Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 21:29:54 2023 -0500 Update VolumeIcon.tsx commit4b4afcdd37Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 21:20:14 2023 -0500 Update HoverButtons.tsx commit609d1dfefbAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 20:49:52 2023 -0500 Update useServerStream.ts commit875ce4b77eAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 20:48:26 2023 -0500 Update useServerStream.ts commit8ed04e496bAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 20:37:59 2023 -0500 Update HoverButtons.tsx commit4b30c132dfAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 20:14:01 2023 -0500 Update useServerStream.ts commitc041c329cfAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 20:07:14 2023 -0500 Update useServerStream.ts commit3e36c16817Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 19:36:21 2023 -0500 Update HoverButtons.tsx commitc7eea96759Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 19:28:03 2023 -0500 Update TextChat.jsx commit5542f8e85dAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 19:21:50 2023 -0500 Update SpeechRecognition.tsx commit9a27e56f8bAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 19:16:01 2023 -0500 Update TextChat.jsx commit7f101bd122Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 19:09:51 2023 -0500 Update SpeechRecognition.tsx commitd405454bf5Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 19:03:34 2023 -0500 Update SpeechRecognition.tsx commit6033eb3ed1Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 19:01:06 2023 -0500 Update TextChat.jsx commit9a3e67fcd2Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 18:53:19 2023 -0500 Update TextChat.jsx commit6583877cb3Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 17:53:18 2023 -0500 Update SubmitButton.jsx commit8d5114bfaeAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 17:39:20 2023 -0500 Update SubmitButton.jsx commit29a5b55883Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 17:28:03 2023 -0500 Update index.ts commitb03001d01dAuthor: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 17:25:43 2023 -0500 Create VolumeIcon.tsx commit863af2c959Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 17:21:43 2023 -0500 Create VolumeMuteIcon.tsx commitad3c78f867Merge:ed4b25b228230d93Author: bsu3338 <bsu3338@users.noreply.github.com> Date: Sun Sep 3 16:49:56 2023 -0500 Merge branch 'danny-avila:main' into Speech-September commited4b25b2c1Author: bsu3338 <bsu3338@yahoo.com> Date: Sun Sep 3 16:49:03 2023 -0500 Squashed commit of the following: commit28230d9305Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Sun Sep 3 02:44:26 2023 +0200 feat: delete button confirm (#875) * base for confirm delete * more like OpenAI commit2b54e3f9feAuthor: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Fri Sep 1 14:20:51 2023 -0400 update: install script (#858) commit1cd0fd9d5aAuthor: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Fri Sep 1 08:12:35 2023 -0400 doc: Hugging Face Deployment (#867) * docs: update ToC * docs: update ToC * update huggingface.md * update render.md * update huggingface.md * update mongodb.md * update huggingface.md * update README.md commitaeeb3d3050Author: Mu Yuan <yuanmu.email@gmail.com> Date: Thu Aug 31 07:21:27 2023 +0800 Update Zh.tsx (#862) * Update Zh.tsx Changed the translation of several words to make it more relevant to Chinese usage habits. * Update Zh.tsx Changed the translation of several words to make it more relevant to Chinese usage habits commit80e2e2675bAuthor: Raí <140329135+itzraiss@users.noreply.github.com> Date: Mon Aug 28 18:05:46 2023 -0300 Translation of 'com_ui_pay_per_call:' to Spanish and Portuguese that were missing. (#857) * Update Br.tsx * Update Es.tsx * Update Br.tsx * Update Es.tsx commit3574d0b823Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 28 14:49:26 2023 -0400 docs: make_your_own.md formatting fix for mkdocs (#855) commitd672ac690dAuthor: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 28 14:24:10 2023 -0400 Release v0.5.8 (#854) * chore: add 'api' image to tag release workflow * docs: update DO deployment docs to include instruction about latest stable release, as well as security best practices * Release v0.5.8 * docs: Update digitalocean.md with firewall section images * docs: make_your_own.md formatting fix for mkdocs commitd3e7627046Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 28 12:03:08 2023 -0400 refactor(plugins): Improve OpenAPI handling, Show Multiple Plugins, & Other Improvements (#845) * feat(PluginsClient.js): add conversationId to options object in the constructor feat(PluginsClient.js): add support for Code Interpreter plugin feat(PluginsClient.js): add support for Code Interpreter plugin in the availableTools manifest feat(CodeInterpreter.js): add CodeInterpreterTools module feat(CodeInterpreter.js): add RunCommand class feat(CodeInterpreter.js): add ReadFile class feat(CodeInterpreter.js): add WriteFile class feat(handleTools.js): add support for loading Code Interpreter plugin * chore(api): update langchain dependency to version 0.0.123 * fix(CodeInterpreter.js): add support for extracting environment from code fix(WriteFile.js): add support for extracting environment from data fix(extractionChain.js): add utility functions for creating extraction chain from Zod schema fix(handleTools.js): refactor getOpenAIKey function to handle user-provided API key fix(handleTools.js): pass model and openAIApiKey to CodeInterpreter constructor * fix(tools): rename CodeInterpreterTools to E2BTools fix(tools): rename code_interpreter pluginKey to e2b_code_interpreter * chore(PluginsClient.js): comment out unused import and function findMessageContent feat(PluginsClient.js): add support for CodeSherpa plugin feat(PluginsClient.js): add CodeSherpaTools to available tools feat(PluginsClient.js): update manifest.json to include CodeSherpa plugin feat(CodeSherpaTools.js): create RunCode and RunCommand classes for CodeSherpa plugin feat(E2BTools.js): Add E2BTools module for extracting environment from code and running commands, reading and writing files fix(codesherpa.js): Remove codesherpa module as it is no longer needed feat(handleTools.js): add support for CodeSherpaTools in loadTools function feat(loadToolSuite.js): create loadToolSuite utility function to load a suite of tools * feat(PluginsClient.js): add support for CodeSherpa v2 plugin feat(PluginsClient.js): add CodeSherpa v1 plugin to available tools feat(PluginsClient.js): add CodeSherpa v2 plugin to available tools feat(PluginsClient.js): update manifest.json for CodeSherpa v1 plugin feat(PluginsClient.js): update manifest.json for CodeSherpa v2 plugin feat(CodeSherpa.js): implement CodeSherpa plugin for interactive code and shell command execution feat(CodeSherpaTools.js): implement RunCode and RunCommand plugins for CodeSherpa v1 feat(CodeSherpaTools.js): update RunCode and RunCommand plugins for CodeSherpa v2 fix(handleTools.js): add CodeSherpa import statement fix(handleTools.js): change pluginKey from 'codesherpa' to 'codesherpa_tools' fix(handleTools.js): remove model and openAIApiKey from options object in e2b_code_interpreter tool fix(handleTools.js): remove openAIApiKey from options object in codesherpa_tools tool fix(loadToolSuite.js): remove model and openAIApiKey parameters from loadToolSuite function * feat(initializeFunctionsAgent.js): add prefix to agentArgs in initializeFunctionsAgent function The prefix is added to the agentArgs in the initializeFunctionsAgent function. This prefix is used to provide instructions to the agent when it receives any instructions from a webpage, plugin, or other tool. The agent will notify the user immediately and ask them if they wish to carry out or ignore the instructions. * feat(PluginsClient.js): add ChatTool to the list of tools if it meets the conditions feat(tools/index.js): import and export ChatTool feat(ChatTool.js): create ChatTool class with necessary properties and methods * fix(initializeFunctionsAgent.js): update PREFIX message to include sharing all output from the tool fix(E2BTools.js): update descriptions for RunCommand, ReadFile, and WriteFile plugins to provide more clarity and context * chore: rebuild package-lock after rebase * chore: remove deleted file from rebase * wip: refactor plugin message handling to mirror chat.openai.com, handle incoming stream for plugin use * wip: new plugin handling * wip: show multiple plugins handling * feat(plugins): save new plugins array * chore: bump langchain * feat(experimental): support streaming in between plugins * refactor(PluginsClient): factor out helper methods to avoid bloating the class, refactor(gptPlugins): use agent action for mapping the name of action * fix(handleTools): fix tests by adding condition to return original toolFunctions map * refactor(MessageContent): Allow the last index to be last in case it has text (may change with streaming) * feat(Plugins): add handleParsingErrors, useful when LLM does not invoke function params * chore: edit out experimental codesherpa integration * refactor(OpenAPIPlugin): rework tool to be 'function-first', as the spec functions are explicitly passed to agent model * refactor(initializeFunctionsAgent): improve error handling and system message * refactor(CodeSherpa, Wolfram): optimize token usage by delegating bulk of instructions to system message * style(Plugins): match official style with input/outputs * chore: remove unnecessary console logs used for testing * fix(abortMiddleware): render markdown when message is aborted * feat(plugins): add BrowserOp * refactor(OpenAPIPlugin): improve prompt handling * fix(useGenerations): hide edit button when message is submitting/streaming * refactor(loadSpecs): optimize OpenAPI spec loading by only loading requested specs instead of all of them * fix(loadSpecs): will retain original behavior when no tools are passed to the function * fix(MessageContent): ensure cursor only shows up for last message and last display index fix(Message): show legacy plugin and pass isLast to Content * chore: remove console.logs * docs: update docs based on breaking changes and new features refactor(structured/SD): use description_for_model for detailed prompting * docs(azure): make plugins section more clear * refactor(structured/SD): change default payload to SD-WebUI to prefer realism and config for SDXL * refactor(structured/SD): further improve system message prompt * docs: update breaking changes after rebase * refactor(MessageContent): factor out EditMessage, types, Container to separate files, rename Content -> Markdown * fix(CodeInterpreter): linting errors * chore: reduce browser console logs from message streams * chore: re-enable debug logs for plugins/langchain to help with user troubleshooting * chore(manifest.json): add [Experimental] tag to CodeInterpreter plugins, which are not intended as the end-all be-all implementation of this feature for Librechat commit66b8580487Author: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Mon Aug 28 09:18:25 2023 -0400 docs: third-party tools (#848) * docs: third-party tools * docs: third-party tools * Update third-party.md * Update third-party.md --------- Co-authored-by: Danny Avila <110412045+danny-avila@users.noreply.github.com> commit9791a78161Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Mon Aug 28 15:14:05 2023 +0200 adjust the animation (#843) commit3797ec6082Author: Ronith <87087292+ronith256@users.noreply.github.com> Date: Mon Aug 28 18:43:50 2023 +0530 feat: Add Code Interpreter Plugin (#837) * feat: Add Code Interpreter Plugin Adds a Simple Code Interpreter Plugin. ## Features: - Runs code using local Python Environment ## Issues - Code execution is not sandboxed. * Add Docker Sandbox for Python Server commite2397076a2Author: Alex Zhang <ztc2011@gmail.com> Date: Mon Aug 28 00:55:34 2023 +0800 🌐: Chinese Translation (#846) commit50c15c704fAuthor: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Sat Aug 26 19:36:59 2023 -0400 Language translation: Polish (#840) * Language translation: Polish * Language translation: Polish * Revert changes in language-contributions.md commit29d3640546Author: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Sat Aug 26 19:36:25 2023 -0400 docs: updates (#841) commit39c626aa8eAuthor: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 25 09:29:19 2023 -0400 fix: isEdited edge case where latest Message is not saved due to aborting too quickly commitae5c06f381Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 25 09:13:50 2023 -0400 fix(chatGPTBrowser): render markdown formatting by setting isCreatedByUser, fix(useMessageHandler): avoid double appearance of cursor by setting latest message at initial response creation time commit9ef1686e18Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Thu Aug 24 20:24:47 2023 -0400 Update mkdocs.yml commit5bbe411569Author: Flynn <dev@flynnbuckingham.com> Date: Thu Aug 24 20:20:37 2023 -0400 Add podman installation instructions. Update dockerfile to stub env (#819) * Added podman container installation docs. Updated dockerfile to stub env file if not present in source * Fix typos commit887fec99caAuthor: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 25 02:11:27 2023 +0200 🌐: Russian Translation (#830) commit007d51ede1Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 25 02:10:48 2023 +0200 feat: facebook login (#820) * Facebook strategy * Update user_auth_system.md * Update user_auth_system.md commita569020312Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Thu Aug 24 21:59:11 2023 +0200 Fix Meilisearch error and refactor of the server index.js (#832) * fix meilisearch error at startup * limit the nesting * disable useless console log * fix(indexSync.js): removed redundant searchEnabled * refactor(index.js): moved configureSocialLogins to a new file * refactor(socialLogins.js): removed unnecessary conditional commit37347d4683Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Wed Aug 23 16:14:17 2023 -0400 fix(registration): Make Username optional (#831) * fix(User.js): update validation schema for username field, allow empty string as a valid value fix(validators.js): update validation schema for username field, allow empty string as a valid value fix(Registration.tsx, validators.js): update validation rules for name and username fields, change minimum length to 2 and maximum length to 80, assure they match and allow empty string as a valid value fix(Eng.tsx): update localization string for com_auth_username, indicate that it is optional * fix(User.js): update regex pattern for username validation to allow special characters @#$%&*() fix(validators.js): update regex pattern for username validation to allow special characters @#$%&*() * fix(Registration.spec.tsx): fix validation error message for username length requirement commitd38e463d34Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Wed Aug 23 13:44:40 2023 -0400 fix(bingAI): markdown and error formatting for final stream response (#829) * fix(bingAI): markdown formatting for final stream response due to new strict payload validation on the frontend * fix: add missing prop to bing Error response commit7dc27b10f1Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Tue Aug 22 18:44:59 2023 -0400 feat: Edit AI Messages, Edit Messages in Place (#825) * refactor: replace lodash import with specific function import fix(api): esm imports to cjs * refactor(Messages.tsx): convert to TS, out-source scrollToDiv logic to a custom hook fix(ScreenshotContext.tsx): change Ref to RefObject in ScreenshotContextType feat(useScrollToRef.ts): add useScrollToRef hook for scrolling to a ref with throttle fix(Chat.tsx): update import path for Messages component fix(Search.tsx): update import path for Messages component * chore(types.ts): add TAskProps and TOptions types refactor(useMessageHandler.ts): use TAskFunction type for ask function signature * refactor(Message/Content): convert to TS, move Plugin component to Content dir * feat(MessageContent.tsx): add MessageContent component for displaying and editing message content feat(index.ts): export MessageContent component from Messages/Content directory * wip(Message.jsx): conversion and use of new component in progress * refactor: convert Message.jsx to TS and fix typing/imports based on changes * refactor: add typed props and refactor MultiMessage to TS, fix typing issues resulting from the conversion * edit message in progress * feat: complete edit AI message logic, refactor continue logic * feat(middleware): add validateMessageReq middleware feat(routes): add validation for message requests using validateMessageReq middleware feat(routes): add create, read, update, and delete routes for messages * feat: complete frontend logic for editing messages in place feat(messages.js): update route for updating a specific message - Change the route for updating a message to include the messageId in the URL - Update the request handler to use the messageId from the request parameters and the text from the request body - Call the updateMessage function with the updated parameters feat(MessageContent.tsx): add functionality to update a message - Import the useUpdateMessageMutation hook from the data provider - Destructure the conversationId, parentMessageId, and messageId from the message object - Create a mutation function using the useUpdateMessageMutation hook - Implement the updateMessage function to call the mutation function with the updated message parameters - Update the messages state to reflect the updated message text feat(api-endpoints.ts): update messages endpoint to include messageId - Update the messages endpoint to include the messageId as an optional parameter feat(data-service.ts): add updateMessage function - Implement the updateMessage function to make a PUT request to * fix(messages.js): make updateMessage function asynchronous and await its execution * style(EditIcon): make icon active for AI message * feat(gptPlugins/anthropic): add edit support * fix(validateMessageReq.js): handle case when conversationId is 'new' and return empty array feat(Message.tsx): pass message prop to SiblingSwitch component refactor(SiblingSwitch.tsx): convert to TS * fix(useMessageHandler.ts): remove message from currentMessages if isContinued is true feat(useMessageHandler.ts): add support for submission messages in setMessages fix(useServerStream.ts): remove unnecessary conditional in setMessages fix(useServerStream.ts): remove isContinued variable from submission * fix(continue): switch to continued message generation when continuing an earlier branch in conversation * fix(abortMiddleware.js): fix condition to check partialText length chore(abortMiddleware.js): add error logging when abortMessage fails * refactor(MessageHeader.tsx): convert to TS fix(Plugin.tsx): add default value for className prop in Plugin component * refactor(MultiMessage.tsx): remove commented out code docs(MultiMessage.tsx): update comment to clarify when siblingIdx is reset * fix(GenerationButtons): optimistic state for continue button * fix(MessageContent.tsx): add data-testid attribute to message text editor fix(messages.spec.ts): update waitForServerStream function to include edit endpoint check feat(messages.spec.ts): add test case for editing messages * fix(HoverButtons & Message & useGenerations): Refactor edit functionality and related conditions - Update enterEdit function signature and prop - Create and utilize hideEditButton variable - Enhance conditions for edit button visibility and active state - Update button event handlers - Introduce isEditableEndpoint in useGenerations and refine continueSupported condition. * fix(useGenerations.ts): fix condition for hideEditButton to include error and searchResult chore(data-provider): bump version to 0.1.6 fix(types.ts): add status property to TError type * chore: bump @dqbd/tiktoken to 1.0.7 * fix(abortMiddleware.js): add required isCreatedByUser property to the error response object * refactor(Message.tsx): remove unnecessary props from SiblingSwitch component, as setLatestMessage is firing on every switch already refactor(SiblingSwitch.tsx): remove unused imports and code * chore(BaseClient.js): move console.debug statements back inside if block commitdb77163f5dAuthor: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Tue Aug 22 14:15:14 2023 +0200 docs: update chimeragpt (#826) * Update free_ai_apis.md * Update free_ai_apis.md commit4a4e803df3Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Mon Aug 21 20:15:18 2023 +0200 style(Dialog): Improved Close Button ("X") position (#824) commit909b00c752Author: Daniel Avila <messagedaniel@protonmail.com> Date: Sun Aug 20 21:04:36 2023 -0400 fix(HoverButtons): light/dark styling to match official site commit61dcb4d307Author: Naosuke Yokoe <ankerasoy@gmail.com> Date: Sat Aug 19 20:11:31 2023 +0900 feat: Azure Cognitive Search Plugin (#815) * feat(AzureCognitiveSearchPlugin) * feat(tools/AzureCognitiveSearch.js): Add a new plugin (not structured version) * feat(tools/structured/AzureCognitiveSearch.js): Add a new plugin (structured version) * feat(tools/manifest.json, tools/index.js, tools/util/handleTools.js): Add configurations for the plugin * feat(api/package.json, package-lock.json): Installed a new package for the plugin (@azure/search-documents) * feat(.env.example): Add new environment variables for the plugin Here is the link to the corresponding discussion page: https://github.com/danny-avila/LibreChat/discussions/567 * docs(AzureCognitiveSearchPlugin) * docs(features/plugins/azure_cognitive_search.md): Add a new document for the plugin * (fix:.env.example) * reverted extra whitespaces removed by the editor * docs(mkdocs.yml) * Add the Azure Cognitive Search Plugin's documentation item to mkdocs.yml. commit3c7f67fa76Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 18 12:40:33 2023 -0400 fix(abortMiddleware): handle early abort error where userMessage.conversationId is undefined. In this case, the userId will be used as the abortKey commitc74c68a135Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 18 12:10:30 2023 -0400 refactor(MessageHandler -> useServerStream): convert all relating files to TS and correct typings based on this change: properly refactor MessageHandler to a custom hook, where it's passed a submission object to instantiate the stream. This is the bare minimum groundwork for potentially having multiple streams running, which would be a big project to modularize a lot of the global state into maps/multiple streams, particular useful for having multiple views in place commit8b4d3c2c21Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 18 12:04:29 2023 -0400 refactor(routes): convert to TS commitd612cfcb45Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 18 12:02:39 2023 -0400 chore(Auth): reorder exports in Auth component fix(PluginAuthForm): handle case when pluginKey is null or undefined fix(PluginStoreDialog): handle case when getAvailablePluginFromKey is null or undefined fix(AuthContext): make authConfig optional in AuthContextProvider feat(hooks): add useServerStream hook fix(conversation): setSubmission to null instead of empty object fix(preset): specify type for presets atom fix(search): specify type for isSearchEnabled atom fix(submission): specify type for submission atom commitc40b95f424Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 18 16:11:00 2023 +0200 feat: Disable Registration with social login (#813) * Google, Github and Discord * update .env.example with ALLOW_SOCIAL_REGISTRATION * fix some conflict * refactor strategy * Update user_auth_system.md * Update user_auth_system.md commit46ed5aaccdAuthor: Patrick <psarnowski@gmail.com> Date: Fri Aug 18 09:38:24 2023 -0400 Show the response scores from Bing. (#814) commit1dacfa49f0Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Thu Aug 17 20:32:31 2023 +0200 update profile picture (#792) commitafd43afb60Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Thu Aug 17 12:50:05 2023 -0400 feat(GPT/Anthropic): Continue Regenerating & Generation Buttons (#808) * feat(useMessageHandler.js/ts): Refactor and add features to handle user messages, support multiple endpoints/models, generate placeholder responses, regeneration, and stopGeneration function fix(conversation.ts, buildTree.ts): Import TMessage type, handle null parentMessageId feat(schemas.ts): Update and add schemas for various AI services, add default values, optional fields, and endpoint-to-schema mapping, create parseConvo function chore(useMessageHandler.js, schemas.ts): Remove unused imports, variables, and chatGPT enum * wip: add generation buttons * refactor(cleanupPreset.ts): simplify cleanupPreset function refactor(getDefaultConversation.js): remove unused code and simplify getDefaultConversation function feat(utils): add getDefaultConversation function This commit adds a new utility function called `getDefaultConversation` to the `client/src/utils/getDefaultConversation.ts` file. This function is responsible for generating a default conversation object based on the provided parameters. The `getDefaultConversation` function takes in an object with the following properties: - `conversation`: The conversation object to be used as a base. - `endpointsConfig`: The configuration object containing information about the available endpoints. - `preset`: An optional preset object that can be used to override the default behavior. The function first tries to determine the target endpoint based on the preset object. If a valid endpoint is found, it is used as the target endpoint. If not, the function tries to retrieve the last conversation setup from the local storage and uses its endpoint if it is valid. If neither the preset nor the local storage contains a valid endpoint, the function falls back to a default endpoint. Once the target endpoint is determined, * fix(utils): remove console.error statement in buildDefaultConversation function fix(schemas): add default values for catch blocks in openAISchema, googleSchema, bingAISchema, anthropicSchema, chatGPTBrowserSchema, and gptPluginsSchema * fix: endpoint not changing on change of preset from other endpoint, wip: refactor * refactor: preset items to TSX * refactor: convert resetConvo to TS * refactor(getDefaultConversation.ts): move defaultEndpoints array to the top of the file for better readability refactor(getDefaultConversation.ts): extract getDefaultEndpoint function for better code organization and reusability * feat(svg): add ContinueIcon component feat(svg): add RegenerateIcon component feat(svg): add ContinueIcon and RegenerateIcon components to index.ts * feat(Button.tsx): add onClick and className props to Button component feat(GenerationButtons.tsx): add logic to display Regenerate or StopGenerating button based on isSubmitting and messages feat(Regenerate.tsx): create Regenerate component with RegenerateIcon and handleRegenerate function feat(StopGenerating.tsx): create StopGenerating component with StopGeneratingIcon and handleStopGenerating function * fix(TextChat.jsx): reorder imports and variables for better readability fix(TextChat.jsx): fix typo in condition for isNotAppendable variable fix(TextChat.jsx): remove unused handleStopGenerating function fix(ContinueIcon.tsx): remove unnecessary closing tags for polygon elements fix(useMessageHandler.ts): add missing type annotations for handleStopGenerating and handleRegenerate functions fix(useMessageHandler.ts): remove unused variables in return statement * fix(getDefaultConversation.ts): refactor code to use getLocalStorageItems function feat(getLocalStorageItems.ts): add utility function to retrieve items from local storage * fix(OpenAIClient.js): add support for streaming result in sendCompletion method feat(OpenAIClient.js): add finish_reason metadata to opts in sendCompletion method feat(Message.js): add finish_reason field to Message model feat(messageSchema.js): add finish_reason field to messageSchema feat(openAI.js): parse chatGptLabel and promptPrefix from req.body and pass rest of the modelOptions to endpointOption feat(openAI.js): add addMetadata function to store metadata in ask function feat(openAI.js): add metadata to response if available feat(schemas.ts): add finish_reason field to tMessageSchema * feat(types.ts): add TOnClick and TGenButtonProps types for button components feat(Continue.tsx): create Continue component for generating button feat(GenerationButtons.tsx): update GenerationButtons component to use Continue component feat(Regenerate.tsx): create Regenerate component for regenerating button feat(Stop.tsx): create Stop component for stop generating button * feat(MessageHandler.jsx): add MessageHandler component to handle messages and conversations fix(Root.jsx): fix import paths for Nav and MessageHandler components * feat(useMessageHandler.ts): add support for generation parameter in ask function feat(useMessageHandler.ts): add support for isEdited parameter in ask function feat(useMessageHandler.ts): add support for continueGeneration function fix(createPayload.ts): replace endpoint URL when isEdited parameter is true * chore(client): set skipLibCheck to true in tsconfig.json * fix(useMessageHandler.ts): remove unused clientId variable fix(schemas.ts): make clientId field in tMessageSchema nullable and optional * wip: edit route for continue generation * refactor(api): move handlers to root of routes dir * fix(useMessageHandler.ts): initialize currentMessages to an empty array if messages is null fix(useMessageHandler.ts): update initialResponse text to use responseText variable fix(useMessageHandler.ts): update setMessages logic for isRegenerate case fix(MessageHandler.jsx): update setMessages logic for cancelHandler, createdHandler, and finalHandler * fix(schemas.ts): make createdAt and updatedAt fields optional and set default values using new Date().toISOString() fix(schemas.ts): change type annotation of TMessage from infer to input * refactor(useMessageHandler.ts): rename AskProps type to TAskProps refactor(useMessageHandler.ts): remove generation property from ask function arguments refactor(useMessageHandler.ts): use nullish coalescing operator (??) instead of logical OR (||) refactor(useMessageHandler.ts): pass the responseMessageId to message prop of submission * fix(BaseClient.js): use nullish coalescing operator (??) instead of logical OR (||) for default values * fix(BaseClient.js): fix responseMessageId assignment in handleStartMethods method feat(BaseClient.js): add support for isEdited flag in sendMessage method feat(BaseClient.js): add generation to responseMessage text in sendMessage method * fix(openAI.js): remove unused imports and commented out code feat(openAI.js): add support for generation parameter in request body fix(openAI.js): remove console.log statement fix(openAI.js): remove unused variables and parameters fix(openAI.js): update response text in case of error fix(openAI.js): handle error and abort message in case of error fix(handlers.js): add generation parameter to createOnProgress function fix(useMessageHandler.ts): update responseText variable to use generation parameter * refactor(api/middleware): move inside server dir * refactor: add endpoint specific, modular functions to build options and initialize clients, create server/utils, move middleware, separate utils into api general utils and server specific utils * fix(abortMiddleware.js): import getConvo and getConvoTitle functions from models feat(abortMiddleware.js): add abortAsk function to abortController to handle aborting of requests fix(openAI.js): import buildOptions and initializeClient functions from endpoints/openAI refactor(openAI.js): use getAbortData function to get data for abortAsk function * refactor: move endpoint specific logic to an endpoints dir * refactor(PluginService.js): fix import path for encrypt and decrypt functions in PluginService.js * feat(openAI): add new endpoint for adding a title to a conversation - Added a new file `addTitle.js` in the `api/server/routes/endpoints/openAI` directory. - The `addTitle.js` file exports a function `addTitle` that takes in request parameters and performs the following actions: - If the `parentMessageId` is `'00000000-0000-0000-0000-000000000000'` and `newConvo` is true, it proceeds with the following steps: - Calls the `titleConvo` function from the `titleConvo` module, passing in the necessary parameters. - Calls the `saveConvo` function from the `saveConvo` module, passing in the user ID and conversation details. - Updated the `index.js` file in the `api/server/routes/endpoints/openAI` directory to export the `addTitle` function. - This change adds * fix(abortMiddleware.js): remove console.log statement refactor(gptPlugins.js): update imports and function parameters feat(gptPlugins.js): add support for abortController and getAbortData refactor(openAI.js): update imports and function parameters feat(openAI.js): add support for abortController and getAbortData fix(openAI.js): refactor code to use modularized functions and middleware fix(buildOptions.js): refactor code to use destructuring and update variable names * refactor(askChatGPTBrowser.js, bingAI.js, google.js): remove duplicate code for setting response headers feat(askChatGPTBrowser.js, bingAI.js, google.js): add setHeaders middleware to set response headers * feat(middleware): validateEndpoint, refactor buildOption to only be concerned of endpointOption * fix(abortMiddleware.js): add 'finish_reason' property with value 'incomplete' to responseMessage object fix(abortMessage.js): remove console.log statement for aborted message fix(handlers.js): modify tokens assignment to handle empty generation string and trailing space * fix(BaseClient.js): import addSpaceIfNeeded function from server/utils fix(BaseClient.js): add space before generation in text property fix(index.js): remove getCitations and citeText exports feat(buildEndpointOption.js): add buildEndpointOption middleware fix(index.js): import buildEndpointOption middleware fix(anthropic.js): remove buildOptions function and use endpointOption from req.body fix(gptPlugins.js): remove buildOptions function and use endpointOption from req.body fix(openAI.js): remove buildOptions function and use endpointOption from req.body feat(utils): add citations.js and handleText.js modules fix(utils): fix import statements in index.js module * refactor(gptPlugins.js): use getResponseSender function from librechat-data-provider * feat(gptPlugins): complete 'continue generating' * wip: anthropic continue regen * feat(middleware): add validateRegistration middleware A new middleware function called `validateRegistration` has been added to the list of exported middleware functions in `index.js`. This middleware is responsible for validating registration data before allowing the registration process to proceed. * feat(Anthropic): complete continue regen * chore: add librechat-data-provider to api/package.json * fix(ci): backend-review will mock meilisearch, also installs data-provider as now needed * chore(ci): remove unneeded SEARCH env var * style(GenerationButtons): make text shorter for sake of space economy, even though this diverges from chat.openai.com * style(GenerationButtons/ScrollToBottom): adjust visibility/position based on screen size * chore(client): 'Editting' typo * feat(GenerationButtons.tsx): add support for endpoint prop in GenerationButtons component feat(OptionsBar.tsx): pass endpoint prop to GenerationButtons component feat(useGenerations.ts): create useGenerations hook to handle generation logic fix(schemas.ts): add searchResult field to tMessageSchema * refactor(HoverButtons): convert to TSX and utilize new useGenerations hook * fix(abortMiddleware): handle error with res headers set, or abortController not found, to ensure proper API error is sent to the client, chore(BaseClient): remove console log for onStart message meant for debugging * refactor(api): remove librechat-data-provider dep for now as it complicates deployed docker build stage, re-use code in CJS, located in server/endpoints/schemas * chore: remove console.logs from test files * ci: add backend tests for AnthropicClient, focusing on new buildMessages logic * refactor(FakeClient): use actual BaseClient sendMessage method for testing * test(BaseClient.test.js): add test for loading chat history test(BaseClient.test.js): add test for sendMessage logic with isEdited flag * fix(buildEndpointOption.js): add support for azureOpenAI in buildFunction object wip(endpoints.js): fetch Azure models from Azure OpenAI API if opts.azure is true * fix(Button.tsx): add data-testid attribute to button component fix(SelectDropDown.tsx): add data-testid attribute to Listbox.Button component fix(messages.spec.ts): add waitForServerStream function to consolidate logic for awaiting the server response feat(messages.spec.ts): add test for stopping and continuing message and improve browser/page context order and closing * refactor(onProgress): speed up time to save initial message for editable routes * chore: disable AI message editing (for now), was accidentally allowed * refactor: ensure continue is only supported for latest message style: improve styling in dark mode and across all hover buttons/icons, including making edit icon for AI invisible (for now) * fix: add test id to generation buttons so they never resolve to 2+ items * chore(package.json): add 'packages/' to the list of ignored directories chore(data-provider/package.json): bump version to 0.1.5 commitae5b7d3d53Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Tue Aug 15 18:27:54 2023 -0400 fix(PluginsClient.js): fix ChatOpenAI Azure Config Issue (#812) * fix(PluginsClient.js): fix issue with creating LLM when using Azure * chore(PluginsClient.js): omit azure logging * refactor(PluginsClient.js): simplify assignment of azure variable The code was simplified by directly assigning the value of `this.azure` to the `azure` variable using object destructuring. This makes the code cleaner and more concise. commitb85f3bf91eAuthor: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Tue Aug 15 18:42:24 2023 +0200 update from lang to localize (#810) commit80aab73bf6Author: Danny Avila <messagedaniel@protonmail.com> Date: Mon Aug 14 19:19:04 2023 -0400 chore: rebuilt package-lock file commitbbe4931a97Author: Danny Avila <messagedaniel@protonmail.com> Date: Mon Aug 14 19:13:24 2023 -0400 refactor(ScreenshotContext): use html-to-image for lighter bundle, faster processing commit74802dd720Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Mon Aug 14 17:51:03 2023 +0200 chore: Translation Fixes, Lint Error Corrections, and Additional Translations (#788) * fix translation and small lint error * changed from localize to useLocalize hook * changed to useLocalize commitb64cc71d88Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 14 10:23:00 2023 -0400 chore(docker-compose.yml): comment out meilisearch ports in docker-compose.yml (#807) commit89f260bc78Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 14 10:12:00 2023 -0400 fix(CodeBlock.tsx): fix copy-to-clipboard functionality. The code has been updated to use the copy function from the copy-to-clipboard library instead of the (#806) avigator.clipboard.writeText method. This should fix the issue with browser incompatibility with navigator SDK and allow users to copy code from the CodeBlock component successfully. commitd00c7354cdAuthor: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 14 09:45:44 2023 -0400 fix: Corrected Registration Validation, Case-Insensitive Variable Handling, Playwright workflow (#805) * feat(auth.js): add validation for registration endpoint using validateRegistration middleware feat(validateRegistration.js): add middleware to validate registration based on ALLOW_REGISTRATION environment variable * fix(config.js): fix registrationEnabled and socialLoginEnabled variables to handle case-insensitive environment variable values * refactor(validateRegistration.js): remove console.log statement * chore(playwright.yml): skip browser download during yarn install chore(playwright.yml): place Playwright binaries to node_modules/@playwright/test chore(playwright.yml): install Playwright dependencies using npx playwright install-deps chore(playwright.yml): install Playwright chromium browser using npx playwright install chromium chore(playwright.yml): install @playwright/test@latest using npm install -D @playwright/test@latest chore(playwright.yml): run Playwright tests using npm run e2e:ci * chore(playwright.yml): change npm install order and update comment The order of the npm install commands in the "Install Playwright Browsers" step has been changed to first install @playwright/test@latest and then install chromium. Additionally, the comment explaining the PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD variable has been updated to mention npm install instead of yarn install. * chore(playwright.yml): remove commented out code for caching and add separate steps for installing Playwright dependencies and browsers commit1aa4b34dc6Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 11 19:02:52 2023 +0200 added the dot (.) username rules (#787) commit28230d9305Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Sun Sep 3 02:44:26 2023 +0200 feat: delete button confirm (#875) * base for confirm delete * more like OpenAI commit2b54e3f9feAuthor: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Fri Sep 1 14:20:51 2023 -0400 update: install script (#858) commit1cd0fd9d5aAuthor: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Fri Sep 1 08:12:35 2023 -0400 doc: Hugging Face Deployment (#867) * docs: update ToC * docs: update ToC * update huggingface.md * update render.md * update huggingface.md * update mongodb.md * update huggingface.md * update README.md commitaeeb3d3050Author: Mu Yuan <yuanmu.email@gmail.com> Date: Thu Aug 31 07:21:27 2023 +0800 Update Zh.tsx (#862) * Update Zh.tsx Changed the translation of several words to make it more relevant to Chinese usage habits. * Update Zh.tsx Changed the translation of several words to make it more relevant to Chinese usage habits commit80e2e2675bAuthor: Raí <140329135+itzraiss@users.noreply.github.com> Date: Mon Aug 28 18:05:46 2023 -0300 Translation of 'com_ui_pay_per_call:' to Spanish and Portuguese that were missing. (#857) * Update Br.tsx * Update Es.tsx * Update Br.tsx * Update Es.tsx commit3574d0b823Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 28 14:49:26 2023 -0400 docs: make_your_own.md formatting fix for mkdocs (#855) commitd672ac690dAuthor: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 28 14:24:10 2023 -0400 Release v0.5.8 (#854) * chore: add 'api' image to tag release workflow * docs: update DO deployment docs to include instruction about latest stable release, as well as security best practices * Release v0.5.8 * docs: Update digitalocean.md with firewall section images * docs: make_your_own.md formatting fix for mkdocs commitd3e7627046Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Mon Aug 28 12:03:08 2023 -0400 refactor(plugins): Improve OpenAPI handling, Show Multiple Plugins, & Other Improvements (#845) * feat(PluginsClient.js): add conversationId to options object in the constructor feat(PluginsClient.js): add support for Code Interpreter plugin feat(PluginsClient.js): add support for Code Interpreter plugin in the availableTools manifest feat(CodeInterpreter.js): add CodeInterpreterTools module feat(CodeInterpreter.js): add RunCommand class feat(CodeInterpreter.js): add ReadFile class feat(CodeInterpreter.js): add WriteFile class feat(handleTools.js): add support for loading Code Interpreter plugin * chore(api): update langchain dependency to version 0.0.123 * fix(CodeInterpreter.js): add support for extracting environment from code fix(WriteFile.js): add support for extracting environment from data fix(extractionChain.js): add utility functions for creating extraction chain from Zod schema fix(handleTools.js): refactor getOpenAIKey function to handle user-provided API key fix(handleTools.js): pass model and openAIApiKey to CodeInterpreter constructor * fix(tools): rename CodeInterpreterTools to E2BTools fix(tools): rename code_interpreter pluginKey to e2b_code_interpreter * chore(PluginsClient.js): comment out unused import and function findMessageContent feat(PluginsClient.js): add support for CodeSherpa plugin feat(PluginsClient.js): add CodeSherpaTools to available tools feat(PluginsClient.js): update manifest.json to include CodeSherpa plugin feat(CodeSherpaTools.js): create RunCode and RunCommand classes for CodeSherpa plugin feat(E2BTools.js): Add E2BTools module for extracting environment from code and running commands, reading and writing files fix(codesherpa.js): Remove codesherpa module as it is no longer needed feat(handleTools.js): add support for CodeSherpaTools in loadTools function feat(loadToolSuite.js): create loadToolSuite utility function to load a suite of tools * feat(PluginsClient.js): add support for CodeSherpa v2 plugin feat(PluginsClient.js): add CodeSherpa v1 plugin to available tools feat(PluginsClient.js): add CodeSherpa v2 plugin to available tools feat(PluginsClient.js): update manifest.json for CodeSherpa v1 plugin feat(PluginsClient.js): update manifest.json for CodeSherpa v2 plugin feat(CodeSherpa.js): implement CodeSherpa plugin for interactive code and shell command execution feat(CodeSherpaTools.js): implement RunCode and RunCommand plugins for CodeSherpa v1 feat(CodeSherpaTools.js): update RunCode and RunCommand plugins for CodeSherpa v2 fix(handleTools.js): add CodeSherpa import statement fix(handleTools.js): change pluginKey from 'codesherpa' to 'codesherpa_tools' fix(handleTools.js): remove model and openAIApiKey from options object in e2b_code_interpreter tool fix(handleTools.js): remove openAIApiKey from options object in codesherpa_tools tool fix(loadToolSuite.js): remove model and openAIApiKey parameters from loadToolSuite function * feat(initializeFunctionsAgent.js): add prefix to agentArgs in initializeFunctionsAgent function The prefix is added to the agentArgs in the initializeFunctionsAgent function. This prefix is used to provide instructions to the agent when it receives any instructions from a webpage, plugin, or other tool. The agent will notify the user immediately and ask them if they wish to carry out or ignore the instructions. * feat(PluginsClient.js): add ChatTool to the list of tools if it meets the conditions feat(tools/index.js): import and export ChatTool feat(ChatTool.js): create ChatTool class with necessary properties and methods * fix(initializeFunctionsAgent.js): update PREFIX message to include sharing all output from the tool fix(E2BTools.js): update descriptions for RunCommand, ReadFile, and WriteFile plugins to provide more clarity and context * chore: rebuild package-lock after rebase * chore: remove deleted file from rebase * wip: refactor plugin message handling to mirror chat.openai.com, handle incoming stream for plugin use * wip: new plugin handling * wip: show multiple plugins handling * feat(plugins): save new plugins array * chore: bump langchain * feat(experimental): support streaming in between plugins * refactor(PluginsClient): factor out helper methods to avoid bloating the class, refactor(gptPlugins): use agent action for mapping the name of action * fix(handleTools): fix tests by adding condition to return original toolFunctions map * refactor(MessageContent): Allow the last index to be last in case it has text (may change with streaming) * feat(Plugins): add handleParsingErrors, useful when LLM does not invoke function params * chore: edit out experimental codesherpa integration * refactor(OpenAPIPlugin): rework tool to be 'function-first', as the spec functions are explicitly passed to agent model * refactor(initializeFunctionsAgent): improve error handling and system message * refactor(CodeSherpa, Wolfram): optimize token usage by delegating bulk of instructions to system message * style(Plugins): match official style with input/outputs * chore: remove unnecessary console logs used for testing * fix(abortMiddleware): render markdown when message is aborted * feat(plugins): add BrowserOp * refactor(OpenAPIPlugin): improve prompt handling * fix(useGenerations): hide edit button when message is submitting/streaming * refactor(loadSpecs): optimize OpenAPI spec loading by only loading requested specs instead of all of them * fix(loadSpecs): will retain original behavior when no tools are passed to the function * fix(MessageContent): ensure cursor only shows up for last message and last display index fix(Message): show legacy plugin and pass isLast to Content * chore: remove console.logs * docs: update docs based on breaking changes and new features refactor(structured/SD): use description_for_model for detailed prompting * docs(azure): make plugins section more clear * refactor(structured/SD): change default payload to SD-WebUI to prefer realism and config for SDXL * refactor(structured/SD): further improve system message prompt * docs: update breaking changes after rebase * refactor(MessageContent): factor out EditMessage, types, Container to separate files, rename Content -> Markdown * fix(CodeInterpreter): linting errors * chore: reduce browser console logs from message streams * chore: re-enable debug logs for plugins/langchain to help with user troubleshooting * chore(manifest.json): add [Experimental] tag to CodeInterpreter plugins, which are not intended as the end-all be-all implementation of this feature for Librechat commit66b8580487Author: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Mon Aug 28 09:18:25 2023 -0400 docs: third-party tools (#848) * docs: third-party tools * docs: third-party tools * Update third-party.md * Update third-party.md --------- Co-authored-by: Danny Avila <110412045+danny-avila@users.noreply.github.com> commit9791a78161Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Mon Aug 28 15:14:05 2023 +0200 adjust the animation (#843) commit3797ec6082Author: Ronith <87087292+ronith256@users.noreply.github.com> Date: Mon Aug 28 18:43:50 2023 +0530 feat: Add Code Interpreter Plugin (#837) * feat: Add Code Interpreter Plugin Adds a Simple Code Interpreter Plugin. ## Features: - Runs code using local Python Environment ## Issues - Code execution is not sandboxed. * Add Docker Sandbox for Python Server commite2397076a2Author: Alex Zhang <ztc2011@gmail.com> Date: Mon Aug 28 00:55:34 2023 +0800 🌐: Chinese Translation (#846) commit50c15c704fAuthor: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Sat Aug 26 19:36:59 2023 -0400 Language translation: Polish (#840) * Language translation: Polish * Language translation: Polish * Revert changes in language-contributions.md commit29d3640546Author: Fuegovic <32828263+fuegovic@users.noreply.github.com> Date: Sat Aug 26 19:36:25 2023 -0400 docs: updates (#841) commit39c626aa8eAuthor: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 25 09:29:19 2023 -0400 fix: isEdited edge case where latest Message is not saved due to aborting too quickly commitae5c06f381Author: Danny Avila <messagedaniel@protonmail.com> Date: Fri Aug 25 09:13:50 2023 -0400 fix(chatGPTBrowser): render markdown formatting by setting isCreatedByUser, fix(useMessageHandler): avoid double appearance of cursor by setting latest message at initial response creation time commit9ef1686e18Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Thu Aug 24 20:24:47 2023 -0400 Update mkdocs.yml commit5bbe411569Author: Flynn <dev@flynnbuckingham.com> Date: Thu Aug 24 20:20:37 2023 -0400 Add podman installation instructions. Update dockerfile to stub env (#819) * Added podman container installation docs. Updated dockerfile to stub env file if not present in source * Fix typos commit887fec99caAuthor: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 25 02:11:27 2023 +0200 🌐: Russian Translation (#830) commit007d51ede1Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Fri Aug 25 02:10:48 2023 +0200 feat: facebook login (#820) * Facebook strategy * Update user_auth_system.md * Update user_auth_system.md commita569020312Author: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Date: Thu Aug 24 21:59:11 2023 +0200 Fix Meilisearch error and refactor of the server index.js (#832) * fix meilisearch error at startup * limit the nesting * disable useless console log * fix(indexSync.js): removed redundant searchEnabled * refactor(index.js): moved configureSocialLogins to a new file * refactor(socialLogins.js): removed unnecessary conditional commit37347d4683Author: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Wed Aug 23 16:14:17 2023 -0400 fix(registration): Make Username optional (#831) * fix(User.js): update validation schema for username field, allow empty string as a valid value fix(validators.js): update validation schema for username field, allow empty string as a valid value fix(Registration.tsx, validators.js): update validation rules for name and username fields, change minimum length to 2 and maximum length to 80, assure they match and allow empty string as a valid value fix(Eng.tsx): update localization string for com_auth_username, indicate that it is optional * fix(User.js): update regex … * Update package-lock.json * Update SubmitButton.tsx * Update SpeechRecognition.tsx * fix: typescript error * style: moved to new UI * fix:(SpeechRecognition) lint error * moved everything to hooks * feat: support stt external * fix(useExternalSpeechRecognition): recording the audio * feat: whisper api support * refactor(SpeechReecognition); fix(HoverButtons): set isSpeakling correctly * fix: spelling errors * fix: renamed files * BIG FIX * feat: whisper support * fixed some ChatForm bugs and added the tts route * handling more errors * Fix audio stream initialization and cleanup in useSpeechToTextExternal * feat: Elevenlabs TTS * fixed some req issues * fix: stt not activating on Mac * fix: send audio blob to frontend * fix(ChatForm): startupConfig var * Update text-to-speech and speech-to-text services * handle more errors correctly * Remove console.log statements * feat: added manual trigger with button * fix: SpeechToText and SpeechToTextExernal + AudioRecorder * refactor: TTS component * chore: removed unused variable * feat: azure stt * feat: dedicated speech panel * feat: STT button switch: fix: TextArea pr value adapted * refactor: textToSpeech function and useTextToSpeechMutation * fix: typo data-service * fix: blob backend to frontend * feat: TTS button for external * feat: librechat.yaml * style: spinner when loading TTS * feat: hold click to download file * style: disabled when apiKey not provided * fix: typo startupConfig?.speechToTextExternal * style: update icons * fix(useTextToSpeech): set isSpeaking when audio finish * fix: small issues with local TTS * style: update settings dark theme * docs: STT & TTS * WIP: chat audio automatic; docs(custom_config): update to new .yaml version; chore: updated librechat.yaml version * fix: send button disabled * fix: interval update * localization * removed unused test code * revert interval update to 100 * feat: auto-send message * fix: chat audio automatic, default false * refactor: moved all logic to hooks * chore: renamed ChatAudio to conversationMode * refactor: organized Speech panel * feat: autoSendText switch * feat: moved chataudio to conversationMode and improved error handling; docs: update localai model * refactor: Auto transcribe audio * test: AutoSendTextSwitch, AutoTranscribeAudioSwitch and ConversationModeSwitch.spec: refactor: removed hark * fix: various speechTab fixes * refactor(useSpeechToTextBrowser):: handle more errors * feat: engine select * feat: advanced mode * chore: converted hooks to TS * feat: cache TTS * feat: delete cache; fix: cache issues * refactor(useTextToSpeechExternal): removed unused import * feat: cache switch; refactor: moved to dir STT/TTS * tests: CacheTTS, TextToSpeech, SpeechToText * feat: custom elevenlabs compatibility * fix(useTextToSpeechExternal): cache switch not working * feat: animation for STT * fix: settings var not working * chore: remove unused var * feat: voice dropdown; refactor: yaml changes * fix(textToSpeech): remove undefined properties * refactor: Remove console logs and unused variable * fix: TTS; feat: support coqui and piper * fix: some STT issues * fix: stt test * fix: STT backend sending wrong data * BREAKING: switch to react-speech-recognition, add regenerator-runtime/runtime in main.jsx * feat: websocket backend * foundations for websocket * first pass elevenlabs streaming * streaming audio * stream changes * input streaming implementation * fix: client build errors * WIP: streaming rewrite * audio stream working but not the loop * WIP: looping audio stream working * WIP tts routes rewrite * feat: track SSE runs by runId, which enables us to better track audio streams per message request * chore: set activeRunId on data.created * rate limit tts and only allow once * WIP: streaming audio * refactor(useSSE): simplify messageId/parentMessageId assignment in message stream * delete unused component * streaming working * first pass but need to investigate forever pending bug * optimize audio stream handling client and initial request * fix(StreamAudio): null exception * refactor(tts): add limiters for db polling and timeout promise by intervals and not elapsed time * refactor(textToSpeech): reduce polling delay * feat(StreamAudio): add caching * refactor: rename global variable, add setIsPlaying, remove mediasource ref * feat: use custom hook for audioRef to help determine audio end state * fix: voices mutation -> query * fix: voices mutation -> query 2/2 * feat: successful TTS for manual playback * fix: tts voice init * feat: playback rate * feat: global audio toggles * chore: Add renderIcon function for chat message hover buttons, update schemas with notes * chore: add debug logging instead of console.logs * fix: edge case undefined user id * feat: Automatic Playback switch * feat: add caching bump data-provider * chore: tts add auth * use global state for audio run * feat: assistants support for TTS read aloud * ci: uncomment tests for now until they are refactored * stream audio tests are WIP * refactor: make automatic playback false as default --------- Co-authored-by: bsu3338 <bsu3338@users.noreply.github.com> Co-authored-by: bsu3338 <bsu3338@yahoo.com> Co-authored-by: Marco Beretta <81851188+Berry-13@users.noreply.github.com> Co-authored-by: Berry-13 <marco13beretta@gmail.com> Co-authored-by: Super User <root@Berry> Co-authored-by: Marco Beretta <marco13beretta@proton.me>
This commit is contained in:
Danny Avila
GitHub
parent
9d8fd92dd3
commit
b3e03b75d0
107 changed files with 3971 additions and 444 deletions
|
|
@ -257,6 +257,14 @@ MEILI_NO_ANALYTICS=true
|
|||
MEILI_HOST=http://0.0.0.0:7700
|
||||
MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
|
||||
|
||||
|
||||
#==================================================#
|
||||
# Speech to Text & Text to Speech #
|
||||
#==================================================#
|
||||
|
||||
STT_API_KEY=
|
||||
TTS_API_KEY=
|
||||
|
||||
#===================================================#
|
||||
# User System #
|
||||
#===================================================#
|
||||
|
|
|
|||
|
|
@ -373,6 +373,14 @@ class BaseClient {
|
|||
const { user, head, isEdited, conversationId, responseMessageId, saveOptions, userMessage } =
|
||||
await this.handleStartMethods(message, opts);
|
||||
|
||||
if (opts.progressCallback) {
|
||||
opts.onProgress = opts.progressCallback.call(null, {
|
||||
...(opts.progressOptions ?? {}),
|
||||
parentMessageId: userMessage.messageId,
|
||||
messageId: responseMessageId,
|
||||
});
|
||||
}
|
||||
|
||||
const { generation = '' } = opts;
|
||||
|
||||
// It's not necessary to push to currentMessages
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ const {
|
|||
createContextHandlers,
|
||||
} = require('./prompts');
|
||||
const { encodeAndFormat } = require('~/server/services/Files/images/encode');
|
||||
const { updateTokenWebsocket } = require('~/server/services/Files/Audio');
|
||||
const { isEnabled, sleep } = require('~/server/utils');
|
||||
const { handleOpenAIErrors } = require('./tools/util');
|
||||
const spendTokens = require('~/models/spendTokens');
|
||||
|
|
@ -594,6 +595,7 @@ class OpenAIClient extends BaseClient {
|
|||
payload,
|
||||
(progressMessage) => {
|
||||
if (progressMessage === '[DONE]') {
|
||||
updateTokenWebsocket('[DONE]');
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1216,6 +1218,7 @@ ${convo}
|
|||
});
|
||||
|
||||
const azureDelay = this.modelOptions.model?.includes('gpt-4') ? 30 : 17;
|
||||
|
||||
for await (const chunk of stream) {
|
||||
const token = chunk.choices[0]?.delta?.content || '';
|
||||
intermediateReply += token;
|
||||
|
|
|
|||
|
|
@ -250,6 +250,7 @@ class PluginsClient extends OpenAIClient {
|
|||
this.setOptions(opts);
|
||||
return super.sendMessage(message, opts);
|
||||
}
|
||||
|
||||
logger.debug('[PluginsClient] sendMessage', { userMessageText: message, opts });
|
||||
const {
|
||||
user,
|
||||
|
|
@ -264,6 +265,14 @@ class PluginsClient extends OpenAIClient {
|
|||
onToolEnd,
|
||||
} = await this.handleStartMethods(message, opts);
|
||||
|
||||
if (opts.progressCallback) {
|
||||
opts.onProgress = opts.progressCallback.call(null, {
|
||||
...(opts.progressOptions ?? {}),
|
||||
parentMessageId: opts.progressOptions?.parentMessageId ?? userMessage.messageId,
|
||||
messageId: responseMessageId,
|
||||
});
|
||||
}
|
||||
|
||||
this.currentMessages.push(userMessage);
|
||||
|
||||
let {
|
||||
|
|
|
|||
6
api/cache/getLogStores.js
vendored
6
api/cache/getLogStores.js
vendored
|
|
@ -7,6 +7,7 @@ const keyvMongo = require('./keyvMongo');
|
|||
|
||||
const { BAN_DURATION, USE_REDIS } = process.env ?? {};
|
||||
const THIRTY_MINUTES = 1800000;
|
||||
const TEN_MINUTES = 600000;
|
||||
|
||||
const duration = math(BAN_DURATION, 7200000);
|
||||
|
||||
|
|
@ -24,6 +25,10 @@ const config = isEnabled(USE_REDIS)
|
|||
? new Keyv({ store: keyvRedis })
|
||||
: new Keyv({ namespace: CacheKeys.CONFIG_STORE });
|
||||
|
||||
const audioRuns = isEnabled(USE_REDIS) // ttl: 30 minutes
|
||||
? new Keyv({ store: keyvRedis, ttl: TEN_MINUTES })
|
||||
: new Keyv({ namespace: CacheKeys.AUDIO_RUNS, ttl: TEN_MINUTES });
|
||||
|
||||
const tokenConfig = isEnabled(USE_REDIS) // ttl: 30 minutes
|
||||
? new Keyv({ store: keyvRedis, ttl: THIRTY_MINUTES })
|
||||
: new Keyv({ namespace: CacheKeys.TOKEN_CONFIG, ttl: THIRTY_MINUTES });
|
||||
|
|
@ -64,6 +69,7 @@ const namespaces = {
|
|||
[CacheKeys.TOKEN_CONFIG]: tokenConfig,
|
||||
[CacheKeys.GEN_TITLE]: genTitle,
|
||||
[CacheKeys.MODEL_QUERIES]: modelQueries,
|
||||
[CacheKeys.AUDIO_RUNS]: audioRuns,
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@
|
|||
"ua-parser-js": "^1.0.36",
|
||||
"winston": "^3.11.0",
|
||||
"winston-daily-rotate-file": "^4.7.1",
|
||||
"ws": "^8.17.0",
|
||||
"zod": "^3.22.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
|||
|
|
@ -105,11 +105,12 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
|
|||
getReqData,
|
||||
onStart,
|
||||
abortController,
|
||||
onProgress: progressCallback.call(null, {
|
||||
progressCallback,
|
||||
progressOptions: {
|
||||
res,
|
||||
text,
|
||||
parentMessageId: overrideParentMessageId || userMessageId,
|
||||
}),
|
||||
// parentMessageId: overrideParentMessageId || userMessageId,
|
||||
},
|
||||
};
|
||||
|
||||
let response = await client.sendMessage(text, messageOptions);
|
||||
|
|
|
|||
|
|
@ -112,11 +112,12 @@ const EditController = async (req, res, next, initializeClient) => {
|
|||
getReqData,
|
||||
onStart,
|
||||
abortController,
|
||||
onProgress: progressCallback.call(null, {
|
||||
progressCallback,
|
||||
progressOptions: {
|
||||
res,
|
||||
text,
|
||||
parentMessageId: overrideParentMessageId || userMessageId,
|
||||
}),
|
||||
// parentMessageId: overrideParentMessageId || userMessageId,
|
||||
},
|
||||
});
|
||||
|
||||
const conversation = await getConvo(user, conversationId);
|
||||
|
|
|
|||
|
|
@ -520,6 +520,7 @@ const chatV2 = async (req, res) => {
|
|||
handlers,
|
||||
thread_id,
|
||||
attachedFileIds,
|
||||
parentMessageId,
|
||||
responseMessage: openai.responseMessage,
|
||||
// streamOptions: {
|
||||
|
||||
|
|
@ -532,6 +533,7 @@ const chatV2 = async (req, res) => {
|
|||
});
|
||||
|
||||
response = streamRunManager;
|
||||
response.text = streamRunManager.intermediateText;
|
||||
};
|
||||
|
||||
await processRun();
|
||||
|
|
@ -554,6 +556,7 @@ const chatV2 = async (req, res) => {
|
|||
/** @type {ResponseMessage} */
|
||||
const responseMessage = {
|
||||
...(response.responseMessage ?? response.finalMessage),
|
||||
text: response.text,
|
||||
parentMessageId: userMessageId,
|
||||
conversationId,
|
||||
user: req.user.id,
|
||||
|
|
|
|||
|
|
@ -174,12 +174,13 @@ router.post(
|
|||
onStart,
|
||||
getPartialText,
|
||||
...endpointOption,
|
||||
onProgress: progressCallback.call(null, {
|
||||
progressCallback,
|
||||
progressOptions: {
|
||||
res,
|
||||
text,
|
||||
parentMessageId: overrideParentMessageId || userMessageId,
|
||||
// parentMessageId: overrideParentMessageId || userMessageId,
|
||||
plugins,
|
||||
}),
|
||||
},
|
||||
abortController,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -153,12 +153,13 @@ router.post(
|
|||
onChainEnd,
|
||||
onStart,
|
||||
...endpointOption,
|
||||
onProgress: progressCallback.call(null, {
|
||||
progressCallback,
|
||||
progressOptions: {
|
||||
res,
|
||||
text,
|
||||
plugin,
|
||||
parentMessageId: overrideParentMessageId || userMessageId,
|
||||
}),
|
||||
// parentMessageId: overrideParentMessageId || userMessageId,
|
||||
},
|
||||
abortController,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ const { createMulterInstance } = require('./multer');
|
|||
const files = require('./files');
|
||||
const images = require('./images');
|
||||
const avatar = require('./avatar');
|
||||
const stt = require('./stt');
|
||||
const tts = require('./tts');
|
||||
|
||||
const initialize = async () => {
|
||||
const router = express.Router();
|
||||
|
|
@ -18,6 +20,9 @@ const initialize = async () => {
|
|||
router.post('/', upload.single('file'));
|
||||
router.post('/images', upload.single('file'));
|
||||
|
||||
router.use('/stt', stt);
|
||||
router.use('/tts', tts);
|
||||
|
||||
router.use('/', files);
|
||||
router.use('/images', images);
|
||||
router.use('/images/avatar', avatar);
|
||||
|
|
|
|||
13
api/server/routes/files/stt.js
Normal file
13
api/server/routes/files/stt.js
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const multer = require('multer');
|
||||
const { requireJwtAuth } = require('~/server/middleware/');
|
||||
const { speechToText } = require('~/server/services/Files/Audio');
|
||||
|
||||
const upload = multer();
|
||||
|
||||
router.post('/', requireJwtAuth, upload.single('audio'), async (req, res) => {
|
||||
await speechToText(req, res);
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
42
api/server/routes/files/tts.js
Normal file
42
api/server/routes/files/tts.js
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
const multer = require('multer');
|
||||
const express = require('express');
|
||||
const { CacheKeys } = require('librechat-data-provider');
|
||||
const { getVoices, streamAudio, textToSpeech } = require('~/server/services/Files/Audio');
|
||||
const { getLogStores } = require('~/cache');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
const router = express.Router();
|
||||
const upload = multer();
|
||||
|
||||
router.post('/manual', upload.none(), async (req, res) => {
|
||||
await textToSpeech(req, res);
|
||||
});
|
||||
|
||||
const logDebugMessage = (req, message) =>
|
||||
logger.debug(`[streamAudio] user: ${req?.user?.id ?? 'UNDEFINED_USER'} | ${message}`);
|
||||
|
||||
// TODO: test caching
|
||||
router.post('/', async (req, res) => {
|
||||
try {
|
||||
const audioRunsCache = getLogStores(CacheKeys.AUDIO_RUNS);
|
||||
const audioRun = await audioRunsCache.get(req.body.runId);
|
||||
logDebugMessage(req, 'start stream audio');
|
||||
if (audioRun) {
|
||||
logDebugMessage(req, 'stream audio already running');
|
||||
return res.status(401).json({ error: 'Audio stream already running' });
|
||||
}
|
||||
audioRunsCache.set(req.body.runId, true);
|
||||
await streamAudio(req, res);
|
||||
logDebugMessage(req, 'end stream audio');
|
||||
res.status(200).end();
|
||||
} catch (error) {
|
||||
logger.error(`[streamAudio] user: ${req.user.id} | Failed to stream audio: ${error}`);
|
||||
res.status(500).json({ error: 'Failed to stream audio' });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/voices', async (req, res) => {
|
||||
await getVoices(req, res);
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
45
api/server/services/Files/Audio/getVoices.js
Normal file
45
api/server/services/Files/Audio/getVoices.js
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
const { logger } = require('~/config');
|
||||
const getCustomConfig = require('~/server/services/Config/getCustomConfig');
|
||||
const { getProvider } = require('./textToSpeech');
|
||||
|
||||
/**
|
||||
* This function retrieves the available voices for the current TTS provider
|
||||
* It first fetches the TTS configuration and determines the provider
|
||||
* Then, based on the provider, it sends the corresponding voices as a JSON response
|
||||
*
|
||||
* @param {Object} req - The request object
|
||||
* @param {Object} res - The response object
|
||||
* @returns {Promise<void>}
|
||||
* @throws {Error} - If the provider is not 'openai' or 'elevenlabs', an error is thrown
|
||||
*/
|
||||
async function getVoices(req, res) {
|
||||
try {
|
||||
const customConfig = await getCustomConfig();
|
||||
|
||||
if (!customConfig || !customConfig?.tts) {
|
||||
throw new Error('Configuration or TTS schema is missing');
|
||||
}
|
||||
|
||||
const ttsSchema = customConfig?.tts;
|
||||
const provider = getProvider(ttsSchema);
|
||||
let voices;
|
||||
|
||||
switch (provider) {
|
||||
case 'openai':
|
||||
voices = ttsSchema.openai?.voices;
|
||||
break;
|
||||
case 'elevenlabs':
|
||||
voices = ttsSchema.elevenlabs?.voices;
|
||||
break;
|
||||
default:
|
||||
throw new Error('Invalid provider');
|
||||
}
|
||||
|
||||
res.json(voices);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to get voices: ${error.message}`);
|
||||
res.status(500).json({ error: 'Failed to get voices' });
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = getVoices;
|
||||
11
api/server/services/Files/Audio/index.js
Normal file
11
api/server/services/Files/Audio/index.js
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
const getVoices = require('./getVoices');
|
||||
const textToSpeech = require('./textToSpeech');
|
||||
const speechToText = require('./speechToText');
|
||||
const { updateTokenWebsocket } = require('./webSocket');
|
||||
|
||||
module.exports = {
|
||||
getVoices,
|
||||
speechToText,
|
||||
...textToSpeech,
|
||||
updateTokenWebsocket,
|
||||
};
|
||||
211
api/server/services/Files/Audio/speechToText.js
Normal file
211
api/server/services/Files/Audio/speechToText.js
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
const axios = require('axios');
|
||||
const { Readable } = require('stream');
|
||||
const { logger } = require('~/config');
|
||||
const getCustomConfig = require('~/server/services/Config/getCustomConfig');
|
||||
const { extractEnvVariable } = require('librechat-data-provider');
|
||||
|
||||
/**
|
||||
* Handle the response from the STT API
|
||||
* @param {Object} response - The response from the STT API
|
||||
*
|
||||
* @returns {string} The text from the response data
|
||||
*
|
||||
* @throws Will throw an error if the response status is not 200 or the response data is missing
|
||||
*/
|
||||
async function handleResponse(response) {
|
||||
if (response.status !== 200) {
|
||||
throw new Error('Invalid response from the STT API');
|
||||
}
|
||||
|
||||
if (!response.data || !response.data.text) {
|
||||
throw new Error('Missing data in response from the STT API');
|
||||
}
|
||||
|
||||
return response.data.text.trim();
|
||||
}
|
||||
|
||||
function getProvider(sttSchema) {
|
||||
if (sttSchema.openai) {
|
||||
return 'openai';
|
||||
}
|
||||
|
||||
throw new Error('Invalid provider');
|
||||
}
|
||||
|
||||
function removeUndefined(obj) {
|
||||
Object.keys(obj).forEach((key) => {
|
||||
if (obj[key] && typeof obj[key] === 'object') {
|
||||
removeUndefined(obj[key]);
|
||||
if (Object.keys(obj[key]).length === 0) {
|
||||
delete obj[key];
|
||||
}
|
||||
} else if (obj[key] === undefined) {
|
||||
delete obj[key];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* This function prepares the necessary data and headers for making a request to the OpenAI API
|
||||
* It uses the provided speech-to-text schema and audio stream to create the request
|
||||
*
|
||||
* @param {Object} sttSchema - The speech-to-text schema containing the OpenAI configuration
|
||||
* @param {Stream} audioReadStream - The audio data to be transcribed
|
||||
*
|
||||
* @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
|
||||
* If an error occurs, it returns an array with three null values and logs the error with logger
|
||||
*/
|
||||
function openAIProvider(sttSchema, audioReadStream) {
|
||||
try {
|
||||
const url = sttSchema.openai?.url || 'https://api.openai.com/v1/audio/transcriptions';
|
||||
const apiKey = sttSchema.openai.apiKey ? extractEnvVariable(sttSchema.openai.apiKey) : '';
|
||||
|
||||
let data = {
|
||||
file: audioReadStream,
|
||||
model: sttSchema.openai.model,
|
||||
};
|
||||
|
||||
let headers = {
|
||||
'Content-Type': 'multipart/form-data',
|
||||
};
|
||||
|
||||
[headers].forEach(removeUndefined);
|
||||
|
||||
if (apiKey) {
|
||||
headers.Authorization = 'Bearer ' + apiKey;
|
||||
}
|
||||
|
||||
return [url, data, headers];
|
||||
} catch (error) {
|
||||
logger.error('An error occurred while preparing the OpenAI API STT request: ', error);
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This function prepares the necessary data and headers for making a request to the Azure API
|
||||
* It uses the provided request and audio stream to create the request
|
||||
*
|
||||
* @param {Object} req - The request object, which should contain the endpoint in its body
|
||||
* @param {Stream} audioReadStream - The audio data to be transcribed
|
||||
*
|
||||
* @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
|
||||
* If an error occurs, it returns an array with three null values and logs the error with logger
|
||||
*/
|
||||
function azureProvider(req, audioReadStream) {
|
||||
try {
|
||||
const { endpoint } = req.body;
|
||||
const azureConfig = req.app.locals[endpoint];
|
||||
|
||||
if (!azureConfig) {
|
||||
throw new Error(`No configuration found for endpoint: ${endpoint}`);
|
||||
}
|
||||
|
||||
const { apiKey, instanceName, whisperModel, apiVersion } = Object.entries(
|
||||
azureConfig.groupMap,
|
||||
).reduce((acc, [, value]) => {
|
||||
if (acc) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
const whisperKey = Object.keys(value.models).find((modelKey) =>
|
||||
modelKey.startsWith('whisper'),
|
||||
);
|
||||
|
||||
if (whisperKey) {
|
||||
return {
|
||||
apiVersion: value.version,
|
||||
apiKey: value.apiKey,
|
||||
instanceName: value.instanceName,
|
||||
whisperModel: value.models[whisperKey]['deploymentName'],
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}, null);
|
||||
|
||||
if (!apiKey || !instanceName || !whisperModel || !apiVersion) {
|
||||
throw new Error('Required Azure configuration values are missing');
|
||||
}
|
||||
|
||||
const baseURL = `https://${instanceName}.openai.azure.com`;
|
||||
|
||||
const url = `${baseURL}/openai/deployments/${whisperModel}/audio/transcriptions?api-version=${apiVersion}`;
|
||||
|
||||
let data = {
|
||||
file: audioReadStream,
|
||||
filename: 'audio.wav',
|
||||
contentType: 'audio/wav',
|
||||
knownLength: audioReadStream.length,
|
||||
};
|
||||
|
||||
const headers = {
|
||||
...data.getHeaders(),
|
||||
'Content-Type': 'multipart/form-data',
|
||||
'api-key': apiKey,
|
||||
};
|
||||
|
||||
return [url, data, headers];
|
||||
} catch (error) {
|
||||
logger.error('An error occurred while preparing the Azure API STT request: ', error);
|
||||
return [null, null, null];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert speech to text
|
||||
* @param {Object} req - The request object
|
||||
* @param {Object} res - The response object
|
||||
*
|
||||
* @returns {Object} The response object with the text from the STT API
|
||||
*
|
||||
* @throws Will throw an error if an error occurs while processing the audio
|
||||
*/
|
||||
|
||||
async function speechToText(req, res) {
|
||||
const customConfig = await getCustomConfig();
|
||||
if (!customConfig) {
|
||||
return res.status(500).send('Custom config not found');
|
||||
}
|
||||
|
||||
if (!req.file || !req.file.buffer) {
|
||||
return res.status(400).json({ message: 'No audio file provided in the FormData' });
|
||||
}
|
||||
|
||||
const audioBuffer = req.file.buffer;
|
||||
const audioReadStream = Readable.from(audioBuffer);
|
||||
audioReadStream.path = 'audio.wav';
|
||||
|
||||
const provider = getProvider(customConfig.stt);
|
||||
|
||||
let [url, data, headers] = [];
|
||||
|
||||
switch (provider) {
|
||||
case 'openai':
|
||||
[url, data, headers] = openAIProvider(customConfig.stt, audioReadStream);
|
||||
break;
|
||||
case 'azure':
|
||||
[url, data, headers] = azureProvider(req, audioReadStream);
|
||||
break;
|
||||
default:
|
||||
throw new Error('Invalid provider');
|
||||
}
|
||||
|
||||
if (!Readable.from) {
|
||||
const audioBlob = new Blob([audioBuffer], { type: req.file.mimetype });
|
||||
delete data['file'];
|
||||
data['file'] = audioBlob;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await axios.post(url, data, { headers: headers });
|
||||
const text = await handleResponse(response);
|
||||
|
||||
res.json({ text });
|
||||
} catch (error) {
|
||||
logger.error('An error occurred while processing the audio:', error);
|
||||
res.sendStatus(500);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = speechToText;
|
||||
91
api/server/services/Files/Audio/streamAudio-wip.js
Normal file
91
api/server/services/Files/Audio/streamAudio-wip.js
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
const { Message } = require('~/models/Message');
|
||||
const { createChunkProcessor } = require('./streamAudio');
|
||||
|
||||
jest.mock('~/models/Message', () => ({
|
||||
Message: {
|
||||
findOne: jest.fn().mockReturnValue({
|
||||
lean: jest.fn(),
|
||||
}),
|
||||
},
|
||||
}));
|
||||
|
||||
describe('processChunks', () => {
|
||||
let processChunks;
|
||||
|
||||
beforeEach(() => {
|
||||
processChunks = createChunkProcessor();
|
||||
Message.findOne.mockClear();
|
||||
Message.findOne().lean.mockClear();
|
||||
});
|
||||
|
||||
it('should return an empty array when the message is not found', async () => {
|
||||
Message.findOne().lean.mockResolvedValueOnce(null);
|
||||
|
||||
const result = await processChunks('non-existent-id');
|
||||
|
||||
expect(result).toEqual([]);
|
||||
expect(Message.findOne).toHaveBeenCalledWith(
|
||||
{ messageId: 'non-existent-id' },
|
||||
'text unfinished',
|
||||
);
|
||||
expect(Message.findOne().lean).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return an empty array when the message does not have a text property', async () => {
|
||||
Message.findOne().lean.mockResolvedValueOnce({ unfinished: true });
|
||||
|
||||
const result = await processChunks('message-id');
|
||||
|
||||
expect(result).toEqual([]);
|
||||
expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');
|
||||
expect(Message.findOne().lean).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return chunks for an unfinished message with separators', async () => {
|
||||
const messageText = 'This is a long message. It should be split into chunks. Lol hi mom';
|
||||
Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: true });
|
||||
|
||||
const result = await processChunks('message-id');
|
||||
|
||||
expect(result).toEqual([
|
||||
{ text: 'This is a long message. It should be split into chunks.', isFinished: false },
|
||||
]);
|
||||
expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');
|
||||
expect(Message.findOne().lean).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return chunks for an unfinished message without separators', async () => {
|
||||
const messageText = 'This is a long message without separators hello there my friend';
|
||||
Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: true });
|
||||
|
||||
const result = await processChunks('message-id');
|
||||
|
||||
expect(result).toEqual([{ text: messageText, isFinished: false }]);
|
||||
expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');
|
||||
expect(Message.findOne().lean).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return the remaining text as a chunk for a finished message', async () => {
|
||||
const messageText = 'This is a finished message.';
|
||||
Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: false });
|
||||
|
||||
const result = await processChunks('message-id');
|
||||
|
||||
expect(result).toEqual([{ text: messageText, isFinished: true }]);
|
||||
expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');
|
||||
expect(Message.findOne().lean).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should return an empty array for a finished message with no remaining text', async () => {
|
||||
const messageText = 'This is a finished message.';
|
||||
Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: false });
|
||||
|
||||
await processChunks('message-id');
|
||||
Message.findOne().lean.mockResolvedValueOnce({ text: messageText, unfinished: false });
|
||||
const result = await processChunks('message-id');
|
||||
|
||||
expect(result).toEqual([]);
|
||||
expect(Message.findOne).toHaveBeenCalledWith({ messageId: 'message-id' }, 'text unfinished');
|
||||
expect(Message.findOne().lean).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
312
api/server/services/Files/Audio/streamAudio.js
Normal file
312
api/server/services/Files/Audio/streamAudio.js
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
const WebSocket = require('ws');
|
||||
const { Message } = require('~/models/Message');
|
||||
|
||||
/**
|
||||
* @param {string[]} voiceIds - Array of voice IDs
|
||||
* @returns {string}
|
||||
*/
|
||||
function getRandomVoiceId(voiceIds) {
|
||||
const randomIndex = Math.floor(Math.random() * voiceIds.length);
|
||||
return voiceIds[randomIndex];
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Object} VoiceSettings
|
||||
* @property {number} similarity_boost
|
||||
* @property {number} stability
|
||||
* @property {boolean} use_speaker_boost
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} GenerateAudioBulk
|
||||
* @property {string} model_id
|
||||
* @property {string} text
|
||||
* @property {VoiceSettings} voice_settings
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} TextToSpeechClient
|
||||
* @property {function(Object): Promise<stream.Readable>} generate
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} AudioChunk
|
||||
* @property {string} audio
|
||||
* @property {boolean} isFinal
|
||||
* @property {Object} alignment
|
||||
* @property {number[]} alignment.char_start_times_ms
|
||||
* @property {number[]} alignment.chars_durations_ms
|
||||
* @property {string[]} alignment.chars
|
||||
* @property {Object} normalizedAlignment
|
||||
* @property {number[]} normalizedAlignment.char_start_times_ms
|
||||
* @property {number[]} normalizedAlignment.chars_durations_ms
|
||||
* @property {string[]} normalizedAlignment.chars
|
||||
*/
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {Record<string, unknown | undefined>} parameters
|
||||
* @returns
|
||||
*/
|
||||
function assembleQuery(parameters) {
|
||||
let query = '';
|
||||
let hasQuestionMark = false;
|
||||
|
||||
for (const [key, value] of Object.entries(parameters)) {
|
||||
if (value == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!hasQuestionMark) {
|
||||
query += '?';
|
||||
hasQuestionMark = true;
|
||||
} else {
|
||||
query += '&';
|
||||
}
|
||||
|
||||
query += `${key}=${value}`;
|
||||
}
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
const SEPARATORS = ['.', '?', '!', '۔', '。', '‥', ';', '¡', '¿', '\n'];
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {string} text
|
||||
* @param {string[] | undefined} [separators]
|
||||
* @returns
|
||||
*/
|
||||
function findLastSeparatorIndex(text, separators = SEPARATORS) {
|
||||
let lastIndex = -1;
|
||||
for (const separator of separators) {
|
||||
const index = text.lastIndexOf(separator);
|
||||
if (index > lastIndex) {
|
||||
lastIndex = index;
|
||||
}
|
||||
}
|
||||
return lastIndex;
|
||||
}
|
||||
|
||||
const MAX_NOT_FOUND_COUNT = 6;
|
||||
const MAX_NO_CHANGE_COUNT = 12;
|
||||
|
||||
/**
|
||||
* @param {string} messageId
|
||||
* @returns {() => Promise<{ text: string, isFinished: boolean }[]>}
|
||||
*/
|
||||
function createChunkProcessor(messageId) {
|
||||
let notFoundCount = 0;
|
||||
let noChangeCount = 0;
|
||||
let processedText = '';
|
||||
if (!messageId) {
|
||||
throw new Error('Message ID is required');
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<{ text: string, isFinished: boolean }[] | string>}
|
||||
*/
|
||||
async function processChunks() {
|
||||
if (notFoundCount >= MAX_NOT_FOUND_COUNT) {
|
||||
return `Message not found after ${MAX_NOT_FOUND_COUNT} attempts`;
|
||||
}
|
||||
|
||||
if (noChangeCount >= MAX_NO_CHANGE_COUNT) {
|
||||
return `No change in message after ${MAX_NO_CHANGE_COUNT} attempts`;
|
||||
}
|
||||
|
||||
const message = await Message.findOne({ messageId }, 'text unfinished').lean();
|
||||
|
||||
if (!message || !message.text) {
|
||||
notFoundCount++;
|
||||
return [];
|
||||
}
|
||||
|
||||
const { text, unfinished } = message;
|
||||
if (text === processedText) {
|
||||
noChangeCount++;
|
||||
}
|
||||
|
||||
const remainingText = text.slice(processedText.length);
|
||||
const chunks = [];
|
||||
|
||||
if (unfinished && remainingText.length >= 20) {
|
||||
const separatorIndex = findLastSeparatorIndex(remainingText);
|
||||
if (separatorIndex !== -1) {
|
||||
const chunkText = remainingText.slice(0, separatorIndex + 1);
|
||||
chunks.push({ text: chunkText, isFinished: false });
|
||||
processedText += chunkText;
|
||||
} else {
|
||||
chunks.push({ text: remainingText, isFinished: false });
|
||||
processedText = text;
|
||||
}
|
||||
} else if (!unfinished && remainingText.trim().length > 0) {
|
||||
chunks.push({ text: remainingText.trim(), isFinished: true });
|
||||
processedText = text;
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
return processChunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Input stream text to speech
|
||||
* @param {Express.Response} res
|
||||
* @param {AsyncIterable<string>} textStream
|
||||
* @param {(token: string) => Promise<boolean>} callback - Whether to continue the stream or not
|
||||
* @returns {AsyncGenerator<AudioChunk>}
|
||||
*/
|
||||
function inputStreamTextToSpeech(res, textStream, callback) {
|
||||
const model = 'eleven_monolingual_v1';
|
||||
const wsUrl = `wss://api.elevenlabs.io/v1/text-to-speech/${getRandomVoiceId()}/stream-input${assembleQuery(
|
||||
{
|
||||
model_id: model,
|
||||
// flush: true,
|
||||
// optimize_streaming_latency: this.settings.optimizeStreamingLatency,
|
||||
optimize_streaming_latency: 1,
|
||||
// output_format: this.settings.outputFormat,
|
||||
},
|
||||
)}`;
|
||||
const socket = new WebSocket(wsUrl);
|
||||
|
||||
socket.onopen = function () {
|
||||
const streamStart = {
|
||||
text: ' ',
|
||||
voice_settings: {
|
||||
stability: 0.5,
|
||||
similarity_boost: 0.8,
|
||||
},
|
||||
xi_api_key: process.env.ELEVENLABS_API_KEY,
|
||||
// generation_config: { chunk_length_schedule: [50, 90, 120, 150, 200] },
|
||||
};
|
||||
|
||||
socket.send(JSON.stringify(streamStart));
|
||||
|
||||
// send stream until done
|
||||
const streamComplete = new Promise((resolve, reject) => {
|
||||
(async () => {
|
||||
let textBuffer = '';
|
||||
let shouldContinue = true;
|
||||
for await (const textDelta of textStream) {
|
||||
textBuffer += textDelta;
|
||||
|
||||
// using ". " as separator: sending in full sentences improves the quality
|
||||
// of the audio output significantly.
|
||||
const separatorIndex = findLastSeparatorIndex(textBuffer);
|
||||
|
||||
// Callback for textStream (will return false if signal is aborted)
|
||||
shouldContinue = await callback(textDelta);
|
||||
|
||||
if (separatorIndex === -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!shouldContinue) {
|
||||
break;
|
||||
}
|
||||
|
||||
const textToProcess = textBuffer.slice(0, separatorIndex);
|
||||
textBuffer = textBuffer.slice(separatorIndex + 1);
|
||||
|
||||
const request = {
|
||||
text: textToProcess,
|
||||
try_trigger_generation: true,
|
||||
};
|
||||
|
||||
socket.send(JSON.stringify(request));
|
||||
}
|
||||
|
||||
// send remaining text:
|
||||
if (shouldContinue && textBuffer.length > 0) {
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
text: `${textBuffer} `, // append space
|
||||
try_trigger_generation: true,
|
||||
}),
|
||||
);
|
||||
}
|
||||
})()
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
});
|
||||
|
||||
streamComplete
|
||||
.then(() => {
|
||||
const endStream = {
|
||||
text: '',
|
||||
};
|
||||
|
||||
socket.send(JSON.stringify(endStream));
|
||||
})
|
||||
.catch((e) => {
|
||||
console.error('Error streaming text to speech:', e);
|
||||
throw e;
|
||||
});
|
||||
};
|
||||
|
||||
return (async function* audioStream() {
|
||||
let isDone = false;
|
||||
let chunks = [];
|
||||
let resolve;
|
||||
let waitForMessage = new Promise((r) => (resolve = r));
|
||||
|
||||
socket.onmessage = function (event) {
|
||||
// console.log(event);
|
||||
const audioChunk = JSON.parse(event.data);
|
||||
if (audioChunk.audio && audioChunk.alignment) {
|
||||
res.write(`event: audio\ndata: ${event.data}\n\n`);
|
||||
chunks.push(audioChunk);
|
||||
resolve(null);
|
||||
waitForMessage = new Promise((r) => (resolve = r));
|
||||
} else if (audioChunk.isFinal) {
|
||||
isDone = true;
|
||||
resolve(null);
|
||||
} else if (audioChunk.message) {
|
||||
console.warn('Received Elevenlabs message:', audioChunk.message);
|
||||
resolve(null);
|
||||
}
|
||||
};
|
||||
|
||||
socket.onerror = function (error) {
|
||||
console.error('WebSocket error:', error);
|
||||
// throw error;
|
||||
};
|
||||
|
||||
socket.onclose = function () {
|
||||
isDone = true;
|
||||
resolve(null);
|
||||
};
|
||||
|
||||
while (!isDone) {
|
||||
await waitForMessage;
|
||||
yield* chunks;
|
||||
chunks = [];
|
||||
}
|
||||
|
||||
res.write('event: end\ndata: \n\n');
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {AsyncIterable<string>} llmStream
|
||||
*/
|
||||
async function* llmMessageSource(llmStream) {
|
||||
for await (const chunk of llmStream) {
|
||||
const message = chunk.choices[0].delta.content;
|
||||
if (message) {
|
||||
yield message;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
inputStreamTextToSpeech,
|
||||
findLastSeparatorIndex,
|
||||
createChunkProcessor,
|
||||
llmMessageSource,
|
||||
getRandomVoiceId,
|
||||
};
|
||||
390
api/server/services/Files/Audio/textToSpeech.js
Normal file
390
api/server/services/Files/Audio/textToSpeech.js
Normal file
|
|
@ -0,0 +1,390 @@
|
|||
const axios = require('axios');
|
||||
const getCustomConfig = require('~/server/services/Config/getCustomConfig');
|
||||
const { getRandomVoiceId, createChunkProcessor } = require('./streamAudio');
|
||||
const { extractEnvVariable } = require('librechat-data-provider');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
/**
|
||||
* getProvider function
|
||||
* This function takes the ttsSchema object and returns the name of the provider
|
||||
* If more than one provider is set or no provider is set, it throws an error
|
||||
*
|
||||
* @param {Object} ttsSchema - The TTS schema containing the provider configuration
|
||||
* @returns {string} The name of the provider
|
||||
* @throws {Error} Throws an error if multiple providers are set or no provider is set
|
||||
*/
|
||||
function getProvider(ttsSchema) {
|
||||
if (!ttsSchema) {
|
||||
throw new Error(`No TTS schema is set. Did you configure TTS in the custom config (librechat.yaml)?
|
||||
# Example TTS configuration`);
|
||||
}
|
||||
const providers = Object.entries(ttsSchema).filter(([, value]) => Object.keys(value).length > 0);
|
||||
|
||||
if (providers.length > 1) {
|
||||
throw new Error('Multiple providers are set. Please set only one provider.');
|
||||
} else if (providers.length === 0) {
|
||||
throw new Error('No provider is set. Please set a provider.');
|
||||
} else {
|
||||
return providers[0][0];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* removeUndefined function
|
||||
* This function takes an object and removes all keys with undefined values
|
||||
* It also removes keys with empty objects as values
|
||||
*
|
||||
* @param {Object} obj - The object to be cleaned
|
||||
* @returns {void} This function does not return a value. It modifies the input object directly
|
||||
*/
|
||||
function removeUndefined(obj) {
|
||||
Object.keys(obj).forEach((key) => {
|
||||
if (obj[key] && typeof obj[key] === 'object') {
|
||||
removeUndefined(obj[key]);
|
||||
if (Object.keys(obj[key]).length === 0) {
|
||||
delete obj[key];
|
||||
}
|
||||
} else if (obj[key] === undefined) {
|
||||
delete obj[key];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* This function prepares the necessary data and headers for making a request to the OpenAI TTS
|
||||
* It uses the provided TTS schema, input text, and voice to create the request
|
||||
*
|
||||
* @param {Object} ttsSchema - The TTS schema containing the OpenAI configuration
|
||||
* @param {string} input - The text to be converted to speech
|
||||
* @param {string} voice - The voice to be used for the speech
|
||||
*
|
||||
* @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
|
||||
* If an error occurs, it throws an error with a message indicating that the selected voice is not available
|
||||
*/
|
||||
function openAIProvider(ttsSchema, input, voice) {
|
||||
const url = ttsSchema.openai?.url || 'https://api.openai.com/v1/audio/speech';
|
||||
|
||||
if (
|
||||
ttsSchema.openai?.voices &&
|
||||
ttsSchema.openai.voices.length > 0 &&
|
||||
!ttsSchema.openai.voices.includes(voice) &&
|
||||
!ttsSchema.openai.voices.includes('ALL')
|
||||
) {
|
||||
throw new Error(`Voice ${voice} is not available.`);
|
||||
}
|
||||
|
||||
let data = {
|
||||
input,
|
||||
model: ttsSchema.openai?.model,
|
||||
voice: ttsSchema.openai?.voices && ttsSchema.openai.voices.length > 0 ? voice : undefined,
|
||||
backend: ttsSchema.openai?.backend,
|
||||
};
|
||||
|
||||
let headers = {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: 'Bearer ' + extractEnvVariable(ttsSchema.openai?.apiKey),
|
||||
};
|
||||
|
||||
[data, headers].forEach(removeUndefined);
|
||||
|
||||
return [url, data, headers];
|
||||
}
|
||||
|
||||
/**
|
||||
* elevenLabsProvider function
|
||||
* This function prepares the necessary data and headers for making a request to the Eleven Labs TTS
|
||||
* It uses the provided TTS schema, input text, and voice to create the request
|
||||
*
|
||||
* @param {Object} ttsSchema - The TTS schema containing the Eleven Labs configuration
|
||||
* @param {string} input - The text to be converted to speech
|
||||
* @param {string} voice - The voice to be used for the speech
|
||||
* @param {boolean} stream - Whether to stream the audio or not
|
||||
*
|
||||
* @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
|
||||
* @throws {Error} Throws an error if the selected voice is not available
|
||||
*/
|
||||
function elevenLabsProvider(ttsSchema, input, voice, stream) {
|
||||
let url =
|
||||
ttsSchema.elevenlabs?.url ||
|
||||
`https://api.elevenlabs.io/v1/text-to-speech/{voice_id}${stream ? '/stream' : ''}`;
|
||||
|
||||
if (
|
||||
!ttsSchema.elevenlabs?.voices.includes(voice) &&
|
||||
!ttsSchema.elevenlabs?.voices.includes('ALL')
|
||||
) {
|
||||
throw new Error(`Voice ${voice} is not available.`);
|
||||
}
|
||||
|
||||
url = url.replace('{voice_id}', voice);
|
||||
|
||||
let data = {
|
||||
model_id: ttsSchema.elevenlabs?.model,
|
||||
text: input,
|
||||
// voice_id: voice,
|
||||
voice_settings: {
|
||||
similarity_boost: ttsSchema.elevenlabs?.voice_settings?.similarity_boost,
|
||||
stability: ttsSchema.elevenlabs?.voice_settings?.stability,
|
||||
style: ttsSchema.elevenlabs?.voice_settings?.style,
|
||||
use_speaker_boost: ttsSchema.elevenlabs?.voice_settings?.use_speaker_boost || undefined,
|
||||
},
|
||||
pronunciation_dictionary_locators: ttsSchema.elevenlabs?.pronunciation_dictionary_locators,
|
||||
};
|
||||
|
||||
let headers = {
|
||||
'Content-Type': 'application/json',
|
||||
'xi-api-key': extractEnvVariable(ttsSchema.elevenlabs?.apiKey),
|
||||
Accept: 'audio/mpeg',
|
||||
};
|
||||
|
||||
[data, headers].forEach(removeUndefined);
|
||||
|
||||
return [url, data, headers];
|
||||
}
|
||||
|
||||
/**
|
||||
* localAIProvider function
|
||||
* This function prepares the necessary data and headers for making a request to the LocalAI TTS
|
||||
* It uses the provided TTS schema, input text, and voice to create the request
|
||||
*
|
||||
* @param {Object} ttsSchema - The TTS schema containing the LocalAI configuration
|
||||
* @param {string} input - The text to be converted to speech
|
||||
* @param {string} voice - The voice to be used for the speech
|
||||
*
|
||||
* @returns {Array} An array containing the URL for the API request, the data to be sent, and the headers for the request
|
||||
* @throws {Error} Throws an error if the selected voice is not available
|
||||
*/
|
||||
function localAIProvider(ttsSchema, input, voice) {
|
||||
let url = ttsSchema.localai?.url;
|
||||
|
||||
if (
|
||||
ttsSchema.localai?.voices &&
|
||||
ttsSchema.localai.voices.length > 0 &&
|
||||
!ttsSchema.localai.voices.includes(voice) &&
|
||||
!ttsSchema.localai.voices.includes('ALL')
|
||||
) {
|
||||
throw new Error(`Voice ${voice} is not available.`);
|
||||
}
|
||||
|
||||
let data = {
|
||||
input,
|
||||
model: ttsSchema.localai?.voices && ttsSchema.localai.voices.length > 0 ? voice : undefined,
|
||||
backend: ttsSchema.localai?.backend,
|
||||
};
|
||||
|
||||
let headers = {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: 'Bearer ' + extractEnvVariable(ttsSchema.localai?.apiKey),
|
||||
};
|
||||
|
||||
[data, headers].forEach(removeUndefined);
|
||||
|
||||
if (extractEnvVariable(ttsSchema.localai.apiKey) === '') {
|
||||
delete headers.Authorization;
|
||||
}
|
||||
|
||||
return [url, data, headers];
|
||||
}
|
||||
|
||||
/* not used */
|
||||
/*
|
||||
async function streamAudioFromWebSocket(req, res) {
|
||||
const { voice } = req.body;
|
||||
const customConfig = await getCustomConfig();
|
||||
|
||||
if (!customConfig) {
|
||||
return res.status(500).send('Custom config not found');
|
||||
}
|
||||
|
||||
const ttsSchema = customConfig.tts;
|
||||
const provider = getProvider(ttsSchema);
|
||||
|
||||
if (provider !== 'elevenlabs') {
|
||||
return res.status(400).send('WebSocket streaming is only supported for Eleven Labs');
|
||||
}
|
||||
|
||||
const url =
|
||||
ttsSchema.elevenlabs.websocketUrl ||
|
||||
'wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input?model_id={model}'
|
||||
.replace('{voice_id}', voice)
|
||||
.replace('{model}', ttsSchema.elevenlabs.model);
|
||||
const ws = new WebSocket(url);
|
||||
|
||||
ws.onopen = () => {
|
||||
logger.debug('WebSocket connection opened');
|
||||
sendTextToWebsocket(ws, (data) => {
|
||||
res.write(data); // Stream data directly to the response
|
||||
});
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
logger.debug('WebSocket connection closed');
|
||||
res.end(); // End the response when the WebSocket is closed
|
||||
};
|
||||
|
||||
ws.onerror = (error) => {
|
||||
logger.error('WebSocket error:', error);
|
||||
res.status(500).send('WebSocket error');
|
||||
};
|
||||
}
|
||||
*/
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {TCustomConfig} customConfig
|
||||
* @param {string} voice
|
||||
* @returns {Promise<ArrayBuffer>}
|
||||
*/
|
||||
async function ttsRequest(
|
||||
customConfig,
|
||||
{ input, voice: _v, stream = true } = { input: '', stream: true },
|
||||
) {
|
||||
const ttsSchema = customConfig.tts;
|
||||
const provider = getProvider(ttsSchema);
|
||||
const voices = ttsSchema[provider].voices.filter(
|
||||
(voice) => voice && voice.toUpperCase() !== 'ALL',
|
||||
);
|
||||
let voice = _v;
|
||||
if (!voice || !voices.includes(voice) || (voice.toUpperCase() === 'ALL' && voices.length > 1)) {
|
||||
voice = getRandomVoiceId(voices);
|
||||
}
|
||||
|
||||
let [url, data, headers] = [];
|
||||
|
||||
switch (provider) {
|
||||
case 'openai':
|
||||
[url, data, headers] = openAIProvider(ttsSchema, input, voice);
|
||||
break;
|
||||
case 'elevenlabs':
|
||||
[url, data, headers] = elevenLabsProvider(ttsSchema, input, voice, stream);
|
||||
break;
|
||||
case 'localai':
|
||||
[url, data, headers] = localAIProvider(ttsSchema, input, voice);
|
||||
break;
|
||||
default:
|
||||
throw new Error('Invalid provider');
|
||||
}
|
||||
|
||||
if (stream) {
|
||||
return await axios.post(url, data, { headers, responseType: 'stream' });
|
||||
}
|
||||
|
||||
return await axios.post(url, data, { headers, responseType: 'arraybuffer' });
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles a text-to-speech request. Extracts input and voice from the request, retrieves the TTS configuration,
|
||||
* and sends a request to the appropriate provider. The resulting audio data is sent in the response
|
||||
*
|
||||
* @param {Object} req - The request object, which should contain the input text and voice in its body
|
||||
* @param {Object} res - The response object, used to send the audio data or an error message
|
||||
*
|
||||
* @returns {Promise<void>} This function does not return a value. It sends the audio data or an error message in the response
|
||||
*
|
||||
* @throws {Error} Throws an error if the provider is invalid
|
||||
*/
|
||||
async function textToSpeech(req, res) {
|
||||
const { input, voice } = req.body;
|
||||
|
||||
if (!input) {
|
||||
return res.status(400).send('Missing text in request body');
|
||||
}
|
||||
|
||||
const customConfig = await getCustomConfig();
|
||||
if (!customConfig) {
|
||||
res.status(500).send('Custom config not found');
|
||||
}
|
||||
|
||||
try {
|
||||
res.setHeader('Content-Type', 'audio/mpeg');
|
||||
const response = await ttsRequest(customConfig, { input, voice });
|
||||
response.data.pipe(res);
|
||||
} catch (error) {
|
||||
logger.error('An error occurred while creating the audio stream:', error);
|
||||
res.status(500).send('An error occurred');
|
||||
}
|
||||
}
|
||||
|
||||
async function streamAudio(req, res) {
|
||||
res.setHeader('Content-Type', 'audio/mpeg');
|
||||
const customConfig = await getCustomConfig();
|
||||
if (!customConfig) {
|
||||
return res.status(500).send('Custom config not found');
|
||||
}
|
||||
|
||||
try {
|
||||
let shouldContinue = true;
|
||||
const processChunks = createChunkProcessor(req.body.messageId);
|
||||
|
||||
while (shouldContinue) {
|
||||
// example updates
|
||||
// const updates = [
|
||||
// { text: 'This is a test.', isFinished: false },
|
||||
// { text: 'This is only a test.', isFinished: false },
|
||||
// { text: 'Your voice is like a combination of Fergie and Jesus!', isFinished: true },
|
||||
// ];
|
||||
|
||||
const updates = await processChunks();
|
||||
if (typeof updates === 'string') {
|
||||
logger.error(`Error processing audio stream updates: ${JSON.stringify(updates)}`);
|
||||
res.status(500).end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (updates.length === 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1250));
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const update of updates) {
|
||||
try {
|
||||
const response = await ttsRequest(customConfig, {
|
||||
input: update.text,
|
||||
stream: true,
|
||||
});
|
||||
|
||||
if (!shouldContinue) {
|
||||
break;
|
||||
}
|
||||
|
||||
logger.debug(`[streamAudio] user: ${req?.user?.id} | writing audio stream`);
|
||||
await new Promise((resolve) => {
|
||||
response.data.pipe(res, { end: false });
|
||||
response.data.on('end', () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
if (update.isFinished) {
|
||||
shouldContinue = false;
|
||||
break;
|
||||
}
|
||||
} catch (innerError) {
|
||||
logger.error('Error processing update:', update, innerError);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).end();
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!shouldContinue) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!res.headersSent) {
|
||||
res.end();
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to fetch audio:', error);
|
||||
if (!res.headersSent) {
|
||||
res.status(500).end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
textToSpeech,
|
||||
getProvider,
|
||||
streamAudio,
|
||||
};
|
||||
31
api/server/services/Files/Audio/webSocket.js
Normal file
31
api/server/services/Files/Audio/webSocket.js
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
let token = '';
|
||||
|
||||
function updateTokenWebsocket(newToken) {
|
||||
console.log('Token:', newToken);
|
||||
token = newToken;
|
||||
}
|
||||
|
||||
function sendTextToWebsocket(ws, onDataReceived) {
|
||||
if (token === '[DONE]') {
|
||||
ws.send(' ');
|
||||
return;
|
||||
}
|
||||
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(token);
|
||||
|
||||
ws.onmessage = function (event) {
|
||||
console.log('Received:', event.data);
|
||||
if (onDataReceived) {
|
||||
onDataReceived(event.data); // Pass the received data to the callback function
|
||||
}
|
||||
};
|
||||
} else {
|
||||
console.error('WebSocket is not open. Ready state is: ' + ws.readyState);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
updateTokenWebsocket,
|
||||
sendTextToWebsocket,
|
||||
};
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
const throttle = require('lodash/throttle');
|
||||
const {
|
||||
StepTypes,
|
||||
ContentTypes,
|
||||
|
|
@ -10,6 +11,7 @@ const { retrieveAndProcessFile } = require('~/server/services/Files/process');
|
|||
const { processRequiredActions } = require('~/server/services/ToolService');
|
||||
const { createOnProgress, sendMessage } = require('~/server/utils');
|
||||
const { processMessages } = require('~/server/services/Threads');
|
||||
const { saveMessage } = require('~/models');
|
||||
const { logger } = require('~/config');
|
||||
|
||||
/**
|
||||
|
|
@ -43,6 +45,8 @@ class StreamRunManager {
|
|||
/** @type {string} */
|
||||
this.apiKey = this.openai.apiKey;
|
||||
/** @type {string} */
|
||||
this.parentMessageId = fields.parentMessageId;
|
||||
/** @type {string} */
|
||||
this.thread_id = fields.thread_id;
|
||||
/** @type {RunCreateAndStreamParams} */
|
||||
this.initialRunBody = fields.runBody;
|
||||
|
|
@ -58,6 +62,8 @@ class StreamRunManager {
|
|||
this.messages = [];
|
||||
/** @type {string} */
|
||||
this.text = '';
|
||||
/** @type {string} */
|
||||
this.intermediateText = '';
|
||||
/** @type {Set<string>} */
|
||||
this.attachedFileIds = fields.attachedFileIds;
|
||||
/** @type {undefined | Promise<ChatCompletion>} */
|
||||
|
|
@ -407,6 +413,7 @@ class StreamRunManager {
|
|||
const content = message.delta.content?.[0];
|
||||
|
||||
if (content && content.type === MessageContentTypes.TEXT) {
|
||||
this.intermediateText += content.text.value;
|
||||
onProgress(content.text.value);
|
||||
}
|
||||
}
|
||||
|
|
@ -523,10 +530,27 @@ class StreamRunManager {
|
|||
const stepKey = message_creation.message_id;
|
||||
const index = this.getStepIndex(stepKey);
|
||||
this.orderedRunSteps.set(index, message_creation);
|
||||
const getText = () => this.intermediateText;
|
||||
// Create the Factory Function to stream the message
|
||||
const { onProgress: progressCallback } = createOnProgress({
|
||||
// todo: add option to save partialText to db
|
||||
// onProgress: () => {},
|
||||
onProgress: throttle(
|
||||
() => {
|
||||
const text = getText();
|
||||
saveMessage({
|
||||
messageId: this.finalMessage.messageId,
|
||||
conversationId: this.finalMessage.conversationId,
|
||||
parentMessageId: this.parentMessageId,
|
||||
model: this.req.body.model,
|
||||
user: this.req.user.id,
|
||||
sender: 'Assistant',
|
||||
unfinished: true,
|
||||
error: false,
|
||||
text,
|
||||
});
|
||||
},
|
||||
2000,
|
||||
{ trailing: false },
|
||||
),
|
||||
});
|
||||
|
||||
// This creates a function that attaches all of the parameters
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ async function saveUserMessage(params) {
|
|||
* @param {Object} params - The parameters of the Assistant message
|
||||
* @param {string} params.user - The user's ID.
|
||||
* @param {string} params.messageId - The message Id.
|
||||
* @param {string} params.text - The concatenated text of the message.
|
||||
* @param {string} params.assistant_id - The assistant Id.
|
||||
* @param {string} params.thread_id - The thread Id.
|
||||
* @param {string} params.model - The model used by the assistant.
|
||||
|
|
@ -134,14 +135,6 @@ async function saveUserMessage(params) {
|
|||
* @return {Promise<Run>} A promise that resolves to the created run object.
|
||||
*/
|
||||
async function saveAssistantMessage(params) {
|
||||
const text = params.content.reduce((acc, part) => {
|
||||
if (!part.value) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
return acc + ' ' + part.value;
|
||||
}, '');
|
||||
|
||||
// const tokenCount = // TODO: need to count each content part
|
||||
|
||||
const message = await recordMessage({
|
||||
|
|
@ -156,7 +149,8 @@ async function saveAssistantMessage(params) {
|
|||
content: params.content,
|
||||
sender: 'Assistant',
|
||||
isCreatedByUser: false,
|
||||
text: text.trim(),
|
||||
text: params.text,
|
||||
unfinished: false,
|
||||
// tokenCount,
|
||||
});
|
||||
|
||||
|
|
@ -302,6 +296,7 @@ async function syncMessages({
|
|||
aggregateMessages: [{ id: apiMessage.id }],
|
||||
model: apiMessage.role === 'user' ? null : apiMessage.assistant_id,
|
||||
user: openai.req.user.id,
|
||||
unfinished: false,
|
||||
};
|
||||
|
||||
if (apiMessage.file_ids?.length) {
|
||||
|
|
|
|||
|
|
@ -79,9 +79,11 @@
|
|||
"react-markdown": "^8.0.6",
|
||||
"react-resizable-panels": "^1.0.9",
|
||||
"react-router-dom": "^6.11.2",
|
||||
"react-speech-recognition": "^3.10.0",
|
||||
"react-textarea-autosize": "^8.4.0",
|
||||
"react-transition-group": "^4.4.5",
|
||||
"recoil": "^0.7.7",
|
||||
"regenerator-runtime": "^0.14.1",
|
||||
"rehype-highlight": "^6.0.0",
|
||||
"rehype-katex": "^6.0.2",
|
||||
"rehype-raw": "^6.1.1",
|
||||
|
|
|
|||
|
|
@ -21,6 +21,21 @@ import type {
|
|||
import type { UseMutationResult } from '@tanstack/react-query';
|
||||
import type { LucideIcon } from 'lucide-react';
|
||||
|
||||
export type AudioChunk = {
|
||||
audio: string;
|
||||
isFinal: boolean;
|
||||
alignment: {
|
||||
char_start_times_ms: number[];
|
||||
chars_durations_ms: number[];
|
||||
chars: string[];
|
||||
};
|
||||
normalizedAlignment: {
|
||||
char_start_times_ms: number[];
|
||||
chars_durations_ms: number[];
|
||||
chars: string[];
|
||||
};
|
||||
};
|
||||
|
||||
export type AssistantListItem = {
|
||||
id: string;
|
||||
name: string;
|
||||
|
|
@ -37,6 +52,7 @@ export type LastSelectedModels = Record<EModelEndpoint, string>;
|
|||
export type LocalizeFunction = (phraseKey: string, ...values: string[]) => string;
|
||||
|
||||
export const mainTextareaId = 'prompt-textarea';
|
||||
export const globalAudioId = 'global-audio';
|
||||
|
||||
export enum IconContext {
|
||||
landing = 'landing',
|
||||
|
|
|
|||
47
client/src/components/Chat/Input/AudioRecorder.tsx
Normal file
47
client/src/components/Chat/Input/AudioRecorder.tsx
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import React from 'react';
|
||||
import { ListeningIcon, Spinner, SpeechIcon } from '~/components/svg';
|
||||
import { TooltipProvider, Tooltip, TooltipTrigger, TooltipContent } from '~/components/ui/';
|
||||
import { useLocalize } from '~/hooks';
|
||||
|
||||
export default function AudioRecorder({
|
||||
isListening,
|
||||
isLoading,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
disabled,
|
||||
}) {
|
||||
const localize = useLocalize();
|
||||
const handleStartRecording = async () => {
|
||||
await startRecording();
|
||||
};
|
||||
|
||||
const handleStopRecording = async () => {
|
||||
await stopRecording();
|
||||
};
|
||||
|
||||
return (
|
||||
<TooltipProvider delayDuration={250}>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
onClick={isListening ? handleStopRecording : handleStartRecording}
|
||||
disabled={disabled}
|
||||
className="absolute bottom-1.5 right-12 flex h-[30px] w-[30px] items-center justify-center rounded-lg p-0.5 transition-colors hover:bg-gray-200 dark:hover:bg-gray-700 md:bottom-3 md:right-12"
|
||||
type="button"
|
||||
>
|
||||
{isListening ? (
|
||||
<SpeechIcon className="stroke-gray-700 dark:stroke-gray-300" />
|
||||
) : isLoading ? (
|
||||
<Spinner className="stroke-gray-700 dark:stroke-gray-300" />
|
||||
) : (
|
||||
<ListeningIcon className="stroke-gray-700 dark:stroke-gray-300" />
|
||||
)}
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top" sideOffset={10}>
|
||||
{localize('com_ui_use_micrphone')}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { memo, useCallback, useRef, useMemo } from 'react';
|
||||
import { useRecoilState, useRecoilValue } from 'recoil';
|
||||
import { memo, useCallback, useRef, useMemo, useEffect } from 'react';
|
||||
import {
|
||||
supportsFiles,
|
||||
mergeFileConfig,
|
||||
|
|
@ -8,12 +8,14 @@ import {
|
|||
fileConfig as defaultFileConfig,
|
||||
} from 'librechat-data-provider';
|
||||
import { useChatContext, useAssistantsMapContext } from '~/Providers';
|
||||
import { useRequiresKey, useTextarea } from '~/hooks';
|
||||
import { useRequiresKey, useTextarea, useSpeechToText } from '~/hooks';
|
||||
import { TextareaAutosize } from '~/components/ui';
|
||||
import { useGetFileConfig } from '~/data-provider';
|
||||
import { cn, removeFocusOutlines } from '~/utils';
|
||||
import AttachFile from './Files/AttachFile';
|
||||
import AudioRecorder from './AudioRecorder';
|
||||
import { mainTextareaId } from '~/common';
|
||||
import StreamAudio from './StreamAudio';
|
||||
import StopButton from './StopButton';
|
||||
import SendButton from './SendButton';
|
||||
import FileRow from './Files/FileRow';
|
||||
|
|
@ -23,6 +25,9 @@ import store from '~/store';
|
|||
const ChatForm = ({ index = 0 }) => {
|
||||
const submitButtonRef = useRef<HTMLButtonElement>(null);
|
||||
const textAreaRef = useRef<HTMLTextAreaElement | null>(null);
|
||||
const SpeechToText = useRecoilValue(store.SpeechToText);
|
||||
const TextToSpeech = useRecoilValue(store.TextToSpeech);
|
||||
const automaticPlayback = useRecoilValue(store.automaticPlayback);
|
||||
const [showStopButton, setShowStopButton] = useRecoilState(store.showStopButtonByIndex(index));
|
||||
const [showMentionPopover, setShowMentionPopover] = useRecoilState(
|
||||
store.showMentionPopoverFamily(index),
|
||||
|
|
@ -67,6 +72,24 @@ const ChatForm = ({ index = 0 }) => {
|
|||
const { endpoint: _endpoint, endpointType } = conversation ?? { endpoint: null };
|
||||
const endpoint = endpointType ?? _endpoint;
|
||||
|
||||
const handleTranscriptionComplete = (text: string) => {
|
||||
if (text) {
|
||||
ask({ text });
|
||||
methods.reset({ text: '' });
|
||||
clearText();
|
||||
}
|
||||
};
|
||||
|
||||
const { isListening, isLoading, startRecording, stopRecording, speechText, clearText } =
|
||||
useSpeechToText(handleTranscriptionComplete);
|
||||
|
||||
useEffect(() => {
|
||||
if (textAreaRef.current) {
|
||||
textAreaRef.current.value = speechText;
|
||||
methods.setValue('text', speechText, { shouldValidate: true });
|
||||
}
|
||||
}, [speechText, methods]);
|
||||
|
||||
const { data: fileConfig = defaultFileConfig } = useGetFileConfig({
|
||||
select: (data) => mergeFileConfig(data),
|
||||
});
|
||||
|
|
@ -87,7 +110,7 @@ const ChatForm = ({ index = 0 }) => {
|
|||
const { ref, ...registerProps } = methods.register('text', {
|
||||
required: true,
|
||||
onChange: (e) => {
|
||||
methods.setValue('text', e.target.value);
|
||||
methods.setValue('text', e.target.value, { shouldValidate: true });
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -135,7 +158,8 @@ const ChatForm = ({ index = 0 }) => {
|
|||
supportsFiles[endpointType ?? endpoint ?? ''] && !endpointFileConfig?.disabled
|
||||
? ' pl-10 md:pl-[55px]'
|
||||
: 'pl-3 md:pl-4',
|
||||
'm-0 w-full resize-none border-0 bg-transparent py-[10px] pr-10 placeholder-black/50 focus:ring-0 focus-visible:ring-0 dark:bg-transparent dark:placeholder-white/50 md:py-3.5 md:pr-12 ',
|
||||
'm-0 w-full resize-none border-0 bg-transparent py-[10px] placeholder-black/50 focus:ring-0 focus-visible:ring-0 dark:bg-transparent dark:placeholder-white/50 md:py-3.5 ',
|
||||
SpeechToText ? 'pr-20 md:pr-[85px]' : 'pr-10 md:pr-12',
|
||||
removeFocusOutlines,
|
||||
'max-h-[65vh] md:max-h-[75vh]',
|
||||
)}
|
||||
|
|
@ -157,6 +181,16 @@ const ChatForm = ({ index = 0 }) => {
|
|||
/>
|
||||
)
|
||||
)}
|
||||
{SpeechToText && (
|
||||
<AudioRecorder
|
||||
isListening={isListening}
|
||||
isLoading={isLoading}
|
||||
startRecording={startRecording}
|
||||
stopRecording={stopRecording}
|
||||
disabled={!!disableInputs}
|
||||
/>
|
||||
)}
|
||||
{TextToSpeech && automaticPlayback && <StreamAudio index={index} />}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -225,7 +225,7 @@ export default function DataTable<TData, TValue>({ columns, data }: DataTablePro
|
|||
)}
|
||||
</div>
|
||||
<Button
|
||||
className="dark:border-gray-500 dark:hover:bg-gray-600 select-none"
|
||||
className="select-none dark:border-gray-500 dark:hover:bg-gray-600"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => table.previousPage()}
|
||||
|
|
@ -234,7 +234,7 @@ export default function DataTable<TData, TValue>({ columns, data }: DataTablePro
|
|||
{localize('com_ui_prev')}
|
||||
</Button>
|
||||
<Button
|
||||
className="dark:border-gray-500 dark:hover:bg-gray-600 select-none"
|
||||
className="select-none dark:border-gray-500 dark:hover:bg-gray-600"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => table.nextPage()}
|
||||
|
|
|
|||
194
client/src/components/Chat/Input/StreamAudio.tsx
Normal file
194
client/src/components/Chat/Input/StreamAudio.tsx
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
import { useParams } from 'react-router-dom';
|
||||
import { QueryKeys } from 'librechat-data-provider';
|
||||
import { useQueryClient } from '@tanstack/react-query';
|
||||
import { useEffect, useCallback } from 'react';
|
||||
import { useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil';
|
||||
import type { TMessage } from 'librechat-data-provider';
|
||||
import { useCustomAudioRef, MediaSourceAppender } from '~/hooks/Audio';
|
||||
import { useAuthContext } from '~/hooks';
|
||||
import { globalAudioId } from '~/common';
|
||||
import store from '~/store';
|
||||
|
||||
function timeoutPromise(ms: number, message?: string) {
|
||||
return new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error(message ?? 'Promise timed out')), ms),
|
||||
);
|
||||
}
|
||||
|
||||
const promiseTimeoutMessage = 'Reader promise timed out';
|
||||
const maxPromiseTime = 15000;
|
||||
|
||||
export default function StreamAudio({ index = 0 }) {
|
||||
const { token } = useAuthContext();
|
||||
|
||||
const cacheTTS = useRecoilValue(store.cacheTTS);
|
||||
const playbackRate = useRecoilValue(store.playbackRate);
|
||||
|
||||
const activeRunId = useRecoilValue(store.activeRunFamily(index));
|
||||
const automaticPlayback = useRecoilValue(store.automaticPlayback);
|
||||
const isSubmitting = useRecoilValue(store.isSubmittingFamily(index));
|
||||
const latestMessage = useRecoilValue(store.latestMessageFamily(index));
|
||||
const setIsPlaying = useSetRecoilState(store.globalAudioPlayingFamily(index));
|
||||
const [audioRunId, setAudioRunId] = useRecoilState(store.audioRunFamily(index));
|
||||
const [isFetching, setIsFetching] = useRecoilState(store.globalAudioFetchingFamily(index));
|
||||
const [globalAudioURL, setGlobalAudioURL] = useRecoilState(store.globalAudioURLFamily(index));
|
||||
|
||||
const { audioRef } = useCustomAudioRef({ setIsPlaying });
|
||||
|
||||
const { conversationId: paramId } = useParams();
|
||||
const queryParam = paramId === 'new' ? paramId : latestMessage?.conversationId ?? paramId ?? '';
|
||||
|
||||
const queryClient = useQueryClient();
|
||||
const getMessages = useCallback(
|
||||
() => queryClient.getQueryData<TMessage[]>([QueryKeys.messages, queryParam]),
|
||||
[queryParam, queryClient],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const shouldFetch =
|
||||
token &&
|
||||
automaticPlayback &&
|
||||
isSubmitting &&
|
||||
latestMessage &&
|
||||
!latestMessage.isCreatedByUser &&
|
||||
(latestMessage.text || latestMessage.content) &&
|
||||
latestMessage.messageId &&
|
||||
!latestMessage.messageId.includes('_') &&
|
||||
!isFetching &&
|
||||
activeRunId &&
|
||||
activeRunId !== audioRunId;
|
||||
|
||||
if (!shouldFetch) {
|
||||
return;
|
||||
}
|
||||
|
||||
async function fetchAudio() {
|
||||
setIsFetching(true);
|
||||
|
||||
try {
|
||||
if (audioRef.current) {
|
||||
audioRef.current.pause();
|
||||
URL.revokeObjectURL(audioRef.current.src);
|
||||
setGlobalAudioURL(null);
|
||||
}
|
||||
|
||||
let cacheKey = latestMessage?.text ?? '';
|
||||
const cache = await caches.open('tts-responses');
|
||||
const cachedResponse = await cache.match(cacheKey);
|
||||
|
||||
if (cachedResponse) {
|
||||
console.log('Audio found in cache');
|
||||
const audioBlob = await cachedResponse.blob();
|
||||
const blobUrl = URL.createObjectURL(audioBlob);
|
||||
setGlobalAudioURL(blobUrl);
|
||||
setAudioRunId(activeRunId);
|
||||
setIsFetching(false);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('Fetching audio...');
|
||||
const response = await fetch('/api/files/tts', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` },
|
||||
body: JSON.stringify({ messageId: latestMessage?.messageId, runId: activeRunId }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to fetch audio');
|
||||
}
|
||||
if (!response.body) {
|
||||
throw new Error('Null Response body');
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const mediaSource = new MediaSourceAppender('audio/mpeg');
|
||||
setGlobalAudioURL(mediaSource.mediaSourceUrl);
|
||||
setAudioRunId(activeRunId);
|
||||
|
||||
let done = false;
|
||||
const chunks: Uint8Array[] = [];
|
||||
|
||||
while (!done) {
|
||||
const readPromise = reader.read();
|
||||
const { value, done: readerDone } = (await Promise.race([
|
||||
readPromise,
|
||||
timeoutPromise(maxPromiseTime, promiseTimeoutMessage),
|
||||
])) as ReadableStreamReadResult<Uint8Array>;
|
||||
|
||||
if (cacheTTS && value) {
|
||||
chunks.push(value);
|
||||
}
|
||||
if (value) {
|
||||
mediaSource.addData(value);
|
||||
}
|
||||
done = readerDone;
|
||||
}
|
||||
|
||||
if (chunks.length) {
|
||||
console.log('Adding audio to cache');
|
||||
const latestMessages = getMessages() ?? [];
|
||||
const targetMessage = latestMessages.find(
|
||||
(msg) => msg.messageId === latestMessage?.messageId,
|
||||
);
|
||||
cacheKey = targetMessage?.text ?? '';
|
||||
if (!cacheKey) {
|
||||
throw new Error('Cache key not found');
|
||||
}
|
||||
const audioBlob = new Blob(chunks, { type: 'audio/mpeg' });
|
||||
cache.put(cacheKey, new Response(audioBlob));
|
||||
}
|
||||
|
||||
console.log('Audio stream reading ended');
|
||||
} catch (error) {
|
||||
if (error?.['message'] !== promiseTimeoutMessage) {
|
||||
console.log(promiseTimeoutMessage);
|
||||
return;
|
||||
}
|
||||
console.error('Error fetching audio:', error);
|
||||
setIsFetching(false);
|
||||
setGlobalAudioURL(null);
|
||||
} finally {
|
||||
setIsFetching(false);
|
||||
}
|
||||
}
|
||||
|
||||
fetchAudio();
|
||||
}, [
|
||||
automaticPlayback,
|
||||
setGlobalAudioURL,
|
||||
setAudioRunId,
|
||||
setIsFetching,
|
||||
latestMessage,
|
||||
isSubmitting,
|
||||
activeRunId,
|
||||
getMessages,
|
||||
isFetching,
|
||||
audioRunId,
|
||||
cacheTTS,
|
||||
audioRef,
|
||||
token,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
if (
|
||||
playbackRate &&
|
||||
globalAudioURL &&
|
||||
audioRef.current &&
|
||||
audioRef.current.playbackRate !== playbackRate
|
||||
) {
|
||||
audioRef.current.playbackRate = playbackRate;
|
||||
}
|
||||
}, [audioRef, globalAudioURL, playbackRate]);
|
||||
|
||||
return (
|
||||
<audio
|
||||
ref={audioRef}
|
||||
controls
|
||||
controlsList="nodownload nofullscreen noremoteplayback"
|
||||
className="absolute h-0 w-0 overflow-hidden"
|
||||
src={globalAudioURL || undefined}
|
||||
id={globalAudioId}
|
||||
autoPlay
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
@ -35,13 +35,14 @@ any) => {
|
|||
/>
|
||||
);
|
||||
})}
|
||||
{!isSubmitting && unfinished && (
|
||||
{/* Temporarily remove this */}
|
||||
{/* {!isSubmitting && unfinished && (
|
||||
<Suspense>
|
||||
<DelayedRender delay={250}>
|
||||
<UnfinishedMessage message={message} key={`unfinished-${messageId}`} />
|
||||
</DelayedRender>
|
||||
</Suspense>
|
||||
)}
|
||||
)} */}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,20 @@
|
|||
import { useState } from 'react';
|
||||
import React, { useState } from 'react';
|
||||
import { useRecoilState } from 'recoil';
|
||||
import type { TConversation, TMessage } from 'librechat-data-provider';
|
||||
import { Clipboard, CheckMark, EditIcon, RegenerateIcon, ContinueIcon } from '~/components/svg';
|
||||
import { useGenerationsByLatest, useLocalize } from '~/hooks';
|
||||
import {
|
||||
Clipboard,
|
||||
CheckMark,
|
||||
EditIcon,
|
||||
RegenerateIcon,
|
||||
ContinueIcon,
|
||||
VolumeIcon,
|
||||
VolumeMuteIcon,
|
||||
Spinner,
|
||||
} from '~/components/svg';
|
||||
import { useGenerationsByLatest, useLocalize, useTextToSpeech } from '~/hooks';
|
||||
import { Fork } from '~/components/Conversations';
|
||||
import { cn } from '~/utils';
|
||||
import store from '~/store';
|
||||
|
||||
type THoverButtons = {
|
||||
isEditing: boolean;
|
||||
|
|
@ -16,9 +27,11 @@ type THoverButtons = {
|
|||
handleContinue: (e: React.MouseEvent<HTMLButtonElement>) => void;
|
||||
latestMessage: TMessage | null;
|
||||
isLast: boolean;
|
||||
index: number;
|
||||
};
|
||||
|
||||
export default function HoverButtons({
|
||||
index,
|
||||
isEditing,
|
||||
enterEdit,
|
||||
copyToClipboard,
|
||||
|
|
@ -34,6 +47,14 @@ export default function HoverButtons({
|
|||
const { endpoint: _endpoint, endpointType } = conversation ?? {};
|
||||
const endpoint = endpointType ?? _endpoint;
|
||||
const [isCopied, setIsCopied] = useState(false);
|
||||
const [TextToSpeech] = useRecoilState<boolean>(store.TextToSpeech);
|
||||
|
||||
const { handleMouseDown, handleMouseUp, toggleSpeech, isSpeaking, isLoading } = useTextToSpeech(
|
||||
message?.text ?? '',
|
||||
isLast,
|
||||
index,
|
||||
);
|
||||
|
||||
const {
|
||||
hideEditButton,
|
||||
regenerateEnabled,
|
||||
|
|
@ -60,15 +81,39 @@ export default function HoverButtons({
|
|||
enterEdit();
|
||||
};
|
||||
|
||||
const renderIcon = (size: string) => {
|
||||
if (isLoading) {
|
||||
return <Spinner size={size} />;
|
||||
}
|
||||
|
||||
if (isSpeaking) {
|
||||
return <VolumeMuteIcon size={size} />;
|
||||
}
|
||||
|
||||
return <VolumeIcon size={size} />;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="visible mt-0 flex justify-center gap-1 self-end text-gray-400 lg:justify-start">
|
||||
{TextToSpeech && (
|
||||
<button
|
||||
className="hover-button rounded-md p-1 pl-0 text-gray-400 hover:text-gray-950 dark:text-gray-400/70 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400 md:group-hover:visible md:group-[.final-completion]:visible"
|
||||
onMouseDown={handleMouseDown}
|
||||
onMouseUp={handleMouseUp}
|
||||
onClick={toggleSpeech}
|
||||
type="button"
|
||||
title={isSpeaking ? localize('com_ui_stop') : localize('com_ui_read_aloud')}
|
||||
>
|
||||
{renderIcon('19')}
|
||||
</button>
|
||||
)}
|
||||
{isEditableEndpoint && (
|
||||
<button
|
||||
className={cn(
|
||||
'hover-button rounded-md p-1 text-gray-400 hover:text-gray-900 dark:text-gray-400/70 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400 md:group-hover:visible md:group-[.final-completion]:visible',
|
||||
isCreatedByUser ? '' : 'active',
|
||||
hideEditButton ? 'opacity-0' : '',
|
||||
isEditing ? 'active bg-gray-200 text-gray-700 dark:bg-gray-700 dark:text-gray-200' : '',
|
||||
isEditing ? 'active text-gray-700 dark:text-gray-200' : '',
|
||||
!isLast ? 'md:opacity-0 md:group-hover:opacity-100' : '',
|
||||
)}
|
||||
onClick={onEdit}
|
||||
|
|
@ -76,7 +121,7 @@ export default function HoverButtons({
|
|||
title={localize('com_ui_edit')}
|
||||
disabled={hideEditButton}
|
||||
>
|
||||
<EditIcon />
|
||||
<EditIcon size="19" />
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
|
|
@ -91,7 +136,7 @@ export default function HoverButtons({
|
|||
isCopied ? localize('com_ui_copied_to_clipboard') : localize('com_ui_copy_to_clipboard')
|
||||
}
|
||||
>
|
||||
{isCopied ? <CheckMark className="h-[18px] w-[18px]" /> : <Clipboard />}
|
||||
{isCopied ? <CheckMark className="h-[18px] w-[18px]" /> : <Clipboard size="19" />}
|
||||
</button>
|
||||
{regenerateEnabled ? (
|
||||
<button
|
||||
|
|
@ -103,7 +148,10 @@ export default function HoverButtons({
|
|||
type="button"
|
||||
title={localize('com_ui_regenerate')}
|
||||
>
|
||||
<RegenerateIcon className="hover:text-gray-700 dark:hover:bg-gray-700 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400" />
|
||||
<RegenerateIcon
|
||||
className="hover:text-gray-700 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400"
|
||||
size="19"
|
||||
/>
|
||||
</button>
|
||||
) : null}
|
||||
<Fork
|
||||
|
|
@ -116,14 +164,14 @@ export default function HoverButtons({
|
|||
{continueSupported ? (
|
||||
<button
|
||||
className={cn(
|
||||
'hover-button active rounded-md p-1 hover:bg-gray-200 hover:text-gray-700 dark:text-gray-400/70 dark:hover:bg-gray-700 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400 md:invisible md:group-hover:visible ',
|
||||
'hover-button active rounded-md p-1 hover:bg-gray-200 hover:text-gray-700 dark:text-gray-400/70 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400 md:invisible md:group-hover:visible ',
|
||||
!isLast ? 'md:opacity-0 md:group-hover:opacity-100' : '',
|
||||
)}
|
||||
onClick={handleContinue}
|
||||
type="button"
|
||||
title={localize('com_ui_continue')}
|
||||
>
|
||||
<ContinueIcon className="h-4 w-4 hover:text-gray-700 dark:hover:bg-gray-700 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400" />
|
||||
<ContinueIcon className="h-4 w-4 hover:text-gray-700 dark:hover:text-gray-200 disabled:dark:hover:text-gray-400" />
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ export default function Message(props: TMessageProps) {
|
|||
const {
|
||||
ask,
|
||||
edit,
|
||||
index,
|
||||
isLast,
|
||||
enterEdit,
|
||||
handleScroll,
|
||||
|
|
@ -102,6 +103,7 @@ export default function Message(props: TMessageProps) {
|
|||
setSiblingIdx={setSiblingIdx}
|
||||
/>
|
||||
<HoverButtons
|
||||
index={index}
|
||||
isEditing={edit}
|
||||
message={message}
|
||||
enterEdit={enterEdit}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ export default function Message(props: TMessageProps) {
|
|||
const {
|
||||
ask,
|
||||
edit,
|
||||
index,
|
||||
isLast,
|
||||
enterEdit,
|
||||
assistant,
|
||||
|
|
@ -90,6 +91,7 @@ export default function Message(props: TMessageProps) {
|
|||
setSiblingIdx={setSiblingIdx}
|
||||
/>
|
||||
<HoverButtons
|
||||
index={index}
|
||||
isEditing={edit}
|
||||
message={message}
|
||||
enterEdit={enterEdit}
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
|
|||
</HoverCard>
|
||||
<div className="grid w-full grid-cols-2 items-center gap-10">
|
||||
<HoverCard openDelay={500}>
|
||||
<HoverCardTrigger className="w-[100px] flex flex-col items-center text-center space-y-4">
|
||||
<HoverCardTrigger className="flex w-[100px] flex-col items-center space-y-4 text-center">
|
||||
<label
|
||||
htmlFor="functions-agent"
|
||||
className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"
|
||||
|
|
@ -106,7 +106,7 @@ export default function Settings({ conversation, setOption, models, readonly }:
|
|||
<OptionHover endpoint={conversation.endpoint ?? ''} type="func" side={ESide.Bottom} />
|
||||
</HoverCard>
|
||||
<HoverCard openDelay={500}>
|
||||
<HoverCardTrigger className="ml-[-60px] w-[100px] flex flex-col items-center text-center space-y-4">
|
||||
<HoverCardTrigger className="ml-[-60px] flex w-[100px] flex-col items-center space-y-4 text-center">
|
||||
<label
|
||||
htmlFor="skip-completion"
|
||||
className="text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70 dark:text-gray-50"
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ export default function Regenerate({ onClick }: TGenButtonProps) {
|
|||
return (
|
||||
<Button onClick={onClick}>
|
||||
<RegenerateIcon className="h-3 w-3 flex-shrink-0 text-gray-600/90 dark:text-gray-400" />
|
||||
{localize('com_ui_regenerate')}
|
||||
{localize('com_ui_regenerate')}
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ export default function Stop({ onClick }: TGenButtonProps) {
|
|||
return (
|
||||
<Button type="stop" onClick={onClick}>
|
||||
<StopGeneratingIcon className="text-gray-600/90 dark:text-gray-400 " />
|
||||
{localize('com_ui_stop')}
|
||||
{localize('com_ui_stop')}
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ import { MessageSquare } from 'lucide-react';
|
|||
import { SettingsTabValues } from 'librechat-data-provider';
|
||||
import type { TDialogProps } from '~/common';
|
||||
import { Dialog, DialogContent, DialogHeader, DialogTitle } from '~/components/ui';
|
||||
import { GearIcon, DataIcon, UserIcon, ExperimentIcon } from '~/components/svg';
|
||||
import { General, Messages, Beta, Data, Account } from './SettingsTabs';
|
||||
import { GearIcon, DataIcon, SpeechIcon, UserIcon, ExperimentIcon } from '~/components/svg';
|
||||
import { General, Messages, Speech, Beta, Data, Account } from './SettingsTabs';
|
||||
import { useMediaQuery, useLocalize } from '~/hooks';
|
||||
import { cn } from '~/utils';
|
||||
|
||||
|
|
@ -83,6 +83,20 @@ export default function Settings({ open, onOpenChange }: TDialogProps) {
|
|||
<ExperimentIcon />
|
||||
{localize('com_nav_setting_beta')}
|
||||
</Tabs.Trigger>
|
||||
<Tabs.Trigger
|
||||
className={cn(
|
||||
'group m-1 flex items-center justify-start gap-2 rounded-md px-2 py-1.5 text-sm text-black radix-state-active:bg-white radix-state-active:text-black dark:text-white dark:radix-state-active:bg-gray-600',
|
||||
isSmallScreen
|
||||
? 'flex-1 flex-col items-center justify-center text-sm dark:text-gray-500 dark:radix-state-active:text-white'
|
||||
: 'bg-white radix-state-active:bg-gray-200',
|
||||
isSmallScreen ? '' : 'dark:bg-gray-700',
|
||||
)}
|
||||
value={SettingsTabValues.SPEECH}
|
||||
style={{ userSelect: 'none' }}
|
||||
>
|
||||
<SpeechIcon className="icon-sm" />
|
||||
{localize('com_nav_setting_speech')}
|
||||
</Tabs.Trigger>
|
||||
<Tabs.Trigger
|
||||
className={cn(
|
||||
'group m-1 flex items-center justify-start gap-2 rounded-md px-2 py-1.5 text-sm text-black radix-state-active:bg-white radix-state-active:text-black dark:text-white dark:radix-state-active:bg-gray-600',
|
||||
|
|
@ -116,6 +130,7 @@ export default function Settings({ open, onOpenChange }: TDialogProps) {
|
|||
<General />
|
||||
<Messages />
|
||||
<Beta />
|
||||
<Speech />
|
||||
<Data />
|
||||
<Account />
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,75 +1,14 @@
|
|||
import React, { useState, useRef } from 'react';
|
||||
import * as Tabs from '@radix-ui/react-tabs';
|
||||
import {
|
||||
useRevokeUserKeyMutation,
|
||||
useRevokeAllUserKeysMutation,
|
||||
useClearConversationsMutation,
|
||||
} from 'librechat-data-provider/react-query';
|
||||
import { useClearConversationsMutation } from 'librechat-data-provider/react-query';
|
||||
import { SettingsTabValues } from 'librechat-data-provider';
|
||||
import React, { useState, useCallback, useRef } from 'react';
|
||||
import { useConversation, useConversations, useOnClickOutside } from '~/hooks';
|
||||
import { RevokeKeysButton } from './RevokeKeysButton';
|
||||
import { DeleteCacheButton } from './DeleteCacheButton';
|
||||
import ImportConversations from './ImportConversations';
|
||||
import { ClearChatsButton } from './ClearChats';
|
||||
import DangerButton from '../DangerButton';
|
||||
import SharedLinks from './SharedLinks';
|
||||
|
||||
export const RevokeKeysButton = ({
|
||||
showText = true,
|
||||
endpoint = '',
|
||||
all = false,
|
||||
disabled = false,
|
||||
}: {
|
||||
showText?: boolean;
|
||||
endpoint?: string;
|
||||
all?: boolean;
|
||||
disabled?: boolean;
|
||||
}) => {
|
||||
const [confirmRevoke, setConfirmRevoke] = useState(false);
|
||||
|
||||
const revokeKeysMutation = useRevokeAllUserKeysMutation();
|
||||
const revokeKeyMutation = useRevokeUserKeyMutation(endpoint);
|
||||
|
||||
const revokeContentRef = useRef(null);
|
||||
useOnClickOutside(revokeContentRef, () => confirmRevoke && setConfirmRevoke(false), []);
|
||||
|
||||
const revokeAllUserKeys = useCallback(() => {
|
||||
if (confirmRevoke) {
|
||||
revokeKeysMutation.mutate({});
|
||||
setConfirmRevoke(false);
|
||||
} else {
|
||||
setConfirmRevoke(true);
|
||||
}
|
||||
}, [confirmRevoke, revokeKeysMutation]);
|
||||
|
||||
const revokeUserKey = useCallback(() => {
|
||||
if (!endpoint) {
|
||||
return;
|
||||
} else if (confirmRevoke) {
|
||||
revokeKeyMutation.mutate({});
|
||||
setConfirmRevoke(false);
|
||||
} else {
|
||||
setConfirmRevoke(true);
|
||||
}
|
||||
}, [confirmRevoke, revokeKeyMutation, endpoint]);
|
||||
|
||||
const onClick = all ? revokeAllUserKeys : revokeUserKey;
|
||||
|
||||
return (
|
||||
<DangerButton
|
||||
ref={revokeContentRef}
|
||||
showText={showText}
|
||||
onClick={onClick}
|
||||
disabled={disabled}
|
||||
confirmClear={confirmRevoke}
|
||||
id={'revoke-all-user-keys'}
|
||||
actionTextCode={'com_ui_revoke'}
|
||||
infoTextCode={'com_ui_revoke_info'}
|
||||
dataTestIdInitial={'revoke-all-keys-initial'}
|
||||
dataTestIdConfirm={'revoke-all-keys-confirm'}
|
||||
mutation={all ? revokeKeysMutation : revokeKeyMutation}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
function Data() {
|
||||
const dataTabRef = useRef(null);
|
||||
const [confirmClearConvos, setConfirmClearConvos] = useState(false);
|
||||
|
|
@ -114,7 +53,9 @@ function Data() {
|
|||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-600">
|
||||
<RevokeKeysButton all={true} />
|
||||
</div>
|
||||
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-600">
|
||||
<DeleteCacheButton />
|
||||
</div>
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-600">
|
||||
<ClearChatsButton
|
||||
confirmClear={confirmClearConvos}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
import React, { useState, useCallback, useRef, useEffect } from 'react';
|
||||
import { useOnClickOutside } from '~/hooks';
|
||||
import DangerButton from '../DangerButton';
|
||||
|
||||
export const DeleteCacheButton = ({
|
||||
showText = true,
|
||||
disabled = false,
|
||||
}: {
|
||||
showText?: boolean;
|
||||
disabled?: boolean;
|
||||
}) => {
|
||||
const [confirmClear, setConfirmClear] = useState(false);
|
||||
const [isCacheEmpty, setIsCacheEmpty] = useState(true);
|
||||
const contentRef = useRef(null);
|
||||
useOnClickOutside(contentRef, () => confirmClear && setConfirmClear(false), []);
|
||||
|
||||
const checkCache = useCallback(async () => {
|
||||
const cache = await caches.open('tts-responses');
|
||||
const keys = await cache.keys();
|
||||
setIsCacheEmpty(keys.length === 0);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
checkCache();
|
||||
}, [confirmClear]);
|
||||
|
||||
const revokeAllUserKeys = useCallback(async () => {
|
||||
if (confirmClear) {
|
||||
const cache = await caches.open('tts-responses');
|
||||
await cache.keys().then((keys) => Promise.all(keys.map((key) => cache.delete(key))));
|
||||
|
||||
setConfirmClear(false);
|
||||
} else {
|
||||
setConfirmClear(true);
|
||||
}
|
||||
}, [confirmClear]);
|
||||
|
||||
return (
|
||||
<DangerButton
|
||||
ref={contentRef}
|
||||
showText={showText}
|
||||
onClick={revokeAllUserKeys}
|
||||
disabled={disabled || isCacheEmpty}
|
||||
confirmClear={confirmClear}
|
||||
id={'delete-cache'}
|
||||
actionTextCode={'com_ui_delete'}
|
||||
infoTextCode={'com_nav_delete_cache_storage'}
|
||||
dataTestIdInitial={'delete-cache-initial'}
|
||||
dataTestIdConfirm={'delete-cache-confirm'}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import {
|
||||
useRevokeAllUserKeysMutation,
|
||||
useRevokeUserKeyMutation,
|
||||
} from 'librechat-data-provider/react-query';
|
||||
import React, { useState, useCallback, useRef } from 'react';
|
||||
import { useOnClickOutside } from '~/hooks';
|
||||
import DangerButton from '../DangerButton';
|
||||
|
||||
export const RevokeKeysButton = ({
|
||||
showText = true,
|
||||
endpoint = '',
|
||||
all = false,
|
||||
disabled = false,
|
||||
}: {
|
||||
showText?: boolean;
|
||||
endpoint?: string;
|
||||
all?: boolean;
|
||||
disabled?: boolean;
|
||||
}) => {
|
||||
const [confirmClear, setConfirmClear] = useState(false);
|
||||
const revokeKeyMutation = useRevokeUserKeyMutation(endpoint);
|
||||
const revokeKeysMutation = useRevokeAllUserKeysMutation();
|
||||
|
||||
const contentRef = useRef(null);
|
||||
useOnClickOutside(contentRef, () => confirmClear && setConfirmClear(false), []);
|
||||
|
||||
const revokeAllUserKeys = useCallback(() => {
|
||||
if (confirmClear) {
|
||||
revokeKeysMutation.mutate({});
|
||||
setConfirmClear(false);
|
||||
} else {
|
||||
setConfirmClear(true);
|
||||
}
|
||||
}, [confirmClear, revokeKeysMutation]);
|
||||
|
||||
const revokeUserKey = useCallback(() => {
|
||||
if (!endpoint) {
|
||||
return;
|
||||
} else if (confirmClear) {
|
||||
revokeKeyMutation.mutate({});
|
||||
setConfirmClear(false);
|
||||
} else {
|
||||
setConfirmClear(true);
|
||||
}
|
||||
}, [confirmClear, revokeKeyMutation, endpoint]);
|
||||
|
||||
const onClick = all ? revokeAllUserKeys : revokeUserKey;
|
||||
|
||||
return (
|
||||
<DangerButton
|
||||
ref={contentRef}
|
||||
showText={showText}
|
||||
onClick={onClick}
|
||||
disabled={disabled}
|
||||
confirmClear={confirmClear}
|
||||
id={'revoke-all-user-keys'}
|
||||
actionTextCode={'com_ui_revoke'}
|
||||
infoTextCode={'com_ui_revoke_info'}
|
||||
dataTestIdInitial={'revoke-all-keys-initial'}
|
||||
dataTestIdConfirm={'revoke-all-keys-confirm'}
|
||||
mutation={all ? revokeKeysMutation : revokeKeyMutation}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import React from 'react';
|
||||
import '@testing-library/jest-dom/extend-expect';
|
||||
import { render, fireEvent } from 'test/layout-test-utils';
|
||||
import ConversationModeSwitch from './ConversationModeSwitch';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
|
||||
describe('ConversationModeSwitch', () => {
|
||||
/**
|
||||
* Mock function to set the auto-send-text state.
|
||||
*/
|
||||
let mockSetConversationMode: jest.Mock<void, [boolean]> | ((value: boolean) => void) | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
mockSetConversationMode = jest.fn();
|
||||
});
|
||||
|
||||
it('renders correctly', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<ConversationModeSwitch />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
|
||||
expect(getByTestId('ConversationMode')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('calls onCheckedChange when the switch is toggled', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<ConversationModeSwitch onCheckedChange={mockSetConversationMode} />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
const switchElement = getByTestId('ConversationMode');
|
||||
fireEvent.click(switchElement);
|
||||
|
||||
expect(mockSetConversationMode).toHaveBeenCalledWith(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { Switch } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function ConversationModeSwitch({
|
||||
onCheckedChange,
|
||||
}: {
|
||||
onCheckedChange?: (value: boolean) => void;
|
||||
}) {
|
||||
const localize = useLocalize();
|
||||
const [conversationMode, setConversationMode] = useRecoilState<boolean>(store.conversationMode);
|
||||
const [advancedMode, setAdvancedMode] = useRecoilState<boolean>(store.advancedMode);
|
||||
const [textToSpeech] = useRecoilState<boolean>(store.TextToSpeech);
|
||||
const [, setAutoSendText] = useRecoilState<boolean>(store.autoSendText);
|
||||
const [, setDecibelValue] = useRecoilState(store.decibelValue);
|
||||
const [, setAutoTranscribeAudio] = useRecoilState<boolean>(store.autoTranscribeAudio);
|
||||
|
||||
const handleCheckedChange = (value: boolean) => {
|
||||
if (!advancedMode) {
|
||||
setAutoTranscribeAudio(value);
|
||||
setAutoSendText(value);
|
||||
setDecibelValue(-45);
|
||||
}
|
||||
setConversationMode(value);
|
||||
if (onCheckedChange) {
|
||||
onCheckedChange(value);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<strong>{localize('com_nav_conversation_mode')}</strong>
|
||||
</div>
|
||||
<div className="flex items-center justify-between">
|
||||
<label
|
||||
className="flex h-auto cursor-pointer items-center rounded border border-gray-500/70 bg-transparent px-2 py-1 text-xs font-medium font-normal transition-colors hover:border-gray-500/95 hover:bg-gray-100 hover:text-green-700 dark:bg-transparent dark:text-white dark:hover:bg-gray-600 dark:hover:text-green-500"
|
||||
onClick={() => setAdvancedMode(!advancedMode)}
|
||||
>
|
||||
<span>{advancedMode ? 'Advanced Mode' : 'Simple Mode'}</span>
|
||||
</label>
|
||||
<div className="w-2" />
|
||||
<Switch
|
||||
id="ConversationMode"
|
||||
checked={conversationMode}
|
||||
onCheckedChange={handleCheckedChange}
|
||||
className="ml-4"
|
||||
data-testid="ConversationMode"
|
||||
disabled={!textToSpeech}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { Switch } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function AutoSendTextSwitch({
|
||||
onCheckedChange,
|
||||
}: {
|
||||
onCheckedChange?: (value: boolean) => void;
|
||||
}) {
|
||||
const localize = useLocalize();
|
||||
const [autoSendText, setAutoSendText] = useRecoilState<boolean>(store.autoSendText);
|
||||
const [SpeechToText] = useRecoilState<boolean>(store.SpeechToText);
|
||||
|
||||
const handleCheckedChange = (value: boolean) => {
|
||||
setAutoSendText(value);
|
||||
if (onCheckedChange) {
|
||||
onCheckedChange(value);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{localize('com_nav_auto_send_text')}</div>
|
||||
<Switch
|
||||
id="AutoSendText"
|
||||
checked={autoSendText}
|
||||
onCheckedChange={handleCheckedChange}
|
||||
className="ml-4"
|
||||
data-testid="AutoSendText"
|
||||
disabled={!SpeechToText}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { Switch } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function AutoTranscribeAudioSwitch({
|
||||
onCheckedChange,
|
||||
}: {
|
||||
onCheckedChange?: (value: boolean) => void;
|
||||
}) {
|
||||
const localize = useLocalize();
|
||||
const [autoTranscribeAudio, setAutoTranscribeAudio] = useRecoilState<boolean>(
|
||||
store.autoTranscribeAudio,
|
||||
);
|
||||
const [speechToText] = useRecoilState<boolean>(store.SpeechToText);
|
||||
|
||||
const handleCheckedChange = (value: boolean) => {
|
||||
setAutoTranscribeAudio(value);
|
||||
if (onCheckedChange) {
|
||||
onCheckedChange(value);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{localize('com_nav_auto_transcribe_audio')}</div>
|
||||
<Switch
|
||||
id="AutoTranscribeAudio"
|
||||
checked={autoTranscribeAudio}
|
||||
onCheckedChange={handleCheckedChange}
|
||||
className="ml-4"
|
||||
data-testid="AutoTranscribeAudio"
|
||||
disabled={!speechToText}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
49
client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx
Executable file
49
client/src/components/Nav/SettingsTabs/Speech/STT/DecibelSelector.tsx
Executable file
|
|
@ -0,0 +1,49 @@
|
|||
import React from 'react';
|
||||
import { useRecoilState, useRecoilValue } from 'recoil';
|
||||
import { Slider, InputNumber } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
import { cn, defaultTextProps, optionText } from '~/utils/';
|
||||
|
||||
export default function DecibelSelector() {
|
||||
const localize = useLocalize();
|
||||
const speechToText = useRecoilValue(store.SpeechToText);
|
||||
const [decibelValue, setDecibelValue] = useRecoilState(store.decibelValue);
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{localize('com_nav_db_sensitivity')}</div>
|
||||
<div className="w-2" />
|
||||
<small className="opacity-40">({localize('com_endpoint_default_with_num', '0.45')})</small>
|
||||
</div>
|
||||
<div className="flex items-center justify-between">
|
||||
<Slider
|
||||
value={[decibelValue ?? -45]}
|
||||
onValueChange={(value) => setDecibelValue(value[0])}
|
||||
doubleClickHandler={() => setDecibelValue(-45)}
|
||||
min={-100}
|
||||
max={-30}
|
||||
step={1}
|
||||
className="ml-4 flex h-4 w-24"
|
||||
disabled={!speechToText}
|
||||
/>
|
||||
<div className="w-2" />
|
||||
<InputNumber
|
||||
value={decibelValue}
|
||||
disabled={!speechToText}
|
||||
onChange={(value) => setDecibelValue(value ? value[0] : 0)}
|
||||
min={-100}
|
||||
max={-30}
|
||||
className={cn(
|
||||
defaultTextProps,
|
||||
cn(
|
||||
optionText,
|
||||
'reset-rc-number-input reset-rc-number-input-text-right h-auto w-12 border-0 group-hover/temp:border-gray-200',
|
||||
),
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { Dropdown } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function EngineSTTDropdown() {
|
||||
const localize = useLocalize();
|
||||
const [endpointSTT, setEndpointSTT] = useRecoilState<string>(store.endpointSTT);
|
||||
const endpointOptions = [
|
||||
{ value: 'browser', display: localize('com_nav_browser') },
|
||||
{ value: 'external', display: localize('com_nav_external') },
|
||||
];
|
||||
|
||||
const handleSelect = (value: string) => {
|
||||
setEndpointSTT(value);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{localize('com_nav_engine')}</div>
|
||||
<Dropdown
|
||||
value={endpointSTT}
|
||||
onChange={handleSelect}
|
||||
options={endpointOptions}
|
||||
width={220}
|
||||
position={'left'}
|
||||
testId="EngineSTTDropdown"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { Switch } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function SpeechToTextSwitch({
|
||||
onCheckedChange,
|
||||
}: {
|
||||
onCheckedChange?: (value: boolean) => void;
|
||||
}) {
|
||||
const localize = useLocalize();
|
||||
const [speechToText, setSpeechToText] = useRecoilState<boolean>(store.SpeechToText);
|
||||
|
||||
const handleCheckedChange = (value: boolean) => {
|
||||
setSpeechToText(value);
|
||||
if (onCheckedChange) {
|
||||
onCheckedChange(value);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<strong>{localize('com_nav_speech_to_text')}</strong>
|
||||
</div>
|
||||
<Switch
|
||||
id="SpeechToText"
|
||||
checked={speechToText}
|
||||
onCheckedChange={handleCheckedChange}
|
||||
className="ml-4"
|
||||
data-testid="SpeechToText"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import React from 'react';
|
||||
import '@testing-library/jest-dom/extend-expect';
|
||||
import { render, fireEvent } from 'test/layout-test-utils';
|
||||
import AutoSendTextSwitch from '../AutoSendTextSwitch';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
|
||||
describe('AutoSendTextSwitch', () => {
|
||||
/**
|
||||
* Mock function to set the auto-send-text state.
|
||||
*/
|
||||
let mockSetAutoSendText: jest.Mock<void, [boolean]> | ((value: boolean) => void) | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
mockSetAutoSendText = jest.fn();
|
||||
});
|
||||
|
||||
it('renders correctly', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<AutoSendTextSwitch />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
|
||||
expect(getByTestId('AutoSendText')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('calls onCheckedChange when the switch is toggled', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<AutoSendTextSwitch onCheckedChange={mockSetAutoSendText} />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
const switchElement = getByTestId('AutoSendText');
|
||||
fireEvent.click(switchElement);
|
||||
|
||||
expect(mockSetAutoSendText).toHaveBeenCalledWith(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
import React from 'react';
|
||||
import '@testing-library/jest-dom/extend-expect';
|
||||
import { render, fireEvent } from 'test/layout-test-utils';
|
||||
import AutoTranscribeAudioSwitch from '../AutoTranscribeAudioSwitch';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
|
||||
describe('AutoTranscribeAudioSwitch', () => {
|
||||
/**
|
||||
* Mock function to set the auto-send-text state.
|
||||
*/
|
||||
let mockSetAutoTranscribeAudio:
|
||||
| jest.Mock<void, [boolean]>
|
||||
| ((value: boolean) => void)
|
||||
| undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
mockSetAutoTranscribeAudio = jest.fn();
|
||||
});
|
||||
|
||||
it('renders correctly', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<AutoTranscribeAudioSwitch />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
|
||||
expect(getByTestId('AutoTranscribeAudio')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('calls onCheckedChange when the switch is toggled', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<AutoTranscribeAudioSwitch onCheckedChange={mockSetAutoTranscribeAudio} />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
const switchElement = getByTestId('AutoTranscribeAudio');
|
||||
fireEvent.click(switchElement);
|
||||
|
||||
expect(mockSetAutoTranscribeAudio).toHaveBeenCalledWith(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import React from 'react';
|
||||
import '@testing-library/jest-dom/extend-expect';
|
||||
import { render, fireEvent } from 'test/layout-test-utils';
|
||||
import SpeechToTextSwitch from '../SpeechToTextSwitch';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
|
||||
describe('SpeechToTextSwitch', () => {
|
||||
/**
|
||||
* Mock function to set the speech-to-text state.
|
||||
*/
|
||||
let mockSetSpeechToText: jest.Mock<void, [boolean]> | ((value: boolean) => void) | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
mockSetSpeechToText = jest.fn();
|
||||
});
|
||||
|
||||
it('renders correctly', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<SpeechToTextSwitch />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
|
||||
expect(getByTestId('SpeechToText')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('calls onCheckedChange when the switch is toggled', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<SpeechToTextSwitch onCheckedChange={mockSetSpeechToText} />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
const switchElement = getByTestId('SpeechToText');
|
||||
fireEvent.click(switchElement);
|
||||
|
||||
expect(mockSetSpeechToText).toHaveBeenCalledWith(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
export { default as AutoSendTextSwitch } from './AutoSendTextSwitch';
|
||||
export { default as SpeechToTextSwitch } from './SpeechToTextSwitch';
|
||||
export { default as EngineSTTDropdown } from './EngineSTTDropdown';
|
||||
export { default as DecibelSelector } from './DecibelSelector';
|
||||
export { default as AutoTranscribeAudioSwitch } from './AutoTranscribeAudioSwitch';
|
||||
93
client/src/components/Nav/SettingsTabs/Speech/Speech.tsx
Normal file
93
client/src/components/Nav/SettingsTabs/Speech/Speech.tsx
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import * as Tabs from '@radix-ui/react-tabs';
|
||||
import { SettingsTabValues } from 'librechat-data-provider';
|
||||
import React, { useState, useRef } from 'react';
|
||||
import { useRecoilState } from 'recoil';
|
||||
import { useOnClickOutside } from '~/hooks';
|
||||
import store from '~/store';
|
||||
import ConversationModeSwitch from './ConversationModeSwitch';
|
||||
import {
|
||||
TextToSpeechSwitch,
|
||||
EngineTTSDropdown,
|
||||
AutomaticPlayback,
|
||||
CacheTTSSwitch,
|
||||
VoiceDropdown,
|
||||
PlaybackRate,
|
||||
} from './TTS';
|
||||
import {
|
||||
DecibelSelector,
|
||||
EngineSTTDropdown,
|
||||
SpeechToTextSwitch,
|
||||
AutoSendTextSwitch,
|
||||
AutoTranscribeAudioSwitch,
|
||||
} from './STT';
|
||||
|
||||
function Speech() {
|
||||
const [confirmClear, setConfirmClear] = useState(false);
|
||||
const [advancedMode] = useRecoilState<boolean>(store.advancedMode);
|
||||
const [autoTranscribeAudio] = useRecoilState<boolean>(store.autoTranscribeAudio);
|
||||
|
||||
const contentRef = useRef(null);
|
||||
useOnClickOutside(contentRef, () => confirmClear && setConfirmClear(false), []);
|
||||
|
||||
return (
|
||||
<Tabs.Content
|
||||
value={SettingsTabValues.SPEECH}
|
||||
role="tabpanel"
|
||||
className="w-full px-4 md:min-h-[300px]"
|
||||
ref={contentRef}
|
||||
>
|
||||
<div className="flex flex-col gap-3 text-sm text-gray-600 dark:text-gray-50">
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<ConversationModeSwitch />
|
||||
</div>
|
||||
<div className="h-px bg-black/20 bg-white/20" role="none" />
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<SpeechToTextSwitch />
|
||||
</div>
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<EngineSTTDropdown />
|
||||
</div>
|
||||
{advancedMode && (
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<AutoTranscribeAudioSwitch />
|
||||
</div>
|
||||
)}
|
||||
{autoTranscribeAudio && advancedMode && (
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<DecibelSelector />
|
||||
</div>
|
||||
)}
|
||||
{advancedMode && (
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<AutoSendTextSwitch />
|
||||
</div>
|
||||
)}
|
||||
<div className="h-px bg-black/20 bg-white/20" role="none" />
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<TextToSpeechSwitch />
|
||||
</div>
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<AutomaticPlayback />
|
||||
</div>
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<EngineTTSDropdown />
|
||||
</div>
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<VoiceDropdown />
|
||||
</div>
|
||||
{advancedMode && (
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<PlaybackRate />
|
||||
</div>
|
||||
)}
|
||||
{advancedMode && (
|
||||
<div className="border-b pb-3 last-of-type:border-b-0 dark:border-gray-700">
|
||||
<CacheTTSSwitch />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</Tabs.Content>
|
||||
);
|
||||
}
|
||||
|
||||
export default React.memo(Speech);
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { Switch } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function AutomaticPlayback({
|
||||
onCheckedChange,
|
||||
}: {
|
||||
onCheckedChange?: (value: boolean) => void;
|
||||
}) {
|
||||
const localize = useLocalize();
|
||||
const [automaticPlayback, setAutomaticPlayback] = useRecoilState(store.automaticPlayback);
|
||||
|
||||
const handleCheckedChange = (value: boolean) => {
|
||||
setAutomaticPlayback(value);
|
||||
if (onCheckedChange) {
|
||||
onCheckedChange(value);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{localize('com_nav_automatic_playback')}</div>
|
||||
<Switch
|
||||
id="AutomaticPlayback"
|
||||
checked={automaticPlayback}
|
||||
onCheckedChange={handleCheckedChange}
|
||||
className="ml-4"
|
||||
data-testid="AutomaticPlayback"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { Switch } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function CacheTTSSwitch({
|
||||
onCheckedChange,
|
||||
}: {
|
||||
onCheckedChange?: (value: boolean) => void;
|
||||
}) {
|
||||
const localize = useLocalize();
|
||||
const [cacheTTS, setCacheTTS] = useRecoilState<boolean>(store.cacheTTS);
|
||||
const [textToSpeech] = useRecoilState<boolean>(store.TextToSpeech);
|
||||
|
||||
const handleCheckedChange = (value: boolean) => {
|
||||
setCacheTTS(value);
|
||||
if (onCheckedChange) {
|
||||
onCheckedChange(value);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{localize('com_nav_enable_cache_tts')}</div>
|
||||
<Switch
|
||||
id="CacheTTS"
|
||||
checked={cacheTTS}
|
||||
onCheckedChange={handleCheckedChange}
|
||||
className="ml-4"
|
||||
data-testid="CacheTTS"
|
||||
disabled={!textToSpeech}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { Dropdown } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function EngineTTSDropdown() {
|
||||
const localize = useLocalize();
|
||||
const [endpointTTS, setEndpointTTS] = useRecoilState<string>(store.endpointTTS);
|
||||
const endpointOptions = [
|
||||
{ value: 'browser', display: localize('com_nav_browser') },
|
||||
{ value: 'external', display: localize('com_nav_external') },
|
||||
];
|
||||
|
||||
const handleSelect = (value: string) => {
|
||||
setEndpointTTS(value);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{localize('com_nav_engine')}</div>
|
||||
<Dropdown
|
||||
value={endpointTTS}
|
||||
onChange={handleSelect}
|
||||
options={endpointOptions}
|
||||
width={220}
|
||||
position={'left'}
|
||||
testId="EngineTTSDropdown"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
49
client/src/components/Nav/SettingsTabs/Speech/TTS/PlaybackRate.tsx
Executable file
49
client/src/components/Nav/SettingsTabs/Speech/TTS/PlaybackRate.tsx
Executable file
|
|
@ -0,0 +1,49 @@
|
|||
import React from 'react';
|
||||
import { useRecoilState, useRecoilValue } from 'recoil';
|
||||
import { Slider, InputNumber } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
import { cn, defaultTextProps, optionText } from '~/utils/';
|
||||
|
||||
export default function DecibelSelector() {
|
||||
const localize = useLocalize();
|
||||
const textToSpeech = useRecoilValue(store.TextToSpeech);
|
||||
const [playbackRate, setPlaybackRate] = useRecoilState(store.playbackRate);
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{localize('com_nav_playback_rate')}</div>
|
||||
<div className="w-2" />
|
||||
<small className="opacity-40">({localize('com_endpoint_default_with_num', '1')})</small>
|
||||
</div>
|
||||
<div className="flex items-center justify-between">
|
||||
<Slider
|
||||
value={[playbackRate ?? 1]}
|
||||
onValueChange={(value) => setPlaybackRate(value[0])}
|
||||
doubleClickHandler={() => setPlaybackRate(null)}
|
||||
min={-0.1}
|
||||
max={2}
|
||||
step={0.1}
|
||||
className="ml-4 flex h-4 w-24"
|
||||
disabled={!textToSpeech}
|
||||
/>
|
||||
<div className="w-2" />
|
||||
<InputNumber
|
||||
value={playbackRate ?? 1}
|
||||
disabled={!textToSpeech}
|
||||
onChange={(value) => setPlaybackRate(value ? value[0] : 0)}
|
||||
min={-0.1}
|
||||
max={2}
|
||||
className={cn(
|
||||
defaultTextProps,
|
||||
cn(
|
||||
optionText,
|
||||
'reset-rc-number-input reset-rc-number-input-text-right h-auto w-12 border-0 group-hover/temp:border-gray-200',
|
||||
),
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import { useRecoilState } from 'recoil';
|
||||
import { Switch } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function TextToSpeechSwitch({
|
||||
onCheckedChange,
|
||||
}: {
|
||||
onCheckedChange?: (value: boolean) => void;
|
||||
}) {
|
||||
const localize = useLocalize();
|
||||
const [TextToSpeech, setTextToSpeech] = useRecoilState<boolean>(store.TextToSpeech);
|
||||
|
||||
const handleCheckedChange = (value: boolean) => {
|
||||
setTextToSpeech(value);
|
||||
if (onCheckedChange) {
|
||||
onCheckedChange(value);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<strong>{localize('com_nav_text_to_speech')}</strong>
|
||||
</div>
|
||||
<Switch
|
||||
id="TextToSpeech"
|
||||
checked={TextToSpeech}
|
||||
onCheckedChange={handleCheckedChange}
|
||||
className="ml-4"
|
||||
data-testid="TextToSpeech"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import { useMemo } from 'react';
|
||||
import { useRecoilState } from 'recoil';
|
||||
import { useVoicesQuery } from '~/data-provider';
|
||||
import { Dropdown } from '~/components/ui';
|
||||
import { useLocalize } from '~/hooks';
|
||||
import store from '~/store';
|
||||
|
||||
export default function VoiceDropdown() {
|
||||
const localize = useLocalize();
|
||||
const [voice, setVoice] = useRecoilState<string>(store.voice);
|
||||
const { data } = useVoicesQuery();
|
||||
|
||||
const voiceOptions = useMemo(
|
||||
() => (data ?? []).map((v: string) => ({ value: v, display: v })),
|
||||
[data],
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between">
|
||||
<div>{localize('com_nav_voice_select')}</div>
|
||||
<Dropdown
|
||||
value={voice}
|
||||
onChange={(value: string) => setVoice(value)}
|
||||
options={voiceOptions}
|
||||
width={220}
|
||||
position={'left'}
|
||||
testId="VoiceDropdown"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import React from 'react';
|
||||
import '@testing-library/jest-dom/extend-expect';
|
||||
import { render, fireEvent } from 'test/layout-test-utils';
|
||||
import CacheTTSSwitch from '../CacheTTSSwitch';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
|
||||
describe('CacheTTSSwitch', () => {
|
||||
/**
|
||||
* Mock function to set the cache-tts state.
|
||||
*/
|
||||
let mockSetCacheTTS: jest.Mock<void, [boolean]> | ((value: boolean) => void) | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
mockSetCacheTTS = jest.fn();
|
||||
});
|
||||
|
||||
it('renders correctly', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<CacheTTSSwitch />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
|
||||
expect(getByTestId('CacheTTS')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('calls onCheckedChange when the switch is toggled', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<CacheTTSSwitch onCheckedChange={mockSetCacheTTS} />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
const switchElement = getByTestId('CacheTTS');
|
||||
fireEvent.click(switchElement);
|
||||
|
||||
expect(mockSetCacheTTS).toHaveBeenCalledWith(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import React from 'react';
|
||||
import '@testing-library/jest-dom/extend-expect';
|
||||
import { render, fireEvent } from 'test/layout-test-utils';
|
||||
import TextToSpeechSwitch from '../TextToSpeechSwitch';
|
||||
import { RecoilRoot } from 'recoil';
|
||||
|
||||
describe('TextToSpeechSwitch', () => {
|
||||
/**
|
||||
* Mock function to set the text-to-speech state.
|
||||
*/
|
||||
let mockSetTextToSpeech: jest.Mock<void, [boolean]> | ((value: boolean) => void) | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
mockSetTextToSpeech = jest.fn();
|
||||
});
|
||||
|
||||
it('renders correctly', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<TextToSpeechSwitch />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
|
||||
expect(getByTestId('TextToSpeech')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('calls onCheckedChange when the switch is toggled', () => {
|
||||
const { getByTestId } = render(
|
||||
<RecoilRoot>
|
||||
<TextToSpeechSwitch onCheckedChange={mockSetTextToSpeech} />
|
||||
</RecoilRoot>,
|
||||
);
|
||||
const switchElement = getByTestId('TextToSpeech');
|
||||
fireEvent.click(switchElement);
|
||||
|
||||
expect(mockSetTextToSpeech).toHaveBeenCalledWith(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
export { default as AutomaticPlayback } from './AutomaticPlayback';
|
||||
export { default as CacheTTSSwitch } from './CacheTTSSwitch';
|
||||
export { default as EngineTTSDropdown } from './EngineTTSDropdown';
|
||||
export { default as PlaybackRate } from './PlaybackRate';
|
||||
export { default as TextToSpeechSwitch } from './TextToSpeechSwitch';
|
||||
export { default as VoiceDropdown } from './VoiceDropdown';
|
||||
|
|
@ -3,5 +3,6 @@ export { default as Messages } from './Messages/Messages';
|
|||
export { ClearChatsButton } from './General/General';
|
||||
export { default as Data } from './Data/Data';
|
||||
export { default as Beta } from './Beta/Beta';
|
||||
export { RevokeKeysButton } from './Data/Data';
|
||||
export { RevokeKeysButton } from './Data/RevokeKeysButton';
|
||||
export { default as Account } from './Account/Account';
|
||||
export { default as Speech } from './Speech/Speech';
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
import React from 'react';
|
||||
import { cn } from '~/utils/';
|
||||
|
||||
export default function Clipboard() {
|
||||
export default function Clipboard({ className = 'icon-md-heavy', size = '1em' }) {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
width="24"
|
||||
height="24"
|
||||
height={size}
|
||||
width={size}
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
className="icon-md-heavy"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
className={cn(className)}
|
||||
>
|
||||
<path
|
||||
fill="currentColor"
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
export default function EditIcon() {
|
||||
import { cn } from '~/utils';
|
||||
|
||||
export default function EditIcon({ className = 'icon-md', size = '1.2em' }) {
|
||||
return (
|
||||
<svg
|
||||
fill="none"
|
||||
strokeWidth="2"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
className="icon-md"
|
||||
height="1em"
|
||||
width="1em"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
height={size}
|
||||
width={size}
|
||||
className={cn(className)}
|
||||
>
|
||||
<path
|
||||
fillRule="evenodd"
|
||||
|
|
|
|||
21
client/src/components/svg/ListeningIcon.tsx
Normal file
21
client/src/components/svg/ListeningIcon.tsx
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { cn } from '~/utils/';
|
||||
|
||||
export default function ListeningIcon({ className }) {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
width="24"
|
||||
height="24"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
className={cn(className)}
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
<path d="M12 2a3 3 0 0 0-3 3v7a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z" />
|
||||
<path d="M19 10v2a7 7 0 0 1-14 0v-2" />
|
||||
<line x1="12" x2="12" y1="19" y2="22" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,13 +1,15 @@
|
|||
import { cn } from '~/utils';
|
||||
|
||||
export default function RegenerateIcon({ className = '' }: { className?: string }) {
|
||||
export default function RegenerateIcon({ className = '', size = '1em' }) {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
width="24"
|
||||
height="24"
|
||||
height={size}
|
||||
width={size}
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
className={cn('icon-md-heavy', className)}
|
||||
>
|
||||
<path
|
||||
|
|
|
|||
25
client/src/components/svg/SpeechIcon.tsx
Normal file
25
client/src/components/svg/SpeechIcon.tsx
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import { cn } from '~/utils/';
|
||||
|
||||
export default function SpeechIcon({ className }) {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
width="24"
|
||||
height="24"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
className={cn(className)}
|
||||
>
|
||||
<path d="M2 10v3" />
|
||||
<path d="M6 6v11" />
|
||||
<path d="M10 3v18" />
|
||||
<path d="M14 8v7" />
|
||||
<path d="M18 5v13" />
|
||||
<path d="M22 10v3" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
21
client/src/components/svg/VolumeIcon.tsx
Normal file
21
client/src/components/svg/VolumeIcon.tsx
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { cn } from '~/utils';
|
||||
|
||||
export default function VolumeIcon({ className = '', size = '1em' }) {
|
||||
return (
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
height={size}
|
||||
width={size}
|
||||
className={cn(className)}
|
||||
>
|
||||
<path
|
||||
fillRule="evenodd"
|
||||
clipRule="evenodd"
|
||||
d="M11 4.9099C11 4.47485 10.4828 4.24734 10.1621 4.54132L6.67572 7.7372C6.49129 7.90626 6.25019 8.00005 6 8.00005H4C3.44772 8.00005 3 8.44776 3 9.00005V15C3 15.5523 3.44772 16 4 16H6C6.25019 16 6.49129 16.0938 6.67572 16.2629L10.1621 19.4588C10.4828 19.7527 11 19.5252 11 19.0902V4.9099ZM8.81069 3.06701C10.4142 1.59714 13 2.73463 13 4.9099V19.0902C13 21.2655 10.4142 22.403 8.81069 20.9331L5.61102 18H4C2.34315 18 1 16.6569 1 15V9.00005C1 7.34319 2.34315 6.00005 4 6.00005H5.61102L8.81069 3.06701ZM20.3166 6.35665C20.8019 6.09313 21.409 6.27296 21.6725 6.75833C22.5191 8.3176 22.9996 10.1042 22.9996 12.0001C22.9996 13.8507 22.5418 15.5974 21.7323 17.1302C21.4744 17.6185 20.8695 17.8054 20.3811 17.5475C19.8927 17.2896 19.7059 16.6846 19.9638 16.1962C20.6249 14.9444 20.9996 13.5175 20.9996 12.0001C20.9996 10.4458 20.6064 8.98627 19.9149 7.71262C19.6514 7.22726 19.8312 6.62017 20.3166 6.35665ZM15.7994 7.90049C16.241 7.5688 16.8679 7.65789 17.1995 8.09947C18.0156 9.18593 18.4996 10.5379 18.4996 12.0001C18.4996 13.3127 18.1094 14.5372 17.4385 15.5604C17.1357 16.0222 16.5158 16.1511 16.0539 15.8483C15.5921 15.5455 15.4632 14.9255 15.766 14.4637C16.2298 13.7564 16.4996 12.9113 16.4996 12.0001C16.4996 10.9859 16.1653 10.0526 15.6004 9.30063C15.2687 8.85905 15.3578 8.23218 15.7994 7.90049Z"
|
||||
fill="currentColor"
|
||||
></path>
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
21
client/src/components/svg/VolumeMuteIcon.tsx
Normal file
21
client/src/components/svg/VolumeMuteIcon.tsx
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { cn } from '~/utils';
|
||||
|
||||
export default function VolumeMuteIcon({ className = '', size = '1em' }) {
|
||||
return (
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
height={size}
|
||||
width={size}
|
||||
className={cn(className)}
|
||||
>
|
||||
<path
|
||||
fillRule="evenodd"
|
||||
clipRule="evenodd"
|
||||
d="M2 12C2 6.47715 6.47715 2 12 2C17.5228 2 22 6.47715 22 12C22 17.5228 17.5228 22 12 22C6.47715 22 2 17.5228 2 12ZM9.5 8.5C8.94772 8.5 8.5 8.94772 8.5 9.5V14.5C8.5 15.0523 8.94772 15.5 9.5 15.5H14.5C15.0523 15.5 15.5 15.0523 15.5 14.5V9.5C15.5 8.94772 15.0523 8.5 14.5 8.5H9.5Z"
|
||||
fill="currentColor"
|
||||
></path>
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
|
@ -41,6 +41,9 @@ export { default as CodeyIcon } from './CodeyIcon';
|
|||
export { default as GeminiIcon } from './GeminiIcon';
|
||||
export { default as GoogleMinimalIcon } from './GoogleMinimalIcon';
|
||||
export { default as AnthropicMinimalIcon } from './AnthropicMinimalIcon';
|
||||
export { default as ListeningIcon } from './ListeningIcon';
|
||||
export { default as VolumeIcon } from './VolumeIcon';
|
||||
export { default as VolumeMuteIcon } from './VolumeMuteIcon';
|
||||
export { default as SendMessageIcon } from './SendMessageIcon';
|
||||
export { default as UserIcon } from './UserIcon';
|
||||
export { default as NewChatIcon } from './NewChatIcon';
|
||||
|
|
@ -49,3 +52,4 @@ export { default as GoogleIconChat } from './GoogleIconChat';
|
|||
export { default as BirthdayIcon } from './BirthdayIcon';
|
||||
export { default as AssistantIcon } from './AssistantIcon';
|
||||
export { default as Sparkles } from './Sparkles';
|
||||
export { default as SpeechIcon } from './SpeechIcon';
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import React, { FC, useContext, useState } from 'react';
|
||||
import React, { FC, useState } from 'react';
|
||||
import { Listbox } from '@headlessui/react';
|
||||
import { cn } from '~/utils/';
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ export default function Landing() {
|
|||
<h1
|
||||
id="landing-title"
|
||||
data-testid="landing-title"
|
||||
className="mb-10 ml-auto mr-auto mt-6 flex items-center justify-center gap-2 dark:text-gray-600 text-center text-4xl font-semibold sm:mb-16 md:mt-[10vh]"
|
||||
className="mb-10 ml-auto mr-auto mt-6 flex items-center justify-center gap-2 text-center text-4xl font-semibold dark:text-gray-600 sm:mb-16 md:mt-[10vh]"
|
||||
>
|
||||
{config?.appTitle || 'LibreChat'}
|
||||
</h1>
|
||||
|
|
|
|||
|
|
@ -547,6 +547,36 @@ export const useUploadAvatarMutation = (
|
|||
});
|
||||
};
|
||||
|
||||
/* Speech to text */
|
||||
export const useSpeechToTextMutation = (
|
||||
options?: t.SpeechToTextOptions,
|
||||
): UseMutationResult<
|
||||
t.SpeechToTextResponse, // response data
|
||||
unknown, // error
|
||||
FormData, // request
|
||||
unknown // context
|
||||
> => {
|
||||
return useMutation([MutationKeys.speechToText], {
|
||||
mutationFn: (variables: FormData) => dataService.speechToText(variables),
|
||||
...(options || {}),
|
||||
});
|
||||
};
|
||||
|
||||
/* Text to speech */
|
||||
export const useTextToSpeechMutation = (
|
||||
options?: t.TextToSpeechOptions,
|
||||
): UseMutationResult<
|
||||
ArrayBuffer, // response data
|
||||
unknown, // error
|
||||
FormData, // request
|
||||
unknown // context
|
||||
> => {
|
||||
return useMutation([MutationKeys.textToSpeech], {
|
||||
mutationFn: (variables: FormData) => dataService.textToSpeech(variables),
|
||||
...(options || {}),
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* ASSISTANTS
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import type {
|
|||
UseInfiniteQueryOptions,
|
||||
QueryObserverResult,
|
||||
UseQueryOptions,
|
||||
UseQueryResult,
|
||||
} from '@tanstack/react-query';
|
||||
import type t from 'librechat-data-provider';
|
||||
import type {
|
||||
|
|
@ -414,3 +415,10 @@ export const useFileDownload = (userId?: string, file_id?: string): QueryObserve
|
|||
},
|
||||
);
|
||||
};
|
||||
|
||||
/** STT/TTS */
|
||||
|
||||
/* Text to speech voices */
|
||||
export const useVoicesQuery = (): UseQueryResult<t.VoiceResponse> => {
|
||||
return useQuery([QueryKeys.voices], () => dataService.getVoices());
|
||||
};
|
||||
|
|
|
|||
41
client/src/hooks/Audio/MediaSourceAppender.ts
Normal file
41
client/src/hooks/Audio/MediaSourceAppender.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
export class MediaSourceAppender {
|
||||
private readonly mediaSource = new MediaSource();
|
||||
private readonly audioChunks: ArrayBuffer[] = [];
|
||||
|
||||
private sourceBuffer?: SourceBuffer;
|
||||
|
||||
constructor(type: string) {
|
||||
this.mediaSource.addEventListener('sourceopen', async () => {
|
||||
this.sourceBuffer = this.mediaSource.addSourceBuffer(type);
|
||||
|
||||
this.sourceBuffer.addEventListener('updateend', () => {
|
||||
this.tryAppendNextChunk();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private tryAppendNextChunk() {
|
||||
if (this.sourceBuffer != null && !this.sourceBuffer.updating && this.audioChunks.length > 0) {
|
||||
this.sourceBuffer.appendBuffer(this.audioChunks.shift()!);
|
||||
}
|
||||
}
|
||||
|
||||
public addBase64Data(base64Data: string) {
|
||||
this.addData(Uint8Array.from(atob(base64Data), (char) => char.charCodeAt(0)).buffer);
|
||||
}
|
||||
|
||||
public addData(data: ArrayBuffer) {
|
||||
this.audioChunks.push(data);
|
||||
this.tryAppendNextChunk();
|
||||
}
|
||||
|
||||
public close() {
|
||||
if (this.mediaSource.readyState === 'open') {
|
||||
this.mediaSource.endOfStream();
|
||||
}
|
||||
}
|
||||
|
||||
public get mediaSourceUrl() {
|
||||
return URL.createObjectURL(this.mediaSource);
|
||||
}
|
||||
}
|
||||
3
client/src/hooks/Audio/index.ts
Normal file
3
client/src/hooks/Audio/index.ts
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
export * from './MediaSourceAppender';
|
||||
export { default as useCustomAudioRef } from './useCustomAudioRef';
|
||||
export { default as usePauseGlobalAudio } from './usePauseGlobalAudio';
|
||||
98
client/src/hooks/Audio/useCustomAudioRef.ts
Normal file
98
client/src/hooks/Audio/useCustomAudioRef.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import { useEffect, useRef } from 'react';
|
||||
|
||||
interface CustomAudioElement extends HTMLAudioElement {
|
||||
customStarted?: boolean;
|
||||
customEnded?: boolean;
|
||||
customPaused?: boolean;
|
||||
customProps?: {
|
||||
customStarted?: boolean;
|
||||
customEnded?: boolean;
|
||||
customPaused?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
type TCustomAudioResult = { audioRef: React.MutableRefObject<CustomAudioElement | null> };
|
||||
|
||||
export default function useCustomAudioRef({
|
||||
setIsPlaying,
|
||||
}: {
|
||||
setIsPlaying: (isPlaying: boolean) => void;
|
||||
}): TCustomAudioResult {
|
||||
const audioRef = useRef<CustomAudioElement | null>(null);
|
||||
useEffect(() => {
|
||||
let lastTimeUpdate: number | null = null;
|
||||
let sameTimeUpdateCount = 0;
|
||||
|
||||
const handleEnded = () => {
|
||||
setIsPlaying(false);
|
||||
console.log('global audio ended');
|
||||
if (audioRef.current) {
|
||||
audioRef.current.customEnded = true;
|
||||
URL.revokeObjectURL(audioRef.current.src);
|
||||
}
|
||||
};
|
||||
|
||||
const handleStart = () => {
|
||||
setIsPlaying(true);
|
||||
console.log('global audio started');
|
||||
if (audioRef.current) {
|
||||
audioRef.current.customStarted = true;
|
||||
}
|
||||
};
|
||||
|
||||
const handlePause = () => {
|
||||
console.log('global audio paused');
|
||||
if (audioRef.current) {
|
||||
audioRef.current.customPaused = true;
|
||||
}
|
||||
};
|
||||
|
||||
const handleTimeUpdate = () => {
|
||||
if (audioRef.current) {
|
||||
const currentTime = audioRef.current.currentTime;
|
||||
// console.log('Current time: ', currentTime);
|
||||
|
||||
if (currentTime === lastTimeUpdate) {
|
||||
sameTimeUpdateCount += 1;
|
||||
} else {
|
||||
sameTimeUpdateCount = 0;
|
||||
}
|
||||
|
||||
lastTimeUpdate = currentTime;
|
||||
|
||||
if (sameTimeUpdateCount >= 1) {
|
||||
console.log('Detected end of audio based on time update');
|
||||
audioRef.current.pause();
|
||||
handleEnded();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const audioElement = audioRef.current;
|
||||
|
||||
if (audioRef.current) {
|
||||
audioRef.current.addEventListener('ended', handleEnded);
|
||||
audioRef.current.addEventListener('play', handleStart);
|
||||
audioRef.current.addEventListener('pause', handlePause);
|
||||
audioRef.current.addEventListener('timeupdate', handleTimeUpdate);
|
||||
|
||||
audioRef.current.customProps = {
|
||||
customStarted: false,
|
||||
customEnded: false,
|
||||
customPaused: false,
|
||||
};
|
||||
}
|
||||
|
||||
return () => {
|
||||
if (audioElement) {
|
||||
audioElement.removeEventListener('ended', handleEnded);
|
||||
audioElement.removeEventListener('play', handleStart);
|
||||
audioElement.removeEventListener('pause', handlePause);
|
||||
audioElement.removeEventListener('timeupdate', handleTimeUpdate);
|
||||
URL.revokeObjectURL(audioElement.src);
|
||||
}
|
||||
};
|
||||
}, [setIsPlaying]);
|
||||
|
||||
return { audioRef };
|
||||
}
|
||||
37
client/src/hooks/Audio/usePauseGlobalAudio.ts
Normal file
37
client/src/hooks/Audio/usePauseGlobalAudio.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import { useCallback } from 'react';
|
||||
import { useRecoilState, useSetRecoilState } from 'recoil';
|
||||
import { globalAudioId } from '~/common';
|
||||
import store from '~/store';
|
||||
|
||||
function usePauseGlobalAudio(index = 0) {
|
||||
/* Global Audio Variables */
|
||||
const setAudioRunId = useSetRecoilState(store.audioRunFamily(index));
|
||||
const setIsGlobalAudioFetching = useSetRecoilState(store.globalAudioFetchingFamily(index));
|
||||
const [globalAudioURL, setGlobalAudioURL] = useRecoilState(store.globalAudioURLFamily(index));
|
||||
const setGlobalIsPlaying = useSetRecoilState(store.globalAudioPlayingFamily(index));
|
||||
|
||||
const pauseGlobalAudio = useCallback(() => {
|
||||
if (globalAudioURL) {
|
||||
const globalAudio = document.getElementById(globalAudioId);
|
||||
if (globalAudio) {
|
||||
console.log('Pausing global audio', globalAudioURL);
|
||||
(globalAudio as HTMLAudioElement).pause();
|
||||
setGlobalIsPlaying(false);
|
||||
}
|
||||
URL.revokeObjectURL(globalAudioURL);
|
||||
setIsGlobalAudioFetching(false);
|
||||
setGlobalAudioURL(null);
|
||||
setAudioRunId(null);
|
||||
}
|
||||
}, [
|
||||
globalAudioURL,
|
||||
setGlobalAudioURL,
|
||||
setGlobalIsPlaying,
|
||||
setIsGlobalAudioFetching,
|
||||
setAudioRunId,
|
||||
]);
|
||||
|
||||
return { pauseGlobalAudio };
|
||||
}
|
||||
|
||||
export default usePauseGlobalAudio;
|
||||
|
|
@ -4,3 +4,5 @@ export { default as useTextarea } from './useTextarea';
|
|||
export { default as useCombobox } from './useCombobox';
|
||||
export { default as useRequiresKey } from './useRequiresKey';
|
||||
export { default as useMultipleKeys } from './useMultipleKeys';
|
||||
export { default as useSpeechToText } from './useSpeechToText';
|
||||
export { default as useTextToSpeech } from './useTextToSpeech';
|
||||
|
|
|
|||
83
client/src/hooks/Input/useSpeechToText.ts
Normal file
83
client/src/hooks/Input/useSpeechToText.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
import { useState, useEffect } from 'react';
|
||||
import useSpeechToTextBrowser from './useSpeechToTextBrowser';
|
||||
import useSpeechToTextExternal from './useSpeechToTextExternal';
|
||||
import { useRecoilState } from 'recoil';
|
||||
import store from '~/store';
|
||||
|
||||
const useSpeechToText = (handleTranscriptionComplete: (text: string) => void) => {
|
||||
const [endpointSTT] = useRecoilState<string>(store.endpointSTT);
|
||||
const useExternalSpeechToText = endpointSTT === 'external';
|
||||
const [animatedText, setAnimatedText] = useState('');
|
||||
|
||||
const {
|
||||
isListening: speechIsListeningBrowser,
|
||||
isLoading: speechIsLoadingBrowser,
|
||||
text: speechTextBrowser,
|
||||
startRecording: startSpeechRecordingBrowser,
|
||||
stopRecording: stopSpeechRecordingBrowser,
|
||||
} = useSpeechToTextBrowser();
|
||||
|
||||
const {
|
||||
isListening: speechIsListeningExternal,
|
||||
isLoading: speechIsLoadingExternal,
|
||||
text: speechTextExternal,
|
||||
externalStartRecording: startSpeechRecordingExternal,
|
||||
externalStopRecording: stopSpeechRecordingExternal,
|
||||
clearText,
|
||||
} = useSpeechToTextExternal(handleTranscriptionComplete);
|
||||
|
||||
const isListening = useExternalSpeechToText
|
||||
? speechIsListeningExternal
|
||||
: speechIsListeningBrowser;
|
||||
const isLoading = useExternalSpeechToText ? speechIsLoadingExternal : speechIsLoadingBrowser;
|
||||
const speechTextForm = useExternalSpeechToText ? speechTextExternal : speechTextBrowser;
|
||||
const startRecording = useExternalSpeechToText
|
||||
? startSpeechRecordingExternal
|
||||
: startSpeechRecordingBrowser;
|
||||
const stopRecording = useExternalSpeechToText
|
||||
? stopSpeechRecordingExternal
|
||||
: stopSpeechRecordingBrowser;
|
||||
const speechText =
|
||||
isListening || (speechTextExternal && speechTextExternal.length > 0)
|
||||
? speechTextExternal
|
||||
: speechTextForm || '';
|
||||
|
||||
const animateTextTyping = (text: string) => {
|
||||
const totalDuration = 2000;
|
||||
const frameRate = 60;
|
||||
const totalFrames = totalDuration / (1000 / frameRate);
|
||||
const charsPerFrame = Math.ceil(text.length / totalFrames);
|
||||
let currentIndex = 0;
|
||||
|
||||
const animate = () => {
|
||||
currentIndex += charsPerFrame;
|
||||
const currentText = text.substring(0, currentIndex);
|
||||
setAnimatedText(currentText);
|
||||
|
||||
if (currentIndex < text.length) {
|
||||
requestAnimationFrame(animate);
|
||||
} else {
|
||||
setAnimatedText(text);
|
||||
}
|
||||
};
|
||||
|
||||
requestAnimationFrame(animate);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (speechText) {
|
||||
animateTextTyping(speechText);
|
||||
}
|
||||
}, [speechText]);
|
||||
|
||||
return {
|
||||
isListening,
|
||||
isLoading,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
speechText: animatedText,
|
||||
clearText,
|
||||
};
|
||||
};
|
||||
|
||||
export default useSpeechToText;
|
||||
52
client/src/hooks/Input/useSpeechToTextBrowser.ts
Normal file
52
client/src/hooks/Input/useSpeechToTextBrowser.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import { useEffect } from 'react';
|
||||
import { useRecoilState } from 'recoil';
|
||||
import { useToastContext } from '~/Providers';
|
||||
import store from '~/store';
|
||||
import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition';
|
||||
|
||||
const useSpeechToTextBrowser = () => {
|
||||
const { showToast } = useToastContext();
|
||||
const [endpointSTT] = useRecoilState<string>(store.endpointSTT);
|
||||
|
||||
const { transcript, listening, resetTranscript, browserSupportsSpeechRecognition } =
|
||||
useSpeechRecognition();
|
||||
|
||||
const toggleListening = () => {
|
||||
if (browserSupportsSpeechRecognition) {
|
||||
if (listening) {
|
||||
SpeechRecognition.stopListening();
|
||||
} else {
|
||||
SpeechRecognition.startListening();
|
||||
}
|
||||
} else {
|
||||
showToast({
|
||||
message: 'Browser does not support SpeechRecognition',
|
||||
status: 'error',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
if (e.shiftKey && e.altKey && e.code === 'KeyL' && endpointSTT === 'browser') {
|
||||
toggleListening();
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener('keydown', handleKeyDown);
|
||||
return () => window.removeEventListener('keydown', handleKeyDown);
|
||||
}, []);
|
||||
|
||||
return {
|
||||
isListening: listening,
|
||||
isLoading: false,
|
||||
text: transcript,
|
||||
startRecording: toggleListening,
|
||||
stopRecording: () => {
|
||||
SpeechRecognition.stopListening();
|
||||
resetTranscript();
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export default useSpeechToTextBrowser;
|
||||
238
client/src/hooks/Input/useSpeechToTextExternal.ts
Normal file
238
client/src/hooks/Input/useSpeechToTextExternal.ts
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
import { useState, useEffect, useRef } from 'react';
|
||||
import { useRecoilState } from 'recoil';
|
||||
import { useSpeechToTextMutation } from '~/data-provider';
|
||||
import { useToastContext } from '~/Providers';
|
||||
import store from '~/store';
|
||||
|
||||
const useSpeechToTextExternal = (onTranscriptionComplete: (text: string) => void) => {
|
||||
const { showToast } = useToastContext();
|
||||
const [endpointSTT] = useRecoilState<string>(store.endpointSTT);
|
||||
const [speechToText] = useRecoilState<boolean>(store.SpeechToText);
|
||||
const [autoTranscribeAudio] = useRecoilState<boolean>(store.autoTranscribeAudio);
|
||||
const [autoSendText] = useRecoilState<boolean>(store.autoSendText);
|
||||
const [text, setText] = useState<string>('');
|
||||
const [isListening, setIsListening] = useState(false);
|
||||
const [permission, setPermission] = useState(false);
|
||||
const [audioChunks, setAudioChunks] = useState<Blob[]>([]);
|
||||
const [isRequestBeingMade, setIsRequestBeingMade] = useState(false);
|
||||
const [minDecibels] = useRecoilState(store.decibelValue);
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const audioStream = useRef<MediaStream | null>(null);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const animationFrameIdRef = useRef<number | null>(null);
|
||||
|
||||
const { mutate: processAudio, isLoading: isProcessing } = useSpeechToTextMutation({
|
||||
onSuccess: (data) => {
|
||||
const extractedText = data.text;
|
||||
setText(extractedText);
|
||||
setIsRequestBeingMade(false);
|
||||
if (autoSendText && speechToText && extractedText.length > 0) {
|
||||
setTimeout(() => {
|
||||
onTranscriptionComplete(extractedText);
|
||||
}, 3000);
|
||||
}
|
||||
},
|
||||
onError: () => {
|
||||
showToast({
|
||||
message: 'An error occurred while processing the audio, maybe the audio was too short',
|
||||
status: 'error',
|
||||
});
|
||||
setIsRequestBeingMade(false);
|
||||
},
|
||||
});
|
||||
|
||||
const cleanup = () => {
|
||||
if (mediaRecorderRef.current) {
|
||||
mediaRecorderRef.current.removeEventListener('dataavailable', handleDataAvailable);
|
||||
mediaRecorderRef.current.removeEventListener('stop', handleStop);
|
||||
mediaRecorderRef.current = null;
|
||||
}
|
||||
};
|
||||
|
||||
const clearText = () => {
|
||||
setText('');
|
||||
};
|
||||
|
||||
const getMicrophonePermission = async () => {
|
||||
try {
|
||||
const streamData = await navigator.mediaDevices.getUserMedia({
|
||||
audio: true,
|
||||
video: false,
|
||||
});
|
||||
setPermission(true);
|
||||
audioStream.current = streamData ?? null;
|
||||
} catch (err) {
|
||||
setPermission(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleDataAvailable = (event: BlobEvent) => {
|
||||
if (event.data.size > 0) {
|
||||
audioChunks.push(event.data);
|
||||
} else {
|
||||
showToast({ message: 'No audio data available', status: 'warning' });
|
||||
}
|
||||
};
|
||||
|
||||
const handleStop = () => {
|
||||
if (audioChunks.length > 0) {
|
||||
const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
|
||||
|
||||
setAudioChunks([]);
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('audio', audioBlob, 'audio.wav');
|
||||
setIsRequestBeingMade(true);
|
||||
cleanup();
|
||||
processAudio(formData);
|
||||
} else {
|
||||
showToast({ message: 'The audio was too short', status: 'warning' });
|
||||
}
|
||||
};
|
||||
|
||||
const monitorSilence = (stream: MediaStream, stopRecording: () => void) => {
|
||||
const audioContext = new AudioContext();
|
||||
const audioStreamSource = audioContext.createMediaStreamSource(stream);
|
||||
const analyser = audioContext.createAnalyser();
|
||||
analyser.minDecibels = minDecibels;
|
||||
audioStreamSource.connect(analyser);
|
||||
|
||||
const bufferLength = analyser.frequencyBinCount;
|
||||
const domainData = new Uint8Array(bufferLength);
|
||||
let lastSoundTime = Date.now();
|
||||
|
||||
const detectSound = () => {
|
||||
analyser.getByteFrequencyData(domainData);
|
||||
const isSoundDetected = domainData.some((value) => value > 0);
|
||||
|
||||
if (isSoundDetected) {
|
||||
lastSoundTime = Date.now();
|
||||
}
|
||||
|
||||
const timeSinceLastSound = Date.now() - lastSoundTime;
|
||||
const isOverSilenceThreshold = timeSinceLastSound > 3000;
|
||||
|
||||
if (isOverSilenceThreshold) {
|
||||
stopRecording();
|
||||
return;
|
||||
}
|
||||
|
||||
animationFrameIdRef.current = window.requestAnimationFrame(detectSound);
|
||||
};
|
||||
|
||||
animationFrameIdRef.current = window.requestAnimationFrame(detectSound);
|
||||
};
|
||||
|
||||
const startRecording = async () => {
|
||||
if (isRequestBeingMade) {
|
||||
showToast({ message: 'A request is already being made. Please wait.', status: 'warning' });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!audioStream.current) {
|
||||
await getMicrophonePermission();
|
||||
}
|
||||
|
||||
if (audioStream.current) {
|
||||
try {
|
||||
setAudioChunks([]);
|
||||
mediaRecorderRef.current = new MediaRecorder(audioStream.current);
|
||||
mediaRecorderRef.current.addEventListener('dataavailable', handleDataAvailable);
|
||||
mediaRecorderRef.current.addEventListener('stop', handleStop);
|
||||
mediaRecorderRef.current.start(100);
|
||||
if (!audioContextRef.current && autoTranscribeAudio && speechToText) {
|
||||
monitorSilence(audioStream.current, stopRecording);
|
||||
}
|
||||
setIsListening(true);
|
||||
} catch (error) {
|
||||
showToast({ message: `Error starting recording: ${error}`, status: 'error' });
|
||||
}
|
||||
} else {
|
||||
showToast({ message: 'Microphone permission not granted', status: 'error' });
|
||||
}
|
||||
};
|
||||
|
||||
const stopRecording = () => {
|
||||
if (!mediaRecorderRef.current) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (mediaRecorderRef.current.state === 'recording') {
|
||||
mediaRecorderRef.current.stop();
|
||||
|
||||
audioStream.current?.getTracks().forEach((track) => track.stop());
|
||||
audioStream.current = null;
|
||||
|
||||
if (animationFrameIdRef.current !== null) {
|
||||
window.cancelAnimationFrame(animationFrameIdRef.current);
|
||||
animationFrameIdRef.current = null;
|
||||
}
|
||||
|
||||
setIsListening(false);
|
||||
} else {
|
||||
showToast({ message: 'MediaRecorder is not recording', status: 'error' });
|
||||
}
|
||||
};
|
||||
|
||||
const externalStartRecording = () => {
|
||||
if (isListening) {
|
||||
showToast({ message: 'Already listening. Please stop recording first.', status: 'warning' });
|
||||
return;
|
||||
}
|
||||
|
||||
startRecording();
|
||||
};
|
||||
|
||||
const externalStopRecording = () => {
|
||||
if (!isListening) {
|
||||
showToast({
|
||||
message: 'Not currently recording. Please start recording first.',
|
||||
status: 'warning',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
stopRecording();
|
||||
};
|
||||
|
||||
const handleKeyDown = async (e: KeyboardEvent) => {
|
||||
if (e.shiftKey && e.altKey && e.code === 'KeyL' && endpointSTT !== 'browser') {
|
||||
if (!window.MediaRecorder) {
|
||||
showToast({ message: 'MediaRecorder is not supported in this browser', status: 'error' });
|
||||
return;
|
||||
}
|
||||
|
||||
if (permission === false) {
|
||||
await getMicrophonePermission();
|
||||
}
|
||||
|
||||
if (isListening) {
|
||||
stopRecording();
|
||||
} else {
|
||||
startRecording();
|
||||
}
|
||||
|
||||
e.preventDefault();
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
window.addEventListener('keydown', handleKeyDown);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener('keydown', handleKeyDown);
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [isListening]);
|
||||
|
||||
return {
|
||||
isListening,
|
||||
isLoading: isProcessing,
|
||||
text,
|
||||
externalStartRecording,
|
||||
externalStopRecording,
|
||||
clearText,
|
||||
};
|
||||
};
|
||||
|
||||
export default useSpeechToTextExternal;
|
||||
67
client/src/hooks/Input/useTextToSpeech.ts
Normal file
67
client/src/hooks/Input/useTextToSpeech.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import { useRef } from 'react';
|
||||
import useTextToSpeechBrowser from './useTextToSpeechBrowser';
|
||||
import useTextToSpeechExternal from './useTextToSpeechExternal';
|
||||
import { usePauseGlobalAudio } from '../Audio';
|
||||
import { useRecoilState } from 'recoil';
|
||||
import store from '~/store';
|
||||
|
||||
const useTextToSpeech = (message: string, isLast: boolean, index = 0) => {
|
||||
const [endpointTTS] = useRecoilState<string>(store.endpointTTS);
|
||||
const useExternalTextToSpeech = endpointTTS === 'external';
|
||||
|
||||
const {
|
||||
generateSpeechLocal: generateSpeechLocal,
|
||||
cancelSpeechLocal: cancelSpeechLocal,
|
||||
isSpeaking: isSpeakingLocal,
|
||||
} = useTextToSpeechBrowser();
|
||||
|
||||
const {
|
||||
generateSpeechExternal: generateSpeechExternal,
|
||||
cancelSpeech: cancelSpeechExternal,
|
||||
isSpeaking: isSpeakingExternal,
|
||||
isLoading: isLoading,
|
||||
} = useTextToSpeechExternal(isLast, index);
|
||||
const { pauseGlobalAudio } = usePauseGlobalAudio(index);
|
||||
|
||||
const generateSpeech = useExternalTextToSpeech ? generateSpeechExternal : generateSpeechLocal;
|
||||
const cancelSpeech = useExternalTextToSpeech ? cancelSpeechExternal : cancelSpeechLocal;
|
||||
const isSpeaking = useExternalTextToSpeech ? isSpeakingExternal : isSpeakingLocal;
|
||||
|
||||
const isMouseDownRef = useRef(false);
|
||||
const timerRef = useRef<number | undefined>(undefined);
|
||||
|
||||
const handleMouseDown = () => {
|
||||
isMouseDownRef.current = true;
|
||||
timerRef.current = window.setTimeout(() => {
|
||||
if (isMouseDownRef.current) {
|
||||
generateSpeech(message, true);
|
||||
}
|
||||
}, 1000);
|
||||
};
|
||||
|
||||
const handleMouseUp = () => {
|
||||
isMouseDownRef.current = false;
|
||||
if (timerRef.current) {
|
||||
window.clearTimeout(timerRef.current);
|
||||
}
|
||||
};
|
||||
|
||||
const toggleSpeech = () => {
|
||||
if (isSpeaking) {
|
||||
cancelSpeech();
|
||||
pauseGlobalAudio();
|
||||
} else {
|
||||
generateSpeech(message, false);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
handleMouseDown,
|
||||
handleMouseUp,
|
||||
toggleSpeech,
|
||||
isSpeaking,
|
||||
isLoading,
|
||||
};
|
||||
};
|
||||
|
||||
export default useTextToSpeech;
|
||||
26
client/src/hooks/Input/useTextToSpeechBrowser.ts
Normal file
26
client/src/hooks/Input/useTextToSpeechBrowser.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import { useState } from 'react';
|
||||
|
||||
function useTextToSpeechBrowser() {
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
|
||||
const generateSpeechLocal = (text: string) => {
|
||||
const synth = window.speechSynthesis;
|
||||
synth.cancel();
|
||||
const utterance = new SpeechSynthesisUtterance(text);
|
||||
utterance.onend = () => {
|
||||
setIsSpeaking(false);
|
||||
};
|
||||
setIsSpeaking(true);
|
||||
synth.speak(utterance);
|
||||
};
|
||||
|
||||
const cancelSpeechLocal = () => {
|
||||
const synth = window.speechSynthesis;
|
||||
synth.cancel();
|
||||
setIsSpeaking(false);
|
||||
};
|
||||
|
||||
return { generateSpeechLocal, cancelSpeechLocal, isSpeaking };
|
||||
}
|
||||
|
||||
export default useTextToSpeechBrowser;
|
||||
155
client/src/hooks/Input/useTextToSpeechExternal.ts
Normal file
155
client/src/hooks/Input/useTextToSpeechExternal.ts
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
import { useRecoilValue } from 'recoil';
|
||||
import { useCallback, useEffect, useState, useMemo } from 'react';
|
||||
import { useTextToSpeechMutation } from '~/data-provider';
|
||||
import { useToastContext } from '~/Providers';
|
||||
import store from '~/store';
|
||||
|
||||
const createFormData = (text: string, voice: string) => {
|
||||
const formData = new FormData();
|
||||
formData.append('input', text);
|
||||
formData.append('voice', voice);
|
||||
return formData;
|
||||
};
|
||||
|
||||
function useTextToSpeechExternal(isLast: boolean, index = 0) {
|
||||
const { showToast } = useToastContext();
|
||||
const voice = useRecoilValue(store.voice);
|
||||
const cacheTTS = useRecoilValue(store.cacheTTS);
|
||||
const playbackRate = useRecoilValue(store.playbackRate);
|
||||
|
||||
const [text, setText] = useState<string | null>(null);
|
||||
const [downloadFile, setDownloadFile] = useState(false);
|
||||
const [isLocalSpeaking, setIsSpeaking] = useState(false);
|
||||
const [blobUrl, setBlobUrl] = useState<string | null>(null);
|
||||
const [audio, setAudio] = useState<HTMLAudioElement | null>(null);
|
||||
|
||||
/* Global Audio Variables */
|
||||
const globalIsFetching = useRecoilValue(store.globalAudioFetchingFamily(index));
|
||||
const globalIsPlaying = useRecoilValue(store.globalAudioPlayingFamily(index));
|
||||
|
||||
const playAudio = (blobUrl: string) => {
|
||||
const newAudio = new Audio(blobUrl);
|
||||
if (playbackRate && playbackRate !== 1) {
|
||||
newAudio.playbackRate = playbackRate;
|
||||
}
|
||||
|
||||
const playPromise = () => newAudio.play().then(() => setIsSpeaking(true));
|
||||
|
||||
playPromise().catch((error: Error) => {
|
||||
if (
|
||||
error?.message &&
|
||||
error.message.includes('The play() request was interrupted by a call to pause()')
|
||||
) {
|
||||
return playPromise().catch(console.error);
|
||||
}
|
||||
console.error(error);
|
||||
showToast({ message: `Error playing audio: ${error.message}`, status: 'error' });
|
||||
});
|
||||
|
||||
newAudio.onended = () => {
|
||||
console.log('Target message audio ended');
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
setIsSpeaking(false);
|
||||
};
|
||||
|
||||
setAudio(newAudio);
|
||||
setBlobUrl(blobUrl);
|
||||
};
|
||||
|
||||
const downloadAudio = (blobUrl: string) => {
|
||||
const a = document.createElement('a');
|
||||
a.href = blobUrl;
|
||||
a.download = 'audio.mp3';
|
||||
a.click();
|
||||
setDownloadFile(false);
|
||||
};
|
||||
|
||||
const { mutate: processAudio, isLoading: isProcessing } = useTextToSpeechMutation({
|
||||
onSuccess: async (data: ArrayBuffer) => {
|
||||
try {
|
||||
const mediaSource = new MediaSource();
|
||||
const audio = new Audio();
|
||||
audio.src = URL.createObjectURL(mediaSource);
|
||||
audio.autoplay = true;
|
||||
|
||||
mediaSource.onsourceopen = () => {
|
||||
const sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg');
|
||||
sourceBuffer.appendBuffer(data);
|
||||
};
|
||||
|
||||
audio.onended = () => {
|
||||
URL.revokeObjectURL(audio.src);
|
||||
setIsSpeaking(false);
|
||||
};
|
||||
|
||||
setAudio(audio);
|
||||
|
||||
if (cacheTTS) {
|
||||
const cache = await caches.open('tts-responses');
|
||||
const request = new Request(text!);
|
||||
const response = new Response(new Blob([data], { type: 'audio/mpeg' }));
|
||||
cache.put(request, response);
|
||||
}
|
||||
|
||||
if (downloadFile) {
|
||||
downloadAudio(audio.src);
|
||||
}
|
||||
} catch (error) {
|
||||
showToast({
|
||||
message: `Error processing audio: ${(error as Error).message}`,
|
||||
status: 'error',
|
||||
});
|
||||
}
|
||||
},
|
||||
onError: (error: unknown) => {
|
||||
showToast({ message: `Error: ${(error as Error).message}`, status: 'error' });
|
||||
},
|
||||
});
|
||||
|
||||
const generateSpeechExternal = async (text: string, download: boolean) => {
|
||||
setText(text);
|
||||
const cachedResponse = await getCachedResponse(text);
|
||||
|
||||
if (cachedResponse && cacheTTS) {
|
||||
handleCachedResponse(cachedResponse, download);
|
||||
} else {
|
||||
const formData = createFormData(text, voice);
|
||||
setDownloadFile(download);
|
||||
processAudio(formData);
|
||||
}
|
||||
};
|
||||
|
||||
const getCachedResponse = async (text: string) => await caches.match(text);
|
||||
|
||||
const handleCachedResponse = async (cachedResponse: Response, download: boolean) => {
|
||||
const audioBlob = await cachedResponse.blob();
|
||||
const blobUrl = URL.createObjectURL(audioBlob);
|
||||
if (download) {
|
||||
downloadAudio(blobUrl);
|
||||
} else {
|
||||
playAudio(blobUrl);
|
||||
}
|
||||
};
|
||||
|
||||
const cancelSpeech = useCallback(() => {
|
||||
if (audio) {
|
||||
audio.pause();
|
||||
blobUrl && URL.revokeObjectURL(blobUrl);
|
||||
setIsSpeaking(false);
|
||||
}
|
||||
}, [audio, blobUrl]);
|
||||
|
||||
useEffect(() => cancelSpeech, [cancelSpeech]);
|
||||
|
||||
const isLoading = useMemo(() => {
|
||||
return isProcessing || (isLast && globalIsFetching && !globalIsPlaying);
|
||||
}, [isProcessing, globalIsFetching, globalIsPlaying, isLast]);
|
||||
|
||||
const isSpeaking = useMemo(() => {
|
||||
return isLocalSpeaking || (isLast && globalIsPlaying);
|
||||
}, [isLocalSpeaking, globalIsPlaying, isLast]);
|
||||
|
||||
return { generateSpeechExternal, cancelSpeech, isLoading, isSpeaking };
|
||||
}
|
||||
|
||||
export default useTextToSpeechExternal;
|
||||
|
|
@ -10,6 +10,7 @@ export default function useMessageHelpers(props: TMessageProps) {
|
|||
|
||||
const {
|
||||
ask,
|
||||
index,
|
||||
regenerate,
|
||||
isSubmitting,
|
||||
conversation,
|
||||
|
|
@ -71,6 +72,7 @@ export default function useMessageHelpers(props: TMessageProps) {
|
|||
return {
|
||||
ask,
|
||||
edit,
|
||||
index,
|
||||
isLast,
|
||||
assistant,
|
||||
enterEdit,
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ type TSyncData = {
|
|||
export default function useSSE(submission: TSubmission | null, index = 0) {
|
||||
const queryClient = useQueryClient();
|
||||
const genTitle = useGenTitleMutation();
|
||||
const setActiveRunId = useSetRecoilState(store.activeRunFamily(index));
|
||||
|
||||
const { conversationId: paramId } = useParams();
|
||||
const { token, isAuthenticated } = useAuthContext();
|
||||
|
|
@ -86,7 +87,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
(data: string, submission: TSubmission) => {
|
||||
const {
|
||||
messages,
|
||||
message,
|
||||
userMessage,
|
||||
plugin,
|
||||
plugins,
|
||||
initialResponse,
|
||||
|
|
@ -99,8 +100,6 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
{
|
||||
...initialResponse,
|
||||
text: data,
|
||||
parentMessageId: message?.overrideParentMessageId ?? null,
|
||||
messageId: message?.overrideParentMessageId + '_',
|
||||
plugin: plugin ?? null,
|
||||
plugins: plugins ?? [],
|
||||
// unfinished: true
|
||||
|
|
@ -109,12 +108,10 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
} else {
|
||||
setMessages([
|
||||
...messages,
|
||||
message,
|
||||
userMessage,
|
||||
{
|
||||
...initialResponse,
|
||||
text: data,
|
||||
parentMessageId: message?.messageId,
|
||||
messageId: message?.messageId + '_',
|
||||
plugin: plugin ?? null,
|
||||
plugins: plugins ?? [],
|
||||
// unfinished: true
|
||||
|
|
@ -175,9 +172,9 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
const syncHandler = useCallback(
|
||||
(data: TSyncData, submission: TSubmission) => {
|
||||
const { conversationId, thread_id, responseMessage, requestMessage } = data;
|
||||
const { initialResponse, messages: _messages, message } = submission;
|
||||
const { initialResponse, messages: _messages, userMessage } = submission;
|
||||
|
||||
const messages = _messages.filter((msg) => msg.messageId !== message.messageId);
|
||||
const messages = _messages.filter((msg) => msg.messageId !== userMessage.messageId);
|
||||
|
||||
setMessages([
|
||||
...messages,
|
||||
|
|
@ -229,35 +226,24 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
|
||||
const createdHandler = useCallback(
|
||||
(data: TResData, submission: TSubmission) => {
|
||||
const { messages, message, initialResponse, isRegenerate = false } = submission;
|
||||
|
||||
const { messages, userMessage, isRegenerate = false } = submission;
|
||||
const initialResponse = {
|
||||
...submission.initialResponse,
|
||||
parentMessageId: userMessage?.messageId,
|
||||
messageId: userMessage?.messageId + '_',
|
||||
};
|
||||
if (isRegenerate) {
|
||||
setMessages([
|
||||
...messages,
|
||||
{
|
||||
...initialResponse,
|
||||
parentMessageId: message?.overrideParentMessageId ?? null,
|
||||
messageId: message?.overrideParentMessageId + '_',
|
||||
},
|
||||
]);
|
||||
setMessages([...messages, initialResponse]);
|
||||
} else {
|
||||
setMessages([
|
||||
...messages,
|
||||
message,
|
||||
{
|
||||
...initialResponse,
|
||||
parentMessageId: message?.messageId,
|
||||
messageId: message?.messageId + '_',
|
||||
},
|
||||
]);
|
||||
setMessages([...messages, userMessage, initialResponse]);
|
||||
}
|
||||
|
||||
const { conversationId, parentMessageId } = message;
|
||||
const { conversationId, parentMessageId } = userMessage;
|
||||
|
||||
let update = {} as TConversation;
|
||||
setConversation((prevState) => {
|
||||
let title = prevState?.title;
|
||||
const parentId = isRegenerate ? message?.overrideParentMessageId : parentMessageId;
|
||||
const parentId = isRegenerate ? userMessage?.overrideParentMessageId : parentMessageId;
|
||||
if (parentId !== Constants.NO_PARENT && title?.toLowerCase()?.includes('new chat')) {
|
||||
const convos = queryClient.getQueryData<ConversationData>([QueryKeys.allConversations]);
|
||||
const cachedConvo = getConversationById(convos, conversationId);
|
||||
|
|
@ -342,11 +328,11 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
|
||||
const errorHandler = useCallback(
|
||||
({ data, submission }: { data?: TResData; submission: TSubmission }) => {
|
||||
const { messages, message, initialResponse } = submission;
|
||||
const { messages, userMessage, initialResponse } = submission;
|
||||
|
||||
setCompleted((prev) => new Set(prev.add(initialResponse.messageId)));
|
||||
|
||||
const conversationId = message?.conversationId ?? submission?.conversationId;
|
||||
const conversationId = userMessage?.conversationId ?? submission?.conversationId;
|
||||
|
||||
const parseErrorResponse = (data: TResData | Partial<TMessage>) => {
|
||||
const metadata = data['responseMessage'] ?? data;
|
||||
|
|
@ -354,7 +340,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
...initialResponse,
|
||||
...metadata,
|
||||
error: true,
|
||||
parentMessageId: message?.messageId,
|
||||
parentMessageId: userMessage?.messageId,
|
||||
};
|
||||
|
||||
if (!errorMessage.messageId) {
|
||||
|
|
@ -371,7 +357,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
...submission,
|
||||
conversationId: convoId,
|
||||
});
|
||||
setMessages([...messages, message, errorResponse]);
|
||||
setMessages([...messages, userMessage, errorResponse]);
|
||||
newConversation({
|
||||
template: { conversationId: convoId },
|
||||
preset: tPresetSchema.parse(submission?.conversation),
|
||||
|
|
@ -383,7 +369,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
if (!conversationId && !data.conversationId) {
|
||||
const convoId = v4();
|
||||
const errorResponse = parseErrorResponse(data);
|
||||
setMessages([...messages, message, errorResponse]);
|
||||
setMessages([...messages, userMessage, errorResponse]);
|
||||
newConversation({
|
||||
template: { conversationId: convoId },
|
||||
preset: tPresetSchema.parse(submission?.conversation),
|
||||
|
|
@ -392,7 +378,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
return;
|
||||
} else if (!data.conversationId) {
|
||||
const errorResponse = parseErrorResponse(data);
|
||||
setMessages([...messages, message, errorResponse]);
|
||||
setMessages([...messages, userMessage, errorResponse]);
|
||||
setIsSubmitting(false);
|
||||
return;
|
||||
}
|
||||
|
|
@ -401,10 +387,10 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
const errorResponse = tMessageSchema.parse({
|
||||
...data,
|
||||
error: true,
|
||||
parentMessageId: message?.messageId,
|
||||
parentMessageId: userMessage?.messageId,
|
||||
});
|
||||
|
||||
setMessages([...messages, message, errorResponse]);
|
||||
setMessages([...messages, userMessage, errorResponse]);
|
||||
if (data.conversationId && paramId === 'new') {
|
||||
newConversation({
|
||||
template: { conversationId: data.conversationId },
|
||||
|
|
@ -466,7 +452,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
};
|
||||
|
||||
const data = {
|
||||
requestMessage: submission.message,
|
||||
requestMessage: submission.userMessage,
|
||||
responseMessage: responseMessage,
|
||||
conversation: submission.conversation,
|
||||
};
|
||||
|
|
@ -493,7 +479,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
error: true,
|
||||
};
|
||||
const errorResponse = tMessageSchema.parse(errorMessage);
|
||||
setMessages([...submission.messages, submission.message, errorResponse]);
|
||||
setMessages([...submission.messages, submission.userMessage, errorResponse]);
|
||||
newConversation({
|
||||
template: { conversationId: convoId },
|
||||
preset: tPresetSchema.parse(submission?.conversation),
|
||||
|
|
@ -509,7 +495,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
return;
|
||||
}
|
||||
|
||||
let { message } = submission;
|
||||
let { userMessage } = submission;
|
||||
|
||||
const payloadData = createPayload(submission);
|
||||
let { payload } = payloadData;
|
||||
|
|
@ -529,20 +515,25 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
|
||||
if (data.final) {
|
||||
const { plugins } = data;
|
||||
finalHandler(data, { ...submission, plugins, message });
|
||||
finalHandler(data, { ...submission, plugins });
|
||||
startupConfig?.checkBalance && balanceQuery.refetch();
|
||||
console.log('final', data);
|
||||
}
|
||||
if (data.created) {
|
||||
message = {
|
||||
...message,
|
||||
const runId = v4();
|
||||
setActiveRunId(runId);
|
||||
userMessage = {
|
||||
...userMessage,
|
||||
...data.message,
|
||||
overrideParentMessageId: message?.overrideParentMessageId,
|
||||
overrideParentMessageId: userMessage?.overrideParentMessageId,
|
||||
};
|
||||
createdHandler(data, { ...submission, message });
|
||||
|
||||
createdHandler(data, { ...submission, userMessage });
|
||||
} else if (data.sync) {
|
||||
const runId = v4();
|
||||
setActiveRunId(runId);
|
||||
/* synchronize messages to Assistants API as well as with real DB ID's */
|
||||
syncHandler(data, { ...submission, message });
|
||||
syncHandler(data, { ...submission, userMessage });
|
||||
} else if (data.type) {
|
||||
const { text, index } = data;
|
||||
if (text && index !== textIndex) {
|
||||
|
|
@ -554,12 +545,28 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
const text = data.text || data.response;
|
||||
const { plugin, plugins } = data;
|
||||
|
||||
const initialResponse = {
|
||||
...submission.initialResponse,
|
||||
parentMessageId: data.parentMessageId,
|
||||
messageId: data.messageId,
|
||||
};
|
||||
|
||||
if (data.message) {
|
||||
messageHandler(text, { ...submission, plugin, plugins, message });
|
||||
messageHandler(text, { ...submission, plugin, plugins, userMessage, initialResponse });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// events.onaudio = (e: MessageEvent) => {
|
||||
// const data = JSON.parse(e.data);
|
||||
// console.log('audio', data);
|
||||
// if (data.audio) {
|
||||
// audioSource.addBase64Data(data.audio);
|
||||
// }
|
||||
// };
|
||||
|
||||
// events.onend = () => audioSource.close();
|
||||
|
||||
events.onopen = () => console.log('connection is opened');
|
||||
|
||||
events.oncancel = async () => {
|
||||
|
|
@ -575,7 +582,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
|
||||
setCompleted((prev) => new Set(prev.add(streamKey)));
|
||||
return await abortConversation(
|
||||
message?.conversationId ?? submission?.conversationId,
|
||||
userMessage?.conversationId ?? submission?.conversationId,
|
||||
submission,
|
||||
);
|
||||
};
|
||||
|
|
@ -594,7 +601,7 @@ export default function useSSE(submission: TSubmission | null, index = 0) {
|
|||
setIsSubmitting(false);
|
||||
}
|
||||
|
||||
errorHandler({ data, submission: { ...submission, message } });
|
||||
errorHandler({ data, submission: { ...submission, userMessage } });
|
||||
};
|
||||
|
||||
setIsSubmitting(true);
|
||||
|
|
|
|||
|
|
@ -23,3 +23,5 @@ export { default as useLocalStorage } from './useLocalStorage';
|
|||
export { default as useDelayedRender } from './useDelayedRender';
|
||||
export { default as useOnClickOutside } from './useOnClickOutside';
|
||||
export { default as useGenerationsByLatest } from './useGenerationsByLatest';
|
||||
export { default as useSpeechToText } from './Input/useSpeechToText';
|
||||
export { default as useTextToSpeech } from './Input/useTextToSpeech';
|
||||
|
|
|
|||
|
|
@ -240,7 +240,7 @@ export default function useChatHelpers(index = 0, paramId: string | undefined) {
|
|||
conversationId,
|
||||
},
|
||||
endpointOption,
|
||||
message: {
|
||||
userMessage: {
|
||||
...currentMsg,
|
||||
generation,
|
||||
responseMessageId,
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ import {
|
|||
} from '~/utils';
|
||||
import useAssistantListMap from './Assistants/useAssistantListMap';
|
||||
import { useDeleteFilesMutation } from '~/data-provider';
|
||||
|
||||
import { usePauseGlobalAudio } from './Audio';
|
||||
import { mainTextareaId } from '~/common';
|
||||
import store from '~/store';
|
||||
|
||||
|
|
@ -47,6 +47,7 @@ const useNewConvo = (index = 0) => {
|
|||
const modelsQuery = useGetModelsQuery();
|
||||
const timeoutIdRef = useRef<NodeJS.Timeout>();
|
||||
const assistantsListMap = useAssistantListMap();
|
||||
const { pauseGlobalAudio } = usePauseGlobalAudio(index);
|
||||
|
||||
const { mutateAsync } = useDeleteFilesMutation({
|
||||
onSuccess: () => {
|
||||
|
|
@ -176,6 +177,8 @@ const useNewConvo = (index = 0) => {
|
|||
buildDefault?: boolean;
|
||||
keepLatestMessage?: boolean;
|
||||
} = {}) => {
|
||||
pauseGlobalAudio();
|
||||
|
||||
const conversation = {
|
||||
conversationId: 'new',
|
||||
title: 'New Chat',
|
||||
|
|
@ -215,7 +218,7 @@ const useNewConvo = (index = 0) => {
|
|||
|
||||
switchToConversation(conversation, preset, modelsData, buildDefault, keepLatestMessage);
|
||||
},
|
||||
[switchToConversation, files, mutateAsync, setFiles, startupConfig],
|
||||
[pauseGlobalAudio, switchToConversation, mutateAsync, setFiles, files, startupConfig],
|
||||
);
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ export default {
|
|||
com_ui_save: 'Save',
|
||||
com_ui_save_submit: 'Save & Submit',
|
||||
com_user_message: 'You',
|
||||
com_ui_read_aloud: 'Read aloud',
|
||||
com_ui_copied: 'Copied!',
|
||||
com_ui_copy_code: 'Copy code',
|
||||
com_ui_copy_to_clipboard: 'Copy to clipboard',
|
||||
|
|
@ -241,6 +242,7 @@ export default {
|
|||
'Uploading "{0}" is taking more time than anticipated. Please wait while the file finishes indexing for retrieval.',
|
||||
com_ui_privacy_policy: 'Privacy policy',
|
||||
com_ui_terms_of_service: 'Terms of service',
|
||||
com_ui_use_micrphone: 'Use microphone',
|
||||
com_ui_min_tags: 'Cannot remove more values, a minimum of {0} are required.',
|
||||
com_ui_max_tags: 'Maximum number allowed is {0}, using latest values.',
|
||||
com_auth_error_login:
|
||||
|
|
@ -478,6 +480,9 @@ export default {
|
|||
com_nav_hide_panel: 'Hide right-most side panel',
|
||||
com_nav_modular_chat: 'Enable switching Endpoints mid-conversation',
|
||||
com_nav_latex_parsing: 'Parsing LaTeX in messages (may affect performance)',
|
||||
com_nav_text_to_speech: 'Text to Speech',
|
||||
com_nav_automatic_playback: 'Autoplay Latest Message (external only)',
|
||||
com_nav_speech_to_text: 'Speech to Text',
|
||||
com_nav_profile_picture: 'Profile Picture',
|
||||
com_nav_change_picture: 'Change picture',
|
||||
com_nav_plugin_store: 'Plugin store',
|
||||
|
|
@ -540,10 +545,22 @@ export default {
|
|||
com_nav_help_faq: 'Help & FAQ',
|
||||
com_nav_settings: 'Settings',
|
||||
com_nav_search_placeholder: 'Search messages',
|
||||
com_nav_conversation_mode: 'Conversation Mode',
|
||||
com_nav_auto_send_text: 'Auto send text (after 3 sec)',
|
||||
com_nav_auto_transcribe_audio: 'Auto transcribe audio',
|
||||
com_nav_db_sensitivity: 'Decibel sensitivity',
|
||||
com_nav_playback_rate: 'Audio Playback Rate',
|
||||
com_nav_engine: 'Engine',
|
||||
com_nav_browser: 'Browser',
|
||||
com_nav_external: 'External',
|
||||
com_nav_delete_cache_storage: 'Delete cache storage',
|
||||
com_nav_enable_cache_tts: 'Enable cache TTS',
|
||||
com_nav_voice_select: 'Voice',
|
||||
com_nav_setting_general: 'General',
|
||||
com_nav_setting_beta: 'Beta features',
|
||||
com_nav_setting_data: 'Data controls',
|
||||
com_nav_setting_account: 'Account',
|
||||
com_nav_setting_speech: 'Speech',
|
||||
com_nav_language: 'Language',
|
||||
com_nav_lang_auto: 'Auto detect',
|
||||
com_nav_lang_english: 'English',
|
||||
|
|
|
|||
|
|
@ -523,6 +523,7 @@ export default {
|
|||
com_nav_setting_general: 'Generali',
|
||||
com_nav_setting_beta: 'Funzionalità beta',
|
||||
com_nav_setting_data: 'Controlli dati',
|
||||
com_nav_setting_speech: 'Voce',
|
||||
com_nav_setting_account: 'Account',
|
||||
/* The following are AI Translated */
|
||||
com_assistants_file_search: 'Ricerca File',
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import 'regenerator-runtime/runtime';
|
||||
import { createRoot } from 'react-dom/client';
|
||||
import App from './App';
|
||||
import './style.css';
|
||||
|
|
|
|||
|
|
@ -120,6 +120,31 @@ const showMentionPopoverFamily = atomFamily<boolean, string | number | null>({
|
|||
default: false,
|
||||
});
|
||||
|
||||
const globalAudioURLFamily = atomFamily<string | null, string | number | null>({
|
||||
key: 'globalAudioURLByIndex',
|
||||
default: null,
|
||||
});
|
||||
|
||||
const globalAudioFetchingFamily = atomFamily<boolean, string | number | null>({
|
||||
key: 'globalAudioisFetchingByIndex',
|
||||
default: false,
|
||||
});
|
||||
|
||||
const globalAudioPlayingFamily = atomFamily<boolean, string | number | null>({
|
||||
key: 'globalAudioisPlayingByIndex',
|
||||
default: false,
|
||||
});
|
||||
|
||||
const activeRunFamily = atomFamily<string | null, string | number | null>({
|
||||
key: 'activeRunByIndex',
|
||||
default: null,
|
||||
});
|
||||
|
||||
const audioRunFamily = atomFamily<string | null, string | number | null>({
|
||||
key: 'audioRunByIndex',
|
||||
default: null,
|
||||
});
|
||||
|
||||
const latestMessageFamily = atomFamily<TMessage | null, string | number | null>({
|
||||
key: 'latestMessageByIndex',
|
||||
default: null,
|
||||
|
|
@ -180,4 +205,9 @@ export default {
|
|||
useClearConvoState,
|
||||
useCreateConversationAtom,
|
||||
showMentionPopoverFamily,
|
||||
globalAudioURLFamily,
|
||||
activeRunFamily,
|
||||
audioRunFamily,
|
||||
globalAudioPlayingFamily,
|
||||
globalAudioFetchingFamily,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,254 +1,69 @@
|
|||
import { atom } from 'recoil';
|
||||
import { SettingsViews, LocalStorageKeys } from 'librechat-data-provider';
|
||||
import { SettingsViews } from 'librechat-data-provider';
|
||||
import type { TOptionSettings } from '~/common';
|
||||
|
||||
const abortScroll = atom<boolean>({
|
||||
key: 'abortScroll',
|
||||
default: false,
|
||||
});
|
||||
|
||||
const showFiles = atom<boolean>({
|
||||
key: 'showFiles',
|
||||
default: false,
|
||||
});
|
||||
|
||||
const optionSettings = atom<TOptionSettings>({
|
||||
key: 'optionSettings',
|
||||
default: {},
|
||||
});
|
||||
|
||||
const showPluginStoreDialog = atom<boolean>({
|
||||
key: 'showPluginStoreDialog',
|
||||
default: false,
|
||||
});
|
||||
|
||||
const showAgentSettings = atom<boolean>({
|
||||
key: 'showAgentSettings',
|
||||
default: false,
|
||||
});
|
||||
|
||||
const currentSettingsView = atom<SettingsViews>({
|
||||
key: 'currentSettingsView',
|
||||
default: SettingsViews.default,
|
||||
});
|
||||
|
||||
const showBingToneSetting = atom<boolean>({
|
||||
key: 'showBingToneSetting',
|
||||
default: false,
|
||||
});
|
||||
|
||||
const showPopover = atom<boolean>({
|
||||
key: 'showPopover',
|
||||
default: false,
|
||||
});
|
||||
|
||||
const autoScroll = atom<boolean>({
|
||||
key: 'autoScroll',
|
||||
default: localStorage.getItem('autoScroll') === 'true',
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem('autoScroll');
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue === 'true');
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'boolean') {
|
||||
localStorage.setItem('autoScroll', newValue.toString());
|
||||
// Improved helper function to create atoms with localStorage
|
||||
function atomWithLocalStorage<T>(key: string, defaultValue: T) {
|
||||
return atom<T>({
|
||||
key,
|
||||
default: defaultValue, // Set the default value directly
|
||||
effects_UNSTABLE: [
|
||||
({ setSelf, onSet }) => {
|
||||
// Load the initial value from localStorage if it exists
|
||||
const savedValue = localStorage.getItem(key);
|
||||
if (savedValue !== null) {
|
||||
setSelf(JSON.parse(savedValue));
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
const showCode = atom<boolean>({
|
||||
key: 'showCode',
|
||||
default: localStorage.getItem('showCode') === 'true',
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem('showCode');
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue === 'true');
|
||||
}
|
||||
// Update localStorage whenever the atom's value changes
|
||||
onSet((newValue: T) => {
|
||||
localStorage.setItem(key, JSON.stringify(newValue));
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'boolean') {
|
||||
localStorage.setItem('showCode', newValue.toString());
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
const hideSidePanel = atom<boolean>({
|
||||
key: 'hideSidePanel',
|
||||
default: localStorage.getItem('hideSidePanel') === 'true',
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem('hideSidePanel');
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue === 'true');
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'boolean') {
|
||||
localStorage.setItem('hideSidePanel', newValue.toString());
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
const modularChat = atom<boolean>({
|
||||
key: 'modularChat',
|
||||
default: true,
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem('modularChat');
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue === 'true');
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'boolean') {
|
||||
localStorage.setItem('modularChat', newValue.toString());
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
const LaTeXParsing = atom<boolean>({
|
||||
key: 'LaTeXParsing',
|
||||
default: true,
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem('LaTeXParsing');
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue === 'true');
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'boolean') {
|
||||
localStorage.setItem('LaTeXParsing', newValue.toString());
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
const forkSetting = atom<string>({
|
||||
key: LocalStorageKeys.FORK_SETTING,
|
||||
default: '',
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem(LocalStorageKeys.FORK_SETTING);
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue);
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'string') {
|
||||
localStorage.setItem(LocalStorageKeys.FORK_SETTING, newValue.toString());
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
const rememberForkOption = atom<boolean>({
|
||||
key: LocalStorageKeys.REMEMBER_FORK_OPTION,
|
||||
default: false,
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem(LocalStorageKeys.REMEMBER_FORK_OPTION);
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue === 'true');
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'boolean') {
|
||||
localStorage.setItem(LocalStorageKeys.REMEMBER_FORK_OPTION, newValue.toString());
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
const splitAtTarget = atom<boolean>({
|
||||
key: LocalStorageKeys.FORK_SPLIT_AT_TARGET,
|
||||
default: false,
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem(LocalStorageKeys.FORK_SPLIT_AT_TARGET);
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue === 'true');
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'boolean') {
|
||||
localStorage.setItem(LocalStorageKeys.FORK_SPLIT_AT_TARGET, newValue.toString());
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
const UsernameDisplay = atom<boolean>({
|
||||
key: 'UsernameDisplay',
|
||||
default: localStorage.getItem('UsernameDisplay') === 'true',
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem('UsernameDisplay');
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue === 'true');
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'boolean') {
|
||||
localStorage.setItem('UsernameDisplay', newValue.toString());
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
const enterToSend = atom<boolean>({
|
||||
key: 'enterToSend',
|
||||
default: true,
|
||||
effects: [
|
||||
({ setSelf, onSet }) => {
|
||||
const savedValue = localStorage.getItem('enterToSend');
|
||||
if (savedValue != null) {
|
||||
setSelf(savedValue === 'true');
|
||||
}
|
||||
|
||||
onSet((newValue: unknown) => {
|
||||
if (typeof newValue === 'boolean') {
|
||||
localStorage.setItem('enterToSend', newValue.toString());
|
||||
}
|
||||
});
|
||||
},
|
||||
] as const,
|
||||
});
|
||||
|
||||
export default {
|
||||
abortScroll,
|
||||
showFiles,
|
||||
optionSettings,
|
||||
showPluginStoreDialog,
|
||||
showAgentSettings,
|
||||
currentSettingsView,
|
||||
showBingToneSetting,
|
||||
showPopover,
|
||||
autoScroll,
|
||||
enterToSend,
|
||||
showCode,
|
||||
hideSidePanel,
|
||||
modularChat,
|
||||
LaTeXParsing,
|
||||
UsernameDisplay,
|
||||
forkSetting,
|
||||
splitAtTarget,
|
||||
rememberForkOption,
|
||||
// Static atoms without localStorage
|
||||
const staticAtoms = {
|
||||
abortScroll: atom<boolean>({ key: 'abortScroll', default: false }),
|
||||
showFiles: atom<boolean>({ key: 'showFiles', default: false }),
|
||||
optionSettings: atom<TOptionSettings>({ key: 'optionSettings', default: {} }),
|
||||
showPluginStoreDialog: atom<boolean>({ key: 'showPluginStoreDialog', default: false }),
|
||||
showAgentSettings: atom<boolean>({ key: 'showAgentSettings', default: false }),
|
||||
currentSettingsView: atom<SettingsViews>({
|
||||
key: 'currentSettingsView',
|
||||
default: SettingsViews.default,
|
||||
}),
|
||||
showBingToneSetting: atom<boolean>({ key: 'showBingToneSetting', default: false }),
|
||||
showPopover: atom<boolean>({ key: 'showPopover', default: false }),
|
||||
};
|
||||
|
||||
// Atoms with localStorage
|
||||
const localStorageAtoms = {
|
||||
autoScroll: atomWithLocalStorage('autoScroll', false),
|
||||
showCode: atomWithLocalStorage('showCode', false),
|
||||
hideSidePanel: atomWithLocalStorage('hideSidePanel', false),
|
||||
modularChat: atomWithLocalStorage('modularChat', false),
|
||||
LaTeXParsing: atomWithLocalStorage('LaTeXParsing', true),
|
||||
UsernameDisplay: atomWithLocalStorage('UsernameDisplay', true),
|
||||
TextToSpeech: atomWithLocalStorage('textToSpeech', true),
|
||||
automaticPlayback: atomWithLocalStorage('automaticPlayback', false),
|
||||
enterToSend: atomWithLocalStorage('enterToSend', true),
|
||||
SpeechToText: atomWithLocalStorage('speechToText', true),
|
||||
conversationMode: atomWithLocalStorage('conversationMode', false),
|
||||
advancedMode: atomWithLocalStorage('advancedMode', false),
|
||||
autoSendText: atomWithLocalStorage('autoSendText', false),
|
||||
autoTranscribeAudio: atomWithLocalStorage('autoTranscribeAudio', false),
|
||||
decibelValue: atomWithLocalStorage('decibelValue', -45),
|
||||
endpointSTT: atomWithLocalStorage('endpointSTT', 'browser'),
|
||||
endpointTTS: atomWithLocalStorage('endpointTTS', 'browser'),
|
||||
cacheTTS: atomWithLocalStorage('cacheTTS', true),
|
||||
voice: atomWithLocalStorage('voice', ''),
|
||||
forkSetting: atomWithLocalStorage('forkSetting', ''),
|
||||
splitAtTarget: atomWithLocalStorage('splitAtTarget', false),
|
||||
rememberForkOption: atomWithLocalStorage('rememberForkOption', true),
|
||||
playbackRate: atomWithLocalStorage<number | null>('playbackRate', null),
|
||||
};
|
||||
|
||||
export default { ...staticAtoms, ...localStorageAtoms };
|
||||
|
|
|
|||
134
config/translations/streamAudioTest.ts
Normal file
134
config/translations/streamAudioTest.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import { WebSocket } from 'ws';
|
||||
// const { ElevenLabsClient } = require('elevenlabs');
|
||||
|
||||
const ELEVENLABS_API_KEY = 'a495399653cc5824ba1e41d914473e07';
|
||||
const VOICE_ID = '1RVpBInY9YUYMLSUQReV';
|
||||
|
||||
interface AudioChunk {
|
||||
audio: string;
|
||||
isFinal: boolean;
|
||||
alignment: {
|
||||
char_start_times_ms: number[];
|
||||
chars_durations_ms: number[];
|
||||
chars: string[];
|
||||
};
|
||||
normalizedAlignment: {
|
||||
char_start_times_ms: number[];
|
||||
chars_durations_ms: number[];
|
||||
chars: string[];
|
||||
};
|
||||
}
|
||||
|
||||
export function inputStreamTextToSpeech(
|
||||
textStream: AsyncIterable<string>,
|
||||
): AsyncGenerator<AudioChunk> {
|
||||
const model = 'eleven_turbo_v2';
|
||||
const wsUrl = `wss://api.elevenlabs.io/v1/text-to-speech/${VOICE_ID}/stream-input?model_id=${model}`;
|
||||
const socket = new WebSocket(wsUrl);
|
||||
|
||||
socket.onopen = function () {
|
||||
const streamStart = {
|
||||
text: ' ',
|
||||
voice_settings: {
|
||||
stability: 0.5,
|
||||
similarity_boost: 0.8,
|
||||
},
|
||||
xi_api_key: ELEVENLABS_API_KEY,
|
||||
};
|
||||
|
||||
socket.send(JSON.stringify(streamStart));
|
||||
|
||||
// send stream until done
|
||||
const streamComplete = new Promise((resolve, reject) => {
|
||||
(async () => {
|
||||
for await (const message of textStream) {
|
||||
const request = {
|
||||
text: message,
|
||||
try_trigger_generation: true,
|
||||
};
|
||||
socket.send(JSON.stringify(request));
|
||||
}
|
||||
})()
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
});
|
||||
|
||||
streamComplete
|
||||
.then(() => {
|
||||
const endStream = {
|
||||
text: '',
|
||||
};
|
||||
|
||||
socket.send(JSON.stringify(endStream));
|
||||
})
|
||||
.catch((e) => {
|
||||
throw e;
|
||||
});
|
||||
};
|
||||
|
||||
return (async function* audioStream() {
|
||||
let isDone = false;
|
||||
let chunks: AudioChunk[] = [];
|
||||
let resolve: (value: unknown) => void;
|
||||
let waitForMessage = new Promise((r) => (resolve = r));
|
||||
|
||||
socket.onmessage = function (event) {
|
||||
console.log(event);
|
||||
const audioChunk = JSON.parse(event.data as string) as AudioChunk;
|
||||
if (audioChunk.audio && audioChunk.alignment) {
|
||||
chunks.push(audioChunk);
|
||||
resolve(null);
|
||||
waitForMessage = new Promise((r) => (resolve = r));
|
||||
}
|
||||
};
|
||||
|
||||
socket.onerror = function (error) {
|
||||
throw error;
|
||||
};
|
||||
|
||||
// Handle socket closing
|
||||
socket.onclose = function () {
|
||||
isDone = true;
|
||||
};
|
||||
|
||||
while (!isDone) {
|
||||
await waitForMessage;
|
||||
yield* chunks;
|
||||
chunks = [];
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
import OpenAI from 'openai';
|
||||
import { ChatCompletionStream } from 'openai/lib/ChatCompletionStream';
|
||||
|
||||
export async function streamCompletion({ systemPrompt, messages }) {
|
||||
const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
||||
return client.beta.chat.completions.stream({
|
||||
model: 'gpt-4-0125-preview',
|
||||
messages: [{ role: 'system', content: systemPrompt }, ...messages],
|
||||
});
|
||||
}
|
||||
|
||||
export async function* llmMessageSource(llmStream: ChatCompletionStream): AsyncIterable<string> {
|
||||
for await (const chunk of llmStream) {
|
||||
const message = chunk.choices[0].delta.content;
|
||||
if (message) {
|
||||
yield message;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main(systemPrompt: string, prompt: string) {
|
||||
const llmStream = await streamCompletion({
|
||||
systemPrompt,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
});
|
||||
const llmMessageStream = llmMessageSource(llmStream);
|
||||
console.log('Streaming LLM messages...');
|
||||
for await (const audio of inputStreamTextToSpeech(llmMessageStream)) {
|
||||
console.log(audio);
|
||||
}
|
||||
}
|
||||
|
||||
main('Hello, how can I help you today?', 'What is the meaning of life?');
|
||||
|
|
@ -25,6 +25,25 @@ registration:
|
|||
# allowedDomains:
|
||||
# - "gmail.com"
|
||||
|
||||
# tts:
|
||||
# url: ''
|
||||
# apiKey: '${TTS_API_KEY}'
|
||||
# model: ''
|
||||
# backend: ''
|
||||
# voice: ''
|
||||
# compatibility: ''
|
||||
# voice_settings:
|
||||
# similarity_boost: ''
|
||||
# stability: ''
|
||||
# style: ''
|
||||
# use_speaker_boost:
|
||||
# pronunciation_dictionary_locators: ['']
|
||||
#
|
||||
# stt:
|
||||
# url: ''
|
||||
# apiKey: '${STT_API_KEY}'
|
||||
# model: ''
|
||||
|
||||
# rateLimits:
|
||||
# fileUploads:
|
||||
# ipMax: 100
|
||||
|
|
|
|||
17
package-lock.json
generated
17
package-lock.json
generated
|
|
@ -102,6 +102,7 @@
|
|||
"ua-parser-js": "^1.0.36",
|
||||
"winston": "^3.11.0",
|
||||
"winston-daily-rotate-file": "^4.7.1",
|
||||
"ws": "^8.17.0",
|
||||
"zod": "^3.22.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
@ -1163,9 +1164,11 @@
|
|||
"react-markdown": "^8.0.6",
|
||||
"react-resizable-panels": "^1.0.9",
|
||||
"react-router-dom": "^6.11.2",
|
||||
"react-speech-recognition": "^3.10.0",
|
||||
"react-textarea-autosize": "^8.4.0",
|
||||
"react-transition-group": "^4.4.5",
|
||||
"recoil": "^0.7.7",
|
||||
"regenerator-runtime": "^0.14.1",
|
||||
"rehype-highlight": "^6.0.0",
|
||||
"rehype-katex": "^6.0.2",
|
||||
"rehype-raw": "^6.1.1",
|
||||
|
|
@ -24436,6 +24439,14 @@
|
|||
"react-dom": ">=16.8"
|
||||
}
|
||||
},
|
||||
"node_modules/react-speech-recognition": {
|
||||
"version": "3.10.0",
|
||||
"resolved": "https://registry.npmjs.org/react-speech-recognition/-/react-speech-recognition-3.10.0.tgz",
|
||||
"integrity": "sha512-EVSr4Ik8l9urwdPiK2r0+ADrLyDDrjB0qBRdUWO+w2MfwEBrj6NuRmy1GD3x7BU/V6/hab0pl8Lupen0zwlJyw==",
|
||||
"peerDependencies": {
|
||||
"react": ">=16.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/react-style-singleton": {
|
||||
"version": "2.2.1",
|
||||
"resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz",
|
||||
|
|
@ -29098,9 +29109,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.16.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz",
|
||||
"integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==",
|
||||
"version": "8.17.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz",
|
||||
"integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -122,3 +122,11 @@ export const files = () => '/api/files';
|
|||
export const images = () => `${files()}/images`;
|
||||
|
||||
export const avatar = () => `${images()}/avatar`;
|
||||
|
||||
export const speechToText = () => `${files()}/stt`;
|
||||
|
||||
export const textToSpeech = () => `${files()}/tts`;
|
||||
|
||||
export const textToSpeechManual = () => `${textToSpeech()}/manual`;
|
||||
|
||||
export const textToSpeechVoices = () => `${textToSpeech()}/voices`;
|
||||
|
|
|
|||
|
|
@ -223,6 +223,53 @@ export const azureEndpointSchema = z
|
|||
export type TAzureConfig = Omit<z.infer<typeof azureEndpointSchema>, 'groups'> &
|
||||
TAzureConfigValidationResult;
|
||||
|
||||
const ttsSchema = z.object({
|
||||
openai: z
|
||||
.object({
|
||||
url: z.string().optional(),
|
||||
apiKey: z.string(),
|
||||
model: z.string(),
|
||||
voices: z.array(z.string()),
|
||||
})
|
||||
.optional(),
|
||||
elevenLabs: z
|
||||
.object({
|
||||
url: z.string().optional(),
|
||||
websocketUrl: z.string().optional(),
|
||||
apiKey: z.string(),
|
||||
model: z.string(),
|
||||
voices: z.array(z.string()),
|
||||
voice_settings: z
|
||||
.object({
|
||||
similarity_boost: z.number().optional(),
|
||||
stability: z.number().optional(),
|
||||
style: z.number().optional(),
|
||||
use_speaker_boost: z.boolean().optional(),
|
||||
})
|
||||
.optional(),
|
||||
pronunciation_dictionary_locators: z.array(z.string()).optional(),
|
||||
})
|
||||
.optional(),
|
||||
localai: z
|
||||
.object({
|
||||
url: z.string(),
|
||||
apiKey: z.string().optional(),
|
||||
voices: z.array(z.string()),
|
||||
backend: z.string(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
const sttSchema = z.object({
|
||||
openai: z
|
||||
.object({
|
||||
url: z.string().optional(),
|
||||
apiKey: z.string().optional(),
|
||||
model: z.string().optional(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
export const rateLimitSchema = z.object({
|
||||
fileUploads: z
|
||||
.object({
|
||||
|
|
@ -289,6 +336,8 @@ export const configSchema = z.object({
|
|||
allowedDomains: z.array(z.string()).optional(),
|
||||
})
|
||||
.default({ socialLogins: defaultSocialLogins }),
|
||||
tts: ttsSchema.optional(),
|
||||
stt: sttSchema.optional(),
|
||||
rateLimits: rateLimitSchema.optional(),
|
||||
fileConfig: fileConfigSchema.optional(),
|
||||
modelSpecs: specsConfigSchema.optional(),
|
||||
|
|
@ -562,6 +611,10 @@ export enum CacheKeys {
|
|||
* Used by Azure OpenAI Assistants.
|
||||
*/
|
||||
ENCODED_DOMAINS = 'encoded_domains',
|
||||
/**
|
||||
* Key for the cached audio run Ids.
|
||||
*/
|
||||
AUDIO_RUNS = 'audioRuns',
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -664,6 +717,10 @@ export enum SettingsTabValues {
|
|||
* Tab for Messages Settings
|
||||
*/
|
||||
MESSAGES = 'messages',
|
||||
/**
|
||||
* Tab for Speech Settings
|
||||
*/
|
||||
SPEECH = 'speech',
|
||||
/**
|
||||
* Tab for Beta Features
|
||||
*/
|
||||
|
|
@ -683,7 +740,7 @@ export enum Constants {
|
|||
/** Key for the app's version. */
|
||||
VERSION = 'v0.7.2',
|
||||
/** Key for the Custom Config's version (librechat.yaml). */
|
||||
CONFIG_VERSION = '1.1.1',
|
||||
CONFIG_VERSION = '1.1.2',
|
||||
/** Standard value for the first message's `parentMessageId` value, to indicate no parent exists. */
|
||||
NO_PARENT = '00000000-0000-0000-0000-000000000000',
|
||||
/** Fixed, encoded domain length for Azure OpenAI Assistants Function name parsing. */
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue